mirror of
https://github.com/go-gitea/gitea
synced 2024-11-09 03:21:45 +01:00
ba4d0b8ffb
The current rpm repository places all packages in the same repository, and different systems (el7,f34) may hit packages that do not belong to this distribution ( #25304 ) , which now supports grouping of rpm.  Fixes #25304 . Fixes #27056 . Refactor: [#25866](https://github.com/go-gitea/gitea/pull/25866)
608 lines
16 KiB
Go
608 lines
16 KiB
Go
// Copyright 2023 The Gitea Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package rpm
|
|
|
|
import (
|
|
"bytes"
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/xml"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
packages_model "code.gitea.io/gitea/models/packages"
|
|
user_model "code.gitea.io/gitea/models/user"
|
|
"code.gitea.io/gitea/modules/json"
|
|
packages_module "code.gitea.io/gitea/modules/packages"
|
|
rpm_module "code.gitea.io/gitea/modules/packages/rpm"
|
|
"code.gitea.io/gitea/modules/util"
|
|
packages_service "code.gitea.io/gitea/services/packages"
|
|
|
|
"github.com/keybase/go-crypto/openpgp"
|
|
"github.com/keybase/go-crypto/openpgp/armor"
|
|
"github.com/keybase/go-crypto/openpgp/packet"
|
|
)
|
|
|
|
// GetOrCreateRepositoryVersion gets or creates the internal repository package
|
|
// The RPM registry needs multiple metadata files which are stored in this package.
|
|
func GetOrCreateRepositoryVersion(ctx context.Context, ownerID int64) (*packages_model.PackageVersion, error) {
|
|
return packages_service.GetOrCreateInternalPackageVersion(ctx, ownerID, packages_model.TypeRpm, rpm_module.RepositoryPackage, rpm_module.RepositoryVersion)
|
|
}
|
|
|
|
// GetOrCreateKeyPair gets or creates the PGP keys used to sign repository metadata files
|
|
func GetOrCreateKeyPair(ctx context.Context, ownerID int64) (string, string, error) {
|
|
priv, err := user_model.GetSetting(ctx, ownerID, rpm_module.SettingKeyPrivate)
|
|
if err != nil && !errors.Is(err, util.ErrNotExist) {
|
|
return "", "", err
|
|
}
|
|
|
|
pub, err := user_model.GetSetting(ctx, ownerID, rpm_module.SettingKeyPublic)
|
|
if err != nil && !errors.Is(err, util.ErrNotExist) {
|
|
return "", "", err
|
|
}
|
|
|
|
if priv == "" || pub == "" {
|
|
priv, pub, err = generateKeypair()
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
|
|
if err := user_model.SetUserSetting(ctx, ownerID, rpm_module.SettingKeyPrivate, priv); err != nil {
|
|
return "", "", err
|
|
}
|
|
|
|
if err := user_model.SetUserSetting(ctx, ownerID, rpm_module.SettingKeyPublic, pub); err != nil {
|
|
return "", "", err
|
|
}
|
|
}
|
|
|
|
return priv, pub, nil
|
|
}
|
|
|
|
func generateKeypair() (string, string, error) {
|
|
e, err := openpgp.NewEntity("", "RPM Registry", "", nil)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
|
|
var priv strings.Builder
|
|
var pub strings.Builder
|
|
|
|
w, err := armor.Encode(&priv, openpgp.PrivateKeyType, nil)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
if err := e.SerializePrivate(w, nil); err != nil {
|
|
return "", "", err
|
|
}
|
|
w.Close()
|
|
|
|
w, err = armor.Encode(&pub, openpgp.PublicKeyType, nil)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
if err := e.Serialize(w); err != nil {
|
|
return "", "", err
|
|
}
|
|
w.Close()
|
|
|
|
return priv.String(), pub.String(), nil
|
|
}
|
|
|
|
type repoChecksum struct {
|
|
Value string `xml:",chardata"`
|
|
Type string `xml:"type,attr"`
|
|
}
|
|
|
|
type repoLocation struct {
|
|
Href string `xml:"href,attr"`
|
|
}
|
|
|
|
type repoData struct {
|
|
Type string `xml:"type,attr"`
|
|
Checksum repoChecksum `xml:"checksum"`
|
|
OpenChecksum repoChecksum `xml:"open-checksum"`
|
|
Location repoLocation `xml:"location"`
|
|
Timestamp int64 `xml:"timestamp"`
|
|
Size int64 `xml:"size"`
|
|
OpenSize int64 `xml:"open-size"`
|
|
}
|
|
|
|
type packageData struct {
|
|
Package *packages_model.Package
|
|
Version *packages_model.PackageVersion
|
|
Blob *packages_model.PackageBlob
|
|
VersionMetadata *rpm_module.VersionMetadata
|
|
FileMetadata *rpm_module.FileMetadata
|
|
}
|
|
|
|
type packageCache = map[*packages_model.PackageFile]*packageData
|
|
|
|
// BuildSpecificRepositoryFiles builds metadata files for the repository
|
|
func BuildRepositoryFiles(ctx context.Context, ownerID int64, compositeKey string) error {
|
|
pv, err := GetOrCreateRepositoryVersion(ctx, ownerID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{
|
|
OwnerID: ownerID,
|
|
PackageType: packages_model.TypeRpm,
|
|
Query: "%.rpm",
|
|
CompositeKey: compositeKey,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Delete the repository files if there are no packages
|
|
if len(pfs) == 0 {
|
|
pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, pf := range pfs {
|
|
if err := packages_service.DeletePackageFile(ctx, pf); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Cache data needed for all repository files
|
|
cache := make(packageCache)
|
|
for _, pf := range pfs {
|
|
pv, err := packages_model.GetVersionByID(ctx, pf.VersionID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
p, err := packages_model.GetPackageByID(ctx, pv.PackageID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
pb, err := packages_model.GetBlobByID(ctx, pf.BlobID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
pps, err := packages_model.GetPropertiesByName(ctx, packages_model.PropertyTypeFile, pf.ID, rpm_module.PropertyMetadata)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
pd := &packageData{
|
|
Package: p,
|
|
Version: pv,
|
|
Blob: pb,
|
|
}
|
|
|
|
if err := json.Unmarshal([]byte(pv.MetadataJSON), &pd.VersionMetadata); err != nil {
|
|
return err
|
|
}
|
|
if len(pps) > 0 {
|
|
if err := json.Unmarshal([]byte(pps[0].Value), &pd.FileMetadata); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
cache[pf] = pd
|
|
}
|
|
|
|
primary, err := buildPrimary(ctx, pv, pfs, cache, compositeKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
filelists, err := buildFilelists(ctx, pv, pfs, cache, compositeKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
other, err := buildOther(ctx, pv, pfs, cache, compositeKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return buildRepomd(
|
|
ctx,
|
|
pv,
|
|
ownerID,
|
|
[]*repoData{
|
|
primary,
|
|
filelists,
|
|
other,
|
|
},
|
|
compositeKey,
|
|
)
|
|
}
|
|
|
|
// https://docs.pulpproject.org/en/2.19/plugins/pulp_rpm/tech-reference/rpm.html#repomd-xml
|
|
func buildRepomd(ctx context.Context, pv *packages_model.PackageVersion, ownerID int64, data []*repoData, compositeKey string) error {
|
|
type Repomd struct {
|
|
XMLName xml.Name `xml:"repomd"`
|
|
Xmlns string `xml:"xmlns,attr"`
|
|
XmlnsRpm string `xml:"xmlns:rpm,attr"`
|
|
Data []*repoData `xml:"data"`
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
buf.WriteString(xml.Header)
|
|
if err := xml.NewEncoder(&buf).Encode(&Repomd{
|
|
Xmlns: "http://linux.duke.edu/metadata/repo",
|
|
XmlnsRpm: "http://linux.duke.edu/metadata/rpm",
|
|
Data: data,
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
|
|
priv, _, err := GetOrCreateKeyPair(ctx, ownerID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
block, err := armor.Decode(strings.NewReader(priv))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
e, err := openpgp.ReadEntity(packet.NewReader(block.Body))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
repomdAscContent, _ := packages_module.NewHashedBuffer()
|
|
defer repomdAscContent.Close()
|
|
|
|
if err := openpgp.ArmoredDetachSign(repomdAscContent, e, bytes.NewReader(buf.Bytes()), nil); err != nil {
|
|
return err
|
|
}
|
|
|
|
repomdContent, _ := packages_module.CreateHashedBufferFromReader(&buf)
|
|
defer repomdContent.Close()
|
|
|
|
for _, file := range []struct {
|
|
Name string
|
|
Data packages_module.HashedSizeReader
|
|
}{
|
|
{"repomd.xml", repomdContent},
|
|
{"repomd.xml.asc", repomdAscContent},
|
|
} {
|
|
_, err = packages_service.AddFileToPackageVersionInternal(
|
|
ctx,
|
|
pv,
|
|
&packages_service.PackageFileCreationInfo{
|
|
PackageFileInfo: packages_service.PackageFileInfo{
|
|
Filename: file.Name,
|
|
CompositeKey: compositeKey,
|
|
},
|
|
Creator: user_model.NewGhostUser(),
|
|
Data: file.Data,
|
|
IsLead: false,
|
|
OverwriteExisting: true,
|
|
},
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// https://docs.pulpproject.org/en/2.19/plugins/pulp_rpm/tech-reference/rpm.html#primary-xml
|
|
func buildPrimary(ctx context.Context, pv *packages_model.PackageVersion, pfs []*packages_model.PackageFile, c packageCache, compositeKey string) (*repoData, error) {
|
|
type Version struct {
|
|
Epoch string `xml:"epoch,attr"`
|
|
Version string `xml:"ver,attr"`
|
|
Release string `xml:"rel,attr"`
|
|
}
|
|
|
|
type Checksum struct {
|
|
Checksum string `xml:",chardata"`
|
|
Type string `xml:"type,attr"`
|
|
Pkgid string `xml:"pkgid,attr"`
|
|
}
|
|
|
|
type Times struct {
|
|
File uint64 `xml:"file,attr"`
|
|
Build uint64 `xml:"build,attr"`
|
|
}
|
|
|
|
type Sizes struct {
|
|
Package int64 `xml:"package,attr"`
|
|
Installed uint64 `xml:"installed,attr"`
|
|
Archive uint64 `xml:"archive,attr"`
|
|
}
|
|
|
|
type Location struct {
|
|
Href string `xml:"href,attr"`
|
|
}
|
|
|
|
type EntryList struct {
|
|
Entries []*rpm_module.Entry `xml:"rpm:entry"`
|
|
}
|
|
|
|
type Format struct {
|
|
License string `xml:"rpm:license"`
|
|
Vendor string `xml:"rpm:vendor"`
|
|
Group string `xml:"rpm:group"`
|
|
Buildhost string `xml:"rpm:buildhost"`
|
|
Sourcerpm string `xml:"rpm:sourcerpm"`
|
|
Provides EntryList `xml:"rpm:provides"`
|
|
Requires EntryList `xml:"rpm:requires"`
|
|
Conflicts EntryList `xml:"rpm:conflicts"`
|
|
Obsoletes EntryList `xml:"rpm:obsoletes"`
|
|
Files []*rpm_module.File `xml:"file"`
|
|
}
|
|
|
|
type Package struct {
|
|
XMLName xml.Name `xml:"package"`
|
|
Type string `xml:"type,attr"`
|
|
Name string `xml:"name"`
|
|
Architecture string `xml:"arch"`
|
|
Version Version `xml:"version"`
|
|
Checksum Checksum `xml:"checksum"`
|
|
Summary string `xml:"summary"`
|
|
Description string `xml:"description"`
|
|
Packager string `xml:"packager"`
|
|
URL string `xml:"url"`
|
|
Time Times `xml:"time"`
|
|
Size Sizes `xml:"size"`
|
|
Location Location `xml:"location"`
|
|
Format Format `xml:"format"`
|
|
}
|
|
|
|
type Metadata struct {
|
|
XMLName xml.Name `xml:"metadata"`
|
|
Xmlns string `xml:"xmlns,attr"`
|
|
XmlnsRpm string `xml:"xmlns:rpm,attr"`
|
|
PackageCount int `xml:"packages,attr"`
|
|
Packages []*Package `xml:"package"`
|
|
}
|
|
|
|
packages := make([]*Package, 0, len(pfs))
|
|
for _, pf := range pfs {
|
|
pd := c[pf]
|
|
|
|
files := make([]*rpm_module.File, 0, 3)
|
|
for _, f := range pd.FileMetadata.Files {
|
|
if f.IsExecutable {
|
|
files = append(files, f)
|
|
}
|
|
}
|
|
packageVersion := fmt.Sprintf("%s-%s", pd.FileMetadata.Version, pd.FileMetadata.Release)
|
|
packages = append(packages, &Package{
|
|
Type: "rpm",
|
|
Name: pd.Package.Name,
|
|
Architecture: pd.FileMetadata.Architecture,
|
|
Version: Version{
|
|
Epoch: pd.FileMetadata.Epoch,
|
|
Version: pd.FileMetadata.Version,
|
|
Release: pd.FileMetadata.Release,
|
|
},
|
|
Checksum: Checksum{
|
|
Type: "sha256",
|
|
Checksum: pd.Blob.HashSHA256,
|
|
Pkgid: "YES",
|
|
},
|
|
Summary: pd.VersionMetadata.Summary,
|
|
Description: pd.VersionMetadata.Description,
|
|
Packager: pd.FileMetadata.Packager,
|
|
URL: pd.VersionMetadata.ProjectURL,
|
|
Time: Times{
|
|
File: pd.FileMetadata.FileTime,
|
|
Build: pd.FileMetadata.BuildTime,
|
|
},
|
|
Size: Sizes{
|
|
Package: pd.Blob.Size,
|
|
Installed: pd.FileMetadata.InstalledSize,
|
|
Archive: pd.FileMetadata.ArchiveSize,
|
|
},
|
|
Location: Location{
|
|
Href: fmt.Sprintf("package/%s/%s/%s/%s", url.PathEscape(pd.Package.Name), url.PathEscape(packageVersion), url.PathEscape(pd.FileMetadata.Architecture), url.PathEscape(fmt.Sprintf("%s-%s.%s.rpm", pd.Package.Name, packageVersion, pd.FileMetadata.Architecture))),
|
|
},
|
|
Format: Format{
|
|
License: pd.VersionMetadata.License,
|
|
Vendor: pd.FileMetadata.Vendor,
|
|
Group: pd.FileMetadata.Group,
|
|
Buildhost: pd.FileMetadata.BuildHost,
|
|
Sourcerpm: pd.FileMetadata.SourceRpm,
|
|
Provides: EntryList{
|
|
Entries: pd.FileMetadata.Provides,
|
|
},
|
|
Requires: EntryList{
|
|
Entries: pd.FileMetadata.Requires,
|
|
},
|
|
Conflicts: EntryList{
|
|
Entries: pd.FileMetadata.Conflicts,
|
|
},
|
|
Obsoletes: EntryList{
|
|
Entries: pd.FileMetadata.Obsoletes,
|
|
},
|
|
Files: files,
|
|
},
|
|
})
|
|
}
|
|
|
|
return addDataAsFileToRepo(ctx, pv, "primary", &Metadata{
|
|
Xmlns: "http://linux.duke.edu/metadata/common",
|
|
XmlnsRpm: "http://linux.duke.edu/metadata/rpm",
|
|
PackageCount: len(pfs),
|
|
Packages: packages,
|
|
}, compositeKey)
|
|
}
|
|
|
|
// https://docs.pulpproject.org/en/2.19/plugins/pulp_rpm/tech-reference/rpm.html#filelists-xml
|
|
func buildFilelists(ctx context.Context, pv *packages_model.PackageVersion, pfs []*packages_model.PackageFile, c packageCache, compositeKey string) (*repoData, error) { //nolint:dupl
|
|
type Version struct {
|
|
Epoch string `xml:"epoch,attr"`
|
|
Version string `xml:"ver,attr"`
|
|
Release string `xml:"rel,attr"`
|
|
}
|
|
|
|
type Package struct {
|
|
Pkgid string `xml:"pkgid,attr"`
|
|
Name string `xml:"name,attr"`
|
|
Architecture string `xml:"arch,attr"`
|
|
Version Version `xml:"version"`
|
|
Files []*rpm_module.File `xml:"file"`
|
|
}
|
|
|
|
type Filelists struct {
|
|
XMLName xml.Name `xml:"filelists"`
|
|
Xmlns string `xml:"xmlns,attr"`
|
|
PackageCount int `xml:"packages,attr"`
|
|
Packages []*Package `xml:"package"`
|
|
}
|
|
|
|
packages := make([]*Package, 0, len(pfs))
|
|
for _, pf := range pfs {
|
|
pd := c[pf]
|
|
|
|
packages = append(packages, &Package{
|
|
Pkgid: pd.Blob.HashSHA256,
|
|
Name: pd.Package.Name,
|
|
Architecture: pd.FileMetadata.Architecture,
|
|
Version: Version{
|
|
Epoch: pd.FileMetadata.Epoch,
|
|
Version: pd.FileMetadata.Version,
|
|
Release: pd.FileMetadata.Release,
|
|
},
|
|
Files: pd.FileMetadata.Files,
|
|
})
|
|
}
|
|
|
|
return addDataAsFileToRepo(ctx, pv, "filelists", &Filelists{
|
|
Xmlns: "http://linux.duke.edu/metadata/other",
|
|
PackageCount: len(pfs),
|
|
Packages: packages,
|
|
},
|
|
compositeKey)
|
|
}
|
|
|
|
// https://docs.pulpproject.org/en/2.19/plugins/pulp_rpm/tech-reference/rpm.html#other-xml
|
|
func buildOther(ctx context.Context, pv *packages_model.PackageVersion, pfs []*packages_model.PackageFile, c packageCache, compositeKey string) (*repoData, error) { //nolint:dupl
|
|
type Version struct {
|
|
Epoch string `xml:"epoch,attr"`
|
|
Version string `xml:"ver,attr"`
|
|
Release string `xml:"rel,attr"`
|
|
}
|
|
|
|
type Package struct {
|
|
Pkgid string `xml:"pkgid,attr"`
|
|
Name string `xml:"name,attr"`
|
|
Architecture string `xml:"arch,attr"`
|
|
Version Version `xml:"version"`
|
|
Changelogs []*rpm_module.Changelog `xml:"changelog"`
|
|
}
|
|
|
|
type Otherdata struct {
|
|
XMLName xml.Name `xml:"otherdata"`
|
|
Xmlns string `xml:"xmlns,attr"`
|
|
PackageCount int `xml:"packages,attr"`
|
|
Packages []*Package `xml:"package"`
|
|
}
|
|
|
|
packages := make([]*Package, 0, len(pfs))
|
|
for _, pf := range pfs {
|
|
pd := c[pf]
|
|
|
|
packages = append(packages, &Package{
|
|
Pkgid: pd.Blob.HashSHA256,
|
|
Name: pd.Package.Name,
|
|
Architecture: pd.FileMetadata.Architecture,
|
|
Version: Version{
|
|
Epoch: pd.FileMetadata.Epoch,
|
|
Version: pd.FileMetadata.Version,
|
|
Release: pd.FileMetadata.Release,
|
|
},
|
|
Changelogs: pd.FileMetadata.Changelogs,
|
|
})
|
|
}
|
|
|
|
return addDataAsFileToRepo(ctx, pv, "other", &Otherdata{
|
|
Xmlns: "http://linux.duke.edu/metadata/other",
|
|
PackageCount: len(pfs),
|
|
Packages: packages,
|
|
}, compositeKey)
|
|
}
|
|
|
|
// writtenCounter counts all written bytes
|
|
type writtenCounter struct {
|
|
written int64
|
|
}
|
|
|
|
func (wc *writtenCounter) Write(buf []byte) (int, error) {
|
|
n := len(buf)
|
|
|
|
wc.written += int64(n)
|
|
|
|
return n, nil
|
|
}
|
|
|
|
func (wc *writtenCounter) Written() int64 {
|
|
return wc.written
|
|
}
|
|
|
|
func addDataAsFileToRepo(ctx context.Context, pv *packages_model.PackageVersion, filetype string, obj any, compositeKey string) (*repoData, error) {
|
|
content, _ := packages_module.NewHashedBuffer()
|
|
gzw := gzip.NewWriter(content)
|
|
wc := &writtenCounter{}
|
|
h := sha256.New()
|
|
|
|
w := io.MultiWriter(gzw, wc, h)
|
|
_, _ = w.Write([]byte(xml.Header))
|
|
|
|
if err := xml.NewEncoder(w).Encode(obj); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if err := gzw.Close(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
filename := filetype + ".xml.gz"
|
|
|
|
_, err := packages_service.AddFileToPackageVersionInternal(
|
|
ctx,
|
|
pv,
|
|
&packages_service.PackageFileCreationInfo{
|
|
PackageFileInfo: packages_service.PackageFileInfo{
|
|
Filename: filename,
|
|
CompositeKey: compositeKey,
|
|
},
|
|
Creator: user_model.NewGhostUser(),
|
|
Data: content,
|
|
IsLead: false,
|
|
OverwriteExisting: true,
|
|
},
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
_, _, hashSHA256, _ := content.Sums()
|
|
|
|
return &repoData{
|
|
Type: filetype,
|
|
Checksum: repoChecksum{
|
|
Type: "sha256",
|
|
Value: hex.EncodeToString(hashSHA256),
|
|
},
|
|
OpenChecksum: repoChecksum{
|
|
Type: "sha256",
|
|
Value: hex.EncodeToString(h.Sum(nil)),
|
|
},
|
|
Location: repoLocation{
|
|
Href: "repodata/" + filename,
|
|
},
|
|
Timestamp: time.Now().Unix(),
|
|
Size: content.Size(),
|
|
OpenSize: wc.Written(),
|
|
}, nil
|
|
}
|