rpmmd: remove unused functions

Removed all functions that have been ported to dnfjson or are no longer
used and all left-over types.
This commit is contained in:
Achilleas Koutsou 2022-04-19 22:33:06 +02:00 committed by Tom Gundersen
parent 8a23a77c5b
commit 0add961bcc
2 changed files with 0 additions and 782 deletions

View file

@ -1,24 +1,17 @@
package rpmmd
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net"
"net/http"
"os"
"path/filepath"
"reflect"
"sort"
"strconv"
"strings"
"time"
"github.com/gobwas/glob"
"github.com/osbuild/osbuild-composer/internal/rhsm"
)
type repository struct {
@ -33,20 +26,6 @@ type repository struct {
ImageTypeTags []string `json:"image_type_tags,omitempty"`
}
type dnfRepoConfig struct {
ID string `json:"id"`
Name string `json:"name,omitempty"`
BaseURL string `json:"baseurl,omitempty"`
Metalink string `json:"metalink,omitempty"`
MirrorList string `json:"mirrorlist,omitempty"`
GPGKey string `json:"gpgkey,omitempty"`
IgnoreSSL bool `json:"ignoressl"`
SSLCACert string `json:"sslcacert,omitempty"`
SSLClientKey string `json:"sslclientkey,omitempty"`
SSLClientCert string `json:"sslclientcert,omitempty"`
MetadataExpire string `json:"metadata_expire,omitempty"`
}
type RepoConfig struct {
Name string
BaseURL string
@ -114,14 +93,6 @@ type PackageSet struct {
Exclude []string
}
// The input to chain depsolve request. A set of packages to include / exclude
// and a set of repository IDs to use, which are represented as indexes to
// an array of repositories provided together with this request.
type chainPackageSet struct {
PackageSet
Repos []int
}
// Append the Include and Exclude package list from another PackageSet and
// return the result.
func (ps PackageSet) Append(other PackageSet) PackageSet {
@ -162,19 +133,6 @@ type PackageSpec struct {
CheckGPG bool `json:"check_gpg,omitempty"`
}
type dnfPackageSpec struct {
Name string `json:"name"`
Epoch uint `json:"epoch"`
Version string `json:"version,omitempty"`
Release string `json:"release,omitempty"`
Arch string `json:"arch,omitempty"`
RepoID string `json:"repo_id,omitempty"`
Path string `json:"path,omitempty"`
RemoteLocation string `json:"remote_location,omitempty"`
Checksum string `json:"checksum,omitempty"`
Secrets string `json:"secrets,omitempty"`
}
type PackageSource struct {
License string `json:"license"`
Version string `json:"version"`
@ -370,345 +328,6 @@ func LoadRepositories(confPaths []string, distro string) (map[string][]RepoConfi
return repoConfigs, nil
}
func runDNF(command string, arguments interface{}, result interface{}) error {
var call = struct {
Command string `json:"command"`
Arguments interface{} `json:"arguments,omitempty"`
}{
command,
arguments,
}
httpc := http.Client{
Transport: &http.Transport{
DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
return net.Dial("unix", "/run/osbuild-dnf-json/api.sock")
},
},
}
bpost, err := json.Marshal(call)
if err != nil {
return err
}
response, err := httpc.Post("http://unix", "application/json", bytes.NewReader(bpost))
if err != nil {
return err
}
defer response.Body.Close()
output, err := ioutil.ReadAll(response.Body)
if err != nil {
return err
}
if response.StatusCode != http.StatusOK {
var dnfError DNFError
err = json.Unmarshal(output, &dnfError)
if err != nil {
return err
}
return &dnfError
}
err = json.Unmarshal(output, result)
if err != nil {
return err
}
return nil
}
type rpmmdImpl struct {
CacheDir string
subscriptions *rhsm.Subscriptions
}
func NewRPMMD(cacheDir string) RPMMD {
subscriptions, err := rhsm.LoadSystemSubscriptions()
if err != nil {
log.Println("Failed to load subscriptions. osbuild-composer will fail to build images if the "+
"configured repositories require them:", err)
} else if err == nil && subscriptions == nil {
log.Println("This host is not subscribed to any RPM repositories. This is fine as long as " +
"the configured sources don't enable \"rhsm\".")
}
return &rpmmdImpl{
CacheDir: cacheDir,
subscriptions: subscriptions,
}
}
func (repo RepoConfig) toDNFRepoConfig(rpmmd *rpmmdImpl, repoID int, arch, releasever string) (dnfRepoConfig, error) {
id := strconv.Itoa(repoID)
dnfRepo := dnfRepoConfig{
ID: id,
Name: repo.Name,
BaseURL: repo.BaseURL,
Metalink: repo.Metalink,
MirrorList: repo.MirrorList,
GPGKey: repo.GPGKey,
IgnoreSSL: repo.IgnoreSSL,
MetadataExpire: repo.MetadataExpire,
}
if repo.RHSM {
if rpmmd.subscriptions == nil {
return dnfRepoConfig{}, fmt.Errorf("This system does not have any valid subscriptions. Subscribe it before specifying rhsm: true in sources.")
}
secrets, err := rpmmd.subscriptions.GetSecretsForBaseurl(repo.BaseURL, arch, releasever)
if err != nil {
return dnfRepoConfig{}, fmt.Errorf("RHSM secrets not found on the host for this baseurl: %s", repo.BaseURL)
}
dnfRepo.SSLCACert = secrets.SSLCACert
dnfRepo.SSLClientKey = secrets.SSLClientKey
dnfRepo.SSLClientCert = secrets.SSLClientCert
}
return dnfRepo, nil
}
func (r *rpmmdImpl) FetchMetadata(repos []RepoConfig, modulePlatformID, arch, releasever string) (PackageList, map[string]string, error) {
var dnfRepoConfigs []dnfRepoConfig
for i, repo := range repos {
dnfRepo, err := repo.toDNFRepoConfig(r, i, arch, releasever)
if err != nil {
return nil, nil, err
}
dnfRepoConfigs = append(dnfRepoConfigs, dnfRepo)
}
var arguments = struct {
Repos []dnfRepoConfig `json:"repos"`
CacheDir string `json:"cachedir"`
ModulePlatformID string `json:"module_platform_id"`
Arch string `json:"arch"`
}{dnfRepoConfigs, r.CacheDir, modulePlatformID, arch}
var reply struct {
Checksums map[string]string `json:"checksums"`
Packages PackageList `json:"packages"`
}
err := runDNF("dump", arguments, &reply)
sort.Slice(reply.Packages, func(i, j int) bool {
return reply.Packages[i].Name < reply.Packages[j].Name
})
checksums := make(map[string]string)
for i, repo := range repos {
checksums[repo.Name] = reply.Checksums[strconv.Itoa(i)]
}
return reply.Packages, checksums, err
}
func (r *rpmmdImpl) Depsolve(packageSet PackageSet, repos []RepoConfig, modulePlatformID, arch, releasever string) ([]PackageSpec, map[string]string, error) {
pkgSetName := "packages"
chainPkgSets, chainRepos, err := chainPackageSets([]string{pkgSetName}, map[string]PackageSet{pkgSetName: packageSet}, repos, nil)
if err != nil {
return nil, nil, err
}
return r.chainDepsolve(chainPkgSets, chainRepos, modulePlatformID, arch, releasever)
}
// ChainPackageSets constructs an array of `ChainPackageSet` based on the provided
// arguments. The provided `packageSets` map is transformed into an array based
// on the order of package set names passed in `packageSetsChain`. Repositories
// provided in `repos` are used for every transaction, while repositories from
// `packageSetsRepos` are used only for package sets with the respective name.
//
// The function returns a `ChainPackageSet` slice, which members are referencing
// repositories from the returned `RepoConfig` by their index. The returned
// `RepoConfig` slice is specific to the requested `packageSetsChain`.
//
// NOTE: Due to implementation limitations of DNF and dnf-json, each package set
// in the chain must use all of the repositories used by its predecessor.
// An error is returned if this requirement is not met.
func chainPackageSets(packageSetsChain []string, packageSets map[string]PackageSet, repos []RepoConfig, packageSetsRepos map[string][]RepoConfig) ([]chainPackageSet, []RepoConfig, error) {
transactions := make([]chainPackageSet, 0, len(packageSetsChain))
transactionsRepos := make([]RepoConfig, 0, len(repos))
transactionsRepos = append(transactionsRepos, repos...)
// These repo IDs will be used for every transaction
baseRepoIDs := make([]int, 0, len(repos))
for idx := range repos {
baseRepoIDs = append(baseRepoIDs, idx)
}
for transactionIdx, pkgSetName := range packageSetsChain {
pkgSet, ok := packageSets[pkgSetName]
if !ok {
return nil, nil, fmt.Errorf("package set %q requested in the 'packageSetsChain' does not exist in provided 'packageSets'", pkgSetName)
}
transaction := chainPackageSet{
PackageSet: pkgSet,
Repos: baseRepoIDs, // Due to its capacity, the slice will be copied if any repo is appended
}
// Add any package-set-specific repos to the list of transaction repos
if pkgSetRepos, ok := packageSetsRepos[pkgSetName]; ok {
for _, pkgSetRepo := range pkgSetRepos {
// Check if the repo has been already used by a transaction
// and if yes, just use its ID. Skip the "base" repos.
pkgSetRepoID := -1
for idx := len(repos); idx < len(transactionsRepos); idx++ {
transactionRepo := transactionsRepos[idx]
if reflect.DeepEqual(pkgSetRepo, transactionRepo) {
pkgSetRepoID = idx
break
}
}
if pkgSetRepoID == -1 {
transactionsRepos = append(transactionsRepos, pkgSetRepo)
pkgSetRepoID = len(transactionsRepos) - 1
}
transaction.Repos = append(transaction.Repos, pkgSetRepoID)
}
}
// Sort the slice of repo IDs to make it easier to compare
sort.Ints(transaction.Repos)
// If more than one transaction, ensure that the transaction uses
// all of the repos from its predecessor
if transactionIdx > 0 {
previousTransRepos := transactions[transactionIdx-1].Repos
if len(transaction.Repos) < len(previousTransRepos) {
return nil, nil, fmt.Errorf("chained packageSet %q does not use all of the repos used by its predecessor", pkgSetName)
}
for idx, repoID := range previousTransRepos {
if repoID != transaction.Repos[idx] {
return nil, nil, fmt.Errorf("chained packageSet %q does not use all of the repos used by its predecessor", pkgSetName)
}
}
}
transactions = append(transactions, transaction)
}
return transactions, transactionsRepos, nil
}
// ChainDepsolve takes a list of required package sets (included and excluded), which should be depsolved
// as separate transactions, list of repositories, platform ID for modularity, architecture and release version.
// It returns a list of all packages (with solved dependencies) that will be installed into the system.
func (r *rpmmdImpl) chainDepsolve(chains []chainPackageSet, repos []RepoConfig, modulePlatformID, arch, releasever string) ([]PackageSpec, map[string]string, error) {
var dnfRepoConfigs []dnfRepoConfig
for i, repo := range repos {
dnfRepo, err := repo.toDNFRepoConfig(r, i, arch, releasever)
if err != nil {
return nil, nil, err
}
dnfRepoConfigs = append(dnfRepoConfigs, dnfRepo)
}
type dnfTransaction struct {
PackageSpecs []string `json:"package-specs"`
ExcludSpecs []string `json:"exclude-specs"`
Repos []int `json:"repos"`
}
var dnfTransactions []dnfTransaction
for _, transaction := range chains {
dnfTransactions = append(dnfTransactions, dnfTransaction{
PackageSpecs: transaction.Include,
ExcludSpecs: transaction.Exclude,
Repos: transaction.Repos,
})
}
var arguments = struct {
Transactions []dnfTransaction `json:"transactions"`
Repos []dnfRepoConfig `json:"repos"`
CacheDir string `json:"cachedir"`
ModulePlatformID string `json:"module_platform_id"`
Arch string `json:"arch"`
}{dnfTransactions, dnfRepoConfigs, r.CacheDir, modulePlatformID, arch}
var reply struct {
Checksums map[string]string `json:"checksums"`
Dependencies []dnfPackageSpec `json:"dependencies"`
}
err := runDNF("chain-depsolve", arguments, &reply)
dependencies := make([]PackageSpec, len(reply.Dependencies))
for i, pack := range reply.Dependencies {
id, err := strconv.Atoi(pack.RepoID)
if err != nil {
panic(err)
}
repo := repos[id]
dep := reply.Dependencies[i]
dependencies[i].Name = dep.Name
dependencies[i].Epoch = dep.Epoch
dependencies[i].Version = dep.Version
dependencies[i].Release = dep.Release
dependencies[i].Arch = dep.Arch
dependencies[i].RemoteLocation = dep.RemoteLocation
dependencies[i].Checksum = dep.Checksum
dependencies[i].CheckGPG = repo.CheckGPG
if repo.RHSM {
dependencies[i].Secrets = "org.osbuild.rhsm"
}
}
return dependencies, reply.Checksums, err
}
// DepsolvePackageSets takes a map of package sets chains, which should be depsolved as separate transactions,
// a map of package sets (included and excluded), a list of common and package-set-specific repositories,
// platform ID for modularity, architecture and release version. It returns a map of Package Specs, depsolved
// in a chain or alone, as defined based on the provided arguments.
func (r *rpmmdImpl) DepsolvePackageSets(
packageSetsChains map[string][]string,
packageSets map[string]PackageSet,
repos []RepoConfig,
packageSetsRepos map[string][]RepoConfig,
modulePlatformID, arch, releasever string) (map[string][]PackageSpec, error) {
packageSpecsSets := make(map[string][]PackageSpec, len(packageSets))
// map to hold package set names which were processed as chains
chainPkgSets := make(map[string]struct{}, len(packageSets))
for name, packageSetChain := range packageSetsChains {
for _, packageSetName := range packageSetChain {
chainPkgSets[packageSetName] = struct{}{}
}
chainPkgSets, chainRepos, err := chainPackageSets(packageSetChain, packageSets, repos, packageSetsRepos)
if err != nil {
return nil, err
}
packageSpecs, _, err := r.chainDepsolve(chainPkgSets, chainRepos, modulePlatformID, arch, releasever)
if err != nil {
return nil, err
}
packageSpecsSets[name] = packageSpecs
}
// Process the remaining package sets not contained in the package set chains
for name, packageSet := range packageSets {
if _, ok := chainPkgSets[name]; ok {
continue
}
chainPkgSets, chainRepos, err := chainPackageSets([]string{name}, map[string]PackageSet{name: packageSet}, repos, packageSetsRepos)
if err != nil {
return nil, err
}
packageSpecs, _, err := r.chainDepsolve(chainPkgSets, chainRepos, modulePlatformID, arch, releasever)
if err != nil {
return nil, err
}
packageSpecsSets[name] = packageSpecs
}
return packageSpecsSets, nil
}
func (packages PackageList) Search(globPatterns ...string) (PackageList, error) {
var globs []glob.Glob
@ -760,8 +379,3 @@ func (packages PackageList) ToPackageInfos() []PackageInfo {
return results
}
func (pkg *PackageInfo) FillDependencies(rpmmd RPMMD, repos []RepoConfig, modulePlatformID, arch, releasever string) (err error) {
pkg.Dependencies, _, err = rpmmd.Depsolve(PackageSet{Include: []string{pkg.Name}}, repos, modulePlatformID, arch, releasever)
return
}

View file

@ -1,396 +0,0 @@
package rpmmd
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
)
func TestChainPackageSets(t *testing.T) {
tests := []struct {
packageSetsChain []string
packageSets map[string]PackageSet
repos []RepoConfig
packageSetsRepos map[string][]RepoConfig
wantChainPkgSets []chainPackageSet
wantRepos []RepoConfig
err bool
}{
// single transaction
{
packageSetsChain: []string{"os"},
packageSets: map[string]PackageSet{
"os": {
Include: []string{"pkg1"},
Exclude: []string{"pkg2"},
},
},
repos: []RepoConfig{
{
Name: "baseos",
BaseURL: "https://example.org/baseos",
},
{
Name: "appstream",
BaseURL: "https://example.org/appstream",
},
},
wantChainPkgSets: []chainPackageSet{
{
PackageSet: PackageSet{
Include: []string{"pkg1"},
Exclude: []string{"pkg2"},
},
Repos: []int{0, 1},
},
},
wantRepos: []RepoConfig{
{
Name: "baseos",
BaseURL: "https://example.org/baseos",
},
{
Name: "appstream",
BaseURL: "https://example.org/appstream",
},
},
},
// 2 transactions + package set specific repo
{
packageSetsChain: []string{"os", "blueprint"},
packageSets: map[string]PackageSet{
"os": {
Include: []string{"pkg1"},
Exclude: []string{"pkg2"},
},
"blueprint": {
Include: []string{"pkg3"},
},
},
repos: []RepoConfig{
{
Name: "baseos",
BaseURL: "https://example.org/baseos",
},
{
Name: "appstream",
BaseURL: "https://example.org/appstream",
},
},
packageSetsRepos: map[string][]RepoConfig{
"blueprint": {
{
Name: "user-repo",
BaseURL: "https://example.org/user-repo",
},
},
},
wantChainPkgSets: []chainPackageSet{
{
PackageSet: PackageSet{
Include: []string{"pkg1"},
Exclude: []string{"pkg2"},
},
Repos: []int{0, 1},
},
{
PackageSet: PackageSet{
Include: []string{"pkg3"},
},
Repos: []int{0, 1, 2},
},
},
wantRepos: []RepoConfig{
{
Name: "baseos",
BaseURL: "https://example.org/baseos",
},
{
Name: "appstream",
BaseURL: "https://example.org/appstream",
},
{
Name: "user-repo",
BaseURL: "https://example.org/user-repo",
},
},
},
// 2 transactions + no package set specific repos
{
packageSetsChain: []string{"os", "blueprint"},
packageSets: map[string]PackageSet{
"os": {
Include: []string{"pkg1"},
Exclude: []string{"pkg2"},
},
"blueprint": {
Include: []string{"pkg3"},
},
},
repos: []RepoConfig{
{
Name: "baseos",
BaseURL: "https://example.org/baseos",
},
{
Name: "appstream",
BaseURL: "https://example.org/appstream",
},
},
wantChainPkgSets: []chainPackageSet{
{
PackageSet: PackageSet{
Include: []string{"pkg1"},
Exclude: []string{"pkg2"},
},
Repos: []int{0, 1},
},
{
PackageSet: PackageSet{
Include: []string{"pkg3"},
},
Repos: []int{0, 1},
},
},
wantRepos: []RepoConfig{
{
Name: "baseos",
BaseURL: "https://example.org/baseos",
},
{
Name: "appstream",
BaseURL: "https://example.org/appstream",
},
},
},
// 3 transactions + package set specific repo used by 2nd and 3rd transaction
{
packageSetsChain: []string{"os", "blueprint", "blueprint2"},
packageSets: map[string]PackageSet{
"os": {
Include: []string{"pkg1"},
Exclude: []string{"pkg2"},
},
"blueprint": {
Include: []string{"pkg3"},
},
"blueprint2": {
Include: []string{"pkg4"},
},
},
repos: []RepoConfig{
{
Name: "baseos",
BaseURL: "https://example.org/baseos",
},
{
Name: "appstream",
BaseURL: "https://example.org/appstream",
},
},
packageSetsRepos: map[string][]RepoConfig{
"blueprint": {
{
Name: "user-repo",
BaseURL: "https://example.org/user-repo",
},
},
"blueprint2": {
{
Name: "user-repo",
BaseURL: "https://example.org/user-repo",
},
},
},
wantChainPkgSets: []chainPackageSet{
{
PackageSet: PackageSet{
Include: []string{"pkg1"},
Exclude: []string{"pkg2"},
},
Repos: []int{0, 1},
},
{
PackageSet: PackageSet{
Include: []string{"pkg3"},
},
Repos: []int{0, 1, 2},
},
{
PackageSet: PackageSet{
Include: []string{"pkg4"},
},
Repos: []int{0, 1, 2},
},
},
wantRepos: []RepoConfig{
{
Name: "baseos",
BaseURL: "https://example.org/baseos",
},
{
Name: "appstream",
BaseURL: "https://example.org/appstream",
},
{
Name: "user-repo",
BaseURL: "https://example.org/user-repo",
},
},
},
// 3 transactions + package set specific repo used by 2nd and 3rd transaction
// + 3rd transaction using another repo
{
packageSetsChain: []string{"os", "blueprint", "blueprint2"},
packageSets: map[string]PackageSet{
"os": {
Include: []string{"pkg1"},
Exclude: []string{"pkg2"},
},
"blueprint": {
Include: []string{"pkg3"},
},
"blueprint2": {
Include: []string{"pkg4"},
},
},
repos: []RepoConfig{
{
Name: "baseos",
BaseURL: "https://example.org/baseos",
},
{
Name: "appstream",
BaseURL: "https://example.org/appstream",
},
},
packageSetsRepos: map[string][]RepoConfig{
"blueprint": {
{
Name: "user-repo",
BaseURL: "https://example.org/user-repo",
},
},
"blueprint2": {
{
Name: "user-repo",
BaseURL: "https://example.org/user-repo",
},
{
Name: "user-repo-2",
BaseURL: "https://example.org/user-repo-2",
},
},
},
wantChainPkgSets: []chainPackageSet{
{
PackageSet: PackageSet{
Include: []string{"pkg1"},
Exclude: []string{"pkg2"},
},
Repos: []int{0, 1},
},
{
PackageSet: PackageSet{
Include: []string{"pkg3"},
},
Repos: []int{0, 1, 2},
},
{
PackageSet: PackageSet{
Include: []string{"pkg4"},
},
Repos: []int{0, 1, 2, 3},
},
},
wantRepos: []RepoConfig{
{
Name: "baseos",
BaseURL: "https://example.org/baseos",
},
{
Name: "appstream",
BaseURL: "https://example.org/appstream",
},
{
Name: "user-repo",
BaseURL: "https://example.org/user-repo",
},
{
Name: "user-repo-2",
BaseURL: "https://example.org/user-repo-2",
},
},
},
// Error: 3 transactions + 3rd one not using repo used by 2nd one
{
packageSetsChain: []string{"os", "blueprint", "blueprint2"},
packageSets: map[string]PackageSet{
"os": {
Include: []string{"pkg1"},
Exclude: []string{"pkg2"},
},
"blueprint": {
Include: []string{"pkg3"},
},
"blueprint2": {
Include: []string{"pkg4"},
},
},
repos: []RepoConfig{
{
Name: "baseos",
BaseURL: "https://example.org/baseos",
},
{
Name: "appstream",
BaseURL: "https://example.org/appstream",
},
},
packageSetsRepos: map[string][]RepoConfig{
"blueprint": {
{
Name: "user-repo",
BaseURL: "https://example.org/user-repo",
},
},
"blueprint2": {
{
Name: "user-repo2",
BaseURL: "https://example.org/user-repo2",
},
},
},
err: true,
},
// Error: requested package set name to chain not defined in provided pkg sets
{
packageSetsChain: []string{"os", "blueprint"},
packageSets: map[string]PackageSet{
"os": {
Include: []string{"pkg1"},
Exclude: []string{"pkg2"},
},
},
err: true,
},
}
for idx, tt := range tests {
t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) {
gotChainPackageSets, gotRepos, err := chainPackageSets(tt.packageSetsChain, tt.packageSets, tt.repos, tt.packageSetsRepos)
if tt.err {
assert.NotNilf(t, err, "expected an error, but got 'nil' instead")
assert.Nilf(t, gotChainPackageSets, "got non-nill []rpmmd.ChainPackageSet, but expected an error")
assert.Nilf(t, gotRepos, "got non-nill []rpmmd.RepoConfig, but expected an error")
} else {
assert.Nilf(t, err, "expected 'nil', but got error instead")
assert.NotNilf(t, gotChainPackageSets, "expected non-nill []rpmmd.ChainPackageSet, but got 'nil' instead")
assert.NotNilf(t, gotRepos, "expected non-nill []rpmmd.RepoConfig, but got 'nil' instead")
assert.Equal(t, tt.wantChainPkgSets, gotChainPackageSets)
assert.Equal(t, tt.wantRepos, gotRepos)
}
})
}
}