From d48da99a1267a79e22a1c197b1159a2ce1e62f51 Mon Sep 17 00:00:00 2001 From: Tomas Hozza Date: Tue, 19 Apr 2022 20:26:59 +0200 Subject: [PATCH] rpmmd/dnf-json: support chain dependency solving Add a new `rpmmdImpl` method `chainDepsolve`, which is able to depsolve multiple chained package sets as separate DNF transactions layered on top of each other. This new method allows to depsolve the `blueprint` package set on top of the base image package set (usually called `packages`). Introduce a helper function `chainPackageSets` for constructing arguments to the `chainDepsolve` method based on the provided arguments: - slice of package set names to chain as transactions - map of package sets - slice of system repositories used by all package sets - map of package-set-specific repositories Extend `dnf-json` with a new command `chain-depsolve` allowing to depsolve multiple transaction in a row, layered on top of each other. Add unit tests where appropriate. --- dnf-json | 57 +++ internal/rpmmd/repository.go | 158 ++++++++ internal/rpmmd/repository_internal_test.go | 396 +++++++++++++++++++++ 3 files changed, 611 insertions(+) create mode 100644 internal/rpmmd/repository_internal_test.go diff --git a/dnf-json b/dnf-json index 74be0c657..4ea3d007f 100755 --- a/dnf-json +++ b/dnf-json @@ -247,6 +247,59 @@ class Solver(): "dependencies": dependencies } + def chain_depsolve(self, transactions): + last_transaction = [] + + for idx, transaction in enumerate(transactions): + self.base.reset(goal=True) + self.base.sack.reset_excludes() + + # don't install weak-deps for transactions after the 1st transaction + if idx > 0: + self.base.conf.install_weak_deps=False + + # set the packages from the last transaction as installed + for installed_pkg in last_transaction: + self.base.package_install(installed_pkg, strict=True) + + # depsolve the current transaction + self.base.install_specs( + transaction.get("package-specs"), + transaction.get("exclude-specs"), + reponame=[str(id) for id in transaction.get("repos")]) + self.base.resolve() + + # store the current transaction result + last_transaction.clear() + for tsi in self.base.transaction: + # Avoid using the install_set() helper, as it does not guarantee + # a stable order + if tsi.action not in dnf.transaction.FORWARD_ACTIONS: + continue + last_transaction.append(tsi.pkg) + + dependencies = [] + for package in last_transaction: + dependencies.append({ + "name": package.name, + "epoch": package.epoch, + "version": package.version, + "release": package.release, + "arch": package.arch, + "repo_id": package.repoid, + "path": package.relativepath, + "remote_location": package.remote_location(), + "checksum": ( + f"{hawkey.chksum_name(package.chksum[0])}:" + f"{package.chksum[1].hex()}" + ) + }) + + return { + "checksums": self._repo_checksums(), + "dependencies": dependencies + } + class DnfJsonRequestHandler(BaseHTTPRequestHandler): """ @@ -354,6 +407,10 @@ class DnfJsonRequestHandler(BaseHTTPRequestHandler): ) ) log.info("depsolve success") + elif command == "chain-depsolve": + self.response_success( + solver.chain_depsolve(arguments["transactions"]) + ) except dnf.exceptions.MarkingErrors as e: log.info("error install_specs") diff --git a/internal/rpmmd/repository.go b/internal/rpmmd/repository.go index 7eabd810b..b5035ef7f 100644 --- a/internal/rpmmd/repository.go +++ b/internal/rpmmd/repository.go @@ -11,6 +11,7 @@ import ( "net/http" "os" "path/filepath" + "reflect" "sort" "strconv" "strings" @@ -113,6 +114,14 @@ type PackageSet struct { Exclude []string } +// The input to chain depsolve request. A set of packages to include / exclude +// and a set of repository IDs to use, which are represented as indexes to +// an array of repositories provided together with this request. +type chainPackageSet struct { + PackageSet + Repos []int +} + // Append the Include and Exclude package list from another PackageSet and // return the result. func (ps PackageSet) Append(other PackageSet) PackageSet { @@ -534,6 +543,155 @@ func (r *rpmmdImpl) Depsolve(packageSet PackageSet, repos []RepoConfig, modulePl return dependencies, reply.Checksums, err } +// ChainPackageSets constructs an array of `ChainPackageSet` based on the provided +// arguments. The provided `packageSets` map is transformed into an array based +// on the order of package set names passed in `packageSetsChain`. Repositories +// provided in `repos` are used for every transaction, while repositories from +// `packageSetsRepos` are used only for package sets with the respective name. +// +// The function returns a `ChainPackageSet` slice, which members are referencing +// repositories from the returned `RepoConfig` by their index. The returned +// `RepoConfig` slice is specific to the requested `packageSetsChain`. +// +// NOTE: Due to implementation limitations of DNF and dnf-json, each package set +// in the chain must use all of the repositories used by its predecessor. +// An error is returned if this requirement is not met. +func chainPackageSets(packageSetsChain []string, packageSets map[string]PackageSet, repos []RepoConfig, packageSetsRepos map[string][]RepoConfig) ([]chainPackageSet, []RepoConfig, error) { + transactions := make([]chainPackageSet, 0, len(packageSetsChain)) + transactionsRepos := make([]RepoConfig, 0, len(repos)) + + transactionsRepos = append(transactionsRepos, repos...) + + // These repo IDs will be used for every transaction + baseRepoIDs := make([]int, 0, len(repos)) + for idx := range repos { + baseRepoIDs = append(baseRepoIDs, idx) + } + + for transactionIdx, pkgSetName := range packageSetsChain { + pkgSet, ok := packageSets[pkgSetName] + if !ok { + return nil, nil, fmt.Errorf("package set %q requested in the 'packageSetsChain' does not exist in provided 'packageSets'", pkgSetName) + } + + transaction := chainPackageSet{ + PackageSet: pkgSet, + Repos: baseRepoIDs, // Due to its capacity, the slice will be copied if any repo is appended + } + + // Add any package-set-specific repos to the list of transaction repos + if pkgSetRepos, ok := packageSetsRepos[pkgSetName]; ok { + for _, pkgSetRepo := range pkgSetRepos { + // Check if the repo has been already used by a transaction + // and if yes, just use its ID. Skip the "base" repos. + pkgSetRepoID := -1 + for idx := len(repos); idx < len(transactionsRepos); idx++ { + transactionRepo := transactionsRepos[idx] + if reflect.DeepEqual(pkgSetRepo, transactionRepo) { + pkgSetRepoID = idx + break + } + } + + if pkgSetRepoID == -1 { + transactionsRepos = append(transactionsRepos, pkgSetRepo) + pkgSetRepoID = len(transactionsRepos) - 1 + } + + transaction.Repos = append(transaction.Repos, pkgSetRepoID) + } + } + + // Sort the slice of repo IDs to make it easier to compare + sort.Ints(transaction.Repos) + + // If more than one transaction, ensure that the transaction uses + // all of the repos from its predecessor + if transactionIdx > 0 { + previousTransRepos := transactions[transactionIdx-1].Repos + if len(transaction.Repos) < len(previousTransRepos) { + return nil, nil, fmt.Errorf("chained packageSet %q does not use all of the repos used by its predecessor", pkgSetName) + } + + for idx, repoID := range previousTransRepos { + if repoID != transaction.Repos[idx] { + return nil, nil, fmt.Errorf("chained packageSet %q does not use all of the repos used by its predecessor", pkgSetName) + } + } + } + + transactions = append(transactions, transaction) + } + + return transactions, transactionsRepos, nil +} + +// ChainDepsolve takes a list of required package sets (included and excluded), which should be depsolved +// as separate transactions, list of repositories, platform ID for modularity, architecture and release version. +// It returns a list of all packages (with solved dependencies) that will be installed into the system. +func (r *rpmmdImpl) chainDepsolve(chains []chainPackageSet, repos []RepoConfig, modulePlatformID, arch, releasever string) ([]PackageSpec, map[string]string, error) { + var dnfRepoConfigs []dnfRepoConfig + for i, repo := range repos { + dnfRepo, err := repo.toDNFRepoConfig(r, i, arch, releasever) + if err != nil { + return nil, nil, err + } + dnfRepoConfigs = append(dnfRepoConfigs, dnfRepo) + } + + type dnfTransaction struct { + PackageSpecs []string `json:"package-specs"` + ExcludSpecs []string `json:"exclude-specs"` + Repos []int `json:"repos"` + } + var dnfTransactions []dnfTransaction + for _, transaction := range chains { + dnfTransactions = append(dnfTransactions, dnfTransaction{ + PackageSpecs: transaction.Include, + ExcludSpecs: transaction.Exclude, + Repos: transaction.Repos, + }) + } + + var arguments = struct { + Transactions []dnfTransaction `json:"transactions"` + Repos []dnfRepoConfig `json:"repos"` + CacheDir string `json:"cachedir"` + ModulePlatformID string `json:"module_platform_id"` + Arch string `json:"arch"` + }{dnfTransactions, dnfRepoConfigs, r.CacheDir, modulePlatformID, arch} + + var reply struct { + Checksums map[string]string `json:"checksums"` + Dependencies []dnfPackageSpec `json:"dependencies"` + } + + err := runDNF("chain-depsolve", arguments, &reply) + + dependencies := make([]PackageSpec, len(reply.Dependencies)) + for i, pack := range reply.Dependencies { + id, err := strconv.Atoi(pack.RepoID) + if err != nil { + panic(err) + } + repo := repos[id] + dep := reply.Dependencies[i] + dependencies[i].Name = dep.Name + dependencies[i].Epoch = dep.Epoch + dependencies[i].Version = dep.Version + dependencies[i].Release = dep.Release + dependencies[i].Arch = dep.Arch + dependencies[i].RemoteLocation = dep.RemoteLocation + dependencies[i].Checksum = dep.Checksum + dependencies[i].CheckGPG = repo.CheckGPG + if repo.RHSM { + dependencies[i].Secrets = "org.osbuild.rhsm" + } + } + + return dependencies, reply.Checksums, err +} + func (packages PackageList) Search(globPatterns ...string) (PackageList, error) { var globs []glob.Glob diff --git a/internal/rpmmd/repository_internal_test.go b/internal/rpmmd/repository_internal_test.go new file mode 100644 index 000000000..8bba4f001 --- /dev/null +++ b/internal/rpmmd/repository_internal_test.go @@ -0,0 +1,396 @@ +package rpmmd + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestChainPackageSets(t *testing.T) { + tests := []struct { + packageSetsChain []string + packageSets map[string]PackageSet + repos []RepoConfig + packageSetsRepos map[string][]RepoConfig + wantChainPkgSets []chainPackageSet + wantRepos []RepoConfig + err bool + }{ + // single transaction + { + packageSetsChain: []string{"os"}, + packageSets: map[string]PackageSet{ + "os": { + Include: []string{"pkg1"}, + Exclude: []string{"pkg2"}, + }, + }, + repos: []RepoConfig{ + { + Name: "baseos", + BaseURL: "https://example.org/baseos", + }, + { + Name: "appstream", + BaseURL: "https://example.org/appstream", + }, + }, + wantChainPkgSets: []chainPackageSet{ + { + PackageSet: PackageSet{ + Include: []string{"pkg1"}, + Exclude: []string{"pkg2"}, + }, + Repos: []int{0, 1}, + }, + }, + wantRepos: []RepoConfig{ + { + Name: "baseos", + BaseURL: "https://example.org/baseos", + }, + { + Name: "appstream", + BaseURL: "https://example.org/appstream", + }, + }, + }, + // 2 transactions + package set specific repo + { + packageSetsChain: []string{"os", "blueprint"}, + packageSets: map[string]PackageSet{ + "os": { + Include: []string{"pkg1"}, + Exclude: []string{"pkg2"}, + }, + "blueprint": { + Include: []string{"pkg3"}, + }, + }, + repos: []RepoConfig{ + { + Name: "baseos", + BaseURL: "https://example.org/baseos", + }, + { + Name: "appstream", + BaseURL: "https://example.org/appstream", + }, + }, + packageSetsRepos: map[string][]RepoConfig{ + "blueprint": { + { + Name: "user-repo", + BaseURL: "https://example.org/user-repo", + }, + }, + }, + wantChainPkgSets: []chainPackageSet{ + { + PackageSet: PackageSet{ + Include: []string{"pkg1"}, + Exclude: []string{"pkg2"}, + }, + Repos: []int{0, 1}, + }, + { + PackageSet: PackageSet{ + Include: []string{"pkg3"}, + }, + Repos: []int{0, 1, 2}, + }, + }, + wantRepos: []RepoConfig{ + { + Name: "baseos", + BaseURL: "https://example.org/baseos", + }, + { + Name: "appstream", + BaseURL: "https://example.org/appstream", + }, + { + Name: "user-repo", + BaseURL: "https://example.org/user-repo", + }, + }, + }, + // 2 transactions + no package set specific repos + { + packageSetsChain: []string{"os", "blueprint"}, + packageSets: map[string]PackageSet{ + "os": { + Include: []string{"pkg1"}, + Exclude: []string{"pkg2"}, + }, + "blueprint": { + Include: []string{"pkg3"}, + }, + }, + repos: []RepoConfig{ + { + Name: "baseos", + BaseURL: "https://example.org/baseos", + }, + { + Name: "appstream", + BaseURL: "https://example.org/appstream", + }, + }, + wantChainPkgSets: []chainPackageSet{ + { + PackageSet: PackageSet{ + Include: []string{"pkg1"}, + Exclude: []string{"pkg2"}, + }, + Repos: []int{0, 1}, + }, + { + PackageSet: PackageSet{ + Include: []string{"pkg3"}, + }, + Repos: []int{0, 1}, + }, + }, + wantRepos: []RepoConfig{ + { + Name: "baseos", + BaseURL: "https://example.org/baseos", + }, + { + Name: "appstream", + BaseURL: "https://example.org/appstream", + }, + }, + }, + // 3 transactions + package set specific repo used by 2nd and 3rd transaction + { + packageSetsChain: []string{"os", "blueprint", "blueprint2"}, + packageSets: map[string]PackageSet{ + "os": { + Include: []string{"pkg1"}, + Exclude: []string{"pkg2"}, + }, + "blueprint": { + Include: []string{"pkg3"}, + }, + "blueprint2": { + Include: []string{"pkg4"}, + }, + }, + repos: []RepoConfig{ + { + Name: "baseos", + BaseURL: "https://example.org/baseos", + }, + { + Name: "appstream", + BaseURL: "https://example.org/appstream", + }, + }, + packageSetsRepos: map[string][]RepoConfig{ + "blueprint": { + { + Name: "user-repo", + BaseURL: "https://example.org/user-repo", + }, + }, + "blueprint2": { + { + Name: "user-repo", + BaseURL: "https://example.org/user-repo", + }, + }, + }, + wantChainPkgSets: []chainPackageSet{ + { + PackageSet: PackageSet{ + Include: []string{"pkg1"}, + Exclude: []string{"pkg2"}, + }, + Repos: []int{0, 1}, + }, + { + PackageSet: PackageSet{ + Include: []string{"pkg3"}, + }, + Repos: []int{0, 1, 2}, + }, + { + PackageSet: PackageSet{ + Include: []string{"pkg4"}, + }, + Repos: []int{0, 1, 2}, + }, + }, + wantRepos: []RepoConfig{ + { + Name: "baseos", + BaseURL: "https://example.org/baseos", + }, + { + Name: "appstream", + BaseURL: "https://example.org/appstream", + }, + { + Name: "user-repo", + BaseURL: "https://example.org/user-repo", + }, + }, + }, + // 3 transactions + package set specific repo used by 2nd and 3rd transaction + // + 3rd transaction using another repo + { + packageSetsChain: []string{"os", "blueprint", "blueprint2"}, + packageSets: map[string]PackageSet{ + "os": { + Include: []string{"pkg1"}, + Exclude: []string{"pkg2"}, + }, + "blueprint": { + Include: []string{"pkg3"}, + }, + "blueprint2": { + Include: []string{"pkg4"}, + }, + }, + repos: []RepoConfig{ + { + Name: "baseos", + BaseURL: "https://example.org/baseos", + }, + { + Name: "appstream", + BaseURL: "https://example.org/appstream", + }, + }, + packageSetsRepos: map[string][]RepoConfig{ + "blueprint": { + { + Name: "user-repo", + BaseURL: "https://example.org/user-repo", + }, + }, + "blueprint2": { + { + Name: "user-repo", + BaseURL: "https://example.org/user-repo", + }, + { + Name: "user-repo-2", + BaseURL: "https://example.org/user-repo-2", + }, + }, + }, + wantChainPkgSets: []chainPackageSet{ + { + PackageSet: PackageSet{ + Include: []string{"pkg1"}, + Exclude: []string{"pkg2"}, + }, + Repos: []int{0, 1}, + }, + { + PackageSet: PackageSet{ + Include: []string{"pkg3"}, + }, + Repos: []int{0, 1, 2}, + }, + { + PackageSet: PackageSet{ + Include: []string{"pkg4"}, + }, + Repos: []int{0, 1, 2, 3}, + }, + }, + wantRepos: []RepoConfig{ + { + Name: "baseos", + BaseURL: "https://example.org/baseos", + }, + { + Name: "appstream", + BaseURL: "https://example.org/appstream", + }, + { + Name: "user-repo", + BaseURL: "https://example.org/user-repo", + }, + { + Name: "user-repo-2", + BaseURL: "https://example.org/user-repo-2", + }, + }, + }, + // Error: 3 transactions + 3rd one not using repo used by 2nd one + { + packageSetsChain: []string{"os", "blueprint", "blueprint2"}, + packageSets: map[string]PackageSet{ + "os": { + Include: []string{"pkg1"}, + Exclude: []string{"pkg2"}, + }, + "blueprint": { + Include: []string{"pkg3"}, + }, + "blueprint2": { + Include: []string{"pkg4"}, + }, + }, + repos: []RepoConfig{ + { + Name: "baseos", + BaseURL: "https://example.org/baseos", + }, + { + Name: "appstream", + BaseURL: "https://example.org/appstream", + }, + }, + packageSetsRepos: map[string][]RepoConfig{ + "blueprint": { + { + Name: "user-repo", + BaseURL: "https://example.org/user-repo", + }, + }, + "blueprint2": { + { + Name: "user-repo2", + BaseURL: "https://example.org/user-repo2", + }, + }, + }, + err: true, + }, + // Error: requested package set name to chain not defined in provided pkg sets + { + packageSetsChain: []string{"os", "blueprint"}, + packageSets: map[string]PackageSet{ + "os": { + Include: []string{"pkg1"}, + Exclude: []string{"pkg2"}, + }, + }, + err: true, + }, + } + for idx, tt := range tests { + t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) { + gotChainPackageSets, gotRepos, err := chainPackageSets(tt.packageSetsChain, tt.packageSets, tt.repos, tt.packageSetsRepos) + if tt.err { + assert.NotNilf(t, err, "expected an error, but got 'nil' instead") + assert.Nilf(t, gotChainPackageSets, "got non-nill []rpmmd.ChainPackageSet, but expected an error") + assert.Nilf(t, gotRepos, "got non-nill []rpmmd.RepoConfig, but expected an error") + } else { + assert.Nilf(t, err, "expected 'nil', but got error instead") + assert.NotNilf(t, gotChainPackageSets, "expected non-nill []rpmmd.ChainPackageSet, but got 'nil' instead") + assert.NotNilf(t, gotRepos, "expected non-nill []rpmmd.RepoConfig, but got 'nil' instead") + + assert.Equal(t, tt.wantChainPkgSets, gotChainPackageSets) + assert.Equal(t, tt.wantRepos, gotRepos) + } + }) + } +}