rpmmd/dnf-json: support chain dependency solving

Add a new `rpmmdImpl` method `chainDepsolve`, which is able to
depsolve multiple chained package sets as separate DNF transactions
layered on top of each other.

This new method allows to depsolve the `blueprint` package set on top of
the base image package set (usually called `packages`).

Introduce a helper function `chainPackageSets` for constructing
arguments to the `chainDepsolve` method based on the provided arguments:
 - slice of package set names to chain as transactions
 - map of package sets
 - slice of system repositories used by all package sets
 - map of package-set-specific repositories

Extend `dnf-json` with a new command `chain-depsolve` allowing to
depsolve multiple transaction in a row, layered on top of each other.

Add unit tests where appropriate.
This commit is contained in:
Tomas Hozza 2022-04-19 20:26:59 +02:00 committed by Ondřej Budai
parent 0f0b2072d5
commit d48da99a12
3 changed files with 611 additions and 0 deletions

View file

@ -247,6 +247,59 @@ class Solver():
"dependencies": dependencies
}
def chain_depsolve(self, transactions):
last_transaction = []
for idx, transaction in enumerate(transactions):
self.base.reset(goal=True)
self.base.sack.reset_excludes()
# don't install weak-deps for transactions after the 1st transaction
if idx > 0:
self.base.conf.install_weak_deps=False
# set the packages from the last transaction as installed
for installed_pkg in last_transaction:
self.base.package_install(installed_pkg, strict=True)
# depsolve the current transaction
self.base.install_specs(
transaction.get("package-specs"),
transaction.get("exclude-specs"),
reponame=[str(id) for id in transaction.get("repos")])
self.base.resolve()
# store the current transaction result
last_transaction.clear()
for tsi in self.base.transaction:
# Avoid using the install_set() helper, as it does not guarantee
# a stable order
if tsi.action not in dnf.transaction.FORWARD_ACTIONS:
continue
last_transaction.append(tsi.pkg)
dependencies = []
for package in last_transaction:
dependencies.append({
"name": package.name,
"epoch": package.epoch,
"version": package.version,
"release": package.release,
"arch": package.arch,
"repo_id": package.repoid,
"path": package.relativepath,
"remote_location": package.remote_location(),
"checksum": (
f"{hawkey.chksum_name(package.chksum[0])}:"
f"{package.chksum[1].hex()}"
)
})
return {
"checksums": self._repo_checksums(),
"dependencies": dependencies
}
class DnfJsonRequestHandler(BaseHTTPRequestHandler):
"""
@ -354,6 +407,10 @@ class DnfJsonRequestHandler(BaseHTTPRequestHandler):
)
)
log.info("depsolve success")
elif command == "chain-depsolve":
self.response_success(
solver.chain_depsolve(arguments["transactions"])
)
except dnf.exceptions.MarkingErrors as e:
log.info("error install_specs")

View file

@ -11,6 +11,7 @@ import (
"net/http"
"os"
"path/filepath"
"reflect"
"sort"
"strconv"
"strings"
@ -113,6 +114,14 @@ type PackageSet struct {
Exclude []string
}
// The input to chain depsolve request. A set of packages to include / exclude
// and a set of repository IDs to use, which are represented as indexes to
// an array of repositories provided together with this request.
type chainPackageSet struct {
PackageSet
Repos []int
}
// Append the Include and Exclude package list from another PackageSet and
// return the result.
func (ps PackageSet) Append(other PackageSet) PackageSet {
@ -534,6 +543,155 @@ func (r *rpmmdImpl) Depsolve(packageSet PackageSet, repos []RepoConfig, modulePl
return dependencies, reply.Checksums, err
}
// ChainPackageSets constructs an array of `ChainPackageSet` based on the provided
// arguments. The provided `packageSets` map is transformed into an array based
// on the order of package set names passed in `packageSetsChain`. Repositories
// provided in `repos` are used for every transaction, while repositories from
// `packageSetsRepos` are used only for package sets with the respective name.
//
// The function returns a `ChainPackageSet` slice, which members are referencing
// repositories from the returned `RepoConfig` by their index. The returned
// `RepoConfig` slice is specific to the requested `packageSetsChain`.
//
// NOTE: Due to implementation limitations of DNF and dnf-json, each package set
// in the chain must use all of the repositories used by its predecessor.
// An error is returned if this requirement is not met.
func chainPackageSets(packageSetsChain []string, packageSets map[string]PackageSet, repos []RepoConfig, packageSetsRepos map[string][]RepoConfig) ([]chainPackageSet, []RepoConfig, error) {
transactions := make([]chainPackageSet, 0, len(packageSetsChain))
transactionsRepos := make([]RepoConfig, 0, len(repos))
transactionsRepos = append(transactionsRepos, repos...)
// These repo IDs will be used for every transaction
baseRepoIDs := make([]int, 0, len(repos))
for idx := range repos {
baseRepoIDs = append(baseRepoIDs, idx)
}
for transactionIdx, pkgSetName := range packageSetsChain {
pkgSet, ok := packageSets[pkgSetName]
if !ok {
return nil, nil, fmt.Errorf("package set %q requested in the 'packageSetsChain' does not exist in provided 'packageSets'", pkgSetName)
}
transaction := chainPackageSet{
PackageSet: pkgSet,
Repos: baseRepoIDs, // Due to its capacity, the slice will be copied if any repo is appended
}
// Add any package-set-specific repos to the list of transaction repos
if pkgSetRepos, ok := packageSetsRepos[pkgSetName]; ok {
for _, pkgSetRepo := range pkgSetRepos {
// Check if the repo has been already used by a transaction
// and if yes, just use its ID. Skip the "base" repos.
pkgSetRepoID := -1
for idx := len(repos); idx < len(transactionsRepos); idx++ {
transactionRepo := transactionsRepos[idx]
if reflect.DeepEqual(pkgSetRepo, transactionRepo) {
pkgSetRepoID = idx
break
}
}
if pkgSetRepoID == -1 {
transactionsRepos = append(transactionsRepos, pkgSetRepo)
pkgSetRepoID = len(transactionsRepos) - 1
}
transaction.Repos = append(transaction.Repos, pkgSetRepoID)
}
}
// Sort the slice of repo IDs to make it easier to compare
sort.Ints(transaction.Repos)
// If more than one transaction, ensure that the transaction uses
// all of the repos from its predecessor
if transactionIdx > 0 {
previousTransRepos := transactions[transactionIdx-1].Repos
if len(transaction.Repos) < len(previousTransRepos) {
return nil, nil, fmt.Errorf("chained packageSet %q does not use all of the repos used by its predecessor", pkgSetName)
}
for idx, repoID := range previousTransRepos {
if repoID != transaction.Repos[idx] {
return nil, nil, fmt.Errorf("chained packageSet %q does not use all of the repos used by its predecessor", pkgSetName)
}
}
}
transactions = append(transactions, transaction)
}
return transactions, transactionsRepos, nil
}
// ChainDepsolve takes a list of required package sets (included and excluded), which should be depsolved
// as separate transactions, list of repositories, platform ID for modularity, architecture and release version.
// It returns a list of all packages (with solved dependencies) that will be installed into the system.
func (r *rpmmdImpl) chainDepsolve(chains []chainPackageSet, repos []RepoConfig, modulePlatformID, arch, releasever string) ([]PackageSpec, map[string]string, error) {
var dnfRepoConfigs []dnfRepoConfig
for i, repo := range repos {
dnfRepo, err := repo.toDNFRepoConfig(r, i, arch, releasever)
if err != nil {
return nil, nil, err
}
dnfRepoConfigs = append(dnfRepoConfigs, dnfRepo)
}
type dnfTransaction struct {
PackageSpecs []string `json:"package-specs"`
ExcludSpecs []string `json:"exclude-specs"`
Repos []int `json:"repos"`
}
var dnfTransactions []dnfTransaction
for _, transaction := range chains {
dnfTransactions = append(dnfTransactions, dnfTransaction{
PackageSpecs: transaction.Include,
ExcludSpecs: transaction.Exclude,
Repos: transaction.Repos,
})
}
var arguments = struct {
Transactions []dnfTransaction `json:"transactions"`
Repos []dnfRepoConfig `json:"repos"`
CacheDir string `json:"cachedir"`
ModulePlatformID string `json:"module_platform_id"`
Arch string `json:"arch"`
}{dnfTransactions, dnfRepoConfigs, r.CacheDir, modulePlatformID, arch}
var reply struct {
Checksums map[string]string `json:"checksums"`
Dependencies []dnfPackageSpec `json:"dependencies"`
}
err := runDNF("chain-depsolve", arguments, &reply)
dependencies := make([]PackageSpec, len(reply.Dependencies))
for i, pack := range reply.Dependencies {
id, err := strconv.Atoi(pack.RepoID)
if err != nil {
panic(err)
}
repo := repos[id]
dep := reply.Dependencies[i]
dependencies[i].Name = dep.Name
dependencies[i].Epoch = dep.Epoch
dependencies[i].Version = dep.Version
dependencies[i].Release = dep.Release
dependencies[i].Arch = dep.Arch
dependencies[i].RemoteLocation = dep.RemoteLocation
dependencies[i].Checksum = dep.Checksum
dependencies[i].CheckGPG = repo.CheckGPG
if repo.RHSM {
dependencies[i].Secrets = "org.osbuild.rhsm"
}
}
return dependencies, reply.Checksums, err
}
func (packages PackageList) Search(globPatterns ...string) (PackageList, error) {
var globs []glob.Glob

View file

@ -0,0 +1,396 @@
package rpmmd
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
)
func TestChainPackageSets(t *testing.T) {
tests := []struct {
packageSetsChain []string
packageSets map[string]PackageSet
repos []RepoConfig
packageSetsRepos map[string][]RepoConfig
wantChainPkgSets []chainPackageSet
wantRepos []RepoConfig
err bool
}{
// single transaction
{
packageSetsChain: []string{"os"},
packageSets: map[string]PackageSet{
"os": {
Include: []string{"pkg1"},
Exclude: []string{"pkg2"},
},
},
repos: []RepoConfig{
{
Name: "baseos",
BaseURL: "https://example.org/baseos",
},
{
Name: "appstream",
BaseURL: "https://example.org/appstream",
},
},
wantChainPkgSets: []chainPackageSet{
{
PackageSet: PackageSet{
Include: []string{"pkg1"},
Exclude: []string{"pkg2"},
},
Repos: []int{0, 1},
},
},
wantRepos: []RepoConfig{
{
Name: "baseos",
BaseURL: "https://example.org/baseos",
},
{
Name: "appstream",
BaseURL: "https://example.org/appstream",
},
},
},
// 2 transactions + package set specific repo
{
packageSetsChain: []string{"os", "blueprint"},
packageSets: map[string]PackageSet{
"os": {
Include: []string{"pkg1"},
Exclude: []string{"pkg2"},
},
"blueprint": {
Include: []string{"pkg3"},
},
},
repos: []RepoConfig{
{
Name: "baseos",
BaseURL: "https://example.org/baseos",
},
{
Name: "appstream",
BaseURL: "https://example.org/appstream",
},
},
packageSetsRepos: map[string][]RepoConfig{
"blueprint": {
{
Name: "user-repo",
BaseURL: "https://example.org/user-repo",
},
},
},
wantChainPkgSets: []chainPackageSet{
{
PackageSet: PackageSet{
Include: []string{"pkg1"},
Exclude: []string{"pkg2"},
},
Repos: []int{0, 1},
},
{
PackageSet: PackageSet{
Include: []string{"pkg3"},
},
Repos: []int{0, 1, 2},
},
},
wantRepos: []RepoConfig{
{
Name: "baseos",
BaseURL: "https://example.org/baseos",
},
{
Name: "appstream",
BaseURL: "https://example.org/appstream",
},
{
Name: "user-repo",
BaseURL: "https://example.org/user-repo",
},
},
},
// 2 transactions + no package set specific repos
{
packageSetsChain: []string{"os", "blueprint"},
packageSets: map[string]PackageSet{
"os": {
Include: []string{"pkg1"},
Exclude: []string{"pkg2"},
},
"blueprint": {
Include: []string{"pkg3"},
},
},
repos: []RepoConfig{
{
Name: "baseos",
BaseURL: "https://example.org/baseos",
},
{
Name: "appstream",
BaseURL: "https://example.org/appstream",
},
},
wantChainPkgSets: []chainPackageSet{
{
PackageSet: PackageSet{
Include: []string{"pkg1"},
Exclude: []string{"pkg2"},
},
Repos: []int{0, 1},
},
{
PackageSet: PackageSet{
Include: []string{"pkg3"},
},
Repos: []int{0, 1},
},
},
wantRepos: []RepoConfig{
{
Name: "baseos",
BaseURL: "https://example.org/baseos",
},
{
Name: "appstream",
BaseURL: "https://example.org/appstream",
},
},
},
// 3 transactions + package set specific repo used by 2nd and 3rd transaction
{
packageSetsChain: []string{"os", "blueprint", "blueprint2"},
packageSets: map[string]PackageSet{
"os": {
Include: []string{"pkg1"},
Exclude: []string{"pkg2"},
},
"blueprint": {
Include: []string{"pkg3"},
},
"blueprint2": {
Include: []string{"pkg4"},
},
},
repos: []RepoConfig{
{
Name: "baseos",
BaseURL: "https://example.org/baseos",
},
{
Name: "appstream",
BaseURL: "https://example.org/appstream",
},
},
packageSetsRepos: map[string][]RepoConfig{
"blueprint": {
{
Name: "user-repo",
BaseURL: "https://example.org/user-repo",
},
},
"blueprint2": {
{
Name: "user-repo",
BaseURL: "https://example.org/user-repo",
},
},
},
wantChainPkgSets: []chainPackageSet{
{
PackageSet: PackageSet{
Include: []string{"pkg1"},
Exclude: []string{"pkg2"},
},
Repos: []int{0, 1},
},
{
PackageSet: PackageSet{
Include: []string{"pkg3"},
},
Repos: []int{0, 1, 2},
},
{
PackageSet: PackageSet{
Include: []string{"pkg4"},
},
Repos: []int{0, 1, 2},
},
},
wantRepos: []RepoConfig{
{
Name: "baseos",
BaseURL: "https://example.org/baseos",
},
{
Name: "appstream",
BaseURL: "https://example.org/appstream",
},
{
Name: "user-repo",
BaseURL: "https://example.org/user-repo",
},
},
},
// 3 transactions + package set specific repo used by 2nd and 3rd transaction
// + 3rd transaction using another repo
{
packageSetsChain: []string{"os", "blueprint", "blueprint2"},
packageSets: map[string]PackageSet{
"os": {
Include: []string{"pkg1"},
Exclude: []string{"pkg2"},
},
"blueprint": {
Include: []string{"pkg3"},
},
"blueprint2": {
Include: []string{"pkg4"},
},
},
repos: []RepoConfig{
{
Name: "baseos",
BaseURL: "https://example.org/baseos",
},
{
Name: "appstream",
BaseURL: "https://example.org/appstream",
},
},
packageSetsRepos: map[string][]RepoConfig{
"blueprint": {
{
Name: "user-repo",
BaseURL: "https://example.org/user-repo",
},
},
"blueprint2": {
{
Name: "user-repo",
BaseURL: "https://example.org/user-repo",
},
{
Name: "user-repo-2",
BaseURL: "https://example.org/user-repo-2",
},
},
},
wantChainPkgSets: []chainPackageSet{
{
PackageSet: PackageSet{
Include: []string{"pkg1"},
Exclude: []string{"pkg2"},
},
Repos: []int{0, 1},
},
{
PackageSet: PackageSet{
Include: []string{"pkg3"},
},
Repos: []int{0, 1, 2},
},
{
PackageSet: PackageSet{
Include: []string{"pkg4"},
},
Repos: []int{0, 1, 2, 3},
},
},
wantRepos: []RepoConfig{
{
Name: "baseos",
BaseURL: "https://example.org/baseos",
},
{
Name: "appstream",
BaseURL: "https://example.org/appstream",
},
{
Name: "user-repo",
BaseURL: "https://example.org/user-repo",
},
{
Name: "user-repo-2",
BaseURL: "https://example.org/user-repo-2",
},
},
},
// Error: 3 transactions + 3rd one not using repo used by 2nd one
{
packageSetsChain: []string{"os", "blueprint", "blueprint2"},
packageSets: map[string]PackageSet{
"os": {
Include: []string{"pkg1"},
Exclude: []string{"pkg2"},
},
"blueprint": {
Include: []string{"pkg3"},
},
"blueprint2": {
Include: []string{"pkg4"},
},
},
repos: []RepoConfig{
{
Name: "baseos",
BaseURL: "https://example.org/baseos",
},
{
Name: "appstream",
BaseURL: "https://example.org/appstream",
},
},
packageSetsRepos: map[string][]RepoConfig{
"blueprint": {
{
Name: "user-repo",
BaseURL: "https://example.org/user-repo",
},
},
"blueprint2": {
{
Name: "user-repo2",
BaseURL: "https://example.org/user-repo2",
},
},
},
err: true,
},
// Error: requested package set name to chain not defined in provided pkg sets
{
packageSetsChain: []string{"os", "blueprint"},
packageSets: map[string]PackageSet{
"os": {
Include: []string{"pkg1"},
Exclude: []string{"pkg2"},
},
},
err: true,
},
}
for idx, tt := range tests {
t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) {
gotChainPackageSets, gotRepos, err := chainPackageSets(tt.packageSetsChain, tt.packageSets, tt.repos, tt.packageSetsRepos)
if tt.err {
assert.NotNilf(t, err, "expected an error, but got 'nil' instead")
assert.Nilf(t, gotChainPackageSets, "got non-nill []rpmmd.ChainPackageSet, but expected an error")
assert.Nilf(t, gotRepos, "got non-nill []rpmmd.RepoConfig, but expected an error")
} else {
assert.Nilf(t, err, "expected 'nil', but got error instead")
assert.NotNilf(t, gotChainPackageSets, "expected non-nill []rpmmd.ChainPackageSet, but got 'nil' instead")
assert.NotNilf(t, gotRepos, "expected non-nill []rpmmd.RepoConfig, but got 'nil' instead")
assert.Equal(t, tt.wantChainPkgSets, gotChainPackageSets)
assert.Equal(t, tt.wantRepos, gotRepos)
}
})
}
}