rpmmd/dnf-json: support chain dependency solving

Add a new `rpmmdImpl` method `chainDepsolve`, which is able to
depsolve multiple chained package sets as separate DNF transactions
layered on top of each other.

This new method allows to depsolve the `blueprint` package set on top of
the base image package set (usually called `packages`).

Introduce a helper function `chainPackageSets` for constructing
arguments to the `chainDepsolve` method based on the provided arguments:
 - slice of package set names to chain as transactions
 - map of package sets
 - slice of system repositories used by all package sets
 - map of package-set-specific repositories

Extend `dnf-json` with a new command `chain-depsolve` allowing to
depsolve multiple transaction in a row, layered on top of each other.

Add unit tests where appropriate.

dnf-json

@@ -247,6 +247,59 @@ class Solver():
             "dependencies": dependencies
         }
 
+    def chain_depsolve(self, transactions):
+        last_transaction = []
+
+        for idx, transaction in enumerate(transactions):
+            self.base.reset(goal=True)
+            self.base.sack.reset_excludes()
+
+            # don't install weak-deps for transactions after the 1st transaction
+            if idx > 0:
+                self.base.conf.install_weak_deps=False
+
+            # set the packages from the last transaction as installed
+            for installed_pkg in last_transaction:
+                self.base.package_install(installed_pkg, strict=True)
+
+            # depsolve the current transaction
+            self.base.install_specs(
+                transaction.get("package-specs"),
+                transaction.get("exclude-specs"),
+                reponame=[str(id) for id in transaction.get("repos")])
+            self.base.resolve()
+
+            # store the current transaction result
+            last_transaction.clear()
+            for tsi in self.base.transaction:
+                # Avoid using the install_set() helper, as it does not guarantee
+                # a stable order
+                if tsi.action not in dnf.transaction.FORWARD_ACTIONS:
+                    continue
+                last_transaction.append(tsi.pkg)
+
+        dependencies = []
+        for package in last_transaction:
+            dependencies.append({
+                "name": package.name,
+                "epoch": package.epoch,
+                "version": package.version,
+                "release": package.release,
+                "arch": package.arch,
+                "repo_id": package.repoid,
+                "path": package.relativepath,
+                "remote_location": package.remote_location(),
+                "checksum": (
+                    f"{hawkey.chksum_name(package.chksum[0])}:"
+                    f"{package.chksum[1].hex()}"
+                )
+            })
+
+        return {
+            "checksums": self._repo_checksums(),
+            "dependencies": dependencies
+        }
+
 
 class DnfJsonRequestHandler(BaseHTTPRequestHandler):
     """
@@ -354,6 +407,10 @@ class DnfJsonRequestHandler(BaseHTTPRequestHandler):
                                 )
                             )
                             log.info("depsolve success")
+                        elif command == "chain-depsolve":
+                            self.response_success(
+                                solver.chain_depsolve(arguments["transactions"])
+                            )
 
                     except dnf.exceptions.MarkingErrors as e:
                         log.info("error install_specs")

internal/rpmmd/repository.go

@@ -11,6 +11,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"reflect"
 	"sort"
 	"strconv"
 	"strings"
@@ -113,6 +114,14 @@ type PackageSet struct {
 	Exclude []string
 }
 
+// The input to chain depsolve request. A set of packages to include / exclude
+// and a set of repository IDs to use, which are represented as indexes to
+// an array of repositories provided together with this request.
+type chainPackageSet struct {
+	PackageSet
+	Repos []int
+}
+
 // Append the Include and Exclude package list from another PackageSet and
 // return the result.
 func (ps PackageSet) Append(other PackageSet) PackageSet {
@@ -534,6 +543,155 @@ func (r *rpmmdImpl) Depsolve(packageSet PackageSet, repos []RepoConfig, modulePl
 	return dependencies, reply.Checksums, err
 }
 
+// ChainPackageSets constructs an array of `ChainPackageSet` based on the provided
+// arguments. The provided `packageSets` map is transformed into an array based
+// on the order of package set names passed in `packageSetsChain`. Repositories
+// provided in `repos` are used for every transaction, while repositories from
+// `packageSetsRepos` are used only for package sets with the respective name.
+//
+// The function returns a `ChainPackageSet` slice, which members are referencing
+// repositories from the returned `RepoConfig` by their index. The returned
+// `RepoConfig` slice is specific to the requested `packageSetsChain`.
+//
+// NOTE: Due to implementation limitations of DNF and dnf-json, each package set
+// in the chain must use all of the repositories used by its predecessor.
+// An error is returned if this requirement is not met.
+func chainPackageSets(packageSetsChain []string, packageSets map[string]PackageSet, repos []RepoConfig, packageSetsRepos map[string][]RepoConfig) ([]chainPackageSet, []RepoConfig, error) {
+	transactions := make([]chainPackageSet, 0, len(packageSetsChain))
+	transactionsRepos := make([]RepoConfig, 0, len(repos))
+
+	transactionsRepos = append(transactionsRepos, repos...)
+
+	// These repo IDs will be used for every transaction
+	baseRepoIDs := make([]int, 0, len(repos))
+	for idx := range repos {
+		baseRepoIDs = append(baseRepoIDs, idx)
+	}
+
+	for transactionIdx, pkgSetName := range packageSetsChain {
+		pkgSet, ok := packageSets[pkgSetName]
+		if !ok {
+			return nil, nil, fmt.Errorf("package set %q requested in the 'packageSetsChain' does not exist in provided 'packageSets'", pkgSetName)
+		}
+
+		transaction := chainPackageSet{
+			PackageSet: pkgSet,
+			Repos:      baseRepoIDs, // Due to its capacity, the slice will be copied if any repo is appended
+		}
+
+		// Add any package-set-specific repos to the list of transaction repos
+		if pkgSetRepos, ok := packageSetsRepos[pkgSetName]; ok {
+			for _, pkgSetRepo := range pkgSetRepos {
+				// Check if the repo has been already used by a transaction
+				// and if yes, just use its ID. Skip the "base" repos.
+				pkgSetRepoID := -1
+				for idx := len(repos); idx < len(transactionsRepos); idx++ {
+					transactionRepo := transactionsRepos[idx]
+					if reflect.DeepEqual(pkgSetRepo, transactionRepo) {
+						pkgSetRepoID = idx
+						break
+					}
+				}
+
+				if pkgSetRepoID == -1 {
+					transactionsRepos = append(transactionsRepos, pkgSetRepo)
+					pkgSetRepoID = len(transactionsRepos) - 1
+				}
+
+				transaction.Repos = append(transaction.Repos, pkgSetRepoID)
+			}
+		}
+
+		// Sort the slice of repo IDs to make it easier to compare
+		sort.Ints(transaction.Repos)
+
+		// If more than one transaction, ensure that the transaction uses
+		// all of the repos from its predecessor
+		if transactionIdx > 0 {
+			previousTransRepos := transactions[transactionIdx-1].Repos
+			if len(transaction.Repos) < len(previousTransRepos) {
+				return nil, nil, fmt.Errorf("chained packageSet %q does not use all of the repos used by its predecessor", pkgSetName)
+			}
+
+			for idx, repoID := range previousTransRepos {
+				if repoID != transaction.Repos[idx] {
+					return nil, nil, fmt.Errorf("chained packageSet %q does not use all of the repos used by its predecessor", pkgSetName)
+				}
+			}
+		}
+
+		transactions = append(transactions, transaction)
+	}
+
+	return transactions, transactionsRepos, nil
+}
+
+// ChainDepsolve takes a list of required package sets (included and excluded), which should be depsolved
+// as separate transactions, list of repositories, platform ID for modularity, architecture and release version.
+// It returns a list of all packages (with solved dependencies) that will be installed into the system.
+func (r *rpmmdImpl) chainDepsolve(chains []chainPackageSet, repos []RepoConfig, modulePlatformID, arch, releasever string) ([]PackageSpec, map[string]string, error) {
+	var dnfRepoConfigs []dnfRepoConfig
+	for i, repo := range repos {
+		dnfRepo, err := repo.toDNFRepoConfig(r, i, arch, releasever)
+		if err != nil {
+			return nil, nil, err
+		}
+		dnfRepoConfigs = append(dnfRepoConfigs, dnfRepo)
+	}
+
+	type dnfTransaction struct {
+		PackageSpecs []string `json:"package-specs"`
+		ExcludSpecs  []string `json:"exclude-specs"`
+		Repos        []int    `json:"repos"`
+	}
+	var dnfTransactions []dnfTransaction
+	for _, transaction := range chains {
+		dnfTransactions = append(dnfTransactions, dnfTransaction{
+			PackageSpecs: transaction.Include,
+			ExcludSpecs:  transaction.Exclude,
+			Repos:        transaction.Repos,
+		})
+	}
+
+	var arguments = struct {
+		Transactions     []dnfTransaction `json:"transactions"`
+		Repos            []dnfRepoConfig  `json:"repos"`
+		CacheDir         string           `json:"cachedir"`
+		ModulePlatformID string           `json:"module_platform_id"`
+		Arch             string           `json:"arch"`
+	}{dnfTransactions, dnfRepoConfigs, r.CacheDir, modulePlatformID, arch}
+
+	var reply struct {
+		Checksums    map[string]string `json:"checksums"`
+		Dependencies []dnfPackageSpec  `json:"dependencies"`
+	}
+
+	err := runDNF("chain-depsolve", arguments, &reply)
+
+	dependencies := make([]PackageSpec, len(reply.Dependencies))
+	for i, pack := range reply.Dependencies {
+		id, err := strconv.Atoi(pack.RepoID)
+		if err != nil {
+			panic(err)
+		}
+		repo := repos[id]
+		dep := reply.Dependencies[i]
+		dependencies[i].Name = dep.Name
+		dependencies[i].Epoch = dep.Epoch
+		dependencies[i].Version = dep.Version
+		dependencies[i].Release = dep.Release
+		dependencies[i].Arch = dep.Arch
+		dependencies[i].RemoteLocation = dep.RemoteLocation
+		dependencies[i].Checksum = dep.Checksum
+		dependencies[i].CheckGPG = repo.CheckGPG
+		if repo.RHSM {
+			dependencies[i].Secrets = "org.osbuild.rhsm"
+		}
+	}
+
+	return dependencies, reply.Checksums, err
+}
+
 func (packages PackageList) Search(globPatterns ...string) (PackageList, error) {
 	var globs []glob.Glob
 

internal/rpmmd/repository_internal_test.go

@@ -0,0 +1,396 @@
+package rpmmd
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestChainPackageSets(t *testing.T) {
+	tests := []struct {
+		packageSetsChain []string
+		packageSets      map[string]PackageSet
+		repos            []RepoConfig
+		packageSetsRepos map[string][]RepoConfig
+		wantChainPkgSets []chainPackageSet
+		wantRepos        []RepoConfig
+		err              bool
+	}{
+		// single transaction
+		{
+			packageSetsChain: []string{"os"},
+			packageSets: map[string]PackageSet{
+				"os": {
+					Include: []string{"pkg1"},
+					Exclude: []string{"pkg2"},
+				},
+			},
+			repos: []RepoConfig{
+				{
+					Name:    "baseos",
+					BaseURL: "https://example.org/baseos",
+				},
+				{
+					Name:    "appstream",
+					BaseURL: "https://example.org/appstream",
+				},
+			},
+			wantChainPkgSets: []chainPackageSet{
+				{
+					PackageSet: PackageSet{
+						Include: []string{"pkg1"},
+						Exclude: []string{"pkg2"},
+					},
+					Repos: []int{0, 1},
+				},
+			},
+			wantRepos: []RepoConfig{
+				{
+					Name:    "baseos",
+					BaseURL: "https://example.org/baseos",
+				},
+				{
+					Name:    "appstream",
+					BaseURL: "https://example.org/appstream",
+				},
+			},
+		},
+		// 2 transactions + package set specific repo
+		{
+			packageSetsChain: []string{"os", "blueprint"},
+			packageSets: map[string]PackageSet{
+				"os": {
+					Include: []string{"pkg1"},
+					Exclude: []string{"pkg2"},
+				},
+				"blueprint": {
+					Include: []string{"pkg3"},
+				},
+			},
+			repos: []RepoConfig{
+				{
+					Name:    "baseos",
+					BaseURL: "https://example.org/baseos",
+				},
+				{
+					Name:    "appstream",
+					BaseURL: "https://example.org/appstream",
+				},
+			},
+			packageSetsRepos: map[string][]RepoConfig{
+				"blueprint": {
+					{
+						Name:    "user-repo",
+						BaseURL: "https://example.org/user-repo",
+					},
+				},
+			},
+			wantChainPkgSets: []chainPackageSet{
+				{
+					PackageSet: PackageSet{
+						Include: []string{"pkg1"},
+						Exclude: []string{"pkg2"},
+					},
+					Repos: []int{0, 1},
+				},
+				{
+					PackageSet: PackageSet{
+						Include: []string{"pkg3"},
+					},
+					Repos: []int{0, 1, 2},
+				},
+			},
+			wantRepos: []RepoConfig{
+				{
+					Name:    "baseos",
+					BaseURL: "https://example.org/baseos",
+				},
+				{
+					Name:    "appstream",
+					BaseURL: "https://example.org/appstream",
+				},
+				{
+					Name:    "user-repo",
+					BaseURL: "https://example.org/user-repo",
+				},
+			},
+		},
+		// 2 transactions + no package set specific repos
+		{
+			packageSetsChain: []string{"os", "blueprint"},
+			packageSets: map[string]PackageSet{
+				"os": {
+					Include: []string{"pkg1"},
+					Exclude: []string{"pkg2"},
+				},
+				"blueprint": {
+					Include: []string{"pkg3"},
+				},
+			},
+			repos: []RepoConfig{
+				{
+					Name:    "baseos",
+					BaseURL: "https://example.org/baseos",
+				},
+				{
+					Name:    "appstream",
+					BaseURL: "https://example.org/appstream",
+				},
+			},
+			wantChainPkgSets: []chainPackageSet{
+				{
+					PackageSet: PackageSet{
+						Include: []string{"pkg1"},
+						Exclude: []string{"pkg2"},
+					},
+					Repos: []int{0, 1},
+				},
+				{
+					PackageSet: PackageSet{
+						Include: []string{"pkg3"},
+					},
+					Repos: []int{0, 1},
+				},
+			},
+			wantRepos: []RepoConfig{
+				{
+					Name:    "baseos",
+					BaseURL: "https://example.org/baseos",
+				},
+				{
+					Name:    "appstream",
+					BaseURL: "https://example.org/appstream",
+				},
+			},
+		},
+		// 3 transactions + package set specific repo used by 2nd and 3rd transaction
+		{
+			packageSetsChain: []string{"os", "blueprint", "blueprint2"},
+			packageSets: map[string]PackageSet{
+				"os": {
+					Include: []string{"pkg1"},
+					Exclude: []string{"pkg2"},
+				},
+				"blueprint": {
+					Include: []string{"pkg3"},
+				},
+				"blueprint2": {
+					Include: []string{"pkg4"},
+				},
+			},
+			repos: []RepoConfig{
+				{
+					Name:    "baseos",
+					BaseURL: "https://example.org/baseos",
+				},
+				{
+					Name:    "appstream",
+					BaseURL: "https://example.org/appstream",
+				},
+			},
+			packageSetsRepos: map[string][]RepoConfig{
+				"blueprint": {
+					{
+						Name:    "user-repo",
+						BaseURL: "https://example.org/user-repo",
+					},
+				},
+				"blueprint2": {
+					{
+						Name:    "user-repo",
+						BaseURL: "https://example.org/user-repo",
+					},
+				},
+			},
+			wantChainPkgSets: []chainPackageSet{
+				{
+					PackageSet: PackageSet{
+						Include: []string{"pkg1"},
+						Exclude: []string{"pkg2"},
+					},
+					Repos: []int{0, 1},
+				},
+				{
+					PackageSet: PackageSet{
+						Include: []string{"pkg3"},
+					},
+					Repos: []int{0, 1, 2},
+				},
+				{
+					PackageSet: PackageSet{
+						Include: []string{"pkg4"},
+					},
+					Repos: []int{0, 1, 2},
+				},
+			},
+			wantRepos: []RepoConfig{
+				{
+					Name:    "baseos",
+					BaseURL: "https://example.org/baseos",
+				},
+				{
+					Name:    "appstream",
+					BaseURL: "https://example.org/appstream",
+				},
+				{
+					Name:    "user-repo",
+					BaseURL: "https://example.org/user-repo",
+				},
+			},
+		},
+		// 3 transactions + package set specific repo used by 2nd and 3rd transaction
+		// + 3rd transaction using another repo
+		{
+			packageSetsChain: []string{"os", "blueprint", "blueprint2"},
+			packageSets: map[string]PackageSet{
+				"os": {
+					Include: []string{"pkg1"},
+					Exclude: []string{"pkg2"},
+				},
+				"blueprint": {
+					Include: []string{"pkg3"},
+				},
+				"blueprint2": {
+					Include: []string{"pkg4"},
+				},
+			},
+			repos: []RepoConfig{
+				{
+					Name:    "baseos",
+					BaseURL: "https://example.org/baseos",
+				},
+				{
+					Name:    "appstream",
+					BaseURL: "https://example.org/appstream",
+				},
+			},
+			packageSetsRepos: map[string][]RepoConfig{
+				"blueprint": {
+					{
+						Name:    "user-repo",
+						BaseURL: "https://example.org/user-repo",
+					},
+				},
+				"blueprint2": {
+					{
+						Name:    "user-repo",
+						BaseURL: "https://example.org/user-repo",
+					},
+					{
+						Name:    "user-repo-2",
+						BaseURL: "https://example.org/user-repo-2",
+					},
+				},
+			},
+			wantChainPkgSets: []chainPackageSet{
+				{
+					PackageSet: PackageSet{
+						Include: []string{"pkg1"},
+						Exclude: []string{"pkg2"},
+					},
+					Repos: []int{0, 1},
+				},
+				{
+					PackageSet: PackageSet{
+						Include: []string{"pkg3"},
+					},
+					Repos: []int{0, 1, 2},
+				},
+				{
+					PackageSet: PackageSet{
+						Include: []string{"pkg4"},
+					},
+					Repos: []int{0, 1, 2, 3},
+				},
+			},
+			wantRepos: []RepoConfig{
+				{
+					Name:    "baseos",
+					BaseURL: "https://example.org/baseos",
+				},
+				{
+					Name:    "appstream",
+					BaseURL: "https://example.org/appstream",
+				},
+				{
+					Name:    "user-repo",
+					BaseURL: "https://example.org/user-repo",
+				},
+				{
+					Name:    "user-repo-2",
+					BaseURL: "https://example.org/user-repo-2",
+				},
+			},
+		},
+		// Error: 3 transactions + 3rd one not using repo used by 2nd one
+		{
+			packageSetsChain: []string{"os", "blueprint", "blueprint2"},
+			packageSets: map[string]PackageSet{
+				"os": {
+					Include: []string{"pkg1"},
+					Exclude: []string{"pkg2"},
+				},
+				"blueprint": {
+					Include: []string{"pkg3"},
+				},
+				"blueprint2": {
+					Include: []string{"pkg4"},
+				},
+			},
+			repos: []RepoConfig{
+				{
+					Name:    "baseos",
+					BaseURL: "https://example.org/baseos",
+				},
+				{
+					Name:    "appstream",
+					BaseURL: "https://example.org/appstream",
+				},
+			},
+			packageSetsRepos: map[string][]RepoConfig{
+				"blueprint": {
+					{
+						Name:    "user-repo",
+						BaseURL: "https://example.org/user-repo",
+					},
+				},
+				"blueprint2": {
+					{
+						Name:    "user-repo2",
+						BaseURL: "https://example.org/user-repo2",
+					},
+				},
+			},
+			err: true,
+		},
+		// Error: requested package set name to chain not defined in provided pkg sets
+		{
+			packageSetsChain: []string{"os", "blueprint"},
+			packageSets: map[string]PackageSet{
+				"os": {
+					Include: []string{"pkg1"},
+					Exclude: []string{"pkg2"},
+				},
+			},
+			err: true,
+		},
+	}
+	for idx, tt := range tests {
+		t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) {
+			gotChainPackageSets, gotRepos, err := chainPackageSets(tt.packageSetsChain, tt.packageSets, tt.repos, tt.packageSetsRepos)
+			if tt.err {
+				assert.NotNilf(t, err, "expected an error, but got 'nil' instead")
+				assert.Nilf(t, gotChainPackageSets, "got non-nill []rpmmd.ChainPackageSet, but expected an error")
+				assert.Nilf(t, gotRepos, "got non-nill []rpmmd.RepoConfig, but expected an error")
+			} else {
+				assert.Nilf(t, err, "expected 'nil', but got error instead")
+				assert.NotNilf(t, gotChainPackageSets, "expected non-nill []rpmmd.ChainPackageSet, but got 'nil' instead")
+				assert.NotNilf(t, gotRepos, "expected non-nill []rpmmd.RepoConfig, but got 'nil' instead")
+
+				assert.Equal(t, tt.wantChainPkgSets, gotChainPackageSets)
+				assert.Equal(t, tt.wantRepos, gotRepos)
+			}
+		})
+	}
+}