From 60ec19f6928fd547bb56b551dfb311bc4d6a4f21 Mon Sep 17 00:00:00 2001 From: Josue David Hernandez Gutierrez Date: Tue, 1 Apr 2025 13:33:33 -0600 Subject: [PATCH] osbuild/solver/dnf.py: Add support for DNF variables for osbuild repos Signed-off-by: Josue David Hernandez Gutierrez --- .mypy.ini | 3 ++ osbuild/solver/dnf.py | 28 ++++++++++----- tools/test/test_depsolve.py | 68 +++++++++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 9 deletions(-) diff --git a/.mypy.ini b/.mypy.ini index 1a0e5d67..dc863d6b 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -15,6 +15,9 @@ ignore_missing_imports = True [mypy-dnf.*] ignore_missing_imports = True +[mypy-libdnf.*] +ignore_missing_imports = True + [mypy-libdnf5.*] ignore_missing_imports = True diff --git a/osbuild/solver/dnf.py b/osbuild/solver/dnf.py index 3fc01872..e51e16e7 100755 --- a/osbuild/solver/dnf.py +++ b/osbuild/solver/dnf.py @@ -10,6 +10,8 @@ from typing import Dict, List import dnf import hawkey +import libdnf +from dnf.i18n import ucd from osbuild.solver import ( DepsolveError, @@ -68,6 +70,12 @@ class DNF(SolverBase): self.base.conf.substitutions['basearch'] = dnf.rpm.basearch(arch) self.base.conf.substitutions['releasever'] = releasever + # variables substitution is only available when root_dir is provided + if root_dir: + # This sets the varsdir to ("{root_dir}/etc/yum/vars/", "{root_dir}/etc/dnf/vars/") for custom variable + # substitution (e.g. CentOS Stream 9's $stream variable) + self.base.conf.substitutions.update_from_etc(root_dir) + if hasattr(self.base.conf, "optional_metadata_types"): # the attribute doesn't exist on older versions of dnf; ignore the option when not available self.base.conf.optional_metadata_types.extend(arguments.get("optional-metadata", [])) @@ -77,15 +85,11 @@ class DNF(SolverBase): try: req_repo_ids = set() for repo in repos: - self.base.repos.add(self._dnfrepo(repo, self.base.conf)) + self.base.repos.add(self._dnfrepo(repo, self.base.conf, root_dir is not None)) # collect repo IDs from the request to separate them from the ones loaded from a root_dir req_repo_ids.add(repo["id"]) if root_dir: - # This sets the varsdir to ("{root_dir}/etc/yum/vars/", "{root_dir}/etc/dnf/vars/") for custom variable - # substitution (e.g. CentOS Stream 9's $stream variable) - self.base.conf.substitutions.update_from_etc(root_dir) - repos_dir = os.path.join(root_dir, "etc/yum.repos.d") self.base.conf.reposdir = repos_dir self.base.read_all_repos() @@ -110,21 +114,27 @@ class DNF(SolverBase): self.license_index_path = license_index_path @staticmethod - def _dnfrepo(desc, parent_conf=None): + def _dnfrepo(desc, parent_conf=None, subs_links=False): """Makes a dnf.repo.Repo out of a JSON repository description""" repo = dnf.repo.Repo(desc["id"], parent_conf) + config = libdnf.conf.ConfigParser if "name" in desc: repo.name = desc["name"] + def subs(basestr): + if subs_links and parent_conf: + return config.substitute(ucd(basestr), parent_conf.substitutions) + return basestr + # at least one is required if "baseurl" in desc: - repo.baseurl = desc["baseurl"] + repo.baseurl = [subs(repo) for repo in desc["baseurl"]] elif "metalink" in desc: - repo.metalink = desc["metalink"] + repo.metalink = subs(desc["metalink"]) elif "mirrorlist" in desc: - repo.mirrorlist = desc["mirrorlist"] + repo.mirrorlist = subs(desc["mirrorlist"]) else: raise ValueError("missing either `baseurl`, `metalink`, or `mirrorlist` in repo") diff --git a/tools/test/test_depsolve.py b/tools/test/test_depsolve.py index 960f4883..375c667d 100644 --- a/tools/test/test_depsolve.py +++ b/tools/test/test_depsolve.py @@ -1544,6 +1544,74 @@ def test_depsolve_config_combos(tmp_path, repo_servers, dnf_config, detect_fn): assert res["solver"] == "dnf" +def set_config_dnfvars(baseurl, dnfvars): + for j, url in enumerate(baseurl): + for var, value in dnfvars.items(): + if value in url: + baseurl[j] = url.replace(value, f"${var}") + return baseurl + + +def create_dnfvars(root_dir, dnfvars): + vars_dir = root_dir / "etc/dnf/vars" + vars_dir.mkdir(parents=True) + + for var, value in dnfvars.items(): + var_path = vars_dir / var + var_path.write_text(value, encoding="utf8") + + +@pytest.mark.parametrize("use_dnfvars", [True, False], ids=["with_dnfvars", "without_dnfvars"]) +@pytest.mark.parametrize("dnf_config, detect_fn", [ + ({}, assert_dnf), + ({"use_dnf5": False}, assert_dnf), + ({"use_dnf5": True}, assert_dnf5), +], ids=["no-config", "dnf4", "dnf5"]) +def test_depsolve_dnfvars(tmp_path, repo_servers, dnf_config, detect_fn, use_dnfvars): + try: + detect_fn() + except RuntimeError as e: + pytest.skip(str(e)) + + test_case = depsolve_test_case_basic_2pkgs_2repos + transactions = test_case["transactions"] + repo_configs = get_test_case_repo_configs(test_case, repo_servers) + root_dir = None + + for index, config in enumerate(repo_configs): + repo_configs[index]["baseurl"] = set_config_dnfvars(config["baseurl"], {"var": "localhost"}) + + if use_dnfvars: + create_dnfvars(tmp_path, {"var": "localhost"}) + root_dir = str(tmp_path) + + res, exit_code = depsolve(transactions, tmp_path.as_posix(), dnf_config, repo_configs, root_dir=root_dir) + + if not use_dnfvars: + assert exit_code != 0 + assert res["kind"] == "RepoError" + assert re.match( + "There was a problem reading a repository: Failed to download metadata", res["reason"], re.DOTALL) + return + + assert exit_code == 0 + assert {pkg["name"] for pkg in res["packages"]} == test_case["results"]["packages"] + assert res["repos"].keys() == test_case["results"]["reponames"] + + # modules is optional here as the dnf5 depsolver never returns any modules + assert res.get("modules", {}).keys() == test_case["results"].get("modules", set()) + + for repo in res["repos"].values(): + assert repo["gpgkeys"] == [TEST_KEY + repo["id"]] + assert repo["sslverify"] is False + + use_dnf5 = dnf_config.get("use_dnf5", False) + if use_dnf5: + assert res["solver"] == "dnf5" + else: + assert res["solver"] == "dnf" + + # pylint: disable=too-many-branches @pytest.mark.parametrize("custom_license_db", [None, "./test/data/spdx/custom-license-index.json"]) @pytest.mark.parametrize("with_sbom", [False, True])