ae72480612
This commit includes the used sovler in the dnfjson reply. This is mostly information (e.g. in service logs) but also useful in tests to ensure that the expected solver was really run. Note that this needs https://github.com/osbuild/images/pull/723 first.
313 lines
13 KiB
Python
Executable file
313 lines
13 KiB
Python
Executable file
import os
|
|
import os.path
|
|
import tempfile
|
|
from datetime import datetime
|
|
from typing import List
|
|
|
|
import dnf
|
|
import hawkey
|
|
|
|
from osbuild.solver import DepsolveError, MarkingError, RepoError, SolverBase, modify_rootdir_path, read_keys
|
|
|
|
|
|
class DNF(SolverBase):
|
|
def __init__(self, request, persistdir, cache_dir):
|
|
arch = request["arch"]
|
|
releasever = request.get("releasever")
|
|
module_platform_id = request["module_platform_id"]
|
|
proxy = request.get("proxy")
|
|
|
|
arguments = request["arguments"]
|
|
repos = arguments.get("repos", [])
|
|
root_dir = arguments.get("root_dir")
|
|
|
|
self.base = dnf.Base()
|
|
|
|
# Enable fastestmirror to ensure we choose the fastest mirrors for
|
|
# downloading metadata (when depsolving) and downloading packages.
|
|
self.base.conf.fastestmirror = True
|
|
|
|
# We use the same cachedir for multiple architectures. Unfortunately,
|
|
# this is something that doesn't work well in certain situations
|
|
# with zchunk:
|
|
# Imagine that we already have cache for arch1. Then, we use dnf-json
|
|
# to depsolve for arch2. If ZChunk is enabled and available (that's
|
|
# the case for Fedora), dnf will try to download only differences
|
|
# between arch1 and arch2 metadata. But, as these are completely
|
|
# different, dnf must basically redownload everything.
|
|
# For downloding deltas, zchunk uses HTTP range requests. Unfortunately,
|
|
# if the mirror doesn't support multi range requests, then zchunk will
|
|
# download one small segment per a request. Because we need to update
|
|
# the whole metadata (10s of MB), this can be extremely slow in some cases.
|
|
# I think that we can come up with a better fix but let's just disable
|
|
# zchunk for now. As we are already downloading a lot of data when
|
|
# building images, I don't care if we download even more.
|
|
self.base.conf.zchunk = False
|
|
|
|
# Set the rest of the dnf configuration.
|
|
self.base.conf.module_platform_id = module_platform_id
|
|
self.base.conf.config_file_path = "/dev/null"
|
|
self.base.conf.persistdir = persistdir
|
|
self.base.conf.cachedir = cache_dir
|
|
self.base.conf.substitutions['arch'] = arch
|
|
self.base.conf.substitutions['basearch'] = dnf.rpm.basearch(arch)
|
|
self.base.conf.substitutions['releasever'] = releasever
|
|
if hasattr(self.base.conf, "optional_metadata_types"):
|
|
# the attribute doesn't exist on older versions of dnf; ignore the option when not available
|
|
self.base.conf.optional_metadata_types.extend(arguments.get("optional-metadata", []))
|
|
if proxy:
|
|
self.base.conf.proxy = proxy
|
|
|
|
try:
|
|
req_repo_ids = set()
|
|
for repo in repos:
|
|
self.base.repos.add(self._dnfrepo(repo, self.base.conf))
|
|
# collect repo IDs from the request to separate them from the ones loaded from a root_dir
|
|
req_repo_ids.add(repo["id"])
|
|
|
|
if root_dir:
|
|
# This sets the varsdir to ("{root_dir}/etc/yum/vars/", "{root_dir}/etc/dnf/vars/") for custom variable
|
|
# substitution (e.g. CentOS Stream 9's $stream variable)
|
|
self.base.conf.substitutions.update_from_etc(root_dir)
|
|
|
|
repos_dir = os.path.join(root_dir, "etc/yum.repos.d")
|
|
self.base.conf.reposdir = repos_dir
|
|
self.base.read_all_repos()
|
|
for repo_id, repo_config in self.base.repos.items():
|
|
if repo_id not in req_repo_ids:
|
|
repo_config.sslcacert = modify_rootdir_path(repo_config.sslcacert, root_dir)
|
|
repo_config.sslclientcert = modify_rootdir_path(repo_config.sslclientcert, root_dir)
|
|
repo_config.sslclientkey = modify_rootdir_path(repo_config.sslclientkey, root_dir)
|
|
|
|
self.base.fill_sack(load_system_repo=False)
|
|
except dnf.exceptions.Error as e:
|
|
raise RepoError from e
|
|
|
|
# pylint: disable=too-many-branches
|
|
@staticmethod
|
|
def _dnfrepo(desc, parent_conf=None):
|
|
"""Makes a dnf.repo.Repo out of a JSON repository description"""
|
|
|
|
repo = dnf.repo.Repo(desc["id"], parent_conf)
|
|
|
|
if "name" in desc:
|
|
repo.name = desc["name"]
|
|
|
|
# at least one is required
|
|
if "baseurl" in desc:
|
|
repo.baseurl = desc["baseurl"]
|
|
elif "metalink" in desc:
|
|
repo.metalink = desc["metalink"]
|
|
elif "mirrorlist" in desc:
|
|
repo.mirrorlist = desc["mirrorlist"]
|
|
else:
|
|
raise ValueError("missing either `baseurl`, `metalink`, or `mirrorlist` in repo")
|
|
|
|
repo.sslverify = desc.get("sslverify", True)
|
|
if "sslcacert" in desc:
|
|
repo.sslcacert = desc["sslcacert"]
|
|
if "sslclientkey" in desc:
|
|
repo.sslclientkey = desc["sslclientkey"]
|
|
if "sslclientcert" in desc:
|
|
repo.sslclientcert = desc["sslclientcert"]
|
|
|
|
if "gpgcheck" in desc:
|
|
repo.gpgcheck = desc["gpgcheck"]
|
|
if "repo_gpgcheck" in desc:
|
|
repo.repo_gpgcheck = desc["repo_gpgcheck"]
|
|
if "gpgkey" in desc:
|
|
repo.gpgkey = [desc["gpgkey"]]
|
|
if "gpgkeys" in desc:
|
|
# gpgkeys can contain a full key, or it can be a URL
|
|
# dnf expects urls, so write the key to a temporary location and add the file://
|
|
# path to repo.gpgkey
|
|
keydir = os.path.join(parent_conf.persistdir, "gpgkeys")
|
|
if not os.path.exists(keydir):
|
|
os.makedirs(keydir, mode=0o700, exist_ok=True)
|
|
|
|
for key in desc["gpgkeys"]:
|
|
if key.startswith("-----BEGIN PGP PUBLIC KEY BLOCK-----"):
|
|
# Not using with because it needs to be a valid file for the duration. It
|
|
# is inside the temporary persistdir so will be cleaned up on exit.
|
|
# pylint: disable=consider-using-with
|
|
keyfile = tempfile.NamedTemporaryFile(dir=keydir, delete=False)
|
|
keyfile.write(key.encode("utf-8"))
|
|
repo.gpgkey.append(f"file://{keyfile.name}")
|
|
keyfile.close()
|
|
else:
|
|
repo.gpgkey.append(key)
|
|
|
|
# In dnf, the default metadata expiration time is 48 hours. However,
|
|
# some repositories never expire the metadata, and others expire it much
|
|
# sooner than that. We therefore allow this to be configured. If nothing
|
|
# is provided we error on the side of checking if we should invalidate
|
|
# the cache. If cache invalidation is not necessary, the overhead of
|
|
# checking is in the hundreds of milliseconds. In order to avoid this
|
|
# overhead accumulating for API calls that consist of several dnf calls,
|
|
# we set the expiration to a short time period, rather than 0.
|
|
repo.metadata_expire = desc.get("metadata_expire", "20s")
|
|
|
|
# This option if True disables modularization filtering. Effectively
|
|
# disabling modularity for given repository.
|
|
if "module_hotfixes" in desc:
|
|
repo.module_hotfixes = desc["module_hotfixes"]
|
|
|
|
return repo
|
|
|
|
@staticmethod
|
|
def _timestamp_to_rfc3339(timestamp):
|
|
return datetime.utcfromtimestamp(timestamp).strftime('%Y-%m-%dT%H:%M:%SZ')
|
|
|
|
def dump(self):
|
|
packages = []
|
|
for package in self.base.sack.query().available():
|
|
packages.append({
|
|
"name": package.name,
|
|
"summary": package.summary,
|
|
"description": package.description,
|
|
"url": package.url,
|
|
"repo_id": package.repoid,
|
|
"epoch": package.epoch,
|
|
"version": package.version,
|
|
"release": package.release,
|
|
"arch": package.arch,
|
|
"buildtime": self._timestamp_to_rfc3339(package.buildtime),
|
|
"license": package.license
|
|
})
|
|
return packages
|
|
|
|
def search(self, args):
|
|
""" Perform a search on the available packages
|
|
|
|
args contains a "search" dict with parameters to use for searching.
|
|
"packages" list of package name globs to search for
|
|
"latest" is a boolean that will return only the latest NEVRA instead
|
|
of all matching builds in the metadata.
|
|
|
|
eg.
|
|
|
|
"search": {
|
|
"latest": false,
|
|
"packages": ["tmux", "vim*", "*ssh*"]
|
|
},
|
|
"""
|
|
pkg_globs = args.get("packages", [])
|
|
|
|
packages = []
|
|
|
|
# NOTE: Build query one piece at a time, don't pass all to filterm at the same
|
|
# time.
|
|
available = self.base.sack.query().available()
|
|
for name in pkg_globs:
|
|
# If the package name glob has * in it, use glob.
|
|
# If it has *name* use substr
|
|
# If it has neither use exact match
|
|
if "*" in name:
|
|
if name[0] != "*" or name[-1] != "*":
|
|
q = available.filter(name__glob=name)
|
|
else:
|
|
q = available.filter(name__substr=name.replace("*", ""))
|
|
else:
|
|
q = available.filter(name__eq=name)
|
|
|
|
if args.get("latest", False):
|
|
q = q.latest()
|
|
|
|
for package in q:
|
|
packages.append({
|
|
"name": package.name,
|
|
"summary": package.summary,
|
|
"description": package.description,
|
|
"url": package.url,
|
|
"repo_id": package.repoid,
|
|
"epoch": package.epoch,
|
|
"version": package.version,
|
|
"release": package.release,
|
|
"arch": package.arch,
|
|
"buildtime": self._timestamp_to_rfc3339(package.buildtime),
|
|
"license": package.license
|
|
})
|
|
return packages
|
|
|
|
def depsolve(self, arguments):
|
|
# # Return an empty list when 'transactions' key is missing or when it is None
|
|
transactions = arguments.get("transactions") or []
|
|
# collect repo IDs from the request so we know whether to translate gpg key paths
|
|
request_repo_ids = set(repo["id"] for repo in arguments.get("repos", []))
|
|
root_dir = arguments.get("root_dir")
|
|
last_transaction: List = []
|
|
|
|
for transaction in transactions:
|
|
self.base.reset(goal=True)
|
|
self.base.sack.reset_excludes()
|
|
|
|
self.base.conf.install_weak_deps = transaction.get("install_weak_deps", False)
|
|
|
|
try:
|
|
# set the packages from the last transaction as installed
|
|
for installed_pkg in last_transaction:
|
|
self.base.package_install(installed_pkg, strict=True)
|
|
|
|
# depsolve the current transaction
|
|
self.base.install_specs(
|
|
transaction.get("package-specs"),
|
|
transaction.get("exclude-specs"),
|
|
reponame=transaction.get("repo-ids"),
|
|
)
|
|
except dnf.exceptions.Error as e:
|
|
raise MarkingError from e
|
|
|
|
try:
|
|
self.base.resolve()
|
|
except dnf.exceptions.Error as e:
|
|
raise DepsolveError from e
|
|
|
|
# store the current transaction result
|
|
last_transaction.clear()
|
|
for tsi in self.base.transaction:
|
|
# Avoid using the install_set() helper, as it does not guarantee
|
|
# a stable order
|
|
if tsi.action not in dnf.transaction.FORWARD_ACTIONS:
|
|
continue
|
|
last_transaction.append(tsi.pkg)
|
|
|
|
packages = []
|
|
pkg_repos = {}
|
|
for package in last_transaction:
|
|
packages.append({
|
|
"name": package.name,
|
|
"epoch": package.epoch,
|
|
"version": package.version,
|
|
"release": package.release,
|
|
"arch": package.arch,
|
|
"repo_id": package.repoid,
|
|
"path": package.relativepath,
|
|
"remote_location": package.remote_location(),
|
|
"checksum": f"{hawkey.chksum_name(package.chksum[0])}:{package.chksum[1].hex()}",
|
|
})
|
|
# collect repository objects by id to create the 'repositories' collection for the response
|
|
pkgrepo = package.repo
|
|
pkg_repos[pkgrepo.id] = pkgrepo
|
|
|
|
repositories = {} # full repository configs for the response
|
|
for repo in pkg_repos.values():
|
|
repositories[repo.id] = {
|
|
"id": repo.id,
|
|
"name": repo.name,
|
|
"baseurl": list(repo.baseurl) if repo.baseurl else None,
|
|
"metalink": repo.metalink,
|
|
"mirrorlist": repo.mirrorlist,
|
|
"gpgcheck": repo.gpgcheck,
|
|
"repo_gpgcheck": repo.repo_gpgcheck,
|
|
"gpgkeys": read_keys(repo.gpgkey, root_dir if repo.id not in request_repo_ids else None),
|
|
"sslverify": bool(repo.sslverify),
|
|
"sslcacert": repo.sslcacert,
|
|
"sslclientkey": repo.sslclientkey,
|
|
"sslclientcert": repo.sslclientcert,
|
|
}
|
|
response = {
|
|
"solver": "dnf",
|
|
"packages": packages,
|
|
"repos": repositories,
|
|
}
|
|
return response
|