stages/dnf: drop stage
This has now been entirely replaced by the rpm stage in all its users. The dnf stage does not fit very nicely into the osbuild module, in particular it requires direct network access, which we would like to avoid. Signed-off-by: Tom Gundersen <teg@jklm.no>
This commit is contained in:
Tom Gundersen
parent
7e80ca9bbe
commit
892342b978
1 changed files with 0 additions and 313 deletions
|
|
@ -1,313 +0,0 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
import osbuild.sources
|
||||
|
||||
STAGE_DESC = "Install packages using DNF"
|
||||
STAGE_INFO = """
|
||||
Depsolves, downloads, and installs packages (and dependencies) using DNF.
|
||||
|
||||
Writes the `repos` into `/tmp/dnf.conf`, does some tree setup, and then runs
|
||||
the buildhost's `dnf` command with `--installroot`, plus the following
|
||||
arguments generated from the stage options:
|
||||
|
||||
* `--forcearch {basearch}`
|
||||
* `--releasever {releasever}`
|
||||
* `--setopt install_weak_deps={install_weak_deps}`
|
||||
* `--setopt fastestmirror={fastestmirror}`
|
||||
* `--config /tmp/dnf.conf`
|
||||
* `--exclude {pkg}` for each item in `exclude_packages`
|
||||
|
||||
Also disables the "generate_completion_cache" plugin, and sets `reposdir` to ""
|
||||
to ensure the buildhost's repo files are *not* being used.
|
||||
|
||||
To prepare the tree, this stage sets `/etc/machine-id` to "ffff..." (32 chars)
|
||||
and bind-mounts `/dev`, `/sys`, and `/proc` from the buildhost into the tree.
|
||||
|
||||
Each repo listed in `repos` needs to have a `checksum` and at least one of
|
||||
`mirrorlist`, `metalink`, or `baseurl`. If a `gpgkey` is provided, `gpgcheck`
|
||||
will be turned on for that repo, and DNF will exit with an error unless every
|
||||
package downloaded from that repo is signed by one of the trusted `gpgkey`s.
|
||||
|
||||
The provided `checksum` must start with "sha256:" and then have the hex-encoded
|
||||
SHA256 of the repo's `repomd.xml` file. If the metadata for any repo has
|
||||
changed and no longer matches `checksum`, this stage will fail after package
|
||||
installation.
|
||||
|
||||
NOTE: Any pipeline that uses a repo that changes frequently (like Fedora's
|
||||
"updates") will quickly become un-reproduceable. This is an unavoidable
|
||||
consequence of Fedora removing "out-of-date" metadata and packages: it's
|
||||
impossible to reproduce a build that requires files that have been deleted.
|
||||
To quote Douglas Adams: "We apologize for the inconvenience."
|
||||
|
||||
After DNF finishes, this stage cleans up the tree by removing
|
||||
`/etc/machine-id`, `/var/lib/systemd/random-seed`, and everything under
|
||||
`/var/cache/dnf`.
|
||||
|
||||
Buildhost commands used: `/bin/sh`, `dnf`, `mkdir`, `mount`, `chmod`.
|
||||
"""
|
||||
STAGE_OPTS = """
|
||||
"required": ["repos", "packages", "releasever", "basearch"],
|
||||
"properties": {
|
||||
"repos": {
|
||||
"description": "Array of checksums to repository sources or repo objects",
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"items": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"metalink": {
|
||||
"description": "metalink URL for this repo",
|
||||
"type": "string"
|
||||
},
|
||||
"mirrorlist": {
|
||||
"description": "mirrorlist URL for this repo",
|
||||
"type": "string"
|
||||
},
|
||||
"baseurl": {
|
||||
"description": "baseurl for this repo",
|
||||
"type": "string"
|
||||
},
|
||||
"checksum": {
|
||||
"description": "checksum for the expected repo metadata",
|
||||
"type": "string",
|
||||
"pattern": "sha256:[a-fA-F0-9]{32}"
|
||||
},
|
||||
"gpgkey": {
|
||||
"description": "GPG public key contents (to check signatures)",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"anyOf": [
|
||||
{"required": ["checksum", "metalink"]},
|
||||
{"required": ["checksum", "mirrorlist"]},
|
||||
{"required": ["checksum", "baseurl"]}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "string"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"packages": {
|
||||
"description": "List of package-specs to pass to DNF",
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"items": { "type": "string" }
|
||||
},
|
||||
"releasever": {
|
||||
"description": "DNF $releasever value",
|
||||
"type": "string"
|
||||
},
|
||||
"basearch": {
|
||||
"description": "DNF $basearch value",
|
||||
"type": "string"
|
||||
},
|
||||
"operation": {
|
||||
"description": "DNF command to use",
|
||||
"type": "string",
|
||||
"default": "install"
|
||||
},
|
||||
"install_weak_deps": {
|
||||
"description": "Whether DNF should install weak deps",
|
||||
"type": "boolean",
|
||||
"default": true
|
||||
},
|
||||
"fastestmirror": {
|
||||
"description": "Whether DNF should choose the fastest mirror available",
|
||||
"type": "boolean",
|
||||
"default": true
|
||||
},
|
||||
"exclude_packages": {
|
||||
"description": "List of package-specs to --exclude",
|
||||
"type": "array",
|
||||
"items": { "type": "string" },
|
||||
"default": []
|
||||
},
|
||||
"module_platform_id": {
|
||||
"description": "DNF's module_platform_id option. Corresponds to PLATFORM_ID from /etc/os-release",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
def fetch_repos(repos):
|
||||
"""Use osbuild's source api to fill in details about those repos in @repos
|
||||
that only have a "checksum" set. All others are passed through unchanged.
|
||||
"""
|
||||
checksums = []
|
||||
result = []
|
||||
for repo in repos:
|
||||
if isinstance(repo, str):
|
||||
checksums.append(repo)
|
||||
else:
|
||||
result.append(repo)
|
||||
|
||||
return result + osbuild.sources.get("org.osbuild.dnf", checksums)
|
||||
|
||||
|
||||
def write_repofile(f, repoid, repo, keydir):
|
||||
f.write(f"[{repoid}]\n")
|
||||
|
||||
def write_option(key, value):
|
||||
f.write(f"{key}={value}\n")
|
||||
|
||||
# silence dnf warning about missing name
|
||||
write_option("name", repoid)
|
||||
|
||||
for key in ("metalink", "mirrorlist", "baseurl"):
|
||||
value = repo.get(key)
|
||||
if value:
|
||||
write_option(key, value)
|
||||
|
||||
for cert in ("sslcacert", "sslclientcert", "sslclientkey"):
|
||||
if cert in repo:
|
||||
path = f"{keydir}/{cert}.pem"
|
||||
with open(path, "w") as certfile:
|
||||
certfile.write(repo[cert])
|
||||
write_option(cert, path)
|
||||
|
||||
if "gpgkey" in repo:
|
||||
keyfile = f"{keydir}/{repoid}.asc"
|
||||
with open(keyfile, "w") as key:
|
||||
key.write(repo["gpgkey"])
|
||||
write_option("gpgcheck", 1)
|
||||
write_option("gpgkey", f"file://{keyfile}")
|
||||
|
||||
|
||||
def dnf_cachedir(repoid, repo, releasever, basearch):
|
||||
"""Return the relative cache directory for a repository.
|
||||
|
||||
Using the same algorithm as libdnf:
|
||||
|
||||
https://github.com/rpm-software-management/libdnf/blob/master/libdnf/repo/Repo.cpp#L1288
|
||||
"""
|
||||
|
||||
if "metalink" in repo:
|
||||
url = repo["metalink"]
|
||||
elif "mirrorlist" in repo:
|
||||
url = repo["mirrorlist"]
|
||||
elif "baseurl" in repo:
|
||||
url = repo["baseurl"]
|
||||
else:
|
||||
raise RuntimeError(f"one of metalink, mirrorlist, or baseurl must be given for repository '{repoid}'")
|
||||
|
||||
url = url.replace("$basearch", basearch).replace("$releasever", releasever)
|
||||
digest = hashlib.sha256(url.encode()).hexdigest()[:16]
|
||||
|
||||
return f"{repoid}-{digest}"
|
||||
|
||||
|
||||
def main(tree, options):
|
||||
repos = fetch_repos(options["repos"])
|
||||
packages = options["packages"]
|
||||
releasever = options["releasever"]
|
||||
basearch = options["basearch"]
|
||||
operation = options.get("operation", "install")
|
||||
weak_deps = options.get("install_weak_deps", True)
|
||||
fastestmirror = options.get("fastestmirror", True)
|
||||
exclude_packages = options.get("exclude_packages", [])
|
||||
module_platform_id = options.get("module_platform_id", None)
|
||||
|
||||
script = f"""
|
||||
set -e
|
||||
mkdir -p {tree}/dev {tree}/sys {tree}/proc
|
||||
mount -o bind /dev {tree}/dev
|
||||
mount -o bind /sys {tree}/sys
|
||||
mount -o bind /proc {tree}/proc
|
||||
"""
|
||||
|
||||
machine_id_set_previously = os.path.exists(f"{tree}/etc/machine-id")
|
||||
if not machine_id_set_previously:
|
||||
# create a fake machine ID to improve reproducibility
|
||||
print("creating a fake machine id")
|
||||
script += f"""
|
||||
mkdir -p {tree}/etc
|
||||
echo "ffffffffffffffffffffffffffffffff" > {tree}/etc/machine-id
|
||||
chmod 0444 {tree}/etc/machine-id
|
||||
"""
|
||||
|
||||
try:
|
||||
subprocess.run(["/bin/sh", "-c", script], check=True)
|
||||
except subprocess.CalledProcessError as err:
|
||||
print(f"setting up API VFS in target tree failed: {err.returncode}")
|
||||
return err.returncode
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="org.osbuild.dnf.") as confdir:
|
||||
dnfconf = f"{confdir}/dnf.conf"
|
||||
|
||||
with open(dnfconf, "w") as conf:
|
||||
if module_platform_id:
|
||||
conf.write("[main]\n")
|
||||
conf.write(f"module_platform_id={module_platform_id}\n")
|
||||
for num, repo in enumerate(repos):
|
||||
write_repofile(conf, f"repo{num}", repo, confdir)
|
||||
|
||||
base_cmd = [
|
||||
"dnf", "-yv",
|
||||
"--installroot", tree,
|
||||
"--forcearch", basearch,
|
||||
"--setopt", "reposdir=",
|
||||
"--setopt", f"install_weak_deps={weak_deps}",
|
||||
"--setopt", f"fastestmirror={fastestmirror}",
|
||||
"--setopt", f"skip_if_unavailable=false",
|
||||
"--releasever", releasever,
|
||||
"--noplugins",
|
||||
"--config", dnfconf
|
||||
]
|
||||
|
||||
cmd = base_cmd + [operation] + packages
|
||||
for x in exclude_packages:
|
||||
cmd += ["--exclude", x]
|
||||
print(" ".join(cmd), flush=True)
|
||||
subprocess.run(cmd, check=True)
|
||||
|
||||
# verify metadata checksum
|
||||
for repoid, repo in enumerate(repos):
|
||||
algorithm, expected_checksum = repo["checksum"].split(":")
|
||||
assert algorithm == "sha256"
|
||||
cachedir = dnf_cachedir(f"repo{repoid}", repo, releasever, basearch)
|
||||
with open(f"{tree}/var/cache/dnf/{cachedir}/repodata/repomd.xml", "rb") as f:
|
||||
repomd = f.read()
|
||||
checksum = hashlib.sha256(repomd).hexdigest()
|
||||
if checksum != expected_checksum:
|
||||
print(f"error: repo was configured with checksum {expected_checksum}, but actually got {checksum}")
|
||||
return 1
|
||||
|
||||
# delete cache manually, because `dnf clean all` leaves some contents behind
|
||||
fd = os.open(f"{tree}/var/cache/dnf", os.O_DIRECTORY)
|
||||
for _, dirs, files, dirfd in os.fwalk(".", topdown=False, dir_fd=fd):
|
||||
for name in files:
|
||||
os.unlink(name, dir_fd=dirfd)
|
||||
for name in dirs:
|
||||
os.rmdir(name, dir_fd=dirfd)
|
||||
os.close(fd)
|
||||
|
||||
# remove temporary machine ID if it was created by us
|
||||
if not machine_id_set_previously:
|
||||
print("deleting the fake machine id")
|
||||
machine_id_file = pathlib.Path(f"{tree}/etc/machine-id")
|
||||
machine_id_file.unlink()
|
||||
machine_id_file.touch()
|
||||
|
||||
# remove random seed from the tree if exists
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(f"{tree}/var/lib/systemd/random-seed")
|
||||
|
||||
return 0
|
||||
|
||||
if __name__ == '__main__':
|
||||
args = json.load(sys.stdin)
|
||||
r = main(args["tree"], args["options"])
|
||||
sys.exit(r)
|
||||
Loading…
Add table
Add a link
Reference in a new issue