1de74ce2c9
Before, the download method was defined in the inherited class of each program. With the same kind of workflow redefined every time. This contribution aims at making the workflow more clear and to generalize what can be in the SourceService class. The download worklow is as follow: Setup -> Filter -> Prepare -> Download The setup mainly step sets up caches. Where the download data will be stored in the end. The filter step is used to discard some of the items to download based on some criterion. By default, it is used to verify if an item is already in the cache using the item's checksum. The Prepare step goes from each element and let the overloading step the ability to alter each item before downloading it. This is used mainly for the curl command which for rhel must generate the subscriptions. Then the download step will call fetch_one for each item. Here the download can be performed sequentially or in parallel depending on the number of workers selected.
125 lines
3.9 KiB
Python
Executable file
125 lines
3.9 KiB
Python
Executable file
#!/usr/bin/python3
|
|
"""Fetch container image from a registry using skopeo
|
|
|
|
Buildhost commands used: `skopeo`.
|
|
"""
|
|
|
|
import errno
|
|
import os
|
|
import sys
|
|
import subprocess
|
|
import tempfile
|
|
import hashlib
|
|
|
|
from osbuild import sources
|
|
from osbuild.util import ctx
|
|
|
|
SCHEMA = """
|
|
"additionalProperties": false,
|
|
"definitions": {
|
|
"item": {
|
|
"description": "The container image to fetch indexed by the container image id",
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"patternProperties": {
|
|
"sha256:[0-9a-f]{64}": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["image"],
|
|
"properties": {
|
|
"image": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["name", "digest"],
|
|
"properties": {
|
|
"name": {
|
|
"type": "string",
|
|
"description": "Name of the image (including registry)."
|
|
},
|
|
"digest": {
|
|
"type": "string",
|
|
"description": "Digest of image in registry.",
|
|
"pattern": "sha256:[0-9a-f]{64}"
|
|
},
|
|
"tls-verify": {
|
|
"type": "boolean",
|
|
"description": "Require https (default true)."
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"properties": {
|
|
"items": {"$ref": "#/definitions/item"},
|
|
"digests": {"$ref": "#/definitions/item"}
|
|
},
|
|
"oneOf": [{
|
|
"required": ["items"]
|
|
}, {
|
|
"required": ["digests"]
|
|
}]
|
|
"""
|
|
|
|
|
|
class SkopeoSource(sources.SourceService):
|
|
|
|
content_type = "org.osbuild.containers"
|
|
|
|
def fetch_one(self, checksum, desc):
|
|
image_id = checksum
|
|
image = desc["image"]
|
|
imagename = image["name"]
|
|
digest = image["digest"]
|
|
tls_verify = image.get("tls-verify", True)
|
|
|
|
with tempfile.TemporaryDirectory(prefix="tmp-download-", dir=self.cache) as tmpdir:
|
|
archive_path = os.path.join(tmpdir, "container-image.tar")
|
|
|
|
source = f"docker://{imagename}@{digest}"
|
|
|
|
# We use the docker format, not oci, because that is the
|
|
# default return image type of real world registries,
|
|
# allowing the image to get the same image id as if you
|
|
# did "podman pull" (rather than converting the image to
|
|
# oci format, changing the id)
|
|
destination = f"docker-archive:{archive_path}"
|
|
|
|
extra_args = []
|
|
|
|
# The archive format can't store signatures, but we still verify them during download
|
|
extra_args.append("--remove-signatures")
|
|
|
|
if not tls_verify:
|
|
extra_args.append("--src-tls-verify=false")
|
|
|
|
subprocess.run(["skopeo", "copy"] + extra_args + [source, destination],
|
|
encoding="utf-8",
|
|
check=True)
|
|
|
|
# Verify that the digest supplied downloaded the correct container image id.
|
|
# The image id is the digest of the config, but skopeo can' currently
|
|
# get the config id, only the full config, so we checksum it ourselves.
|
|
res = subprocess.run(["skopeo", "inspect", "--raw", "--config", destination],
|
|
capture_output=True,
|
|
check=True)
|
|
downloaded_id = "sha256:" + hashlib.sha256(res.stdout).hexdigest()
|
|
if downloaded_id != image_id:
|
|
raise RuntimeError(
|
|
f"Downloaded image {imagename}@{digest} has a id of {downloaded_id}, but expected {image_id}")
|
|
|
|
# Atomically move download dir into place on successful download
|
|
os.chmod(tmpdir, 0o755)
|
|
with ctx.suppress_oserror(errno.ENOTEMPTY, errno.EEXIST):
|
|
os.rename(tmpdir, f"{self.cache}/{image_id}")
|
|
|
|
|
|
def main():
|
|
service = SkopeoSource.from_args(sys.argv[1:])
|
|
service.main()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|