diff --git a/README.md b/README.md index c97711ad..c69f3b7d 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ Additionally, the built-in stages require: * `rpm >= 4.15` * `tar >= 1.32` * `util-linux >= 235` + * `skopeo` At build-time, the following software is required: diff --git a/inputs/org.osbuild.containers b/inputs/org.osbuild.containers new file mode 100755 index 00000000..eb9575dc --- /dev/null +++ b/inputs/org.osbuild.containers @@ -0,0 +1,180 @@ +#!/usr/bin/python3 +"""Inputs for container images + +This reads images from the `org.osbuild.containers` directory in the +sources store. + +The store is indexed by the "container image id", which is the digest +of the container configuration file (rather than the outer manifest) +and is what will be shown in the "podman images" output when the image +is installed. This digest is stable as opposed to the manifest digest +which can change during transfer and storage due to +e.g. recompression. + +""" + +import os +import sys +import pathlib + +from osbuild import inputs + + +SCHEMA = r""" +"definitions": { + "source-options": { + "type": "object", + "required": ["name"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "The name to use for the image" + } + } + }, + "source-object-ref": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "patternProperties": { + ".*": { + "$ref": "#/definitions/source-options" + } + } + }, + "source-origin": { + "type": "string", + "description": "When the origin of the input is a source", + "enum": ["org.osbuild.source"] + }, + "pipeline-options": { + "type": "object", + "additionalProperties": false, + "required": ["name", "file", "format"], + "properties": { + "name": { + "type": "string", + "description": "The name to use for the image" + }, + "file": { + "description": "File to access with in a pipeline", + "type": "string" + }, + "format": { + "description": "Container archive format", + "enum": ["oci-archive", "docker-archive"] + } + } + }, + "pipeline-object-ref": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "patternProperties": { + ".*": { + "$ref": "#/definitions/pipeline-options" + } + } + }, + "pipeline-origin": { + "type": "string", + "description": "When the origin of the input is a pipeline", + "enum": ["org.osbuild.pipeline"] + } +}, +"additionalProperties": true, +"oneOf": [ + { + "additionalProperties": false, + "required": ["type", "origin", "references"], + "properties": { + "type": { + "enum": ["org.osbuild.containers"] + }, + "origin": { + "description": "The org.osbuild.source origin case", + "$ref": "#/definitions/source-origin" + }, + "references": { + "description": "Container image id", + "$ref": "#/definitions/source-object-ref" + } + } + }, + { + "additionalProperties": false, + "required": ["type", "origin", "references"], + "properties": { + "type": { + "enum": ["org.osbuild.containers"] + }, + "origin": { + "description": "The org.osbuild.source origin case", + "$ref": "#/definitions/pipeline-origin" + }, + "references": { + "description": "References to pipelines", + "$ref": "#/definitions/pipeline-object-ref" + } + } + } +] +""" + + +class ContainersInput(inputs.InputService): + + @staticmethod + def map_source_ref(source, ref, data, target): + cache_dir = os.path.join(source, ref) + os.link(os.path.join(cache_dir, "container-image.tar"), os.path.join(target, ref)) + + return ref, "docker-archive" + + @staticmethod + def map_pipeline_ref(store, ref, data, target): + filepath = data["file"].lstrip("/") + container_format = data["format"] + + # prepare the mount point + filename = pathlib.Path(target, filepath) + os.makedirs(filename.parent, exist_ok=True) + filename.touch() + + store.read_tree_at(ref, filename, filepath) + + return filepath, container_format + + def map(self, store, origin, refs, target, _options): + source = store.source("org.osbuild.containers") + images = {} + + for ref, data in refs.items(): + if origin == "org.osbuild.source": + ref, container_format = self.map_source_ref(source, ref, data, target) + else: + ref, container_format = self.map_pipeline_ref(store, ref, data, target) + + images[ref] = { + "format": container_format, + "name": data["name"] + } + images[ref]["name"] = data["name"] + + reply = { + "path": target, + "data": { + "archives": images + } + } + return reply + + +def main(): + service = ContainersInput.from_args(sys.argv[1:]) + service.main() + + +if __name__ == '__main__': + main() diff --git a/osbuild/buildroot.py b/osbuild/buildroot.py index 36604893..8dfd8b94 100644 --- a/osbuild/buildroot.py +++ b/osbuild/buildroot.py @@ -225,6 +225,11 @@ class BuildRoot(contextlib.AbstractContextManager): os.path.join(self._rootdir, "etc/mke2fs.conf"), "/etc/mke2fs.conf"] + # Skopeo needs things like /etc/containers/policy.json, so take them from buildroot + mounts += ["--ro-bind-try", + os.path.join(self._rootdir, "etc/containers"), + "/etc/containers"] + # We execute our own modules by bind-mounting them from the host into # the build-root. We have minimal requirements on the build-root, so # these modules can be executed. Everything else we provide ourselves. diff --git a/sources/org.osbuild.skopeo b/sources/org.osbuild.skopeo new file mode 100755 index 00000000..c9f7b230 --- /dev/null +++ b/sources/org.osbuild.skopeo @@ -0,0 +1,134 @@ +#!/usr/bin/python3 +"""Fetch container image from a registry using skopeo + +Buildhost commands used: `skopeo`. +""" + +import errno +import os +import sys +import subprocess +import tempfile +import hashlib + +from osbuild import sources +from osbuild.util import ctx + +SCHEMA = """ +"additionalProperties": false, +"definitions": { + "item": { + "description": "The container image to fetch indexed by the container image id", + "type": "object", + "additionalProperties": false, + "patternProperties": { + "sha256:[0-9a-f]{64}": { + "type": "object", + "additionalProperties": false, + "required": ["image"], + "properties": { + "image": { + "type": "object", + "additionalProperties": false, + "required": ["name", "digest"], + "properties": { + "name": { + "type": "string", + "description": "Name of the image (including registry)." + }, + "digest": { + "type": "string", + "description": "Digest of image in registry.", + "pattern": "sha256:[0-9a-f]{64}" + }, + "tls-verify": { + "type": "boolean", + "description": "Require https (default true)." + } + } + } + } + } + } + } +}, +"properties": { + "items": {"$ref": "#/definitions/item"}, + "digests": {"$ref": "#/definitions/item"} +}, +"oneOf": [{ + "required": ["items"] +}, { + "required": ["digests"] +}] +""" + + +def download(items, cache): + for image_id, item in items.items(): + image = item["image"] + imagename = image["name"] + digest = image["digest"] + tls_verify = image.get("tls-verify", True) + + path = f"{cache}/{image_id}" + if os.path.isdir(path): + continue + + with tempfile.TemporaryDirectory(prefix="tmp-download-", dir=cache) as tmpdir: + archive_path = os.path.join(tmpdir, "container-image.tar") + + source = f"docker://{imagename}@{digest}" + + # We use the docker format, not oci, because that is the + # default return image type of real world registries, + # allowing the image to get the same image id as if you + # did "podman pull" (rather than converting the image to + # oci format, changing the id) + destination = f"docker-archive:{archive_path}" + + extra_args = [] + + # The archive format can't store signatures, but we still verify them during download + extra_args.append("--remove-signatures") + + if not tls_verify: + extra_args.append("--src-tls-verify=false") + + subprocess.run(["skopeo", "copy"] + extra_args + [source, destination], + encoding="utf-8", + check=True) + + # Verify that the digest supplied downloaded the correct container image id. + # The image id is the digest of the config, but skopeo can' currently + # get the config id, only the full config, so we checksum it ourselves. + res = subprocess.run(["skopeo", "inspect", "--raw", "--config", destination], + capture_output=True, + check=True) + downloaded_id = "sha256:" + hashlib.sha256(res.stdout).hexdigest() + if downloaded_id != image_id: + raise RuntimeError( + f"Downloaded image {imagename}@{digest} has a id of {downloaded_id}, but expected {image_id}") + + # Atomically move download dir into place on successful download + os.chmod(tmpdir, 0o755) + with ctx.suppress_oserror(errno.ENOTEMPTY, errno.EEXIST): + os.rename(tmpdir, path) + + +class SkopeoSource(sources.SourceService): + + def download(self, items, cache, _options): + cache = os.path.join(cache, "org.osbuild.containers") + os.makedirs(cache, exist_ok=True) + + download(items, cache) + + +def main(): + service = SkopeoSource.from_args(sys.argv[1:]) + service.main() + + +if __name__ == '__main__': + main() diff --git a/stages/org.osbuild.skopeo b/stages/org.osbuild.skopeo new file mode 100755 index 00000000..87357261 --- /dev/null +++ b/stages/org.osbuild.skopeo @@ -0,0 +1,107 @@ +#!/usr/bin/python3 +""" +Install an container image into the container store. +This supports both oci archives and docker archives, and uses the containers +input (reading from a skopeo source or a file in a pipeline). + +Buildhost commands used: `skopeo`. +""" + +import os +import subprocess +import sys +import tempfile + +import osbuild.api + + +SCHEMA_2 = r""" +"inputs": { + "type": "object", + "additionalProperties": false, + "required": ["images"], + "properties": { + "images": { + "type": "object", + "additionalProperties": true + } + } +}, +"options": { + "additionalProperties": false, + "required": ["destination"], + "properties": { + "destination": { + "type": "object", + "additionalProperties": false, + "required": ["type"], + "properties": { + "type": { + "enum": ["containers-storage"] + }, + "storage-path": { + "description": "Container storage location (default /var/lib/containers/storage).", + "type": "string" + }, + "storage-driver": { + "description": "The container storage driver to use (default overlay).", + "type": "string" + } + } + } + } +} +""" + + +def parse_input(inputs): + images = inputs["images"] + archives = images["data"]["archives"] + + res = [] + for filename, data in archives.items(): + filepath = os.path.join(images["path"], filename) + + res.append((filepath, data)) + return res + + +def main(inputs, output, options): + files = parse_input(inputs) + + destination = options["destination"] + # The destination type is always containers-storage atm, so ignore "type" + + storage_root = destination.get("storage-path", "/var/lib/containers/storage") + storage_driver = destination.get("storage-driver", "overlay") + + for source, source_data in files: + container_format = source_data["format"] + image_name = source_data["name"] + + # We can't have special characters like ":" in the source names because containers/image + # treats them special, like e.g. /some/path:tag, so we make a symlink to the real name + # and pass the symlink name to skopeo to make it work with anything + with tempfile.TemporaryDirectory() as tmpdir: + linkname = os.path.join(tmpdir, "image.tar") + os.symlink(source, linkname) + + if container_format == "docker-archive": + source = f"docker-archive:{linkname}" + elif container_format == "oci-archive": + source = f"oci-archive:{linkname}" + else: + raise RuntimeError(f"Unknown container format {container_format}") + + dest = f"containers-storage:[{storage_driver}@{output}{storage_root}+/run/containers/storage]{image_name}" + + subprocess.run(["skopeo", "copy", source, dest], + check=True) + + return 0 + + +if __name__ == '__main__': + args = osbuild.api.arguments() + r = main(args["inputs"], args["tree"], args["options"]) + sys.exit(r)