Add support for installing containers in images

This adds a stage called org.osbuild.skopeo that installs docker and
oci archive files into the container storage of the tree being
constructed.

The source can either be a file from another pipeline, for example one
created with the existing org.osbuild.oci-archive stage, or it can
be using the new org.osbuild.skopeo source and org.osbuild.containers
input, which will download an image from a registry and install that.

There is an optional option in the install stage that lets you
configure a custom storage location, which allows the use of the
additionalimagestores option in the container storage.conf
to use a read-only image stores (instead of /var/lib/container).

Note: skopeo fails to start if /etc/containers/policy.json is
not available, so we bind mount it from the build tree to the
buildroot if available.

README.md

@@ -38,6 +38,7 @@ Additionally, the built-in stages require:
  * `rpm >= 4.15`
  * `tar >= 1.32`
  * `util-linux >= 235`
+ * `skopeo`
 
 At build-time, the following software is required:
 

inputs/org.osbuild.containers

@@ -0,0 +1,180 @@
+#!/usr/bin/python3
+"""Inputs for container images
+
+This reads images from the `org.osbuild.containers` directory in the
+sources store.
+
+The store is indexed by the "container image id", which is the digest
+of the container configuration file (rather than the outer manifest)
+and is what will be shown in the "podman images" output when the image
+is installed. This digest is stable as opposed to the manifest digest
+which can change during transfer and storage due to
+e.g. recompression.
+
+"""
+
+import os
+import sys
+import pathlib
+
+from osbuild import inputs
+
+
+SCHEMA = r"""
+"definitions": {
+  "source-options": {
+    "type": "object",
+    "required": ["name"],
+    "additionalProperties": false,
+    "properties": {
+      "name": {
+        "type": "string",
+        "description": "The name to use for the image"
+      }
+    }
+  },
+  "source-object-ref": {
+    "type": "object",
+    "additionalProperties": false,
+    "minProperties": 1,
+    "patternProperties": {
+      ".*": {
+        "$ref": "#/definitions/source-options"
+      }
+    }
+  },
+  "source-origin": {
+    "type": "string",
+    "description": "When the origin of the input is a source",
+    "enum": ["org.osbuild.source"]
+  },
+  "pipeline-options": {
+    "type": "object",
+    "additionalProperties": false,
+    "required": ["name", "file", "format"],
+    "properties": {
+      "name": {
+        "type": "string",
+        "description": "The name to use for the image"
+      },
+      "file": {
+        "description": "File to access with in a pipeline",
+        "type": "string"
+      },
+      "format": {
+        "description": "Container archive format",
+        "enum": ["oci-archive", "docker-archive"]
+      }
+    }
+  },
+  "pipeline-object-ref": {
+    "type": "object",
+    "additionalProperties": false,
+    "minProperties": 1,
+    "patternProperties": {
+      ".*": {
+        "$ref": "#/definitions/pipeline-options"
+      }
+    }
+  },
+  "pipeline-origin": {
+    "type": "string",
+    "description": "When the origin of the input is a pipeline",
+    "enum": ["org.osbuild.pipeline"]
+  }
+},
+"additionalProperties": true,
+"oneOf": [
+  {
+    "additionalProperties": false,
+    "required": ["type", "origin", "references"],
+    "properties": {
+      "type": {
+        "enum": ["org.osbuild.containers"]
+      },
+      "origin": {
+        "description": "The org.osbuild.source origin case",
+        "$ref": "#/definitions/source-origin"
+      },
+      "references": {
+        "description": "Container image id",
+        "$ref": "#/definitions/source-object-ref"
+      }
+    }
+  },
+  {
+    "additionalProperties": false,
+    "required": ["type", "origin", "references"],
+    "properties": {
+      "type": {
+        "enum": ["org.osbuild.containers"]
+      },
+      "origin": {
+        "description": "The org.osbuild.source origin case",
+        "$ref": "#/definitions/pipeline-origin"
+      },
+      "references": {
+        "description": "References to pipelines",
+        "$ref": "#/definitions/pipeline-object-ref"
+      }
+    }
+  }
+]
+"""
+
+
+class ContainersInput(inputs.InputService):
+
+    @staticmethod
+    def map_source_ref(source, ref, data, target):
+        cache_dir = os.path.join(source, ref)
+        os.link(os.path.join(cache_dir, "container-image.tar"), os.path.join(target, ref))
+
+        return ref, "docker-archive"
+
+    @staticmethod
+    def map_pipeline_ref(store, ref, data, target):
+        filepath = data["file"].lstrip("/")
+        container_format = data["format"]
+
+        # prepare the mount point
+        filename = pathlib.Path(target, filepath)
+        os.makedirs(filename.parent, exist_ok=True)
+        filename.touch()
+
+        store.read_tree_at(ref, filename, filepath)
+
+        return filepath, container_format
+
+    def map(self, store, origin, refs, target, _options):
+        source = store.source("org.osbuild.containers")
+        images = {}
+
+        for ref, data in refs.items():
+            if origin == "org.osbuild.source":
+                ref, container_format = self.map_source_ref(source, ref, data, target)
+            else:
+                ref, container_format = self.map_pipeline_ref(store, ref, data, target)
+
+            images[ref] = {
+                "format": container_format,
+                "name": data["name"]
+            }
+            images[ref]["name"] = data["name"]
+
+        reply = {
+            "path": target,
+            "data": {
+                "archives": images
+            }
+        }
+        return reply
+
+
+def main():
+    service = ContainersInput.from_args(sys.argv[1:])
+    service.main()
+
+
+if __name__ == '__main__':
+    main()

osbuild/buildroot.py

@@ -225,6 +225,11 @@ class BuildRoot(contextlib.AbstractContextManager):
                    os.path.join(self._rootdir, "etc/mke2fs.conf"),
                    "/etc/mke2fs.conf"]
 
+        # Skopeo needs things like /etc/containers/policy.json, so take them from buildroot
+        mounts += ["--ro-bind-try",
+                   os.path.join(self._rootdir, "etc/containers"),
+                   "/etc/containers"]
+
         # We execute our own modules by bind-mounting them from the host into
         # the build-root. We have minimal requirements on the build-root, so
         # these modules can be executed. Everything else we provide ourselves.

sources/org.osbuild.skopeo

@@ -0,0 +1,134 @@
+#!/usr/bin/python3
+"""Fetch container image from a registry using skopeo
+
+Buildhost commands used: `skopeo`.
+"""
+
+import errno
+import os
+import sys
+import subprocess
+import tempfile
+import hashlib
+
+from osbuild import sources
+from osbuild.util import ctx
+
+SCHEMA = """
+"additionalProperties": false,
+"definitions": {
+  "item": {
+    "description": "The container image to fetch indexed by the container image id",
+    "type": "object",
+    "additionalProperties": false,
+    "patternProperties": {
+      "sha256:[0-9a-f]{64}": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["image"],
+        "properties": {
+          "image": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["name", "digest"],
+            "properties": {
+              "name": {
+                "type": "string",
+                "description": "Name of the image (including registry)."
+              },
+              "digest": {
+                "type": "string",
+                "description": "Digest of image in registry.",
+                "pattern": "sha256:[0-9a-f]{64}"
+              },
+              "tls-verify": {
+                "type": "boolean",
+                "description": "Require https (default true)."
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+},
+"properties": {
+  "items": {"$ref": "#/definitions/item"},
+  "digests": {"$ref": "#/definitions/item"}
+},
+"oneOf": [{
+  "required": ["items"]
+}, {
+  "required": ["digests"]
+}]
+"""
+
+
+def download(items, cache):
+    for image_id, item in items.items():
+        image = item["image"]
+        imagename = image["name"]
+        digest = image["digest"]
+        tls_verify = image.get("tls-verify", True)
+
+        path = f"{cache}/{image_id}"
+        if os.path.isdir(path):
+            continue
+
+        with tempfile.TemporaryDirectory(prefix="tmp-download-", dir=cache) as tmpdir:
+            archive_path = os.path.join(tmpdir, "container-image.tar")
+
+            source = f"docker://{imagename}@{digest}"
+
+            # We use the docker format, not oci, because that is the
+            # default return image type of real world registries,
+            # allowing the image to get the same image id as if you
+            # did "podman pull" (rather than converting the image to
+            # oci format, changing the id)
+            destination = f"docker-archive:{archive_path}"
+
+            extra_args = []
+
+            # The archive format can't store signatures, but we still verify them during download
+            extra_args.append("--remove-signatures")
+
+            if not tls_verify:
+                extra_args.append("--src-tls-verify=false")
+
+            subprocess.run(["skopeo", "copy"] + extra_args + [source, destination],
+                           encoding="utf-8",
+                           check=True)
+
+            # Verify that the digest supplied downloaded the correct container image id.
+            # The image id is the digest of the config, but skopeo can' currently
+            # get the config id, only the full config, so we checksum it ourselves.
+            res = subprocess.run(["skopeo", "inspect", "--raw", "--config", destination],
+                                 capture_output=True,
+                                 check=True)
+            downloaded_id = "sha256:" + hashlib.sha256(res.stdout).hexdigest()
+            if downloaded_id != image_id:
+                raise RuntimeError(
+                    f"Downloaded image {imagename}@{digest} has a id of {downloaded_id}, but expected {image_id}")
+
+            # Atomically move download dir into place on successful download
+            os.chmod(tmpdir, 0o755)
+            with ctx.suppress_oserror(errno.ENOTEMPTY, errno.EEXIST):
+                os.rename(tmpdir, path)
+
+
+class SkopeoSource(sources.SourceService):
+
+    def download(self, items, cache, _options):
+        cache = os.path.join(cache, "org.osbuild.containers")
+        os.makedirs(cache, exist_ok=True)
+
+        download(items, cache)
+
+
+def main():
+    service = SkopeoSource.from_args(sys.argv[1:])
+    service.main()
+
+
+if __name__ == '__main__':
+    main()

stages/org.osbuild.skopeo

@@ -0,0 +1,107 @@
+#!/usr/bin/python3
+"""
+Install an container image into the container store.
+This supports both oci archives and docker archives, and uses the containers
+input (reading from a skopeo source or a file in a pipeline).
+
+Buildhost commands used: `skopeo`.
+"""
+
+import os
+import subprocess
+import sys
+import tempfile
+
+import osbuild.api
+
+
+SCHEMA_2 = r"""
+"inputs": {
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["images"],
+  "properties": {
+    "images": {
+      "type": "object",
+      "additionalProperties": true
+    }
+  }
+},
+"options": {
+  "additionalProperties": false,
+  "required": ["destination"],
+  "properties": {
+    "destination": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["type"],
+      "properties": {
+        "type": {
+          "enum": ["containers-storage"]
+        },
+        "storage-path": {
+          "description": "Container storage location (default /var/lib/containers/storage).",
+          "type": "string"
+        },
+        "storage-driver": {
+          "description": "The container storage driver to use (default overlay).",
+          "type": "string"
+        }
+      }
+    }
+  }
+}
+"""
+
+
+def parse_input(inputs):
+    images = inputs["images"]
+    archives = images["data"]["archives"]
+
+    res = []
+    for filename, data in archives.items():
+        filepath = os.path.join(images["path"], filename)
+
+        res.append((filepath, data))
+    return res
+
+
+def main(inputs, output, options):
+    files = parse_input(inputs)
+
+    destination = options["destination"]
+    # The destination type is always containers-storage atm, so ignore "type"
+
+    storage_root = destination.get("storage-path", "/var/lib/containers/storage")
+    storage_driver = destination.get("storage-driver", "overlay")
+
+    for source, source_data in files:
+        container_format = source_data["format"]
+        image_name = source_data["name"]
+
+        # We can't have special characters like ":" in the source names because containers/image
+        # treats them special, like e.g. /some/path:tag, so we make a symlink to the real name
+        # and pass the symlink name to skopeo to make it work with anything
+        with tempfile.TemporaryDirectory() as tmpdir:
+            linkname = os.path.join(tmpdir, "image.tar")
+            os.symlink(source, linkname)
+
+            if container_format == "docker-archive":
+                source = f"docker-archive:{linkname}"
+            elif container_format == "oci-archive":
+                source = f"oci-archive:{linkname}"
+            else:
+                raise RuntimeError(f"Unknown container format {container_format}")
+
+            dest = f"containers-storage:[{storage_driver}@{output}{storage_root}+/run/containers/storage]{image_name}"
+
+            subprocess.run(["skopeo", "copy", source, dest],
+                           check=True)
+
+    return 0
+
+
+if __name__ == '__main__':
+    args = osbuild.api.arguments()
+    r = main(args["inputs"], args["tree"], args["options"])
+    sys.exit(r)