sources/skopeo: change local container format

Change the local storage format for containers to the `dir` format.
The `dir` format will be used to retain signatures and manifests.

The remove-signatures option is removed since the storage format now
supports them.

The final move (os.rename()) at the end of the fetch_one() method now
creates the checksum directory if it doesn't exist and moves the child
archive into it, adding to any existing archives that might exist in
other formats (from a previous version downloading a `docker-archive`).

Dropped the .tar suffix from the symlink in the skopeo stage since it's
not necessary and the target of the link might be a directory now.

The parent class exists() method checks if there is a *file* in the
sources cache that matches the checksum.  For containers, this used to
be a file called container-image.tar under a directory that matches the
checksum, so for containers it always returned False.  Added an override
for the skopeo source that checks for the new directory archive.

sources/org.osbuild.skopeo

@@ -1,6 +1,16 @@
 #!/usr/bin/python3
 """Fetch container image from a registry using skopeo
 
+The image is stored in a directory called `image` under a directory indexed by
+the "container image id", which is the digest of the container configuration
+file (rather than the outer manifest) and is what will be shown in the "podman
+images" output when the image is installed. This digest is stable as opposed to
+the manifest digest which can change during transfer and storage due to e.g.
+recompression.
+
+The local storage format for containers is the `dir` format which supports
+retaining signatures and manifests.
+
 Buildhost commands used: `skopeo`.
 """
 
@@ -68,6 +78,8 @@ class SkopeoSource(sources.SourceService):
 
     content_type = "org.osbuild.containers"
 
+    dir_name = "image"
+
     def fetch_one(self, checksum, desc):
         image_id = checksum
         image = desc["image"]
@@ -79,22 +91,14 @@ class SkopeoSource(sources.SourceService):
             archive_dir = os.path.join(tmpdir, "container-archive")
             os.makedirs(archive_dir)
             os.chmod(archive_dir, 0o755)
-            archive_path = os.path.join(archive_dir, "container-image.tar")
 
             source = f"docker://{imagename}@{digest}"
 
-            # We use the docker format, not oci, because that is the
-            # default return image type of real world registries,
-            # allowing the image to get the same image id as if you
-            # did "podman pull" (rather than converting the image to
-            # oci format, changing the id)
-            destination = f"docker-archive:{archive_path}"
+            # We use the dir format because it is the most powerful in terms of feature support and is the closest to a
+            # direct serialisation of the registry data.
+            destination = f"dir:{archive_dir}/{self.dir_name}"
 
             extra_args = []
-
-            # The archive format can't store signatures, but we still verify them during download
-            extra_args.append("--remove-signatures")
-
             if not tls_verify:
                 extra_args.append("--src-tls-verify=false")
 
@@ -111,9 +115,14 @@ class SkopeoSource(sources.SourceService):
                 raise RuntimeError(
                     f"Downloaded image {imagename}@{digest} has a id of {downloaded_id}, but expected {image_id}")
 
-            # Atomically move download dir into place on successful download
+            # Atomically move download archive into place on successful download
             with ctx.suppress_oserror(errno.ENOTEMPTY, errno.EEXIST):
-                os.rename(archive_dir, f"{self.cache}/{image_id}")
+                os.makedirs(os.path.join(self.cache, image_id), exist_ok=True)
+                os.rename(os.path.join(archive_dir, self.dir_name), os.path.join(self.cache, image_id, self.dir_name))
+
+    def exists(self, checksum, _desc):
+        path = os.path.join(self.cache, checksum, self.dir_name)
+        return os.path.exists(path)
 
 
 def main():

stages/org.osbuild.skopeo

@@ -82,7 +82,7 @@ def main(inputs, output, options):
         # treats them special, like e.g. /some/path:tag, so we make a symlink to the real name
         # and pass the symlink name to skopeo to make it work with anything
         with tempfile.TemporaryDirectory() as tmpdir:
-            linkname = os.path.join(tmpdir, "image.tar")
+            linkname = os.path.join(tmpdir, "image")
             os.symlink(source, linkname)
 
             if container_format == "dir":