assemblers: add org.osbuild.rawfs

This assembler outputs an image file which only contains the file
system.

.travis.yml

@@ -35,3 +35,6 @@ jobs:
     - name: locale
       before_install: sudo apt-get install -y systemd-container yum tar
       script: sudo env "PATH=$PATH" python3 -m test --case locale --build-pipeline samples/build-from-yum.json  
+    - name: assemblers
+      before_install: sudo apt-get install -y systemd-container yum tar
+      script: sudo python3 -m unittest test.test_assemblers

assemblers/org.osbuild.rawfs

@@ -0,0 +1,48 @@
+#!/usr/bin/python3
+
+import contextlib
+import json
+import os
+import socket
+import subprocess
+import sys
+import osbuild.remoteloop as remoteloop
+
+
+@contextlib.contextmanager
+def mount(source, dest, *options):
+    os.makedirs(dest, 0o755, True)
+    subprocess.run(["mount", *options, source, dest], check=True)
+    try:
+        yield
+    finally:
+        subprocess.run(["umount", "-R", dest], check=True)
+
+
+def main(tree, output_dir, options, loop_client):
+    filename = options["filename"]
+    root_fs_uuid = options["root_fs_uuid"]
+    size = options["size"]
+
+    image = f"/var/tmp/osbuild-image.raw"
+    mountpoint = f"/tmp/osbuild-mnt"
+
+    subprocess.run(["truncate", "--size", str(size), image], check=True)
+    subprocess.run(["mkfs.ext4", "-U", root_fs_uuid, image], input="y", encoding='utf-8', check=True)
+
+    # Copy the tree into the target image
+    with loop_client.device(image) as loop, mount(loop, mountpoint):
+        subprocess.run(["cp", "-a", f"{tree}/.", mountpoint], check=True)
+
+    subprocess.run(["mv", image, f"{output_dir}/{filename}"], check=True)
+
+
+if __name__ == '__main__':
+    args = json.load(sys.stdin)
+
+    with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
+        sock.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1)
+        sock.connect("/run/osbuild/api/remoteloop")
+        r = main(args["tree"], args["output_dir"], args["options"], remoteloop.LoopClient(sock))
+
+    sys.exit(r)

samples/base-rawfs.json

@@ -0,0 +1,78 @@
+{
+  "stages": [
+    {
+      "name": "org.osbuild.dnf",
+      "options": {
+        "releasever": "30",
+        "basearch": "x86_64",
+        "install_weak_deps": true,
+        "repos": [
+          {
+            "metalink": "https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever&arch=$basearch",
+            "gpgkey": "F1D8 EC98 F241 AAF2 0DF6  9420 EF3C 111F CFC6 59B9",
+            "checksum": "sha256:9f596e18f585bee30ac41c11fb11a83ed6b11d5b341c1cb56ca4015d7717cb97"
+          }
+        ],
+        "packages": [
+          "@Core",
+          "chrony",
+          "kernel",
+          "selinux-policy-targeted",
+          "grub2-pc",
+          "spice-vdagent",
+          "qemu-guest-agent",
+          "xen-libs",
+          "langpacks-en"
+        ],
+        "exclude_packages": [
+          "dracut-config-rescue"
+        ]
+      }
+    },
+    {
+      "name": "org.osbuild.locale",
+      "options": {
+        "language": "en_US"
+      }
+    },
+    {
+      "name": "org.osbuild.fstab",
+      "options": {
+        "filesystems": [
+          {
+            "uuid": "76a22bf4-f153-4541-b6c7-0332c0dfaeac",
+            "vfs_type": "ext4",
+            "path": "/",
+            "freq": "1",
+            "passno": "1"
+          }
+        ]
+      }
+    },
+    {
+      "name": "org.osbuild.grub2",
+      "options": {
+        "root_fs_uuid": "76a22bf4-f153-4541-b6c7-0332c0dfaeac",
+        "kernel_opts": "ro biosdevname=0 net.ifnames=0"
+      }
+    },
+    {
+      "name": "org.osbuild.selinux",
+      "options": {
+        "file_contexts": "etc/selinux/targeted/contexts/files/file_contexts"
+      }
+    },
+    {
+      "name": "org.osbuild.fix-bls"
+    }
+  ],
+  "assembler":
+    {
+      "name": "org.osbuild.rawfs",
+      "options": {
+        "filename": "image.raw",
+        "root_fs_uuid": "76a22bf4-f153-4541-b6c7-0332c0dfaeac",
+        "size": 3221225472
+      }
+    }
+}

test/osbuildtest.py

@@ -28,7 +28,7 @@ class TestCase(unittest.TestCase):
         if not os.getenv("OSBUILD_TEST_STORE"):
             shutil.rmtree(self.store)
 
-    def run_osbuild(self, pipeline):
+    def run_osbuild(self, pipeline, input=None):
             osbuild_cmd = ["python3", "-m", "osbuild", "--json", "--store", self.store, "--libdir", ".", pipeline]
 
             build_pipeline = os.getenv("OSBUILD_TEST_BUILD_PIPELINE", None)
@@ -36,7 +36,7 @@ class TestCase(unittest.TestCase):
                 osbuild_cmd.append("--build-pipeline")
                 osbuild_cmd.append(build_pipeline)
 
-            r = subprocess.run(osbuild_cmd, encoding="utf-8", stdout=subprocess.PIPE, check=True)
+            r = subprocess.run(osbuild_cmd, encoding="utf-8", input=input, stdout=subprocess.PIPE, check=True)
 
             result = json.loads(r.stdout)
             return result["tree_id"], result["output_id"]

test/pipelines/f30-base.json

@@ -0,0 +1,55 @@
+{
+  "build": {
+    "stages": [
+      {
+        "name": "org.osbuild.dnf",
+        "options": {
+          "releasever": "30",
+          "basearch": "x86_64",
+          "install_weak_deps": false,
+          "repos": [
+            {
+              "metalink": "https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever&arch=$basearch",
+              "gpgkey": "F1D8 EC98 F241 AAF2 0DF6  9420 EF3C 111F CFC6 59B9",
+              "checksum": "sha256:9f596e18f585bee30ac41c11fb11a83ed6b11d5b341c1cb56ca4015d7717cb97"
+            }
+          ],
+          "packages": [
+            "dnf",
+            "e2fsprogs",
+            "grub2-pc",
+            "policycoreutils",
+            "qemu-img",
+            "systemd"
+          ]
+        }
+      }
+    ]
+  },
+  "stages": [
+    {
+      "name": "org.osbuild.dnf",
+      "options": {
+        "releasever": "30",
+        "basearch": "x86_64",
+        "repos": [
+          {
+            "metalink": "https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever&arch=$basearch",
+            "gpgkey": "F1D8 EC98 F241 AAF2 0DF6  9420 EF3C 111F CFC6 59B9",
+            "checksum": "sha256:9f596e18f585bee30ac41c11fb11a83ed6b11d5b341c1cb56ca4015d7717cb97"
+          }
+        ],
+        "packages": [
+          "@Core",
+          "selinux-policy-targeted"
+        ]
+      }
+    },
+    {
+      "name": "org.osbuild.selinux",
+      "options": {
+        "file_contexts": "etc/selinux/targeted/contexts/files/file_contexts"
+      }
+    }
+  ]
+}

test/test_assemblers.py

@@ -0,0 +1,53 @@
+
+import contextlib
+import json
+import os
+import subprocess
+import tempfile
+import uuid
+
+from . import osbuildtest
+
+
+class TestAssemblers(osbuildtest.TestCase):
+    def test_rawfs(self):
+        with open("test/pipelines/f30-base.json") as f:
+            base = json.load(f)
+
+        options = {
+            "filename": "image.raw",
+            "root_fs_uuid": str(uuid.uuid4()),
+            "size": 2 * 1024 * 1024 * 1024
+        }
+
+        pipeline = dict(base, assembler = {
+            "name": "org.osbuild.rawfs",
+            "options": options
+        })
+        tree_id, output_id = self.run_osbuild("-", json.dumps(pipeline))
+
+        image = f"{self.store}/refs/{output_id}/image.raw"
+
+        self.assertEqual(os.path.getsize(image), options["size"])
+
+        output = subprocess.check_output(["blkid", "--output", "export", image], encoding="utf-8")
+        blkid = dict(line.split("=") for line in output.strip().split("\n"))
+        self.assertEqual(blkid["UUID"], options["root_fs_uuid"])
+        self.assertEqual(blkid["TYPE"], "ext4")
+
+        with mount(image) as target_tree:
+            tree = f"{self.store}/refs/{tree_id}"
+            diff = json.loads(subprocess.check_output(["./tree-diff", tree, target_tree]))
+            self.assertEqual(diff["added_files"], ["/lost+found"])
+            self.assertEqual(diff["deleted_files"], [])
+            self.assertEqual(diff["differences"], {})
+
+
+@contextlib.contextmanager
+def mount(device):
+    with tempfile.TemporaryDirectory() as mountpoint:
+        subprocess.run(["mount", "-o", "ro", device, mountpoint], check=True)
+        try:
+            yield mountpoint
+        finally:
+            subprocess.run(["umount", "--lazy", mountpoint], check=True)