ostree: introduce optional subpath feature

osbuild/testutil/net.py

@@ -4,8 +4,10 @@ network related utilities
 """
 import contextlib
 import http.server
+import os
 import socket
 import ssl
+import sys
 import threading
 
 try:
@@ -25,6 +27,12 @@ except ImportError:
 from .atomic import AtomicCounter
 
 
+def print_dir(directory):
+    for root, _, files in os.walk(directory):
+        for fn in files:
+            print(os.path.join(root, fn))
+
+
 def _get_free_port():
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     s.bind(("localhost", 0))
@@ -47,6 +55,8 @@ class SilentHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
 class DirHTTPServer(ThreadingHTTPServer):
     def __init__(self, *args, directory=None, simulate_failures=0, **kwargs):
         super().__init__(*args, **kwargs)
+        print("Serving:", file=sys.stderr)
+        print_dir(directory)
         self.directory = directory
         self.simulate_failures = AtomicCounter(simulate_failures)
         self.reqs = AtomicCounter()

sources/org.osbuild.ostree

@@ -16,9 +16,11 @@ supported are:
   entitlement certificate. Secret fields `consumer_cert` and
   `consumer_key` must be set.
 
-To skip TLS verification, set OSBUILD_SOURCES_OSTREE_INSECURE environment
-variable to "true". To set a HTTP(S) proxy, set OSBUILD_SOURCES_OSTREE_PROXY
-environment variable to the proxy URL.
+When one or more subpaths are specified, only those are pulled, otherwise the
+whole tree is pulled. To skip TLS verification, set
+OSBUILD_SOURCES_OSTREE_INSECURE environment variable to "true". To set a
+HTTP(S) proxy, set OSBUILD_SOURCES_OSTREE_PROXY environment variable to the
+proxy URL.
 """
 
 
@@ -57,6 +59,13 @@ SCHEMA = """
                 "type": "string",
                 "description": "content URL of the repository."
               },
+              "subpaths": {
+                "type": "array",
+                "items": {
+                  "type": "string",
+                  "description": "only include path(s) when fetching (fetch all when empty)"
+                }
+              },
               "gpgkeys": {
                 "type": "array",
                 "items": {
@@ -114,6 +123,9 @@ class OSTreeSource(sources.SourceService):
     def fetch_one(self, checksum, desc):
         commit = checksum
         remote = desc["remote"]
+        subpaths = remote.get("subpaths", [])
+        subpaths = [f'--subpath={p}' for p in subpaths]
+
         # This is a temporary remote so we'll just use a random name
         name = str(uuid.uuid4())
 
@@ -121,7 +133,7 @@ class OSTreeSource(sources.SourceService):
 
         # Transfer the commit: remote → cache
         print(f"pulling {commit}", file=sys.stderr)
-        ostree.cli("pull", name, commit, repo=self.repo)
+        ostree.cli("pull", name, commit, *subpaths, repo=self.repo)
 
         # Remove the temporary remote again
         ostree.cli("remote", "delete", name, repo=self.repo)

sources/test/test_ostree_source.py

@@ -1,5 +1,6 @@
 #!/usr/bin/python3
 
+import os
 import pathlib
 import tempfile
 
@@ -27,11 +28,12 @@ def test_ostree_source_exists(tmp_path, sources_service):
     assert sources_service.exists("sha256:" + commit, None)
 
 
-def make_test_sources(proto, port, fake_commit, **secrets):
+def make_test_sources(proto, port, fake_commit, subpaths, **secrets):
     sources = {
         fake_commit: {
             "remote": {
                 "url": f"{proto}://localhost:{port}",
+                "subpaths": subpaths,
             }
         }
     }
@@ -42,7 +44,9 @@ def make_test_sources(proto, port, fake_commit, **secrets):
 
 def make_repo(root):
     with tempfile.TemporaryDirectory() as empty_tmpdir:
-        ostree.cli("init", f"--repo={root}")
+        ostree.cli("init", "--mode=archive", f"--repo={root}")
+        os.mknod(os.path.join(empty_tmpdir, "a.txt"))
+        os.mknod(os.path.join(empty_tmpdir, "b.txt"))
         return ostree.cli("commit", f"--repo={root}", "--orphan", empty_tmpdir).stdout.rstrip()
 
 
@@ -53,12 +57,27 @@ def test_ostree_pull_plain(tmp_path, sources_service):
     fake_commit = make_repo(fake_httpd_root)
 
     with http_serve_directory(fake_httpd_root) as httpd:
-        test_sources = make_test_sources("http", httpd.server_port, fake_commit)
+        test_sources = make_test_sources("http", httpd.server_port, fake_commit, [])
         sources_service.setup({"cache": tmp_path, "options": {}})
         sources_service.fetch_all(test_sources)
         assert sources_service.exists("sha256:" + fake_commit, None)
 
 
+@pytest.mark.skipif(not has_executable("ostree"), reason="need ostree")
+def test_ostree_pull_subtree(tmp_path, sources_service, capsys):
+    fake_httpd_root = tmp_path / "fake-httpd-root"
+    fake_httpd_root.mkdir(exist_ok=True)
+    fake_commit = make_repo(fake_httpd_root)
+
+    with http_serve_directory(fake_httpd_root) as httpd:
+        test_sources = make_test_sources("http", httpd.server_port, fake_commit, ["/b.txt"])
+        sources_service.setup({"cache": tmp_path, "options": {}})
+        sources_service.fetch_all(test_sources)
+        assert sources_service.exists("sha256:" + fake_commit, None)
+
+    assert "--subpath=/b.txt" in capsys.readouterr().err
+
+
 @pytest.mark.skipif(not has_executable("ostree"), reason="need ostree")
 def test_ostree_pull_plain_mtls(tmp_path, sources_service, monkeypatch):
     fake_httpd_root = tmp_path / "fake-httpd-root"
@@ -73,7 +92,7 @@ def test_ostree_pull_plain_mtls(tmp_path, sources_service, monkeypatch):
 
     with https_serve_directory(fake_httpd_root, cert1, key1) as httpd:
         monkeypatch.setenv("OSBUILD_SOURCES_OSTREE_INSECURE", "1")
-        test_sources = make_test_sources("https", httpd.server_port, fake_commit, name="org.osbuild.mtls")
+        test_sources = make_test_sources("https", httpd.server_port, fake_commit, [], name="org.osbuild.mtls")
         sources_service.setup({"cache": tmp_path, "options": {}})
         sources_service.fetch_all(test_sources)
         assert sources_service.exists("sha256:" + fake_commit, None)