From 3bb5bedd8e7dc3ac465e3399f8cd7fd32e071580 Mon Sep 17 00:00:00 2001 From: Lukas Zapletal Date: Mon, 10 Feb 2025 09:28:53 +0100 Subject: [PATCH] ostree: introduce optional subpath feature --- osbuild/testutil/net.py | 10 ++++++++++ sources/org.osbuild.ostree | 20 ++++++++++++++++---- sources/test/test_ostree_source.py | 27 +++++++++++++++++++++++---- 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/osbuild/testutil/net.py b/osbuild/testutil/net.py index bb8e2885..183fdb58 100644 --- a/osbuild/testutil/net.py +++ b/osbuild/testutil/net.py @@ -4,8 +4,10 @@ network related utilities """ import contextlib import http.server +import os import socket import ssl +import sys import threading try: @@ -25,6 +27,12 @@ except ImportError: from .atomic import AtomicCounter +def print_dir(directory): + for root, _, files in os.walk(directory): + for fn in files: + print(os.path.join(root, fn)) + + def _get_free_port(): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(("localhost", 0)) @@ -47,6 +55,8 @@ class SilentHTTPRequestHandler(http.server.SimpleHTTPRequestHandler): class DirHTTPServer(ThreadingHTTPServer): def __init__(self, *args, directory=None, simulate_failures=0, **kwargs): super().__init__(*args, **kwargs) + print("Serving:", file=sys.stderr) + print_dir(directory) self.directory = directory self.simulate_failures = AtomicCounter(simulate_failures) self.reqs = AtomicCounter() diff --git a/sources/org.osbuild.ostree b/sources/org.osbuild.ostree index ed307433..8c9aaaba 100755 --- a/sources/org.osbuild.ostree +++ b/sources/org.osbuild.ostree @@ -16,9 +16,11 @@ supported are: entitlement certificate. Secret fields `consumer_cert` and `consumer_key` must be set. -To skip TLS verification, set OSBUILD_SOURCES_OSTREE_INSECURE environment -variable to "true". To set a HTTP(S) proxy, set OSBUILD_SOURCES_OSTREE_PROXY -environment variable to the proxy URL. +When one or more subpaths are specified, only those are pulled, otherwise the +whole tree is pulled. To skip TLS verification, set +OSBUILD_SOURCES_OSTREE_INSECURE environment variable to "true". To set a +HTTP(S) proxy, set OSBUILD_SOURCES_OSTREE_PROXY environment variable to the +proxy URL. """ @@ -57,6 +59,13 @@ SCHEMA = """ "type": "string", "description": "content URL of the repository." }, + "subpaths": { + "type": "array", + "items": { + "type": "string", + "description": "only include path(s) when fetching (fetch all when empty)" + } + }, "gpgkeys": { "type": "array", "items": { @@ -114,6 +123,9 @@ class OSTreeSource(sources.SourceService): def fetch_one(self, checksum, desc): commit = checksum remote = desc["remote"] + subpaths = remote.get("subpaths", []) + subpaths = [f'--subpath={p}' for p in subpaths] + # This is a temporary remote so we'll just use a random name name = str(uuid.uuid4()) @@ -121,7 +133,7 @@ class OSTreeSource(sources.SourceService): # Transfer the commit: remote → cache print(f"pulling {commit}", file=sys.stderr) - ostree.cli("pull", name, commit, repo=self.repo) + ostree.cli("pull", name, commit, *subpaths, repo=self.repo) # Remove the temporary remote again ostree.cli("remote", "delete", name, repo=self.repo) diff --git a/sources/test/test_ostree_source.py b/sources/test/test_ostree_source.py index 8d1ab750..48df06f7 100644 --- a/sources/test/test_ostree_source.py +++ b/sources/test/test_ostree_source.py @@ -1,5 +1,6 @@ #!/usr/bin/python3 +import os import pathlib import tempfile @@ -27,11 +28,12 @@ def test_ostree_source_exists(tmp_path, sources_service): assert sources_service.exists("sha256:" + commit, None) -def make_test_sources(proto, port, fake_commit, **secrets): +def make_test_sources(proto, port, fake_commit, subpaths, **secrets): sources = { fake_commit: { "remote": { "url": f"{proto}://localhost:{port}", + "subpaths": subpaths, } } } @@ -42,7 +44,9 @@ def make_test_sources(proto, port, fake_commit, **secrets): def make_repo(root): with tempfile.TemporaryDirectory() as empty_tmpdir: - ostree.cli("init", f"--repo={root}") + ostree.cli("init", "--mode=archive", f"--repo={root}") + os.mknod(os.path.join(empty_tmpdir, "a.txt")) + os.mknod(os.path.join(empty_tmpdir, "b.txt")) return ostree.cli("commit", f"--repo={root}", "--orphan", empty_tmpdir).stdout.rstrip() @@ -53,12 +57,27 @@ def test_ostree_pull_plain(tmp_path, sources_service): fake_commit = make_repo(fake_httpd_root) with http_serve_directory(fake_httpd_root) as httpd: - test_sources = make_test_sources("http", httpd.server_port, fake_commit) + test_sources = make_test_sources("http", httpd.server_port, fake_commit, []) sources_service.setup({"cache": tmp_path, "options": {}}) sources_service.fetch_all(test_sources) assert sources_service.exists("sha256:" + fake_commit, None) +@pytest.mark.skipif(not has_executable("ostree"), reason="need ostree") +def test_ostree_pull_subtree(tmp_path, sources_service, capsys): + fake_httpd_root = tmp_path / "fake-httpd-root" + fake_httpd_root.mkdir(exist_ok=True) + fake_commit = make_repo(fake_httpd_root) + + with http_serve_directory(fake_httpd_root) as httpd: + test_sources = make_test_sources("http", httpd.server_port, fake_commit, ["/b.txt"]) + sources_service.setup({"cache": tmp_path, "options": {}}) + sources_service.fetch_all(test_sources) + assert sources_service.exists("sha256:" + fake_commit, None) + + assert "--subpath=/b.txt" in capsys.readouterr().err + + @pytest.mark.skipif(not has_executable("ostree"), reason="need ostree") def test_ostree_pull_plain_mtls(tmp_path, sources_service, monkeypatch): fake_httpd_root = tmp_path / "fake-httpd-root" @@ -73,7 +92,7 @@ def test_ostree_pull_plain_mtls(tmp_path, sources_service, monkeypatch): with https_serve_directory(fake_httpd_root, cert1, key1) as httpd: monkeypatch.setenv("OSBUILD_SOURCES_OSTREE_INSECURE", "1") - test_sources = make_test_sources("https", httpd.server_port, fake_commit, name="org.osbuild.mtls") + test_sources = make_test_sources("https", httpd.server_port, fake_commit, [], name="org.osbuild.mtls") sources_service.setup({"cache": tmp_path, "options": {}}) sources_service.fetch_all(test_sources) assert sources_service.exists("sha256:" + fake_commit, None)