diff --git a/.travis.yml b/.travis.yml index e518da4b..84e1ee94 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,6 +25,9 @@ jobs: script: - sudo env "PATH=$PATH" python3 -m osbuild --libdir . --build-env samples/ubuntu1804.json samples/noop.json - sudo env "PATH=$PATH" python3 -m osbuild --libdir . --build-env samples/ubuntu1804.json samples/noop.json + - name: sources-tests + before_install: sudo apt-get install -y rpm + script: sudo env "PATH=$PATH" python3 -m unittest -v test.test_sources - name: f30-boot before_install: sudo apt-get install -y systemd-container yum qemu-kvm script: sudo env "PATH=$PATH" "OSBUILD_TEST_BUILD_ENV=samples/f27-build-from-ubuntu1804.json" python3 -m unittest -v test.test_boot diff --git a/osbuild.spec b/osbuild.spec index 87fa3353..806425dd 100644 --- a/osbuild.spec +++ b/osbuild.spec @@ -16,6 +16,7 @@ BuildRequires: python3-devel Requires: bash Requires: coreutils +Requires: curl Requires: dnf Requires: e2fsprogs Requires: glibc diff --git a/osbuild/pipeline.py b/osbuild/pipeline.py index 8984a0a7..5d3ccb6d 100644 --- a/osbuild/pipeline.py +++ b/osbuild/pipeline.py @@ -73,23 +73,26 @@ class Stage: tree, runner, build_tree, + cache, interactive=False, libdir=None, var="/var/tmp", source_options=None, secrets=None): - with buildroot.BuildRoot(build_tree, runner, libdir=libdir, var=var) as build_root: + with buildroot.BuildRoot(build_tree, runner, libdir=libdir, var=var) as build_root, \ + tempfile.TemporaryDirectory(prefix="osbuild-sources-output-", dir=var) as sources_output: if interactive: print_header(f"{self.name}: {self.id}", self.options) args = { "tree": "/run/osbuild/tree", + "sources": "/run/osbuild/sources", "options": self.options, } sources_dir = f"{libdir}/sources" if libdir else "/usr/lib/osbuild/sources" - ro_binds = [] + ro_binds = [f"{sources_output}:/run/osbuild/sources"] if not libdir: osbuild_module_path = os.path.dirname(importlib.util.find_spec('osbuild').origin) # This is a temporary workaround, once we have a common way to include osbuild in the @@ -98,7 +101,12 @@ class Stage: ro_binds.append(f"{osbuild_module_path}:/run/osbuild/lib/stages/osbuild") with API(f"{build_root.api}/osbuild", args, interactive) as api, \ - sources.SourcesServer(f"{build_root.api}/sources", sources_dir, source_options, secrets): + sources.SourcesServer(f"{build_root.api}/sources", + sources_dir, + source_options, + f"{cache}/sources", + sources_output, + secrets): r = build_root.run( [f"/run/osbuild/lib/stages/{self.name}"], binds=[f"{tree}:/run/osbuild/tree"], @@ -267,6 +275,7 @@ class Pipeline: r = stage.run(tree, self.runner, build_tree, + store, interactive=interactive, libdir=libdir, var=store, diff --git a/osbuild/sources.py b/osbuild/sources.py index c3770dc9..38eeef51 100644 --- a/osbuild/sources.py +++ b/osbuild/sources.py @@ -6,23 +6,29 @@ import threading class SourcesServer: - def __init__(self, socket_address, sources_dir, source_options, secrets=None): + # pylint: disable=too-many-instance-attributes + def __init__(self, socket_address, sources_libdir, options, cache, output, secrets=None): self.socket_address = socket_address - self.sources_dir = sources_dir - self.source_options = source_options or {} + self.sources_libdir = sources_libdir + self.cache = cache + self.output = output + self.options = options or {} self.secrets = secrets or {} self.event_loop = asyncio.new_event_loop() self.thread = threading.Thread(target=self._run_event_loop) + self.barrier = threading.Barrier(2) def _run_source(self, source, checksums): msg = { - "options": self.source_options.get(source, {}), + "options": self.options.get(source, {}), "secrets": self.secrets.get(source, {}), + "cache": f"{self.cache}/{source}", + "output": f"{self.output}/{source}", "checksums": checksums } r = subprocess.run( - [f"{self.sources_dir}/{source}"], + [f"{self.sources_libdir}/{source}"], input=json.dumps(msg), stdout=subprocess.PIPE, encoding="utf-8", @@ -43,6 +49,7 @@ class SourcesServer: def _run_event_loop(self): sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) sock.bind(self.socket_address) + self.barrier.wait() self.event_loop.add_reader(sock, self._dispatch, sock) asyncio.set_event_loop(self.event_loop) self.event_loop.run_forever() @@ -51,6 +58,7 @@ class SourcesServer: def __enter__(self): self.thread.start() + self.barrier.wait() return self def __exit__(self, *args): @@ -58,10 +66,10 @@ class SourcesServer: self.thread.join() -def get(source, checksums): +def get(source, checksums, api_path="/run/osbuild/api/sources"): with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock: sock.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1) - sock.connect("/run/osbuild/api/sources") + sock.connect(api_path) msg = { "source": source, "checksums": checksums diff --git a/sources/org.osbuild.files b/sources/org.osbuild.files new file mode 100755 index 00000000..430e5387 --- /dev/null +++ b/sources/org.osbuild.files @@ -0,0 +1,108 @@ +#!/usr/bin/python3 + +import concurrent.futures +import itertools +import json +import os +import subprocess +import sys +import tempfile + + +def verify_checksum(filename, checksum): + algorithm, checksum = checksum.split(":", 1) + if algorithm not in ("md5", "sha1", "sha256", "sha384", "sha512"): + raise RuntimeError(f"unsupported checksum algorithm: {algorithm}") + + ret = subprocess.run( + [f"{algorithm}sum", "-c"], + input=f"{checksum} {filename}", + stdout=subprocess.DEVNULL, + encoding="utf-8", + check=False + ) + + return ret.returncode == 0 + + +def fetch(url, checksum, directory): + # Invariant: all files in @directory must be named after their (verified) checksum. + if os.path.isfile(f"{directory}/{checksum}"): + return + + # Download to a temporary directory until we have verified the checksum. Use a + # subdirectory, so we avoid copying accross block devices. + with tempfile.TemporaryDirectory(prefix="osbuild-unverified-file-", dir=directory) as tmpdir: + # some mirrors are broken sometimes. retry manually, because curl doesn't on 404 + for _ in range(3): + curl = subprocess.run([ + "curl", + "--silent", + "--show-error", + "--fail", + "--location", + "--output", checksum, + url + ], encoding="utf-8", cwd=tmpdir, check=False) + if curl.returncode == 0: + break + else: + raise RuntimeError(f"error downloading {url}") + + if not verify_checksum(f"{tmpdir}/{checksum}", checksum): + raise RuntimeError(f"checksum mismatch: {checksum} {url}") + + # The checksum has been verified, move the file into place. in case we race + # another download of the same file, we simply ignore the error as their + # contents are guaranteed to be the same. + try: + os.rename(f"{tmpdir}/{checksum}", f"{directory}/{checksum}") + except FileExistsError: + pass + + +def main(options, checksums, cache, output): + urls = options.get("urls", {}) + + os.makedirs(cache, exist_ok=True) + os.makedirs(output, exist_ok=True) + + with concurrent.futures.ProcessPoolExecutor(max_workers=10) as executor: + requested_urls = [] + for checksum in checksums: + try: + requested_urls.append(urls[checksum]) + except KeyError: + json.dump({"error": f"unknown file: {checksum}"}, sys.stdout) + return 1 + results = executor.map(fetch, requested_urls, checksums, itertools.repeat(cache)) + + try: + for _ in results: + pass + except RuntimeError as e: + json.dump({"error": e.args[0]}, sys.stdout) + return 1 + + for checksum in checksums: + try: + subprocess.run([ + "cp", + "--reflink=auto", + f"{cache}/{checksum}", + f"{output}/{checksum}"], + check=True) + except FileExistsError: + continue + except Exception as e: + json.dump({"error": e.message}, sys.stdout) + return 1 + + json.dump({}, sys.stdout) + return 0 + + +if __name__ == '__main__': + args = json.load(sys.stdin) + r = main(args["options"], args["checksums"], args["cache"], args["output"]) + sys.exit(r) diff --git a/test/sources_tests/org.osbuild.files/cases/empty.json b/test/sources_tests/org.osbuild.files/cases/empty.json new file mode 100644 index 00000000..2a645c83 --- /dev/null +++ b/test/sources_tests/org.osbuild.files/cases/empty.json @@ -0,0 +1,4 @@ +{ + "expects": "success", + "checksums": [] +} diff --git a/test/sources_tests/org.osbuild.files/cases/invalid_checksum.json b/test/sources_tests/org.osbuild.files/cases/invalid_checksum.json new file mode 100644 index 00000000..4ceb9500 --- /dev/null +++ b/test/sources_tests/org.osbuild.files/cases/invalid_checksum.json @@ -0,0 +1,6 @@ +{ + "expects": "error", + "checksums": [ + "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] +} diff --git a/test/sources_tests/org.osbuild.files/cases/missing_file.json b/test/sources_tests/org.osbuild.files/cases/missing_file.json new file mode 100644 index 00000000..1ded1c91 --- /dev/null +++ b/test/sources_tests/org.osbuild.files/cases/missing_file.json @@ -0,0 +1,6 @@ +{ + "expects": "error", + "checksums": [ + "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + ] +} diff --git a/test/sources_tests/org.osbuild.files/cases/success.json b/test/sources_tests/org.osbuild.files/cases/success.json new file mode 100644 index 00000000..e85714df --- /dev/null +++ b/test/sources_tests/org.osbuild.files/cases/success.json @@ -0,0 +1,31 @@ +{ + "expects": "success", + "checksums": [ + "sha256:87428fc522803d31065e7bce3cf03fe475096631e5e07bbd7a0fde60c4cf25c7", + "sha256:0263829989b6fd954f72baaf2fc64bc2e2f01d692d4de72986ea808f6e99813f", + "sha256:a3a5e715f0cc574a73c3f9bebb6bc24f32ffd5b67b387244c2c909da779a1478", + "sha256:8d74beec1be996322ad76813bafb92d40839895d6dd7ee808b17ca201eac98be", + "sha256:a2bbdb2de53523b8099b37013f251546f3d65dbe7a0774fa41af0a4176992fd4", + "sha256:092fcfbbcfca3b5be7ae1b5e58538e92c35ab273ae13664fed0d67484c8e78a6", + "sha256:768c71d785bf6bbbf8c4d6af6582041f2659027140a962cd0c55b11eddfd5e3d", + "sha256:91ee5e9f42ba3d34e414443b36a27b797a56a47aad6bb1e4c1769e69c77ce0ca", + "sha256:50c393f158c3de2db92fa9661bfb00eda5b67c3a777c88524ed3417509631625", + "sha256:cee00b08a818db87e17e703273818e5194f83280e1ef3eae9214ff14675d9e6d", + "sha256:19732980d68fbd00358a0a4d98246c960400b87e4fa2a2e155db98be2b42ed6c", + "sha256:6d7ebc44c5bc26207e62f4f628f912e1a0f41ed11764891aa7dd99eab83228e7", + "sha256:01a60e35df88d8b49546cb3f8f4ba4f406870f9b8e1f394c9d48ab73548d748d", + "sha256:a4fb621495a0122493b2203591c448903c472e306a1ede54fabad829e01075c0", + "sha256:7427d152005f9ed0fa31c76ef9963cf4bb47dce6e2768111d9eb0edbfe59c704", + "sha256:fd6641673e7f3bf6e80e4bc5401fcb2821a1e117206c8e1c65cef23a58dc37ff", + "sha256:4adc33bd9fe74303c344be46e5916d65182fb218e248fe80452ab3f025b06c64", + "sha256:8e54b0ca18020275e4aef1ca0eb5e197e066c065c1864817652a8a39c55402cd", + "sha256:cbc80bb5c0c0f8944bf73b3a429505ac5cde16644978bc9a1e74c5755f8ca556", + "sha256:fe8edeeb98cc6d3b93cf2d57000254b84bd9eba34b4df7ce4b87db8b937b7703", + "sha256:ea46748e171abd2dd4dba5b86bb6589334d86bba2df8d50cbb16b36c83b0856a", + "sha256:73324e1ab1db72ee9eb4fdf1c90a586d67e00ab58330d1cbfea26ecd0a77fa4d", + "sha256:cf945b5236e101dbe0471d5200f28b1ae64f21c1f35bf55fcf40cd0fe42cd8e7", + "sha256:73cb3858a687a8494ca3323053016282f3dad39d42cf62ca4e79dda2aac7d9ac", + "sha256:3bb2abb69ebb27fbfe63c7639624c6ec5e331b841a5bc8c3ebc10b9285e90877", + "sha256:c865f6c5ab8d1b0bcd383a5e1e3879d22681c96bf462c269b7581d523fbe70ab" + ] +} diff --git a/test/sources_tests/org.osbuild.files/cases/unknown_checksum.json b/test/sources_tests/org.osbuild.files/cases/unknown_checksum.json new file mode 100644 index 00000000..16bbdb51 --- /dev/null +++ b/test/sources_tests/org.osbuild.files/cases/unknown_checksum.json @@ -0,0 +1,6 @@ +{ + "expects": "error", + "checksums": [ + "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + ] +} diff --git a/test/sources_tests/org.osbuild.files/data/a b/test/sources_tests/org.osbuild.files/data/a new file mode 100644 index 00000000..78981922 --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/a @@ -0,0 +1 @@ +a diff --git a/test/sources_tests/org.osbuild.files/data/b b/test/sources_tests/org.osbuild.files/data/b new file mode 100644 index 00000000..61780798 --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/b @@ -0,0 +1 @@ +b diff --git a/test/sources_tests/org.osbuild.files/data/c b/test/sources_tests/org.osbuild.files/data/c new file mode 100644 index 00000000..f2ad6c76 --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/c @@ -0,0 +1 @@ +c diff --git a/test/sources_tests/org.osbuild.files/data/d b/test/sources_tests/org.osbuild.files/data/d new file mode 100644 index 00000000..4bcfe98e --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/d @@ -0,0 +1 @@ +d diff --git a/test/sources_tests/org.osbuild.files/data/e b/test/sources_tests/org.osbuild.files/data/e new file mode 100644 index 00000000..d905d9da --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/e @@ -0,0 +1 @@ +e diff --git a/test/sources_tests/org.osbuild.files/data/f b/test/sources_tests/org.osbuild.files/data/f new file mode 100644 index 00000000..6a69f920 --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/f @@ -0,0 +1 @@ +f diff --git a/test/sources_tests/org.osbuild.files/data/g b/test/sources_tests/org.osbuild.files/data/g new file mode 100644 index 00000000..01058d84 --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/g @@ -0,0 +1 @@ +g diff --git a/test/sources_tests/org.osbuild.files/data/h b/test/sources_tests/org.osbuild.files/data/h new file mode 100644 index 00000000..6e9f0da1 --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/h @@ -0,0 +1 @@ +h diff --git a/test/sources_tests/org.osbuild.files/data/i b/test/sources_tests/org.osbuild.files/data/i new file mode 100644 index 00000000..0ddf2bae --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/i @@ -0,0 +1 @@ +i diff --git a/test/sources_tests/org.osbuild.files/data/j b/test/sources_tests/org.osbuild.files/data/j new file mode 100644 index 00000000..4c559f78 --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/j @@ -0,0 +1 @@ +j diff --git a/test/sources_tests/org.osbuild.files/data/k b/test/sources_tests/org.osbuild.files/data/k new file mode 100644 index 00000000..b68fde2a --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/k @@ -0,0 +1 @@ +k diff --git a/test/sources_tests/org.osbuild.files/data/l b/test/sources_tests/org.osbuild.files/data/l new file mode 100644 index 00000000..1f9d725a --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/l @@ -0,0 +1 @@ +l diff --git a/test/sources_tests/org.osbuild.files/data/m b/test/sources_tests/org.osbuild.files/data/m new file mode 100644 index 00000000..28ce6a8b --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/m @@ -0,0 +1 @@ +m diff --git a/test/sources_tests/org.osbuild.files/data/n b/test/sources_tests/org.osbuild.files/data/n new file mode 100644 index 00000000..8ba3a163 --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/n @@ -0,0 +1 @@ +n diff --git a/test/sources_tests/org.osbuild.files/data/o b/test/sources_tests/org.osbuild.files/data/o new file mode 100644 index 00000000..13e7564e --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/o @@ -0,0 +1 @@ +o diff --git a/test/sources_tests/org.osbuild.files/data/p b/test/sources_tests/org.osbuild.files/data/p new file mode 100644 index 00000000..1a9cc2b7 --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/p @@ -0,0 +1 @@ +p diff --git a/test/sources_tests/org.osbuild.files/data/q b/test/sources_tests/org.osbuild.files/data/q new file mode 100644 index 00000000..bca70f35 --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/q @@ -0,0 +1 @@ +q diff --git a/test/sources_tests/org.osbuild.files/data/r b/test/sources_tests/org.osbuild.files/data/r new file mode 100644 index 00000000..4286f428 --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/r @@ -0,0 +1 @@ +r diff --git a/test/sources_tests/org.osbuild.files/data/s b/test/sources_tests/org.osbuild.files/data/s new file mode 100644 index 00000000..b4785957 --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/s @@ -0,0 +1 @@ +s diff --git a/test/sources_tests/org.osbuild.files/data/t b/test/sources_tests/org.osbuild.files/data/t new file mode 100644 index 00000000..718f4d2f --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/t @@ -0,0 +1 @@ +t diff --git a/test/sources_tests/org.osbuild.files/data/u b/test/sources_tests/org.osbuild.files/data/u new file mode 100644 index 00000000..4ae8ef02 --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/u @@ -0,0 +1 @@ +u diff --git a/test/sources_tests/org.osbuild.files/data/v b/test/sources_tests/org.osbuild.files/data/v new file mode 100644 index 00000000..110ed9b9 --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/v @@ -0,0 +1 @@ +v diff --git a/test/sources_tests/org.osbuild.files/data/w b/test/sources_tests/org.osbuild.files/data/w new file mode 100644 index 00000000..e556b830 --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/w @@ -0,0 +1 @@ +w diff --git a/test/sources_tests/org.osbuild.files/data/x b/test/sources_tests/org.osbuild.files/data/x new file mode 100644 index 00000000..587be6b4 --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/x @@ -0,0 +1 @@ +x diff --git a/test/sources_tests/org.osbuild.files/data/y b/test/sources_tests/org.osbuild.files/data/y new file mode 100644 index 00000000..975fbec8 --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/y @@ -0,0 +1 @@ +y diff --git a/test/sources_tests/org.osbuild.files/data/z b/test/sources_tests/org.osbuild.files/data/z new file mode 100644 index 00000000..b6802534 --- /dev/null +++ b/test/sources_tests/org.osbuild.files/data/z @@ -0,0 +1 @@ +z diff --git a/test/sources_tests/org.osbuild.files/sources.json b/test/sources_tests/org.osbuild.files/sources.json new file mode 100644 index 00000000..c81897e9 --- /dev/null +++ b/test/sources_tests/org.osbuild.files/sources.json @@ -0,0 +1,34 @@ +{ + "org.osbuild.files": { + "urls": { + "sha256:87428fc522803d31065e7bce3cf03fe475096631e5e07bbd7a0fde60c4cf25c7": "http://localhost/test/sources_tests/org.osbuild.files/data/a", + "sha256:0263829989b6fd954f72baaf2fc64bc2e2f01d692d4de72986ea808f6e99813f": "http://localhost/test/sources_tests/org.osbuild.files/data/b", + "sha256:a3a5e715f0cc574a73c3f9bebb6bc24f32ffd5b67b387244c2c909da779a1478": "http://localhost/test/sources_tests/org.osbuild.files/data/c", + "sha256:8d74beec1be996322ad76813bafb92d40839895d6dd7ee808b17ca201eac98be": "http://localhost/test/sources_tests/org.osbuild.files/data/d", + "sha256:a2bbdb2de53523b8099b37013f251546f3d65dbe7a0774fa41af0a4176992fd4": "http://localhost/test/sources_tests/org.osbuild.files/data/e", + "sha256:092fcfbbcfca3b5be7ae1b5e58538e92c35ab273ae13664fed0d67484c8e78a6": "http://localhost/test/sources_tests/org.osbuild.files/data/f", + "sha256:768c71d785bf6bbbf8c4d6af6582041f2659027140a962cd0c55b11eddfd5e3d": "http://localhost/test/sources_tests/org.osbuild.files/data/g", + "sha256:91ee5e9f42ba3d34e414443b36a27b797a56a47aad6bb1e4c1769e69c77ce0ca": "http://localhost/test/sources_tests/org.osbuild.files/data/h", + "sha256:50c393f158c3de2db92fa9661bfb00eda5b67c3a777c88524ed3417509631625": "http://localhost/test/sources_tests/org.osbuild.files/data/i", + "sha256:cee00b08a818db87e17e703273818e5194f83280e1ef3eae9214ff14675d9e6d": "http://localhost/test/sources_tests/org.osbuild.files/data/j", + "sha256:19732980d68fbd00358a0a4d98246c960400b87e4fa2a2e155db98be2b42ed6c": "http://localhost/test/sources_tests/org.osbuild.files/data/k", + "sha256:6d7ebc44c5bc26207e62f4f628f912e1a0f41ed11764891aa7dd99eab83228e7": "http://localhost/test/sources_tests/org.osbuild.files/data/l", + "sha256:01a60e35df88d8b49546cb3f8f4ba4f406870f9b8e1f394c9d48ab73548d748d": "http://localhost/test/sources_tests/org.osbuild.files/data/m", + "sha256:a4fb621495a0122493b2203591c448903c472e306a1ede54fabad829e01075c0": "http://localhost/test/sources_tests/org.osbuild.files/data/n", + "sha256:7427d152005f9ed0fa31c76ef9963cf4bb47dce6e2768111d9eb0edbfe59c704": "http://localhost/test/sources_tests/org.osbuild.files/data/o", + "sha256:fd6641673e7f3bf6e80e4bc5401fcb2821a1e117206c8e1c65cef23a58dc37ff": "http://localhost/test/sources_tests/org.osbuild.files/data/p", + "sha256:4adc33bd9fe74303c344be46e5916d65182fb218e248fe80452ab3f025b06c64": "http://localhost/test/sources_tests/org.osbuild.files/data/q", + "sha256:8e54b0ca18020275e4aef1ca0eb5e197e066c065c1864817652a8a39c55402cd": "http://localhost/test/sources_tests/org.osbuild.files/data/r", + "sha256:cbc80bb5c0c0f8944bf73b3a429505ac5cde16644978bc9a1e74c5755f8ca556": "http://localhost/test/sources_tests/org.osbuild.files/data/s", + "sha256:fe8edeeb98cc6d3b93cf2d57000254b84bd9eba34b4df7ce4b87db8b937b7703": "http://localhost/test/sources_tests/org.osbuild.files/data/t", + "sha256:ea46748e171abd2dd4dba5b86bb6589334d86bba2df8d50cbb16b36c83b0856a": "http://localhost/test/sources_tests/org.osbuild.files/data/u", + "sha256:73324e1ab1db72ee9eb4fdf1c90a586d67e00ab58330d1cbfea26ecd0a77fa4d": "http://localhost/test/sources_tests/org.osbuild.files/data/v", + "sha256:cf945b5236e101dbe0471d5200f28b1ae64f21c1f35bf55fcf40cd0fe42cd8e7": "http://localhost/test/sources_tests/org.osbuild.files/data/w", + "sha256:73cb3858a687a8494ca3323053016282f3dad39d42cf62ca4e79dda2aac7d9ac": "http://localhost/test/sources_tests/org.osbuild.files/data/x", + "sha256:3bb2abb69ebb27fbfe63c7639624c6ec5e331b841a5bc8c3ebc10b9285e90877": "http://localhost/test/sources_tests/org.osbuild.files/data/y", + "sha256:c865f6c5ab8d1b0bcd383a5e1e3879d22681c96bf462c269b7581d523fbe70ab": "http://localhost/test/sources_tests/org.osbuild.files/data/z", + "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": "http://localhost/test/sources_tests/org.osbuild.files/data/a", + "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb": "http://localhost/test/sources_tests/org.osbuild.files/data/missing" + } + } +} diff --git a/test/test_sources.py b/test/test_sources.py new file mode 100644 index 00000000..b5aa4da3 --- /dev/null +++ b/test/test_sources.py @@ -0,0 +1,96 @@ +import contextlib +import ctypes +import json +import os +import osbuild.sources +import socketserver +import subprocess +import tempfile +import threading +import unittest + +from http import server + +def errcheck(ret, func, args): + if ret == -1: + e = ctypes.get_errno() + raise OSError(e, os.strerror(e)) + + +CLONE_NEWNET = 0x40000000 +libc = ctypes.CDLL('libc.so.6', use_errno=True) +libc.setns.errcheck = errcheck + + +@contextlib.contextmanager +def netns(): + # Grab a reference to the current namespace. + with open("/proc/self/ns/net") as oldnet: + # Create a new namespace and enter it. + libc.unshare(CLONE_NEWNET) + # Up the loopback device in the new namespace. + subprocess.run(["ip", "link", "set", "up", "dev", "lo"], check=True) + try: + yield + finally: + # Revert to the old namespace, dropping our + # reference to the new one. + libc.setns(oldnet.fileno(), CLONE_NEWNET) + + +@contextlib.contextmanager +def fileServer(path): + with netns(): + # This is leaked until the program exits, but inaccessible after the with + # due to the network namespace. + barrier = threading.Barrier(2) + thread = threading.Thread(target=runFileServer, args=(path, barrier)) + thread.daemon = True + thread.start() + barrier.wait() + yield + + +def runFileServer(path, barrier): + httpd = socketserver.TCPServer(('', 80), server.SimpleHTTPRequestHandler) + barrier.wait() + httpd.serve_forever() + + +class TestSources(unittest.TestCase): + def setUp(self): + self.sources = 'test/sources_tests' + + + def check_case(self, source, case, destdir, api_path): + expects = case["expects"] + if expects == "error": + with self.assertRaises(RuntimeError): + osbuild.sources.get(source, case["checksums"], api_path=api_path) + elif expects == "success": + r = osbuild.sources.get(source, case["checksums"], api_path=api_path) + self.assertEqual(r, {}) + else: + raise ValueError(f"invalid expectation: {expects}") + + + def check_source(self, source): + source_options = {} + with open(f"{self.sources}/{source}/sources.json") as f: + source_options = json.load(f) + for case in os.listdir(f"{self.sources}/{source}/cases"): + with self.subTest(case=case): + case_options = {} + with open(f"{self.sources}/{source}/cases/{case}") as f: + case_options = json.load(f) + with tempfile.TemporaryDirectory() as tmpdir, \ + fileServer(f"{self.sources}/{source}/data"), \ + osbuild.sources.SourcesServer(f"{tmpdir}/sources-api", "./sources", source_options, f"{tmpdir}/cache", f"{tmpdir}/dst"): + self.check_case(source, case_options, f"{tmpdir}/dst", f"{tmpdir}/sources-api") + self.check_case(source, case_options, f"{tmpdir}/dst", f"{tmpdir}/sources-api") + + + def test_sources(self): + for source in os.listdir(self.sources): + with self.subTest(source=source): + self.check_source(source)