diff --git a/sources/test/test_curl_source.py b/sources/test/test_curl_source.py index 409a4d36..626915db 100644 --- a/sources/test/test_curl_source.py +++ b/sources/test/test_curl_source.py @@ -1,10 +1,15 @@ #!/usr/bin/python3 import contextlib +import hashlib import os +import re +import shutil import pytest +import osbuild.testutil.net + SOURCES_NAME = "org.osbuild.curl" @@ -80,3 +85,85 @@ def test_curl_source_amend_secrets_subscription_mgr(sources_service): checksum = "sha256:1234567890123456789012345678901234567890909b14ffb032aa20fa23d9ad6" checksum, desc = sources_service.amend_secrets(checksum, desc) assert desc["secrets"] == "secret-for-http://localhost:80/a" + + +def test_curl_download_many_fail(sources_service): + TEST_SOURCES = { + "sha:1111111111111111111111111111111111111111111111111111111111111111": { + "url": "http://localhost:9876/random-not-exists", + }, + } + with pytest.raises(RuntimeError) as exp: + sources_service.fetch_all(TEST_SOURCES) + assert str(exp.value) == 'curl: error downloading http://localhost:9876/random-not-exists: error code 7' + + +def make_test_sources(fake_httpd_root, port, n_files): + """ + Create test sources for n_file. All files have the names + 0,1,2... + and the content that matches their name (i.e. file "0" has content "0") + + Returns a sources dict that can be used as input for "fetch_all()" with + the correct hash/urls. + """ + fake_httpd_root.mkdir(exist_ok=True) + sources = {} + for i in range(n_files): + name = f"{i}" + sources[f"sha256:{hashlib.sha256(name.encode()).hexdigest()}"] = { + "url": f"http://localhost:{port}/{name}", + } + (fake_httpd_root / name).write_text(name, encoding="utf8") + + return sources + + +def test_curl_download_many_with_retry(tmp_path, sources_service): + fake_httpd_root = tmp_path / "fake-httpd-root" + + simulate_failures = 2 + with osbuild.testutil.net.http_serve_directory(fake_httpd_root, simulate_failures=simulate_failures) as httpd: + test_sources = make_test_sources(fake_httpd_root, httpd.server_port, 5) + + sources_service.cache = tmp_path / "curl-download-dir" + sources_service.cache.mkdir() + sources_service.fetch_all(test_sources) + # we simulated N failures and we need to fetch K files + assert httpd.reqs == simulate_failures + len(test_sources) + # double downloads happend in the expected format + for chksum in test_sources: + assert (sources_service.cache / chksum).exists() + + +def test_curl_download_many_chksum_validate(tmp_path, sources_service): + fake_httpd_root = tmp_path / "fake-httpd-root" + + with osbuild.testutil.net.http_serve_directory(fake_httpd_root) as httpd: + test_sources = make_test_sources(fake_httpd_root, httpd.server_port, 5) + # "break" the hash of file "1" by replacing the content to no longer + # match the checksum + (fake_httpd_root / "1").write_text("hash-no-longer-matches", encoding="utf8") + + sources_service.cache = tmp_path / "curl-download-dir" + sources_service.cache.mkdir() + with pytest.raises(RuntimeError) as exp: + sources_service.fetch_all(test_sources) + assert re.search(r"checksum mismatch: sha256:.* http://localhost:.*/1", str(exp.value)) + + +def test_curl_download_many_retries(tmp_path, sources_service): + fake_httpd_root = tmp_path / "fake-httpd-root" + + with osbuild.testutil.net.http_serve_directory(fake_httpd_root) as httpd: + test_sources = make_test_sources(fake_httpd_root, httpd.server_port, 5) + # remove all the sources + shutil.rmtree(fake_httpd_root) + + sources_service.cache = tmp_path / "curl-download-dir" + sources_service.cache.mkdir() + with pytest.raises(RuntimeError) as exp: + sources_service.fetch_all(test_sources) + # curl will retry 10 times + assert httpd.reqs == 10 * len(test_sources) + assert "curl: error downloading http://localhost:" in str(exp.value)