From 7a48c3b73443addc8fea072acdd20912e094b04a Mon Sep 17 00:00:00 2001 From: Christian Kellner Date: Tue, 27 Apr 2021 18:31:29 +0000 Subject: [PATCH] test/sources: check Source.download Instead of testing the legacy `SourcesServer` and `sources.get`, test the `Source.download` method, which is the new and exclusive way how sources are used in osbuild. For this, the complete info for a specific source is now included the specific test case. For the `org.osbuild.curl` source this means that the respective information is moved from `sources.json`, which is then unused and thus removed. The test case that checks for an unknown checksum is also removed because `Source.download` just fetches everything instead of a subset. --- .../sources/org.osbuild.curl/cases/empty.json | 4 +- .../cases/invalid_checksum.json | 10 +++- .../org.osbuild.curl/cases/missing_file.json | 10 +++- .../org.osbuild.curl/cases/success.json | 58 ++++++++++--------- .../cases/unknown_checksum.json | 6 -- .../sources/org.osbuild.curl/sources.json | 34 ----------- test/run/test_sources.py | 36 ++++++------ 7 files changed, 66 insertions(+), 92 deletions(-) delete mode 100644 test/data/sources/org.osbuild.curl/cases/unknown_checksum.json delete mode 100644 test/data/sources/org.osbuild.curl/sources.json diff --git a/test/data/sources/org.osbuild.curl/cases/empty.json b/test/data/sources/org.osbuild.curl/cases/empty.json index 2a645c83..dc9ae3e0 100644 --- a/test/data/sources/org.osbuild.curl/cases/empty.json +++ b/test/data/sources/org.osbuild.curl/cases/empty.json @@ -1,4 +1,6 @@ { "expects": "success", - "checksums": [] + "org.osbuild.curl": { + "items": {} + } } diff --git a/test/data/sources/org.osbuild.curl/cases/invalid_checksum.json b/test/data/sources/org.osbuild.curl/cases/invalid_checksum.json index 4ceb9500..c88fb6a6 100644 --- a/test/data/sources/org.osbuild.curl/cases/invalid_checksum.json +++ b/test/data/sources/org.osbuild.curl/cases/invalid_checksum.json @@ -1,6 +1,10 @@ { "expects": "error", - "checksums": [ - "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - ] + "org.osbuild.curl": { + "items": { + "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": { + "url": "http://localhost/sources/org.osbuild.curl/data/a" + } + } + } } diff --git a/test/data/sources/org.osbuild.curl/cases/missing_file.json b/test/data/sources/org.osbuild.curl/cases/missing_file.json index 1ded1c91..9ed3dedb 100644 --- a/test/data/sources/org.osbuild.curl/cases/missing_file.json +++ b/test/data/sources/org.osbuild.curl/cases/missing_file.json @@ -1,6 +1,10 @@ { "expects": "error", - "checksums": [ - "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - ] + "org.osbuild.curl": { + "items": { + "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb": { + "url": "http://localhost/sources/org.osbuild.curl/data/missing" + } + } + } } diff --git a/test/data/sources/org.osbuild.curl/cases/success.json b/test/data/sources/org.osbuild.curl/cases/success.json index e85714df..aeaa7eca 100644 --- a/test/data/sources/org.osbuild.curl/cases/success.json +++ b/test/data/sources/org.osbuild.curl/cases/success.json @@ -1,31 +1,33 @@ { "expects": "success", - "checksums": [ - "sha256:87428fc522803d31065e7bce3cf03fe475096631e5e07bbd7a0fde60c4cf25c7", - "sha256:0263829989b6fd954f72baaf2fc64bc2e2f01d692d4de72986ea808f6e99813f", - "sha256:a3a5e715f0cc574a73c3f9bebb6bc24f32ffd5b67b387244c2c909da779a1478", - "sha256:8d74beec1be996322ad76813bafb92d40839895d6dd7ee808b17ca201eac98be", - "sha256:a2bbdb2de53523b8099b37013f251546f3d65dbe7a0774fa41af0a4176992fd4", - "sha256:092fcfbbcfca3b5be7ae1b5e58538e92c35ab273ae13664fed0d67484c8e78a6", - "sha256:768c71d785bf6bbbf8c4d6af6582041f2659027140a962cd0c55b11eddfd5e3d", - "sha256:91ee5e9f42ba3d34e414443b36a27b797a56a47aad6bb1e4c1769e69c77ce0ca", - "sha256:50c393f158c3de2db92fa9661bfb00eda5b67c3a777c88524ed3417509631625", - "sha256:cee00b08a818db87e17e703273818e5194f83280e1ef3eae9214ff14675d9e6d", - "sha256:19732980d68fbd00358a0a4d98246c960400b87e4fa2a2e155db98be2b42ed6c", - "sha256:6d7ebc44c5bc26207e62f4f628f912e1a0f41ed11764891aa7dd99eab83228e7", - "sha256:01a60e35df88d8b49546cb3f8f4ba4f406870f9b8e1f394c9d48ab73548d748d", - "sha256:a4fb621495a0122493b2203591c448903c472e306a1ede54fabad829e01075c0", - "sha256:7427d152005f9ed0fa31c76ef9963cf4bb47dce6e2768111d9eb0edbfe59c704", - "sha256:fd6641673e7f3bf6e80e4bc5401fcb2821a1e117206c8e1c65cef23a58dc37ff", - "sha256:4adc33bd9fe74303c344be46e5916d65182fb218e248fe80452ab3f025b06c64", - "sha256:8e54b0ca18020275e4aef1ca0eb5e197e066c065c1864817652a8a39c55402cd", - "sha256:cbc80bb5c0c0f8944bf73b3a429505ac5cde16644978bc9a1e74c5755f8ca556", - "sha256:fe8edeeb98cc6d3b93cf2d57000254b84bd9eba34b4df7ce4b87db8b937b7703", - "sha256:ea46748e171abd2dd4dba5b86bb6589334d86bba2df8d50cbb16b36c83b0856a", - "sha256:73324e1ab1db72ee9eb4fdf1c90a586d67e00ab58330d1cbfea26ecd0a77fa4d", - "sha256:cf945b5236e101dbe0471d5200f28b1ae64f21c1f35bf55fcf40cd0fe42cd8e7", - "sha256:73cb3858a687a8494ca3323053016282f3dad39d42cf62ca4e79dda2aac7d9ac", - "sha256:3bb2abb69ebb27fbfe63c7639624c6ec5e331b841a5bc8c3ebc10b9285e90877", - "sha256:c865f6c5ab8d1b0bcd383a5e1e3879d22681c96bf462c269b7581d523fbe70ab" - ] + "org.osbuild.curl": { + "items": { + "sha256:87428fc522803d31065e7bce3cf03fe475096631e5e07bbd7a0fde60c4cf25c7": "http://localhost/sources/org.osbuild.curl/data/a", + "sha256:0263829989b6fd954f72baaf2fc64bc2e2f01d692d4de72986ea808f6e99813f": "http://localhost/sources/org.osbuild.curl/data/b", + "sha256:a3a5e715f0cc574a73c3f9bebb6bc24f32ffd5b67b387244c2c909da779a1478": "http://localhost/sources/org.osbuild.curl/data/c", + "sha256:8d74beec1be996322ad76813bafb92d40839895d6dd7ee808b17ca201eac98be": "http://localhost/sources/org.osbuild.curl/data/d", + "sha256:a2bbdb2de53523b8099b37013f251546f3d65dbe7a0774fa41af0a4176992fd4": "http://localhost/sources/org.osbuild.curl/data/e", + "sha256:092fcfbbcfca3b5be7ae1b5e58538e92c35ab273ae13664fed0d67484c8e78a6": "http://localhost/sources/org.osbuild.curl/data/f", + "sha256:768c71d785bf6bbbf8c4d6af6582041f2659027140a962cd0c55b11eddfd5e3d": "http://localhost/sources/org.osbuild.curl/data/g", + "sha256:91ee5e9f42ba3d34e414443b36a27b797a56a47aad6bb1e4c1769e69c77ce0ca": "http://localhost/sources/org.osbuild.curl/data/h", + "sha256:50c393f158c3de2db92fa9661bfb00eda5b67c3a777c88524ed3417509631625": "http://localhost/sources/org.osbuild.curl/data/i", + "sha256:cee00b08a818db87e17e703273818e5194f83280e1ef3eae9214ff14675d9e6d": "http://localhost/sources/org.osbuild.curl/data/j", + "sha256:19732980d68fbd00358a0a4d98246c960400b87e4fa2a2e155db98be2b42ed6c": "http://localhost/sources/org.osbuild.curl/data/k", + "sha256:6d7ebc44c5bc26207e62f4f628f912e1a0f41ed11764891aa7dd99eab83228e7": "http://localhost/sources/org.osbuild.curl/data/l", + "sha256:01a60e35df88d8b49546cb3f8f4ba4f406870f9b8e1f394c9d48ab73548d748d": "http://localhost/sources/org.osbuild.curl/data/m", + "sha256:a4fb621495a0122493b2203591c448903c472e306a1ede54fabad829e01075c0": "http://localhost/sources/org.osbuild.curl/data/n", + "sha256:7427d152005f9ed0fa31c76ef9963cf4bb47dce6e2768111d9eb0edbfe59c704": "http://localhost/sources/org.osbuild.curl/data/o", + "sha256:fd6641673e7f3bf6e80e4bc5401fcb2821a1e117206c8e1c65cef23a58dc37ff": "http://localhost/sources/org.osbuild.curl/data/p", + "sha256:4adc33bd9fe74303c344be46e5916d65182fb218e248fe80452ab3f025b06c64": "http://localhost/sources/org.osbuild.curl/data/q", + "sha256:8e54b0ca18020275e4aef1ca0eb5e197e066c065c1864817652a8a39c55402cd": "http://localhost/sources/org.osbuild.curl/data/r", + "sha256:cbc80bb5c0c0f8944bf73b3a429505ac5cde16644978bc9a1e74c5755f8ca556": "http://localhost/sources/org.osbuild.curl/data/s", + "sha256:fe8edeeb98cc6d3b93cf2d57000254b84bd9eba34b4df7ce4b87db8b937b7703": "http://localhost/sources/org.osbuild.curl/data/t", + "sha256:ea46748e171abd2dd4dba5b86bb6589334d86bba2df8d50cbb16b36c83b0856a": "http://localhost/sources/org.osbuild.curl/data/u", + "sha256:73324e1ab1db72ee9eb4fdf1c90a586d67e00ab58330d1cbfea26ecd0a77fa4d": "http://localhost/sources/org.osbuild.curl/data/v", + "sha256:cf945b5236e101dbe0471d5200f28b1ae64f21c1f35bf55fcf40cd0fe42cd8e7": "http://localhost/sources/org.osbuild.curl/data/w", + "sha256:73cb3858a687a8494ca3323053016282f3dad39d42cf62ca4e79dda2aac7d9ac": "http://localhost/sources/org.osbuild.curl/data/x", + "sha256:3bb2abb69ebb27fbfe63c7639624c6ec5e331b841a5bc8c3ebc10b9285e90877": "http://localhost/sources/org.osbuild.curl/data/y", + "sha256:c865f6c5ab8d1b0bcd383a5e1e3879d22681c96bf462c269b7581d523fbe70ab": "http://localhost/sources/org.osbuild.curl/data/z" + } + } } diff --git a/test/data/sources/org.osbuild.curl/cases/unknown_checksum.json b/test/data/sources/org.osbuild.curl/cases/unknown_checksum.json deleted file mode 100644 index 16bbdb51..00000000 --- a/test/data/sources/org.osbuild.curl/cases/unknown_checksum.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "expects": "error", - "checksums": [ - "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" - ] -} diff --git a/test/data/sources/org.osbuild.curl/sources.json b/test/data/sources/org.osbuild.curl/sources.json deleted file mode 100644 index f7b0a5ef..00000000 --- a/test/data/sources/org.osbuild.curl/sources.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "org.osbuild.curl": { - "urls": { - "sha256:87428fc522803d31065e7bce3cf03fe475096631e5e07bbd7a0fde60c4cf25c7": "http://localhost/sources/org.osbuild.curl/data/a", - "sha256:0263829989b6fd954f72baaf2fc64bc2e2f01d692d4de72986ea808f6e99813f": "http://localhost/sources/org.osbuild.curl/data/b", - "sha256:a3a5e715f0cc574a73c3f9bebb6bc24f32ffd5b67b387244c2c909da779a1478": "http://localhost/sources/org.osbuild.curl/data/c", - "sha256:8d74beec1be996322ad76813bafb92d40839895d6dd7ee808b17ca201eac98be": "http://localhost/sources/org.osbuild.curl/data/d", - "sha256:a2bbdb2de53523b8099b37013f251546f3d65dbe7a0774fa41af0a4176992fd4": "http://localhost/sources/org.osbuild.curl/data/e", - "sha256:092fcfbbcfca3b5be7ae1b5e58538e92c35ab273ae13664fed0d67484c8e78a6": "http://localhost/sources/org.osbuild.curl/data/f", - "sha256:768c71d785bf6bbbf8c4d6af6582041f2659027140a962cd0c55b11eddfd5e3d": "http://localhost/sources/org.osbuild.curl/data/g", - "sha256:91ee5e9f42ba3d34e414443b36a27b797a56a47aad6bb1e4c1769e69c77ce0ca": "http://localhost/sources/org.osbuild.curl/data/h", - "sha256:50c393f158c3de2db92fa9661bfb00eda5b67c3a777c88524ed3417509631625": "http://localhost/sources/org.osbuild.curl/data/i", - "sha256:cee00b08a818db87e17e703273818e5194f83280e1ef3eae9214ff14675d9e6d": "http://localhost/sources/org.osbuild.curl/data/j", - "sha256:19732980d68fbd00358a0a4d98246c960400b87e4fa2a2e155db98be2b42ed6c": "http://localhost/sources/org.osbuild.curl/data/k", - "sha256:6d7ebc44c5bc26207e62f4f628f912e1a0f41ed11764891aa7dd99eab83228e7": "http://localhost/sources/org.osbuild.curl/data/l", - "sha256:01a60e35df88d8b49546cb3f8f4ba4f406870f9b8e1f394c9d48ab73548d748d": "http://localhost/sources/org.osbuild.curl/data/m", - "sha256:a4fb621495a0122493b2203591c448903c472e306a1ede54fabad829e01075c0": "http://localhost/sources/org.osbuild.curl/data/n", - "sha256:7427d152005f9ed0fa31c76ef9963cf4bb47dce6e2768111d9eb0edbfe59c704": "http://localhost/sources/org.osbuild.curl/data/o", - "sha256:fd6641673e7f3bf6e80e4bc5401fcb2821a1e117206c8e1c65cef23a58dc37ff": "http://localhost/sources/org.osbuild.curl/data/p", - "sha256:4adc33bd9fe74303c344be46e5916d65182fb218e248fe80452ab3f025b06c64": "http://localhost/sources/org.osbuild.curl/data/q", - "sha256:8e54b0ca18020275e4aef1ca0eb5e197e066c065c1864817652a8a39c55402cd": "http://localhost/sources/org.osbuild.curl/data/r", - "sha256:cbc80bb5c0c0f8944bf73b3a429505ac5cde16644978bc9a1e74c5755f8ca556": "http://localhost/sources/org.osbuild.curl/data/s", - "sha256:fe8edeeb98cc6d3b93cf2d57000254b84bd9eba34b4df7ce4b87db8b937b7703": "http://localhost/sources/org.osbuild.curl/data/t", - "sha256:ea46748e171abd2dd4dba5b86bb6589334d86bba2df8d50cbb16b36c83b0856a": "http://localhost/sources/org.osbuild.curl/data/u", - "sha256:73324e1ab1db72ee9eb4fdf1c90a586d67e00ab58330d1cbfea26ecd0a77fa4d": "http://localhost/sources/org.osbuild.curl/data/v", - "sha256:cf945b5236e101dbe0471d5200f28b1ae64f21c1f35bf55fcf40cd0fe42cd8e7": "http://localhost/sources/org.osbuild.curl/data/w", - "sha256:73cb3858a687a8494ca3323053016282f3dad39d42cf62ca4e79dda2aac7d9ac": "http://localhost/sources/org.osbuild.curl/data/x", - "sha256:3bb2abb69ebb27fbfe63c7639624c6ec5e331b841a5bc8c3ebc10b9285e90877": "http://localhost/sources/org.osbuild.curl/data/y", - "sha256:c865f6c5ab8d1b0bcd383a5e1e3879d22681c96bf462c269b7581d523fbe70ab": "http://localhost/sources/org.osbuild.curl/data/z", - "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": "http://localhost/sources/org.osbuild.curl/data/a", - "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb": "http://localhost/sources/org.osbuild.curl/data/missing" - } - } -} diff --git a/test/run/test_sources.py b/test/run/test_sources.py index 3c04f623..0b80c8be 100644 --- a/test/run/test_sources.py +++ b/test/run/test_sources.py @@ -14,6 +14,8 @@ import threading import pytest +import osbuild.objectstore +import osbuild.meta import osbuild.sources from .. import test @@ -76,36 +78,36 @@ def runFileServer(barrier, directory): httpd.serve_forever() -def check_case(source, case, api_path): +def check_case(source, case, store, libdir): expects = case["expects"] if expects == "error": with pytest.raises(RuntimeError): - osbuild.sources.get( - source, case["checksums"], api_path=api_path) + source.download(store, libdir) elif expects == "success": - r = osbuild.sources.get( - source, case["checksums"], api_path=api_path) - assert r == {} + source.download(store, libdir) else: raise ValueError(f"invalid expectation: {expects}") def check_source(source, sources): - source_options = {} - with open(f"{sources}/{source}/sources.json") as f: - source_options = json.load(f) + index = osbuild.meta.Index(os.curdir) + for case in os.listdir(f"{sources}/{source}/cases"): - case_options = {} with open(f"{sources}/{source}/cases/{case}") as f: case_options = json.load(f) + + info = index.get_module_info("Source", source) + desc = case_options[source] + items = desc.get("items", {}) + options = desc.get("options", {}) + + src = osbuild.sources.Source(info, items, options) + with tempfile.TemporaryDirectory() as tmpdir, \ - fileServer(test.TestBase.locate_test_data()), \ - osbuild.sources.SourcesServer( - "./", source_options, - f"{tmpdir}/cache", f"{tmpdir}/dst", - socket_address=f"{tmpdir}/sources-api"): - check_case(source, case_options, f"{tmpdir}/sources-api") - check_case(source, case_options, f"{tmpdir}/sources-api") + osbuild.objectstore.ObjectStore(tmpdir) as store, \ + fileServer(test.TestBase.locate_test_data()): + check_case(src, case_options, store, index.path) + check_case(src, case_options, store, index.path) @pytest.mark.skipif(not test.TestBase.have_test_data(), reason="no test-data access")