diff --git a/sources/org.osbuild.curl b/sources/org.osbuild.curl index 8add8f39..aab7ba94 100755 --- a/sources/org.osbuild.curl +++ b/sources/org.osbuild.curl @@ -100,7 +100,7 @@ SCHEMA = """ # We are not just using %{json} here because older curl (7.76) will # write {"http_connect":000} which python cannot parse so we write our # own json subset -CURL_WRITE_OUT = r'\{\"url\": \"%{url}\"\, \"filename_effective\": \"%{filename_effective}\", \"exitcode\": %{exitcode}, \"errormsg\": \"%{errormsg}\" \}\n' +CURL_WRITE_OUT_FMT = r'\{\"url\": \"%{url}\"\, \"filename_effective\": \"%{filename_effective}\", \"exitcode\": %{exitcode}, \"errormsg\": \"%{errormsg}\" \}\n' NR_RETRYS = 10 @@ -141,31 +141,37 @@ def _quote_url(url: str) -> str: def gen_curl_download_config(config_path: pathlib.Path, chksum_desc_tuple: List[Tuple[str, Dict]], parallel=False): with open(config_path, "w", encoding="utf8") as fp: - # global options - if parallel: - fp.write("parallel\n") - fp.write(textwrap.dedent(f"""\ + # Because we use --next which resets the parser state we need to set + # these options for each url. + per_url_opts = textwrap.dedent(f"""\ user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)" silent speed-limit = 1000 connect-timeout = 30 fail location - """)) + """) if parallel: - fp.write(textwrap.dedent(f"""\ - write-out = "{CURL_WRITE_OUT}" - """)) - + per_url_opts += textwrap.dedent(f"""\ + write-out = "{CURL_WRITE_OUT_FMT}" + """) proxy = os.getenv("OSBUILD_SOURCES_CURL_PROXY") if proxy: - fp.write(f'proxy = "{proxy}"\n') - fp.write("\n") - # per url options - for checksum, desc in chksum_desc_tuple: + per_url_opts += f'proxy = "{proxy}"\n' + # start with the global option(s) + if parallel: + fp.write(textwrap.dedent("""\ + # global options + parallel + + """)) + # then generate the per-url config + fp.write("# per-url options\n") + for i, (checksum, desc) in enumerate(chksum_desc_tuple): url = _quote_url(desc.get("url")) fp.write(f'url = "{url}"\n') fp.write(f'output = "{checksum}"\n') + fp.write(f'{per_url_opts}') secrets = desc.get("secrets") if secrets: ssl_ca_cert = secrets.get('ssl_ca_cert') @@ -182,7 +188,8 @@ def gen_curl_download_config(config_path: pathlib.Path, chksum_desc_tuple: List[ fp.write('insecure\n') else: fp.write('no-insecure\n') - fp.write("\n") + if i + 1 < len(chksum_desc_tuple): + fp.write("next\n\n") def try_parse_curl_line(line): diff --git a/sources/test/test_curl_source.py b/sources/test/test_curl_source.py index 7d278b95..6f128a03 100644 --- a/sources/test/test_curl_source.py +++ b/sources/test/test_curl_source.py @@ -261,17 +261,16 @@ def test_curl_gen_download_config_old_curl(tmp_path, sources_module): assert config_path.exists() assert config_path.read_text(encoding="utf8") == textwrap.dedent(f"""\ + # per-url options + url = "http://example.com/file/0" + output = "sha256:5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9" user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)" silent speed-limit = 1000 connect-timeout = 30 fail location - - url = "http://example.com/file/0" - output = "sha256:5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9" no-insecure - """) @@ -281,34 +280,59 @@ def test_curl_gen_download_config_parallel(tmp_path, sources_module): assert config_path.exists() assert config_path.read_text(encoding="utf8") == textwrap.dedent(f"""\ + # global options parallel + + # per-url options + url = "http://example.com/file/0" + output = "sha256:5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9" user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)" silent speed-limit = 1000 connect-timeout = 30 fail location - write-out = "{sources_module.CURL_WRITE_OUT}" - - url = "http://example.com/file/0" - output = "sha256:5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9" + write-out = "{sources_module.CURL_WRITE_OUT_FMT}" no-insecure + next url = "http://example.com/file/1" output = "sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b" + user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)" + silent + speed-limit = 1000 + connect-timeout = 30 + fail + location + write-out = "{sources_module.CURL_WRITE_OUT_FMT}" insecure + next url = "http://example.com/file/2" output = "sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35" + user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)" + silent + speed-limit = 1000 + connect-timeout = 30 + fail + location + write-out = "{sources_module.CURL_WRITE_OUT_FMT}" cacert = "some-ssl_ca_cert" no-insecure + next url = "http://example.com/file/3" output = "sha256:4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce" + user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)" + silent + speed-limit = 1000 + connect-timeout = 30 + fail + location + write-out = "{sources_module.CURL_WRITE_OUT_FMT}" cert = "some-ssl_client_cert" key = "some-ssl_client_key" no-insecure - """)