particle-os/debian-forge
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

sources(curl): use --next for each url in curl config

Browse source 
curl keeps a global parser state. This means that if there are
multiple "cacert =" values they are just overriden and the last
one wins. This is why the `test_curl_download_many_mixed_certs`
test did not work - the second `cacert = ` overwrites the previous
one.

To fix this we need to use `--next` when we need to change options
on a per url (like `cacert`) basis. With `--next` curl starts a
new parser state for the next url (but keeps the options for the
previous ones set). This commit does that in a slightly naive
way by just repeating our options for each url. Technically
we could sort the sources so that we have less repetition but
other then slightly smaller auto-generated files it has no
advantage.

With this commit the `test_curl_download_many_mixed_certs` test
works.
This commit is contained in:
Michael Vogt 2024-07-11 17:23:14 +02:00
parent 6ccd5d5cfe
commit a50dbb14c2
2 changed files with 55 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

37
sources/org.osbuild.curl
View file

@ -100,7 +100,7 @@ SCHEMA = """
# We are not just using %{json} here because older curl (7.76) will
# write {"http_connect":000} which python cannot parse so we write our
# own json subset
CURL_WRITE_OUT = r'\{\"url\": \"%{url}\"\, \"filename_effective\": \"%{filename_effective}\", \"exitcode\": %{exitcode}, \"errormsg\": \"%{errormsg}\" \}\n'
CURL_WRITE_OUT_FMT = r'\{\"url\": \"%{url}\"\, \"filename_effective\": \"%{filename_effective}\", \"exitcode\": %{exitcode}, \"errormsg\": \"%{errormsg}\" \}\n'
NR_RETRYS = 10
@ -141,31 +141,37 @@ def _quote_url(url: str) -> str:
def gen_curl_download_config(config_path: pathlib.Path, chksum_desc_tuple: List[Tuple[str, Dict]], parallel=False):
with open(config_path, "w", encoding="utf8") as fp:
# global options
if parallel:
fp.write("parallel\n")
fp.write(textwrap.dedent(f"""\
# Because we use --next which resets the parser state we need to set
# these options for each url.
per_url_opts = textwrap.dedent(f"""\
user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)"
silent
speed-limit = 1000
connect-timeout = 30
fail
location
"""))
""")
if parallel:
fp.write(textwrap.dedent(f"""\
write-out = "{CURL_WRITE_OUT}"
"""))
per_url_opts += textwrap.dedent(f"""\
write-out = "{CURL_WRITE_OUT_FMT}"
""")
proxy = os.getenv("OSBUILD_SOURCES_CURL_PROXY")
if proxy:
fp.write(f'proxy = "{proxy}"\n')
fp.write("\n")
# per url options
for checksum, desc in chksum_desc_tuple:
per_url_opts += f'proxy = "{proxy}"\n'
# start with the global option(s)
if parallel:
fp.write(textwrap.dedent("""\
# global options
parallel
"""))
# then generate the per-url config
fp.write("# per-url options\n")
for i, (checksum, desc) in enumerate(chksum_desc_tuple):
url = _quote_url(desc.get("url"))
fp.write(f'url = "{url}"\n')
fp.write(f'output = "{checksum}"\n')
fp.write(f'{per_url_opts}')
secrets = desc.get("secrets")
if secrets:
ssl_ca_cert = secrets.get('ssl_ca_cert')
@ -182,7 +188,8 @@ def gen_curl_download_config(config_path: pathlib.Path, chksum_desc_tuple: List[
fp.write('insecure\n')
else:
fp.write('no-insecure\n')
fp.write("\n")
if i + 1 < len(chksum_desc_tuple):
fp.write("next\n\n")
def try_parse_curl_line(line):

42
sources/test/test_curl_source.py
View file

@ -261,17 +261,16 @@ def test_curl_gen_download_config_old_curl(tmp_path, sources_module):
assert config_path.exists()
assert config_path.read_text(encoding="utf8") == textwrap.dedent(f"""\
# per-url options
url = "http://example.com/file/0"
output = "sha256:5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9"
user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)"
silent
speed-limit = 1000
connect-timeout = 30
fail
location
url = "http://example.com/file/0"
output = "sha256:5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9"
no-insecure
""")
@ -281,34 +280,59 @@ def test_curl_gen_download_config_parallel(tmp_path, sources_module):
assert config_path.exists()
assert config_path.read_text(encoding="utf8") == textwrap.dedent(f"""\
# global options
parallel
# per-url options
url = "http://example.com/file/0"
output = "sha256:5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9"
user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)"
silent
speed-limit = 1000
connect-timeout = 30
fail
location
write-out = "{sources_module.CURL_WRITE_OUT}"
url = "http://example.com/file/0"
output = "sha256:5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9"
write-out = "{sources_module.CURL_WRITE_OUT_FMT}"
no-insecure
next
url = "http://example.com/file/1"
output = "sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b"
user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)"
silent
speed-limit = 1000
connect-timeout = 30
fail
location
write-out = "{sources_module.CURL_WRITE_OUT_FMT}"
insecure
next
url = "http://example.com/file/2"
output = "sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35"
user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)"
silent
speed-limit = 1000
connect-timeout = 30
fail
location
write-out = "{sources_module.CURL_WRITE_OUT_FMT}"
cacert = "some-ssl_ca_cert"
no-insecure
next
url = "http://example.com/file/3"
output = "sha256:4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce"
user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)"
silent
speed-limit = 1000
connect-timeout = 30
fail
location
write-out = "{sources_module.CURL_WRITE_OUT_FMT}"
cert = "some-ssl_client_cert"
key = "some-ssl_client_key"
no-insecure
""")