sources(curl): use --next for each url in curl config

curl keeps a global parser state. This means that if there are
multiple "cacert =" values they are just overriden and the last
one wins. This is why the `test_curl_download_many_mixed_certs`
test did not work - the second `cacert = ` overwrites the previous
one.

To fix this we need to use `--next` when we need to change options
on a per url (like `cacert`) basis. With `--next` curl starts a
new parser state for the next url (but keeps the options for the
previous ones set). This commit does that in a slightly naive
way by just repeating our options for each url. Technically
we could sort the sources so that we have less repetition but
other then slightly smaller auto-generated files it has no
advantage.

With this commit the `test_curl_download_many_mixed_certs` test
works.

sources/org.osbuild.curl

@@ -100,7 +100,7 @@ SCHEMA = """
 # We are not just using %{json} here because older curl (7.76) will
 # write {"http_connect":000} which python cannot parse so we write our
 # own json subset
-CURL_WRITE_OUT = r'\{\"url\": \"%{url}\"\, \"filename_effective\": \"%{filename_effective}\", \"exitcode\": %{exitcode}, \"errormsg\": \"%{errormsg}\" \}\n'
+CURL_WRITE_OUT_FMT = r'\{\"url\": \"%{url}\"\, \"filename_effective\": \"%{filename_effective}\", \"exitcode\": %{exitcode}, \"errormsg\": \"%{errormsg}\" \}\n'
 
 NR_RETRYS = 10
 
@@ -141,31 +141,37 @@ def _quote_url(url: str) -> str:
 
 def gen_curl_download_config(config_path: pathlib.Path, chksum_desc_tuple: List[Tuple[str, Dict]], parallel=False):
     with open(config_path, "w", encoding="utf8") as fp:
-        # global options
+        # Because we use --next which resets the parser state we need to set
-        if parallel:
+        # these options for each url.
-            fp.write("parallel\n")
+        per_url_opts = textwrap.dedent(f"""\
-        fp.write(textwrap.dedent(f"""\
         user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)"
         silent
         speed-limit = 1000
         connect-timeout = 30
         fail
         location
-        """))
+        """)
         if parallel:
-            fp.write(textwrap.dedent(f"""\
+            per_url_opts += textwrap.dedent(f"""\
-            write-out = "{CURL_WRITE_OUT}"
+            write-out = "{CURL_WRITE_OUT_FMT}"
-            """))
+            """)
-
         proxy = os.getenv("OSBUILD_SOURCES_CURL_PROXY")
         if proxy:
-            fp.write(f'proxy = "{proxy}"\n')
+            per_url_opts += f'proxy = "{proxy}"\n'
-        fp.write("\n")
+        # start with the global option(s)
-        # per url options
+        if parallel:
-        for checksum, desc in chksum_desc_tuple:
+            fp.write(textwrap.dedent("""\
+            # global options
+            parallel
+
+            """))
+        # then generate the per-url config
+        fp.write("# per-url options\n")
+        for i, (checksum, desc) in enumerate(chksum_desc_tuple):
             url = _quote_url(desc.get("url"))
             fp.write(f'url = "{url}"\n')
             fp.write(f'output = "{checksum}"\n')
+            fp.write(f'{per_url_opts}')
             secrets = desc.get("secrets")
             if secrets:
                 ssl_ca_cert = secrets.get('ssl_ca_cert')
@@ -182,7 +188,8 @@ def gen_curl_download_config(config_path: pathlib.Path, chksum_desc_tuple: List[
                 fp.write('insecure\n')
             else:
                 fp.write('no-insecure\n')
-            fp.write("\n")
+            if i + 1 < len(chksum_desc_tuple):
+                fp.write("next\n\n")
 
 
 def try_parse_curl_line(line):

sources/test/test_curl_source.py

@@ -261,17 +261,16 @@ def test_curl_gen_download_config_old_curl(tmp_path, sources_module):
 
     assert config_path.exists()
     assert config_path.read_text(encoding="utf8") == textwrap.dedent(f"""\
+    # per-url options
+    url = "http://example.com/file/0"
+    output = "sha256:5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9"
     user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)"
     silent
     speed-limit = 1000
     connect-timeout = 30
     fail
     location
-
-    url = "http://example.com/file/0"
-    output = "sha256:5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9"
     no-insecure
-
     """)
 
 
@@ -281,34 +280,59 @@ def test_curl_gen_download_config_parallel(tmp_path, sources_module):
 
     assert config_path.exists()
     assert config_path.read_text(encoding="utf8") == textwrap.dedent(f"""\
+    # global options
     parallel
+
+    # per-url options
+    url = "http://example.com/file/0"
+    output = "sha256:5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9"
     user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)"
     silent
     speed-limit = 1000
     connect-timeout = 30
     fail
     location
-    write-out = "{sources_module.CURL_WRITE_OUT}"
+    write-out = "{sources_module.CURL_WRITE_OUT_FMT}"
-
-    url = "http://example.com/file/0"
-    output = "sha256:5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9"
     no-insecure
+    next
 
     url = "http://example.com/file/1"
     output = "sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b"
+    user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)"
+    silent
+    speed-limit = 1000
+    connect-timeout = 30
+    fail
+    location
+    write-out = "{sources_module.CURL_WRITE_OUT_FMT}"
     insecure
+    next
 
     url = "http://example.com/file/2"
     output = "sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35"
+    user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)"
+    silent
+    speed-limit = 1000
+    connect-timeout = 30
+    fail
+    location
+    write-out = "{sources_module.CURL_WRITE_OUT_FMT}"
     cacert = "some-ssl_ca_cert"
     no-insecure
+    next
 
     url = "http://example.com/file/3"
     output = "sha256:4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce"
+    user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)"
+    silent
+    speed-limit = 1000
+    connect-timeout = 30
+    fail
+    location
+    write-out = "{sources_module.CURL_WRITE_OUT_FMT}"
     cert = "some-ssl_client_cert"
     key = "some-ssl_client_key"
     no-insecure
-
     """)