plugins: support for repo package sets

This adds support for specifing the package sets for repositories;
on the command line this can be done via `--repo-package-set` with
and argument of `;` separated package set names. This will result
in repo information being transported via dict instead of plain
strings. Thus the hub plugin's schema was modified accordingly.
Last but not least, the builder plugin now can decode these dicts
and setup the repos accordingly.
Test were added for plugins as well as the integration test changed
to use this new feature.
The first upstream commit that supports this feature is pinned.

test/integration/test_koji.py

@@ -12,11 +12,13 @@ import subprocess
 
 REPOS = {
     "fedora": [
-        "http://download.fedoraproject.org/pub/fedora/linux/releases/$release/Everything/$arch/os"
+        {"url": "http://download.fedoraproject.org/pub/fedora/linux/releases/$release/Everything/$arch/os"}
     ],
     "rhel": [
-        "http://download.devel.redhat.com/released/RHEL-8/$release/BaseOS/x86_64/os/",
-        "http://download.devel.redhat.com/released/RHEL-8/$release/AppStream/x86_64/os/",
+        {"url": "http://download.devel.redhat.com/released/RHEL-8/$release/BaseOS/x86_64/os/",
+         "package_sets": "blueprint; build; packages"},
+        {"url": "http://download.devel.redhat.com/released/RHEL-8/$release/AppStream/x86_64/os/",
+         "package_sets": "blueprint; build; packages"},
     ]
 }
 
@@ -98,9 +100,13 @@ class TestIntegration(unittest.TestCase):
 
         repos = []
         for repo in REPOS[name]:
-            tpl = string.Template(repo)
+            baseurl = repo["url"]
+            package_sets = repo.get("package_sets")
+            tpl = string.Template(baseurl)
             url = tpl.safe_substitute({"release": release})
             repos += ["--repo", url]
+            if package_sets:
+                repos += ["--repo-package-sets", package_sets]
 
         package = f"{name.lower()}-guest"
 

test/unit/test_builder.py

@@ -2,6 +2,8 @@
 # koji hub plugin unit tests
 #
 
+#pylint: disable=too-many-lines
+
 import configparser
 import json
 import os
@@ -960,3 +962,52 @@ class TestBuilderPlugin(PluginTest):
         ireq_refs = [i["ostree"]["ref"] for i in ireqs]
         diff = set(f"osbuild/{a}/r" for a in arches) ^ set(ireq_refs)
         self.assertEqual(diff, set())
+
+    @httpretty.activate
+    def test_compose_repo_complex(self):
+        # Check we properly handle ostree compose requests
+        session = self.mock_session()
+        handler = self.make_handler(session=session)
+
+        arches = ["x86_64", "s390x"]
+        repos = [
+            {"baseurl": "https://first.repo/$arch",
+             "package_sets": ["a", "b", "c", "d"]},
+            {"baseurl": "https://second.repo/$arch",
+             "package_sets": ["alpha"]},
+            {"baseurl": "https://third.repo/$arch"}
+        ]
+        args = ["name", "version", "distro",
+                ["image_type"],
+                "fedora-candidate",
+                arches,
+                {"repo": repos}]
+
+        url = self.plugin.DEFAULT_COMPOSER_URL
+        composer = MockComposer(url, architectures=arches)
+        composer.httpretty_regsiter()
+
+        res = handler.handler(*args)
+        assert res, "invalid compose result"
+        compose_id = res["composer"]["id"]
+        compose = composer.composes.get(compose_id)
+        self.assertIsNotNone(compose)
+
+        ireqs = compose["request"]["image_requests"]
+
+        # Check we got all the requested architectures
+        ireq_arches = [i["architecture"] for i in ireqs]
+        diff = set(arches) ^ set(ireq_arches)
+        self.assertEqual(diff, set())
+
+        for ir in ireqs:
+            arch = ir["architecture"]
+            repos = ir["repositories"]
+            assert len(repos) == 3
+
+            for r in repos:
+                baseurl = r["baseurl"]
+                assert baseurl.endswith(arch)
+                if baseurl.startswith("https://first.repo"):
+                    ps = r.get("package_sets")
+                    assert ps and ps == ["a", "b", "c", "d"]

test/unit/test_cli.py

@@ -182,6 +182,65 @@ class TestCliPlugin(PluginTest):
         r = self.plugin.handle_osbuild_image(options, session, argv)
         self.assertEqual(r, 0)
 
+    def test_repo_package_sets(self):
+        # Check we properly handle ostree specific options
+
+        argv = [
+            # the required positional arguments
+            "name", "version", "distro", "target", "arch1",
+            # optional keyword arguments
+            "--repo", "https://first.repo",
+            "--repo-package-sets", "a; b; c",
+            "--repo-package-sets", "d",
+            "--repo", "https://second.repo",
+            "--repo-package-sets", "alpha",
+            "--repo", "https://third.repo",  # NB: no `--repo-package-set`
+            "--release", "20200202.n2",
+        ]
+
+        expected_args = ["name", "version", "distro",
+                         ['guest-image'],  # the default image type
+                         "target",
+                         ['arch1']]
+
+        expected_opts = {
+            "release": "20200202.n2",
+            "repo": [
+                {"baseurl": "https://first.repo",
+                 "package_sets": ["a", "b", "c", "d"]},
+                {"baseurl": "https://second.repo",
+                 "package_sets": ["alpha"]},
+                {"baseurl": "https://third.repo"}
+            ],
+        }
+
+        task_result = {"compose_id": "42", "build_id": 23}
+        task_id = 1
+        koji_lib = self.mock_koji_lib()
+
+        options = self.mock_options()
+        session = flexmock()
+
+        self.mock_session_add_valid_tag(session)
+
+        session.should_receive("osbuildImage") \
+               .with_args(*expected_args, opts=expected_opts) \
+               .and_return(task_id) \
+               .once()
+
+        session.should_receive("logout") \
+               .with_args() \
+               .once()
+
+        session.should_receive("getTaskResult") \
+               .with_args(task_id) \
+               .and_return(task_result) \
+               .once()
+
+        setattr(self.plugin, "kl", koji_lib)
+        r = self.plugin.handle_osbuild_image(options, session, argv)
+        self.assertEqual(r, 0)
+
     def test_target_check(self):
         # unknown build target
         session = flexmock()