From 23bf01bb45dec42e147fe3f2cf11d925bb4d9c14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Fri, 8 Feb 2019 10:10:41 +0100 Subject: [PATCH] hybrid: Apply filters to debuginfo and source packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For binary packages the filters are handled at the depsolver level. However sources and debuginfo is added later in the process, so the filters have to be explicitly applied. JIRA: COMPOSE-3114 Signed-off-by: Lubomír Sedlář --- pungi/phases/gather/methods/method_hybrid.py | 17 +++- tests/test_gather_method_hybrid.py | 101 +++++++++++++++---- 2 files changed, 94 insertions(+), 24 deletions(-) diff --git a/pungi/phases/gather/methods/method_hybrid.py b/pungi/phases/gather/methods/method_hybrid.py index 830f7cd9..c2f2ca0a 100644 --- a/pungi/phases/gather/methods/method_hybrid.py +++ b/pungi/phases/gather/methods/method_hybrid.py @@ -191,17 +191,16 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase): # Filters are received as tuples (name, arch), we should convert it to # strings. - filter_packages = [_fmt_pkg(*p) for p in filter_packages] + filters = [_fmt_pkg(*p) for p in filter_packages] - nvrs, out_modules = self.run_solver( - variant, arch, packages, platform, filter_packages - ) + nvrs, out_modules = self.run_solver(variant, arch, packages, platform, filters) filter_modules(variant, arch, out_modules) return expand_packages( self._get_pkg_map(arch), variant.arch_mmds.get(arch, {}), pungi.phases.gather.get_lookaside_repos(self.compose, arch, variant), nvrs, + filter_packages=filter_packages, ) # maybe check invalid sigkeys @@ -475,7 +474,7 @@ def _make_result(paths): return [{"path": path, "flags": []} for path in sorted(paths)] -def expand_packages(nevra_to_pkg, variant_modules, lookasides, nvrs): +def expand_packages(nevra_to_pkg, variant_modules, lookasides, nvrs, filter_packages): """For each package add source RPM and possibly also debuginfo.""" # This will server as the final result. We collect sets of paths to the # packages. @@ -483,6 +482,8 @@ def expand_packages(nevra_to_pkg, variant_modules, lookasides, nvrs): srpms = set() debuginfo = set() + filters = set(filter_packages) + # Collect list of all packages in lookaside. These will not be added to the # result. Fus handles this in part: if a package is explicitly mentioned as # input (which can happen with comps group expansion), it will be in the @@ -520,6 +521,9 @@ def expand_packages(nevra_to_pkg, variant_modules, lookasides, nvrs): # debuginfo is explicitly listed in the output, and we don't # want anything more. srpm_arches[srpm_nevra].add(arch) + if (srpm.name, "src") in filters: + # Filtered package, skipping + continue if srpm.file_path not in lookaside_packages: srpms.add(srpm.file_path) except KeyError: @@ -531,6 +535,9 @@ def expand_packages(nevra_to_pkg, variant_modules, lookasides, nvrs): # for architecture that has at least one binary package, we include it too. for pkg in nevra_to_pkg.values(): if pkg_is_debug(pkg) and pkg.arch in srpm_arches[_get_srpm_nevra(pkg)]: + if set([(pkg.name, pkg.arch), (pkg.name, None)]) & filters: + # Filtered package, skipping + continue if pkg.file_path not in lookaside_packages: debuginfo.add(pkg.file_path) diff --git a/tests/test_gather_method_hybrid.py b/tests/test_gather_method_hybrid.py index 1abb5aed..458b0570 100644 --- a/tests/test_gather_method_hybrid.py +++ b/tests/test_gather_method_hybrid.py @@ -79,6 +79,7 @@ class TestMethodHybrid(helpers.PungiTestCase): {}, glr.return_value, m.run_solver.return_value[0], + filter_packages=[], ) ], ) @@ -365,7 +366,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): "x86_64", [], platform="pl", - filter_packages=["foo"], + filter_packages=[("foo", None)], ) self.assertEqual(res, po.return_value) @@ -390,7 +391,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): [self._repo("repo"), self._repo("module_repo_Server")], [], platform="pl", - filter_packages=["foo"], + filter_packages=[("foo", None)], ) ], ) @@ -744,11 +745,13 @@ class TestExpandPackages(helpers.PungiTestCase): nevra_to_pkg = {"pkg-3:1-2.x86_64": pkg} if src or debug_arch: nevra_to_pkg["pkg-3:1-2.src"] = pkg._replace( - arch="src", file_path="/tmp/spkg.rpm" + name="pkg", arch="src", file_path="/tmp/pkg.src.rpm" ) if debug_arch: nevra_to_pkg["pkg-debuginfo-3:1-2.%s" % debug_arch] = pkg._replace( - name="pkg-debuginfo", arch=debug_arch, file_path="/tmp/d1.rpm" + name="pkg-debuginfo", + arch=debug_arch, + file_path="/tmp/pkg-debuginfo.%s.rpm" % debug_arch ) return nevra_to_pkg @@ -756,7 +759,7 @@ class TestExpandPackages(helpers.PungiTestCase): nevra_to_pkg = self._mk_packages() res = hybrid.expand_packages( - nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64", [])] + nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64", [])], [] ) self.assertEqual( @@ -772,15 +775,75 @@ class TestExpandPackages(helpers.PungiTestCase): nevra_to_pkg = self._mk_packages(debug_arch="x86_64") res = hybrid.expand_packages( - nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64", [])] + nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64", [])], [] ) self.assertEqual( res, { "rpm": [{"path": "/tmp/pkg.rpm", "flags": []}], - "srpm": [{"path": "/tmp/spkg.rpm", "flags": []}], - "debuginfo": [{"path": "/tmp/d1.rpm", "flags": []}], + "srpm": [{"path": "/tmp/pkg.src.rpm", "flags": []}], + "debuginfo": [{"path": "/tmp/pkg-debuginfo.x86_64.rpm", "flags": []}], + }, + ) + + def test_filter_src_and_debuginfo(self): + nevra_to_pkg = self._mk_packages(debug_arch="x86_64") + + res = hybrid.expand_packages( + nevra_to_pkg, + {}, + [], + [("pkg-3:1-2", "x86_64", [])], + filter_packages=[("pkg-debuginfo", "x86_64"), ("pkg", "src")], + ) + + self.assertEqual( + res, + { + "rpm": [{"path": "/tmp/pkg.rpm", "flags": []}], + "srpm": [], + "debuginfo": [], + }, + ) + + def test_filter_debuginfo_missing_arch(self): + nevra_to_pkg = self._mk_packages(debug_arch="x86_64") + + res = hybrid.expand_packages( + nevra_to_pkg, + {}, + [], + [("pkg-3:1-2", "x86_64", [])], + filter_packages=[("pkg-debuginfo", None)], + ) + + self.assertEqual( + res, + { + "rpm": [{"path": "/tmp/pkg.rpm", "flags": []}], + "srpm": [{"path": "/tmp/pkg.src.rpm", "flags": []}], + "debuginfo": [], + }, + ) + + def test_filter_debuginfo_different_arch(self): + nevra_to_pkg = self._mk_packages(debug_arch="x86_64") + + res = hybrid.expand_packages( + nevra_to_pkg, + {}, + [], + [("pkg-3:1-2", "x86_64", [])], + filter_packages=[("pkg-debuginfo", "aarch64")], + ) + + self.assertEqual( + res, + { + "rpm": [{"path": "/tmp/pkg.rpm", "flags": []}], + "srpm": [{"path": "/tmp/pkg.src.rpm", "flags": []}], + "debuginfo": [{"path": "/tmp/pkg-debuginfo.x86_64.rpm", "flags": []}], }, ) @@ -788,14 +851,14 @@ class TestExpandPackages(helpers.PungiTestCase): nevra_to_pkg = self._mk_packages(debug_arch="x86_64") res = hybrid.expand_packages( - nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64", ["modular"])] + nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64", ["modular"])], [] ) self.assertEqual( res, { "rpm": [{"path": "/tmp/pkg.rpm", "flags": []}], - "srpm": [{"path": "/tmp/spkg.rpm", "flags": []}], + "srpm": [{"path": "/tmp/pkg.src.rpm", "flags": []}], "debuginfo": [], }, ) @@ -804,15 +867,15 @@ class TestExpandPackages(helpers.PungiTestCase): nevra_to_pkg = self._mk_packages(debug_arch="x86_64") res = hybrid.expand_packages( - nevra_to_pkg, {}, [], [("pkg-debuginfo-3:1-2", "x86_64", ["modular"])] + nevra_to_pkg, {}, [], [("pkg-debuginfo-3:1-2", "x86_64", ["modular"])], [] ) self.assertEqual( res, { "rpm": [], - "srpm": [{"path": "/tmp/spkg.rpm", "flags": []}], - "debuginfo": [{"path": "/tmp/d1.rpm", "flags": []}], + "srpm": [{"path": "/tmp/pkg.src.rpm", "flags": []}], + "debuginfo": [{"path": "/tmp/pkg-debuginfo.x86_64.rpm", "flags": []}], }, ) @@ -820,14 +883,14 @@ class TestExpandPackages(helpers.PungiTestCase): nevra_to_pkg = self._mk_packages(debug_arch="i686") res = hybrid.expand_packages( - nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64", [])] + nevra_to_pkg, {}, [], [("pkg-3:1-2", "x86_64", [])], [] ) self.assertEqual( res, { "rpm": [{"path": "/tmp/pkg.rpm", "flags": []}], - "srpm": [{"path": "/tmp/spkg.rpm", "flags": []}], + "srpm": [{"path": "/tmp/pkg.src.rpm", "flags": []}], "debuginfo": [], }, ) @@ -841,20 +904,20 @@ class TestExpandPackages(helpers.PungiTestCase): name="pkg", arch="src", location_base="file:///tmp/", - location_href="spkg.rpm", + location_href="pkg.src.rpm", ), "def": NamedMock( name="pkg-debuginfo", arch="x86_64", location_base="file:///tmp/", - location_href="d1.rpm", + location_href="pkg-debuginfo.x86_64.rpm", ), } cr.Metadata.return_value.keys.return_value = repo.keys() cr.Metadata.return_value.get.side_effect = lambda key: repo[key] res = hybrid.expand_packages( - nevra_to_pkg, {}, lookasides, [("pkg-3:1-2", "x86_64", [])] + nevra_to_pkg, {}, lookasides, [("pkg-3:1-2", "x86_64", [])], [] ) self.assertEqual( @@ -882,7 +945,7 @@ class TestExpandPackages(helpers.PungiTestCase): cr.Metadata.return_value.get.side_effect = lambda key: repo[key] res = hybrid.expand_packages( - nevra_to_pkg, {}, lookasides, [("pkg-3:1-2", "x86_64", [])] + nevra_to_pkg, {}, lookasides, [("pkg-3:1-2", "x86_64", [])], [] ) self.assertEqual(res, {"rpm": [], "srpm": [], "debuginfo": []})