diff --git a/pungi/phases/gather/methods/method_deps.py b/pungi/phases/gather/methods/method_deps.py index 7c9e8fb6..e144415f 100644 --- a/pungi/phases/gather/methods/method_deps.py +++ b/pungi/phases/gather/methods/method_deps.py @@ -18,6 +18,7 @@ import os from kobo.shortcuts import run from kobo.pkgset import SimpleRpmWrapper, RpmWrapper +from kobo.rpmlib import parse_nvra from pungi.util import rmtree, get_arch_variant_data from pungi.wrappers.pungi import PungiWrapper @@ -43,10 +44,28 @@ class GatherMethodDeps(pungi.phases.gather.method.GatherMethodBase): fulltree_excludes=fulltree_excludes, prepopulate=prepopulate, source_name=self.source_name) result, missing_deps = resolve_deps(self.compose, arch, variant, source_name=self.source_name) + raise_on_invalid_sigkeys(arch, variant, package_sets, result) check_deps(self.compose, arch, variant, missing_deps) return result +def raise_on_invalid_sigkeys(arch, variant, package_sets, result): + """ + Raises RuntimeError if some package in compose is signed with an invalid + sigkey. + """ + invalid_sigkeys_rpms = [] + for package in result["rpm"]: + name = parse_nvra(package["path"])["name"] + for forbidden_package in package_sets["global"].invalid_sigkeys_rpms(): + if name == forbidden_package["name"]: + invalid_sigkeys_rpms.append(forbidden_package) + + if invalid_sigkeys_rpms: + package_sets["global"].raise_invalid_sigkeys_exception( + invalid_sigkeys_rpms) + + def _format_packages(pkgs): """Sort packages and merge name with arch.""" for pkg, pkg_arch in sorted(pkgs): diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py index 4d902f4f..8ede7815 100644 --- a/pungi/phases/pkgset/pkgsets.py +++ b/pungi/phases/pkgset/pkgsets.py @@ -67,7 +67,8 @@ class ReaderThread(WorkerThread): class PackageSetBase(kobo.log.LoggingBase): - def __init__(self, sigkey_ordering, arches=None, logger=None): + def __init__(self, sigkey_ordering, arches=None, logger=None, + allow_invalid_sigkeys=False): super(PackageSetBase, self).__init__(logger=logger) self.file_cache = kobo.pkgset.FileCache(kobo.pkgset.SimpleRpmWrapper) self.sigkey_ordering = sigkey_ordering or [None] @@ -76,6 +77,10 @@ class PackageSetBase(kobo.log.LoggingBase): self.srpms_by_name = {} # RPMs not found for specified sigkeys self._invalid_sigkey_rpms = [] + self._allow_invalid_sigkeys = allow_invalid_sigkeys + + def invalid_sigkeys_rpms(self): + return self._invalid_sigkey_rpms def __getitem__(self, name): return self.file_cache[name] @@ -96,6 +101,19 @@ class PackageSetBase(kobo.log.LoggingBase): self._logger = None self.__dict__.update(data) + def raise_invalid_sigkeys_exception(self, rpminfos): + """ + Raises RuntimeError containing details of RPMs with invalid + sigkeys defined in `rpminfos`. + """ + def nvr_formatter(package_info): + # joins NVR parts of the package with '-' character. + return '-'.join((package_info['name'], package_info['version'], package_info['release'])) + raise RuntimeError( + "RPM(s) not found for sigs: %s. Check log for details. Unsigned packages:\n%s" % ( + self.sigkey_ordering, + '\n'.join(sorted(set([nvr_formatter(rpminfo) for rpminfo in rpminfos]))))) + def read_packages(self, rpms, srpms): srpm_pool = ReaderPool(self, self._logger) rpm_pool = ReaderPool(self, self._logger) @@ -123,14 +141,8 @@ class PackageSetBase(kobo.log.LoggingBase): rpm_pool.stop() self.log_debug("Package set: worker threads stopped (RPMs)") - if self._invalid_sigkey_rpms: - def nvr_formatter(package_info): - # joins NVR parts of the package with '-' character. - return '-'.join((package_info['name'], package_info['version'], package_info['release'])) - raise RuntimeError( - "RPM(s) not found for sigs: %s. Check log for details. Unsigned packages:\n%s" % ( - self.sigkey_ordering, - '\n'.join(sorted(set([nvr_formatter(rpminfo) for rpminfo in self._invalid_sigkey_rpms]))))) + if not self._allow_invalid_sigkeys and self._invalid_sigkey_rpms: + self.raise_invalid_sigkeys_exception(self._invalid_sigkey_rpms) return self.rpms_by_arch @@ -222,9 +234,10 @@ class FilelistPackageSet(PackageSetBase): class KojiPackageSet(PackageSetBase): def __init__(self, koji_wrapper, sigkey_ordering, arches=None, logger=None, - packages=None): + packages=None, allow_invalid_sigkeys=False): super(KojiPackageSet, self).__init__(sigkey_ordering=sigkey_ordering, - arches=arches, logger=logger) + arches=arches, logger=logger, + allow_invalid_sigkeys=allow_invalid_sigkeys) self.koji_wrapper = koji_wrapper # Names of packages to look for in the Koji tag. self.packages = set(packages or []) @@ -270,6 +283,14 @@ class KojiPackageSet(PackageSetBase): if os.path.isfile(rpm_path): return rpm_path + if self._allow_invalid_sigkeys: + # use an unsigned copy (if allowed) + rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.rpm(rpm_info)) + paths.append(rpm_path) + if os.path.isfile(rpm_path): + self._invalid_sigkey_rpms.append(rpm_info) + return rpm_path + self._invalid_sigkey_rpms.append(rpm_info) self.log_error("RPM %s not found for sigs: %s. Paths checked: %s" % (rpm_info, self.sigkey_ordering, paths)) diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 98eed625..9cc0bc34 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -214,11 +214,23 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): for group in groups: packages_to_gather += comps.get_packages(group) + # In case we use "deps" gather_method, there might be some packages in + # the Koji tag which are not signed with proper sigkey. However, these + # packages might never end up in a compose depending on which packages + # from the Koji tag are requested how the deps are resolved in the end. + # In this case, we allow even packages with invalid sigkeys to be returned + # by PKGSET phase and later, the gather phase checks its results and if + # there are some packages with invalid sigkeys, it raises an exception. + if compose.conf["gather_method"] == "deps": + allow_invalid_sigkeys = True + else: + allow_invalid_sigkeys = False + session = get_pdc_client_session(compose) for variant in compose.all_variants.values(): variant.pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet( koji_wrapper, compose.conf["sigkeys"], logger=compose._logger, - arches=all_arches) + arches=all_arches, allow_invalid_sigkeys=allow_invalid_sigkeys) variant_tags[variant] = [] pdc_module_file = os.path.join(compose.paths.work.topdir(arch="global"), "pdc-module-%s.json" % variant.uid) @@ -285,7 +297,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): else: global_pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet( koji_wrapper, compose.conf["sigkeys"], logger=compose._logger, - arches=all_arches) + arches=all_arches, allow_invalid_sigkeys=allow_invalid_sigkeys) # Get package set for each compose tag and merge it to global package # list. Also prepare per-variant pkgset, because we do not have list # of binary RPMs in module definition - there is just list of SRPMs. @@ -294,7 +306,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): "'%s'" % compose_tag) pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet( koji_wrapper, compose.conf["sigkeys"], logger=compose._logger, - arches=all_arches, packages=packages_to_gather) + arches=all_arches, packages=packages_to_gather, + allow_invalid_sigkeys=allow_invalid_sigkeys) # Create a filename for log with package-to-tag mapping. The tag # name is included in filename, so any slashes in it are replaced # with underscores just to be safe. diff --git a/tests/test_pkgset_pkgsets.py b/tests/test_pkgset_pkgsets.py index 2c5d00e0..4479737c 100644 --- a/tests/test_pkgset_pkgsets.py +++ b/tests/test_pkgset_pkgsets.py @@ -265,6 +265,24 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase): re.DOTALL) self.assertRegexpMatches(str(ctx.exception), figure) + def test_can_not_find_signed_package_allow_invalid_sigkeys(self): + pkgset = pkgsets.KojiPackageSet(self.koji_wrapper, ['cafebabe'], arches=['x86_64'], + allow_invalid_sigkeys=True) + + pkgset.populate('f25') + + self.assertEqual( + self.koji_wrapper.koji_proxy.mock_calls, + [mock.call.listTaggedRPMS('f25', event=None, inherit=True, latest=True)]) + + with self.assertRaises(RuntimeError) as ctx: + pkgset.raise_invalid_sigkeys_exception(pkgset.invalid_sigkeys_rpms()) + + figure = re.compile( + r'^RPM\(s\) not found for sigs: .+Check log for details.+bash-4\.3\.42-4\.fc24.+bash-debuginfo-4\.3\.42-4\.fc24$', + re.DOTALL) + self.assertRegexpMatches(str(ctx.exception), figure) + def test_can_not_find_any_package(self): pkgset = pkgsets.KojiPackageSet(self.koji_wrapper, ['cafebabe', None], arches=['x86_64'])