robojerk/tumbi-assembler
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Support multiple sources in one variant

Browse source 
With this patch the gather_source option is no longer used. Instead, all
sources are always used. If they return at least some input packages,
then a configured method is used and the returned lists of packages from
all sources are merged.

The method used for gathering can be configured for each variant and
gather source separately.

Additional packages are only added to the comps source.

Each gathering step is logged separately. All the logs are preserved for
later inspection.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2017-11-07 14:16:37 +01:00
parent 0074fe3f2c
commit 364d7f5229
12 changed files with 199 additions and 136 deletions
Download patch file Download diff file Expand all files Collapse all files

8
tests/data/dummy-pungi.conf
View file

@ -36,8 +36,12 @@ createrepo_checksum = "sha256"
# GATHER
gather_source = "comps"
gather_method = "deps"
gather_method = {
"^.*$": {
"module": "nodeps",
"comps": "deps",
}
}
greedy_method = "build"
check_deps = False
hashed_directories = True

1
tests/helpers.py
View file

@ -186,7 +186,6 @@ BASE_CONFIG = dict(
runroot=False,
createrepo_checksum='sha256',
gather_method='deps',
gather_source='none',
sigkeys=[],
)

25
tests/test_config.py
View file

@ -213,31 +213,6 @@ class CreaterepoConfigTestCase(ConfigTestCase):
class GatherConfigTestCase(ConfigTestCase):
def test_source_comps_requires_comps(self):
cfg = load_config(
pkgset_source='koji',
pkgset_koji_tag="f25",
gather_source='comps',
gather_source_mapping='foo'
)
self.assertValidation(
cfg,
[checks.REQUIRES.format('gather_source', 'comps', 'comps_file'),
checks.CONFLICTS.format('gather_source', 'comps', 'gather_source_mapping')])
def test_source_json_requires_mapping(self):
cfg = load_config(
pkgset_source='koji',
pkgset_koji_tag="f25",
gather_source='json',
comps_file='comps',
)
self.assertValidation(
cfg,
[checks.REQUIRES.format('gather_source', 'json', 'gather_source_mapping')])
def test_dnf_backend_is_default_on_py3(self):
cfg = load_config(
pkgset_source='koji',

51
tests/test_gather_phase.py
View file

@ -460,7 +460,7 @@ class TestGetVariantPackages(helpers.PungiTestCase):
def test_no_variant(self):
compose = helpers.DummyCompose(self.topdir, {})
packages, groups, filter_packages = gather.get_variant_packages(
compose, 'x86_64', None)
compose, 'x86_64', None, 'comps')
self.assertItemsEqual(packages, [])
self.assertItemsEqual(groups, [])
self.assertItemsEqual(filter_packages, [])
@ -473,7 +473,7 @@ class TestGetVariantPackages(helpers.PungiTestCase):
compose = helpers.DummyCompose(self.topdir, {})
packages, groups, filter_packages = gather.get_variant_packages(
compose, 'x86_64', compose.variants['Server'])
compose, 'x86_64', compose.variants['Server'], 'comps')
self.assertItemsEqual(packages, ['foo'])
self.assertItemsEqual(groups, ['core'])
self.assertItemsEqual(filter_packages, [])
@ -490,7 +490,7 @@ class TestGetVariantPackages(helpers.PungiTestCase):
)
packages, groups, filter_packages = gather.get_variant_packages(
compose, 'x86_64', compose.variants['Server'], package_sets={'x86_64': pkgset})
compose, 'x86_64', compose.variants['Server'], 'comps', package_sets={'x86_64': pkgset})
self.assertItemsEqual(packages, [('system-release-server', None)])
self.assertItemsEqual(groups, [])
self.assertItemsEqual(filter_packages, [('system-release', None)])
@ -509,7 +509,7 @@ class TestGetVariantPackages(helpers.PungiTestCase):
)
packages, groups, filter_packages = gather.get_variant_packages(
compose, 'x86_64', compose.variants['Server'], package_sets={'x86_64': pkgset})
compose, 'x86_64', compose.variants['Server'], 'comps', package_sets={'x86_64': pkgset})
self.assertItemsEqual(packages, [])
self.assertItemsEqual(groups, [])
self.assertItemsEqual(filter_packages, [])
@ -534,7 +534,7 @@ class TestGetVariantPackages(helpers.PungiTestCase):
)
packages, groups, filter_packages = gather.get_variant_packages(
compose, 'x86_64', compose.all_variants['Server-optional'])
compose, 'x86_64', compose.all_variants['Server-optional'], 'comps')
self.assertItemsEqual(packages, ['server-pkg', 'addon-pkg', 'opt-pkg'])
self.assertItemsEqual(groups, ['server-group', 'addon-group', 'opt-group'])
self.assertItemsEqual(filter_packages, [])
@ -554,7 +554,7 @@ class TestGetVariantPackages(helpers.PungiTestCase):
)
packages, groups, filter_packages = gather.get_variant_packages(
compose, 'x86_64', compose.all_variants['Server-optional'])
compose, 'x86_64', compose.all_variants['Server-optional'], 'comps')
self.assertItemsEqual(packages, [])
self.assertItemsEqual(groups, [])
self.assertItemsEqual(filter_packages, [])
@ -572,7 +572,7 @@ class TestGetVariantPackages(helpers.PungiTestCase):
)
packages, groups, filter_packages = gather.get_variant_packages(
compose, 'x86_64', compose.all_variants['Server'])
compose, 'x86_64', compose.all_variants['Server'], 'comps')
self.assertItemsEqual(packages, [('pkg', None), ('foo', 'x86_64')])
self.assertItemsEqual(groups, [])
self.assertItemsEqual(filter_packages, [])
@ -591,7 +591,7 @@ class TestGetVariantPackages(helpers.PungiTestCase):
with self.assertRaises(ValueError) as ctx:
packages, groups, filter_packages = gather.get_variant_packages(
compose, 'x86_64', compose.all_variants['Server'])
compose, 'x86_64', compose.all_variants['Server'], 'comps')
self.assertIn('Incompatible package arch', str(ctx.exception))
@ -641,17 +641,19 @@ class TestGatherPackages(helpers.PungiTestCase):
pkg_set = mock.Mock()
self.assertEqual(
gather.gather_packages(compose, 'x86_64', compose.variants['Server'], pkg_set),
get_gather_method.return_value.return_value.return_value
{'rpm': [], 'srpm': [], 'debuginfo': []}
)
self.assertEqual(get_gather_method.call_args_list,
[mock.call(compose.conf['gather_method'])])
[mock.call(compose.conf['gather_method'])] * 3)
self.assertEqual(get_variant_packages.call_args_list,
[mock.call(compose, 'x86_64', compose.variants['Server'], pkg_set)])
[mock.call(compose, 'x86_64', compose.variants['Server'], 'module', pkg_set),
mock.call(compose, 'x86_64', compose.variants['Server'], 'comps', pkg_set),
mock.call(compose, 'x86_64', compose.variants['Server'], 'json', pkg_set)])
self.assertEqual(
get_gather_method.return_value.return_value.call_args_list,
[mock.call('x86_64', compose.variants['Server'], packages, groups,
filters, set(), set(), pkg_set, fulltree_excludes=set(),
prepopulate=set())]
prepopulate=set())] * 3
)
@mock.patch('pungi.phases.gather.get_variant_packages')
@ -679,19 +681,36 @@ class TestGatherPackages(helpers.PungiTestCase):
pkg_set = mock.Mock()
self.assertEqual(
gather.gather_packages(compose, 'x86_64', compose.variants['Server'], pkg_set),
get_gather_method.return_value.return_value.return_value
{'rpm': [], 'srpm': [], 'debuginfo': []}
)
self.assertEqual(get_gather_method.call_args_list,
[mock.call(compose.conf['gather_method'])])
[mock.call(compose.conf['gather_method'])] * 3)
self.assertEqual(get_variant_packages.call_args_list,
[mock.call(compose, 'x86_64', compose.variants['Server'], pkg_set)])
[mock.call(compose, 'x86_64', compose.variants['Server'], 'module', pkg_set),
mock.call(compose, 'x86_64', compose.variants['Server'], 'comps', pkg_set),
mock.call(compose, 'x86_64', compose.variants['Server'], 'json', pkg_set)])
self.assertEqual(
get_gather_method.return_value.return_value.call_args_list,
[mock.call('x86_64', compose.variants['Server'], packages, groups,
filters, set(['white']), set(['black']), pkg_set,
fulltree_excludes=set(), prepopulate=set())]
fulltree_excludes=set(), prepopulate=set())] * 3
)
@mock.patch('pungi.phases.gather.get_variant_packages')
@mock.patch('pungi.phases.gather.get_gather_method')
def test_per_source_method(self, get_gather_method, get_variant_packages):
packages, groups, filters = mock.Mock(), mock.Mock(), mock.Mock()
get_variant_packages.return_value = (packages, groups, filters)
compose = helpers.DummyCompose(self.topdir, {
'multilib_whitelist': {'*': ['white']},
'multilib_blacklist': {'*': ['black']},
'gather_method': {'^Server$': {'comps': 'deps', 'module': 'nodeps', 'json': 'deps'}},
})
pkg_set = mock.Mock()
gather.gather_packages(compose, 'x86_64', compose.variants['Server'], pkg_set),
self.assertEqual(get_gather_method.call_args_list,
[mock.call('nodeps'), mock.call('deps'), mock.call('deps')])
class TestWritePrepopulate(helpers.PungiTestCase):
def test_without_config(self):

1
tests/test_pkgset_source_koji.py
View file

@ -204,7 +204,6 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
pickle_dumps):
self.compose = helpers.DummyCompose(self.topdir, {
'gather_method': 'nodeps',
'gather_source': 'none',
'pkgset_koji_tag': 'f25',
'sigkeys': mock.Mock(),
'additional_packages': [