robojerk/tumbi-assembler
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add common global settings for images

Browse source 
The live_images, live_media and image_build phases have same options
that need to be specified on each image. This leads to a lot of
duplication in the config file. This patch adds global settings and
phase-level settings that allow to significantly reduce duplication.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2016-04-14 14:23:42 +02:00
parent a8ff48ce92
commit 084994538b
10 changed files with 534 additions and 62 deletions
Download patch file Download diff file Expand all files Collapse all files

65
pungi/phases/base.py
View file

@ -15,6 +15,7 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
from pungi.checks import validate_options
from pungi import util
class PhaseBase(object):
@ -84,3 +85,67 @@ class ConfigGuardedPhase(PhaseBase):
self.compose.log_info("Config section '%s' was not found. Skipping." % self.name)
return True
return False
class ImageConfigMixin(object):
"""
A mixin for phase that needs to access image related settings: ksurl,
version, target and release.
First, it checks config object given as argument, then it checks
phase-level configuration and finally falls back to global configuration.
"""
def __init__(self, *args, **kwargs):
super(ImageConfigMixin, self).__init__(*args, **kwargs)
self._phase_ksurl = None
def get_config(self, cfg, opt):
return cfg.get(
opt, self.compose.conf.get(
'{}_{}'.format(self.name, opt), self.compose.conf.get(
'global_{}'.format(opt))))
def get_release(self, cfg):
"""
If release is set explicitly to None, replace it with date and respin.
Uses configuration passed as argument, phase specific settings and
global settings.
"""
for key, conf in [('release', cfg),
('{}_release'.format(self.name), self.compose.conf),
('global_release', self.compose.conf)]:
if key in conf:
return conf[key] or self.compose.image_release
return None
def get_ksurl(self, cfg):
"""
Get ksurl from `cfg`. If not present, fall back to phase defined one or
global one.
"""
if 'ksurl' in cfg:
return util.resolve_git_url(cfg['ksurl'])
if '{}_ksurl'.format(self.name) in self.compose.conf:
return self.phase_ksurl
if 'global_ksurl' in self.compose.conf:
return self.global_ksurl
return None
@property
def phase_ksurl(self):
"""Get phase level ksurl, making sure to resolve it only once."""
# The phase-level setting is cached as instance attribute of the phase.
if not self._phase_ksurl:
ksurl = self.compose.conf.get('{}_ksurl'.format(self.name))
self._phase_ksurl = util.resolve_git_url(ksurl)
return self._phase_ksurl
@property
def global_ksurl(self):
"""Get global ksurl setting, making sure to resolve it only once."""
# The global setting is cached in the configuration object.
if '_global_ksurl' not in self.compose.conf:
ksurl = self.compose.conf.get('global_ksurl')
self.compose.conf['_global_ksurl'] = util.resolve_git_url(ksurl)
return self.compose.conf['_global_ksurl']

46
pungi/phases/image_build.py
View file

@ -5,8 +5,8 @@ import os
import time
from kobo import shortcuts
from pungi.util import get_variant_data, resolve_git_url, makedirs, get_mtime, get_file_size, failable
from pungi.phases.base import ConfigGuardedPhase
from pungi.util import get_variant_data, makedirs, get_mtime, get_file_size, failable
from pungi.phases import base
from pungi.linker import Linker
from pungi.paths import translate_path
from pungi.wrappers.kojiwrapper import KojiWrapper
@ -14,10 +14,38 @@ from kobo.threads import ThreadPool, WorkerThread
from productmd.images import Image
class ImageBuildPhase(ConfigGuardedPhase):
class ImageBuildPhase(base.ImageConfigMixin, base.ConfigGuardedPhase):
"""class for wrapping up koji image-build"""
name = "image_build"
config_options = [
{
"name": "image_build",
"expected_types": [dict],
"optional": True,
},
{
"name": "image_build_ksurl",
"expected_types": [str],
"optional": True,
},
{
"name": "image_build_target",
"expected_types": [str],
"optional": True,
},
{
"name": "image_build_release",
"expected_types": [str, type(None)],
"optional": True,
},
{
"name": "image_build_version",
"expected_types": [str],
"optional": True,
},
]
def __init__(self, compose):
super(ImageBuildPhase, self).__init__(compose)
self.pool = ThreadPool(logger=self.compose._logger)
@ -92,14 +120,20 @@ class ImageBuildPhase(ConfigGuardedPhase):
continue
# Replace possible ambiguous ref name with explicit hash.
if 'ksurl' in image_conf['image-build']:
image_conf["image-build"]['ksurl'] = resolve_git_url(image_conf["image-build"]['ksurl'])
ksurl = self.get_ksurl(image_conf['image-build'])
if ksurl:
image_conf["image-build"]['ksurl'] = ksurl
image_conf["image-build"]["variant"] = variant
image_conf["image-build"]["install_tree"] = self._get_install_tree(image_conf['image-build'], variant)
self._set_release(image_conf['image-build'])
release = self.get_release(image_conf['image-build'])
if release:
image_conf['image-build']['release'] = release
image_conf['image-build']['version'] = self.get_config(image_conf['image-build'], 'version')
image_conf['image-build']['target'] = self.get_config(image_conf['image-build'], 'target')
# transform format into right 'format' for image-build
# e.g. 'docker,qcow2'

21
pungi/phases/init.py
View file

@ -150,6 +150,27 @@ class InitPhase(PhaseBase):
"optional": True,
},
# Configuration shared by all image building phases.
{
"name": "global_ksurl",
"expected_types": [str],
"optional": True,
},
{
"name": "global_target",
"expected_types": [str],
"optional": True,
},
{
"name": "global_release",
"expected_types": [str, type(None)],
"optional": True,
},
{
"name": "global_version",
"expected_types": [str],
"optional": True,
},
)

40
pungi/phases/live_images.py
View file

@ -27,8 +27,8 @@ from productmd.images import Image
from pungi.wrappers.kojiwrapper import KojiWrapper
from pungi.wrappers.iso import IsoWrapper
from pungi.phases.base import ConfigGuardedPhase
from pungi.util import get_arch_variant_data, resolve_git_url, makedirs, get_mtime, get_file_size, failable
from pungi.phases import base
from pungi.util import get_arch_variant_data, makedirs, get_mtime, get_file_size, failable
from pungi.paths import translate_path
@ -38,7 +38,7 @@ if sys.version_info[0] == 3:
return (a > b) - (a < b)
class LiveImagesPhase(ConfigGuardedPhase):
class LiveImagesPhase(base.ImageConfigMixin, base.ConfigGuardedPhase):
name = "live_images"
config_options = (
@ -71,7 +71,22 @@ class LiveImagesPhase(ConfigGuardedPhase):
"name": "live_images_no_rename",
"expected_types": [bool],
"optional": True,
}
},
{
"name": "live_images_ksurl",
"expected_types": [str],
"optional": True,
},
{
"name": "live_images_release",
"expected_types": [str, type(None)],
"optional": True,
},
{
"name": "live_images_version",
"expected_types": [str],
"optional": True,
},
)
def __init__(self, compose):
@ -102,19 +117,13 @@ class LiveImagesPhase(ConfigGuardedPhase):
repos.extend(self._get_extra_repos(arch, variant, force_list(data.get('repo_from', []))))
return repos
def _get_release(self, image_conf):
"""If release is set explicitly to None, replace it with date and respin."""
if 'release' in image_conf and image_conf['release'] is None:
return self.compose.image_release
return image_conf.get('release', None)
def run(self):
symlink_isos_to = self.compose.conf.get("symlink_isos_to", None)
commands = []
for variant in self.compose.variants.values():
for arch in variant.arches + ["src"]:
for data in get_arch_variant_data(self.compose.conf, "live_images", arch, variant):
for data in get_arch_variant_data(self.compose.conf, self.name, arch, variant):
subvariant = data.get('subvariant', variant.uid)
type = data.get('type', 'live')
@ -131,12 +140,12 @@ class LiveImagesPhase(ConfigGuardedPhase):
cmd = {
"name": data.get('name'),
"version": data.get("version", None),
"release": self._get_release(data),
"version": self.get_config(data, 'version'),
"release": self.get_release(data),
"dest_dir": dest_dir,
"build_arch": arch,
"ks_file": data['kickstart'],
"ksurl": None,
"ksurl": self.get_ksurl(data),
# Used for images wrapped in RPM
"specfile": data.get("specfile", None),
# Scratch (only taken in consideration if specfile
@ -150,9 +159,6 @@ class LiveImagesPhase(ConfigGuardedPhase):
"subvariant": subvariant,
}
if 'ksurl' in data:
cmd['ksurl'] = resolve_git_url(data['ksurl'])
cmd["repos"] = self._get_repos(arch, variant, data)
# Signing of the rpm wrapped image

41
pungi/phases/livemedia_phase.py
View file

@ -4,8 +4,8 @@ import os
import time
from kobo import shortcuts
from pungi.util import get_variant_data, resolve_git_url, makedirs, get_mtime, get_file_size, failable
from pungi.phases.base import ConfigGuardedPhase
from pungi.util import get_variant_data, makedirs, get_mtime, get_file_size, failable
from pungi.phases.base import ConfigGuardedPhase, ImageConfigMixin
from pungi.linker import Linker
from pungi.paths import translate_path
from pungi.wrappers.kojiwrapper import KojiWrapper
@ -13,7 +13,7 @@ from kobo.threads import ThreadPool, WorkerThread
from productmd.images import Image
class LiveMediaPhase(ConfigGuardedPhase):
class LiveMediaPhase(ImageConfigMixin, ConfigGuardedPhase):
"""class for wrapping up koji spin-livemedia"""
name = 'live_media'
@ -48,7 +48,6 @@ class LiveMediaPhase(ConfigGuardedPhase):
def __init__(self, compose):
super(LiveMediaPhase, self).__init__(compose)
self.pool = ThreadPool(logger=self.compose._logger)
self._global_ksurl = None
def _get_repos(self, image_conf, variant):
"""
@ -81,15 +80,6 @@ class LiveMediaPhase(ConfigGuardedPhase):
arches = set(image_conf.get('arches', [])) & arches
return sorted(arches)
def _get_release(self, image_conf):
"""If release is set explicitly to None, replace it with date and respin.
Uses both image configuration and global config.
"""
for key, conf in [('release', image_conf), ('live_media_release', self.compose.conf)]:
if key in conf and conf[key] is None:
return self.compose.image_release
return image_conf.get('release', self.compose.conf.get('live_media_release'))
def _get_install_tree(self, image_conf, variant):
if 'install_tree_from' in image_conf:
variant_uid = image_conf['install_tree_from']
@ -104,23 +94,6 @@ class LiveMediaPhase(ConfigGuardedPhase):
self.compose.paths.compose.os_tree('$basearch', variant, create_dir=False)
)
@property
def global_ksurl(self):
"""Get globally configure kickstart URL. It will only be resolved once."""
if not self._global_ksurl:
ksurl = self.compose.conf.get('live_media_ksurl')
self._global_ksurl = resolve_git_url(ksurl)
return self._global_ksurl
def _get_ksurl(self, image_conf):
"""Get ksurl from `image_conf`. If not present, fall back to global one."""
if 'ksurl' in image_conf:
return resolve_git_url(image_conf['ksurl'])
return self.global_ksurl
def _get_config(self, image_conf, opt):
return image_conf.get(opt, self.compose.conf.get('live_media_' + opt))
def run(self):
for variant in self.compose.get_variants():
arches = set([x for x in variant.arches if x != 'src'])
@ -129,20 +102,20 @@ class LiveMediaPhase(ConfigGuardedPhase):
name = image_conf.get(
'name', "%s-%s-Live" % (self.compose.ci_base.release.short, subvariant))
config = {
'target': self._get_config(image_conf, 'target'),
'target': self.get_config(image_conf, 'target'),
'arches': self._get_arches(image_conf, arches),
'ksfile': image_conf['kickstart'],
'ksurl': self._get_ksurl(image_conf),
'ksurl': self.get_ksurl(image_conf),
'ksversion': image_conf.get('ksversion'),
'scratch': image_conf.get('scratch', False),
'release': self._get_release(image_conf),
'release': self.get_release(image_conf),
'skip_tag': image_conf.get('skip_tag'),
'name': name,
'subvariant': subvariant,
'title': image_conf.get('title'),
'repo': self._get_repos(image_conf, variant),
'install_tree': self._get_install_tree(image_conf, variant),
'version': self._get_config(image_conf, 'version'),
'version': self.get_config(image_conf, 'version'),
}
self.pool.add(LiveMediaThread(self.pool))
self.pool.queue_put((self.compose, variant, config))