implement image-build tasks on the builder

This commit is contained in:
Jay Greguske 2013-08-26 17:18:51 -04:00 committed by Mike McLean
parent 752ec8da74
commit 15964c3225

View file

@ -37,6 +37,7 @@ from koji.util import parseStatus, isSuccess
import os
import pwd
import grp
import random
import re
import rpm
import shutil
@ -46,6 +47,7 @@ import socket
import sys
import time
import traceback
import xml.dom.minidom
import xmlrpclib
import zipfile
import Cheetah.Template
@ -62,8 +64,14 @@ try:
import pykickstart.parser as ksparser
import pykickstart.handlers.control as kscontrol
import pykickstart.errors as kserrors
from imgfac.BuildDispatcher import BuildDispatcher
from imgfac.PluginManager import PluginManager
from imgfac.ReservationManager import ReservationManager
import hashlib
import iso9660 # from pycdio
plugin_mgr = PluginManager('/etc/imagefactory/plugins.d')
plugin_mgr.load()
from imgfac.ApplicationConfiguration import ApplicationConfiguration
image_enabled = True
except ImportError:
pass
@ -1555,7 +1563,7 @@ class WrapperRPMTask(BaseBuildTask):
elif task['method'] == 'vmExec':
self.copy_fields(task_result, values, 'epoch', 'name', 'version', 'release')
values['win_info'] = {'platform': task_result['platform']}
elif task['method'] == 'createLiveCD' or task['method'] == 'createAppliance':
elif task['method'] in ('createLiveCD', 'createAppliance', 'createBaseImage'):
self.copy_fields(task_result, values, 'epoch', 'name', 'version', 'release')
else:
# can't happen
@ -1798,6 +1806,98 @@ class BuildImageTask(MultiPlatformTask):
"""return the next available release number for an N-V"""
return self.session.getNextRelease(dict(name=name, version=ver))
class BuildBaseImageTask(BuildImageTask):
Methods = ['baseImage']
def handler(self, name, version, arches, target, inst_tree, opts=None):
"""Governing task for building an appliance using Oz"""
target_info = self.session.getBuildTarget(target, strict=True)
build_tag = target_info['build_tag']
repo_info = self.getRepo(build_tag)
if not opts:
opts = {}
if not image_enabled:
self.logger.error("Appliance features require the following dependencies: pykickstart, imagefactory, oz and possibly python-hashlib")
raise koji.ApplianceError, 'Appliance functions not available'
# build image(s)
try:
release = opts.get('release')
if not release:
release = self.getRelease(name, version)
if '-' in version:
raise koji.ApplianceError('The Version may not have a hyphen')
if '-' in release:
raise koji.ApplianceError('The Release may not have a hyphen')
bld_info = None
if not opts.get('scratch'):
bld_info = self.initImageBuild(name, version, release,
target_info, opts)
subtasks = {}
self.logger.debug("Spawning jobs for image arches: %r" % (arches))
for arch in arches:
inst_url = inst_tree.replace('$arch', arch)
subtasks[arch] = self.session.host.subtask(
method='createBaseImage',
arglist=[name, version, release, arch, target_info,
build_tag, repo_info, inst_url, opts],
label=arch, parent=self.id, arch=arch)
self.logger.debug("Got image subtasks: %r" % (subtasks))
self.logger.debug("Waiting on image subtasks...")
results = self.wait(subtasks.values(), all=True, failany=True)
# wrap in an RPM if asked
rpm_results = None
spec_url = opts.get('specfile')
for arch in arches:
# get around an xmlrpc limitation, use arches for keys instead
results[arch] = results[subtasks[arch]]
del results[subtasks[arch]]
if spec_url:
subtask = subtasks[arch]
results[arch]['rpmresults'] = self.buildWrapperRPM(
spec_url, subtask, target_info, bld_info,
repo_info['id'])
if opts.get('scratch'):
self.session.host.moveImageBuildToScratch(self.id, results)
else:
self.session.host.completeImageBuild(self.id, bld_info['id'],
results)
except (SystemExit,ServerExit,KeyboardInterrupt):
#we do not trap these
raise
except:
if not opts.get('scratch'):
#scratch builds do not get imported
if bld_info:
self.session.host.failBuild(self.id, bld_info['id'])
# reraise the exception
raise
# tag it
if not opts.get('scratch') and not opts.get('skip_tag'):
tag_task_id = self.session.host.subtask(method='tagBuild',
arglist=[target_info['dest_tag'], bld_info['id'], False, None, True],
label='tag', parent=self.id, arch='noarch')
self.wait(tag_task_id)
# report results
report = ''
if opts.get('scratch'):
respath = ', '.join(
[os.path.join(koji.pathinfo.work(), koji.pathinfo.taskrelpath(tid)) for tid in subtasks.values()])
report += 'Scratch '
else:
respath = koji.pathinfo.imagebuild(bld_info)
report += 'image build results in: %s' % respath
return report
class BuildApplianceTask(BuildImageTask):
Methods = ['appliance']
@ -1827,7 +1927,7 @@ class BuildApplianceTask(BuildImageTask):
arglist=[name, version, release, arch, target_info, build_tag,
repo_info, ksfile, opts],
label='appliance', parent=self.id, arch=arch)
results = self.wait(create_task_id)[create_task_id]
results = self.wait(create_task_id)
self.logger.info('image build task (%s) completed' % create_task_id)
self.logger.info('results: %s' % results)
@ -1835,16 +1935,18 @@ class BuildApplianceTask(BuildImageTask):
rpm_results = None
spec_url = opts.get('specfile')
if spec_url:
rpm_results = self.buildWrapperRPM(spec_url, create_task_id,
results[create_task_id]['rpmresults'] = self.buildWrapperRPM(
spec_url, create_task_id,
target_info, bld_info, repo_info['id'])
results[str(create_task_id)] = results[create_task_id]
del results[create_task_id]
# import the image (move it too)
if not opts.get('scratch'):
self.session.host.importImage(self.id, bld_info['id'],
results, rpm_results)
else:
self.session.host.moveImageBuildToScratch(self.id, results,
rpm_results)
self.session.host.moveImageBuildToScratch(self.id, results)
except (SystemExit,ServerExit,KeyboardInterrupt):
#we do not trap these
@ -1879,7 +1981,7 @@ class BuildLiveCDTask(BuildImageTask):
Methods = ['livecd']
def handler(self, name, version, arch, target, ksfile, opts=None):
"""Governing task for building an appliance"""
"""Governing task for building LiveCDs"""
target_info = self.session.getBuildTarget(target, strict=True)
build_tag = target_info['build_tag']
repo_info = self.getRepo(build_tag)
@ -1904,7 +2006,7 @@ class BuildLiveCDTask(BuildImageTask):
arglist=[name, version, release, arch, target_info, build_tag,
repo_info, ksfile, opts],
label='livecd', parent=self.id, arch=arch)
results = self.wait(create_task_id)[create_task_id]
results = self.wait(create_task_id)
self.logger.info('image build task (%s) completed' % create_task_id)
self.logger.info('results: %s' % results)
@ -1912,16 +2014,18 @@ class BuildLiveCDTask(BuildImageTask):
spec_url = opts.get('specfile')
rpm_results = None
if spec_url:
rpm_results = self.buildWrapperRPM(spec_url, create_task_id,
results[create_task_id]['rpmresults'] = self.buildWrapperRPM(
spec_url, create_task_id,
target_info, bld_info, repo_info['id'])
results[str(create_task_id)] = results[create_task_id]
del results[create_task_id]
# import it (and move)
if not opts.get('scratch'):
self.session.host.importImage(self.id, bld_info['id'],
results, rpm_results)
else:
self.session.host.moveImageBuildToScratch(self.id, results,
rpm_results)
self.session.host.moveImageBuildToScratch(self.id, results)
except (SystemExit,ServerExit,KeyboardInterrupt):
#we do not trap these
@ -1952,14 +2056,14 @@ class BuildLiveCDTask(BuildImageTask):
os.path.join(koji.pathinfo.imagebuild(bld_info),
results['files'][0])
# A generic task for building cd or disk images. Other handlers should inherit
# this.
# A generic task for building cd or disk images using chroot-based tools.
# Other chroot-based image handlers should inherit this.
class ImageTask(BaseTaskHandler):
Methods = []
def makeImgBuildRoot(self, buildtag, repoinfo, arch, inst_group):
"""
Create and prepare the chroot we're going to build an image in.
Create and prepare the chroot we're going to build an image in.
Binds necessary directories and creates needed device files.
@args:
@ -2366,6 +2470,396 @@ class LiveCDTask(ImageTask):
broot.expire()
return imgdata
# A generic task for building disk images using Oz
# Other Oz-based image handlers should inherit this.
class OzImageTask(BaseTaskHandler):
Methods = []
def fetchKickstart(self):
"""
Retrieve the kickstart file we were given (locally or remotely) and
upload it.
Note that if the KS file existed locally, then "ksfile" is a relative
path to it in the /mnt/koji/work directory. If not, then it is still
the parameter the user passed in initially, and we assume it is a
relative path in a remote scm. The user should have passed in an scm
url with --ksurl.
@args: None, use self.opts for options
@returns: absolute path to the retrieved kickstart file
"""
ksfile = self.opts.get('kickstart')
self.logger.debug("ksfile = %s" % ksfile)
if self.opts.get('ksurl'):
scm = SCM(self.opts['ksurl'])
scm.assert_allowed(self.options.allowed_scms)
logfile = os.path.join(self.workdir, 'checkout.log')
scmsrcdir = scm.checkout(self.workdir, self.session,
self.getUploadDir(), logfile)
kspath = os.path.join(scmsrcdir, os.path.basename(ksfile))
else:
tops = dict([(k, getattr(self.options, k)) for k in 'topurl','topdir'])
ks_src = koji.openRemoteFile(ksfile, **tops)
kspath = os.path.join(self.workdir, os.path.basename(ksfile))
ks_dest = open(kspath, 'w')
ks_dest.write(ks_src.read())
ks_dest.close()
self.logger.debug('uploading kickstart from here: %s' % kspath)
self.uploadFile(kspath) # upload the original ks file
return kspath # absolute path to the ks file
def readKickstart(self, kspath):
"""
Read a kickstart file and save the ks object as a task member.
@args:
kspath: path to a kickstart file
@returns: None
"""
# XXX: If the ks file came from a local path and has %include
# macros, *-creator will fail because the included
# kickstarts were not copied into the chroot. For now we
# require users to flatten their kickstart file if submitting
# the task with a local path.
#
# Note that if an SCM URL was used instead, %include macros
# may not be a problem if the included kickstarts are present
# in the repository we checked out.
if self.opts.get('ksversion'):
version = ksparser.makeVersion(
ksparser.stringToVersion(self.opts['ksversion']))
else:
version = ksparser.makeVersion()
self.ks = ksparser.KickstartParser(version)
self.logger.debug('attempting to read kickstart: %s' % kspath)
try:
self.ks.readKickstart(kspath)
except IOError, e:
raise koji.BuildError("Failed to read kickstart file "
"'%s' : %s" % (kspath, e))
except kserrors.KickstartError, e:
raise koji.BuildError("Failed to parse kickstart file "
"'%s' : %s" % (kspath, e))
def prepareKickstart(self, repo_info, target_info, arch):
"""
Process the ks file to be used for controlled image generation. This
method also uploads the modified kickstart file to the task output
area.
@args:
target_info: a sesion.getBuildTarget() object
repo_info: session.getRepo() object
arch: canonical architecture name
@returns:
absolute path to a processed kickstart file
"""
# Now we do some kickstart manipulation. If the user passed in a repo
# url with --repo, then we substitute that in for the repo(s) specified
# in the kickstart file. If --repo wasn't specified, then we use the
# repo associated with the target passed in initially.
self.ks.handler.repo.repoList = [] # delete whatever the ks file told us
repo_class = kscontrol.dataMap[self.ks.version]['RepoData']
if self.opts.get('repo'):
# the user used --repo at least once
user_repos = self.opts.get('repo')
index = 0
for user_repo in user_repos:
repo_url = user_repo.replace('$arch', arch)
self.ks.handler.repo.repoList.append(repo_class(
baseurl=repo_url, name='koji-override-%i' % index))
index += 1
else:
# --repo was not given, so we use the target's build repo
path_info = koji.PathInfo(topdir=self.options.topurl)
repopath = path_info.repo(repo_info['id'],
target_info['build_tag_name'])
baseurl = '%s/%s' % (repopath, arch)
self.logger.debug('BASEURL: %s' % baseurl)
self.ks.handler.repo.repoList.append(repo_class(
baseurl=baseurl, name='koji-override-0'))
# Write out the new ks file. Note that things may not be in the same
# order and comments in the original ks file may be lost.
kskoji = os.path.join(self.workdir, 'koji-image-%s-%i.ks' %
(target_info['build_tag_name'], self.id))
self.logger.debug('modified ks file: %s' % kskoji)
outfile = open(kskoji, 'w')
outfile.write(str(self.ks.handler))
outfile.close()
# put the new ksfile in the output directory
if not os.path.exists(kskoji):
raise koji.BuildError, "KS file missing: %s" % kskoji
self.uploadFile(kskoji) # upload the modified ks file
return kskoji
def makeConfig(self):
"""
Generate a configuration dict for ImageFactory. This will override
anything in the /etc config files.
"""
return {
#Oz specific
'oz_data_dir': os.path.join(self.workdir, 'oz_data'),
'oz_screenshot_dir': os.path.join(self.workdir, 'oz_screenshots'),
#IF specific
'imgdir': os.path.join(self.workdir, 'scratch_images'),
'tmpdir': os.path.join(self.workdir, 'oz-tmp'),
'verbose' : True,
'timeout': 3600,
'output': 'log',
'raw': False,
'debug': True,
'image_manager': 'file',
'plugins': '/etc/imagefactory/plugins.d',
'tdl_require_root_pw': False,
'image_manager_args': {
'storage_path': os.path.join(self.workdir, 'output_image')},
}
def makeTemplate(self, imgname, arch, inst_tree):
"""
Generate a simple template for ImageFactory
"""
# we have to split this up so the variable substitution works
distname, distver = self.parseDistro(self.opts.get('distro'))
template = """<template>
<name>%s</name>
<os>
<name>%s</name>
<version>%s</version>
<arch>%s</arch>
<install type='url'>
<url>%s</url>
</install>
""" % (imgname, distname, distver, arch, inst_tree)
template += """<icicle>
<extra_command>rpm -qa --qf '%{NAME},%{VERSION},%{RELEASE},%{ARCH},%{EPOCH},%{SIZE},%{SIGMD5},%{BUILDTIME}\n'</extra_command>
</icicle>
"""
template += """</os>
<description>%s OS</description>
</template>
""" % imgname
return template
def parseDistro(self, distro):
"""
Figure out the distribution name and version we are going to build an
image on.
"""
if distro.startswith('RHEL'):
return distro.split('.')
elif distro.startswith('Fedora'):
return distro.split('-')
else:
raise BuildError('Unknown or supported distro given: %s' % distro)
def fixImageXML(self, format, imgname, filename, xmltext):
"""
The XML generated by Oz/ImageFactory knows nothing about the name
or image format conversions Koji does. We fix those values in the
libvirt XML and write the changes out to a file, the path of which
we return.
@args:
format = raw, qcow2, vmdk, etc... a string representation
name = the (file) name of the image
xmltext = the libvirt XML to start with
@return:
an absolute path to the modified XML
"""
newxml = xml.dom.minidom.parseString(xmltext)
ename = newxml.getElementsByTagName('name')[0]
ename.firstChild.nodeValue = imgname
esources = newxml.getElementsByTagName('source')
for e in esources:
if e.hasAttribute('file'):
e.setAttribute('file', '%s.%s' % (imgname, format))
edriver = newxml.getElementsByTagName('driver')[0]
edriver.setAttribute('type', format)
xml_path = os.path.join(self.workdir, filename)
xmlfd = open(xml_path, 'w')
xmlfd.write(newxml.toprettyxml())
xmlfd.close()
return xml_path
def getScreenshot(self):
"""
Locate a screenshot taken by libvirt in the case of build failure,
if it exists. If it does, return the path, else return None.
"""
shotdir = os.path.join(self.workdir, 'oz_screenshots')
screenshot = None
found = glob.glob(os.path.join(shotdir, '*.ppm'))
if len(found) > 0:
screenshot = found[0]
found = glob.glob(os.path.join(shotdir, '*.png'))
if len(found) > 0:
screenshot = found[0]
return screenshot
class BaseImageTask(OzImageTask):
Methods = ['createBaseImage']
_taskWeight = 1.5
def getRootDevice(self):
"""
Return the device name for the / partition, as specified in the
kickstart file. Appliances should have this defined.
"""
for part in self.ks.handler.partition.partitions:
if part.mountpoint == '/':
return part.disk
raise koji.ApplianceError, 'kickstart lacks a "/" mountpoint'
def handler(self, name, version, release, arch, target_info, build_tag, repo_info, inst_tree, opts=None):
if opts == None:
opts = {}
self.opts = opts
formats = opts.get('format')
for f in formats:
if f not in ('vmdk', 'qcow', 'qcow2', 'vdi', 'raw'):
raise koji.ApplianceError('Invalid format: %s' % f)
# First, prepare the kickstart to use the repos we tell it
kspath = self.fetchKickstart()
self.readKickstart(kspath)
kskoji = self.prepareKickstart(repo_info, target_info, arch)
# auto-generate a TDL file and config dict for ImageFactory
imgname = '%s-%s-%s.%s' % (name, version, release, arch)
template = self.makeTemplate(imgname, arch, inst_tree)
self.logger.debug('oz template: %s' % template)
config = self.makeConfig()
self.logger.debug('IF config object: %s' % config)
ApplicationConfiguration(configuration=config)
# ImageFactory picks a port to the guest VM using a rolling integer.
# This is a problem for concurrency, so we override the port it picks
# here using the task ID. (not a perfect solution but good enough:
# the likelihood of image tasks clashing here is very small)
rm = ReservationManager()
rm._listen_port = rm.MIN_PORT + self.id % (rm.MAX_PORT - rm.MIN_PORT)
# invoke ImageFactory, capture its logging
ozlogname = 'oz-%s.log' % arch
ozlog = os.path.join(self.workdir, ozlogname)
fhandler = logging.FileHandler(ozlog)
bd = BuildDispatcher()
tlog = logging.getLogger()
tlog.setLevel(logging.DEBUG)
tlog.addHandler(fhandler)
params = {'install_script': str(self.ks.handler)} #ks contents
random.seed() # necessary to ensure a unique mac address
try:
ozif = bd.builder_for_base_image(template, parameters=params)
ozif.base_thread.join()
finally:
# upload log even if we failed to help diagnose an issue
tlog.removeHandler(fhandler)
self.uploadFile(ozlog)
if ozif.base_image.status == 'FAILED':
scrnshot = self.getScreenshot()
if scrnshot:
ext = scrnshot[-3:]
self.uploadFile(scrnshot, remoteName='screenshot.%s' % ext)
ozif.os_plugin.abort() # forcibly tear down the VM
# TODO do this when a task is CANCELLED
raise koji.ApplianceError('Image status is %s: %s' %
(ozif.base_image.status, ozif.base_image.status_detail))
# structure the results to pass back to the hub:
imgdata = {
'arch': arch,
'task_id': self.id,
'logs': [ozlogname, os.path.basename(kspath),
os.path.basename(kskoji)],
'name': name,
'version': version,
'release': release,
'rpmlist': [],
'files': []
}
# record the RPMs that were installed
if not opts.get('scratch'):
fields = ('name', 'version', 'release', 'arch', 'epoch', 'size',
'payloadhash', 'buildtime')
icicle = xml.dom.minidom.parseString(ozif.base_image.icicle)
self.logger.debug('ICICLE: %s' % ozif.base_image.icicle)
for p in icicle.getElementsByTagName('extra'):
bits = p.firstChild.nodeValue.split(',')
rpm = {
'name': bits[0],
'version': bits[1],
'release': bits[2],
'arch': bits[3],
# epoch is a special case, as usual
'size': int(bits[5]),
'payloadhash': bits[6],
'buildtime': int(bits[7])
}
if bits[4] == '(none)':
rpm['epoch'] = None
else:
rpm['epoch'] = int(bits[4])
imgdata['rpmlist'].append(rpm)
# TODO: hack to make this work for now, need to refactor
br = BuildRoot(self.session, self.options, build_tag, arch,
self.id, repo_id=repo_info['id'])
br.markExternalRPMs(imgdata['rpmlist'])
# If a user requests 1 or more image formats (with --format) we do not
# by default include the raw disk image in the results, because it is
# 10G in size. To override this behavior, the user must specify
# "--format raw" in their command. If --format was not used at all,
# then we do include the raw disk image by itself.
if len(formats) == 0:
# we only want a raw disk image (no format option given)
formats.append('raw')
for f in formats:
if f == 'raw':
newimg = os.path.join(self.workdir, imgname + '.raw')
imgdata['files'].append(os.path.basename(newimg))
self.uploadFile(ozif.base_image.data,
remoteName=os.path.basename(newimg))
lxml = self.fixImageXML(f, imgname,
'libvirt-%s-%s.xml' % (f, arch),
ozif.base_image.parameters['libvirt_xml'])
self.uploadFile(lxml)
imgdata['files'].append(os.path.basename(lxml))
else:
# transform the image to the desired format(s)
newimg = os.path.join(self.workdir, imgname + '.%s' % f)
cmd = ['/usr/bin/qemu-img', 'convert', '-f', 'raw', '-O',
f, ozif.base_image.data, newimg]
if f in ('qcow', 'qcow2'):
cmd.insert(2, '-c') # enable compression for qcow images
log_output(self.session, cmd[0], cmd,
os.path.join(self.workdir, 'qemu-img-%s-%s.log' % (f, arch)),
self.getUploadDir(), logerror=1)
imgdata['files'].append(os.path.basename(newimg))
self.uploadFile(newimg)
lxml = self.fixImageXML(f, imgname,
'libvirt-%s-%s.xml' % (f, arch),
ozif.base_image.parameters['libvirt_xml'])
self.uploadFile(lxml)
imgdata['files'].append(os.path.basename(lxml))
tdl_path = os.path.join(self.workdir, 'tdl-%s.xml' % arch)
tdl = open(tdl_path, 'w')
tdl.write(ozif.base_image.template)
tdl.close()
self.uploadFile(tdl_path)
imgdata['files'].append(os.path.basename(tdl_path))
# no need to delete anything since self.workdir will get scrubbed
return imgdata
class BuildSRPMFromSCMTask(BaseBuildTask):
Methods = ['buildSRPMFromSCM']