particle-os/debian-koji
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
debian-koji/builder/kojid
Mike Bonnet ab53d2deb0 update copyright
2014-10-28 23:54:21 -04:00

4615 lines
194 KiB
Python
Executable file
Raw Blame History

#!/usr/bin/python
# Koji build daemon
# Copyright (c) 2005-2014 Red Hat, Inc.
#
# Koji is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation;
# version 2.1 of the License.
#
# This software is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this software; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#
# Authors:
# Mike McLean <mikem@redhat.com>
# Mike Bonnet <mikeb@redhat.com>
try:
import krbV
except ImportError:
pass
import koji
import koji.plugin
import koji.util
import koji.tasks
import glob
import logging
import logging.handlers
from koji.daemon import incremental_upload, log_output, TaskManager, SCM
from koji.tasks import ServerExit, ServerRestart, BaseTaskHandler, MultiPlatformTask
from koji.util import parseStatus, isSuccess, dslice, dslice_ex
import os
import pwd
import grp
import random
import re
import rpm
import shutil
import signal
import smtplib
import socket
import sys
import time
import traceback
import xml.dom.minidom
import xmlrpclib
import zipfile
import copy
import Cheetah.Template
from ConfigParser import ConfigParser
from fnmatch import fnmatch
from gzip import GzipFile
from optparse import OptionParser, SUPPRESS_HELP
from StringIO import StringIO
from yum import repoMDObject
#imports for LiveCD and Appliance handler
image_enabled = False
try:
import pykickstart.parser as ksparser
import pykickstart.handlers.control as kscontrol
import pykickstart.errors as kserrors
import hashlib
import iso9660 # from pycdio
image_enabled = True
except ImportError:
pass
ozif_enabled = False
try:
from imgfac.BuildDispatcher import BuildDispatcher
from imgfac.Builder import Builder
from imgfac.PluginManager import PluginManager
from imgfac.ReservationManager import ReservationManager
plugin_mgr = PluginManager('/etc/imagefactory/plugins.d')
plugin_mgr.load()
from imgfac.ApplicationConfiguration import ApplicationConfiguration
from imgfac.PersistentImageManager import PersistentImageManager
from imgfac.BaseImage import BaseImage
from imgfac.TargetImage import TargetImage
ozif_enabled = True
except ImportError:
pass
def main(options, session):
logger = logging.getLogger("koji.build")
logger.info('Starting up')
koji.util.setup_rlimits(options.__dict__, logger)
tm = TaskManager(options, session)
tm.findHandlers(globals())
tm.findHandlers(vars(koji.tasks))
if options.plugin:
#load plugins
pt = koji.plugin.PluginTracker(path=options.pluginpath.split(':'))
for name in options.plugin:
logger.info('Loading plugin: %s' % name)
tm.scanPlugin(pt.load(name))
def shutdown(*args):
raise SystemExit
def restart(*args):
logger.warn("Initiating graceful restart")
tm.restart_pending = True
signal.signal(signal.SIGTERM,shutdown)
signal.signal(signal.SIGUSR1,restart)
while 1:
try:
taken = False
tm.updateBuildroots()
tm.updateTasks()
taken = tm.getNextTask()
except (SystemExit,ServerExit,KeyboardInterrupt):
logger.warn("Exiting")
break
except ServerRestart:
logger.warn("Restarting")
os.execv(sys.argv[0], sys.argv)
except koji.AuthExpired:
logger.error('Session expired')
break
except koji.RetryError:
raise
except:
# XXX - this is a little extreme
# log the exception and continue
logger.error(''.join(traceback.format_exception(*sys.exc_info())))
try:
if not taken:
# Only sleep if we didn't take a task, otherwise retry immediately.
# The load-balancing code in getNextTask() will prevent a single builder
# from getting overloaded.
time.sleep(options.sleeptime)
except (SystemExit,KeyboardInterrupt):
logger.warn("Exiting")
break
logger.warn("Shutting down, please wait...")
tm.shutdown()
session.logout()
sys.exit(0)
class BuildRoot(object):
def __init__(self,session,options,*args,**kwargs):
self.logger = logging.getLogger("koji.build.buildroot")
self.session = session
self.options = options
if len(args) + len(kwargs) == 1:
# manage an existing mock buildroot
self._load(*args,**kwargs)
else:
self._new(*args,**kwargs)
def _load(self, data):
#manage an existing buildroot
if isinstance(data, dict):
#assume data already pulled from db
self.id = data['id']
else:
self.id = data
data = self.session.getBuildroot(self.id)
self.task_id = data['task_id']
self.tag_id = data['tag_id']
self.tag_name = data['tag_name']
self.repoid = data['repo_id']
self.repo_info = self.session.repoInfo(self.repoid, strict=True)
self.event_id = self.repo_info['create_event']
self.br_arch = data['arch']
self.name = "%(tag_name)s-%(id)s-%(repoid)s" % vars(self)
self.config = self.session.getBuildConfig(self.tag_id, event=self.event_id)
def _new(self, tag, arch, task_id, repo_id=None, install_group='build',
setup_dns=False, bind_opts=None, maven_opts=None, maven_envs=None, deps=None):
"""Create a brand new repo"""
if not repo_id:
raise koji.BuildrootError, "A repo id must be provided"
repo_info = self.session.repoInfo(repo_id, strict=True)
self.repo_info = repo_info
self.repoid = self.repo_info['id']
self.event_id = self.repo_info['create_event']
self.task_id = task_id
self.config = self.session.getBuildConfig(tag, event=self.event_id)
if not self.config:
raise koji.BuildrootError("Could not get config info for tag: %s" % tag)
self.tag_id = self.config['id']
self.tag_name = self.config['name']
if self.config['id'] != repo_info['tag_id']:
raise koji.BuildrootError, "tag/repo mismatch: %s vs %s" \
% (self.config['name'], repo_info['tag_name'])
repo_state = koji.REPO_STATES[repo_info['state']]
if repo_state == 'EXPIRED':
# This should be ok. Expired repos are still intact, just not
# up-to-date (which may be the point in some cases).
self.logger.info("Requested repo (%i) is no longer current" % repo_id)
elif repo_state != 'READY':
raise koji.BuildrootError, "Requested repo (%i) is %s" % (repo_id, repo_state)
self.br_arch = koji.canonArch(arch)
# armhfp is not a valid arch according to autoconf
if arch == 'armhfp':
self.target_arch = 'arm'
else:
self.target_arch = arch
self.logger.debug("New buildroot: %(tag_name)s/%(br_arch)s/%(repoid)s" % vars(self))
id = self.session.host.newBuildRoot(self.repoid, self.br_arch, task_id=task_id)
if id is None:
raise koji.BuildrootError, "failed to get a buildroot id"
self.id = id
self.name = "%(tag_name)s-%(id)s-%(repoid)s" % vars(self)
self.install_group = install_group
self.setup_dns = setup_dns
self.bind_opts = bind_opts
self.maven_opts = maven_opts
self.maven_envs = maven_envs
self.deps = deps
self._writeMockConfig()
def _writeMockConfig(self):
# mock config
configdir = '/etc/mock/koji'
configfile = "%s/%s.cfg" % (configdir,self.name)
self.mockcfg = "koji/%s" % self.name
opts = {}
for k in ('repoid', 'tag_name'):
if hasattr(self, k):
opts[k] = getattr(self, k)
for k in ('mockdir', 'topdir', 'topurl', 'topurls', 'packager', 'vendor', 'distribution', 'mockhost', 'yum_proxy', 'rpmbuild_timeout'):
if hasattr(self.options, k):
opts[k] = getattr(self.options, k)
opts['buildroot_id'] = self.id
opts['use_host_resolv'] = self.setup_dns
opts['install_group'] = self.install_group
opts['maven_opts'] = self.maven_opts
opts['maven_envs'] = self.maven_envs
opts['bind_opts'] = self.bind_opts
opts['target_arch'] = self.target_arch
output = koji.genMockConfig(self.name, self.br_arch, managed=True, **opts)
#write config
fo = file(configfile,'w')
fo.write(output)
fo.close()
def _repositoryEntries(self, pi, plugin=False):
entries = []
if plugin:
tag_name = 'pluginRepository'
id_suffix = 'plugin-repo'
name_prefix = 'Plugin repository for Koji'
else:
tag_name = 'repository'
id_suffix = 'repo'
name_prefix = 'Repository for Koji'
for dep in self.deps:
if isinstance(dep, (int, long)):
# dep is a task ID, the url points to the task output directory
repo_type = 'task'
dep_url = pi.task(dep)
snapshots = 'true'
else:
# dep is a build NVR, the url points to the build output directory
repo_type = 'build'
build = koji.parse_NVR(dep)
dep_url = pi.mavenbuild(build)
snapshots = 'false'
repo_id = 'koji-%(repo_type)s-%(dep)s-%(id_suffix)s' % locals()
entry = """
<%(tag_name)s>
<id>%(repo_id)s</id>
<name>%(name_prefix)s %(repo_type)s %(dep)s</name>
<url>%(dep_url)s</url>
<layout>default</layout>
<releases>
<enabled>true</enabled>
<updatePolicy>never</updatePolicy>
<checksumPolicy>fail</checksumPolicy>
</releases>
<snapshots>
<enabled>%(snapshots)s</enabled>
<updatePolicy>never</updatePolicy>
<checksumPolicy>fail</checksumPolicy>
</snapshots>
</%(tag_name)s>""" % locals()
entries.append((repo_id, entry))
return entries
def writeMavenSettings(self, destfile, outputdir):
"""
Write the Maven settings.xml file to the specified destination.
"""
task_id = self.task_id
repo_id = self.repoid
tag_name = self.tag_name
deploy_dir = outputdir[len(self.rootdir()):]
pi = koji.PathInfo(topdir=self.options.topurl)
repourl = pi.repo(repo_id, tag_name) + '/maven'
mirror_spec = '*'
settings = """<settings xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/settings-1.0.0.xsd">
<interactiveMode>false</interactiveMode>
<mirrors>
<mirror>
<id>koji-maven-repo-%(tag_name)s-%(repo_id)i</id>
<name>Koji-managed Maven repository (%(tag_name)s-%(repo_id)i)</name>
<url>%(repourl)s</url>
<mirrorOf>%(mirror_spec)s</mirrorOf>
</mirror>
</mirrors>
<profiles>
<profile>
<id>koji-task-%(task_id)s</id>
<properties>
<altDeploymentRepository>koji-output::default::file://%(deploy_dir)s</altDeploymentRepository>
</properties>"""
if self.deps:
settings += """
<repositories>"""
for dep_repo_id, dep_repo_entry in self._repositoryEntries(pi):
mirror_spec += ',!' + dep_repo_id
settings += dep_repo_entry
settings += """
</repositories>
<pluginRepositories>"""
for dep_repo_id, dep_repo_entry in self._repositoryEntries(pi, plugin=True):
mirror_spec += ',!' + dep_repo_id
settings += dep_repo_entry
settings += """
</pluginRepositories>"""
settings += """
</profile>
</profiles>
<activeProfiles>
<activeProfile>koji-task-%(task_id)s</activeProfile>
</activeProfiles>
</settings>
"""
settings = settings % locals()
fo = file(self.rootdir() + destfile, 'w')
fo.write(settings)
fo.close()
def mock(self, args):
"""Run mock"""
mockpath = getattr(self.options,"mockpath","/usr/bin/mock")
cmd = [mockpath, "-r", self.mockcfg]
#if self.options.debug_mock:
# cmd.append('--debug')
# TODO: should we pass something like --verbose --trace instead?
cmd.extend(args)
self.logger.info(' '.join(cmd))
workdir = getattr(self, 'workdir', None)
mocklog = 'mock_output.log'
pid = os.fork()
if pid:
resultdir = self.resultdir()
uploadpath = self.getUploadPath()
logs = {}
finished = False
while not finished:
time.sleep(1)
status = os.waitpid(pid, os.WNOHANG)
if status[0] != 0:
finished = True
try:
results = os.listdir(resultdir)
except OSError:
# will happen when mock hasn't created the resultdir yet
continue
for fname in results:
if fname.endswith('.log') and not logs.has_key(fname):
fpath = os.path.join(resultdir, fname)
logs[fname] = (None, None, 0, fpath)
if workdir and mocklog not in logs:
fpath = os.path.join(workdir, mocklog)
if os.path.exists(fpath):
logs[mocklog] = (None, None, 0, fpath)
for (fname, (fd, inode, size, fpath)) in logs.items():
try:
stat_info = os.stat(fpath)
if not fd or stat_info.st_ino != inode or stat_info.st_size < size:
# either a file we haven't opened before, or mock replaced a file we had open with
# a new file and is writing to it, or truncated the file we're reading,
# but our fd is pointing to the previous location in the old file
if fd:
self.logger.info('Rereading %s, inode: %s -> %s, size: %s -> %s' %
(fpath, inode, stat_info.st_ino, size, stat_info.st_size))
fd.close()
fd = file(fpath, 'r')
logs[fname] = (fd, stat_info.st_ino, stat_info.st_size, fpath)
except:
self.logger.error("Error reading mock log: %s", fpath)
self.logger.error(''.join(traceback.format_exception(*sys.exc_info())))
continue
incremental_upload(self.session, fname, fd, uploadpath, logger=self.logger)
#clean up and return exit status of command
for (fname, (fd, inode, size, fpath)) in logs.items():
if fd:
fd.close()
return status[1]
else:
#in no case should exceptions propagate past here
try:
self.session._forget()
if workdir:
outfile = os.path.join(workdir, mocklog)
flags = os.O_CREAT | os.O_WRONLY | os.O_APPEND
fd = os.open(outfile, flags, 0666)
os.dup2(fd, 1)
os.dup2(fd, 2)
if os.getuid() == 0 and hasattr(self.options,"mockuser"):
self.logger.info('Running mock as %s' % self.options.mockuser)
uid,gid = pwd.getpwnam(self.options.mockuser)[2:4]
os.setgroups([grp.getgrnam('mock')[2]])
os.setregid(gid,gid)
os.setreuid(uid,uid)
os.execvp(cmd[0],cmd)
except:
#diediedie
print "Failed to exec mock"
print ''.join(traceback.format_exception(*sys.exc_info()))
os._exit(1)
def getUploadPath(self):
"""Get the path that should be used when uploading files to
the hub."""
return koji.pathinfo.taskrelpath(self.task_id)
def uploadDir(self, dirpath, suffix=None):
"""Upload the contents of the given directory to the
task output directory on the hub. If suffix is provided,
append '.' + suffix to the filenames, so that successive uploads
of the same directory won't overwrite each other, if the files have
the same name but different contents."""
if not os.path.isdir(dirpath):
return
uploadpath = self.getUploadPath()
for filename in os.listdir(dirpath):
filepath = os.path.join(dirpath, filename)
if os.stat(filepath).st_size > 0:
if suffix:
filename = '%s.%s' % (filename, suffix)
self.session.uploadWrapper(filepath, uploadpath, filename)
def init(self):
rv = self.mock(['--init'])
if rv:
self.expire()
raise koji.BuildrootError, "could not init mock buildroot, %s" % self._mockResult(rv)
self.session.host.setBuildRootList(self.id,self.getPackageList())
def _mockResult(self, rv, logfile=None):
if logfile:
pass
elif os.WIFEXITED(rv) and os.WEXITSTATUS(rv) == 1:
logfile = 'build.log'
else:
logfile = 'root.log'
msg = '; see %s for more information' % logfile
return parseStatus(rv, 'mock') + msg
def build_srpm(self, specfile, sourcedir, source_cmd):
self.session.host.setBuildRootState(self.id,'BUILDING')
alt_sources_dir = "%s/SOURCES" % sourcedir
if self.options.support_rpm_source_layout and os.path.isdir(alt_sources_dir):
sources_dir = alt_sources_dir
else:
sources_dir = sourcedir
if source_cmd:
# call the command defined by source_cmd in the chroot so any required files not stored in
# the SCM can be retrieved
chroot_sourcedir = sourcedir[len(self.rootdir()):]
args = ['--no-clean', '--unpriv', '--cwd', chroot_sourcedir, '--chroot']
args.extend(source_cmd)
rv = self.mock(args)
if rv:
self.expire()
raise koji.BuildError, "error retrieving sources, %s" % self._mockResult(rv)
args = ['--no-clean', '--buildsrpm', '--spec', specfile, '--sources', sources_dir,
'--target', 'noarch']
rv = self.mock(args)
if rv:
self.expire()
raise koji.BuildError, "error building srpm, %s" % self._mockResult(rv)
def build(self,srpm,arch=None):
# run build
self.session.host.setBuildRootState(self.id,'BUILDING')
args = ['--no-clean']
if arch:
args.extend(['--target', arch])
args.extend(['--rebuild', srpm])
rv = self.mock(args)
self.session.host.updateBuildRootList(self.id,self.getPackageList())
if rv:
self.expire()
raise koji.BuildError, "error building package (arch %s), %s" % (arch, self._mockResult(rv))
def getPackageList(self):
"""Return a list of packages from the buildroot
Each member of the list is a dictionary containing the following fields:
- name
- version
- release
- epoch
- arch
- payloadhash
- size
- buildtime
"""
fields = ('name',
'version',
'release',
'epoch',
'arch',
'sigmd5',
'size',
'buildtime')
rpm.addMacro("_dbpath", "%s/var/lib/rpm" % self.rootdir())
ret = []
try:
ts = rpm.TransactionSet()
for h in ts.dbMatch():
pkg = koji.get_header_fields(h,fields)
#skip our fake packages
if pkg['name'] in ['buildsys-build', 'gpg-pubkey']:
#XXX config
continue
pkg['payloadhash'] = koji.hex_string(pkg['sigmd5'])
del pkg['sigmd5']
ret.append(pkg)
finally:
rpm.delMacro("_dbpath")
self.markExternalRPMs(ret)
return ret
def getMavenPackageList(self, repodir):
"""Return a list of Maven packages that were installed into the local repo
to satisfy build requirements.
Each member of the list is a dictionary containing the following fields:
- maven_info: a dict of Maven info containing the groupId, artifactId, and version fields
- files: a list of files associated with that POM
"""
packages = []
for path, dirs, files in os.walk(repodir):
relpath = path[len(repodir) + 1:]
maven_files = []
for repofile in files:
if koji.util.multi_fnmatch(repofile, self.options.maven_repo_ignore) or \
koji.util.multi_fnmatch(os.path.join(relpath, repofile), self.options.maven_repo_ignore):
continue
if relpath == '' and repofile in ['scm-sources.zip', 'patches.zip']:
# special-case the archives of the sources and patches, since we drop them in
# root of the output directory
continue
maven_files.append({'path': relpath, 'filename': repofile,
'size': os.path.getsize(os.path.join(path, repofile))})
if maven_files:
path_comps = relpath.split('/')
if len(path_comps) < 3:
raise koji.BuildrootError, 'files found in unexpected path in local Maven repo, directory: %s, files: %s' % \
(relpath, ', '.join([f['filename'] for f in maven_files]))
# extract the Maven info from the path within the local repo
maven_info = {'version': path_comps[-1],
'artifact_id': path_comps[-2],
'group_id': '.'.join(path_comps[:-2])}
packages.append({'maven_info': maven_info, 'files': maven_files})
return packages
def mavenBuild(self, sourcedir, outputdir, repodir,
props=None, profiles=None, options=None, goals=None):
self.session.host.setBuildRootState(self.id, 'BUILDING')
cmd = ['--no-clean', '--chroot', '--unpriv', '--cwd', sourcedir[len(self.rootdir()):], '--',
'/usr/bin/mvn', '-C']
if options:
cmd.extend(options)
if profiles:
cmd.append('-P%s' % ','.join(profiles))
if props:
for name, value in props.items():
if value is not None:
cmd.append('-D%s=%s' % (name, value))
else:
cmd.append('-D%s' % name)
if goals:
cmd.extend(goals)
cmd.extend(['deploy'])
rv = self.mock(cmd)
# if the deploy command failed, don't raise an error on unknown artifacts, because that
# will mask the underlying failure
ignore_unknown = False
if rv:
ignore_unknown = True
self.session.host.updateMavenBuildRootList(self.id, self.task_id, self.getMavenPackageList(repodir),
ignore=self.getMavenPackageList(outputdir),
project=True, ignore_unknown=ignore_unknown,
extra_deps=self.deps)
if rv:
self.expire()
raise koji.BuildrootError, 'error building Maven package, %s' % self._mockResult(rv, logfile='root.log')
def scrub(self):
"Non-mock implementation of clean"
raise koji.FunctionDeprecated, "no longer needed and deprecated. use clean()"
def markExternalRPMs(self, rpmlist):
"""Check rpms against pkgorigins and add external repo data to the external ones
Modifies rpmlist in place. No return
"""
external_repos = self.session.getExternalRepoList(self.repo_info['tag_id'],
event=self.repo_info['create_event'])
if not external_repos:
#nothing to do
return
#index external repos by expanded url
erepo_idx = {}
for erepo in external_repos:
# substitute $arch in the url with the arch of the repo we're generating
ext_url = erepo['url'].replace('$arch', self.br_arch)
erepo_idx[ext_url] = erepo
pathinfo = koji.PathInfo(topdir='')
#XXX - cheap hack to get relative paths
repodir = pathinfo.repo(self.repo_info['id'], self.repo_info['tag_name'])
repomdpath = os.path.join(repodir, self.br_arch, 'repodata', 'repomd.xml')
opts = dict([(k, getattr(self.options, k)) for k in 'topurl','topdir'])
fo = koji.openRemoteFile(repomdpath, **opts)
try:
repodata = repoMDObject.RepoMD('ourrepo', fo)
except:
raise koji.BuildError, "Unable to parse repomd.xml file for %s" % os.path.join(repodir, self.br_arch)
data = repodata.getData('origin')
pkgorigins = data.location[1]
relpath = os.path.join(repodir, self.br_arch, pkgorigins)
fo = koji.openRemoteFile(relpath, **opts)
#at this point we know there were external repos at the create event,
#so there should be an origins file.
origin_idx = {}
fo2 = GzipFile(fileobj=fo, mode='r')
for line in fo2:
parts=line.split(None, 2)
if len(parts) < 2:
continue
#first field is formated by yum as [e:]n-v-r.a
nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" % koji.parse_NVRA(parts[0])
origin_idx[nvra] = parts[1]
fo2.close()
fo.close()
# mergerepo starts from a local repo in the task workdir, so internal
# rpms have an odd-looking origin that we need to look for
localtail = '/repo_%s_premerge/' % self.repo_info['id']
for rpm_info in rpmlist:
key = "%(name)s-%(version)s-%(release)s.%(arch)s" % rpm_info
# src rpms should not show up in rpmlist so we do not have to
# worry about fixing the arch for them
ext_url = origin_idx.get(key)
if not ext_url:
raise koji.BuildError, "No origin for %s" % key
erepo = erepo_idx.get(ext_url)
if not erepo:
if ext_url.startswith('file://') and ext_url.endswith(localtail):
# internal rpm
continue
raise koji.BuildError, "Unknown origin for %s: %s" % (key, ext_url)
rpm_info['external_repo'] = erepo
rpm_info['location'] = erepo['external_repo_id']
def resultdir(self):
return "%s/%s/result" % (self.options.mockdir, self.name)
def rootdir(self):
return "%s/%s/root" % (self.options.mockdir, self.name)
def expire(self):
self.session.host.setBuildRootState(self.id,'EXPIRED')
class ChainBuildTask(BaseTaskHandler):
Methods = ['chainbuild']
#mostly just waiting on other tasks
_taskWeight = 0.1
def handler(self, srcs, target, opts=None):
"""Run a chain build
target and opts are passed on to the build tasks
srcs is a list of "build levels"
each build level is a list of strings, each string may be one of:
- a build src (SCM url only)
- an n-v-r
each build level is processed in order
successive levels are only started once the previous levels have completed
and gotten into the repo.
"""
if opts.get('scratch'):
raise koji.BuildError, "--scratch is not allowed with chain-builds"
target_info = self.session.getBuildTarget(target)
if not target_info:
raise koji.GenericError, 'unknown build target: %s' % target
nvrs = []
for n_level, build_level in enumerate(srcs):
#if there are any nvrs to wait on, do so
if nvrs:
task_id = self.session.host.subtask(method='waitrepo',
arglist=[target_info['build_tag_name'], None, nvrs],
label="wait %i" % n_level,
parent=self.id)
self.wait(task_id, all=True, failany=True)
nvrs = []
#kick off the builds for this level
build_tasks = []
for n_src, src in enumerate(build_level):
if SCM.is_scm_url(src):
task_id = self.session.host.subtask(method='build',
arglist=[src, target, opts],
label="build %i,%i" % (n_level, n_src),
parent=self.id)
build_tasks.append(task_id)
else:
nvrs.append(src)
#next pass will wait for these
if build_tasks:
#the level could have been all nvrs
self.wait(build_tasks, all=True, failany=True)
#see what builds we created in this batch so the next pass can wait for them also
for build_task in build_tasks:
builds = self.session.listBuilds(taskID=build_task)
if builds:
nvrs.append(builds[0]['nvr'])
class BuildTask(BaseTaskHandler):
Methods = ['build']
#we mostly just wait on other tasks
_taskWeight = 0.2
def handler(self, src, target, opts=None):
"""Handler for the master build task"""
if opts is None:
opts = {}
self.opts = opts
if opts.get('arch_override') and not opts.get('scratch'):
raise koji.BuildError, "arch_override is only allowed for scratch builds"
if opts.get('repo_id') is not None:
repo_info = self.session.repoInfo(opts['repo_id'])
if not repo_info:
raise koji.BuildError, 'No such repo: %s' % opts['repo_id']
repo_state = koji.REPO_STATES[repo_info['state']]
if repo_state not in ('READY', 'EXPIRED'):
raise koji.BuildError, 'Bad repo: %s (%s)' % (repo_info['id'], repo_state)
self.event_id = repo_info['create_event']
else:
repo_info = None
#we'll wait for a repo later (self.getRepo)
self.event_id = None
task_info = self.session.getTaskInfo(self.id)
target_info = None
if target:
target_info = self.session.getBuildTarget(target, event=self.event_id)
if target_info:
dest_tag = target_info['dest_tag']
build_tag = target_info['build_tag']
if repo_info is not None:
#make sure specified repo matches target
if repo_info['tag_id'] != target_info['build_tag']:
raise koji.BuildError, 'Repo/Target mismatch: %s/%s' \
% (repo_info['tag_name'], target_info['build_tag_name'])
else:
# if repo_id is specified, we can allow the 'target' arg to simply specify
# the destination tag (since the repo specifies the build tag).
if repo_info is None:
raise koji.GenericError, 'unknown build target: %s' % target
build_tag = repo_info['tag_id']
if target is None:
#ok, call it skip-tag for the buildroot tag
self.opts['skip_tag'] = True
dest_tag = build_tag
else:
taginfo = self.session.getTag(target, event=self.event_id)
if not taginfo:
raise koji.GenericError, 'neither tag nor target: %s' % target
dest_tag = taginfo['id']
#policy checks...
policy_data = {
'user_id' : task_info['owner'],
'source' : src,
'task_id' : self.id,
'build_tag' : build_tag, #id
'skip_tag' : bool(self.opts.get('skip_tag')),
}
if target_info:
policy_data['target'] = target_info['id'],
if not self.opts.get('skip_tag'):
policy_data['tag'] = dest_tag #id
if not SCM.is_scm_url(src) and not opts.get('scratch'):
#let hub policy decide
self.session.host.assertPolicy('build_from_srpm', policy_data)
if opts.get('repo_id') is not None:
# use of this option is governed by policy
self.session.host.assertPolicy('build_from_repo_id', policy_data)
if not repo_info:
repo_info = self.getRepo(build_tag) #(subtask)
self.event_id = self.session.getLastEvent()['id']
srpm = self.getSRPM(src, build_tag, repo_info['id'])
h = self.readSRPMHeader(srpm)
data = koji.get_header_fields(h,['name','version','release','epoch'])
data['task_id'] = self.id
extra_arches = None
self.logger.info("Reading package config for %(name)s" % data)
pkg_cfg = self.session.getPackageConfig(dest_tag,data['name'],event=self.event_id)
self.logger.debug("%r" % pkg_cfg)
if pkg_cfg is not None:
extra_arches = pkg_cfg.get('extra_arches')
if not self.opts.get('skip_tag') and not self.opts.get('scratch'):
# Make sure package is on the list for this tag
if pkg_cfg is None:
raise koji.BuildError, "package %s not in list for tag %s" \
% (data['name'], target_info['dest_tag_name'])
elif pkg_cfg['blocked']:
raise koji.BuildError, "package %s is blocked for tag %s" \
% (data['name'], target_info['dest_tag_name'])
# TODO - more pre tests
archlist = self.getArchList(build_tag, h, extra=extra_arches)
#let the system know about the build we're attempting
if not self.opts.get('scratch'):
#scratch builds do not get imported
build_id = self.session.host.initBuild(data)
#(initBuild raises an exception if there is a conflict)
try:
self.extra_information = { "src": src, "data": data, "target": target }
srpm,rpms,brmap,logs = self.runBuilds(srpm,build_tag,archlist,repo_info['id'])
if opts.get('scratch'):
#scratch builds do not get imported
self.session.host.moveBuildToScratch(self.id,srpm,rpms,logs=logs)
else:
self.session.host.completeBuild(self.id,build_id,srpm,rpms,brmap,logs=logs)
except (SystemExit,ServerExit,KeyboardInterrupt):
#we do not trap these
raise
except:
if not self.opts.get('scratch'):
#scratch builds do not get imported
self.session.host.failBuild(self.id, build_id)
# reraise the exception
raise
if not self.opts.get('skip_tag') and not self.opts.get('scratch'):
self.tagBuild(build_id,dest_tag)
def getSRPM(self, src, build_tag, repo_id):
"""Get srpm from src"""
if isinstance(src,str):
if SCM.is_scm_url(src):
return self.getSRPMFromSCM(src, build_tag, repo_id)
else:
#assume this is a path under uploads
return src
else:
raise koji.BuildError, 'Invalid source specification: %s' % src
#XXX - other methods?
def getSRPMFromSCM(self, url, build_tag, repo_id):
#TODO - allow different ways to get the srpm
task_id = self.session.host.subtask(method='buildSRPMFromSCM',
arglist=[url, build_tag, {'repo_id': repo_id}],
label='srpm',
parent=self.id)
# wait for subtask to finish
result = self.wait(task_id)[task_id]
srpm = result['srpm']
return srpm
def readSRPMHeader(self, srpm):
#srpm arg should be a path relative to <BASEDIR>/work
self.logger.debug("Reading SRPM")
relpath = "work/%s" % srpm
opts = dict([(k, getattr(self.options, k)) for k in 'topurl','topdir'])
fo = koji.openRemoteFile(relpath, **opts)
h = koji.get_rpm_header(fo)
fo.close()
if h[rpm.RPMTAG_SOURCEPACKAGE] != 1:
raise koji.BuildError, "%s is not a source package" % srpm
return h
def getArchList(self, build_tag, h, extra=None):
# get list of arches to build for
buildconfig = self.session.getBuildConfig(build_tag, event=self.event_id)
arches = buildconfig['arches']
if not arches:
#XXX - need to handle this better
raise koji.BuildError, "No arches for tag %(name)s [%(id)s]" % buildconfig
tag_archlist = [koji.canonArch(a) for a in arches.split()]
self.logger.debug('arches: %s' % arches)
if extra:
self.logger.debug('Got extra arches: %s' % extra)
arches = "%s %s" % (arches,extra)
archlist = arches.split()
self.logger.debug('base archlist: %r' % archlist)
# - adjust arch list based on srpm macros
buildarchs = h[rpm.RPMTAG_BUILDARCHS]
exclusivearch = h[rpm.RPMTAG_EXCLUSIVEARCH]
excludearch = h[rpm.RPMTAG_EXCLUDEARCH]
if buildarchs:
archlist = buildarchs
self.logger.debug('archlist after buildarchs: %r' % archlist)
if exclusivearch:
archlist = [ a for a in archlist if a in exclusivearch ]
self.logger.debug('archlist after exclusivearch: %r' % archlist)
if excludearch:
archlist = [ a for a in archlist if a not in excludearch ]
self.logger.debug('archlist after excludearch: %r' % archlist)
#noarch is funny
if 'noarch' not in excludearch and \
( 'noarch' in buildarchs or 'noarch' in exclusivearch ):
archlist.append('noarch')
override = self.opts.get('arch_override')
if self.opts.get('scratch') and override:
#only honor override for scratch builds
self.logger.debug('arch override: %s' % override)
archlist = override.split()
archdict = {}
for a in archlist:
# Filter based on canonical arches for tag
# This prevents building for an arch that we can't handle
if a == 'noarch' or koji.canonArch(a) in tag_archlist:
archdict[a] = 1
if not archdict:
raise koji.BuildError, "No matching arches were found"
return archdict.keys()
def runBuilds(self, srpm, build_tag, archlist, repo_id):
self.logger.debug("Spawning jobs for arches: %r" % (archlist))
subtasks = {}
keep_srpm = True
for arch in archlist:
if koji.util.multi_fnmatch(arch, self.options.literal_task_arches):
taskarch = arch
else:
taskarch = koji.canonArch(arch)
subtasks[arch] = self.session.host.subtask(method='buildArch',
arglist=[srpm, build_tag, arch, keep_srpm, {'repo_id': repo_id}],
label=arch,
parent=self.id,
arch=taskarch)
keep_srpm = False
self.logger.debug("Got subtasks: %r" % (subtasks))
self.logger.debug("Waiting on subtasks...")
# wait for subtasks to finish
results = self.wait(subtasks.values(), all=True, failany=True)
# finalize import
# merge data into needed args for completeBuild call
rpms = []
brmap = {}
logs = {}
built_srpm = None
for (arch, task_id) in subtasks.iteritems():
result = results[task_id]
self.logger.debug("DEBUG: %r : %r " % (arch,result,))
brootid = result['brootid']
for fn in result['rpms']:
rpms.append(fn)
brmap[fn] = brootid
for fn in result['logs']:
logs.setdefault(arch,[]).append(fn)
if result['srpms']:
if built_srpm:
raise koji.BuildError, "multiple builds returned a srpm. task %i" % self.id
else:
built_srpm = result['srpms'][0]
brmap[result['srpms'][0]] = brootid
if built_srpm:
srpm = built_srpm
else:
raise koji.BuildError("could not find a built srpm")
return srpm,rpms,brmap,logs
def tagBuild(self,build_id,dest_tag):
#XXX - need options to skip tagging and to force tagging
#create the tagBuild subtask
#this will handle the "post tests"
task_id = self.session.host.subtask(method='tagBuild',
arglist=[dest_tag,build_id,False,None,True],
label='tag',
parent=self.id,
arch='noarch')
self.wait(task_id)
class BaseBuildTask(BaseTaskHandler):
"""Base class for tasks the create a build root"""
def checkHostArch(self, tag, hostdata, event=None):
tagref = tag
if isinstance(tag, dict):
tagref = tag.get('id') or tag.get('name')
opts = {}
if event is not None:
opts['event'] = event
tag = self.session.getBuildConfig(tagref, **opts)
if tag and tag['arches']:
tag_arches = [koji.canonArch(a) for a in tag['arches'].split()]
host_arches = hostdata['arches'].split()
if not set(tag_arches).intersection(host_arches):
self.logger.info('Task %s (%s): tag arches (%s) and ' \
'host arches (%s) are disjoint' % \
(self.id, self.method,
', '.join(tag_arches), ', '.join(host_arches)))
return False
#otherwise...
# This is in principle an error condition, but this is not a good place
# to fail. Instead we proceed and let the task fail normally.
return True
class BuildArchTask(BaseBuildTask):
Methods = ['buildArch']
def weight(self):
return 1.5
def updateWeight(self, name):
"""
Update the weight of this task based on the package we're building.
weight is scaled from a minimum of 1.5 to a maximum of 6, based on
the average duration of a build of this package.
"""
avg = self.session.getAverageBuildDuration(name)
if not avg:
return
if avg < 0:
self.logger.warn("Negative average build duration for %s: %s", name, avg)
return
# increase the task weight by 0.75 for every hour of build duration
adj = (avg / 4800.0)
# cap the adjustment at +4.5
weight = self.weight() + min(4.5, adj)
self.session.host.setTaskWeight(self.id, weight)
def checkHost(self, hostdata):
tag = self.params[1]
return self.checkHostArch(tag, hostdata)
def srpm_sanity_checks(self, filename):
header = koji.get_rpm_header(filename)
if not header[rpm.RPMTAG_PACKAGER]:
raise koji.BuildError, "The build system failed to set the packager tag"
if not header[rpm.RPMTAG_VENDOR]:
raise koji.BuildError, "The build system failed to set the vendor tag"
if not header[rpm.RPMTAG_DISTRIBUTION]:
raise koji.BuildError, "The build system failed to set the distribution tag"
def handler(self, pkg, root, arch, keep_srpm, opts=None):
"""Build a package in a buildroot for one arch"""
ret = {}
if opts is None:
opts = {}
repo_id = opts.get('repo_id')
if not repo_id:
raise koji.BuildError, "A repo id must be provided"
repo_info = self.session.repoInfo(repo_id, strict=True)
event_id = repo_info['create_event']
# starting srpm should already have been uploaded by parent
self.logger.debug("Reading SRPM")
fn = self.localPath("work/%s" % pkg)
if not os.path.exists(fn):
raise koji.BuildError, "SRPM file missing: %s" % fn
# peel E:N-V-R from package
h = koji.get_rpm_header(fn)
name = h[rpm.RPMTAG_NAME]
ver = h[rpm.RPMTAG_VERSION]
rel = h[rpm.RPMTAG_RELEASE]
epoch = h[rpm.RPMTAG_EPOCH]
if h[rpm.RPMTAG_SOURCEPACKAGE] != 1:
raise koji.BuildError, "not a source package"
# Disable checking for distribution in the initial SRPM because it
# might have been built outside of the build system
# if not h[rpm.RPMTAG_DISTRIBUTION]:
# raise koji.BuildError, "the distribution tag is not set in the original srpm"
self.updateWeight(name)
rootopts = {
'repo_id': repo_id
}
br_arch = self.find_arch(arch, self.session.host.getHost(), self.session.getBuildConfig(root, event=event_id))
broot = BuildRoot(self.session, self.options, root, br_arch, self.id, **rootopts)
broot.workdir = self.workdir
self.logger.debug("Initializing buildroot")
broot.init()
# run build
self.logger.debug("Running build")
broot.build(fn,arch)
# extract results
resultdir = broot.resultdir()
rpm_files = []
srpm_files = []
log_files = []
unexpected = []
for f in os.listdir(resultdir):
# files here should have one of two extensions: .log and .rpm
if f[-4:] == ".log":
log_files.append(f)
elif f[-8:] == ".src.rpm":
srpm_files.append(f)
elif f[-4:] == ".rpm":
rpm_files.append(f)
else:
unexpected.append(f)
self.logger.debug("rpms: %r" % rpm_files)
self.logger.debug("srpms: %r" % srpm_files)
self.logger.debug("logs: %r" % log_files)
self.logger.debug("unexpected: %r" % unexpected)
# upload files to storage server
uploadpath = broot.getUploadPath()
for f in rpm_files:
self.uploadFile("%s/%s" % (resultdir,f))
self.logger.debug("keep srpm %i %s %s" % (self.id, keep_srpm, opts))
if keep_srpm:
if len(srpm_files) == 0:
raise koji.BuildError, "no srpm files found for task %i" % self.id
if len(srpm_files) > 1:
raise koji.BuildError, "mulitple srpm files found for task %i: %s" % (self.id, srpm_files)
# Run sanity checks. Any failures will throw a BuildError
self.srpm_sanity_checks("%s/%s" % (resultdir,srpm_files[0]))
self.logger.debug("uploading %s/%s to %s" % (resultdir,srpm_files[0], uploadpath))
self.uploadFile("%s/%s" % (resultdir,srpm_files[0]))
if rpm_files:
ret['rpms'] = [ "%s/%s" % (uploadpath,f) for f in rpm_files ]
else:
ret['rpms'] = []
if keep_srpm:
ret['srpms'] = [ "%s/%s" % (uploadpath,f) for f in srpm_files ]
else:
ret['srpms'] = []
ret['logs'] = [ "%s/%s" % (uploadpath,f) for f in log_files ]
ret['brootid'] = broot.id
broot.expire()
#Let TaskManager clean up
return ret
class MavenTask(MultiPlatformTask):
Methods = ['maven']
_taskWeight = 0.2
def handler(self, url, target, opts=None):
"""Use Maven to build the source from the given url"""
if opts is None:
opts = {}
self.opts = opts
target_info = self.session.getBuildTarget(target)
if not target_info:
raise koji.BuildError, 'unknown build target: %s' % target
dest_tag = self.session.getTag(target_info['dest_tag'], strict=True)
build_tag = self.session.getTag(target_info['build_tag'], strict=True)
repo_id = opts.get('repo_id')
if not repo_id:
repo = self.session.getRepo(build_tag['id'])
if repo:
repo_id = repo['id']
else:
raise koji.BuildError, 'no repo for tag %s' % build_tag['name']
build_opts = dslice(opts, ['goals', 'profiles', 'properties', 'envs', 'patches',
'packages', 'jvm_options', 'maven_options', 'deps'],
strict=False)
build_opts['repo_id'] = repo_id
self.build_task_id = self.session.host.subtask(method='buildMaven',
arglist=[url, build_tag, build_opts],
label='build',
parent=self.id,
arch='noarch')
maven_results = self.wait(self.build_task_id)[self.build_task_id]
maven_results['task_id'] = self.build_task_id
build_info = None
if not self.opts.get('scratch'):
maven_info = maven_results['maven_info']
if maven_info['version'].endswith('-SNAPSHOT'):
raise koji.BuildError, '-SNAPSHOT versions are only supported in scratch builds'
build_info = koji.maven_info_to_nvr(maven_info)
if not self.opts.get('skip_tag'):
dest_cfg = self.session.getPackageConfig(dest_tag['id'], build_info['name'])
# Make sure package is on the list for this tag
if dest_cfg is None:
raise koji.BuildError, "package %s not in list for tag %s" \
% (build_info['name'], dest_tag['name'])
elif dest_cfg['blocked']:
raise koji.BuildError, "package %s is blocked for tag %s" \
% (build_info['name'], dest_tag['name'])
build_info = self.session.host.initMavenBuild(self.id, build_info, maven_info)
self.build_id = build_info['id']
try:
rpm_results = None
spec_url = self.opts.get('specfile')
if spec_url:
rpm_results = self.buildWrapperRPM(spec_url, self.build_task_id, target_info, build_info, repo_id)
if self.opts.get('scratch'):
self.session.host.moveMavenBuildToScratch(self.id, maven_results, rpm_results)
else:
self.session.host.completeMavenBuild(self.id, self.build_id, maven_results, rpm_results)
except (SystemExit, ServerExit, KeyboardInterrupt):
# we do not trap these
raise
except:
if not self.opts.get('scratch'):
#scratch builds do not get imported
self.session.host.failBuild(self.id, self.build_id)
# reraise the exception
raise
if not self.opts.get('scratch') and not self.opts.get('skip_tag'):
tag_task_id = self.session.host.subtask(method='tagBuild',
arglist=[dest_tag['id'], self.build_id, False, None, True],
label='tag',
parent=self.id,
arch='noarch')
self.wait(tag_task_id)
class BuildMavenTask(BaseBuildTask):
Methods = ['buildMaven']
_taskWeight = 1.5
def _zip_dir(self, rootdir, filename):
rootbase = os.path.basename(rootdir)
roottrim = len(rootdir) - len(rootbase)
zfo = zipfile.ZipFile(filename, 'w', zipfile.ZIP_DEFLATED)
for dirpath, dirnames, filenames in os.walk(rootdir):
for skip in ['CVS', '.svn', '.git']:
if skip in dirnames:
dirnames.remove(skip)
for filename in filenames:
filepath = os.path.join(dirpath, filename)
zfo.write(filepath, filepath[roottrim:])
zfo.close()
def checkHost(self, hostdata):
tag = self.params[1]
return self.checkHostArch(tag, hostdata)
def handler(self, url, build_tag, opts=None):
if opts is None:
opts = {}
self.opts = opts
scm = SCM(url)
scm.assert_allowed(self.options.allowed_scms)
repo_id = opts.get('repo_id')
if not repo_id:
raise koji.BuildError, 'A repo_id must be provided'
repo_info = self.session.repoInfo(repo_id, strict=True)
event_id = repo_info['create_event']
br_arch = self.find_arch('noarch', self.session.host.getHost(), session.getBuildConfig(build_tag['id'], event=event_id))
maven_opts = opts.get('jvm_options')
if not maven_opts:
maven_opts = []
for opt in maven_opts:
if opt.startswith('-Xmx'):
break
else:
# Give the JVM 2G to work with by default, if the build isn't specifying its own max. memory
maven_opts.append('-Xmx2048m')
buildroot = BuildRoot(self.session, self.options, build_tag['id'], br_arch, self.id,
install_group='maven-build', setup_dns=True, repo_id=repo_id,
maven_opts=maven_opts, maven_envs=opts.get('envs'),
deps=opts.get('deps'))
buildroot.workdir = self.workdir
self.logger.debug("Initializing buildroot")
buildroot.init()
packages = opts.get('packages')
if packages:
rv = buildroot.mock(['--install'] + packages)
self.session.host.setBuildRootState(buildroot.id, 'BUILDING')
self.session.host.updateBuildRootList(buildroot.id, buildroot.getPackageList())
if rv:
buildroot.expire()
raise koji.BuildrootError, 'error installing packages, %s' % buildroot._mockResult(rv, logfile='mock_output.log')
if not os.path.exists('%s/usr/bin/mvn' % buildroot.rootdir()):
raise koji.BuildError, '/usr/bin/mvn was not found in the buildroot'
scmdir = '%s/maven/build' % buildroot.rootdir()
outputdir = '%s/maven/output' % buildroot.rootdir()
m2dir = '%s/builddir/.m2' % buildroot.rootdir()
repodir = '%s/builddir/.m2/repository' % buildroot.rootdir()
patchdir = '%s/maven/patches' % buildroot.rootdir()
koji.ensuredir(scmdir)
koji.ensuredir(outputdir)
koji.ensuredir(repodir)
koji.ensuredir(patchdir)
logfile = self.workdir + '/checkout.log'
uploadpath = self.getUploadDir()
# Check out sources from the SCM
sourcedir = scm.checkout(scmdir, self.session, uploadpath, logfile)
# zip up pristine sources for auditing purposes
self._zip_dir(sourcedir, os.path.join(outputdir, 'scm-sources.zip'))
# Checkout out patches, if present
if self.opts.get('patches'):
patchlog = self.workdir + '/patches.log'
patch_scm = SCM(self.opts.get('patches'))
patch_scm.assert_allowed(self.options.allowed_scms)
# never try to check out a common/ dir when checking out patches
patch_scm.use_common = False
patchcheckoutdir = patch_scm.checkout(patchdir, self.session, uploadpath, patchlog)
self._zip_dir(patchcheckoutdir, os.path.join(outputdir, 'patches.zip'))
# Apply patches, if present
if self.opts.get('patches'):
# filter out directories and files beginning with . (probably scm metadata)
patches = [patch for patch in os.listdir(patchcheckoutdir) if \
os.path.isfile(os.path.join(patchcheckoutdir, patch)) and \
patch.endswith('.patch')]
if not patches:
raise koji.BuildError, 'no patches found at %s' % self.opts.get('patches')
patches.sort()
for patch in patches:
cmd = ['/usr/bin/patch', '--verbose', '--no-backup-if-mismatch', '-d', sourcedir, '-p1', '-i', os.path.join(patchcheckoutdir, patch)]
ret = log_output(self.session, cmd[0], cmd, patchlog, uploadpath, logerror=1, append=1)
if ret:
raise koji.BuildError, 'error applying patches from %s, see patches.log for details' % self.opts.get('patches')
# Set ownership of the entire source tree to the mock user
uid = pwd.getpwnam(self.options.mockuser)[2]
gid = grp.getgrnam('mock')[2]
self.chownTree(scmdir, uid, gid)
self.chownTree(outputdir, uid, gid)
self.chownTree(m2dir, uid, gid)
if self.opts.get('patches'):
self.chownTree(patchdir, uid, gid)
settingsfile = '/builddir/.m2/settings.xml'
buildroot.writeMavenSettings(settingsfile, outputdir)
pomfile = 'pom.xml'
maven_options = self.opts.get('maven_options', [])
for i, opt in enumerate(maven_options):
if opt == '-f' or opt == '--file':
if len(maven_options) > (i + 1):
pomfile = maven_options[i + 1]
break
else:
raise koji.BuildError, '%s option requires a file path' % opt
elif opt.startswith('-f=') or opt.startswith('--file='):
pomfile = opt.split('=', 1)[1]
break
elif opt.startswith('-f'):
pomfile = opt[2:]
break
buildroot.mavenBuild(sourcedir, outputdir, repodir,
props=self.opts.get('properties'), profiles=self.opts.get('profiles'),
options=self.opts.get('maven_options'), goals=self.opts.get('goals'))
build_pom = os.path.join(sourcedir, pomfile)
if not os.path.exists(build_pom):
raise koji.BuildError, '%s does not exist' % pomfile
pom_info = koji.parse_pom(build_pom)
maven_info = koji.pom_to_maven_info(pom_info)
# give the zip files more descriptive names
os.rename(os.path.join(outputdir, 'scm-sources.zip'),
os.path.join(outputdir, maven_info['artifact_id'] + '-' +
maven_info['version'] + '-scm-sources.zip'))
if self.opts.get('patches'):
os.rename(os.path.join(outputdir, 'patches.zip'),
os.path.join(outputdir, maven_info['artifact_id'] + '-' +
maven_info['version'] + '-patches.zip'))
logs = ['checkout.log']
if self.opts.get('patches'):
logs.append('patches.log')
output_files = {}
for path, dirs, files in os.walk(outputdir):
if not files:
continue
reldir = path[len(outputdir) + 1:]
for filename in files:
root, ext = os.path.splitext(filename)
if ext == '.log':
logs.append(os.path.join(reldir, filename))
else:
output_files.setdefault(reldir, []).append(filename)
# upload the build output
for filepath in logs:
self.uploadFile(os.path.join(outputdir, filepath),
relPath=os.path.dirname(filepath))
for relpath, files in output_files.iteritems():
for filename in files:
self.uploadFile(os.path.join(outputdir, relpath, filename),
relPath=relpath)
# Should only find log files in the mock result directory.
# Don't upload these log files, they've already been streamed
# the hub.
for filename in os.listdir(buildroot.resultdir()):
root, ext = os.path.splitext(filename)
if ext == '.log':
filepath = os.path.join(buildroot.resultdir(), filename)
if os.path.isfile(filepath) and os.stat(filepath).st_size > 0:
# only files with content get uploaded to the hub
logs.append(filename)
buildroot.expire()
return {'maven_info': maven_info,
'buildroot_id': buildroot.id,
'logs': logs,
'files': output_files}
class WrapperRPMTask(BaseBuildTask):
"""Build a wrapper rpm around archives output from a Maven or Windows build.
May either be called as a subtask or as a separate
top-level task. In the latter case it can either associate the new rpms
with the existing build or create a new build."""
Methods = ['wrapperRPM']
_taskWeight = 1.5
def copy_fields(self, src, tgt, *fields):
for field in fields:
tgt[field] = src.get(field)
def spec_sanity_checks(self, filename):
spec = open(filename).read()
for tag in ("Packager", "Distribution", "Vendor"):
if re.match("%s:" % tag, spec, re.M):
raise koji.BuildError, "%s is not allowed to be set in spec file" % tag
for tag in ("packager", "distribution", "vendor"):
if re.match("%%define\s+%s\s+" % tag, spec, re.M):
raise koji.BuildError, "%s is not allowed to be defined in spec file" % tag
def checkHost(self, hostdata):
target = self.params[1]
return self.checkHostArch(target['build_tag'], hostdata)
def handler(self, spec_url, build_target, build, task, opts=None):
if not opts:
opts = {}
if not (build or task):
raise koji.BuildError, 'build and/or task must be specified'
values = {}
if build:
maven_info = self.session.getMavenBuild(build['id'], strict=False)
win_info = self.session.getWinBuild(build['id'], strict=False)
image_info = self.session.getImageBuild(build['id'], strict=False)
else:
maven_info = None
win_info = None
image_info = None
# list of artifact paths relative to kojiroot (not exposed to the specfile)
artifact_relpaths = []
# map of file extension to a list of files
artifacts = {}
# list of all files
all_artifacts = []
# list of all files with their repo path
all_artifacts_with_path = []
# makes generating relative paths easier
self.pathinfo = koji.PathInfo(topdir='')
if task:
# called as a subtask of a build
artifact_paths = self.session.listTaskOutput(task['id'])
for artifact_path in artifact_paths:
artifact_name = os.path.basename(artifact_path)
base, ext = os.path.splitext(artifact_name)
if ext == '.log':
# Exclude log files for consistency with the output of listArchives() used below
continue
relpath = os.path.join(self.pathinfo.task(task['id']), artifact_path)[1:]
artifact_relpaths.append(relpath)
artifacts.setdefault(ext, []).append(artifact_name)
all_artifacts.append(artifact_name)
all_artifacts_with_path.append(artifact_path)
else:
# called as a top-level task to create wrapper rpms for an existing build
# verify that the build is complete
if not build['state'] == koji.BUILD_STATES['COMPLETE']:
raise koji.BuildError, 'cannot call wrapperRPM on a build that did not complete successfully'
# get the list of files from the build instead of the task, because the task output directory may
# have already been cleaned up
if maven_info:
build_artifacts = self.session.listArchives(buildID=build['id'], type='maven')
elif win_info:
build_artifacts = self.session.listArchives(buildID=build['id'], type='win')
elif image_info:
build_artifacts = self.session.listArchives(buildID=build['id'], type='image')
else:
raise koji.BuildError, 'unsupported build type'
for artifact in build_artifacts:
artifact_name = artifact['filename']
base, ext = os.path.splitext(artifact_name)
artifacts.setdefault(ext, []).append(artifact_name)
all_artifacts.append(artifact_name)
if ext == '.log':
# listArchives() should never return .log files, but we check for completeness
continue
if maven_info:
repopath = self.pathinfo.mavenfile(artifact)
relpath = os.path.join(self.pathinfo.mavenbuild(build), repopath)[1:]
artifact_relpaths.append(relpath)
all_artifacts_with_path.append(repopath)
elif win_info:
repopath = self.pathinfo.winfile(artifact)
relpath = os.path.join(self.pathinfo.winbuild(build), repopath)[1:]
artifact_relpaths.append(relpath)
all_artifacts_with_path.append(repopath)
elif image_info:
ipath = self.pathinfo.imagebuild(build)
relpath = os.path.join(ipath, artifact_name)[1:]
artifact_relpaths.append(relpath)
all_artifacts_with_path.append(artifact_name)
else:
# can't happen
assert False
if not artifacts:
raise koji.BuildError, 'no output found for %s' % (task and koji.taskLabel(task) or koji.buildLabel(build))
values['artifacts'] = artifacts
values['all_artifacts'] = all_artifacts
values['all_artifacts_with_path'] = all_artifacts_with_path
if build:
self.copy_fields(build, values, 'epoch', 'name', 'version', 'release')
if maven_info:
values['maven_info'] = maven_info
elif win_info:
values['win_info'] = win_info
elif image_info:
values['image_info'] = image_info
else:
# can't happen
assert False
else:
task_result = self.session.getTaskResult(task['id'])
if task['method'] == 'buildMaven':
maven_info = task_result['maven_info']
maven_nvr = koji.maven_info_to_nvr(maven_info)
maven_nvr['release'] = '0.scratch'
self.copy_fields(maven_nvr, values, 'epoch', 'name', 'version', 'release')
values['maven_info'] = maven_info
elif task['method'] == 'vmExec':
self.copy_fields(task_result, values, 'epoch', 'name', 'version', 'release')
values['win_info'] = {'platform': task_result['platform']}
elif task['method'] in ('createLiveCD', 'createAppliance', 'createImage'):
self.copy_fields(task_result, values, 'epoch', 'name', 'version', 'release')
else:
# can't happen
assert False
scm = SCM(spec_url)
scm.assert_allowed(self.options.allowed_scms)
repo_id = opts.get('repo_id')
if not repo_id:
raise koji.BuildError, "A repo id must be provided"
repo_info = self.session.repoInfo(repo_id, strict=True)
event_id = repo_info['create_event']
build_tag = self.session.getTag(build_target['build_tag'], strict=True)
br_arch = self.find_arch('noarch', self.session.host.getHost(), self.session.getBuildConfig(build_tag['id'], event=event_id))
buildroot = BuildRoot(self.session, self.options, build_tag['id'], br_arch, self.id, install_group='wrapper-rpm-build', repo_id=repo_id)
buildroot.workdir = self.workdir
self.logger.debug("Initializing buildroot")
buildroot.init()
logfile = os.path.join(self.workdir, 'checkout.log')
scmdir = buildroot.rootdir() + '/tmp/scmroot'
koji.ensuredir(scmdir)
specdir = scm.checkout(scmdir, self.session, self.getUploadDir(), logfile)
spec_template = None
for path, dir, files in os.walk(specdir):
files.sort()
for filename in files:
if filename.endswith('.spec.tmpl'):
spec_template = os.path.join(path, filename)
break
if not spec_template:
raise koji.BuildError, 'no spec file template found at URL: %s' % spec_url
# Put the jars into the same directory as the specfile. This directory will be
# set to the rpm _sourcedir so other files in the SCM may be referenced in the
# specfile as well.
specdir = os.path.dirname(spec_template)
for relpath in artifact_relpaths:
localpath = self.localPath(relpath)
# RPM requires all SOURCE files in the srpm to be in the same directory, so
# we flatten any directory structure of the output files here.
# If multiple files in the build have the same basename, duplicate files will
# have their relative path prepended to their name, with / replaced with -.
destpath = os.path.join(specdir, os.path.basename(relpath))
if os.path.exists(destpath):
destpath = os.path.join(specdir, relpath.replace('/', '-'))
shutil.copy(localpath, destpath)
# change directory to the specdir to the template can reference files there
os.chdir(specdir)
contents = Cheetah.Template.Template(file=spec_template,
searchList=[values]).respond()
contents = contents.encode('utf-8')
specfile = spec_template[:-5]
specfd = file(specfile, 'w')
specfd.write(contents)
specfd.close()
# Run spec file sanity checks. Any failures will throw a BuildError
self.spec_sanity_checks(specfile)
# chown the specdir to the mock user, because srpm creation happens
# as an unprivileged user
uid = pwd.getpwnam(self.options.mockuser)[2]
gid = grp.getgrnam('mock')[2]
self.chownTree(specdir, uid, gid)
#build srpm
self.logger.debug("Running srpm build")
buildroot.build_srpm(specfile, specdir, None)
srpms = glob.glob('%s/*.src.rpm' % buildroot.resultdir())
if len(srpms) == 0:
raise koji.BuildError, 'no srpms found in %s' % buildroot.resultdir()
elif len(srpms) > 1:
raise koji.BuildError, 'multiple srpms found in %s: %s' % (buildroot.resultdir(), ', '.join(srpms))
else:
srpm = srpms[0]
shutil.move(srpm, self.workdir)
srpm = os.path.join(self.workdir, os.path.basename(srpm))
self.new_build_id = None
if opts.get('create_build') and not opts.get('scratch'):
h = koji.get_rpm_header(srpm)
data = koji.get_header_fields(h, ['name', 'version', 'release', 'epoch'])
data['task_id'] = self.id
self.logger.info("Reading package config for %(name)s" % data)
pkg_cfg = self.session.getPackageConfig(build_target['dest_tag'], data['name'])
if not opts.get('skip_tag'):
# Make sure package is on the list for this tag
if pkg_cfg is None:
raise koji.BuildError, "package %s not in list for tag %s" \
% (data['name'], build_target['dest_tag_name'])
elif pkg_cfg['blocked']:
raise koji.BuildError, "package %s is blocked for tag %s" \
% (data['name'], build_target['dest_tag_name'])
self.new_build_id = self.session.host.initBuild(data)
try:
buildroot.build(srpm)
except (SystemExit, ServerExit, KeyboardInterrupt):
raise
except:
if self.new_build_id:
self.session.host.failBuild(self.id, self.new_build_id)
raise
resultdir = buildroot.resultdir()
srpm = None
rpms = []
logs = ['checkout.log']
for filename in os.listdir(resultdir):
if filename.endswith('.src.rpm'):
if not srpm:
srpm = filename
else:
if self.new_build_id:
self.session.host.failBuild(self.id, self.new_build_id)
raise koji.BuildError, 'multiple srpms found in %s: %s, %s' % \
(resultdir, srpm, filename)
elif filename.endswith('.rpm'):
rpms.append(filename)
elif filename.endswith('.log'):
logs.append(filename)
else:
if self.new_build_id:
self.session.host.failBuild(self.id, self.new_build_id)
raise koji.BuildError, 'unexpected file found in %s: %s' % \
(resultdir, filename)
if not srpm:
if self.new_build_id:
self.session.host.failBuild(self.id, self.new_build_id)
raise koji.BuildError, 'no srpm found'
if not rpms:
if self.new_build_id:
self.session.host.failBuild(self.id, self.new_build_id)
raise koji.BuildError, 'no rpms found'
try:
for rpm in [srpm] + rpms:
self.uploadFile(os.path.join(resultdir, rpm))
except (SystemExit, ServerExit, KeyboardInterrupt):
raise
except:
if self.new_build_id:
self.session.host.failBuild(self.id, self.new_build_id)
raise
results = {'buildroot_id': buildroot.id,
'srpm': srpm,
'rpms': rpms,
'logs': logs}
if not task:
# Called as a standalone top-level task, so handle the rpms now.
# Otherwise we let the parent task handle it.
uploaddir = self.getUploadDir()
relsrpm = uploaddir + '/' + srpm
relrpms = [uploaddir + '/' + r for r in rpms]
rellogs = [uploaddir + '/' + l for l in logs]
if opts.get('scratch'):
self.session.host.moveBuildToScratch(self.id, relsrpm, relrpms, {'noarch': rellogs})
else:
if opts.get('create_build'):
brmap = dict.fromkeys([relsrpm] + relrpms, buildroot.id)
try:
self.session.host.completeBuild(self.id, self.new_build_id,
relsrpm, relrpms, brmap, {'noarch': rellogs})
except (SystemExit, ServerExit, KeyboardInterrupt):
raise
except:
self.session.host.failBuild(self.id, self.new_build_id)
raise
if not opts.get('skip_tag'):
tag_task_id = self.session.host.subtask(method='tagBuild',
arglist=[build_target['dest_tag'],
self.new_build_id, False, None, True],
label='tag', parent=self.id, arch='noarch')
self.wait(tag_task_id)
else:
self.session.host.importWrapperRPMs(self.id, build['id'], results)
# no need to upload logs, they've already been streamed to the hub
# during the build process
buildroot.expire()
return results
class ChainMavenTask(MultiPlatformTask):
Methods = ['chainmaven']
_taskWeight = 0.2
def handler(self, builds, target, opts=None):
"""Run a sequence of Maven builds in dependency order"""
if not opts:
opts = {}
target_info = self.session.getBuildTarget(target)
if not target_info:
raise koji.BuildError, 'unknown build target: %s' % target
dest_tag = self.session.getTag(target_info['dest_tag'], strict=True)
if not (opts.get('scratch') or opts.get('skip_tag')):
for package in builds:
dest_cfg = self.session.getPackageConfig(dest_tag['id'], package)
# Make sure package is on the list for this tag
if dest_cfg is None:
raise koji.BuildError, "package %s not in list for tag %s" \
% (package, dest_tag['name'])
elif dest_cfg['blocked']:
raise koji.BuildError, "package %s is blocked for tag %s" \
% (package, dest_tag['name'])
self.depmap = {}
for package, params in builds.items():
self.depmap[package] = set(params.get('buildrequires', []))
todo = copy.deepcopy(self.depmap)
running = {}
self.done = {}
self.results = []
while True:
ready = [package for package, deps in todo.items() if not deps]
if not ready and not running:
break
for package in ready:
params = builds[package]
buildtype = params.get('type', 'maven')
task_url = params['scmurl']
task_opts = dslice_ex(params, ['scmurl', 'buildrequires', 'type'], strict=False)
if buildtype == 'maven':
task_deps = list(self.depset(package))
if task_deps:
task_opts['deps'] = task_deps
if not opts.get('force'):
# check for a duplicate build (a build performed with the
# same scmurl and options)
dup_build = self.get_duplicate_build(dest_tag['name'], package, params, task_opts)
# if we find one, mark the package as built and remove it from todo
if dup_build:
self.done[package] = dup_build['nvr']
for deps in todo.values():
deps.discard(package)
del todo[package]
self.results.append('%s previously built from %s' % (dup_build['nvr'], task_url))
continue
task_opts.update(dslice(opts, ['skip_tag', 'scratch'], strict=False))
if buildtype == 'maven':
if opts.get('debug'):
task_opts.setdefault('maven_options', []).append('--debug')
task_id = self.subtask('maven', [task_url, target, task_opts],
label=package)
elif buildtype == 'wrapper':
pkg_to_wrap = params['buildrequires'][0]
to_wrap = self.done[pkg_to_wrap]
if isinstance(to_wrap, (int, long)):
task_to_wrap = self.session.getTaskInfo(to_wrap, request=True)
build_to_wrap = None
else:
build_to_wrap = self.session.getBuild(to_wrap, strict=True)
task_to_wrap = None
target_info = self.session.getBuildTarget(target, strict=True)
repo_info = self.getRepo(target_info['build_tag'])
task_opts['repo_id'] = repo_info['id']
task_id = self.subtask('wrapperRPM', [task_url, target_info,
build_to_wrap, task_to_wrap,
task_opts],
label=package)
else:
raise koji.BuilError, 'unsupported build type: %s' % buildtype
running[task_id] = package
del todo[package]
try:
results = self.wait(running.keys())
except (xmlrpclib.Fault, koji.GenericError), e:
# One task has failed, wait for the rest to complete before the
# chainmaven task fails. self.wait(all=True) should thrown an exception.
self.wait(all=True)
raise
# if we get here, results is a map whose keys are the ids of tasks
# that have completed successfully
for task_id in results:
package = running.pop(task_id)
task_url = builds[package]['scmurl']
if opts.get('scratch'):
if builds[package].get('type') == 'wrapper':
self.done[package] = task_id
else:
children = self.session.getTaskChildren(task_id)
for child in children:
# we want the ID of the buildMaven task because the
# output dir of that task is where the Maven repo is
if child['method'] == 'buildMaven':
self.done[package] = child['id']
break
else:
raise koji.BuildError, 'could not find buildMaven subtask of %s' % task_id
self.results.append('%s built from %s by task %s' % \
(package, task_url, task_id))
else:
task_builds = self.session.listBuilds(taskID=task_id)
if not task_builds:
raise koji.BuildError, 'could not find build for task %s' % task_id
task_build = task_builds[0]
self.done[package] = task_build['nvr']
self.results.append('%s built from %s' % (task_build['nvr'], task_url))
for deps in todo.values():
deps.discard(package)
if todo:
# should never happen, the client should have checked for circular dependencies
raise koji.BuildError, 'unable to run chain build, circular dependencies'
return self.results
def depset(self, package):
deps = set()
for dep in self.depmap[package]:
deps.add(self.done[dep])
deps.update(self.depset(dep))
return deps
def dicts_equal(self, a, b):
"""Check if two dicts are equal. They are considered equal if they
have the same keys and those keys have the same values. If a value is
list, it will be considered equal to a list with the same values in
a different order."""
akeys = a.keys()
bkeys = b.keys()
if sorted(akeys) != sorted(bkeys):
return False
for key in akeys:
aval = a.get(key)
bval = b.get(key)
if type(aval) != type(bval):
return False
if isinstance(aval, dict):
if not self.dicts_equal(aval, bval):
return False
elif isinstance(aval, list):
if not sorted(aval) == sorted(bval):
return False
else:
if not aval == bval:
return False
return True
def get_duplicate_build(self, tag, package, params, task_opts):
"""Find the latest build of package in tag and compare it to the
scmurl and task_opts. If they're identical, return the build."""
builds = self.session.getLatestBuilds(tag, package=package)
if not builds:
return None
build = builds[0]
if not build['task_id']:
return None
build_task = session.getTaskInfo(build['task_id'], request=True)
request = build_task['request']
if request[0] != params['scmurl']:
return None
if params.get('type') == 'wrapper':
wrapped_build = request[2]
pkg_to_wrap = params['buildrequires'][0]
nvr_to_wrap = self.done[pkg_to_wrap]
if wrapped_build['nvr'] != nvr_to_wrap:
return None
# For a wrapper-rpm build, the only parameters that really matter
# are the scmurl and the wrapped NVR. These both match, so
# return the existing build.
return build
if len(request) > 2:
build_opts = dslice_ex(request[2], ['skip_tag', 'scratch'], strict=False)
else:
build_opts = {}
task_opts = copy.deepcopy(task_opts)
# filter out options that don't affect the build output
# to avoid unnecessary rebuilds
for opts in [build_opts, task_opts]:
if 'maven_options' in opts:
maven_options = opts['maven_options']
for opt in ['-e', '--errors', '-q', '--quiet',
'-V', '--show-version', '-X', '--debug']:
if opt in maven_options:
maven_options.remove(opt)
if not maven_options:
del opts['maven_options']
if 'jvm_options' in opts:
del opts['jvm_options']
if not self.dicts_equal(build_opts, task_opts):
return None
# everything matches
return build
class TagBuildTask(BaseTaskHandler):
Methods = ['tagBuild']
#XXX - set weight?
def handler(self, tag_id, build_id, force=False, fromtag=None, ignore_success=False):
task = self.session.getTaskInfo(self.id)
user_id = task['owner']
try:
build = self.session.getBuild(build_id, strict=True)
tag = self.session.getTag(tag_id, strict=True)
#several basic sanity checks have already been run (and will be run
#again when we make the final call). Our job is to perform the more
#computationally expensive 'post' tests.
#XXX - add more post tests
self.session.host.tagBuild(self.id,tag_id,build_id,force=force,fromtag=fromtag)
self.session.host.tagNotification(True, tag_id, fromtag, build_id, user_id, ignore_success)
except Exception, e:
exctype, value = sys.exc_info()[:2]
self.session.host.tagNotification(False, tag_id, fromtag, build_id, user_id, ignore_success, "%s: %s" % (exctype, value))
raise e
class BuildImageTask(MultiPlatformTask):
def initImageBuild(self, name, version, release, target_info, opts):
"""create a build object for this image build"""
pkg_cfg = self.session.getPackageConfig(target_info['dest_tag_name'],
name)
self.logger.debug("%r" % pkg_cfg)
if not opts.get('skip_tag') and not opts.get('scratch'):
# Make sure package is on the list for this tag
if pkg_cfg is None:
raise koji.BuildError, "package (image) %s not in list for tag %s" % (name, target_info['dest_tag_name'])
elif pkg_cfg['blocked']:
raise koji.BuildError, "package (image) %s is blocked for tag %s" % (name, target_info['dest_tag_name'])
return self.session.host.initImageBuild(self.id,
dict(name=name, version=version, release=release, epoch=0))
def getRelease(self, name, ver):
"""return the next available release number for an N-V"""
return self.session.getNextRelease(dict(name=name, version=ver))
class BuildBaseImageTask(BuildImageTask):
Methods = ['image']
def handler(self, name, version, arches, target, inst_tree, opts=None):
"""Governing task for building an appliance using Oz"""
target_info = self.session.getBuildTarget(target, strict=True)
build_tag = target_info['build_tag']
repo_info = self.getRepo(build_tag)
#check requested arches against build tag
buildconfig = self.session.getBuildConfig(build_tag)
if not buildconfig['arches']:
raise koji.BuildError, "No arches for tag %(name)s [%(id)s]" % buildconfig
tag_archlist = [koji.canonArch(a) for a in buildconfig['arches'].split()]
for arch in arches:
if koji.canonArch(arch) not in tag_archlist:
raise koji.BuildError, "Invalid arch for build tag: %s" % arch
if not opts:
opts = {}
if not ozif_enabled:
self.logger.error("ImageFactory features require the following dependencies: pykickstart, imagefactory, oz and possibly python-hashlib")
raise koji.ApplianceError, 'ImageFactory functions not available'
# build image(s)
bld_info = None
try:
release = opts.get('release')
if not release:
release = self.getRelease(name, version)
if '-' in version:
raise koji.ApplianceError('The Version may not have a hyphen')
if '-' in release:
raise koji.ApplianceError('The Release may not have a hyphen')
if not opts.get('scratch'):
bld_info = self.initImageBuild(name, version, release,
target_info, opts)
subtasks = {}
self.logger.debug("Spawning jobs for image arches: %r" % (arches))
for arch in arches:
inst_url = inst_tree.replace('$arch', arch)
subtasks[arch] = self.session.host.subtask(
method='createImage',
arglist=[name, version, release, arch, target_info,
build_tag, repo_info, inst_url, opts],
label=arch, parent=self.id, arch=arch)
self.logger.debug("Got image subtasks: %r" % (subtasks))
self.logger.debug("Waiting on image subtasks...")
results = self.wait(subtasks.values(), all=True, failany=True)
# wrap in an RPM if asked
rpm_results = None
spec_url = opts.get('specfile')
for arch in arches:
# get around an xmlrpc limitation, use arches for keys instead
results[arch] = results[subtasks[arch]]
del results[subtasks[arch]]
if spec_url:
subtask = subtasks[arch]
results[arch]['rpmresults'] = self.buildWrapperRPM(
spec_url, subtask, target_info, bld_info,
repo_info['id'])
# make sure we only import the user-submitted kickstart file one
# time, otherwise we will have collisions. Remove it from exactly
# 1 results hash from the subtasks
if opts.has_key('kickstart'):
saw_ks = False
for arch in results.keys():
ks = os.path.basename(opts.get('kickstart'))
if ks in results[arch]['files']:
if saw_ks:
results[arch]['files'].remove(ks)
saw_ks = True
self.logger.debug('Image Results for hub: %s' % results)
if opts.get('scratch'):
self.session.host.moveImageBuildToScratch(self.id, results)
else:
self.session.host.completeImageBuild(self.id, bld_info['id'],
results)
except (SystemExit,ServerExit,KeyboardInterrupt):
#we do not trap these
raise
except:
if not opts.get('scratch'):
#scratch builds do not get imported
if bld_info:
self.session.host.failBuild(self.id, bld_info['id'])
# reraise the exception
raise
# tag it
if not opts.get('scratch') and not opts.get('skip_tag'):
tag_task_id = self.session.host.subtask(method='tagBuild',
arglist=[target_info['dest_tag'], bld_info['id'], False, None, True],
label='tag', parent=self.id, arch='noarch')
self.wait(tag_task_id)
# report results
report = ''
if opts.get('scratch'):
respath = ', '.join(
[os.path.join(koji.pathinfo.work(), koji.pathinfo.taskrelpath(tid)) for tid in subtasks.values()])
report += 'Scratch '
else:
respath = koji.pathinfo.imagebuild(bld_info)
report += 'image build results in: %s' % respath
return report
class BuildApplianceTask(BuildImageTask):
Methods = ['appliance']
def handler(self, name, version, arch, target, ksfile, opts=None):
"""Governing task for building an appliance"""
target_info = self.session.getBuildTarget(target, strict=True)
build_tag = target_info['build_tag']
repo_info = self.getRepo(build_tag)
#check requested arch against build tag
buildconfig = self.session.getBuildConfig(build_tag)
if not buildconfig['arches']:
raise koji.BuildError, "No arches for tag %(name)s [%(id)s]" % buildconfig
tag_archlist = [koji.canonArch(a) for a in buildconfig['arches'].split()]
if koji.canonArch(arch) not in tag_archlist:
raise koji.BuildError, "Invalid arch for build tag: %s" % arch
if not opts:
opts = {}
if not image_enabled:
self.logger.error("Appliance features require the following dependencies: pykickstart, and possibly python-hashlib")
raise koji.ApplianceError, 'Appliance functions not available'
# build image
try:
release = opts.get('release')
if not release:
release = self.getRelease(name, version)
bld_info = None
if not opts.get('scratch'):
bld_info = self.initImageBuild(name, version, release,
target_info, opts)
create_task_id = self.session.host.subtask(method='createAppliance',
arglist=[name, version, release, arch, target_info, build_tag,
repo_info, ksfile, opts],
label='appliance', parent=self.id, arch=arch)
results = self.wait(create_task_id)
self.logger.info('image build task (%s) completed' % create_task_id)
self.logger.info('results: %s' % results)
# wrap in an RPM if asked
rpm_results = None
spec_url = opts.get('specfile')
if spec_url:
results[create_task_id]['rpmresults'] = self.buildWrapperRPM(
spec_url, create_task_id,
target_info, bld_info, repo_info['id'])
results[str(create_task_id)] = results[create_task_id]
del results[create_task_id]
# import the image (move it too)
if not opts.get('scratch'):
self.session.host.completeImageBuild(self.id, bld_info['id'], results)
else:
self.session.host.moveImageBuildToScratch(self.id, results)
except (SystemExit,ServerExit,KeyboardInterrupt):
#we do not trap these
raise
except:
if not opts.get('scratch'):
#scratch builds do not get imported
if bld_info:
self.session.host.failBuild(self.id, bld_info['id'])
# reraise the exception
raise
# tag it
if not opts.get('scratch') and not opts.get('skip_tag'):
tag_task_id = self.session.host.subtask(method='tagBuild',
arglist=[target_info['dest_tag'], bld_info['id'], False, None, True],
label='tag', parent=self.id, arch='noarch')
self.wait(tag_task_id)
# report results
if opts.get('scratch'):
respath = os.path.join(koji.pathinfo.work(),
koji.pathinfo.taskrelpath(create_task_id))
report = 'Scratch '
else:
respath = koji.pathinfo.imagebuild(bld_info)
report = ''
report += 'appliance build results in: %s' % respath
return report
class BuildLiveCDTask(BuildImageTask):
Methods = ['livecd']
def handler(self, name, version, arch, target, ksfile, opts=None):
"""Governing task for building LiveCDs"""
target_info = self.session.getBuildTarget(target, strict=True)
build_tag = target_info['build_tag']
repo_info = self.getRepo(build_tag)
#check requested arch against build tag
buildconfig = self.session.getBuildConfig(build_tag)
if not buildconfig['arches']:
raise koji.BuildError, "No arches for tag %(name)s [%(id)s]" % buildconfig
tag_archlist = [koji.canonArch(a) for a in buildconfig['arches'].split()]
if koji.canonArch(arch) not in tag_archlist:
raise koji.BuildError, "Invalid arch for build tag: %s" % arch
if not opts:
opts = {}
if not image_enabled:
self.logger.error("LiveCD features require the following dependencies: "
"pykickstart, pycdio, and possibly python-hashlib")
raise koji.LiveCDError, 'LiveCD functions not available'
# build the image
try:
release = opts.get('release')
if not release:
release = self.getRelease(name, version)
bld_info = None
if not opts.get('scratch'):
bld_info = self.initImageBuild(name, version, release,
target_info, opts)
create_task_id = self.session.host.subtask(method='createLiveCD',
arglist=[name, version, release, arch, target_info, build_tag,
repo_info, ksfile, opts],
label='livecd', parent=self.id, arch=arch)
results = self.wait(create_task_id)
self.logger.info('image build task (%s) completed' % create_task_id)
self.logger.info('results: %s' % results)
# wrap in an RPM if needed
spec_url = opts.get('specfile')
rpm_results = None
if spec_url:
results[create_task_id]['rpmresults'] = self.buildWrapperRPM(
spec_url, create_task_id,
target_info, bld_info, repo_info['id'])
results[str(create_task_id)] = results[create_task_id]
del results[create_task_id]
# import it (and move)
if not opts.get('scratch'):
self.session.host.completeImageBuild(self.id, bld_info['id'], results)
else:
self.session.host.moveImageBuildToScratch(self.id, results)
except (SystemExit,ServerExit,KeyboardInterrupt):
#we do not trap these
raise
except:
if not opts.get('scratch'):
#scratch builds do not get imported
if bld_info:
self.session.host.failBuild(self.id, bld_info['id'])
# reraise the exception
raise
# tag it if necessary
if not opts.get('scratch') and not opts.get('skip_tag'):
tag_task_id = self.session.host.subtask(method='tagBuild',
arglist=[target_info['dest_tag'], bld_info['id'], False, None, True],
label='tag', parent=self.id, arch='noarch')
self.wait(tag_task_id)
# report the results
if opts.get('scratch'):
respath = os.path.join(koji.pathinfo.work(),
koji.pathinfo.taskrelpath(create_task_id))
report = 'Scratch '
else:
respath = koji.pathinfo.imagebuild(bld_info)
report = ''
report += 'livecd build results in: %s' % respath
return report
# A generic task for building cd or disk images using chroot-based tools.
# Other chroot-based image handlers should inherit this.
class ImageTask(BaseTaskHandler):
Methods = []
def makeImgBuildRoot(self, buildtag, repoinfo, arch, inst_group):
"""
Create and prepare the chroot we're going to build an image in.
Binds necessary directories and creates needed device files.
@args:
buildtag: a build tag
repoinfo: a session.getRepo() object
arch: a canonical architecture name
inst_group: a string representing the yum group to install with
@returns: a buildroot object
"""
# Here we configure mock to bind mount a set of /dev directories
bind_opts = {'dirs' : {
'/dev' : '/dev',
'/selinux' : '/selinux'}
}
rootopts = {'install_group': inst_group,
'setup_dns': True,
'repo_id': repoinfo['id'],
'bind_opts' : bind_opts}
broot = BuildRoot(self.session, self.options, buildtag, arch, self.id, **rootopts)
broot.workdir = self.workdir
# create the mock chroot
self.logger.debug("Initializing image buildroot")
broot.init()
self.logger.debug("Image buildroot ready: " + broot.rootdir())
return broot
def fetchKickstart(self, broot, ksfile):
"""
Retrieve the kickstart file we were given (locally or remotely) and
upload it.
Note that if the KS file existed locally, then "ksfile" is a relative
path to it in the /mnt/koji/work directory. If not, then it is still
the parameter the user passed in initially, and we assume it is a
relative path in a remote scm. The user should have passed in an scm
url with --ksurl.
@args:
broot: a buildroot object
ksfile: path to a kickstart file
@returns: absolute path to the retrieved kickstart file
"""
scmdir = os.path.join(broot.rootdir(), 'tmp')
koji.ensuredir(scmdir)
self.logger.debug("ksfile = %s" % ksfile)
if self.opts.get('ksurl'):
scm = SCM(self.opts['ksurl'])
scm.assert_allowed(self.options.allowed_scms)
logfile = os.path.join(self.workdir, 'checkout.log')
scmsrcdir = scm.checkout(scmdir, self.session, self.getUploadDir(), logfile)
kspath = os.path.join(scmsrcdir, ksfile)
else:
kspath = self.localPath("work/%s" % ksfile)
self.uploadFile(kspath) # upload the original ks file
return kspath # full absolute path to the file in the chroot
def readKickstart(self, kspath, opts):
"""
Read a kickstart file and save the ks object as a task member.
@args:
kspath: path to a kickstart file
@returns: None
"""
# XXX: If the ks file came from a local path and has %include
# macros, *-creator will fail because the included
# kickstarts were not copied into the chroot. For now we
# require users to flatten their kickstart file if submitting
# the task with a local path.
#
# Note that if an SCM URL was used instead, %include macros
# may not be a problem if the included kickstarts are present
# in the repository we checked out.
if opts.get('ksversion'):
version = ksparser.version.makeVersion(ksparser.stringToVersion(opts['ksversion']))
else:
version = ksparser.version.makeVersion()
self.ks = ksparser.KickstartParser(version)
try:
self.ks.readKickstart(kspath)
except IOError, e:
raise koji.LiveCDError("Failed to read kickstart file "
"'%s' : %s" % (kspath, e))
except kserrors.KickstartError, e:
raise koji.LiveCDError("Failed to parse kickstart file "
"'%s' : %s" % (kspath, e))
def prepareKickstart(self, repo_info, target_info, arch, broot, opts):
"""
Process the ks file to be used for controlled image generation. This
method also uploads the modified kickstart file to the task output
area.
@args:
target_info: a sesion.getBuildTarget() object
repo_info: session.getRepo() object
arch: canonical architecture name
broot: a buildroot object
kspath: absolute path to a kickstart file
@returns:
absolute path to a processed kickstart file within the buildroot
"""
# Now we do some kickstart manipulation. If the user passed in a repo
# url with --repo, then we substitute that in for the repo(s) specified
# in the kickstart file. If --repo wasn't specified, then we use the
# repo associated with the target passed in initially.
repo_class = kscontrol.dataMap[self.ks.version]['RepoData']
self.ks.handler.repo.repoList = [] # delete whatever the ks file told us
if opts.get('repo'):
user_repos = opts['repo']
if isinstance(user_repos, basestring):
user_repos = user_repos.split(',')
index = 0
for user_repo in user_repos:
self.ks.handler.repo.repoList.append(repo_class(baseurl=user_repo, name='koji-override-%i' % index))
index += 1
else:
path_info = koji.PathInfo(topdir=self.options.topurl)
repopath = path_info.repo(repo_info['id'],
target_info['build_tag_name'])
baseurl = '%s/%s' % (repopath, arch)
self.logger.debug('BASEURL: %s' % baseurl)
self.ks.handler.repo.repoList.append(repo_class(baseurl=baseurl, name='koji-%s-%i' % (target_info['build_tag_name'], repo_info['id'])))
# Write out the new ks file. Note that things may not be in the same
# order and comments in the original ks file may be lost.
kskoji = os.path.join('/tmp', 'koji-image-%s-%i.ks' %
(target_info['build_tag_name'], self.id))
kojikspath = os.path.join(broot.rootdir(), kskoji[1:])
outfile = open(kojikspath, 'w')
outfile.write(str(self.ks.handler))
outfile.close()
# put the new ksfile in the output directory
if not os.path.exists(kojikspath):
raise koji.LiveCDError, "KS file missing: %s" % kojikspath
self.uploadFile(kojikspath)
return kskoji # absolute path within chroot
def getImagePackages(self, cachepath):
"""
Read RPM header information from the yum cache available in the
given path. Returns a list of dictionaries for each RPM included.
"""
found = False
hdrlist = []
fields = ['name', 'version', 'release', 'epoch', 'arch', \
'buildtime', 'sigmd5']
for root, dirs, files in os.walk(cachepath):
for f in files:
if fnmatch(f, '*.rpm'):
pkgfile = os.path.join(root, f)
hdr = koji.get_header_fields(pkgfile, fields)
hdr['size'] = os.path.getsize(pkgfile)
hdr['payloadhash'] = koji.hex_string(hdr['sigmd5'])
del hdr['sigmd5']
hdrlist.append(hdr)
found = True
if not found:
raise koji.LiveCDError, 'No repos found in yum cache!'
return hdrlist
# ApplianceTask begins with a mock chroot, and then installs appliance-tools
# into it via the appliance-build group. appliance-creator is then executed
# in the chroot to create the appliance image.
#
class ApplianceTask(ImageTask):
Methods = ['createAppliance']
_taskWeight = 1.5
def getRootDevice(self):
"""
Return the device name for the / partition, as specified in the
kickstart file. Appliances should have this defined.
"""
for part in self.ks.handler.partition.partitions:
if part.mountpoint == '/':
return part.disk
raise koji.ApplianceError, 'kickstart lacks a "/" mountpoint'
def handler(self, name, version, release, arch, target_info, build_tag, repo_info, ksfile, opts=None):
if opts == None:
opts = {}
self.opts = opts
broot = self.makeImgBuildRoot(build_tag, repo_info, arch,
'appliance-build')
kspath = self.fetchKickstart(broot, ksfile)
self.readKickstart(kspath, opts)
kskoji = self.prepareKickstart(repo_info, target_info, arch, broot, opts)
# Figure out appliance-creator arguments, let it fail if something
# is wrong.
odir = 'app-output'
opath = os.path.join(broot.rootdir(), 'tmp', odir)
cachedir = '/tmp/koji-appliance' # arbitrary paths in chroot
app_log = '/tmp/appliance.log'
os.mkdir(opath)
cmd = ['/usr/bin/appliance-creator', '-c', kskoji, '-d', '-v',
'--logfile', app_log, '--cache', cachedir, '-o', odir]
for arg_name in ('vmem', 'vcpu', 'format'):
arg = opts.get(arg_name)
if arg != None:
cmd.extend(['--%s' % arg_name, arg])
appname = '%s-%s-%s' % (name, version, release)
cmd.extend(['--name', appname])
cmd.extend(['--version', version, '--release', release])
# Run appliance-creator
rv = broot.mock(['--cwd', '/tmp', '--chroot', '--'] + cmd)
self.uploadFile(os.path.join(broot.rootdir(), app_log[1:]))
if rv:
raise koji.ApplianceError, \
"Could not create appliance: %s" % parseStatus(rv, 'appliance-creator') + "; see root.log or appliance.log for more information"
# Find the results
results = []
for directory, subdirs, files in os.walk(opath):
for f in files:
results.append(os.path.join(broot.rootdir(), 'tmp',
directory, f))
self.logger.debug('output: %s' % results)
if len(results) == 0:
raise koji.ApplianceError, "Could not find image build results!"
imgdata = {
'arch': arch,
'rootdev': self.getRootDevice(),
'task_id': self.id,
'logs': ['build.log', 'mock_output.log', 'root.log', 'state.log',
'appliance.log', os.path.basename(ksfile),
os.path.basename(kskoji)],
'name': name,
'version': version,
'release': release
}
imgdata['files'] = []
for ofile in results:
self.uploadFile(ofile)
imgdata['files'].append(os.path.basename(ofile))
# TODO: get file manifest from the appliance
if not opts.get('scratch'):
hdrlist = self.getImagePackages(os.path.join(broot.rootdir(),
cachedir[1:]))
broot.markExternalRPMs(hdrlist)
imgdata['rpmlist'] = hdrlist
broot.expire()
return imgdata
# LiveCDTask begins with a mock chroot, and then installs livecd-tools into it
# via the livecd-build group. livecd-creator is then executed in the chroot
# to create the LiveCD image.
#
class LiveCDTask(ImageTask):
Methods = ['createLiveCD']
_taskWeight = 1.5
def genISOManifest(self, image, manifile):
"""
Using iso9660 from pycdio, get the file manifest of the given image,
and save it to the text file manifile.
"""
fd = open(manifile, 'w')
if not fd:
raise koji.GenericError, \
'Unable to open manifest file (%s) for writing!' % manifile
iso = iso9660.ISO9660.IFS(source=image)
if not iso.is_open():
raise koji.GenericError, \
'Could not open %s as an ISO-9660 image!' % image
# image metadata
id = iso.get_application_id()
if id is not None: fd.write("Application ID: %s\n" % id)
id = iso.get_preparer_id()
if id is not None: fd.write("Preparer ID: %s\n" % id)
id = iso.get_publisher_id()
if id is not None: fd.write("Publisher ID: %s\n" % id)
id = iso.get_system_id()
if id is not None: fd.write("System ID: %s\n" % id)
id = iso.get_volume_id()
if id is not None: fd.write("Volume ID: %s\n" % id)
id = iso.get_volumeset_id()
if id is not None: fd.write("Volumeset ID: %s\n" % id)
fd.write('\nSize(bytes) File Name\n')
manifest = self.listISODir(iso, '/')
for a_file in manifest:
fd.write(a_file)
fd.close()
iso.close()
def listISODir(self, iso, path):
"""
Helper function called recursively by genISOManifest. Returns a
listing of files/directories at the given path in an iso image obj.
"""
manifest = []
file_stats = iso.readdir(path)
for stat in file_stats:
filename = stat[0]
size = stat[2]
is_dir = stat[4] == 2
if filename == '..':
continue
elif filename == '.':
# path should always end in a trailing /
filepath = path
else:
filepath = path + filename
# identify directories with a trailing /
if is_dir:
filepath += '/'
if is_dir and filename != '.':
# recurse into subdirectories
manifest.extend(self.listISODir(iso, filepath))
else:
# output information for the current directory and files
manifest.append("%-10d %s\n" % (size, filepath))
return manifest
def handler(self, name, version, release, arch, target_info, build_tag, repo_info, ksfile, opts=None):
if opts == None:
opts = {}
self.opts = opts
broot = self.makeImgBuildRoot(build_tag, repo_info, arch,
'livecd-build')
kspath = self.fetchKickstart(broot, ksfile)
self.readKickstart(kspath, opts)
kskoji = self.prepareKickstart(repo_info, target_info, arch, broot, opts)
cachedir = '/tmp/koji-livecd' # arbitrary paths in chroot
livecd_log = '/tmp/livecd.log'
cmd = ['/usr/bin/livecd-creator', '-c', kskoji, '-d', '-v',
'--logfile', livecd_log, '--cache', cachedir]
# we set the fs label to the same as the isoname if it exists,
# taking at most 32 characters
isoname = '%s-%s-%s' % (name, version, release)
cmd.extend(['-f', isoname[:32]])
# Run livecd-creator
rv = broot.mock(['--cwd', '/tmp', '--chroot', '--'] + cmd)
self.uploadFile(os.path.join(broot.rootdir(), livecd_log[1:]))
if rv:
raise koji.LiveCDError, \
'Could not create LiveCD: %s' % parseStatus(rv, 'livecd-creator') + '; see root.log or livecd.log for more information'
# Find the resultant iso
# The cwd of the livecd-creator process is /tmp in the chroot, so
# that is where it writes the .iso
files = os.listdir(os.path.join(broot.rootdir(), 'tmp'))
isofile = None
for afile in files:
if afile.endswith('.iso'):
if not isofile:
isofile = afile
else:
raise koji.LiveCDError, 'multiple .iso files found: %s and %s' % (isofile, afile)
if not isofile:
raise koji.LiveCDError, 'could not find iso file in chroot'
isosrc = os.path.join(broot.rootdir(), 'tmp', isofile)
# copy the iso out of the chroot. If we were given an isoname,
# this is where the renaming happens.
self.logger.debug('uploading image: %s' % isosrc)
isoname += '.iso'
# Generate the file manifest of the image, upload the results
manifest = os.path.join(broot.resultdir(), 'manifest.log')
self.genISOManifest(isosrc, manifest)
self.uploadFile(manifest)
self.uploadFile(isosrc, remoteName=isoname)
imgdata = {'arch': arch,
'files': [isoname],
'rootdev': None,
'task_id': self.id,
'logs': ['build.log', 'mock_output.log', 'root.log', 'state.log',
'livecd.log', os.path.basename(ksfile),
os.path.basename(kskoji)],
'name': name,
'version': version,
'release': release
}
if not opts.get('scratch'):
hdrlist = self.getImagePackages(os.path.join(broot.rootdir(),
cachedir[1:]))
imgdata ['rpmlist'] = hdrlist
broot.markExternalRPMs(hdrlist)
broot.expire()
return imgdata
# A generic task for building disk images using Oz
# Other Oz-based image handlers should inherit this.
class OzImageTask(BaseTaskHandler):
Methods = []
def fetchKickstart(self):
"""
Retrieve the kickstart file we were given (locally or remotely) and
upload it to the hub.
Note that if the KS file existed locally, then "ksfile" is a relative
path to it in the /mnt/koji/work directory. If not, then it is still
the parameter the user passed in initially, and we assume it is a
relative path in a remote scm. The user should have passed in an scm
url with --ksurl.
@args: None, use self.opts for options
@returns:
absolute path to the retrieved kickstart file
"""
ksfile = self.opts.get('kickstart')
self.logger.debug("ksfile = %s" % ksfile)
if self.opts.get('ksurl'):
scm = SCM(self.opts['ksurl'])
scm.assert_allowed(self.options.allowed_scms)
logfile = os.path.join(self.workdir, 'checkout-%s.log' % self.arch)
scmsrcdir = scm.checkout(self.workdir, self.session,
self.getUploadDir(), logfile)
kspath = os.path.join(scmsrcdir, os.path.basename(ksfile))
else:
tops = dict([(k, getattr(self.options, k)) for k in 'topurl','topdir'])
ks_src = koji.openRemoteFile(ksfile, **tops)
kspath = os.path.join(self.workdir, os.path.basename(ksfile))
ks_dest = open(kspath, 'w')
ks_dest.write(ks_src.read())
ks_dest.close()
self.logger.debug('uploading kickstart from here: %s' % kspath)
self.uploadFile(kspath) # upload the original ks file
return kspath # absolute path to the ks file
def readKickstart(self, kspath):
"""
Read a kickstart file and save the ks object as a task member.
@args:
kspath: path to a kickstart file
@returns:
a kickstart object returned by pykickstart
"""
# XXX: If the ks file came from a local path and has %include
# macros, Oz will fail because it can only handle flat files.
# We require users to flatten their kickstart file.
if self.opts.get('ksversion'):
version = ksparser.version.makeVersion(
ksparser.stringToVersion(self.opts['ksversion']))
else:
version = ksparser.version.makeVersion()
ks = ksparser.KickstartParser(version)
self.logger.debug('attempting to read kickstart: %s' % kspath)
try:
ks.readKickstart(kspath)
except IOError, e:
raise koji.BuildError("Failed to read kickstart file "
"'%s' : %s" % (kspath, e))
except kserrors.KickstartError, e:
raise koji.BuildError("Failed to parse kickstart file "
"'%s' : %s" % (kspath, e))
return ks
def prepareKickstart(self, kspath):
"""
Process the ks file to be used for controlled image generation. This
method also uploads the modified kickstart file to the task output
area on the hub.
@args:
kspath: a path to a kickstart file
@returns:
a kickstart object with koji-specific modifications
"""
ks = self.readKickstart(kspath)
# Now we do some kickstart manipulation. If the user passed in a repo
# url with --repo, then we substitute that in for the repo(s) specified
# in the kickstart file. If --repo wasn't specified, then we use the
# repo associated with the target passed in initially.
ks.handler.repo.repoList = [] # delete whatever the ks file told us
repo_class = kscontrol.dataMap[ks.version]['RepoData']
# TODO: sensibly use "url" and "repo" commands in kickstart
if self.opts.get('repo'):
# the user used --repo at least once
user_repos = self.opts.get('repo')
index = 0
for user_repo in user_repos:
repo_url = user_repo.replace('$arch', self.arch)
ks.handler.repo.repoList.append(repo_class(
baseurl=repo_url, name='koji-override-%i' % index))
index += 1
else:
# --repo was not given, so we use the target's build repo
path_info = koji.PathInfo(topdir=self.options.topurl)
repopath = path_info.repo(self.repo_info['id'],
self.target_info['build_tag_name'])
baseurl = '%s/%s' % (repopath, self.arch)
self.logger.debug('BASEURL: %s' % baseurl)
ks.handler.repo.repoList.append(repo_class(
baseurl=baseurl, name='koji-override-0'))
return ks
def writeKickstart(self, ksobj, ksname):
"""
Write out the new ks file. Note that things may not be in the same
order and comments in the original ks file may be lost.
@args:
ksobj: a pykickstart object of what we want to write
ksname: file name for the kickstart
@returns:
an absolute path to the kickstart file we wrote
"""
kspath = os.path.join(self.workdir, ksname)
outfile = open(kspath, 'w')
outfile.write(str(ksobj.handler))
outfile.close()
# put the new ksfile in the output directory
if not os.path.exists(kspath):
raise koji.BuildError, "KS file missing: %s" % kspath
self.uploadFile(kspath) # upload the modified ks file
return kspath
def makeConfig(self):
"""
Generate a configuration dict for ImageFactory. This will override
anything in the /etc config files. We do this forcibly so that it is
impossible for Koji to use any image caches or leftover metadata from
other images created by the service.
@args: none
@returns:
a dictionary used for configuring ImageFactory to built an image
the way we want
"""
return {
#Oz specific
'oz_data_dir': os.path.join(self.workdir, 'oz_data'),
'oz_screenshot_dir': os.path.join(self.workdir, 'oz_screenshots'),
#IF specific
'imgdir': os.path.join(self.workdir, 'scratch_images'),
'tmpdir': os.path.join(self.workdir, 'oz-tmp'),
'verbose' : True,
'timeout': 7200,
'output': 'log',
'raw': False,
'debug': True,
'image_manager': 'file',
'plugins': '/etc/imagefactory/plugins.d',
'rhevm_image_format': 'qcow2',
'tdl_require_root_pw': False,
'image_manager_args': {
'storage_path': os.path.join(self.workdir, 'output_image')},
}
def makeTemplate(self, name, inst_tree):
"""
Generate a simple "TDL" for ImageFactory to build an image with.
@args:
name: a name for the image
inst_tree: a string, a URL to the install tree (a compose)
@returns:
An XML string that imagefactory can consume
"""
# we have to split this up so the variable substitution works
# XXX: using a packages section (which we don't) will have IF boot the
# image and attempt to ssh in. This breaks docker image creation.
# TODO: intelligently guess the distro based on the install tree URL
distname, distver = self.parseDistro(self.opts.get('distro'))
template = """<template>
<name>%s</name>
<os>
<name>%s</name>
<version>%s</version>
<arch>%s</arch>
<install type='url'>
<url>%s</url>
</install>
""" % (name, distname, distver, self.arch, inst_tree)
template += """<icicle>
<extra_command>rpm -qa --qf '%{NAME},%{VERSION},%{RELEASE},%{ARCH},%{EPOCH},%{SIZE},%{SIGMD5},%{BUILDTIME}\n'</extra_command>
</icicle>
"""
# TODO: intelligently guess the size based on the kickstart file
template += """</os>
<description>%s OS</description>
<disk>
<size>%sG</size>
</disk>
</template>
""" % (name, self.opts.get('disk_size'))
return template
def parseDistro(self, distro):
"""
Figure out the distribution name and version we are going to build an
image on.
args:
a string of the form: RHEL-X.Y, Fedora-NN, CentOS-X.Y, or SL-X.Y
returns:
a 2-element list, depends on the distro where the split happened
"""
if distro.startswith('RHEL'):
major, minor = distro.split('.')
if major == 'RHEL-5':
minor = 'U' + minor
return major, minor
elif distro.startswith('Fedora'):
return distro.split('-')
elif distro.startswith('CentOS'):
return distro.split('.')
elif distro.startswith('SL'):
return distro.split('.')
else:
raise BuildError('Unknown or supported distro given: %s' % distro)
def fixImageXML(self, format, filename, xmltext):
"""
The XML generated by Oz/ImageFactory knows nothing about the name
or image format conversions Koji does. We fix those values in the
libvirt XML and write the changes out to a file, the path of which
we return.
@args:
format = raw, qcow2, vmdk, etc... a string representation
filename = the name of the XML file we will save this too
xmltext = the libvirt XML to start with
@return:
an absolute path to the modified XML
"""
newxml = xml.dom.minidom.parseString(xmltext)
ename = newxml.getElementsByTagName('name')[0]
ename.firstChild.nodeValue = self.imgname
esources = newxml.getElementsByTagName('source')
for e in esources:
if e.hasAttribute('file'):
e.setAttribute('file', '%s.%s' % (self.imgname, format))
edriver = newxml.getElementsByTagName('driver')[0]
edriver.setAttribute('type', format)
xml_path = os.path.join(self.workdir, filename)
xmlfd = open(xml_path, 'w')
xmlfd.write(newxml.toprettyxml())
xmlfd.close()
return xml_path
def getScreenshot(self):
"""
Locate a screenshot taken by libvirt in the case of build failure,
if it exists. If it does, return the path, else return None.
@args: none
@returns: a path to a screenshot take by libvirt
"""
shotdir = os.path.join(self.workdir, 'oz_screenshots')
screenshot = None
found = glob.glob(os.path.join(shotdir, '*.ppm'))
if len(found) > 0:
screenshot = found[0]
found = glob.glob(os.path.join(shotdir, '*.png'))
if len(found) > 0:
screenshot = found[0]
return screenshot
class BaseImageTask(OzImageTask):
Methods = ['createImage']
_taskWeight = 2.0
def _format_deps(self, formats):
"""
Return a dictionary where the keys are the image formats we need to
build/convert, and the values are booleans that indicate whether the
output should be included in the task results.
Some image formats require others to be processed first, which is why
we have to do this. raw files in particular may not be kept.
"""
supported = ('raw', 'raw-xz', 'vmdk', 'qcow', 'qcow2', 'vdi', 'rhevm-ova', 'vsphere-ova', 'docker')
for f in formats:
if f not in supported:
raise koji.ApplianceError('Invalid format: %s' % f)
f_dict = dict((f, True) for f in formats)
# If a user requests 1 or more image formats (with --format) we do not
# by default include the raw disk image in the results, because it is
# 10G in size. To override this behavior, the user must specify
# "--format raw" in their command. If --format was not used at all,
# then we do include the raw disk image by itself.
if len(formats) == 0:
# we only want a raw disk image (no format option given)
f_dict['raw'] = True
elif 'raw' not in f_dict.keys():
f_dict['raw'] = False
self.logger.debug('Image delivery plan: %s' % f_dict)
return f_dict
def do_images(self, ks, template, inst_tree):
"""
Call out to ImageFactory to build the image(s) we want. Returns a dict
of details for each image type we had to ask ImageFactory to build
"""
fcalls = {'raw': self._buildBase,
'raw-xz': self._buildXZ,
'vmdk': self._buildConvert,
'vdi': self._buildConvert,
'qcow': self._buildConvert,
'qcow2': self._buildConvert,
'rhevm-ova': self._buildOVA,
'vsphere-ova': self._buildOVA,
'docker': self._buildDocker
}
# add a handler to the logger so that we capture ImageFactory's logging
self.fhandler = logging.FileHandler(self.ozlog)
self.bd = BuildDispatcher()
self.tlog = logging.getLogger()
self.tlog.setLevel(logging.DEBUG)
self.tlog.addHandler(self.fhandler)
images = {}
random.seed() # necessary to ensure a unique mac address
params = {'install_script': str(ks.handler),
'offline_icicle': True}
# build the base (raw) image
self.base_img = self._buildBase(template, params)
images['raw'] = {'image': self.base_img.base_image.data,
'icicle': self.base_img.base_image.icicle}
# Do the rest of the image types (everything but raw)
for format in self.formats:
if format == 'raw':
continue
self.logger.info('dispatching %s image builder' % format)
images[format] = fcalls[format](format)
imginfo = self._processXML(images)
self.tlog.removeHandler(self.fhandler)
self.uploadFile(self.ozlog)
return imginfo
def _processXML(self, images):
"""
Produce XML that libvirt can import to create a domain based on image(s)
we produced. We save the location of the XML file in the dictionary
it corresponds to here.
@args:
images - a dict where the keys are image formats, and the values
are dicts with details about the image (location, icicle, etc)
@returns:
a dictionary just like "images" but with a new key called "libvirt"
that points to the path of the XML file for that image
"""
imginfo = {}
for fmt in images.keys():
imginfo[fmt] = images[fmt]
lxml = self.fixImageXML(fmt, 'libvirt-%s-%s.xml' % (fmt, self.arch),
self.base_img.base_image.parameters['libvirt_xml'])
imginfo[fmt]['libvirt'] = lxml
return imginfo
def _checkImageState(self, image):
"""
Query ImageFactory for details of a dispatched image build. If it is
FAILED we raise an exception.
@args:
image - a build dispatcher object returned by a BuildDispatcher
@returns: nothing
"""
if image.target_image:
status = image.target_image.status
details = image.target_image.status_detail['error']
else:
status = image.base_image.status
details = image.base_image.status_detail['error']
self.logger.debug('check image results: %s' % status)
if status == 'FAILED':
scrnshot = self.getScreenshot()
if scrnshot:
ext = scrnshot[-3:]
self.uploadFile(scrnshot, remoteName='screenshot.%s' % ext)
image.os_plugin.abort() # forcibly tear down the VM
# TODO abort when a task is CANCELLED
if not self.session.checkUpload('', os.path.basename(self.ozlog)):
self.tlog.removeHandler(self.fhandler)
self.uploadFile(self.ozlog)
if 'No disk activity' in details:
details = 'Automated install failed or prompted for input. See the screenshot in the task results for more information.'
raise koji.ApplianceError('Image status is %s: %s' %
(status, details))
def _buildBase(self, template, params, wait=True):
"""
Build a base image using ImageFactory. This is a "raw" image.
@args:
template - an XML string for the TDL
params - a dict that controls some ImageFactory settings
wait - call join() on the building thread if True
@returns:
a dict with some metadata about the image (includes an icicle)
"""
# TODO: test the failure case where IF itself throws an exception
# ungracefully (missing a plugin for example)
# may need to still upload ozlog and remove the log handler
self.logger.info('dispatching a baseimg builder')
self.logger.debug('templates: %s' % template)
self.logger.debug('params: %s' % params)
base = self.bd.builder_for_base_image(template, parameters=params)
if wait:
base.base_thread.join()
self._checkImageState(base)
return base
def _buildXZ(self, format):
"""
Use xz to compress a raw disk image. This is very straightforward.
@args:
format - a string representing the image format, "raw-xz"
@returns:
a dict with some metadata about the image
"""
newimg = os.path.join(self.workdir, self.imgname + '.raw.xz')
rawimg = os.path.join(self.workdir, self.imgname + '.raw')
cmd = ['/bin/cp', self.base_img.base_image.data, rawimg]
conlog = os.path.join(self.workdir,
'xz-cp-%s-%s.log' % (format, self.arch))
log_output(self.session, cmd[0], cmd, conlog, self.getUploadDir(),
logerror=1)
cmd = ['/usr/bin/xz', '-z', rawimg]
conlog = os.path.join(self.workdir,
'xz-%s-%s.log' % (format, self.arch))
log_output(self.session, cmd[0], cmd, conlog, self.getUploadDir(),
logerror=1)
return {'image': newimg}
def _buildOVA(self, format):
"""
Build an OVA target image. This is a format supported by RHEV and
vSphere
@args:
format - a string representing the image format, "rhevm-ova"
@returns:
a dict with some metadata about the image
"""
img_opts = {}
if self.opts.get('ova_option'):
img_opts = dict([o.split('=') for o in self.opts.get('ova_option')])
targ = self._do_target_image(self.base_img.base_image.identifier,
format.replace('-ova', ''))
targ2 = self._do_target_image(targ.target_image.identifier, 'OVA',
img_opts=img_opts)
return {'image': targ2.target_image.data}
def _buildDocker(self, format):
"""
Build a base docker image. This image will be tagged with the NVR.A
automatically because we name it that way in the ImageFactory TDL.
@args:
format - the string "docker"
@returns:
a dict with some metadata about the image
"""
img_opts = {'compress': 'gzip'}
targ = self._do_target_image(self.base_img.base_image.identifier,
'docker', img_opts=img_opts)
return {'image': targ.target_image.data}
def _do_target_image(self, base_id, image_type, img_opts={}):
"""
A generic method for building what ImageFactory calls "target images".
These are images based on a raw disk that was built before using the
_buildBase method.
@args:
base_id - a string ID of the image to build off of
image_type - a string representing the target type. ImageFactory
uses this to figure out what plugin to run
img_opts - a dict of additional options that specific to the target
type we pass in via image_type
@returns:
A Builder() object from ImageFactory that contains information
about the image building include state and progress.
"""
# TODO: test the failure case where IF itself throws an exception
# ungracefully (missing a plugin for example)
# may need to still upload ozlog and remove the log handler
self.logger.debug('img_opts: %s' % img_opts)
target = self.bd.builder_for_target_image(image_type,
image_id=base_id, template=None, parameters=img_opts)
target.target_thread.join()
self._checkImageState(target)
return target
def _buildConvert(self, format):
"""
Build an image by converting the format using qemu-img. This is method
enables a variety of formats like qcow, qcow2, vmdk, and vdi.
@args:
format - a string representing the image format, "qcow2"
@returns
a dict with some metadata about the image
"""
newimg = os.path.join(self.workdir, self.imgname + '.%s' % format)
cmd = ['/usr/bin/qemu-img', 'convert', '-f', 'raw', '-O',
format, self.base_img.base_image.data, newimg]
if format in ('qcow', 'qcow2'):
cmd.insert(2, '-c') # enable compression for qcow images
conlog = os.path.join(self.workdir,
'qemu-img-%s-%s.log' % (format, self.arch))
log_output(self.session, cmd[0], cmd, conlog,
self.getUploadDir(), logerror=1)
return {'image': newimg}
def handler(self, name, version, release, arch, target_info, build_tag, repo_info, inst_tree, opts=None):
if opts == None:
opts = {}
self.arch = arch
self.target_info = target_info
self.repo_info = repo_info
self.opts = opts
self.formats = self._format_deps(opts.get('format'))
# First, prepare the kickstart to use the repos we tell it
kspath = self.fetchKickstart()
ks = self.prepareKickstart(kspath)
kskoji = self.writeKickstart(ks,
os.path.join(self.workdir, 'koji-%s-%i-base.ks' %
(self.target_info['build_tag_name'], self.id)))
# auto-generate a TDL file and config dict for ImageFactory
self.imgname = '%s-%s-%s.%s' % (name, version, release, self.arch)
template = self.makeTemplate(self.imgname, inst_tree)
self.logger.debug('oz template: %s' % template)
config = self.makeConfig()
self.logger.debug('IF config object: %s' % config)
ApplicationConfiguration(configuration=config)
tdl_path = os.path.join(self.workdir, 'tdl-%s.xml' % self.arch)
tdl = open(tdl_path, 'w')
tdl.write(template)
tdl.close()
self.uploadFile(tdl_path)
# ImageFactory picks a port to the guest VM using a rolling integer.
# This is a problem for concurrency, so we override the port it picks
# here using the task ID. (not a perfect solution but good enough:
# the likelihood of image tasks clashing here is very small)
rm = ReservationManager()
rm._listen_port = rm.MIN_PORT + self.id % (rm.MAX_PORT - rm.MIN_PORT)
ozlogname = 'oz-%s.log' % self.arch
self.ozlog = os.path.join(self.workdir, ozlogname)
# invoke the image builds
images = self.do_images(ks, template, inst_tree)
images['raw']['tdl'] = os.path.basename(tdl_path),
# structure the results to pass back to the hub:
imgdata = {
'arch': self.arch,
'task_id': self.id,
'logs': [ozlogname],
'name': name,
'version': version,
'release': release,
'rpmlist': [],
'files': [os.path.basename(tdl_path),
os.path.basename(kspath),
os.path.basename(kskoji)]
}
# record the RPMs that were installed
if not opts.get('scratch'):
fields = ('name', 'version', 'release', 'arch', 'epoch', 'size',
'payloadhash', 'buildtime')
icicle = xml.dom.minidom.parseString(images['raw']['icicle'])
self.logger.debug('ICICLE: %s' % images['raw']['icicle'])
for p in icicle.getElementsByTagName('extra'):
bits = p.firstChild.nodeValue.split(',')
rpm = {
'name': bits[0],
'version': bits[1],
'release': bits[2],
'arch': bits[3],
# epoch is a special case, as usual
'size': int(bits[5]),
'payloadhash': bits[6],
'buildtime': int(bits[7])
}
if rpm['name'] in ['buildsys-build', 'gpg-pubkey']:
continue
if bits[4] == '(none)':
rpm['epoch'] = None
else:
rpm['epoch'] = int(bits[4])
imgdata['rpmlist'].append(rpm)
# TODO: hack to make this work for now, need to refactor
br = BuildRoot(self.session, self.options, build_tag, self.arch,
self.id, repo_id=self.repo_info['id'])
br.markExternalRPMs(imgdata['rpmlist'])
# upload the results
for format in (f for f in self.formats.keys() if self.formats[f]):
newimg = images[format]['image']
if 'ova' in format or format == 'raw-xz':
newname = self.imgname + '.' + format.replace('-', '.')
elif format == 'docker':
newname = self.imgname + '.' + 'tar.gz'
else:
newname = self.imgname + '.' + format
if format != 'docker':
lxml = images[format]['libvirt']
imgdata['files'].append(os.path.basename(lxml))
self.uploadFile(lxml)
imgdata['files'].append(os.path.basename(newname))
self.uploadFile(newimg, remoteName=newname)
# no need to delete anything since self.workdir will get scrubbed
return imgdata
class BuildIndirectionImageTask(OzImageTask):
Methods = ['indirectionimage']
# So, these are copied directly from the base image class
# Realistically, we want to inherit methods from both BuildImageTask
# and OzImageTask.
# TODO: refactor - my initial suggestion would be to have OzImageTask
# be a child of BuildImageTask
def initImageBuild(self, name, version, release, target_info, opts):
"""create a build object for this image build"""
pkg_cfg = self.session.getPackageConfig(target_info['dest_tag_name'],
name)
self.logger.debug("%r" % pkg_cfg)
if not opts.get('skip_tag') and not opts.get('scratch'):
# Make sure package is on the list for this tag
if pkg_cfg is None:
raise koji.BuildError, "package (image) %s not in list for tag %s" % (name, target_info['dest_tag_name'])
elif pkg_cfg['blocked']:
raise koji.BuildError, "package (image) %s is blocked for tag %s" % (name, target_info['dest_tag_name'])
return self.session.host.initImageBuild(self.id,
dict(name=name, version=version, release=release, epoch=0))
def getRelease(self, name, ver):
"""return the next available release number for an N-V"""
return self.session.getNextRelease(dict(name=name, version=ver))
# END inefficient base image task method copies
def fetchHubOrSCM(self, filepath, fileurl):
"""
Retrieve a file either from the hub or a remote scm
If fileurl is None we assume we are being asked to retrieve from
the hub and that filepath is relative to /mnt/koji/work.
if fileurl contains a value we assume a remote SCM.
If retrieving remote we assume that filepath is the file name and
fileurl is the path in the remote SCM where that file can be found.
@returns: absolute path to the retrieved file
"""
# TODO: A small change to the base image build code could allow this method
# to be shared between both tasks. I wanted this initial implementation
# to be entirely self contained. Revisit if anyone feels like a refactor.
self.logger.debug("filepath = %s" % filepath)
if fileurl:
scm = SCM(fileurl)
scm.assert_allowed(self.options.allowed_scms)
logfile = os.path.join(self.workdir, 'checkout.log')
scmsrcdir = scm.checkout(self.workdir, self.session,
self.getUploadDir(), logfile)
final_path = os.path.join(scmsrcdir, os.path.basename(filepath))
else:
tops = dict([(k, getattr(self.options, k)) for k in 'topurl','topdir'])
remote_fileobj = koji.openRemoteFile(filepath, **tops)
final_path = os.path.join(self.workdir, os.path.basename(filepath))
final_fileobj = open(final_path, 'w')
final_fileobj.write(remote_fileobj.read())
final_fileobj.close()
self.logger.debug('uploading retrieved file from here: %s' % final_path)
self.uploadFile(final_path) # upload the original ks file
return final_path # absolute path to the ks file
def handler(self, opts):
"""Governing task for building an image with two other images using Factory Indirection"""
# TODO: Add mode of operation where full build details are given for
# either base or utility or both, then spawn subtasks to do them first
def _task_to_image(task_id):
""" Take a task ID and turn it into an Image Factory Base Image object """
pim = PersistentImageManager.default_manager()
taskinfo = self.session.getTaskInfo(task_id)
taskstate = koji.TASK_STATES[taskinfo['state']].lower()
if taskstate != 'closed':
raise koji.BuildError("Input task (%d) must be in closed state - current state is (%s)" %
(task_id, taskstate))
taskmethod = taskinfo['method']
if taskmethod != "createImage":
raise koji.BuildError("Input task method must be 'createImage' - actual method (%s)" %
(taskmethod))
result = self.session.getTaskResult(task_id)
files = self.session.listTaskOutput(task_id)
# This approach works for both scratch and saved/formal images
# The downside is that we depend on the output file naming convention
def _match_name(inlist, namere):
for filename in inlist:
if re.search(namere, filename):
return filename
task_diskimage = _match_name(result['files'], ".*qcow2$")
task_tdl = _match_name(result['files'], "tdl.*xml")
task_dir = os.path.join(koji.pathinfo.work(),koji.pathinfo.taskrelpath(task_id))
diskimage_full = os.path.join(task_dir, task_diskimage)
tdl_full = os.path.join(task_dir, task_tdl)
if not (os.path.isfile(diskimage_full) and os.path.isfile(tdl_full)):
raise koji.BuildError("Missing TDL or qcow2 image for task (%d) - possible expired scratch build" % (task_id))
# The sequence to recreate a valid persistent image is as follows
# Create a new BaseImage object
factory_base_image = BaseImage()
# Add it to the persistence layer
pim.add_image(factory_base_image)
# Now replace the data and template with the files referenced above
# and mark it as a complete image
# Factory doesn't attempt to modify a disk image after it is COMPLETE so
# this will work safely on read-only NFS mounts
factory_base_image.data = diskimage_full
factory_base_image.template = open(tdl_full).read()
factory_base_image.status = 'COMPLETE'
# Now save it
pim.save_image(factory_base_image)
# We can now reference this object directly or via its UUID in persistent storage
return factory_base_image
def _nvr_to_image(nvr, arch):
""" Take a build ID or NVR plus arch and turn it into an Image Factory Base Image object """
pim = PersistentImageManager.default_manager()
build = self.session.getBuild(nvr)
if not build:
raise koji.BuildError("Could not find build for (%s)" % (nvr))
buildarchives = self.session.listArchives(build['id'])
if not buildarchives:
raise koji.Builderror("Could not retrieve archives for build (%s) from NVR (%s)" %
(build['id'], nvr))
buildfiles = [ x['filename'] for x in buildarchives ]
builddir = koji.pathinfo.imagebuild(build)
def _match_name(inlist, namere):
for filename in inlist:
if re.search(namere, filename):
return filename
build_diskimage = _match_name(buildfiles, ".*%s\.qcow2$" % (arch))
build_tdl = _match_name(buildfiles, "tdl.%s\.xml" % (arch))
diskimage_full = os.path.join(builddir, build_diskimage)
tdl_full = os.path.join(builddir, build_tdl)
if not (os.path.isfile(diskimage_full) and os.path.isfile(tdl_full)):
raise koji.BuildError("Missing TDL (%s) or qcow2 (%s) image for image (%s) - this should never happen" %
(build_tdl, build_diskimage, nvr))
# The sequence to recreate a valid persistent image is as follows
# Create a new BaseImage object
factory_base_image = BaseImage()
# Add it to the persistence layer
pim.add_image(factory_base_image)
# Now replace the data and template with the files referenced above
# and mark it as a complete image
# Factory doesn't attempt to modify a disk image after it is COMPLETE so
# this will work safely on read-only NFS mounts
factory_base_image.data = diskimage_full
factory_base_image.template = open(tdl_full).read()
factory_base_image.status = 'COMPLETE'
# Now save it
pim.save_image(factory_base_image)
# We can now reference this object directly or via its UUID in persistent storage
return factory_base_image
if opts == None:
opts = {}
self.opts = opts
config = self.makeConfig()
self.logger.debug('IF config object: %s' % config)
ApplicationConfiguration(configuration=config)
ozlogname = 'oz-indirection.log'
ozlog = os.path.join(self.workdir, ozlogname)
# END shared code
fhandler = logging.FileHandler(ozlog)
bd = BuildDispatcher()
tlog = logging.getLogger()
tlog.setLevel(logging.DEBUG)
tlog.addHandler(fhandler)
# TODO: Copy-paste from BaseImage - refactor
target_info = self.session.getBuildTarget(opts['target'], strict=True)
build_tag = target_info['build_tag']
repo_info = self.getRepo(build_tag)
name = opts['name']
version = opts['version']
release = opts['release']
# TODO: Another mostly copy-paste
if not release:
release = self.getRelease(name, version)
if '-' in version:
raise koji.ApplianceError('The Version may not have a hyphen')
if '-' in release:
raise koji.ApplianceError('The Release may not have a hyphen')
indirection_template = self.fetchHubOrSCM(opts.get('indirection_template'),
opts.get('indirection_template_url'))
self.logger.debug('Got indirection template %s' % (indirection_template))
try:
if opts['utility_image_build']:
utility_factory_image = _nvr_to_image(opts['utility_image_build'], opts['arch'])
else:
utility_factory_image = _task_to_image(int(opts['utility_image_task']))
if opts['base_image_build']:
base_factory_image = _nvr_to_image(opts['base_image_build'], opts['arch'])
else:
base_factory_image = _task_to_image(int(opts['base_image_task']))
except Exception, e:
self.logger.exception(e)
raise
# OK - We have a template and two input images - lets build
bld_info = None
if not opts['scratch']:
bld_info = self.initImageBuild(name, version, release,
target_info, opts)
try:
return self._do_indirection(opts, base_factory_image, utility_factory_image,
indirection_template, tlog, ozlog, fhandler,
bld_info, target_info, bd)
except:
if not opts.get('scratch'):
#scratch builds do not get imported
if bld_info:
self.session.host.failBuild(self.id, bld_info['id'])
# reraise the exception
raise
def _do_indirection(self, opts, base_factory_image, utility_factory_image,
indirection_template, tlog, ozlog, fhandler, bld_info,
target_info, bd):
# TODO: The next several lines are shared with the handler for other Factory tasks
# refactor in such a way that this can be a helper in OzImageTask
# ImageFactory picks a port to the guest VM using a rolling integer.
# This is a problem for concurrency, so we override the port it picks
# here using the task ID. (not a perfect solution but good enough:
# the likelihood of image tasks clashing here is very small)
rm = ReservationManager()
rm._listen_port = rm.MIN_PORT + self.id % (rm.MAX_PORT - rm.MIN_PORT)
utility_customizations = open(indirection_template).read()
results_loc = opts.get('results_loc', None)
if results_loc[0] != "/":
results_loc = "/" + results_loc
params = {'utility_image': str(utility_factory_image.identifier),
'utility_customizations': utility_customizations,
'results_location': results_loc }
random.seed() # necessary to ensure a unique mac address
try:
try:
# Embedded deep debug option - if template is just the string MOCK
# skip the actual build and create a mock target image instead
if utility_customizations.strip() == "MOCK":
target = Builder()
target_image = TargetImage()
pim = PersistentImageManager.default_manager()
pim.add_image(target_image)
target.target_image = target_image
open(target_image.data, "w").write("Mock build from task ID: %s" %
(str(self.id)))
target_image.status='COMPLETE'
else:
target = bd.builder_for_target_image('indirection',
image_id=base_factory_image.identifier,
parameters=params)
target.target_thread.join()
except Exception, e:
self.logger.debug("Exception encountered during target build")
self.logger.exception(e)
finally:
# upload log even if we failed to help diagnose an issue
tlog.removeHandler(fhandler)
self.uploadFile(ozlog)
self.logger.debug('Target image results: %s' % target.target_image.status)
if target.target_image.status == 'FAILED':
# TODO abort when a task is CANCELLED
if not self.session.checkUpload('', os.path.basename(ozlog)):
tlog.removeHandler(fhandler)
self.uploadFile(ozlog)
raise koji.ApplianceError('Image status is %s: %s' %
(target.target_image.status, target.target_image.status_detail))
self.uploadFile(target.target_image.data, remoteName=os.path.basename(results_loc))
myresults = { }
myresults['task_id'] = self.id
myresults['files'] = [ os.path.basename(results_loc) ]
myresults['logs'] = [ os.path.basename(ozlog) ]
myresults['arch'] = opts['arch']
# TODO: This should instead track the two input images: base and utility
myresults['rpmlist'] = [ ]
# This is compatible with some helper methods originally implemented for the base
# image build. In the original usage, the dict contains an entry per build arch
# TODO: If adding multiarch support, keep this in mind
results = { str(self.id): myresults }
self.logger.debug('Image Results for hub: %s' % results)
if opts['scratch']:
self.session.host.moveImageBuildToScratch(self.id, results)
else:
self.session.host.completeImageBuild(self.id, bld_info['id'],
results)
# tag it
if not opts.get('scratch') and not opts.get('skip_tag'):
tag_task_id = self.session.host.subtask(method='tagBuild',
arglist=[target_info['dest_tag'], bld_info['id'], False, None, True],
label='tag', parent=self.id, arch='noarch')
self.wait(tag_task_id)
# report results
report = ''
if opts.get('scratch'):
respath = ', '.join(
[os.path.join(koji.pathinfo.work(), koji.pathinfo.taskrelpath(tid)) for tid in [self.id] ])
report += 'Scratch '
else:
respath = koji.pathinfo.imagebuild(bld_info)
report += 'image build results in: %s' % respath
return report
class BuildSRPMFromSCMTask(BaseBuildTask):
Methods = ['buildSRPMFromSCM']
_taskWeight = 1.0
def spec_sanity_checks(self, filename):
spec = open(filename).read()
for tag in ("Packager", "Distribution", "Vendor"):
if re.match("%s:" % tag, spec, re.M):
raise koji.BuildError, "%s is not allowed to be set in spec file" % tag
for tag in ("packager", "distribution", "vendor"):
if re.match("%%define\s+%s\s+" % tag, spec, re.M):
raise koji.BuildError, "%s is not allowed to be defined in spec file" % tag
def patch_scm_source(self, sourcedir, logfile, opts):
# override if desired
pass
def checkHost(self, hostdata):
tag = self.params[1]
return self.checkHostArch(tag, hostdata)
def handler(self, url, build_tag, opts=None):
# will throw a BuildError if the url is invalid
scm = SCM(url)
scm.assert_allowed(self.options.allowed_scms)
if opts is None:
opts = {}
repo_id = opts.get('repo_id')
if not repo_id:
raise koji.BuildError, "A repo id must be provided"
repo_info = self.session.repoInfo(repo_id, strict=True)
event_id = repo_info['create_event']
build_tag = self.session.getTag(build_tag, strict=True, event=event_id)
# need DNS in the chroot because "make srpm" may need to contact
# a SCM or lookaside cache to retrieve the srpm contents
rootopts = {'install_group': 'srpm-build',
'setup_dns': True,
'repo_id': repo_id}
br_arch = self.find_arch('noarch', self.session.host.getHost(), self.session.getBuildConfig(build_tag['id'], event=event_id))
broot = BuildRoot(self.session, self.options, build_tag['id'], br_arch, self.id, **rootopts)
broot.workdir = self.workdir
self.logger.debug("Initializing buildroot")
broot.init()
# Setup files and directories for SRPM creation
# We can't put this under the mock homedir because that directory
# is completely blown away and recreated on every mock invocation
scmdir = broot.rootdir() + '/tmp/scmroot'
koji.ensuredir(scmdir)
logfile = self.workdir + '/checkout.log'
uploadpath = self.getUploadDir()
# Check out spec file, etc. from SCM
sourcedir = scm.checkout(scmdir, self.session, uploadpath, logfile)
# chown the sourcedir and everything under it to the mockuser
# so we can build the srpm as non-root
uid = pwd.getpwnam(self.options.mockuser)[2]
# rpmbuild seems to complain if it's running in the "mock" group but
# files are in a different group
gid = grp.getgrnam('mock')[2]
self.chownTree(scmdir, uid, gid)
# Hook for patching spec file in place
self.patch_scm_source(sourcedir, logfile, opts)
# Find and verify that there is only one spec file.
spec_files = glob.glob("%s/*.spec" % sourcedir)
if not spec_files and self.options.support_rpm_source_layout:
# also check SPECS dir
spec_files = glob.glob("%s/SPECS/*.spec" % sourcedir)
if len(spec_files) == 0:
raise koji.BuildError("No spec file found")
elif len(spec_files) > 1:
raise koji.BuildError("Multiple spec files found: %s" % spec_files)
spec_file = spec_files[0]
# Run spec file sanity checks. Any failures will throw a BuildError
self.spec_sanity_checks(spec_file)
#build srpm
self.logger.debug("Running srpm build")
broot.build_srpm(spec_file, sourcedir, scm.source_cmd)
srpms = glob.glob('%s/*.src.rpm' % broot.resultdir())
if len(srpms) == 0:
raise koji.BuildError, "No srpms found in %s" % sourcedir
elif len(srpms) > 1:
raise koji.BuildError, "Multiple srpms found in %s: %s" % (sourcedir, ", ".join(srpms))
else:
srpm = srpms[0]
# check srpm name
h = koji.get_rpm_header(srpm)
name = h[rpm.RPMTAG_NAME]
version = h[rpm.RPMTAG_VERSION]
release = h[rpm.RPMTAG_RELEASE]
srpm_name = "%(name)s-%(version)s-%(release)s.src.rpm" % locals()
if srpm_name != os.path.basename(srpm):
raise koji.BuildError, 'srpm name mismatch: %s != %s' % (srpm_name, os.path.basename(srpm))
#upload srpm and return
self.uploadFile(srpm)
brootid = broot.id
log_files = glob.glob('%s/*.log' % broot.resultdir())
broot.expire()
return {'srpm': "%s/%s" % (uploadpath, srpm_name),
'logs': ["%s/%s" % (uploadpath, os.path.basename(f))
for f in log_files],
'brootid': brootid,
}
class TagNotificationTask(BaseTaskHandler):
Methods = ['tagNotification']
_taskWeight = 0.1
message_templ = \
"""From: %(from_addr)s\r
Subject: %(nvr)s %(result)s %(operation)s by %(user_name)s\r
To: %(to_addrs)s\r
X-Koji-Package: %(pkg_name)s\r
X-Koji-NVR: %(nvr)s\r
X-Koji-User: %(user_name)s\r
X-Koji-Status: %(status)s\r
%(tag_headers)s\r
\r
Package: %(pkg_name)s\r
NVR: %(nvr)s\r
User: %(user_name)s\r
Status: %(status)s\r
%(operation_details)s\r
%(nvr)s %(result)s %(operation)s by %(user_name)s\r
%(failure_info)s\r
"""
def handler(self, recipients, is_successful, tag_info, from_info, build_info, user_info, ignore_success=None, failure_msg=''):
if len(recipients) == 0:
self.logger.debug('task %i: no recipients, not sending notifications', self.id)
return
if ignore_success and is_successful:
self.logger.debug('task %i: tag operation successful and ignore success is true, not sending notifications', self.id)
return
build = self.session.getBuild(build_info)
user = self.session.getUser(user_info)
pkg_name = build['package_name']
nvr = koji.buildLabel(build)
user_name = user['name']
from_addr = self.options.from_addr
to_addrs = ', '.join(recipients)
operation = '%(action)s'
operation_details = 'Tag Operation: %(action)s\r\n'
tag_headers = ''
if from_info:
from_tag = self.session.getTag(from_info)
from_tag_name = from_tag['name']
operation += ' from %s' % from_tag_name
operation_details += 'From Tag: %s\r\n' % from_tag_name
tag_headers += 'X-Koji-Tag: %s' % from_tag_name
action = 'untagged'
if tag_info:
tag = self.session.getTag(tag_info)
tag_name = tag['name']
operation += ' into %s' % tag_name
operation_details += 'Into Tag: %s\r\n' % tag_name
if tag_headers:
tag_headers += '\r\n'
tag_headers += 'X-Koji-Tag: %s' % tag_name
action = 'tagged'
if tag_info and from_info:
action = 'moved'
operation = operation % locals()
operation_details = operation_details % locals()
if is_successful:
result = 'successfully'
status = 'complete'
failure_info = ''
else:
result = 'unsuccessfully'
status = 'failed'
failure_info = "Operation failed with the error:\r\n %s\r\n" % failure_msg
message = self.message_templ % locals()
# ensure message is in UTF-8
message = koji.fixEncoding(message)
server = smtplib.SMTP(self.options.smtphost)
#server.set_debuglevel(True)
server.sendmail(from_addr, recipients, message)
server.quit()
return 'sent notification of tag operation %i to: %s' % (self.id, to_addrs)
class BuildNotificationTask(BaseTaskHandler):
Methods = ['buildNotification']
_taskWeight = 0.1
# XXX externalize these templates somewhere
subject_templ = """Package: %(build_nvr)s Tag: %(dest_tag)s Status: %(status)s Built by: %(build_owner)s"""
message_templ = \
"""From: %(from_addr)s\r
Subject: %(subject)s\r
To: %(to_addrs)s\r
X-Koji-Tag: %(dest_tag)s\r
X-Koji-Package: %(build_pkg_name)s\r
X-Koji-Builder: %(build_owner)s\r
X-Koji-Status: %(status)s\r
\r
Package: %(build_nvr)s\r
Tag: %(dest_tag)s\r
Status: %(status)s%(cancel_info)s\r
Built by: %(build_owner)s\r
ID: %(build_id)i\r
Started: %(creation_time)s\r
Finished: %(completion_time)s\r
%(changelog)s\r
%(failure)s\r
%(output)s\r
Task Info: %(weburl)s/taskinfo?taskID=%(task_id)i\r
Build Info: %(weburl)s/buildinfo?buildID=%(build_id)i\r
"""
def _getTaskData(self, task_id, data=None):
if not data:
data = {}
taskinfo = self.session.getTaskInfo(task_id)
if not taskinfo:
# invalid task_id
return data
if taskinfo['host_id']:
hostinfo = self.session.getHost(taskinfo['host_id'])
else:
hostinfo = None
result = None
try:
result = self.session.getTaskResult(task_id)
except:
excClass, result = sys.exc_info()[:2]
if hasattr(result, 'faultString'):
result = result.faultString
else:
result = '%s: %s' % (excClass.__name__, result)
result = result.strip()
# clear the exception, since we're just using
# it for display purposes
sys.exc_clear()
if not result:
result = 'Unknown'
files = self.session.listTaskOutput(task_id)
logs = [filename for filename in files if filename.endswith('.log')]
rpms = [filename for filename in files if filename.endswith('.rpm') and not filename.endswith('.src.rpm')]
srpms = [filename for filename in files if filename.endswith('.src.rpm')]
misc = [filename for filename in files if filename not in logs + rpms + srpms]
logs.sort()
rpms.sort()
misc.sort()
data[task_id] = {}
data[task_id]['id'] = taskinfo['id']
data[task_id]['method'] = taskinfo['method']
data[task_id]['arch'] = taskinfo['arch']
data[task_id]['build_arch'] = taskinfo['label']
data[task_id]['host'] = hostinfo and hostinfo['name'] or None
data[task_id]['state'] = koji.TASK_STATES[taskinfo['state']].lower()
data[task_id]['result'] = result
data[task_id]['request'] = self.session.getTaskRequest(task_id)
data[task_id]['logs'] = logs
data[task_id]['rpms'] = rpms
data[task_id]['srpms'] = srpms
data[task_id]['misc'] = misc
children = self.session.getTaskChildren(task_id)
for child in children:
data = self._getTaskData(child['id'], data)
return data
def handler(self, recipients, build, target, weburl):
if len(recipients) == 0:
self.logger.debug('task %i: no recipients, not sending notifications', self.id)
return
build_pkg_name = build['package_name']
build_pkg_evr = '%s%s-%s' % ((build['epoch'] and str(build['epoch']) + ':' or ''), build['version'], build['release'])
build_nvr = koji.buildLabel(build)
build_id = build['id']
build_owner = build['owner_name']
# target comes from session.py:_get_build_target()
dest_tag = None
if target is not None:
dest_tag = target['dest_tag_name']
status = koji.BUILD_STATES[build['state']].lower()
creation_time = koji.formatTimeLong(build['creation_time'])
completion_time = koji.formatTimeLong(build['completion_time'])
task_id = build['task_id']
task_data = self._getTaskData(task_id)
cancel_info = ''
failure_info = ''
if build['state'] == koji.BUILD_STATES['CANCELED']:
# The owner of the buildNotification task is the one
# who canceled the task, it turns out.
this_task = self.session.getTaskInfo(self.id)
if this_task['owner']:
canceler = self.session.getUser(this_task['owner'])
cancel_info = "\r\nCanceled by: %s" % canceler['name']
elif build['state'] == koji.BUILD_STATES['FAILED']:
failure_data = task_data[task_id]['result']
failed_hosts = ['%s (%s)' % (task['host'], task['arch']) for task in task_data.values() if task['host'] and task['state'] == 'failed']
failure_info = "\r\n%s (%d) failed on %s:\r\n %s" % (build_nvr, build_id,
', '.join(failed_hosts),
failure_data)
failure = failure_info or cancel_info or ''
tasks = {'failed' : [task for task in task_data.values() if task['state'] == 'failed'],
'canceled' : [task for task in task_data.values() if task['state'] == 'canceled'],
'closed' : [task for task in task_data.values() if task['state'] == 'closed']}
srpms = []
for taskinfo in task_data.values():
for srpmfile in taskinfo['srpms']:
srpms.append(srpmfile)
srpms = self.uniq(srpms)
srpms.sort()
if srpms:
output = "SRPMS:\r\n"
for srpm in srpms:
output += " %s" % srpm
output += "\r\n\r\n"
else:
output = ''
pathinfo = koji.PathInfo(topdir=self.options.topurl)
buildurl = pathinfo.build(build)
# list states here to make them go in the correct order
for task_state in ['failed', 'canceled', 'closed']:
if tasks[task_state]:
output += "%s tasks:\r\n" % task_state.capitalize()
output += "%s-------\r\n\r\n" % ("-" * len(task_state))
for task in tasks[task_state]:
output += "Task %s" % task['id']
if task['host']:
output += " on %s\r\n" % task['host']
else:
output += "\r\n"
output += "Task Type: %s\r\n" % koji.taskLabel(task)
for filetype in ['logs', 'rpms', 'misc']:
if task[filetype]:
output += "%s:\r\n" % filetype
for file in task[filetype]:
if filetype == 'rpms':
output += " %s\r\n" % '/'.join([buildurl, task['build_arch'], file])
elif filetype == 'logs':
if tasks[task_state] != 'closed':
output += " %s/getfile?taskID=%s&name=%s\r\n" % (weburl, task['id'], file)
else:
output += " %s\r\n" % '/'.join([buildurl, 'data', 'logs', task['build_arch'], file])
elif task[filetype] == 'misc':
output += " %s/getfile?taskID=%s&name=%s\r\n" % (weburl, task['id'], file)
output += "\r\n"
output += "\r\n"
changelog = koji.util.formatChangelog(self.session.getChangelogEntries(build_id, queryOpts={'limit': 3})).replace("\n","\r\n")
if changelog:
changelog = "Changelog:\r\n%s" % changelog
from_addr = self.options.from_addr
to_addrs = ', '.join(recipients)
subject = self.subject_templ % locals()
message = self.message_templ % locals()
# ensure message is in UTF-8
message = koji.fixEncoding(message)
server = smtplib.SMTP(self.options.smtphost)
# server.set_debuglevel(True)
server.sendmail(from_addr, recipients, message)
server.quit()
return 'sent notification of build %i to: %s' % (build_id, to_addrs)
def uniq(self, items):
"""Remove duplicates from the list of items, and sort the list."""
m = dict(zip(items, [1] * len(items)))
l = m.keys()
l.sort()
return l
class NewRepoTask(BaseTaskHandler):
Methods = ['newRepo']
_taskWeight = 0.1
def handler(self, tag, event=None, src=False, debuginfo=False):
tinfo = self.session.getTag(tag, strict=True, event=event)
kwargs = {}
if event is not None:
kwargs['event'] = event
if src:
kwargs['with_src'] = True
if debuginfo:
kwargs['with_debuginfo'] = True
repo_id, event_id = self.session.host.repoInit(tinfo['id'], **kwargs)
path = koji.pathinfo.repo(repo_id, tinfo['name'])
if not os.path.isdir(path):
raise koji.GenericError, "Repo directory missing: %s" % path
arches = []
for fn in os.listdir(path):
if fn != 'groups' and os.path.isfile("%s/%s/pkglist" % (path, fn)):
arches.append(fn)
#see if we can find a previous repo to update from
#only shadowbuild tags should start with SHADOWBUILD, their repos are auto
#expired. so lets get the most recent expired tag for newRepo shadowbuild tasks.
if tinfo['name'].startswith('SHADOWBUILD'):
oldrepo = self.session.getRepo(tinfo['id'], state=koji.REPO_EXPIRED)
else:
oldrepo = self.session.getRepo(tinfo['id'], state=koji.REPO_READY)
subtasks = {}
for arch in arches:
arglist = [repo_id, arch, oldrepo]
subtasks[arch] = self.session.host.subtask(method='createrepo',
arglist=arglist,
label=arch,
parent=self.id,
arch='noarch')
# wait for subtasks to finish
results = self.wait(subtasks.values(), all=True, failany=True)
data = {}
for (arch, task_id) in subtasks.iteritems():
data[arch] = results[task_id]
self.logger.debug("DEBUG: %r : %r " % (arch,data[arch],))
kwargs = {}
if event is not None:
kwargs['expire'] = True
self.session.host.repoDone(repo_id, data, **kwargs)
return repo_id, event_id
class CreaterepoTask(BaseTaskHandler):
Methods = ['createrepo']
_taskWeight = 1.5
def handler(self, repo_id, arch, oldrepo):
#arch is the arch of the repo, not the task
rinfo = self.session.repoInfo(repo_id, strict=True)
if rinfo['state'] != koji.REPO_INIT:
raise koji.GenericError, "Repo %(id)s not in INIT state (got %(state)s)" % rinfo
self.repo_id = rinfo['id']
self.pathinfo = koji.PathInfo(self.options.topdir)
toprepodir = self.pathinfo.repo(repo_id, rinfo['tag_name'])
self.repodir = '%s/%s' % (toprepodir, arch)
if not os.path.isdir(self.repodir):
raise koji.GenericError, "Repo directory missing: %s" % self.repodir
groupdata = os.path.join(toprepodir, 'groups', 'comps.xml')
#set up our output dir
self.outdir = '%s/repo' % self.workdir
self.datadir = '%s/repodata' % self.outdir
pkglist = os.path.join(self.repodir, 'pkglist')
if os.path.getsize(pkglist) == 0:
pkglist = None
self.create_local_repo(rinfo, arch, pkglist, groupdata, oldrepo)
external_repos = self.session.getExternalRepoList(rinfo['tag_id'], event=rinfo['create_event'])
if external_repos:
self.merge_repos(external_repos, arch, groupdata)
elif pkglist is None:
fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w')
fo.write("This repo is empty because its tag has no content for this arch\n")
fo.close()
uploadpath = self.getUploadDir()
files = []
for f in os.listdir(self.datadir):
files.append(f)
self.session.uploadWrapper('%s/%s' % (self.datadir, f), uploadpath, f)
return [uploadpath, files]
def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo):
koji.ensuredir(self.outdir)
cmd = ['/usr/bin/createrepo', '-vd', '-o', self.outdir]
if pkglist is not None:
cmd.extend(['-i', pkglist])
if os.path.isfile(groupdata):
cmd.extend(['-g', groupdata])
#attempt to recycle repodata from last repo
if pkglist and oldrepo and self.options.createrepo_update:
oldpath = self.pathinfo.repo(oldrepo['id'], rinfo['tag_name'])
olddatadir = '%s/%s/repodata' % (oldpath, arch)
if not os.path.isdir(olddatadir):
self.logger.warn("old repodata is missing: %s" % olddatadir)
else:
shutil.copytree(olddatadir, self.datadir)
oldorigins = os.path.join(self.datadir, 'pkgorigins.gz')
if os.path.isfile(oldorigins):
# remove any previous origins file and rely on mergerepos
# to rewrite it (if we have external repos to merge)
os.unlink(oldorigins)
cmd.append('--update')
if self.options.createrepo_skip_stat:
cmd.append('--skip-stat')
# note: we can't easily use a cachedir because we do not have write
# permission. The good news is that with --update we won't need to
# be scanning many rpms.
if pkglist is None:
cmd.append(self.outdir)
else:
cmd.append(self.repodir)
logfile = '%s/createrepo.log' % self.workdir
status = log_output(self.session, cmd[0], cmd, logfile, self.getUploadDir(), logerror=True)
if not isSuccess(status):
raise koji.GenericError, 'failed to create repo: %s' \
% parseStatus(status, ' '.join(cmd))
def merge_repos(self, external_repos, arch, groupdata):
repos = []
localdir = '%s/repo_%s_premerge' % (self.workdir, self.repo_id)
os.rename(self.outdir, localdir)
koji.ensuredir(self.outdir)
repos.append('file://' + localdir + '/')
for repo in external_repos:
ext_url = repo['url']
# substitute $arch in the url with the arch of the repo we're generating
ext_url = ext_url.replace('$arch', arch)
repos.append(ext_url)
blocklist = self.repodir + '/blocklist'
cmd = ['/usr/libexec/kojid/mergerepos', '-a', arch, '-b', blocklist, '-o', self.outdir]
if os.path.isfile(groupdata):
cmd.extend(['-g', groupdata])
for repo in repos:
cmd.extend(['-r', repo])
logfile = '%s/mergerepos.log' % self.workdir
status = log_output(self.session, cmd[0], cmd, logfile, self.getUploadDir(), logerror=True)
if not isSuccess(status):
raise koji.GenericError, 'failed to merge repos: %s' \
% parseStatus(status, ' '.join(cmd))
class WaitrepoTask(BaseTaskHandler):
Methods = ['waitrepo']
#mostly just waiting
_taskWeight = 0.2
PAUSE = 60
# time in minutes before we fail this task
TIMEOUT = 120
def handler(self, tag, newer_than=None, nvrs=None):
"""Wait for a repo for the tag, subject to given conditions
newer_than: create_event timestamp should be newer than this
nvr: repo should contain this nvr (which may not exist at first)
Only one of the options may be specified. If neither is, then
the call will wait for the first ready repo.
Returns the repo info (from getRepo) of the chosen repo
"""
start = time.time()
taginfo = self.session.getTag(tag, strict=True)
targets = self.session.getBuildTargets(buildTagID=taginfo['id'])
if not targets:
raise koji.GenericError("No build target for tag: %s" % taginfo['name'])
if isinstance(newer_than, basestring) and newer_than.lower() == "now":
newer_than = start
if not isinstance(newer_than, (type(None), int, long, float)):
raise koji.GenericError, "Invalid value for newer_than: %s" % newer_than
if newer_than and nvrs:
raise koji.GenericError, "only one of (newer_than, nvrs) may be specified"
if not nvrs:
nvrs = []
builds = [koji.parse_NVR(nvr) for nvr in nvrs]
last_repo = None
while True:
repo = self.session.getRepo(taginfo['id'])
if repo and repo != last_repo:
if builds:
if koji.util.checkForBuilds(self.session, taginfo['id'], builds, repo['create_event']):
self.logger.debug("Successfully waited %s for %s to appear in the %s repo" % \
(koji.util.duration(start), koji.util.printList(nvrs), taginfo['name']))
return repo
elif newer_than:
if repo['create_ts'] > newer_than:
self.logger.debug("Successfully waited %s for a new %s repo" % \
(koji.util.duration(start), taginfo['name']))
return repo
else:
#no check requested -- return first ready repo
return repo
if (time.time() - start) > (self.TIMEOUT * 60.0):
if builds:
raise koji.GenericError, "Unsuccessfully waited %s for %s to appear in the %s repo" % \
(koji.util.duration(start), koji.util.printList(nvrs), taginfo['name'])
else:
raise koji.GenericError, "Unsuccessfully waited %s for a new %s repo" % \
(koji.util.duration(start), taginfo['name'])
time.sleep(self.PAUSE)
last_repo = repo
def get_options():
"""process options from command line and config file"""
# parse command line args
logger = logging.getLogger("koji.build")
parser = OptionParser()
parser.add_option("-c", "--config", dest="configFile",
help="use alternate configuration file", metavar="FILE",
default="/etc/kojid/kojid.conf")
parser.add_option("--user", help="specify user")
parser.add_option("--password", help="specify password")
parser.add_option("-f", "--fg", dest="daemon",
action="store_false", default=True,
help="run in foreground")
parser.add_option("--force-lock", action="store_true", default=False,
help="force lock for exclusive session")
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="show verbose output")
parser.add_option("-d", "--debug", action="store_true", default=False,
help="show debug output")
parser.add_option("--debug-task", action="store_true", default=False,
help="enable debug output for tasks")
parser.add_option("--debug-xmlrpc", action="store_true", default=False,
help="show xmlrpc debug output")
parser.add_option("--debug-mock", action="store_true", default=False,
#obsolete option
help=SUPPRESS_HELP)
parser.add_option("--skip-main", action="store_true", default=False,
help="don't actually run main")
parser.add_option("--maxjobs", type='int', help="Specify maxjobs")
parser.add_option("--minspace", type='int', help="Specify minspace")
parser.add_option("--sleeptime", type='int', help="Specify the polling interval")
parser.add_option("--admin-emails", help="Address(es) to send error notices to")
parser.add_option("--topdir", help="Specify topdir")
parser.add_option("--topurl", help="Specify topurl")
parser.add_option("--workdir", help="Specify workdir")
parser.add_option("--pluginpath", help="Specify plugin search path")
parser.add_option("--plugin", action="append", help="Load specified plugin")
parser.add_option("--mockdir", help="Specify mockdir")
parser.add_option("--mockuser", help="User to run mock as")
parser.add_option("-s", "--server", help="url of XMLRPC server")
parser.add_option("--pkgurl", help=SUPPRESS_HELP)
(options, args) = parser.parse_args()
if args:
parser.error("incorrect number of arguments")
#not reached
assert False
# load local config
config = ConfigParser()
config.read(options.configFile)
for x in config.sections():
if x != 'kojid':
quit('invalid section found in config file: %s' % x)
defaults = {'sleeptime': 15,
'maxjobs': 10,
'literal_task_arches': '',
'minspace': 8192,
'admin_emails': None,
'log_level': None,
'topdir': '/mnt/koji',
'topurl': None,
'workdir': '/var/tmp/koji',
'pluginpath': '/usr/lib/koji-builder-plugins',
'mockdir': '/var/lib/mock',
'mockuser': 'kojibuilder',
'packager': 'Koji',
'vendor': 'Koji',
'distribution': 'Koji',
'mockhost': 'koji-linux-gnu',
'smtphost': 'example.com',
'from_addr': 'Koji Build System <buildsys@example.com>',
'krb_principal': None,
'host_principal_format': 'compile/%s@EXAMPLE.COM',
'keytab': '/etc/kojid/kojid.keytab',
'ccache': '/var/tmp/kojid.ccache',
'krbservice': 'host',
'server': None,
'user': None,
'password': None,
'retry_interval': 60,
'max_retries': 120,
'offline_retry': True,
'offline_retry_interval': 120,
'keepalive' : True,
'timeout' : None,
'use_fast_upload': True,
'createrepo_skip_stat': True,
'createrepo_update': True,
'pkgurl': None,
'allowed_scms': '',
'support_rpm_source_layout': True,
'yum_proxy': None,
'maven_repo_ignore': '*.md5 *.sha1 maven-metadata*.xml _maven.repositories '
'resolver-status.properties *.lastUpdated',
'failed_buildroot_lifetime' : 3600 * 4,
'rpmbuild_timeout' : 3600 * 24,
'cert': '/etc/kojid/client.crt',
'ca': '/etc/kojid/clientca.crt',
'serverca': '/etc/kojid/serverca.crt'}
if config.has_section('kojid'):
for name, value in config.items('kojid'):
if name in ['sleeptime', 'maxjobs', 'minspace', 'retry_interval',
'max_retries', 'offline_retry_interval', 'failed_buildroot_lifetime',
'timeout', 'rpmbuild_timeout',]:
try:
defaults[name] = int(value)
except ValueError:
quit("value for %s option must be a valid integer" % name)
elif name in ['offline_retry', 'createrepo_skip_stat', 'createrepo_update',
'keepalive', 'use_fast_upload', 'support_rpm_source_layout']:
defaults[name] = config.getboolean('kojid', name)
elif name in ['plugin', 'plugins']:
defaults['plugin'] = value.split()
elif name in defaults.keys():
defaults[name] = value
elif name.upper().startswith('RLIMIT_'):
defaults[name.upper()] = value
else:
quit("unknown config option: %s" % name)
for name, value in defaults.items():
if getattr(options, name, None) is None:
setattr(options, name, value)
#honor topdir
if options.topdir:
koji.BASEDIR = options.topdir
koji.pathinfo.topdir = options.topdir
#make sure workdir exists
if not os.path.exists(options.workdir):
koji.ensuredir(options.workdir)
if not options.server:
msg = "the server option is required"
logger.error(msg)
parser.error(msg)
if not options.topurl:
msg = "the topurl option is required"
logger.error(msg)
parser.error(msg)
topurls = options.topurl.split()
options.topurls = topurls
if len(topurls) > 1:
# XXX - fix the rest of the code so this is not necessary
options.topurl = topurls[0]
if options.pkgurl:
logger.warning("The pkgurl option is obsolete")
if options.debug_mock:
logger.warning("The debug-mock option is obsolete")
return options
def quit(msg=None, code=1):
if msg:
logging.getLogger("koji.build").error(msg)
sys.stderr.write('%s\n' % msg)
sys.stderr.flush()
sys.exit(code)
if __name__ == "__main__":
koji.add_file_logger("koji", "/var/log/kojid.log")
#note we're setting logging params for all of koji*
options = get_options()
if options.log_level:
lvl = getattr(logging, options.log_level, None)
if lvl is None:
quit("Invalid log level: %s" % options.log_level)
logging.getLogger("koji").setLevel(lvl)
else:
logging.getLogger("koji").setLevel(logging.WARN)
if options.debug:
logging.getLogger("koji").setLevel(logging.DEBUG)
elif options.verbose:
logging.getLogger("koji").setLevel(logging.INFO)
if options.debug_task:
logging.getLogger("koji.build.BaseTaskHandler").setLevel(logging.DEBUG)
if options.admin_emails:
koji.add_mail_logger("koji", options.admin_emails)
#build session options
session_opts = {}
for k in ('user', 'password', 'krbservice', 'debug_xmlrpc', 'debug',
'retry_interval', 'max_retries', 'offline_retry', 'offline_retry_interval',
'keepalive', 'timeout', 'use_fast_upload',
):
v = getattr(options, k, None)
if v is not None:
session_opts[k] = v
#start a session and login
session = koji.ClientSession(options.server, session_opts)
if os.path.isfile(options.cert):
try:
# authenticate using SSL client certificates
session.ssl_login(options.cert, options.ca,
options.serverca)
except koji.AuthError, e:
quit("Error: Unable to log in: %s" % e)
except xmlrpclib.ProtocolError:
quit("Error: Unable to connect to server %s" % (options.server))
elif options.user:
try:
# authenticate using user/password
session.login()
except koji.AuthError:
quit("Error: Unable to log in. Bad credentials?")
except xmlrpclib.ProtocolError:
quit("Error: Unable to connect to server %s" % (options.server))
elif sys.modules.has_key('krbV'):
krb_principal = options.krb_principal
if krb_principal is None:
krb_principal = options.host_principal_format % socket.getfqdn()
try:
session.krb_login(principal=krb_principal,
keytab=options.keytab,
ccache=options.ccache)
except krbV.Krb5Error, e:
quit("Kerberos authentication failed: '%s' (%s)" % (e.args[1], e.args[0]))
except socket.error, e:
quit("Could not connect to Kerberos authentication service: '%s'" % e.args[1])
else:
quit("No username/password supplied and Kerberos missing or not configured")
#make session exclusive
try:
session.exclusiveSession(force=options.force_lock)
except koji.AuthLockError:
quit("Error: Unable to get lock. Trying using --force-lock")
if not session.logged_in:
quit("Error: Unknown login error")
#make sure it works
try:
ret = session.echo("OK")
except xmlrpclib.ProtocolError:
quit("Error: Unable to connect to server %s" % (options.server))
if ret != ["OK"]:
quit("Error: incorrect server response: %r" % (ret))
# run main
if options.daemon:
#detach
koji.daemonize()
main(options, session)
# not reached
assert False
elif not options.skip_main:
koji.add_stderr_logger("koji")
main(options, session)
Reference in a new issue View git blame Copy permalink