eaa0943a1a
In some cases it makes sense to continue with sibling task even if some of them already failed. E.g. with kernel build it could be of use if submitter knows for which archs it succeed and for which it fails. Repeated builds could take a lot of time and resources. Option is turned on by 'build_arch_can_fail' in kojid.conf and it is turned on per builder. Note, that it shouldn't be enabled by default as it could result in unnecessary resource consumption.
5143 lines
215 KiB
Python
Executable file
5143 lines
215 KiB
Python
Executable file
#!/usr/bin/python
|
|
|
|
# Koji build daemon
|
|
# Copyright (c) 2005-2014 Red Hat, Inc.
|
|
#
|
|
# Koji is free software; you can redistribute it and/or
|
|
# modify it under the terms of the GNU Lesser General Public
|
|
# License as published by the Free Software Foundation;
|
|
# version 2.1 of the License.
|
|
#
|
|
# This software is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
# Lesser General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Lesser General Public
|
|
# License along with this software; if not, write to the Free Software
|
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
#
|
|
# Authors:
|
|
# Mike McLean <mikem@redhat.com>
|
|
# Mike Bonnet <mikeb@redhat.com>
|
|
|
|
try:
|
|
import krbV
|
|
except ImportError:
|
|
pass
|
|
import koji
|
|
import koji.plugin
|
|
import koji.util
|
|
import koji.tasks
|
|
import glob
|
|
import logging
|
|
import logging.handlers
|
|
from koji.daemon import incremental_upload, log_output, TaskManager, SCM
|
|
from koji.tasks import ServerExit, ServerRestart, BaseTaskHandler, MultiPlatformTask
|
|
from koji.util import parseStatus, isSuccess, dslice, dslice_ex
|
|
import os
|
|
import pwd
|
|
import grp
|
|
import random
|
|
import re
|
|
import rpm
|
|
import shutil
|
|
import signal
|
|
import smtplib
|
|
import socket
|
|
import sys
|
|
import time
|
|
import traceback
|
|
import xml.dom.minidom
|
|
import xmlrpclib
|
|
import zipfile
|
|
import copy
|
|
import Cheetah.Template
|
|
from ConfigParser import ConfigParser
|
|
from fnmatch import fnmatch
|
|
from gzip import GzipFile
|
|
from optparse import OptionParser, SUPPRESS_HELP
|
|
from StringIO import StringIO
|
|
from yum import repoMDObject
|
|
|
|
#imports for LiveCD, LiveMedia, and Appliance handler
|
|
image_enabled = False
|
|
try:
|
|
import pykickstart.parser as ksparser
|
|
import pykickstart.handlers.control as kscontrol
|
|
import pykickstart.errors as kserrors
|
|
import hashlib
|
|
import iso9660 # from pycdio
|
|
image_enabled = True
|
|
except ImportError:
|
|
pass
|
|
|
|
ozif_enabled = False
|
|
try:
|
|
from imgfac.BuildDispatcher import BuildDispatcher
|
|
from imgfac.Builder import Builder
|
|
from imgfac.PluginManager import PluginManager
|
|
from imgfac.ReservationManager import ReservationManager
|
|
plugin_mgr = PluginManager('/etc/imagefactory/plugins.d')
|
|
plugin_mgr.load()
|
|
from imgfac.ApplicationConfiguration import ApplicationConfiguration
|
|
from imgfac.PersistentImageManager import PersistentImageManager
|
|
from imgfac.BaseImage import BaseImage
|
|
from imgfac.TargetImage import TargetImage
|
|
# NOTE: import below requires Factory 1.1.7 or higher
|
|
from imgfac.FactoryUtils import qemu_convert_cmd
|
|
ozif_enabled = True
|
|
except ImportError:
|
|
pass
|
|
|
|
def main(options, session):
|
|
logger = logging.getLogger("koji.build")
|
|
logger.info('Starting up')
|
|
koji.util.setup_rlimits(options.__dict__, logger)
|
|
tm = TaskManager(options, session)
|
|
tm.findHandlers(globals())
|
|
tm.findHandlers(vars(koji.tasks))
|
|
if options.plugin:
|
|
#load plugins
|
|
pt = koji.plugin.PluginTracker(path=options.pluginpath.split(':'))
|
|
for name in options.plugin:
|
|
logger.info('Loading plugin: %s' % name)
|
|
tm.scanPlugin(pt.load(name))
|
|
def shutdown(*args):
|
|
raise SystemExit
|
|
def restart(*args):
|
|
logger.warn("Initiating graceful restart")
|
|
tm.restart_pending = True
|
|
signal.signal(signal.SIGTERM,shutdown)
|
|
signal.signal(signal.SIGUSR1,restart)
|
|
while 1:
|
|
try:
|
|
taken = False
|
|
tm.updateBuildroots()
|
|
tm.updateTasks()
|
|
taken = tm.getNextTask()
|
|
except (SystemExit,ServerExit,KeyboardInterrupt):
|
|
logger.warn("Exiting")
|
|
break
|
|
except ServerRestart:
|
|
logger.warn("Restarting")
|
|
os.execv(sys.argv[0], sys.argv)
|
|
except koji.AuthExpired:
|
|
logger.error('Session expired')
|
|
break
|
|
except koji.RetryError:
|
|
raise
|
|
except:
|
|
# XXX - this is a little extreme
|
|
# log the exception and continue
|
|
logger.error(''.join(traceback.format_exception(*sys.exc_info())))
|
|
try:
|
|
if not taken:
|
|
# Only sleep if we didn't take a task, otherwise retry immediately.
|
|
# The load-balancing code in getNextTask() will prevent a single builder
|
|
# from getting overloaded.
|
|
time.sleep(options.sleeptime)
|
|
except (SystemExit,KeyboardInterrupt):
|
|
logger.warn("Exiting")
|
|
break
|
|
logger.warn("Shutting down, please wait...")
|
|
tm.shutdown()
|
|
session.logout()
|
|
sys.exit(0)
|
|
|
|
|
|
class BuildRoot(object):
|
|
|
|
def __init__(self,session,options,*args,**kwargs):
|
|
self.logger = logging.getLogger("koji.build.buildroot")
|
|
self.session = session
|
|
self.options = options
|
|
if len(args) + len(kwargs) == 1:
|
|
# manage an existing mock buildroot
|
|
self._load(*args,**kwargs)
|
|
else:
|
|
self._new(*args,**kwargs)
|
|
|
|
def _load(self, data):
|
|
#manage an existing buildroot
|
|
if isinstance(data, dict):
|
|
#assume data already pulled from db
|
|
self.id = data['id']
|
|
else:
|
|
self.id = data
|
|
data = self.session.getBuildroot(self.id)
|
|
self.task_id = data['task_id']
|
|
self.tag_id = data['tag_id']
|
|
self.tag_name = data['tag_name']
|
|
self.repoid = data['repo_id']
|
|
self.repo_info = self.session.repoInfo(self.repoid, strict=True)
|
|
self.event_id = self.repo_info['create_event']
|
|
self.br_arch = data['arch']
|
|
self.name = "%(tag_name)s-%(id)s-%(repoid)s" % vars(self)
|
|
self.config = self.session.getBuildConfig(self.tag_id, event=self.event_id)
|
|
|
|
def _new(self, tag, arch, task_id, repo_id=None, install_group='build',
|
|
setup_dns=False, bind_opts=None, maven_opts=None, maven_envs=None, deps=None):
|
|
"""Create a brand new repo"""
|
|
if not repo_id:
|
|
raise koji.BuildrootError, "A repo id must be provided"
|
|
repo_info = self.session.repoInfo(repo_id, strict=True)
|
|
self.repo_info = repo_info
|
|
self.repoid = self.repo_info['id']
|
|
self.event_id = self.repo_info['create_event']
|
|
self.task_id = task_id
|
|
self.config = self.session.getBuildConfig(tag, event=self.event_id)
|
|
if not self.config:
|
|
raise koji.BuildrootError("Could not get config info for tag: %s" % tag)
|
|
self.tag_id = self.config['id']
|
|
self.tag_name = self.config['name']
|
|
if self.config['id'] != repo_info['tag_id']:
|
|
raise koji.BuildrootError, "tag/repo mismatch: %s vs %s" \
|
|
% (self.config['name'], repo_info['tag_name'])
|
|
repo_state = koji.REPO_STATES[repo_info['state']]
|
|
if repo_state == 'EXPIRED':
|
|
# This should be ok. Expired repos are still intact, just not
|
|
# up-to-date (which may be the point in some cases).
|
|
self.logger.info("Requested repo (%i) is no longer current" % repo_id)
|
|
elif repo_state != 'READY':
|
|
raise koji.BuildrootError, "Requested repo (%i) is %s" % (repo_id, repo_state)
|
|
self.br_arch = koji.canonArch(arch)
|
|
# armhfp is not a valid arch according to autoconf
|
|
if arch == 'armhfp':
|
|
self.target_arch = 'armv7hl'
|
|
else:
|
|
self.target_arch = arch
|
|
self.logger.debug("New buildroot: %(tag_name)s/%(br_arch)s/%(repoid)s" % vars(self))
|
|
id = self.session.host.newBuildRoot(self.repoid, self.br_arch, task_id=task_id)
|
|
if id is None:
|
|
raise koji.BuildrootError, "failed to get a buildroot id"
|
|
self.id = id
|
|
self.name = "%(tag_name)s-%(id)s-%(repoid)s" % vars(self)
|
|
self.install_group = install_group
|
|
self.setup_dns = setup_dns
|
|
self.bind_opts = bind_opts
|
|
self.maven_opts = maven_opts
|
|
self.maven_envs = maven_envs
|
|
self.deps = deps
|
|
self._writeMockConfig()
|
|
|
|
def _writeMockConfig(self):
|
|
# mock config
|
|
configdir = '/etc/mock/koji'
|
|
configfile = "%s/%s.cfg" % (configdir,self.name)
|
|
self.mockcfg = "koji/%s" % self.name
|
|
|
|
opts = {}
|
|
for k in ('repoid', 'tag_name'):
|
|
if hasattr(self, k):
|
|
opts[k] = getattr(self, k)
|
|
for k in ('mockdir', 'topdir', 'topurl', 'topurls', 'packager', 'vendor', 'distribution', 'mockhost', 'yum_proxy', 'rpmbuild_timeout'):
|
|
if hasattr(self.options, k):
|
|
opts[k] = getattr(self.options, k)
|
|
opts['buildroot_id'] = self.id
|
|
opts['use_host_resolv'] = self.setup_dns
|
|
opts['install_group'] = self.install_group
|
|
opts['maven_opts'] = self.maven_opts
|
|
opts['maven_envs'] = self.maven_envs
|
|
opts['bind_opts'] = self.bind_opts
|
|
opts['target_arch'] = self.target_arch
|
|
if 'mock.package_manager' in self.config['extra']:
|
|
opts['package_manager'] = self.config['extra']['mock.package_manager']
|
|
output = koji.genMockConfig(self.name, self.br_arch, managed=True, **opts)
|
|
|
|
#write config
|
|
fo = file(configfile,'w')
|
|
fo.write(output)
|
|
fo.close()
|
|
|
|
def _repositoryEntries(self, pi, plugin=False):
|
|
entries = []
|
|
if plugin:
|
|
tag_name = 'pluginRepository'
|
|
else:
|
|
tag_name = 'repository'
|
|
id_suffix = 'repo'
|
|
name_prefix = 'Repository for Koji'
|
|
for dep in self.deps:
|
|
if isinstance(dep, (int, long)):
|
|
# dep is a task ID, the url points to the task output directory
|
|
repo_type = 'task'
|
|
dep_url = pi.task(dep)
|
|
snapshots = 'true'
|
|
else:
|
|
# dep is a build NVR, the url points to the build output directory
|
|
repo_type = 'build'
|
|
build = koji.parse_NVR(dep)
|
|
dep_url = pi.mavenbuild(build)
|
|
snapshots = 'false'
|
|
repo_id = 'koji-%(repo_type)s-%(dep)s-%(id_suffix)s' % locals()
|
|
entry = """
|
|
<%(tag_name)s>
|
|
<id>%(repo_id)s</id>
|
|
<name>%(name_prefix)s %(repo_type)s %(dep)s</name>
|
|
<url>%(dep_url)s</url>
|
|
<layout>default</layout>
|
|
<releases>
|
|
<enabled>true</enabled>
|
|
<updatePolicy>never</updatePolicy>
|
|
<checksumPolicy>fail</checksumPolicy>
|
|
</releases>
|
|
<snapshots>
|
|
<enabled>%(snapshots)s</enabled>
|
|
<updatePolicy>never</updatePolicy>
|
|
<checksumPolicy>fail</checksumPolicy>
|
|
</snapshots>
|
|
</%(tag_name)s>""" % locals()
|
|
entries.append((repo_id, entry))
|
|
return entries
|
|
|
|
def writeMavenSettings(self, destfile, outputdir):
|
|
"""
|
|
Write the Maven settings.xml file to the specified destination.
|
|
"""
|
|
task_id = self.task_id
|
|
repo_id = self.repoid
|
|
tag_name = self.tag_name
|
|
deploy_dir = outputdir[len(self.rootdir()):]
|
|
|
|
pi = koji.PathInfo(topdir=self.options.topurl)
|
|
repourl = pi.repo(repo_id, tag_name) + '/maven'
|
|
|
|
mirror_spec = '*'
|
|
settings = """<settings xmlns="http://maven.apache.org/POM/4.0.0"
|
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
|
|
http://maven.apache.org/xsd/settings-1.0.0.xsd">
|
|
<interactiveMode>false</interactiveMode>
|
|
<mirrors>
|
|
<mirror>
|
|
<id>koji-maven-repo-%(tag_name)s-%(repo_id)i</id>
|
|
<name>Koji-managed Maven repository (%(tag_name)s-%(repo_id)i)</name>
|
|
<url>%(repourl)s</url>
|
|
<mirrorOf>%(mirror_spec)s</mirrorOf>
|
|
</mirror>
|
|
</mirrors>
|
|
<profiles>
|
|
<profile>
|
|
<id>koji-task-%(task_id)s</id>
|
|
<properties>
|
|
<altDeploymentRepository>koji-output::default::file://%(deploy_dir)s</altDeploymentRepository>
|
|
</properties>"""
|
|
if self.deps:
|
|
settings += """
|
|
<repositories>"""
|
|
for dep_repo_id, dep_repo_entry in self._repositoryEntries(pi):
|
|
mirror_spec += ',!' + dep_repo_id
|
|
settings += dep_repo_entry
|
|
settings += """
|
|
</repositories>
|
|
<pluginRepositories>"""
|
|
for dep_repo_id, dep_repo_entry in self._repositoryEntries(pi, plugin=True):
|
|
mirror_spec += ',!' + dep_repo_id
|
|
settings += dep_repo_entry
|
|
settings += """
|
|
</pluginRepositories>"""
|
|
settings += """
|
|
</profile>
|
|
</profiles>
|
|
<activeProfiles>
|
|
<activeProfile>koji-task-%(task_id)s</activeProfile>
|
|
</activeProfiles>
|
|
</settings>
|
|
"""
|
|
settings = settings % locals()
|
|
fo = file(self.rootdir() + destfile, 'w')
|
|
fo.write(settings)
|
|
fo.close()
|
|
|
|
def mock(self, args):
|
|
"""Run mock"""
|
|
mockpath = getattr(self.options,"mockpath","/usr/bin/mock")
|
|
cmd = [mockpath, "-r", self.mockcfg]
|
|
#if self.options.debug_mock:
|
|
# cmd.append('--debug')
|
|
# TODO: should we pass something like --verbose --trace instead?
|
|
cmd.extend(args)
|
|
self.logger.info(' '.join(cmd))
|
|
workdir = getattr(self, 'workdir', None)
|
|
mocklog = 'mock_output.log'
|
|
pid = os.fork()
|
|
if pid:
|
|
resultdir = self.resultdir()
|
|
uploadpath = self.getUploadPath()
|
|
logs = {}
|
|
|
|
finished = False
|
|
while not finished:
|
|
time.sleep(1)
|
|
status = os.waitpid(pid, os.WNOHANG)
|
|
if status[0] != 0:
|
|
finished = True
|
|
|
|
try:
|
|
results = os.listdir(resultdir)
|
|
except OSError:
|
|
# will happen when mock hasn't created the resultdir yet
|
|
results = []
|
|
|
|
for fname in results:
|
|
if fname.endswith('.log') and not logs.has_key(fname):
|
|
fpath = os.path.join(resultdir, fname)
|
|
logs[fname] = (None, None, 0, fpath)
|
|
if workdir and mocklog not in logs:
|
|
fpath = os.path.join(workdir, mocklog)
|
|
if os.path.exists(fpath):
|
|
logs[mocklog] = (None, None, 0, fpath)
|
|
|
|
for (fname, (fd, inode, size, fpath)) in logs.items():
|
|
try:
|
|
stat_info = os.stat(fpath)
|
|
if not fd or stat_info.st_ino != inode or stat_info.st_size < size:
|
|
# either a file we haven't opened before, or mock replaced a file we had open with
|
|
# a new file and is writing to it, or truncated the file we're reading,
|
|
# but our fd is pointing to the previous location in the old file
|
|
if fd:
|
|
self.logger.info('Rereading %s, inode: %s -> %s, size: %s -> %s' %
|
|
(fpath, inode, stat_info.st_ino, size, stat_info.st_size))
|
|
fd.close()
|
|
fd = file(fpath, 'r')
|
|
logs[fname] = (fd, stat_info.st_ino, stat_info.st_size, fpath)
|
|
except:
|
|
self.logger.error("Error reading mock log: %s", fpath)
|
|
self.logger.error(''.join(traceback.format_exception(*sys.exc_info())))
|
|
continue
|
|
|
|
incremental_upload(self.session, fname, fd, uploadpath, logger=self.logger)
|
|
#clean up and return exit status of command
|
|
for (fname, (fd, inode, size, fpath)) in logs.items():
|
|
if fd:
|
|
fd.close()
|
|
return status[1]
|
|
|
|
else:
|
|
#in no case should exceptions propagate past here
|
|
try:
|
|
self.session._forget()
|
|
if workdir:
|
|
outfile = os.path.join(workdir, mocklog)
|
|
flags = os.O_CREAT | os.O_WRONLY | os.O_APPEND
|
|
fd = os.open(outfile, flags, 0666)
|
|
os.dup2(fd, 1)
|
|
os.dup2(fd, 2)
|
|
if os.getuid() == 0 and hasattr(self.options,"mockuser"):
|
|
self.logger.info('Running mock as %s' % self.options.mockuser)
|
|
uid,gid = pwd.getpwnam(self.options.mockuser)[2:4]
|
|
os.setgroups([grp.getgrnam('mock')[2]])
|
|
os.setregid(gid,gid)
|
|
os.setreuid(uid,uid)
|
|
os.execvp(cmd[0],cmd)
|
|
except:
|
|
#diediedie
|
|
print "Failed to exec mock"
|
|
print ''.join(traceback.format_exception(*sys.exc_info()))
|
|
os._exit(1)
|
|
|
|
def getUploadPath(self):
|
|
"""Get the path that should be used when uploading files to
|
|
the hub."""
|
|
return koji.pathinfo.taskrelpath(self.task_id)
|
|
|
|
def uploadDir(self, dirpath, suffix=None):
|
|
"""Upload the contents of the given directory to the
|
|
task output directory on the hub. If suffix is provided,
|
|
append '.' + suffix to the filenames, so that successive uploads
|
|
of the same directory won't overwrite each other, if the files have
|
|
the same name but different contents."""
|
|
if not os.path.isdir(dirpath):
|
|
return
|
|
uploadpath = self.getUploadPath()
|
|
for filename in os.listdir(dirpath):
|
|
filepath = os.path.join(dirpath, filename)
|
|
if os.stat(filepath).st_size > 0:
|
|
if suffix:
|
|
filename = '%s.%s' % (filename, suffix)
|
|
self.session.uploadWrapper(filepath, uploadpath, filename)
|
|
|
|
def init(self):
|
|
rv = self.mock(['--init'])
|
|
|
|
if rv:
|
|
self.expire()
|
|
raise koji.BuildrootError, "could not init mock buildroot, %s" % self._mockResult(rv)
|
|
self.session.host.setBuildRootList(self.id,self.getPackageList())
|
|
|
|
def _mockResult(self, rv, logfile=None):
|
|
if logfile:
|
|
pass
|
|
elif os.WIFEXITED(rv) and os.WEXITSTATUS(rv) == 1:
|
|
logfile = 'build.log'
|
|
else:
|
|
logfile = 'root.log'
|
|
msg = '; see %s for more information' % logfile
|
|
return parseStatus(rv, 'mock') + msg
|
|
|
|
def build_srpm(self, specfile, sourcedir, source_cmd):
|
|
self.session.host.setBuildRootState(self.id,'BUILDING')
|
|
alt_sources_dir = "%s/SOURCES" % sourcedir
|
|
if self.options.support_rpm_source_layout and os.path.isdir(alt_sources_dir):
|
|
sources_dir = alt_sources_dir
|
|
else:
|
|
sources_dir = sourcedir
|
|
if source_cmd:
|
|
# call the command defined by source_cmd in the chroot so any required files not stored in
|
|
# the SCM can be retrieved
|
|
chroot_sourcedir = sourcedir[len(self.rootdir()):]
|
|
args = ['--no-clean', '--unpriv', '--cwd', chroot_sourcedir, '--chroot']
|
|
args.extend(source_cmd)
|
|
rv = self.mock(args)
|
|
if rv:
|
|
self.expire()
|
|
raise koji.BuildError, "error retrieving sources, %s" % self._mockResult(rv)
|
|
|
|
args = ['--no-clean', '--buildsrpm', '--spec', specfile, '--sources', sources_dir,
|
|
'--target', 'noarch']
|
|
|
|
rv = self.mock(args)
|
|
|
|
if rv:
|
|
self.expire()
|
|
raise koji.BuildError, "error building srpm, %s" % self._mockResult(rv)
|
|
|
|
def build(self,srpm,arch=None):
|
|
# run build
|
|
self.session.host.setBuildRootState(self.id,'BUILDING')
|
|
args = ['--no-clean']
|
|
if arch:
|
|
args.extend(['--target', arch])
|
|
args.extend(['--rebuild', srpm])
|
|
rv = self.mock(args)
|
|
|
|
self.session.host.updateBuildRootList(self.id,self.getPackageList())
|
|
if rv:
|
|
self.expire()
|
|
raise koji.BuildError, "error building package (arch %s), %s" % (arch, self._mockResult(rv))
|
|
|
|
def getPackageList(self):
|
|
"""Return a list of packages from the buildroot
|
|
|
|
Each member of the list is a dictionary containing the following fields:
|
|
- name
|
|
- version
|
|
- release
|
|
- epoch
|
|
- arch
|
|
- payloadhash
|
|
- size
|
|
- buildtime
|
|
"""
|
|
fields = ('name',
|
|
'version',
|
|
'release',
|
|
'epoch',
|
|
'arch',
|
|
'sigmd5',
|
|
'size',
|
|
'buildtime')
|
|
rpm.addMacro("_dbpath", "%s/var/lib/rpm" % self.rootdir())
|
|
ret = []
|
|
try:
|
|
ts = rpm.TransactionSet()
|
|
for h in ts.dbMatch():
|
|
pkg = koji.get_header_fields(h,fields)
|
|
#skip our fake packages
|
|
if pkg['name'] in ['buildsys-build', 'gpg-pubkey']:
|
|
#XXX config
|
|
continue
|
|
pkg['payloadhash'] = koji.hex_string(pkg['sigmd5'])
|
|
del pkg['sigmd5']
|
|
ret.append(pkg)
|
|
finally:
|
|
rpm.delMacro("_dbpath")
|
|
self.markExternalRPMs(ret)
|
|
return ret
|
|
|
|
def getMavenPackageList(self, repodir):
|
|
"""Return a list of Maven packages that were installed into the local repo
|
|
to satisfy build requirements.
|
|
|
|
Each member of the list is a dictionary containing the following fields:
|
|
- maven_info: a dict of Maven info containing the groupId, artifactId, and version fields
|
|
- files: a list of files associated with that POM
|
|
"""
|
|
packages = []
|
|
for path, dirs, files in os.walk(repodir):
|
|
relpath = path[len(repodir) + 1:]
|
|
maven_files = []
|
|
for repofile in files:
|
|
if koji.util.multi_fnmatch(repofile, self.options.maven_repo_ignore) or \
|
|
koji.util.multi_fnmatch(os.path.join(relpath, repofile), self.options.maven_repo_ignore):
|
|
continue
|
|
if relpath == '' and repofile in ['scm-sources.zip', 'patches.zip']:
|
|
# special-case the archives of the sources and patches, since we drop them in
|
|
# root of the output directory
|
|
continue
|
|
maven_files.append({'path': relpath, 'filename': repofile,
|
|
'size': os.path.getsize(os.path.join(path, repofile))})
|
|
if maven_files:
|
|
path_comps = relpath.split('/')
|
|
if len(path_comps) < 3:
|
|
raise koji.BuildrootError, 'files found in unexpected path in local Maven repo, directory: %s, files: %s' % \
|
|
(relpath, ', '.join([f['filename'] for f in maven_files]))
|
|
# extract the Maven info from the path within the local repo
|
|
maven_info = {'version': path_comps[-1],
|
|
'artifact_id': path_comps[-2],
|
|
'group_id': '.'.join(path_comps[:-2])}
|
|
packages.append({'maven_info': maven_info, 'files': maven_files})
|
|
|
|
return packages
|
|
|
|
def mavenBuild(self, sourcedir, outputdir, repodir,
|
|
props=None, profiles=None, options=None, goals=None):
|
|
self.session.host.setBuildRootState(self.id, 'BUILDING')
|
|
cmd = ['--no-clean', '--chroot', '--unpriv', '--cwd', sourcedir[len(self.rootdir()):], '--',
|
|
'/usr/bin/mvn', '-C']
|
|
if options:
|
|
cmd.extend(options)
|
|
if profiles:
|
|
cmd.append('-P%s' % ','.join(profiles))
|
|
if props:
|
|
for name, value in props.items():
|
|
if value is not None:
|
|
cmd.append('-D%s=%s' % (name, value))
|
|
else:
|
|
cmd.append('-D%s' % name)
|
|
if goals:
|
|
cmd.extend(goals)
|
|
cmd.extend(['deploy'])
|
|
rv = self.mock(cmd)
|
|
|
|
# if the deploy command failed, don't raise an error on unknown artifacts, because that
|
|
# will mask the underlying failure
|
|
ignore_unknown = False
|
|
if rv:
|
|
ignore_unknown = True
|
|
self.session.host.updateMavenBuildRootList(self.id, self.task_id, self.getMavenPackageList(repodir),
|
|
ignore=self.getMavenPackageList(outputdir),
|
|
project=True, ignore_unknown=ignore_unknown,
|
|
extra_deps=self.deps)
|
|
if rv:
|
|
self.expire()
|
|
raise koji.BuildrootError, 'error building Maven package, %s' % self._mockResult(rv, logfile='root.log')
|
|
|
|
def scrub(self):
|
|
"Non-mock implementation of clean"
|
|
raise koji.FunctionDeprecated, "no longer needed and deprecated. use clean()"
|
|
|
|
def markExternalRPMs(self, rpmlist):
|
|
"""Check rpms against pkgorigins and add external repo data to the external ones
|
|
|
|
Modifies rpmlist in place. No return
|
|
"""
|
|
external_repos = self.session.getExternalRepoList(self.repo_info['tag_id'],
|
|
event=self.repo_info['create_event'])
|
|
if not external_repos:
|
|
#nothing to do
|
|
return
|
|
#index external repos by expanded url
|
|
erepo_idx = {}
|
|
for erepo in external_repos:
|
|
# substitute $arch in the url with the arch of the repo we're generating
|
|
ext_url = erepo['url'].replace('$arch', self.br_arch)
|
|
erepo_idx[ext_url] = erepo
|
|
pathinfo = koji.PathInfo(topdir='')
|
|
#XXX - cheap hack to get relative paths
|
|
repodir = pathinfo.repo(self.repo_info['id'], self.repo_info['tag_name'])
|
|
repomdpath = os.path.join(repodir, self.br_arch, 'repodata', 'repomd.xml')
|
|
|
|
opts = dict([(k, getattr(self.options, k)) for k in 'topurl','topdir'])
|
|
opts['tempdir'] = self.options.workdir
|
|
fo = koji.openRemoteFile(repomdpath, **opts)
|
|
try:
|
|
repodata = repoMDObject.RepoMD('ourrepo', fo)
|
|
except:
|
|
raise koji.BuildError, "Unable to parse repomd.xml file for %s" % os.path.join(repodir, self.br_arch)
|
|
data = repodata.getData('origin')
|
|
pkgorigins = data.location[1]
|
|
|
|
relpath = os.path.join(repodir, self.br_arch, pkgorigins)
|
|
fo = koji.openRemoteFile(relpath, **opts)
|
|
#at this point we know there were external repos at the create event,
|
|
#so there should be an origins file.
|
|
origin_idx = {}
|
|
fo2 = GzipFile(fileobj=fo, mode='r')
|
|
for line in fo2:
|
|
parts=line.split(None, 2)
|
|
if len(parts) < 2:
|
|
continue
|
|
#first field is formated by yum as [e:]n-v-r.a
|
|
nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" % koji.parse_NVRA(parts[0])
|
|
origin_idx[nvra] = parts[1]
|
|
fo2.close()
|
|
fo.close()
|
|
# mergerepo starts from a local repo in the task workdir, so internal
|
|
# rpms have an odd-looking origin that we need to look for
|
|
localtail = '/repo_%s_premerge/' % self.repo_info['id']
|
|
for rpm_info in rpmlist:
|
|
key = "%(name)s-%(version)s-%(release)s.%(arch)s" % rpm_info
|
|
# src rpms should not show up in rpmlist so we do not have to
|
|
# worry about fixing the arch for them
|
|
ext_url = origin_idx.get(key)
|
|
if not ext_url:
|
|
raise koji.BuildError, "No origin for %s" % key
|
|
erepo = erepo_idx.get(ext_url)
|
|
if not erepo:
|
|
if ext_url.startswith('file://') and ext_url.endswith(localtail):
|
|
# internal rpm
|
|
continue
|
|
raise koji.BuildError, "Unknown origin for %s: %s" % (key, ext_url)
|
|
rpm_info['external_repo'] = erepo
|
|
rpm_info['location'] = erepo['external_repo_id']
|
|
|
|
def resultdir(self):
|
|
return "%s/%s/result" % (self.options.mockdir, self.name)
|
|
|
|
def rootdir(self):
|
|
return "%s/%s/root" % (self.options.mockdir, self.name)
|
|
|
|
def expire(self):
|
|
self.session.host.setBuildRootState(self.id,'EXPIRED')
|
|
|
|
|
|
class ChainBuildTask(BaseTaskHandler):
|
|
|
|
Methods = ['chainbuild']
|
|
#mostly just waiting on other tasks
|
|
_taskWeight = 0.1
|
|
|
|
def handler(self, srcs, target, opts=None):
|
|
"""Run a chain build
|
|
|
|
target and opts are passed on to the build tasks
|
|
srcs is a list of "build levels"
|
|
each build level is a list of strings, each string may be one of:
|
|
- a build src (SCM url only)
|
|
- an n-v-r
|
|
each build level is processed in order
|
|
successive levels are only started once the previous levels have completed
|
|
and gotten into the repo.
|
|
"""
|
|
if opts.get('scratch'):
|
|
raise koji.BuildError, "--scratch is not allowed with chain-builds"
|
|
target_info = self.session.getBuildTarget(target)
|
|
if not target_info:
|
|
raise koji.GenericError, 'unknown build target: %s' % target
|
|
nvrs = []
|
|
for n_level, build_level in enumerate(srcs):
|
|
#if there are any nvrs to wait on, do so
|
|
if nvrs:
|
|
task_id = self.session.host.subtask(method='waitrepo',
|
|
arglist=[target_info['build_tag_name'], None, nvrs],
|
|
label="wait %i" % n_level,
|
|
parent=self.id)
|
|
self.wait(task_id, all=True, failany=True)
|
|
nvrs = []
|
|
#kick off the builds for this level
|
|
build_tasks = []
|
|
for n_src, src in enumerate(build_level):
|
|
if SCM.is_scm_url(src):
|
|
task_id = self.session.host.subtask(method='build',
|
|
arglist=[src, target, opts],
|
|
label="build %i,%i" % (n_level, n_src),
|
|
parent=self.id)
|
|
build_tasks.append(task_id)
|
|
else:
|
|
nvrs.append(src)
|
|
#next pass will wait for these
|
|
if build_tasks:
|
|
#the level could have been all nvrs
|
|
self.wait(build_tasks, all=True, failany=True)
|
|
#see what builds we created in this batch so the next pass can wait for them also
|
|
for build_task in build_tasks:
|
|
builds = self.session.listBuilds(taskID=build_task)
|
|
if builds:
|
|
nvrs.append(builds[0]['nvr'])
|
|
|
|
|
|
class BuildTask(BaseTaskHandler):
|
|
|
|
Methods = ['build']
|
|
#we mostly just wait on other tasks
|
|
_taskWeight = 0.2
|
|
|
|
def handler(self, src, target, opts=None):
|
|
"""Handler for the master build task"""
|
|
if opts is None:
|
|
opts = {}
|
|
self.opts = opts
|
|
if opts.get('arch_override') and not opts.get('scratch'):
|
|
raise koji.BuildError, "arch_override is only allowed for scratch builds"
|
|
if opts.get('repo_id') is not None:
|
|
repo_info = self.session.repoInfo(opts['repo_id'])
|
|
if not repo_info:
|
|
raise koji.BuildError, 'No such repo: %s' % opts['repo_id']
|
|
repo_state = koji.REPO_STATES[repo_info['state']]
|
|
if repo_state not in ('READY', 'EXPIRED'):
|
|
raise koji.BuildError, 'Bad repo: %s (%s)' % (repo_info['id'], repo_state)
|
|
self.event_id = repo_info['create_event']
|
|
else:
|
|
repo_info = None
|
|
#we'll wait for a repo later (self.getRepo)
|
|
self.event_id = None
|
|
task_info = self.session.getTaskInfo(self.id)
|
|
target_info = None
|
|
if target:
|
|
target_info = self.session.getBuildTarget(target, event=self.event_id)
|
|
if target_info:
|
|
dest_tag = target_info['dest_tag']
|
|
build_tag = target_info['build_tag']
|
|
if repo_info is not None:
|
|
#make sure specified repo matches target
|
|
if repo_info['tag_id'] != target_info['build_tag']:
|
|
raise koji.BuildError, 'Repo/Target mismatch: %s/%s' \
|
|
% (repo_info['tag_name'], target_info['build_tag_name'])
|
|
else:
|
|
# if repo_id is specified, we can allow the 'target' arg to simply specify
|
|
# the destination tag (since the repo specifies the build tag).
|
|
if repo_info is None:
|
|
raise koji.GenericError, 'unknown build target: %s' % target
|
|
build_tag = repo_info['tag_id']
|
|
if target is None:
|
|
#ok, call it skip-tag for the buildroot tag
|
|
self.opts['skip_tag'] = True
|
|
dest_tag = build_tag
|
|
else:
|
|
taginfo = self.session.getTag(target, event=self.event_id)
|
|
if not taginfo:
|
|
raise koji.GenericError, 'neither tag nor target: %s' % target
|
|
dest_tag = taginfo['id']
|
|
#policy checks...
|
|
policy_data = {
|
|
'user_id' : task_info['owner'],
|
|
'source' : src,
|
|
'task_id' : self.id,
|
|
'build_tag' : build_tag, #id
|
|
'skip_tag' : bool(self.opts.get('skip_tag')),
|
|
}
|
|
if target_info:
|
|
policy_data['target'] = target_info['id'],
|
|
if not self.opts.get('skip_tag'):
|
|
policy_data['tag'] = dest_tag #id
|
|
if not SCM.is_scm_url(src) and not opts.get('scratch'):
|
|
#let hub policy decide
|
|
self.session.host.assertPolicy('build_from_srpm', policy_data)
|
|
if opts.get('repo_id') is not None:
|
|
# use of this option is governed by policy
|
|
self.session.host.assertPolicy('build_from_repo_id', policy_data)
|
|
if not repo_info:
|
|
repo_info = self.getRepo(build_tag) #(subtask)
|
|
self.event_id = self.session.getLastEvent()['id']
|
|
srpm = self.getSRPM(src, build_tag, repo_info['id'])
|
|
h = self.readSRPMHeader(srpm)
|
|
data = koji.get_header_fields(h,['name','version','release','epoch'])
|
|
data['task_id'] = self.id
|
|
extra_arches = None
|
|
self.logger.info("Reading package config for %(name)s" % data)
|
|
pkg_cfg = self.session.getPackageConfig(dest_tag,data['name'],event=self.event_id)
|
|
self.logger.debug("%r" % pkg_cfg)
|
|
if pkg_cfg is not None:
|
|
extra_arches = pkg_cfg.get('extra_arches')
|
|
if not self.opts.get('skip_tag') and not self.opts.get('scratch'):
|
|
# Make sure package is on the list for this tag
|
|
if pkg_cfg is None:
|
|
raise koji.BuildError, "package %s not in list for tag %s" \
|
|
% (data['name'], target_info['dest_tag_name'])
|
|
elif pkg_cfg['blocked']:
|
|
raise koji.BuildError, "package %s is blocked for tag %s" \
|
|
% (data['name'], target_info['dest_tag_name'])
|
|
# TODO - more pre tests
|
|
archlist = self.getArchList(build_tag, h, extra=extra_arches)
|
|
#let the system know about the build we're attempting
|
|
if not self.opts.get('scratch'):
|
|
#scratch builds do not get imported
|
|
build_id = self.session.host.initBuild(data)
|
|
#(initBuild raises an exception if there is a conflict)
|
|
try:
|
|
self.extra_information = { "src": src, "data": data, "target": target }
|
|
srpm,rpms,brmap,logs = self.runBuilds(srpm,build_tag,archlist,repo_info['id'])
|
|
|
|
if opts.get('scratch'):
|
|
#scratch builds do not get imported
|
|
self.session.host.moveBuildToScratch(self.id,srpm,rpms,logs=logs)
|
|
else:
|
|
self.session.host.completeBuild(self.id,build_id,srpm,rpms,brmap,logs=logs)
|
|
except (SystemExit,ServerExit,KeyboardInterrupt):
|
|
#we do not trap these
|
|
raise
|
|
except:
|
|
if not self.opts.get('scratch'):
|
|
#scratch builds do not get imported
|
|
self.session.host.failBuild(self.id, build_id)
|
|
# reraise the exception
|
|
raise
|
|
if not self.opts.get('skip_tag') and not self.opts.get('scratch'):
|
|
self.tagBuild(build_id,dest_tag)
|
|
|
|
def getSRPM(self, src, build_tag, repo_id):
|
|
"""Get srpm from src"""
|
|
if isinstance(src,str):
|
|
if SCM.is_scm_url(src):
|
|
return self.getSRPMFromSCM(src, build_tag, repo_id)
|
|
else:
|
|
#assume this is a path under uploads
|
|
return src
|
|
else:
|
|
raise koji.BuildError, 'Invalid source specification: %s' % src
|
|
#XXX - other methods?
|
|
|
|
def getSRPMFromSCM(self, url, build_tag, repo_id):
|
|
#TODO - allow different ways to get the srpm
|
|
task_id = self.session.host.subtask(method='buildSRPMFromSCM',
|
|
arglist=[url, build_tag, {'repo_id': repo_id}],
|
|
label='srpm',
|
|
parent=self.id)
|
|
# wait for subtask to finish
|
|
result = self.wait(task_id)[task_id]
|
|
srpm = result['srpm']
|
|
return srpm
|
|
|
|
def readSRPMHeader(self, srpm):
|
|
#srpm arg should be a path relative to <BASEDIR>/work
|
|
self.logger.debug("Reading SRPM")
|
|
relpath = "work/%s" % srpm
|
|
opts = dict([(k, getattr(self.options, k)) for k in 'topurl','topdir'])
|
|
opts['tempdir'] = self.workdir
|
|
fo = koji.openRemoteFile(relpath, **opts)
|
|
h = koji.get_rpm_header(fo)
|
|
fo.close()
|
|
if h[rpm.RPMTAG_SOURCEPACKAGE] != 1:
|
|
raise koji.BuildError, "%s is not a source package" % srpm
|
|
return h
|
|
|
|
def getArchList(self, build_tag, h, extra=None):
|
|
# get list of arches to build for
|
|
buildconfig = self.session.getBuildConfig(build_tag, event=self.event_id)
|
|
arches = buildconfig['arches']
|
|
if not arches:
|
|
#XXX - need to handle this better
|
|
raise koji.BuildError, "No arches for tag %(name)s [%(id)s]" % buildconfig
|
|
tag_archlist = [koji.canonArch(a) for a in arches.split()]
|
|
self.logger.debug('arches: %s' % arches)
|
|
if extra:
|
|
self.logger.debug('Got extra arches: %s' % extra)
|
|
arches = "%s %s" % (arches,extra)
|
|
archlist = arches.split()
|
|
self.logger.debug('base archlist: %r' % archlist)
|
|
# - adjust arch list based on srpm macros
|
|
buildarchs = h[rpm.RPMTAG_BUILDARCHS]
|
|
exclusivearch = h[rpm.RPMTAG_EXCLUSIVEARCH]
|
|
excludearch = h[rpm.RPMTAG_EXCLUDEARCH]
|
|
if buildarchs:
|
|
archlist = buildarchs
|
|
self.logger.debug('archlist after buildarchs: %r' % archlist)
|
|
if exclusivearch:
|
|
archlist = [ a for a in archlist if a in exclusivearch ]
|
|
self.logger.debug('archlist after exclusivearch: %r' % archlist)
|
|
if excludearch:
|
|
archlist = [ a for a in archlist if a not in excludearch ]
|
|
self.logger.debug('archlist after excludearch: %r' % archlist)
|
|
#noarch is funny
|
|
if 'noarch' not in excludearch and \
|
|
( 'noarch' in buildarchs or 'noarch' in exclusivearch ):
|
|
archlist.append('noarch')
|
|
override = self.opts.get('arch_override')
|
|
if self.opts.get('scratch') and override:
|
|
#only honor override for scratch builds
|
|
self.logger.debug('arch override: %s' % override)
|
|
archlist = override.split()
|
|
archdict = {}
|
|
for a in archlist:
|
|
# Filter based on canonical arches for tag
|
|
# This prevents building for an arch that we can't handle
|
|
if a == 'noarch' or koji.canonArch(a) in tag_archlist:
|
|
archdict[a] = 1
|
|
if not archdict:
|
|
raise koji.BuildError, "No matching arches were found"
|
|
return archdict.keys()
|
|
|
|
def runBuilds(self, srpm, build_tag, archlist, repo_id):
|
|
self.logger.debug("Spawning jobs for arches: %r" % (archlist))
|
|
subtasks = {}
|
|
keep_srpm = True
|
|
for arch in archlist:
|
|
if koji.util.multi_fnmatch(arch, self.options.literal_task_arches):
|
|
taskarch = arch
|
|
else:
|
|
taskarch = koji.canonArch(arch)
|
|
subtasks[arch] = self.session.host.subtask(method='buildArch',
|
|
arglist=[srpm, build_tag, arch, keep_srpm, {'repo_id': repo_id}],
|
|
label=arch,
|
|
parent=self.id,
|
|
arch=taskarch)
|
|
keep_srpm = False
|
|
|
|
self.logger.debug("Got subtasks: %r" % (subtasks))
|
|
self.logger.debug("Waiting on subtasks...")
|
|
|
|
# wait for subtasks to finish
|
|
failany = not getattr(self.options, 'build_arch_can_fail', False)
|
|
results = self.wait(subtasks.values(), all=True, failany=failany)
|
|
|
|
# finalize import
|
|
# merge data into needed args for completeBuild call
|
|
rpms = []
|
|
brmap = {}
|
|
logs = {}
|
|
built_srpm = None
|
|
for (arch, task_id) in subtasks.iteritems():
|
|
result = results[task_id]
|
|
self.logger.debug("DEBUG: %r : %r " % (arch,result,))
|
|
brootid = result['brootid']
|
|
for fn in result['rpms']:
|
|
rpms.append(fn)
|
|
brmap[fn] = brootid
|
|
for fn in result['logs']:
|
|
logs.setdefault(arch,[]).append(fn)
|
|
if result['srpms']:
|
|
if built_srpm:
|
|
raise koji.BuildError, "multiple builds returned a srpm. task %i" % self.id
|
|
else:
|
|
built_srpm = result['srpms'][0]
|
|
brmap[result['srpms'][0]] = brootid
|
|
if built_srpm:
|
|
srpm = built_srpm
|
|
else:
|
|
raise koji.BuildError("could not find a built srpm")
|
|
|
|
return srpm,rpms,brmap,logs
|
|
|
|
def tagBuild(self,build_id,dest_tag):
|
|
#XXX - need options to skip tagging and to force tagging
|
|
#create the tagBuild subtask
|
|
#this will handle the "post tests"
|
|
task_id = self.session.host.subtask(method='tagBuild',
|
|
arglist=[dest_tag,build_id,False,None,True],
|
|
label='tag',
|
|
parent=self.id,
|
|
arch='noarch')
|
|
self.wait(task_id)
|
|
|
|
|
|
class BaseBuildTask(BaseTaskHandler):
|
|
"""Base class for tasks the create a build root"""
|
|
|
|
def checkHostArch(self, tag, hostdata, event=None):
|
|
tagref = tag
|
|
if isinstance(tag, dict):
|
|
tagref = tag.get('id') or tag.get('name')
|
|
opts = {}
|
|
if event is not None:
|
|
opts['event'] = event
|
|
tag = self.session.getBuildConfig(tagref, **opts)
|
|
if tag and tag['arches']:
|
|
tag_arches = [koji.canonArch(a) for a in tag['arches'].split()]
|
|
host_arches = hostdata['arches'].split()
|
|
if not set(tag_arches).intersection(host_arches):
|
|
self.logger.info('Task %s (%s): tag arches (%s) and ' \
|
|
'host arches (%s) are disjoint' % \
|
|
(self.id, self.method,
|
|
', '.join(tag_arches), ', '.join(host_arches)))
|
|
return False
|
|
#otherwise...
|
|
# This is in principle an error condition, but this is not a good place
|
|
# to fail. Instead we proceed and let the task fail normally.
|
|
return True
|
|
|
|
|
|
class BuildArchTask(BaseBuildTask):
|
|
|
|
Methods = ['buildArch']
|
|
|
|
def weight(self):
|
|
return 1.5
|
|
|
|
def updateWeight(self, name):
|
|
"""
|
|
Update the weight of this task based on the package we're building.
|
|
weight is scaled from a minimum of 1.5 to a maximum of 6, based on
|
|
the average duration of a build of this package.
|
|
"""
|
|
avg = self.session.getAverageBuildDuration(name)
|
|
if not avg:
|
|
return
|
|
if avg < 0:
|
|
self.logger.warn("Negative average build duration for %s: %s", name, avg)
|
|
return
|
|
# increase the task weight by 0.75 for every hour of build duration
|
|
adj = (avg / 4800.0)
|
|
# cap the adjustment at +4.5
|
|
weight = self.weight() + min(4.5, adj)
|
|
self.session.host.setTaskWeight(self.id, weight)
|
|
|
|
def checkHost(self, hostdata):
|
|
tag = self.params[1]
|
|
return self.checkHostArch(tag, hostdata)
|
|
|
|
def srpm_sanity_checks(self, filename):
|
|
header = koji.get_rpm_header(filename)
|
|
|
|
if not header[rpm.RPMTAG_PACKAGER]:
|
|
raise koji.BuildError, "The build system failed to set the packager tag"
|
|
if not header[rpm.RPMTAG_VENDOR]:
|
|
raise koji.BuildError, "The build system failed to set the vendor tag"
|
|
if not header[rpm.RPMTAG_DISTRIBUTION]:
|
|
raise koji.BuildError, "The build system failed to set the distribution tag"
|
|
|
|
def handler(self, pkg, root, arch, keep_srpm, opts=None):
|
|
"""Build a package in a buildroot for one arch"""
|
|
ret = {}
|
|
if opts is None:
|
|
opts = {}
|
|
repo_id = opts.get('repo_id')
|
|
if not repo_id:
|
|
raise koji.BuildError, "A repo id must be provided"
|
|
repo_info = self.session.repoInfo(repo_id, strict=True)
|
|
event_id = repo_info['create_event']
|
|
|
|
# starting srpm should already have been uploaded by parent
|
|
self.logger.debug("Reading SRPM")
|
|
fn = self.localPath("work/%s" % pkg)
|
|
if not os.path.exists(fn):
|
|
raise koji.BuildError, "SRPM file missing: %s" % fn
|
|
# peel E:N-V-R from package
|
|
h = koji.get_rpm_header(fn)
|
|
name = h[rpm.RPMTAG_NAME]
|
|
ver = h[rpm.RPMTAG_VERSION]
|
|
rel = h[rpm.RPMTAG_RELEASE]
|
|
epoch = h[rpm.RPMTAG_EPOCH]
|
|
if h[rpm.RPMTAG_SOURCEPACKAGE] != 1:
|
|
raise koji.BuildError, "not a source package"
|
|
# Disable checking for distribution in the initial SRPM because it
|
|
# might have been built outside of the build system
|
|
# if not h[rpm.RPMTAG_DISTRIBUTION]:
|
|
# raise koji.BuildError, "the distribution tag is not set in the original srpm"
|
|
|
|
self.updateWeight(name)
|
|
|
|
rootopts = {
|
|
'repo_id': repo_id
|
|
}
|
|
br_arch = self.find_arch(arch, self.session.host.getHost(), self.session.getBuildConfig(root, event=event_id))
|
|
broot = BuildRoot(self.session, self.options, root, br_arch, self.id, **rootopts)
|
|
broot.workdir = self.workdir
|
|
|
|
self.logger.debug("Initializing buildroot")
|
|
broot.init()
|
|
|
|
# run build
|
|
self.logger.debug("Running build")
|
|
broot.build(fn,arch)
|
|
|
|
# extract results
|
|
resultdir = broot.resultdir()
|
|
rpm_files = []
|
|
srpm_files = []
|
|
log_files = []
|
|
unexpected = []
|
|
for f in os.listdir(resultdir):
|
|
# files here should have one of two extensions: .log and .rpm
|
|
if f[-4:] == ".log":
|
|
log_files.append(f)
|
|
elif f[-8:] == ".src.rpm":
|
|
srpm_files.append(f)
|
|
elif f[-4:] == ".rpm":
|
|
rpm_files.append(f)
|
|
else:
|
|
unexpected.append(f)
|
|
self.logger.debug("rpms: %r" % rpm_files)
|
|
self.logger.debug("srpms: %r" % srpm_files)
|
|
self.logger.debug("logs: %r" % log_files)
|
|
self.logger.debug("unexpected: %r" % unexpected)
|
|
|
|
# upload files to storage server
|
|
uploadpath = broot.getUploadPath()
|
|
for f in rpm_files:
|
|
self.uploadFile("%s/%s" % (resultdir,f))
|
|
self.logger.debug("keep srpm %i %s %s" % (self.id, keep_srpm, opts))
|
|
if keep_srpm:
|
|
if len(srpm_files) == 0:
|
|
raise koji.BuildError, "no srpm files found for task %i" % self.id
|
|
if len(srpm_files) > 1:
|
|
raise koji.BuildError, "mulitple srpm files found for task %i: %s" % (self.id, srpm_files)
|
|
|
|
# Run sanity checks. Any failures will throw a BuildError
|
|
self.srpm_sanity_checks("%s/%s" % (resultdir,srpm_files[0]))
|
|
|
|
self.logger.debug("uploading %s/%s to %s" % (resultdir,srpm_files[0], uploadpath))
|
|
self.uploadFile("%s/%s" % (resultdir,srpm_files[0]))
|
|
if rpm_files:
|
|
ret['rpms'] = [ "%s/%s" % (uploadpath,f) for f in rpm_files ]
|
|
else:
|
|
ret['rpms'] = []
|
|
if keep_srpm:
|
|
ret['srpms'] = [ "%s/%s" % (uploadpath,f) for f in srpm_files ]
|
|
else:
|
|
ret['srpms'] = []
|
|
ret['logs'] = [ "%s/%s" % (uploadpath,f) for f in log_files ]
|
|
|
|
ret['brootid'] = broot.id
|
|
|
|
broot.expire()
|
|
#Let TaskManager clean up
|
|
|
|
return ret
|
|
|
|
class MavenTask(MultiPlatformTask):
|
|
|
|
Methods = ['maven']
|
|
|
|
_taskWeight = 0.2
|
|
|
|
def handler(self, url, target, opts=None):
|
|
"""Use Maven to build the source from the given url"""
|
|
if opts is None:
|
|
opts = {}
|
|
self.opts = opts
|
|
target_info = self.session.getBuildTarget(target)
|
|
if not target_info:
|
|
raise koji.BuildError, 'unknown build target: %s' % target
|
|
dest_tag = self.session.getTag(target_info['dest_tag'], strict=True)
|
|
build_tag = self.session.getTag(target_info['build_tag'], strict=True)
|
|
|
|
repo_id = opts.get('repo_id')
|
|
if not repo_id:
|
|
repo = self.session.getRepo(build_tag['id'])
|
|
if repo:
|
|
repo_id = repo['id']
|
|
else:
|
|
raise koji.BuildError, 'no repo for tag %s' % build_tag['name']
|
|
|
|
build_opts = dslice(opts, ['goals', 'profiles', 'properties', 'envs', 'patches',
|
|
'packages', 'jvm_options', 'maven_options', 'deps'],
|
|
strict=False)
|
|
build_opts['repo_id'] = repo_id
|
|
|
|
self.build_task_id = self.session.host.subtask(method='buildMaven',
|
|
arglist=[url, build_tag, build_opts],
|
|
label='build',
|
|
parent=self.id,
|
|
arch='noarch')
|
|
maven_results = self.wait(self.build_task_id)[self.build_task_id]
|
|
maven_results['task_id'] = self.build_task_id
|
|
|
|
build_info = None
|
|
if not self.opts.get('scratch'):
|
|
maven_info = maven_results['maven_info']
|
|
if maven_info['version'].endswith('-SNAPSHOT'):
|
|
raise koji.BuildError, '-SNAPSHOT versions are only supported in scratch builds'
|
|
build_info = koji.maven_info_to_nvr(maven_info)
|
|
|
|
if not self.opts.get('skip_tag'):
|
|
dest_cfg = self.session.getPackageConfig(dest_tag['id'], build_info['name'])
|
|
# Make sure package is on the list for this tag
|
|
if dest_cfg is None:
|
|
raise koji.BuildError, "package %s not in list for tag %s" \
|
|
% (build_info['name'], dest_tag['name'])
|
|
elif dest_cfg['blocked']:
|
|
raise koji.BuildError, "package %s is blocked for tag %s" \
|
|
% (build_info['name'], dest_tag['name'])
|
|
|
|
build_info = self.session.host.initMavenBuild(self.id, build_info, maven_info)
|
|
self.build_id = build_info['id']
|
|
|
|
try:
|
|
rpm_results = None
|
|
spec_url = self.opts.get('specfile')
|
|
if spec_url:
|
|
rpm_results = self.buildWrapperRPM(spec_url, self.build_task_id, target_info, build_info, repo_id)
|
|
|
|
if self.opts.get('scratch'):
|
|
self.session.host.moveMavenBuildToScratch(self.id, maven_results, rpm_results)
|
|
else:
|
|
self.session.host.completeMavenBuild(self.id, self.build_id, maven_results, rpm_results)
|
|
except (SystemExit, ServerExit, KeyboardInterrupt):
|
|
# we do not trap these
|
|
raise
|
|
except:
|
|
if not self.opts.get('scratch'):
|
|
#scratch builds do not get imported
|
|
self.session.host.failBuild(self.id, self.build_id)
|
|
# reraise the exception
|
|
raise
|
|
|
|
if not self.opts.get('scratch') and not self.opts.get('skip_tag'):
|
|
tag_task_id = self.session.host.subtask(method='tagBuild',
|
|
arglist=[dest_tag['id'], self.build_id, False, None, True],
|
|
label='tag',
|
|
parent=self.id,
|
|
arch='noarch')
|
|
self.wait(tag_task_id)
|
|
|
|
class BuildMavenTask(BaseBuildTask):
|
|
|
|
Methods = ['buildMaven']
|
|
|
|
_taskWeight = 1.5
|
|
|
|
def _zip_dir(self, rootdir, filename):
|
|
rootbase = os.path.basename(rootdir)
|
|
roottrim = len(rootdir) - len(rootbase)
|
|
zfo = zipfile.ZipFile(filename, 'w', zipfile.ZIP_DEFLATED)
|
|
for dirpath, dirnames, filenames in os.walk(rootdir):
|
|
for skip in ['CVS', '.svn', '.git']:
|
|
if skip in dirnames:
|
|
dirnames.remove(skip)
|
|
for filename in filenames:
|
|
filepath = os.path.join(dirpath, filename)
|
|
if os.path.islink(filepath):
|
|
content = os.readlink(filepath)
|
|
st = os.lstat(filepath)
|
|
mtime = time.localtime(st.st_mtime)
|
|
info = zipfile.ZipInfo(filepath[roottrim:])
|
|
info.external_attr |= 0120000 << 16L # symlink file type
|
|
info.compress_type = zipfile.ZIP_STORED
|
|
info.date_time = mtime[:6]
|
|
zfo.writestr(info, content)
|
|
else:
|
|
zfo.write(filepath, filepath[roottrim:])
|
|
zfo.close()
|
|
|
|
def checkHost(self, hostdata):
|
|
tag = self.params[1]
|
|
return self.checkHostArch(tag, hostdata)
|
|
|
|
def handler(self, url, build_tag, opts=None):
|
|
if opts is None:
|
|
opts = {}
|
|
self.opts = opts
|
|
|
|
scm = SCM(url)
|
|
scm.assert_allowed(self.options.allowed_scms)
|
|
|
|
repo_id = opts.get('repo_id')
|
|
if not repo_id:
|
|
raise koji.BuildError, 'A repo_id must be provided'
|
|
repo_info = self.session.repoInfo(repo_id, strict=True)
|
|
event_id = repo_info['create_event']
|
|
|
|
br_arch = self.find_arch('noarch', self.session.host.getHost(), session.getBuildConfig(build_tag['id'], event=event_id))
|
|
maven_opts = opts.get('jvm_options')
|
|
if not maven_opts:
|
|
maven_opts = []
|
|
for opt in maven_opts:
|
|
if opt.startswith('-Xmx'):
|
|
break
|
|
else:
|
|
# Give the JVM 2G to work with by default, if the build isn't specifying its own max. memory
|
|
maven_opts.append('-Xmx2048m')
|
|
buildroot = BuildRoot(self.session, self.options, build_tag['id'], br_arch, self.id,
|
|
install_group='maven-build', setup_dns=True, repo_id=repo_id,
|
|
maven_opts=maven_opts, maven_envs=opts.get('envs'),
|
|
deps=opts.get('deps'))
|
|
buildroot.workdir = self.workdir
|
|
self.logger.debug("Initializing buildroot")
|
|
buildroot.init()
|
|
|
|
packages = opts.get('packages')
|
|
if packages:
|
|
rv = buildroot.mock(['--install'] + packages)
|
|
self.session.host.setBuildRootState(buildroot.id, 'BUILDING')
|
|
self.session.host.updateBuildRootList(buildroot.id, buildroot.getPackageList())
|
|
if rv:
|
|
buildroot.expire()
|
|
raise koji.BuildrootError, 'error installing packages, %s' % buildroot._mockResult(rv, logfile='mock_output.log')
|
|
|
|
if not os.path.exists('%s/usr/bin/mvn' % buildroot.rootdir()):
|
|
raise koji.BuildError, '/usr/bin/mvn was not found in the buildroot'
|
|
|
|
scmdir = '%s/maven/build' % buildroot.rootdir()
|
|
outputdir = '%s/maven/output' % buildroot.rootdir()
|
|
m2dir = '%s/builddir/.m2' % buildroot.rootdir()
|
|
repodir = '%s/builddir/.m2/repository' % buildroot.rootdir()
|
|
patchdir = '%s/maven/patches' % buildroot.rootdir()
|
|
|
|
koji.ensuredir(scmdir)
|
|
koji.ensuredir(outputdir)
|
|
koji.ensuredir(repodir)
|
|
koji.ensuredir(patchdir)
|
|
|
|
logfile = self.workdir + '/checkout.log'
|
|
uploadpath = self.getUploadDir()
|
|
|
|
# Check out sources from the SCM
|
|
sourcedir = scm.checkout(scmdir, self.session, uploadpath, logfile)
|
|
|
|
# zip up pristine sources for auditing purposes
|
|
self._zip_dir(sourcedir, os.path.join(outputdir, 'scm-sources.zip'))
|
|
|
|
# Checkout out patches, if present
|
|
if self.opts.get('patches'):
|
|
patchlog = self.workdir + '/patches.log'
|
|
patch_scm = SCM(self.opts.get('patches'))
|
|
patch_scm.assert_allowed(self.options.allowed_scms)
|
|
# never try to check out a common/ dir when checking out patches
|
|
patch_scm.use_common = False
|
|
patchcheckoutdir = patch_scm.checkout(patchdir, self.session, uploadpath, patchlog)
|
|
self._zip_dir(patchcheckoutdir, os.path.join(outputdir, 'patches.zip'))
|
|
|
|
# Apply patches, if present
|
|
if self.opts.get('patches'):
|
|
# filter out directories and files beginning with . (probably scm metadata)
|
|
patches = [patch for patch in os.listdir(patchcheckoutdir) if \
|
|
os.path.isfile(os.path.join(patchcheckoutdir, patch)) and \
|
|
patch.endswith('.patch')]
|
|
if not patches:
|
|
raise koji.BuildError, 'no patches found at %s' % self.opts.get('patches')
|
|
patches.sort()
|
|
for patch in patches:
|
|
cmd = ['/usr/bin/patch', '--verbose', '--no-backup-if-mismatch', '-d', sourcedir, '-p1', '-i', os.path.join(patchcheckoutdir, patch)]
|
|
ret = log_output(self.session, cmd[0], cmd, patchlog, uploadpath, logerror=1, append=1)
|
|
if ret:
|
|
raise koji.BuildError, 'error applying patches from %s, see patches.log for details' % self.opts.get('patches')
|
|
|
|
# Set ownership of the entire source tree to the mock user
|
|
uid = pwd.getpwnam(self.options.mockuser)[2]
|
|
gid = grp.getgrnam('mock')[2]
|
|
self.chownTree(scmdir, uid, gid)
|
|
self.chownTree(outputdir, uid, gid)
|
|
self.chownTree(m2dir, uid, gid)
|
|
if self.opts.get('patches'):
|
|
self.chownTree(patchdir, uid, gid)
|
|
|
|
settingsfile = '/builddir/.m2/settings.xml'
|
|
buildroot.writeMavenSettings(settingsfile, outputdir)
|
|
|
|
pomfile = 'pom.xml'
|
|
maven_options = self.opts.get('maven_options', [])
|
|
for i, opt in enumerate(maven_options):
|
|
if opt == '-f' or opt == '--file':
|
|
if len(maven_options) > (i + 1):
|
|
pomfile = maven_options[i + 1]
|
|
break
|
|
else:
|
|
raise koji.BuildError, '%s option requires a file path' % opt
|
|
elif opt.startswith('-f=') or opt.startswith('--file='):
|
|
pomfile = opt.split('=', 1)[1]
|
|
break
|
|
elif opt.startswith('-f'):
|
|
pomfile = opt[2:]
|
|
break
|
|
|
|
buildroot.mavenBuild(sourcedir, outputdir, repodir,
|
|
props=self.opts.get('properties'), profiles=self.opts.get('profiles'),
|
|
options=self.opts.get('maven_options'), goals=self.opts.get('goals'))
|
|
|
|
build_pom = os.path.join(sourcedir, pomfile)
|
|
if not os.path.exists(build_pom):
|
|
raise koji.BuildError, '%s does not exist' % pomfile
|
|
pom_info = koji.parse_pom(build_pom)
|
|
maven_info = koji.pom_to_maven_info(pom_info)
|
|
|
|
# give the zip files more descriptive names
|
|
os.rename(os.path.join(outputdir, 'scm-sources.zip'),
|
|
os.path.join(outputdir, maven_info['artifact_id'] + '-' +
|
|
maven_info['version'] + '-scm-sources.zip'))
|
|
if self.opts.get('patches'):
|
|
os.rename(os.path.join(outputdir, 'patches.zip'),
|
|
os.path.join(outputdir, maven_info['artifact_id'] + '-' +
|
|
maven_info['version'] + '-patches.zip'))
|
|
|
|
logs = ['checkout.log']
|
|
if self.opts.get('patches'):
|
|
logs.append('patches.log')
|
|
output_files = {}
|
|
|
|
for path, dirs, files in os.walk(outputdir):
|
|
if not files:
|
|
continue
|
|
reldir = path[len(outputdir) + 1:]
|
|
for filename in files:
|
|
root, ext = os.path.splitext(filename)
|
|
if ext == '.log':
|
|
logs.append(os.path.join(reldir, filename))
|
|
else:
|
|
output_files.setdefault(reldir, []).append(filename)
|
|
|
|
# upload the build output
|
|
for filepath in logs:
|
|
self.uploadFile(os.path.join(outputdir, filepath),
|
|
relPath=os.path.dirname(filepath))
|
|
for relpath, files in output_files.iteritems():
|
|
for filename in files:
|
|
self.uploadFile(os.path.join(outputdir, relpath, filename),
|
|
relPath=relpath)
|
|
|
|
# Should only find log files in the mock result directory.
|
|
# Don't upload these log files, they've already been streamed
|
|
# the hub.
|
|
for filename in os.listdir(buildroot.resultdir()):
|
|
root, ext = os.path.splitext(filename)
|
|
if ext == '.log':
|
|
filepath = os.path.join(buildroot.resultdir(), filename)
|
|
if os.path.isfile(filepath) and os.stat(filepath).st_size > 0:
|
|
# only files with content get uploaded to the hub
|
|
logs.append(filename)
|
|
|
|
buildroot.expire()
|
|
|
|
return {'maven_info': maven_info,
|
|
'buildroot_id': buildroot.id,
|
|
'logs': logs,
|
|
'files': output_files}
|
|
|
|
class WrapperRPMTask(BaseBuildTask):
|
|
"""Build a wrapper rpm around archives output from a Maven or Windows build.
|
|
May either be called as a subtask or as a separate
|
|
top-level task. In the latter case it can either associate the new rpms
|
|
with the existing build or create a new build."""
|
|
|
|
Methods = ['wrapperRPM']
|
|
|
|
_taskWeight = 1.5
|
|
|
|
def copy_fields(self, src, tgt, *fields):
|
|
for field in fields:
|
|
tgt[field] = src.get(field)
|
|
|
|
def spec_sanity_checks(self, filename):
|
|
spec = open(filename).read()
|
|
for tag in ("Packager", "Distribution", "Vendor"):
|
|
if re.match("%s:" % tag, spec, re.M):
|
|
raise koji.BuildError, "%s is not allowed to be set in spec file" % tag
|
|
for tag in ("packager", "distribution", "vendor"):
|
|
if re.match("%%define\s+%s\s+" % tag, spec, re.M):
|
|
raise koji.BuildError, "%s is not allowed to be defined in spec file" % tag
|
|
|
|
def checkHost(self, hostdata):
|
|
target = self.params[1]
|
|
return self.checkHostArch(target['build_tag'], hostdata)
|
|
|
|
def handler(self, spec_url, build_target, build, task, opts=None):
|
|
if not opts:
|
|
opts = {}
|
|
|
|
if not (build or task):
|
|
raise koji.BuildError, 'build and/or task must be specified'
|
|
|
|
values = {}
|
|
|
|
if build:
|
|
maven_info = self.session.getMavenBuild(build['id'], strict=False)
|
|
win_info = self.session.getWinBuild(build['id'], strict=False)
|
|
image_info = self.session.getImageBuild(build['id'], strict=False)
|
|
else:
|
|
maven_info = None
|
|
win_info = None
|
|
image_info = None
|
|
|
|
# list of artifact paths relative to kojiroot (not exposed to the specfile)
|
|
artifact_relpaths = []
|
|
# map of file extension to a list of files
|
|
artifacts = {}
|
|
# list of all files
|
|
all_artifacts = []
|
|
# list of all files with their repo path
|
|
all_artifacts_with_path = []
|
|
|
|
# makes generating relative paths easier
|
|
self.pathinfo = koji.PathInfo(topdir='')
|
|
|
|
if task:
|
|
# called as a subtask of a build
|
|
artifact_paths = self.session.listTaskOutput(task['id'])
|
|
|
|
for artifact_path in artifact_paths:
|
|
artifact_name = os.path.basename(artifact_path)
|
|
base, ext = os.path.splitext(artifact_name)
|
|
if ext == '.log':
|
|
# Exclude log files for consistency with the output of listArchives() used below
|
|
continue
|
|
relpath = os.path.join(self.pathinfo.task(task['id']), artifact_path)[1:]
|
|
artifact_relpaths.append(relpath)
|
|
artifacts.setdefault(ext, []).append(artifact_name)
|
|
all_artifacts.append(artifact_name)
|
|
all_artifacts_with_path.append(artifact_path)
|
|
else:
|
|
# called as a top-level task to create wrapper rpms for an existing build
|
|
# verify that the build is complete
|
|
if not build['state'] == koji.BUILD_STATES['COMPLETE']:
|
|
raise koji.BuildError, 'cannot call wrapperRPM on a build that did not complete successfully'
|
|
|
|
# get the list of files from the build instead of the task, because the task output directory may
|
|
# have already been cleaned up
|
|
if maven_info:
|
|
build_artifacts = self.session.listArchives(buildID=build['id'], type='maven')
|
|
elif win_info:
|
|
build_artifacts = self.session.listArchives(buildID=build['id'], type='win')
|
|
elif image_info:
|
|
build_artifacts = self.session.listArchives(buildID=build['id'], type='image')
|
|
else:
|
|
raise koji.BuildError, 'unsupported build type'
|
|
|
|
for artifact in build_artifacts:
|
|
artifact_name = artifact['filename']
|
|
base, ext = os.path.splitext(artifact_name)
|
|
artifacts.setdefault(ext, []).append(artifact_name)
|
|
all_artifacts.append(artifact_name)
|
|
if ext == '.log':
|
|
# listArchives() should never return .log files, but we check for completeness
|
|
continue
|
|
if maven_info:
|
|
repopath = self.pathinfo.mavenfile(artifact)
|
|
relpath = os.path.join(self.pathinfo.mavenbuild(build), repopath)[1:]
|
|
artifact_relpaths.append(relpath)
|
|
all_artifacts_with_path.append(repopath)
|
|
elif win_info:
|
|
repopath = self.pathinfo.winfile(artifact)
|
|
relpath = os.path.join(self.pathinfo.winbuild(build), repopath)[1:]
|
|
artifact_relpaths.append(relpath)
|
|
all_artifacts_with_path.append(repopath)
|
|
elif image_info:
|
|
ipath = self.pathinfo.imagebuild(build)
|
|
relpath = os.path.join(ipath, artifact_name)[1:]
|
|
artifact_relpaths.append(relpath)
|
|
all_artifacts_with_path.append(artifact_name)
|
|
else:
|
|
# can't happen
|
|
assert False
|
|
|
|
if not artifacts:
|
|
raise koji.BuildError, 'no output found for %s' % (task and koji.taskLabel(task) or koji.buildLabel(build))
|
|
|
|
values['artifacts'] = artifacts
|
|
values['all_artifacts'] = all_artifacts
|
|
values['all_artifacts_with_path'] = all_artifacts_with_path
|
|
|
|
if build:
|
|
self.copy_fields(build, values, 'epoch', 'name', 'version', 'release')
|
|
if maven_info:
|
|
values['maven_info'] = maven_info
|
|
elif win_info:
|
|
values['win_info'] = win_info
|
|
elif image_info:
|
|
values['image_info'] = image_info
|
|
else:
|
|
# can't happen
|
|
assert False
|
|
else:
|
|
task_result = self.session.getTaskResult(task['id'])
|
|
if task['method'] == 'buildMaven':
|
|
maven_info = task_result['maven_info']
|
|
maven_nvr = koji.maven_info_to_nvr(maven_info)
|
|
maven_nvr['release'] = '0.scratch'
|
|
self.copy_fields(maven_nvr, values, 'epoch', 'name', 'version', 'release')
|
|
values['maven_info'] = maven_info
|
|
elif task['method'] == 'vmExec':
|
|
self.copy_fields(task_result, values, 'epoch', 'name', 'version', 'release')
|
|
values['win_info'] = {'platform': task_result['platform']}
|
|
elif task['method'] in ('createLiveCD', 'createAppliance', 'createImage'):
|
|
self.copy_fields(task_result, values, 'epoch', 'name', 'version', 'release')
|
|
else:
|
|
# can't happen
|
|
assert False
|
|
|
|
scm = SCM(spec_url)
|
|
scm.assert_allowed(self.options.allowed_scms)
|
|
|
|
repo_id = opts.get('repo_id')
|
|
if not repo_id:
|
|
raise koji.BuildError, "A repo id must be provided"
|
|
|
|
repo_info = self.session.repoInfo(repo_id, strict=True)
|
|
event_id = repo_info['create_event']
|
|
build_tag = self.session.getTag(build_target['build_tag'], strict=True)
|
|
br_arch = self.find_arch('noarch', self.session.host.getHost(), self.session.getBuildConfig(build_tag['id'], event=event_id))
|
|
|
|
buildroot = BuildRoot(self.session, self.options, build_tag['id'], br_arch, self.id, install_group='wrapper-rpm-build', repo_id=repo_id)
|
|
buildroot.workdir = self.workdir
|
|
self.logger.debug("Initializing buildroot")
|
|
buildroot.init()
|
|
|
|
logfile = os.path.join(self.workdir, 'checkout.log')
|
|
scmdir = buildroot.rootdir() + '/tmp/scmroot'
|
|
koji.ensuredir(scmdir)
|
|
specdir = scm.checkout(scmdir, self.session, self.getUploadDir(), logfile)
|
|
|
|
spec_template = None
|
|
for path, dir, files in os.walk(specdir):
|
|
files.sort()
|
|
for filename in files:
|
|
if filename.endswith('.spec.tmpl'):
|
|
spec_template = os.path.join(path, filename)
|
|
break
|
|
if not spec_template:
|
|
raise koji.BuildError, 'no spec file template found at URL: %s' % spec_url
|
|
|
|
# Put the jars into the same directory as the specfile. This directory will be
|
|
# set to the rpm _sourcedir so other files in the SCM may be referenced in the
|
|
# specfile as well.
|
|
specdir = os.path.dirname(spec_template)
|
|
for relpath in artifact_relpaths:
|
|
localpath = self.localPath(relpath)
|
|
# RPM requires all SOURCE files in the srpm to be in the same directory, so
|
|
# we flatten any directory structure of the output files here.
|
|
# If multiple files in the build have the same basename, duplicate files will
|
|
# have their relative path prepended to their name, with / replaced with -.
|
|
destpath = os.path.join(specdir, os.path.basename(relpath))
|
|
if os.path.exists(destpath):
|
|
destpath = os.path.join(specdir, relpath.replace('/', '-'))
|
|
shutil.copy(localpath, destpath)
|
|
|
|
# change directory to the specdir to the template can reference files there
|
|
os.chdir(specdir)
|
|
contents = Cheetah.Template.Template(file=spec_template,
|
|
searchList=[values]).respond()
|
|
contents = contents.encode('utf-8')
|
|
|
|
specfile = spec_template[:-5]
|
|
specfd = file(specfile, 'w')
|
|
specfd.write(contents)
|
|
specfd.close()
|
|
|
|
# Run spec file sanity checks. Any failures will throw a BuildError
|
|
self.spec_sanity_checks(specfile)
|
|
|
|
# chown the specdir to the mock user, because srpm creation happens
|
|
# as an unprivileged user
|
|
uid = pwd.getpwnam(self.options.mockuser)[2]
|
|
gid = grp.getgrnam('mock')[2]
|
|
self.chownTree(specdir, uid, gid)
|
|
|
|
#build srpm
|
|
self.logger.debug("Running srpm build")
|
|
buildroot.build_srpm(specfile, specdir, None)
|
|
|
|
srpms = glob.glob('%s/*.src.rpm' % buildroot.resultdir())
|
|
if len(srpms) == 0:
|
|
raise koji.BuildError, 'no srpms found in %s' % buildroot.resultdir()
|
|
elif len(srpms) > 1:
|
|
raise koji.BuildError, 'multiple srpms found in %s: %s' % (buildroot.resultdir(), ', '.join(srpms))
|
|
else:
|
|
srpm = srpms[0]
|
|
|
|
shutil.move(srpm, self.workdir)
|
|
srpm = os.path.join(self.workdir, os.path.basename(srpm))
|
|
|
|
self.new_build_id = None
|
|
if opts.get('create_build') and not opts.get('scratch'):
|
|
h = koji.get_rpm_header(srpm)
|
|
data = koji.get_header_fields(h, ['name', 'version', 'release', 'epoch'])
|
|
data['task_id'] = self.id
|
|
self.logger.info("Reading package config for %(name)s" % data)
|
|
pkg_cfg = self.session.getPackageConfig(build_target['dest_tag'], data['name'])
|
|
if not opts.get('skip_tag'):
|
|
# Make sure package is on the list for this tag
|
|
if pkg_cfg is None:
|
|
raise koji.BuildError, "package %s not in list for tag %s" \
|
|
% (data['name'], build_target['dest_tag_name'])
|
|
elif pkg_cfg['blocked']:
|
|
raise koji.BuildError, "package %s is blocked for tag %s" \
|
|
% (data['name'], build_target['dest_tag_name'])
|
|
self.new_build_id = self.session.host.initBuild(data)
|
|
|
|
try:
|
|
buildroot.build(srpm)
|
|
except (SystemExit, ServerExit, KeyboardInterrupt):
|
|
raise
|
|
except:
|
|
if self.new_build_id:
|
|
self.session.host.failBuild(self.id, self.new_build_id)
|
|
raise
|
|
|
|
resultdir = buildroot.resultdir()
|
|
srpm = None
|
|
rpms = []
|
|
logs = ['checkout.log']
|
|
|
|
for filename in os.listdir(resultdir):
|
|
if filename.endswith('.src.rpm'):
|
|
if not srpm:
|
|
srpm = filename
|
|
else:
|
|
if self.new_build_id:
|
|
self.session.host.failBuild(self.id, self.new_build_id)
|
|
raise koji.BuildError, 'multiple srpms found in %s: %s, %s' % \
|
|
(resultdir, srpm, filename)
|
|
elif filename.endswith('.rpm'):
|
|
rpms.append(filename)
|
|
elif filename.endswith('.log'):
|
|
logs.append(filename)
|
|
else:
|
|
if self.new_build_id:
|
|
self.session.host.failBuild(self.id, self.new_build_id)
|
|
raise koji.BuildError, 'unexpected file found in %s: %s' % \
|
|
(resultdir, filename)
|
|
|
|
if not srpm:
|
|
if self.new_build_id:
|
|
self.session.host.failBuild(self.id, self.new_build_id)
|
|
raise koji.BuildError, 'no srpm found'
|
|
|
|
if not rpms:
|
|
if self.new_build_id:
|
|
self.session.host.failBuild(self.id, self.new_build_id)
|
|
raise koji.BuildError, 'no rpms found'
|
|
|
|
try:
|
|
for rpm in [srpm] + rpms:
|
|
self.uploadFile(os.path.join(resultdir, rpm))
|
|
except (SystemExit, ServerExit, KeyboardInterrupt):
|
|
raise
|
|
except:
|
|
if self.new_build_id:
|
|
self.session.host.failBuild(self.id, self.new_build_id)
|
|
raise
|
|
|
|
results = {'buildroot_id': buildroot.id,
|
|
'srpm': srpm,
|
|
'rpms': rpms,
|
|
'logs': logs}
|
|
|
|
if not task:
|
|
# Called as a standalone top-level task, so handle the rpms now.
|
|
# Otherwise we let the parent task handle it.
|
|
uploaddir = self.getUploadDir()
|
|
relsrpm = uploaddir + '/' + srpm
|
|
relrpms = [uploaddir + '/' + r for r in rpms]
|
|
rellogs = [uploaddir + '/' + l for l in logs]
|
|
if opts.get('scratch'):
|
|
self.session.host.moveBuildToScratch(self.id, relsrpm, relrpms, {'noarch': rellogs})
|
|
else:
|
|
if opts.get('create_build'):
|
|
brmap = dict.fromkeys([relsrpm] + relrpms, buildroot.id)
|
|
try:
|
|
self.session.host.completeBuild(self.id, self.new_build_id,
|
|
relsrpm, relrpms, brmap, {'noarch': rellogs})
|
|
except (SystemExit, ServerExit, KeyboardInterrupt):
|
|
raise
|
|
except:
|
|
self.session.host.failBuild(self.id, self.new_build_id)
|
|
raise
|
|
if not opts.get('skip_tag'):
|
|
tag_task_id = self.session.host.subtask(method='tagBuild',
|
|
arglist=[build_target['dest_tag'],
|
|
self.new_build_id, False, None, True],
|
|
label='tag', parent=self.id, arch='noarch')
|
|
self.wait(tag_task_id)
|
|
else:
|
|
self.session.host.importWrapperRPMs(self.id, build['id'], results)
|
|
|
|
# no need to upload logs, they've already been streamed to the hub
|
|
# during the build process
|
|
|
|
buildroot.expire()
|
|
|
|
return results
|
|
|
|
class ChainMavenTask(MultiPlatformTask):
|
|
|
|
Methods = ['chainmaven']
|
|
|
|
_taskWeight = 0.2
|
|
|
|
def handler(self, builds, target, opts=None):
|
|
"""Run a sequence of Maven builds in dependency order"""
|
|
if not opts:
|
|
opts = {}
|
|
target_info = self.session.getBuildTarget(target)
|
|
if not target_info:
|
|
raise koji.BuildError, 'unknown build target: %s' % target
|
|
dest_tag = self.session.getTag(target_info['dest_tag'], strict=True)
|
|
|
|
if not (opts.get('scratch') or opts.get('skip_tag')):
|
|
for package in builds:
|
|
dest_cfg = self.session.getPackageConfig(dest_tag['id'], package)
|
|
# Make sure package is on the list for this tag
|
|
if dest_cfg is None:
|
|
raise koji.BuildError, "package %s not in list for tag %s" \
|
|
% (package, dest_tag['name'])
|
|
elif dest_cfg['blocked']:
|
|
raise koji.BuildError, "package %s is blocked for tag %s" \
|
|
% (package, dest_tag['name'])
|
|
|
|
self.depmap = {}
|
|
for package, params in builds.items():
|
|
self.depmap[package] = set(params.get('buildrequires', []))
|
|
|
|
todo = copy.deepcopy(self.depmap)
|
|
running = {}
|
|
self.done = {}
|
|
self.results = []
|
|
|
|
while True:
|
|
ready = [package for package, deps in todo.items() if not deps]
|
|
if not ready and not running:
|
|
break
|
|
for package in ready:
|
|
params = builds[package]
|
|
buildtype = params.get('type', 'maven')
|
|
task_url = params['scmurl']
|
|
task_opts = dslice_ex(params, ['scmurl', 'buildrequires', 'type'], strict=False)
|
|
if buildtype == 'maven':
|
|
task_deps = list(self.depset(package))
|
|
if task_deps:
|
|
task_opts['deps'] = task_deps
|
|
|
|
if not opts.get('force'):
|
|
# check for a duplicate build (a build performed with the
|
|
# same scmurl and options)
|
|
dup_build = self.get_duplicate_build(dest_tag['name'], package, params, task_opts)
|
|
# if we find one, mark the package as built and remove it from todo
|
|
if dup_build:
|
|
self.done[package] = dup_build['nvr']
|
|
for deps in todo.values():
|
|
deps.discard(package)
|
|
del todo[package]
|
|
self.results.append('%s previously built from %s' % (dup_build['nvr'], task_url))
|
|
continue
|
|
task_opts.update(dslice(opts, ['skip_tag', 'scratch'], strict=False))
|
|
|
|
if buildtype == 'maven':
|
|
if opts.get('debug'):
|
|
task_opts.setdefault('maven_options', []).append('--debug')
|
|
task_id = self.subtask('maven', [task_url, target, task_opts],
|
|
label=package)
|
|
elif buildtype == 'wrapper':
|
|
pkg_to_wrap = params['buildrequires'][0]
|
|
to_wrap = self.done[pkg_to_wrap]
|
|
|
|
if isinstance(to_wrap, (int, long)):
|
|
task_to_wrap = self.session.getTaskInfo(to_wrap, request=True)
|
|
build_to_wrap = None
|
|
else:
|
|
build_to_wrap = self.session.getBuild(to_wrap, strict=True)
|
|
task_to_wrap = None
|
|
target_info = self.session.getBuildTarget(target, strict=True)
|
|
repo_info = self.getRepo(target_info['build_tag'])
|
|
task_opts['repo_id'] = repo_info['id']
|
|
task_id = self.subtask('wrapperRPM', [task_url, target_info,
|
|
build_to_wrap, task_to_wrap,
|
|
task_opts],
|
|
label=package)
|
|
else:
|
|
raise koji.BuildError, 'unsupported build type: %s' % buildtype
|
|
|
|
running[task_id] = package
|
|
del todo[package]
|
|
try:
|
|
results = self.wait(running.keys())
|
|
except (xmlrpclib.Fault, koji.GenericError), e:
|
|
# One task has failed, wait for the rest to complete before the
|
|
# chainmaven task fails. self.wait(all=True) should thrown an exception.
|
|
self.wait(all=True)
|
|
raise
|
|
# if we get here, results is a map whose keys are the ids of tasks
|
|
# that have completed successfully
|
|
for task_id in results:
|
|
package = running.pop(task_id)
|
|
task_url = builds[package]['scmurl']
|
|
if opts.get('scratch'):
|
|
if builds[package].get('type') == 'wrapper':
|
|
self.done[package] = task_id
|
|
else:
|
|
children = self.session.getTaskChildren(task_id)
|
|
for child in children:
|
|
# we want the ID of the buildMaven task because the
|
|
# output dir of that task is where the Maven repo is
|
|
if child['method'] == 'buildMaven':
|
|
self.done[package] = child['id']
|
|
break
|
|
else:
|
|
raise koji.BuildError, 'could not find buildMaven subtask of %s' % task_id
|
|
self.results.append('%s built from %s by task %s' % \
|
|
(package, task_url, task_id))
|
|
else:
|
|
task_builds = self.session.listBuilds(taskID=task_id)
|
|
if not task_builds:
|
|
raise koji.BuildError, 'could not find build for task %s' % task_id
|
|
task_build = task_builds[0]
|
|
self.done[package] = task_build['nvr']
|
|
self.results.append('%s built from %s' % (task_build['nvr'], task_url))
|
|
for deps in todo.values():
|
|
deps.discard(package)
|
|
|
|
if todo:
|
|
# should never happen, the client should have checked for circular dependencies
|
|
raise koji.BuildError, 'unable to run chain build, circular dependencies'
|
|
return self.results
|
|
|
|
def depset(self, package):
|
|
deps = set()
|
|
for dep in self.depmap[package]:
|
|
deps.add(self.done[dep])
|
|
deps.update(self.depset(dep))
|
|
return deps
|
|
|
|
def dicts_equal(self, a, b):
|
|
"""Check if two dicts are equal. They are considered equal if they
|
|
have the same keys and those keys have the same values. If a value is
|
|
list, it will be considered equal to a list with the same values in
|
|
a different order."""
|
|
akeys = a.keys()
|
|
bkeys = b.keys()
|
|
if sorted(akeys) != sorted(bkeys):
|
|
return False
|
|
for key in akeys:
|
|
aval = a.get(key)
|
|
bval = b.get(key)
|
|
if type(aval) != type(bval):
|
|
return False
|
|
if isinstance(aval, dict):
|
|
if not self.dicts_equal(aval, bval):
|
|
return False
|
|
elif isinstance(aval, list):
|
|
if not sorted(aval) == sorted(bval):
|
|
return False
|
|
else:
|
|
if not aval == bval:
|
|
return False
|
|
return True
|
|
|
|
def get_duplicate_build(self, tag, package, params, task_opts):
|
|
"""Find the latest build of package in tag and compare it to the
|
|
scmurl and task_opts. If they're identical, return the build."""
|
|
builds = self.session.getLatestBuilds(tag, package=package)
|
|
if not builds:
|
|
return None
|
|
build = builds[0]
|
|
if not build['task_id']:
|
|
return None
|
|
build_task = session.getTaskInfo(build['task_id'], request=True)
|
|
request = build_task['request']
|
|
if request[0] != params['scmurl']:
|
|
return None
|
|
if params.get('type') == 'wrapper':
|
|
wrapped_build = request[2]
|
|
pkg_to_wrap = params['buildrequires'][0]
|
|
nvr_to_wrap = self.done[pkg_to_wrap]
|
|
if wrapped_build['nvr'] != nvr_to_wrap:
|
|
return None
|
|
# For a wrapper-rpm build, the only parameters that really matter
|
|
# are the scmurl and the wrapped NVR. These both match, so
|
|
# return the existing build.
|
|
return build
|
|
if len(request) > 2:
|
|
build_opts = dslice_ex(request[2], ['skip_tag', 'scratch'], strict=False)
|
|
else:
|
|
build_opts = {}
|
|
task_opts = copy.deepcopy(task_opts)
|
|
# filter out options that don't affect the build output
|
|
# to avoid unnecessary rebuilds
|
|
for opts in [build_opts, task_opts]:
|
|
if 'maven_options' in opts:
|
|
maven_options = opts['maven_options']
|
|
for opt in ['-e', '--errors', '-q', '--quiet',
|
|
'-V', '--show-version', '-X', '--debug']:
|
|
if opt in maven_options:
|
|
maven_options.remove(opt)
|
|
if not maven_options:
|
|
del opts['maven_options']
|
|
if 'jvm_options' in opts:
|
|
del opts['jvm_options']
|
|
if not self.dicts_equal(build_opts, task_opts):
|
|
return None
|
|
# everything matches
|
|
return build
|
|
|
|
class TagBuildTask(BaseTaskHandler):
|
|
|
|
Methods = ['tagBuild']
|
|
#XXX - set weight?
|
|
|
|
def handler(self, tag_id, build_id, force=False, fromtag=None, ignore_success=False):
|
|
task = self.session.getTaskInfo(self.id)
|
|
user_id = task['owner']
|
|
try:
|
|
build = self.session.getBuild(build_id, strict=True)
|
|
tag = self.session.getTag(tag_id, strict=True)
|
|
|
|
#several basic sanity checks have already been run (and will be run
|
|
#again when we make the final call). Our job is to perform the more
|
|
#computationally expensive 'post' tests.
|
|
|
|
#XXX - add more post tests
|
|
self.session.host.tagBuild(self.id,tag_id,build_id,force=force,fromtag=fromtag)
|
|
self.session.host.tagNotification(True, tag_id, fromtag, build_id, user_id, ignore_success)
|
|
except Exception, e:
|
|
exctype, value = sys.exc_info()[:2]
|
|
self.session.host.tagNotification(False, tag_id, fromtag, build_id, user_id, ignore_success, "%s: %s" % (exctype, value))
|
|
raise e
|
|
|
|
class BuildImageTask(MultiPlatformTask):
|
|
|
|
def initImageBuild(self, name, version, release, target_info, opts):
|
|
"""create a build object for this image build"""
|
|
pkg_cfg = self.session.getPackageConfig(target_info['dest_tag_name'],
|
|
name)
|
|
self.logger.debug("%r" % pkg_cfg)
|
|
if not opts.get('skip_tag') and not opts.get('scratch'):
|
|
# Make sure package is on the list for this tag
|
|
if pkg_cfg is None:
|
|
raise koji.BuildError, "package (image) %s not in list for tag %s" % (name, target_info['dest_tag_name'])
|
|
elif pkg_cfg['blocked']:
|
|
raise koji.BuildError, "package (image) %s is blocked for tag %s" % (name, target_info['dest_tag_name'])
|
|
return self.session.host.initImageBuild(self.id,
|
|
dict(name=name, version=version, release=release, epoch=0))
|
|
|
|
def getRelease(self, name, ver):
|
|
"""return the next available release number for an N-V"""
|
|
return self.session.getNextRelease(dict(name=name, version=ver))
|
|
|
|
class BuildBaseImageTask(BuildImageTask):
|
|
Methods = ['image']
|
|
|
|
def handler(self, name, version, arches, target, inst_tree, opts=None):
|
|
"""Governing task for building an appliance using Oz"""
|
|
target_info = self.session.getBuildTarget(target, strict=True)
|
|
build_tag = target_info['build_tag']
|
|
repo_info = self.getRepo(build_tag)
|
|
#check requested arches against build tag
|
|
buildconfig = self.session.getBuildConfig(build_tag)
|
|
if not buildconfig['arches']:
|
|
raise koji.BuildError, "No arches for tag %(name)s [%(id)s]" % buildconfig
|
|
tag_archlist = [koji.canonArch(a) for a in buildconfig['arches'].split()]
|
|
for arch in arches:
|
|
if koji.canonArch(arch) not in tag_archlist:
|
|
raise koji.BuildError, "Invalid arch for build tag: %s" % arch
|
|
|
|
if not opts:
|
|
opts = {}
|
|
|
|
if not ozif_enabled:
|
|
self.logger.error("ImageFactory features require the following dependencies: pykickstart, imagefactory, oz and possibly python-hashlib")
|
|
raise koji.ApplianceError, 'ImageFactory functions not available'
|
|
|
|
# build image(s)
|
|
bld_info = None
|
|
try:
|
|
release = opts.get('release')
|
|
if not release:
|
|
release = self.getRelease(name, version)
|
|
if '-' in version:
|
|
raise koji.ApplianceError('The Version may not have a hyphen')
|
|
if '-' in release:
|
|
raise koji.ApplianceError('The Release may not have a hyphen')
|
|
if not opts.get('scratch'):
|
|
bld_info = self.initImageBuild(name, version, release,
|
|
target_info, opts)
|
|
|
|
subtasks = {}
|
|
self.logger.debug("Spawning jobs for image arches: %r" % (arches))
|
|
for arch in arches:
|
|
inst_url = inst_tree.replace('$arch', arch)
|
|
subtasks[arch] = self.session.host.subtask(
|
|
method='createImage',
|
|
arglist=[name, version, release, arch, target_info,
|
|
build_tag, repo_info, inst_url, opts],
|
|
label=arch, parent=self.id, arch=arch)
|
|
self.logger.debug("Got image subtasks: %r" % (subtasks))
|
|
self.logger.debug("Waiting on image subtasks...")
|
|
results = self.wait(subtasks.values(), all=True, failany=True)
|
|
|
|
# wrap in an RPM if asked
|
|
rpm_results = None
|
|
spec_url = opts.get('specfile')
|
|
for arch in arches:
|
|
# get around an xmlrpc limitation, use arches for keys instead
|
|
results[arch] = results[subtasks[arch]]
|
|
del results[subtasks[arch]]
|
|
if spec_url:
|
|
subtask = subtasks[arch]
|
|
results[arch]['rpmresults'] = self.buildWrapperRPM(
|
|
spec_url, subtask, target_info, bld_info,
|
|
repo_info['id'])
|
|
|
|
# make sure we only import the user-submitted kickstart file one
|
|
# time, otherwise we will have collisions. Remove it from exactly
|
|
# 1 results hash from the subtasks
|
|
if opts.has_key('kickstart'):
|
|
saw_ks = False
|
|
for arch in results.keys():
|
|
ks = os.path.basename(opts.get('kickstart'))
|
|
if ks in results[arch]['files']:
|
|
if saw_ks:
|
|
results[arch]['files'].remove(ks)
|
|
saw_ks = True
|
|
|
|
self.logger.debug('Image Results for hub: %s' % results)
|
|
if opts.get('scratch'):
|
|
self.session.host.moveImageBuildToScratch(self.id, results)
|
|
else:
|
|
self.session.host.completeImageBuild(self.id, bld_info['id'],
|
|
results)
|
|
|
|
except (SystemExit,ServerExit,KeyboardInterrupt):
|
|
#we do not trap these
|
|
raise
|
|
except:
|
|
if not opts.get('scratch'):
|
|
#scratch builds do not get imported
|
|
if bld_info:
|
|
self.session.host.failBuild(self.id, bld_info['id'])
|
|
# reraise the exception
|
|
raise
|
|
|
|
# tag it
|
|
if not opts.get('scratch') and not opts.get('skip_tag'):
|
|
tag_task_id = self.session.host.subtask(method='tagBuild',
|
|
arglist=[target_info['dest_tag'], bld_info['id'], False, None, True],
|
|
label='tag', parent=self.id, arch='noarch')
|
|
self.wait(tag_task_id)
|
|
|
|
# report results
|
|
report = ''
|
|
if opts.get('scratch'):
|
|
respath = ', '.join(
|
|
[os.path.join(koji.pathinfo.work(), koji.pathinfo.taskrelpath(tid)) for tid in subtasks.values()])
|
|
report += 'Scratch '
|
|
else:
|
|
respath = koji.pathinfo.imagebuild(bld_info)
|
|
report += 'image build results in: %s' % respath
|
|
return report
|
|
|
|
|
|
class BuildApplianceTask(BuildImageTask):
|
|
Methods = ['appliance']
|
|
|
|
def handler(self, name, version, arch, target, ksfile, opts=None):
|
|
"""Governing task for building an appliance"""
|
|
target_info = self.session.getBuildTarget(target, strict=True)
|
|
build_tag = target_info['build_tag']
|
|
repo_info = self.getRepo(build_tag)
|
|
#check requested arch against build tag
|
|
buildconfig = self.session.getBuildConfig(build_tag)
|
|
if not buildconfig['arches']:
|
|
raise koji.BuildError, "No arches for tag %(name)s [%(id)s]" % buildconfig
|
|
tag_archlist = [koji.canonArch(a) for a in buildconfig['arches'].split()]
|
|
if koji.canonArch(arch) not in tag_archlist:
|
|
raise koji.BuildError, "Invalid arch for build tag: %s" % arch
|
|
|
|
|
|
if not opts:
|
|
opts = {}
|
|
|
|
if not image_enabled:
|
|
self.logger.error("Appliance features require the following dependencies: pykickstart, and possibly python-hashlib")
|
|
raise koji.ApplianceError, 'Appliance functions not available'
|
|
|
|
# build image
|
|
bld_info = None
|
|
try:
|
|
release = opts.get('release')
|
|
if not release:
|
|
release = self.getRelease(name, version)
|
|
if not opts.get('scratch'):
|
|
bld_info = self.initImageBuild(name, version, release,
|
|
target_info, opts)
|
|
create_task_id = self.session.host.subtask(method='createAppliance',
|
|
arglist=[name, version, release, arch, target_info, build_tag,
|
|
repo_info, ksfile, opts],
|
|
label='appliance', parent=self.id, arch=arch)
|
|
results = self.wait(create_task_id)
|
|
self.logger.info('image build task (%s) completed' % create_task_id)
|
|
self.logger.info('results: %s' % results)
|
|
|
|
# wrap in an RPM if asked
|
|
rpm_results = None
|
|
spec_url = opts.get('specfile')
|
|
if spec_url:
|
|
results[create_task_id]['rpmresults'] = self.buildWrapperRPM(
|
|
spec_url, create_task_id,
|
|
target_info, bld_info, repo_info['id'])
|
|
results[str(create_task_id)] = results[create_task_id]
|
|
del results[create_task_id]
|
|
|
|
# import the image (move it too)
|
|
if not opts.get('scratch'):
|
|
self.session.host.completeImageBuild(self.id, bld_info['id'], results)
|
|
else:
|
|
self.session.host.moveImageBuildToScratch(self.id, results)
|
|
|
|
except (SystemExit,ServerExit,KeyboardInterrupt):
|
|
#we do not trap these
|
|
raise
|
|
except:
|
|
if not opts.get('scratch'):
|
|
#scratch builds do not get imported
|
|
if bld_info:
|
|
self.session.host.failBuild(self.id, bld_info['id'])
|
|
# reraise the exception
|
|
raise
|
|
|
|
# tag it
|
|
if not opts.get('scratch') and not opts.get('skip_tag'):
|
|
tag_task_id = self.session.host.subtask(method='tagBuild',
|
|
arglist=[target_info['dest_tag'], bld_info['id'], False, None, True],
|
|
label='tag', parent=self.id, arch='noarch')
|
|
self.wait(tag_task_id)
|
|
|
|
# report results
|
|
if opts.get('scratch'):
|
|
respath = os.path.join(koji.pathinfo.work(),
|
|
koji.pathinfo.taskrelpath(create_task_id))
|
|
report = 'Scratch '
|
|
else:
|
|
respath = koji.pathinfo.imagebuild(bld_info)
|
|
report = ''
|
|
report += 'appliance build results in: %s' % respath
|
|
return report
|
|
|
|
class BuildLiveCDTask(BuildImageTask):
|
|
Methods = ['livecd']
|
|
|
|
def handler(self, name, version, arch, target, ksfile, opts=None):
|
|
"""Governing task for building LiveCDs"""
|
|
target_info = self.session.getBuildTarget(target, strict=True)
|
|
build_tag = target_info['build_tag']
|
|
repo_info = self.getRepo(build_tag)
|
|
#check requested arch against build tag
|
|
buildconfig = self.session.getBuildConfig(build_tag)
|
|
if not buildconfig['arches']:
|
|
raise koji.BuildError, "No arches for tag %(name)s [%(id)s]" % buildconfig
|
|
tag_archlist = [koji.canonArch(a) for a in buildconfig['arches'].split()]
|
|
if koji.canonArch(arch) not in tag_archlist:
|
|
raise koji.BuildError, "Invalid arch for build tag: %s" % arch
|
|
|
|
if not opts:
|
|
opts = {}
|
|
if not image_enabled:
|
|
self.logger.error("LiveCD features require the following dependencies: "
|
|
"pykickstart, pycdio, and possibly python-hashlib")
|
|
raise koji.LiveCDError, 'LiveCD functions not available'
|
|
|
|
# build the image
|
|
bld_info = None
|
|
try:
|
|
release = opts.get('release')
|
|
if not release:
|
|
release = self.getRelease(name, version)
|
|
if not opts.get('scratch'):
|
|
bld_info = self.initImageBuild(name, version, release,
|
|
target_info, opts)
|
|
create_task_id = self.session.host.subtask(method='createLiveCD',
|
|
arglist=[name, version, release, arch, target_info, build_tag,
|
|
repo_info, ksfile, opts],
|
|
label='livecd', parent=self.id, arch=arch)
|
|
results = self.wait(create_task_id)
|
|
self.logger.info('image build task (%s) completed' % create_task_id)
|
|
self.logger.info('results: %s' % results)
|
|
|
|
# wrap in an RPM if needed
|
|
spec_url = opts.get('specfile')
|
|
rpm_results = None
|
|
if spec_url:
|
|
results[create_task_id]['rpmresults'] = self.buildWrapperRPM(
|
|
spec_url, create_task_id,
|
|
target_info, bld_info, repo_info['id'])
|
|
results[str(create_task_id)] = results[create_task_id]
|
|
del results[create_task_id]
|
|
|
|
# import it (and move)
|
|
if not opts.get('scratch'):
|
|
self.session.host.completeImageBuild(self.id, bld_info['id'], results)
|
|
else:
|
|
self.session.host.moveImageBuildToScratch(self.id, results)
|
|
|
|
except (SystemExit,ServerExit,KeyboardInterrupt):
|
|
#we do not trap these
|
|
raise
|
|
except:
|
|
if not opts.get('scratch'):
|
|
#scratch builds do not get imported
|
|
if bld_info:
|
|
self.session.host.failBuild(self.id, bld_info['id'])
|
|
# reraise the exception
|
|
raise
|
|
|
|
# tag it if necessary
|
|
if not opts.get('scratch') and not opts.get('skip_tag'):
|
|
tag_task_id = self.session.host.subtask(method='tagBuild',
|
|
arglist=[target_info['dest_tag'], bld_info['id'], False, None, True],
|
|
label='tag', parent=self.id, arch='noarch')
|
|
self.wait(tag_task_id)
|
|
|
|
# report the results
|
|
if opts.get('scratch'):
|
|
respath = os.path.join(koji.pathinfo.work(),
|
|
koji.pathinfo.taskrelpath(create_task_id))
|
|
report = 'Scratch '
|
|
else:
|
|
respath = koji.pathinfo.imagebuild(bld_info)
|
|
report = ''
|
|
report += 'livecd build results in: %s' % respath
|
|
return report
|
|
|
|
|
|
class BuildLiveMediaTask(BuildImageTask):
|
|
Methods = ['livemedia']
|
|
|
|
def handler(self, name, version, arches, target, ksfile, opts=None):
|
|
"""Governing task for building live media"""
|
|
|
|
target_info = self.session.getBuildTarget(target, strict=True)
|
|
build_tag = target_info['build_tag']
|
|
repo_info = self.getRepo(build_tag)
|
|
#check requested arch against build tag
|
|
buildconfig = self.session.getBuildConfig(build_tag)
|
|
if not buildconfig['arches']:
|
|
raise koji.BuildError, "No arches for tag %(name)s [%(id)s]" % buildconfig
|
|
tag_archlist = [koji.canonArch(a) for a in buildconfig['arches'].split()]
|
|
|
|
# check arches and remove duplicates
|
|
arches = set(arches)
|
|
for arch in arches:
|
|
if koji.canonArch(arch) not in tag_archlist:
|
|
raise koji.BuildError, "Invalid arch for build tag: %s" % arch
|
|
|
|
if not opts:
|
|
opts = {}
|
|
if not image_enabled:
|
|
# XXX - are these still required here?
|
|
self.logger.error("Missing the following dependencies: "
|
|
"pykickstart, pycdio, and possibly python-hashlib")
|
|
raise koji.PreBuildError, 'Live Media functions not available'
|
|
|
|
# build the image
|
|
bld_info = None
|
|
try:
|
|
release = opts.get('release')
|
|
if not release:
|
|
release = self.getRelease(name, version)
|
|
if not opts.get('scratch'):
|
|
bld_info = self.initImageBuild(name, version, release,
|
|
target_info, opts)
|
|
subtasks = {}
|
|
for arch in arches:
|
|
subtasks[arch] = self.subtask('createLiveMedia',
|
|
[name, version, release, arch, target_info, build_tag,
|
|
repo_info, ksfile, opts],
|
|
label='livemedia %s' % arch, arch=arch)
|
|
|
|
|
|
self.logger.debug("Got image subtasks: %r", subtasks)
|
|
self.logger.debug("Waiting on livemedia subtasks...")
|
|
results = self.wait(subtasks.values(), all=True, failany=True)
|
|
|
|
self.logger.debug('subtask results: %r', results)
|
|
|
|
# wrap each image an RPM if needed
|
|
spec_url = opts.get('specfile')
|
|
if spec_url:
|
|
wrapper_tasks = {}
|
|
for arch in arches:
|
|
subtask_id = subtasks[arch]
|
|
result = results[subtask_id]
|
|
tinfo = self.session.getTaskInfo(subtask_id)
|
|
arglist = [spec_url, target_info, bld_info, tinfo,
|
|
{'repo_id': repo_info['id']}]
|
|
wrapper_tasks[arch] = self.subtask('wrapperRPM', arglist,
|
|
label='wrapper %s' % arch, arch='noarch')
|
|
|
|
results2 = self.wait(wrapper_tasks.values(), all=True, failany=True)
|
|
self.logger.debug('wrapper results: %r', results2)
|
|
|
|
# add wrapper rpm results into main results
|
|
for arch in arches:
|
|
result = results[subtasks[arch]]
|
|
result2 = results2[wrapper_tasks[arch]]
|
|
result['rpmresults'] = result2
|
|
|
|
# re-key results for xmlrpc friendliness
|
|
results = dict([(str(k), results[k]) for k in results])
|
|
|
|
# import it (and move)
|
|
if not opts.get('scratch'):
|
|
self.session.host.completeImageBuild(self.id, bld_info['id'], results)
|
|
else:
|
|
self.session.host.moveImageBuildToScratch(self.id, results)
|
|
|
|
except (SystemExit, ServerExit, KeyboardInterrupt):
|
|
#we do not trap these
|
|
raise
|
|
except:
|
|
if not opts.get('scratch'):
|
|
#scratch builds do not get imported
|
|
if bld_info:
|
|
self.session.host.failBuild(self.id, bld_info['id'])
|
|
# reraise the exception
|
|
raise
|
|
|
|
# tag it if necessary
|
|
if not opts.get('scratch') and not opts.get('skip_tag'):
|
|
tag_task_id = self.session.host.subtask(method='tagBuild',
|
|
arglist=[target_info['dest_tag'], bld_info['id'], False, None, True],
|
|
label='tag', parent=self.id, arch='noarch')
|
|
self.wait(tag_task_id)
|
|
|
|
# report the results
|
|
if opts.get('scratch'):
|
|
respath = ', '.join(
|
|
[os.path.join(koji.pathinfo.work(), koji.pathinfo.taskrelpath(tid)) for tid in subtasks.values()])
|
|
report = 'Scratch '
|
|
else:
|
|
respath = koji.pathinfo.imagebuild(bld_info)
|
|
report = ''
|
|
report += 'livecd build results in: %s' % respath
|
|
return report
|
|
|
|
|
|
# A generic task for building cd or disk images using chroot-based tools.
|
|
# Other chroot-based image handlers should inherit this.
|
|
class ImageTask(BaseTaskHandler):
|
|
Methods = []
|
|
|
|
def makeImgBuildRoot(self, buildtag, repoinfo, arch, inst_group):
|
|
"""
|
|
Create and prepare the chroot we're going to build an image in.
|
|
Binds necessary directories and creates needed device files.
|
|
|
|
@args:
|
|
buildtag: a build tag
|
|
repoinfo: a session.getRepo() object
|
|
arch: a canonical architecture name
|
|
inst_group: a string representing the yum group to install with
|
|
@returns: a buildroot object
|
|
"""
|
|
|
|
# Here we configure mock to bind mount a set of /dev directories
|
|
bind_opts = {'dirs' : {'/dev' : '/dev',}}
|
|
if os.path.exists('/selinux'):
|
|
bind_opts['dirs']['/selinux'] = '/selinux'
|
|
rootopts = {'install_group': inst_group,
|
|
'setup_dns': True,
|
|
'repo_id': repoinfo['id'],
|
|
'bind_opts' : bind_opts}
|
|
|
|
broot = BuildRoot(self.session, self.options, buildtag, arch, self.id, **rootopts)
|
|
broot.workdir = self.workdir
|
|
|
|
# create the mock chroot
|
|
self.logger.debug("Initializing image buildroot")
|
|
broot.init()
|
|
self.logger.debug("Image buildroot ready: " + broot.rootdir())
|
|
return broot
|
|
|
|
def fetchKickstart(self, broot, ksfile):
|
|
"""
|
|
Retrieve the kickstart file we were given (locally or remotely) and
|
|
upload it.
|
|
|
|
Note that if the KS file existed locally, then "ksfile" is a relative
|
|
path to it in the /mnt/koji/work directory. If not, then it is still
|
|
the parameter the user passed in initially, and we assume it is a
|
|
relative path in a remote scm. The user should have passed in an scm
|
|
url with --ksurl.
|
|
|
|
@args:
|
|
broot: a buildroot object
|
|
ksfile: path to a kickstart file
|
|
@returns: absolute path to the retrieved kickstart file
|
|
"""
|
|
scmdir = os.path.join(broot.rootdir(), 'tmp')
|
|
koji.ensuredir(scmdir)
|
|
self.logger.debug("ksfile = %s" % ksfile)
|
|
if self.opts.get('ksurl'):
|
|
scm = SCM(self.opts['ksurl'])
|
|
scm.assert_allowed(self.options.allowed_scms)
|
|
logfile = os.path.join(self.workdir, 'checkout.log')
|
|
scmsrcdir = scm.checkout(scmdir, self.session, self.getUploadDir(), logfile)
|
|
kspath = os.path.join(scmsrcdir, ksfile)
|
|
else:
|
|
kspath = self.localPath("work/%s" % ksfile)
|
|
|
|
self.uploadFile(kspath) # upload the original ks file
|
|
return kspath # full absolute path to the file in the chroot
|
|
|
|
def readKickstart(self, kspath, opts):
|
|
"""
|
|
Read a kickstart file and save the ks object as a task member.
|
|
|
|
@args:
|
|
kspath: path to a kickstart file
|
|
@returns: None
|
|
"""
|
|
# XXX: If the ks file came from a local path and has %include
|
|
# macros, *-creator will fail because the included
|
|
# kickstarts were not copied into the chroot. For now we
|
|
# require users to flatten their kickstart file if submitting
|
|
# the task with a local path.
|
|
#
|
|
# Note that if an SCM URL was used instead, %include macros
|
|
# may not be a problem if the included kickstarts are present
|
|
# in the repository we checked out.
|
|
if opts.get('ksversion'):
|
|
version = ksparser.version.makeVersion(opts['ksversion'])
|
|
else:
|
|
version = ksparser.version.makeVersion()
|
|
self.ks = ksparser.KickstartParser(version)
|
|
try:
|
|
self.ks.readKickstart(kspath)
|
|
except IOError, e:
|
|
raise koji.LiveCDError("Failed to read kickstart file "
|
|
"'%s' : %s" % (kspath, e))
|
|
except kserrors.KickstartError, e:
|
|
raise koji.LiveCDError("Failed to parse kickstart file "
|
|
"'%s' : %s" % (kspath, e))
|
|
|
|
def prepareKickstart(self, repo_info, target_info, arch, broot, opts):
|
|
"""
|
|
Process the ks file to be used for controlled image generation. This
|
|
method also uploads the modified kickstart file to the task output
|
|
area.
|
|
|
|
@args:
|
|
target_info: a sesion.getBuildTarget() object
|
|
repo_info: session.getRepo() object
|
|
arch: canonical architecture name
|
|
broot: a buildroot object
|
|
kspath: absolute path to a kickstart file
|
|
@returns:
|
|
absolute path to a processed kickstart file within the buildroot
|
|
"""
|
|
# Now we do some kickstart manipulation. If the user passed in a repo
|
|
# url with --repo, then we substitute that in for the repo(s) specified
|
|
# in the kickstart file. If --repo wasn't specified, then we use the
|
|
# repo associated with the target passed in initially.
|
|
repo_class = kscontrol.dataMap[self.ks.version]['RepoData']
|
|
self.ks.handler.repo.repoList = [] # delete whatever the ks file told us
|
|
if opts.get('repo'):
|
|
user_repos = opts['repo']
|
|
if isinstance(user_repos, basestring):
|
|
user_repos = user_repos.split(',')
|
|
index = 0
|
|
for user_repo in user_repos:
|
|
self.ks.handler.repo.repoList.append(repo_class(baseurl=user_repo, name='koji-override-%i' % index))
|
|
index += 1
|
|
else:
|
|
path_info = koji.PathInfo(topdir=self.options.topurl)
|
|
repopath = path_info.repo(repo_info['id'],
|
|
target_info['build_tag_name'])
|
|
baseurl = '%s/%s' % (repopath, arch)
|
|
self.logger.debug('BASEURL: %s' % baseurl)
|
|
self.ks.handler.repo.repoList.append(repo_class(baseurl=baseurl, name='koji-%s-%i' % (target_info['build_tag_name'], repo_info['id'])))
|
|
#inject url if provided
|
|
if opts.get('install_tree_url'):
|
|
self.ks.handler.url(url=opts['install_tree_url'])
|
|
|
|
# Write out the new ks file. Note that things may not be in the same
|
|
# order and comments in the original ks file may be lost.
|
|
kskoji = os.path.join('/tmp', 'koji-image-%s-%i.ks' %
|
|
(target_info['build_tag_name'], self.id))
|
|
kojikspath = os.path.join(broot.rootdir(), kskoji[1:])
|
|
outfile = open(kojikspath, 'w')
|
|
outfile.write(str(self.ks.handler))
|
|
outfile.close()
|
|
|
|
# put the new ksfile in the output directory
|
|
if not os.path.exists(kojikspath):
|
|
raise koji.LiveCDError, "KS file missing: %s" % kojikspath
|
|
self.uploadFile(kojikspath)
|
|
return kskoji # absolute path within chroot
|
|
|
|
def getImagePackages(self, cachepath):
|
|
"""
|
|
Read RPM header information from the yum cache available in the
|
|
given path. Returns a list of dictionaries for each RPM included.
|
|
"""
|
|
found = False
|
|
hdrlist = []
|
|
fields = ['name', 'version', 'release', 'epoch', 'arch', \
|
|
'buildtime', 'sigmd5']
|
|
for root, dirs, files in os.walk(cachepath):
|
|
for f in files:
|
|
if fnmatch(f, '*.rpm'):
|
|
pkgfile = os.path.join(root, f)
|
|
hdr = koji.get_header_fields(pkgfile, fields)
|
|
hdr['size'] = os.path.getsize(pkgfile)
|
|
hdr['payloadhash'] = koji.hex_string(hdr['sigmd5'])
|
|
del hdr['sigmd5']
|
|
hdrlist.append(hdr)
|
|
found = True
|
|
if not found:
|
|
raise koji.LiveCDError, 'No repos found in yum cache!'
|
|
return hdrlist
|
|
|
|
def _shortenVolID(self, name, version, release):
|
|
# Based on code from pungi
|
|
substitutions = {
|
|
'MATE_Compiz': 'MATE',
|
|
'Security': 'Sec',
|
|
'Electronic_Lab': 'Elec',
|
|
'Robotics': 'Robo',
|
|
'Scientific_KDE': 'SciK',
|
|
'Design_suite': 'Dsgn',
|
|
'Games': 'Game',
|
|
'Jam_KDE': 'Jam',
|
|
'Workstation': 'WS',
|
|
'Server': 'S',
|
|
'Cloud': 'C',
|
|
'Alpha': 'A',
|
|
'Beta': 'B',
|
|
'TC': 'T',
|
|
}
|
|
|
|
for k, v in substitutions.iteritems():
|
|
if k in name:
|
|
name = name.replace(k, v)
|
|
if k in version:
|
|
version = version.replace(k, v)
|
|
if k in release:
|
|
release = release.replace(k, v)
|
|
|
|
volid = "%s-%s-%s" % (name, version, release)
|
|
return volid[:32]
|
|
|
|
|
|
# ApplianceTask begins with a mock chroot, and then installs appliance-tools
|
|
# into it via the appliance-build group. appliance-creator is then executed
|
|
# in the chroot to create the appliance image.
|
|
#
|
|
class ApplianceTask(ImageTask):
|
|
|
|
Methods = ['createAppliance']
|
|
_taskWeight = 1.5
|
|
|
|
def getRootDevice(self):
|
|
"""
|
|
Return the device name for the / partition, as specified in the
|
|
kickstart file. Appliances should have this defined.
|
|
"""
|
|
for part in self.ks.handler.partition.partitions:
|
|
if part.mountpoint == '/':
|
|
return part.disk
|
|
raise koji.ApplianceError, 'kickstart lacks a "/" mountpoint'
|
|
|
|
def handler(self, name, version, release, arch, target_info, build_tag, repo_info, ksfile, opts=None):
|
|
|
|
if opts == None:
|
|
opts = {}
|
|
self.opts = opts
|
|
broot = self.makeImgBuildRoot(build_tag, repo_info, arch,
|
|
'appliance-build')
|
|
kspath = self.fetchKickstart(broot, ksfile)
|
|
self.readKickstart(kspath, opts)
|
|
kskoji = self.prepareKickstart(repo_info, target_info, arch, broot, opts)
|
|
# Figure out appliance-creator arguments, let it fail if something
|
|
# is wrong.
|
|
odir = 'app-output'
|
|
opath = os.path.join(broot.rootdir(), 'tmp', odir)
|
|
cachedir = '/tmp/koji-appliance' # arbitrary paths in chroot
|
|
app_log = '/tmp/appliance.log'
|
|
os.mkdir(opath)
|
|
|
|
cmd = ['/usr/bin/appliance-creator', '-c', kskoji, '-d', '-v',
|
|
'--logfile', app_log, '--cache', cachedir, '-o', odir]
|
|
for arg_name in ('vmem', 'vcpu', 'format'):
|
|
arg = opts.get(arg_name)
|
|
if arg != None:
|
|
cmd.extend(['--%s' % arg_name, arg])
|
|
appname = '%s-%s-%s' % (name, version, release)
|
|
cmd.extend(['--name', appname])
|
|
cmd.extend(['--version', version, '--release', release])
|
|
|
|
# Run appliance-creator
|
|
rv = broot.mock(['--cwd', '/tmp', '--chroot', '--'] + cmd)
|
|
self.uploadFile(os.path.join(broot.rootdir(), app_log[1:]))
|
|
if rv:
|
|
raise koji.ApplianceError, \
|
|
"Could not create appliance: %s" % parseStatus(rv, 'appliance-creator') + "; see root.log or appliance.log for more information"
|
|
|
|
# Find the results
|
|
results = []
|
|
for directory, subdirs, files in os.walk(opath):
|
|
for f in files:
|
|
results.append(os.path.join(broot.rootdir(), 'tmp',
|
|
directory, f))
|
|
self.logger.debug('output: %s' % results)
|
|
if len(results) == 0:
|
|
raise koji.ApplianceError, "Could not find image build results!"
|
|
imgdata = {
|
|
'arch': arch,
|
|
'rootdev': self.getRootDevice(),
|
|
'task_id': self.id,
|
|
'logs': ['build.log', 'mock_output.log', 'root.log', 'state.log',
|
|
'appliance.log', os.path.basename(ksfile),
|
|
os.path.basename(kskoji)],
|
|
'name': name,
|
|
'version': version,
|
|
'release': release
|
|
}
|
|
imgdata['files'] = []
|
|
for ofile in results:
|
|
self.uploadFile(ofile)
|
|
imgdata['files'].append(os.path.basename(ofile))
|
|
|
|
# TODO: get file manifest from the appliance
|
|
|
|
if not opts.get('scratch'):
|
|
hdrlist = self.getImagePackages(os.path.join(broot.rootdir(),
|
|
cachedir[1:]))
|
|
broot.markExternalRPMs(hdrlist)
|
|
imgdata['rpmlist'] = hdrlist
|
|
|
|
broot.expire()
|
|
return imgdata
|
|
|
|
# LiveCDTask begins with a mock chroot, and then installs livecd-tools into it
|
|
# via the livecd-build group. livecd-creator is then executed in the chroot
|
|
# to create the LiveCD image.
|
|
#
|
|
class LiveCDTask(ImageTask):
|
|
|
|
Methods = ['createLiveCD']
|
|
_taskWeight = 1.5
|
|
|
|
def genISOManifest(self, image, manifile):
|
|
"""
|
|
Using iso9660 from pycdio, get the file manifest of the given image,
|
|
and save it to the text file manifile.
|
|
"""
|
|
fd = open(manifile, 'w')
|
|
if not fd:
|
|
raise koji.GenericError, \
|
|
'Unable to open manifest file (%s) for writing!' % manifile
|
|
iso = iso9660.ISO9660.IFS(source=image)
|
|
if not iso.is_open():
|
|
raise koji.GenericError, \
|
|
'Could not open %s as an ISO-9660 image!' % image
|
|
|
|
# image metadata
|
|
id = iso.get_application_id()
|
|
if id is not None: fd.write("Application ID: %s\n" % id)
|
|
id = iso.get_preparer_id()
|
|
if id is not None: fd.write("Preparer ID: %s\n" % id)
|
|
id = iso.get_publisher_id()
|
|
if id is not None: fd.write("Publisher ID: %s\n" % id)
|
|
id = iso.get_system_id()
|
|
if id is not None: fd.write("System ID: %s\n" % id)
|
|
id = iso.get_volume_id()
|
|
if id is not None: fd.write("Volume ID: %s\n" % id)
|
|
id = iso.get_volumeset_id()
|
|
if id is not None: fd.write("Volumeset ID: %s\n" % id)
|
|
|
|
fd.write('\nSize(bytes) File Name\n')
|
|
manifest = self.listISODir(iso, '/')
|
|
for a_file in manifest:
|
|
fd.write(a_file)
|
|
fd.close()
|
|
iso.close()
|
|
|
|
def listISODir(self, iso, path):
|
|
"""
|
|
Helper function called recursively by genISOManifest. Returns a
|
|
listing of files/directories at the given path in an iso image obj.
|
|
"""
|
|
manifest = []
|
|
file_stats = iso.readdir(path)
|
|
for stat in file_stats:
|
|
filename = stat[0]
|
|
size = stat[2]
|
|
is_dir = stat[4] == 2
|
|
|
|
if filename == '..':
|
|
continue
|
|
elif filename == '.':
|
|
# path should always end in a trailing /
|
|
filepath = path
|
|
else:
|
|
filepath = path + filename
|
|
# identify directories with a trailing /
|
|
if is_dir:
|
|
filepath += '/'
|
|
|
|
if is_dir and filename != '.':
|
|
# recurse into subdirectories
|
|
manifest.extend(self.listISODir(iso, filepath))
|
|
else:
|
|
# output information for the current directory and files
|
|
manifest.append("%-10d %s\n" % (size, filepath))
|
|
|
|
return manifest
|
|
|
|
|
|
def handler(self, name, version, release, arch, target_info, build_tag, repo_info, ksfile, opts=None):
|
|
|
|
if opts == None:
|
|
opts = {}
|
|
self.opts = opts
|
|
|
|
broot = self.makeImgBuildRoot(build_tag, repo_info, arch,
|
|
'livecd-build')
|
|
kspath = self.fetchKickstart(broot, ksfile)
|
|
self.readKickstart(kspath, opts)
|
|
kskoji = self.prepareKickstart(repo_info, target_info, arch, broot, opts)
|
|
|
|
cachedir = '/tmp/koji-livecd' # arbitrary paths in chroot
|
|
livecd_log = '/tmp/livecd.log'
|
|
cmd = ['/usr/bin/livecd-creator', '-c', kskoji, '-d', '-v',
|
|
'--logfile', livecd_log, '--cache', cachedir]
|
|
isoname = '%s-%s-%s' % (name, version, release)
|
|
cmd.extend(['-f', self._shortenVolID(name, version, release)])
|
|
|
|
# Run livecd-creator
|
|
rv = broot.mock(['--cwd', '/tmp', '--chroot', '--'] + cmd)
|
|
self.uploadFile(os.path.join(broot.rootdir(), livecd_log[1:]))
|
|
if rv:
|
|
raise koji.LiveCDError, \
|
|
'Could not create LiveCD: %s' % parseStatus(rv, 'livecd-creator') + '; see root.log or livecd.log for more information'
|
|
|
|
# Find the resultant iso
|
|
# The cwd of the livecd-creator process is /tmp in the chroot, so
|
|
# that is where it writes the .iso
|
|
files = os.listdir(os.path.join(broot.rootdir(), 'tmp'))
|
|
isofile = None
|
|
for afile in files:
|
|
if afile.endswith('.iso'):
|
|
if not isofile:
|
|
isofile = afile
|
|
else:
|
|
raise koji.LiveCDError, 'multiple .iso files found: %s and %s' % (isofile, afile)
|
|
if not isofile:
|
|
raise koji.LiveCDError, 'could not find iso file in chroot'
|
|
isosrc = os.path.join(broot.rootdir(), 'tmp', isofile)
|
|
|
|
# copy the iso out of the chroot. If we were given an isoname,
|
|
# this is where the renaming happens.
|
|
self.logger.debug('uploading image: %s' % isosrc)
|
|
isoname += '.iso'
|
|
|
|
# Generate the file manifest of the image, upload the results
|
|
manifest = os.path.join(broot.resultdir(), 'manifest.log')
|
|
self.genISOManifest(isosrc, manifest)
|
|
self.uploadFile(manifest)
|
|
self.uploadFile(isosrc, remoteName=isoname)
|
|
|
|
imgdata = {'arch': arch,
|
|
'files': [isoname],
|
|
'rootdev': None,
|
|
'task_id': self.id,
|
|
'logs': ['build.log', 'mock_output.log', 'root.log', 'state.log',
|
|
'livecd.log', os.path.basename(ksfile),
|
|
os.path.basename(kskoji)],
|
|
'name': name,
|
|
'version': version,
|
|
'release': release
|
|
}
|
|
if not opts.get('scratch'):
|
|
hdrlist = self.getImagePackages(os.path.join(broot.rootdir(),
|
|
cachedir[1:]))
|
|
imgdata ['rpmlist'] = hdrlist
|
|
broot.markExternalRPMs(hdrlist)
|
|
|
|
broot.expire()
|
|
return imgdata
|
|
|
|
|
|
|
|
## livemedia-creator
|
|
class LiveMediaTask(ImageTask):
|
|
|
|
Methods = ['createLiveMedia']
|
|
_taskWeight = 1.5
|
|
|
|
def genISOManifest(self, image, manifile):
|
|
"""
|
|
Using iso9660 from pycdio, get the file manifest of the given image,
|
|
and save it to the text file manifile.
|
|
"""
|
|
fd = open(manifile, 'w')
|
|
if not fd:
|
|
raise koji.GenericError, \
|
|
'Unable to open manifest file (%s) for writing!' % manifile
|
|
iso = iso9660.ISO9660.IFS(source=image)
|
|
if not iso.is_open():
|
|
raise koji.GenericError, \
|
|
'Could not open %s as an ISO-9660 image!' % image
|
|
|
|
# image metadata
|
|
id = iso.get_application_id()
|
|
if id is not None: fd.write("Application ID: %s\n" % id)
|
|
id = iso.get_preparer_id()
|
|
if id is not None: fd.write("Preparer ID: %s\n" % id)
|
|
id = iso.get_publisher_id()
|
|
if id is not None: fd.write("Publisher ID: %s\n" % id)
|
|
id = iso.get_system_id()
|
|
if id is not None: fd.write("System ID: %s\n" % id)
|
|
id = iso.get_volume_id()
|
|
if id is not None: fd.write("Volume ID: %s\n" % id)
|
|
id = iso.get_volumeset_id()
|
|
if id is not None: fd.write("Volumeset ID: %s\n" % id)
|
|
|
|
fd.write('\nSize(bytes) File Name\n')
|
|
manifest = self.listISODir(iso, '/')
|
|
for a_file in manifest:
|
|
fd.write(a_file)
|
|
fd.close()
|
|
iso.close()
|
|
|
|
def listISODir(self, iso, path):
|
|
"""
|
|
Helper function called recursively by genISOManifest. Returns a
|
|
listing of files/directories at the given path in an iso image obj.
|
|
"""
|
|
manifest = []
|
|
file_stats = iso.readdir(path)
|
|
for stat in file_stats:
|
|
filename = stat[0]
|
|
size = stat[2]
|
|
is_dir = stat[4] == 2
|
|
|
|
if filename == '..':
|
|
continue
|
|
elif filename == '.':
|
|
# path should always end in a trailing /
|
|
filepath = path
|
|
else:
|
|
filepath = path + filename
|
|
# identify directories with a trailing /
|
|
if is_dir:
|
|
filepath += '/'
|
|
|
|
if is_dir and filename != '.':
|
|
# recurse into subdirectories
|
|
manifest.extend(self.listISODir(iso, filepath))
|
|
else:
|
|
# output information for the current directory and files
|
|
manifest.append("%-10d %s\n" % (size, filepath))
|
|
|
|
return manifest
|
|
|
|
def handler(self, name, version, release, arch, target_info, build_tag, repo_info, ksfile, opts=None):
|
|
|
|
if opts == None:
|
|
opts = {}
|
|
self.opts = opts
|
|
|
|
broot = self.makeImgBuildRoot(build_tag, repo_info, arch,
|
|
'livemedia-build')
|
|
kspath = self.fetchKickstart(broot, ksfile)
|
|
self.readKickstart(kspath, opts)
|
|
kskoji = self.prepareKickstart(repo_info, target_info, arch, broot, opts)
|
|
|
|
cachedir = '/tmp/koji-livemedia' # arbitrary paths in chroot
|
|
livemedia_log = '/tmp/lmc-logs/livemedia-out.log'
|
|
resultdir = '/tmp/lmc'
|
|
|
|
|
|
# Common LMC command setup, needs extending
|
|
cmd = ['/sbin/livemedia-creator',
|
|
'--ks', kskoji,
|
|
'--logfile', livemedia_log,
|
|
'--no-virt',
|
|
'--resultdir', resultdir,
|
|
'--project', name,
|
|
#'--tmp', '/tmp'
|
|
]
|
|
|
|
# note: at the moment, we are only generating live isos. We may add support
|
|
# for other types in the future
|
|
|
|
cmd.extend(['--make-iso',
|
|
'--volid', self._shortenVolID(name, version, release),
|
|
'--iso-only',
|
|
])
|
|
|
|
isoname='%s-%s-%s-%s.iso' % (name, arch, version, release)
|
|
title = self.opts.get('title', name)
|
|
cmd.extend(['--iso-name', isoname,
|
|
'--releasever', version,
|
|
'--title', title,
|
|
])
|
|
|
|
|
|
if arch == 'x86_64':
|
|
cmd.append('--macboot')
|
|
|
|
|
|
# Run livemedia-creator
|
|
rv = broot.mock(['--cwd', '/tmp', '--chroot', '--'] + cmd)
|
|
|
|
# upload logs
|
|
logdirs = [
|
|
os.path.join(broot.rootdir(), 'tmp/lmc-logs'),
|
|
os.path.join(broot.rootdir(), 'tmp/lmc-logs/anaconda'),
|
|
]
|
|
for logdir in logdirs:
|
|
if not os.path.isdir(logdir):
|
|
continue
|
|
for filename in os.listdir(logdir):
|
|
if not filename.endswith('.log'):
|
|
continue
|
|
filepath = os.path.join(logdir, filename)
|
|
if os.stat(filepath).st_size == 0:
|
|
continue
|
|
# avoid file duplication between directories by prefixing anaconda logs
|
|
if logdir.endswith('anaconda'):
|
|
self.uploadFile(os.path.join(filepath), remoteName='anaconda-%s' % filename)
|
|
continue
|
|
|
|
self.uploadFile(os.path.join(filepath))
|
|
|
|
if rv:
|
|
raise koji.LiveMediaError, \
|
|
'Could not create LiveMedia: %s' % parseStatus(rv, 'livemedia-creator') + '; see root.log or livemedia-out.log for more information'
|
|
|
|
# Find the resultant iso
|
|
# The cwd of the livemedia-creator process is /tmp in the chroot, so
|
|
# that is where it writes the .iso
|
|
rootresultsdir = os.path.join(broot.rootdir(), resultdir.lstrip('/'))
|
|
files = os.listdir(rootresultsdir)
|
|
isofile = None
|
|
for afile in files:
|
|
if afile.endswith('.iso'):
|
|
if not isofile:
|
|
isofile = afile
|
|
else:
|
|
raise koji.LiveMediaError, 'multiple .iso files found: %s and %s' % (isofile, afile)
|
|
if not isofile:
|
|
raise koji.LiveMediaError, 'could not find iso file in chroot'
|
|
isosrc = os.path.join(rootresultsdir, isofile)
|
|
|
|
|
|
# Generate the file manifest of the image, upload the results
|
|
manifest = os.path.join(broot.resultdir(), 'manifest.log')
|
|
self.genISOManifest(isosrc, manifest)
|
|
self.uploadFile(manifest)
|
|
self.logger.debug('uploading image: %s' % isosrc)
|
|
self.uploadFile(isosrc, remoteName=isoname)
|
|
|
|
imgdata = {'arch': arch,
|
|
'files': [isoname],
|
|
'rootdev': None,
|
|
'task_id': self.id,
|
|
'logs': ['build.log', 'mock_output.log', 'root.log', 'state.log',
|
|
'livemedia-out.log', os.path.basename(ksfile),
|
|
os.path.basename(kskoji)],
|
|
'name': name,
|
|
'version': version,
|
|
'release': release
|
|
}
|
|
if not opts.get('scratch'):
|
|
# TODO - generate list of rpms in image
|
|
# (getImagePackages doesn't work here)
|
|
#hdrlist = self.getImagePackages(os.path.join(broot.rootdir(),
|
|
# cachedir[1:]))
|
|
imgdata ['rpmlist'] = []
|
|
#broot.markExternalRPMs(hdrlist)
|
|
|
|
broot.expire()
|
|
return imgdata
|
|
|
|
# A generic task for building disk images using Oz
|
|
# Other Oz-based image handlers should inherit this.
|
|
class OzImageTask(BaseTaskHandler):
|
|
Methods = []
|
|
|
|
def fetchKickstart(self):
|
|
"""
|
|
Retrieve the kickstart file we were given (locally or remotely) and
|
|
upload it to the hub.
|
|
|
|
Note that if the KS file existed locally, then "ksfile" is a relative
|
|
path to it in the /mnt/koji/work directory. If not, then it is still
|
|
the parameter the user passed in initially, and we assume it is a
|
|
relative path in a remote scm. The user should have passed in an scm
|
|
url with --ksurl.
|
|
|
|
@args: None, use self.opts for options
|
|
@returns:
|
|
absolute path to the retrieved kickstart file
|
|
"""
|
|
ksfile = self.opts.get('kickstart')
|
|
self.logger.debug("ksfile = %s" % ksfile)
|
|
if self.opts.get('ksurl'):
|
|
scm = SCM(self.opts['ksurl'])
|
|
scm.assert_allowed(self.options.allowed_scms)
|
|
logfile = os.path.join(self.workdir, 'checkout-%s.log' % self.arch)
|
|
scmsrcdir = scm.checkout(self.workdir, self.session,
|
|
self.getUploadDir(), logfile)
|
|
kspath = os.path.join(scmsrcdir, os.path.basename(ksfile))
|
|
else:
|
|
tops = dict([(k, getattr(self.options, k)) for k in 'topurl','topdir'])
|
|
tops['tempdir'] = self.workdir
|
|
ks_src = koji.openRemoteFile(ksfile, **tops)
|
|
kspath = os.path.join(self.workdir, os.path.basename(ksfile))
|
|
ks_dest = open(kspath, 'w')
|
|
ks_dest.write(ks_src.read())
|
|
ks_dest.close()
|
|
self.logger.debug('uploading kickstart from here: %s' % kspath)
|
|
self.uploadFile(kspath) # upload the original ks file
|
|
return kspath # absolute path to the ks file
|
|
|
|
def readKickstart(self, kspath):
|
|
"""
|
|
Read a kickstart file and save the ks object as a task member.
|
|
|
|
@args:
|
|
kspath: path to a kickstart file
|
|
@returns:
|
|
a kickstart object returned by pykickstart
|
|
"""
|
|
# XXX: If the ks file came from a local path and has %include
|
|
# macros, Oz will fail because it can only handle flat files.
|
|
# We require users to flatten their kickstart file.
|
|
if self.opts.get('ksversion'):
|
|
version = ksparser.version.makeVersion(self.opts['ksversion'])
|
|
else:
|
|
version = ksparser.version.makeVersion()
|
|
ks = ksparser.KickstartParser(version)
|
|
self.logger.debug('attempting to read kickstart: %s' % kspath)
|
|
try:
|
|
ks.readKickstart(kspath)
|
|
except IOError, e:
|
|
raise koji.BuildError("Failed to read kickstart file "
|
|
"'%s' : %s" % (kspath, e))
|
|
except kserrors.KickstartError, e:
|
|
raise koji.BuildError("Failed to parse kickstart file "
|
|
"'%s' : %s" % (kspath, e))
|
|
return ks
|
|
|
|
def prepareKickstart(self, kspath, install_tree):
|
|
"""
|
|
Process the ks file to be used for controlled image generation. This
|
|
method also uploads the modified kickstart file to the task output
|
|
area on the hub.
|
|
|
|
@args:
|
|
kspath: a path to a kickstart file
|
|
@returns:
|
|
a kickstart object with koji-specific modifications
|
|
"""
|
|
ks = self.readKickstart(kspath)
|
|
# Now we do some kickstart manipulation. If the user passed in a repo
|
|
# url with --repo, then we substitute that in for the repo(s) specified
|
|
# in the kickstart file. If --repo wasn't specified, then we use the
|
|
# repo associated with the target passed in initially.
|
|
ks.handler.repo.repoList = [] # delete whatever the ks file told us
|
|
repo_class = kscontrol.dataMap[ks.version]['RepoData']
|
|
# TODO: sensibly use "url" and "repo" commands in kickstart
|
|
if self.opts.get('repo'):
|
|
# the user used --repo at least once
|
|
user_repos = self.opts.get('repo')
|
|
index = 0
|
|
for user_repo in user_repos:
|
|
repo_url = user_repo.replace('$arch', self.arch)
|
|
ks.handler.repo.repoList.append(repo_class(
|
|
baseurl=repo_url, name='koji-override-%i' % index))
|
|
index += 1
|
|
else:
|
|
# --repo was not given, so we use the target's build repo
|
|
path_info = koji.PathInfo(topdir=self.options.topurl)
|
|
repopath = path_info.repo(self.repo_info['id'],
|
|
self.target_info['build_tag_name'])
|
|
baseurl = '%s/%s' % (repopath, self.arch)
|
|
self.logger.debug('BASEURL: %s' % baseurl)
|
|
ks.handler.repo.repoList.append(repo_class(
|
|
baseurl=baseurl, name='koji-override-0'))
|
|
# inject the URL of the install tree into the kickstart
|
|
ks.handler.url(url=install_tree)
|
|
return ks
|
|
|
|
def writeKickstart(self, ksobj, ksname):
|
|
"""
|
|
Write out the new ks file. Note that things may not be in the same
|
|
order and comments in the original ks file may be lost.
|
|
|
|
@args:
|
|
ksobj: a pykickstart object of what we want to write
|
|
ksname: file name for the kickstart
|
|
@returns:
|
|
an absolute path to the kickstart file we wrote
|
|
"""
|
|
kspath = os.path.join(self.workdir, ksname)
|
|
outfile = open(kspath, 'w')
|
|
outfile.write(str(ksobj.handler))
|
|
outfile.close()
|
|
|
|
# put the new ksfile in the output directory
|
|
if not os.path.exists(kspath):
|
|
raise koji.BuildError, "KS file missing: %s" % kspath
|
|
self.uploadFile(kspath) # upload the modified ks file
|
|
return kspath
|
|
|
|
def makeConfig(self):
|
|
"""
|
|
Generate a configuration dict for ImageFactory. This will override
|
|
anything in the /etc config files. We do this forcibly so that it is
|
|
impossible for Koji to use any image caches or leftover metadata from
|
|
other images created by the service.
|
|
|
|
@args: none
|
|
@returns:
|
|
a dictionary used for configuring ImageFactory to built an image
|
|
the way we want
|
|
"""
|
|
return {
|
|
#Oz specific
|
|
'oz_data_dir': os.path.join(self.workdir, 'oz_data'),
|
|
'oz_screenshot_dir': os.path.join(self.workdir, 'oz_screenshots'),
|
|
#IF specific
|
|
'imgdir': os.path.join(self.workdir, 'scratch_images'),
|
|
'tmpdir': os.path.join(self.workdir, 'oz-tmp'),
|
|
'verbose' : True,
|
|
'timeout': 7200,
|
|
'output': 'log',
|
|
'raw': False,
|
|
'debug': True,
|
|
'image_manager': 'file',
|
|
'plugins': '/etc/imagefactory/plugins.d',
|
|
'rhevm_image_format': 'qcow2',
|
|
'tdl_require_root_pw': False,
|
|
'image_manager_args': {
|
|
'storage_path': os.path.join(self.workdir, 'output_image')},
|
|
}
|
|
|
|
def makeTemplate(self, name, inst_tree):
|
|
"""
|
|
Generate a simple "TDL" for ImageFactory to build an image with.
|
|
|
|
@args:
|
|
name: a name for the image
|
|
inst_tree: a string, a URL to the install tree (a compose)
|
|
@returns:
|
|
An XML string that imagefactory can consume
|
|
"""
|
|
# we have to split this up so the variable substitution works
|
|
# XXX: using a packages section (which we don't) will have IF boot the
|
|
# image and attempt to ssh in. This breaks docker image creation.
|
|
# TODO: intelligently guess the distro based on the install tree URL
|
|
distname, distver = self.parseDistro(self.opts.get('distro'))
|
|
if self.arch in ['armhfp','armv7hnl','armv7hl']:
|
|
arch = 'armv7l'
|
|
else:
|
|
arch = self.arch
|
|
template = """<template>
|
|
<name>%s</name>
|
|
<os>
|
|
<name>%s</name>
|
|
<version>%s</version>
|
|
<arch>%s</arch>
|
|
<install type='url'>
|
|
<url>%s</url>
|
|
</install>
|
|
""" % (name, distname, distver, arch, inst_tree)
|
|
template += """<icicle>
|
|
<extra_command>rpm -qa --qf '%{NAME},%{VERSION},%{RELEASE},%{ARCH},%{EPOCH},%{SIZE},%{SIGMD5},%{BUILDTIME}\n'</extra_command>
|
|
</icicle>
|
|
"""
|
|
# TODO: intelligently guess the size based on the kickstart file
|
|
template += """</os>
|
|
<description>%s OS</description>
|
|
<disk>
|
|
<size>%sG</size>
|
|
</disk>
|
|
</template>
|
|
""" % (name, self.opts.get('disk_size'))
|
|
return template
|
|
|
|
def parseDistro(self, distro):
|
|
"""
|
|
Figure out the distribution name and version we are going to build an
|
|
image on.
|
|
|
|
args:
|
|
a string of the form: RHEL-X.Y, Fedora-NN, CentOS-X.Y, or SL-X.Y
|
|
returns:
|
|
a 2-element list, depends on the distro where the split happened
|
|
"""
|
|
if distro.startswith('RHEL'):
|
|
major, minor = distro.split('.')
|
|
if major == 'RHEL-5':
|
|
minor = 'U' + minor
|
|
return major, minor
|
|
elif distro.startswith('Fedora'):
|
|
return distro.split('-')
|
|
elif distro.startswith('CentOS'):
|
|
return distro.split('.')
|
|
elif distro.startswith('SL'):
|
|
return distro.split('.')
|
|
else:
|
|
raise koji.BuildError('Unknown or supported distro given: %s' % distro)
|
|
|
|
def fixImageXML(self, format, filename, xmltext):
|
|
|
|
"""
|
|
The XML generated by Oz/ImageFactory knows nothing about the name
|
|
or image format conversions Koji does. We fix those values in the
|
|
libvirt XML and write the changes out to a file, the path of which
|
|
we return.
|
|
|
|
@args:
|
|
format = raw, qcow2, vmdk, etc... a string representation
|
|
filename = the name of the XML file we will save this too
|
|
xmltext = the libvirt XML to start with
|
|
@return:
|
|
an absolute path to the modified XML
|
|
"""
|
|
newxml = xml.dom.minidom.parseString(xmltext)
|
|
ename = newxml.getElementsByTagName('name')[0]
|
|
ename.firstChild.nodeValue = self.imgname
|
|
esources = newxml.getElementsByTagName('source')
|
|
for e in esources:
|
|
if e.hasAttribute('file'):
|
|
e.setAttribute('file', '%s.%s' % (self.imgname, format))
|
|
edriver = newxml.getElementsByTagName('driver')[0]
|
|
edriver.setAttribute('type', format)
|
|
xml_path = os.path.join(self.workdir, filename)
|
|
xmlfd = open(xml_path, 'w')
|
|
xmlfd.write(newxml.toprettyxml())
|
|
xmlfd.close()
|
|
return xml_path
|
|
|
|
def getScreenshot(self):
|
|
"""
|
|
Locate a screenshot taken by libvirt in the case of build failure,
|
|
if it exists. If it does, return the path, else return None.
|
|
|
|
@args: none
|
|
@returns: a path to a screenshot take by libvirt
|
|
"""
|
|
shotdir = os.path.join(self.workdir, 'oz_screenshots')
|
|
screenshot = None
|
|
found = glob.glob(os.path.join(shotdir, '*.ppm'))
|
|
if len(found) > 0:
|
|
screenshot = found[0]
|
|
found = glob.glob(os.path.join(shotdir, '*.png'))
|
|
if len(found) > 0:
|
|
screenshot = found[0]
|
|
return screenshot
|
|
|
|
class BaseImageTask(OzImageTask):
|
|
|
|
Methods = ['createImage']
|
|
_taskWeight = 2.0
|
|
|
|
def _format_deps(self, formats):
|
|
"""
|
|
Return a dictionary where the keys are the image formats we need to
|
|
build/convert, and the values are booleans that indicate whether the
|
|
output should be included in the task results.
|
|
|
|
Some image formats require others to be processed first, which is why
|
|
we have to do this. raw files in particular may not be kept.
|
|
"""
|
|
supported = ('raw', 'raw-xz', 'liveimg-squashfs', 'vmdk', 'qcow', 'qcow2', 'vdi', 'rhevm-ova', 'vsphere-ova', 'docker', 'vagrant-virtualbox', 'vagrant-libvirt', 'vagrant-vmware-fusion', 'vagrant-hyperv', 'vpc', "tar-gz")
|
|
for f in formats:
|
|
if f not in supported:
|
|
raise koji.ApplianceError('Invalid format: %s' % f)
|
|
f_dict = dict((f, True) for f in formats)
|
|
|
|
# If a user requests 1 or more image formats (with --format) we do not
|
|
# by default include the raw disk image in the results, because it is
|
|
# 10G in size. To override this behavior, the user must specify
|
|
# "--format raw" in their command. If --format was not used at all,
|
|
# then we do include the raw disk image by itself.
|
|
if len(formats) == 0:
|
|
# we only want a raw disk image (no format option given)
|
|
f_dict['raw'] = True
|
|
elif 'raw' not in f_dict.keys():
|
|
f_dict['raw'] = False
|
|
self.logger.debug('Image delivery plan: %s' % f_dict)
|
|
return f_dict
|
|
|
|
def do_images(self, ks, template, inst_tree):
|
|
"""
|
|
Call out to ImageFactory to build the image(s) we want. Returns a dict
|
|
of details for each image type we had to ask ImageFactory to build
|
|
"""
|
|
fcalls = {'raw': self._buildBase,
|
|
'raw-xz': self._buildXZ,
|
|
'tar-gz': self._buildTarGZ,
|
|
'liveimg-squashfs': self._buildSquashfs,
|
|
'vmdk': self._buildConvert,
|
|
'vdi': self._buildConvert,
|
|
'qcow': self._buildConvert,
|
|
'qcow2': self._buildConvert,
|
|
'vpc': self._buildConvert,
|
|
'rhevm-ova': self._buildOVA,
|
|
'vsphere-ova': self._buildOVA,
|
|
'vagrant-virtualbox': self._buildOVA,
|
|
'vagrant-libvirt': self._buildOVA,
|
|
'vagrant-vmware-fusion': self._buildOVA,
|
|
'vagrant-hyperv': self._buildOVA,
|
|
'docker': self._buildDocker
|
|
}
|
|
# add a handler to the logger so that we capture ImageFactory's logging
|
|
self.fhandler = logging.FileHandler(self.ozlog)
|
|
self.bd = BuildDispatcher()
|
|
self.tlog = logging.getLogger()
|
|
self.tlog.setLevel(logging.DEBUG)
|
|
self.tlog.addHandler(self.fhandler)
|
|
images = {}
|
|
random.seed() # necessary to ensure a unique mac address
|
|
params = {'install_script': str(ks.handler),
|
|
'offline_icicle': True}
|
|
# build the base (raw) image
|
|
self.base_img = self._buildBase(template, params)
|
|
images['raw'] = {'image': self.base_img.base_image.data,
|
|
'icicle': self.base_img.base_image.icicle}
|
|
# Do the rest of the image types (everything but raw)
|
|
for format in self.formats:
|
|
if format == 'raw':
|
|
continue
|
|
self.logger.info('dispatching %s image builder' % format)
|
|
images[format] = fcalls[format](format)
|
|
imginfo = self._processXML(images)
|
|
self.tlog.removeHandler(self.fhandler)
|
|
self.uploadFile(self.ozlog)
|
|
return imginfo
|
|
|
|
def _processXML(self, images):
|
|
"""
|
|
Produce XML that libvirt can import to create a domain based on image(s)
|
|
we produced. We save the location of the XML file in the dictionary
|
|
it corresponds to here.
|
|
|
|
@args:
|
|
images - a dict where the keys are image formats, and the values
|
|
are dicts with details about the image (location, icicle, etc)
|
|
@returns:
|
|
a dictionary just like "images" but with a new key called "libvirt"
|
|
that points to the path of the XML file for that image
|
|
"""
|
|
imginfo = {}
|
|
for fmt in images.keys():
|
|
imginfo[fmt] = images[fmt]
|
|
lxml = self.fixImageXML(fmt, 'libvirt-%s-%s.xml' % (fmt, self.arch),
|
|
self.base_img.base_image.parameters['libvirt_xml'])
|
|
imginfo[fmt]['libvirt'] = lxml
|
|
return imginfo
|
|
|
|
def _checkImageState(self, image):
|
|
"""
|
|
Query ImageFactory for details of a dispatched image build. If it is
|
|
FAILED we raise an exception.
|
|
|
|
@args:
|
|
image - a build dispatcher object returned by a BuildDispatcher
|
|
@returns: nothing
|
|
"""
|
|
if image.target_image:
|
|
status = image.target_image.status
|
|
details = image.target_image.status_detail['error']
|
|
else:
|
|
status = image.base_image.status
|
|
details = image.base_image.status_detail['error']
|
|
self.logger.debug('check image results: %s' % status)
|
|
if status == 'FAILED':
|
|
scrnshot = self.getScreenshot()
|
|
if scrnshot:
|
|
ext = scrnshot[-3:]
|
|
self.uploadFile(scrnshot, remoteName='screenshot.%s' % ext)
|
|
image.os_plugin.abort() # forcibly tear down the VM
|
|
# TODO abort when a task is CANCELLED
|
|
if not self.session.checkUpload('', os.path.basename(self.ozlog)):
|
|
self.tlog.removeHandler(self.fhandler)
|
|
self.uploadFile(self.ozlog)
|
|
if 'No disk activity' in details:
|
|
details = 'Automated install failed or prompted for input. See the screenshot in the task results for more information.'
|
|
raise koji.ApplianceError('Image status is %s: %s' %
|
|
(status, details))
|
|
|
|
def _mergeFactoryParams(self, img_opts, fixed_params):
|
|
"""
|
|
Merge any KV pairs passed in via --factory-parameter CLI options
|
|
into an existing dictionary that will eventually be passed to
|
|
factory build commands. This allows a fairly generic mechanism
|
|
for parameter passthrough to plugins and the base builder that does
|
|
not require patching the builder and CLI each time.
|
|
|
|
@args:
|
|
img_opts - an existing dict with any pre-existing parameters
|
|
fixed_params - list of parameters that must not be overriden
|
|
@returns:
|
|
nothing - dict is modified in place
|
|
"""
|
|
if self.opts.get('factory_parameter'):
|
|
for kvp in self.opts.get('factory_parameter'):
|
|
if kvp[0] not in fixed_params:
|
|
img_opts[kvp[0]] = kvp[1]
|
|
|
|
def _buildBase(self, template, params, wait=True):
|
|
"""
|
|
Build a base image using ImageFactory. This is a "raw" image.
|
|
|
|
@args:
|
|
template - an XML string for the TDL
|
|
params - a dict that controls some ImageFactory settings
|
|
wait - call join() on the building thread if True
|
|
@returns:
|
|
a dict with some metadata about the image (includes an icicle)
|
|
"""
|
|
# TODO: test the failure case where IF itself throws an exception
|
|
# ungracefully (missing a plugin for example)
|
|
# may need to still upload ozlog and remove the log handler
|
|
self.logger.info('dispatching a baseimg builder')
|
|
self.logger.debug('templates: %s' % template)
|
|
self.logger.debug('pre-merge params: %s' % params)
|
|
# We enforce various things related to the ks file - do not allow override
|
|
self._mergeFactoryParams(params, [ 'install_script' ])
|
|
self.logger.debug('post-merge params: %s' % params)
|
|
base = self.bd.builder_for_base_image(template, parameters=params)
|
|
if wait:
|
|
base.base_thread.join()
|
|
self._checkImageState(base)
|
|
return base
|
|
|
|
def _buildXZ(self, format):
|
|
"""
|
|
Use xz to compress a raw disk image. This is very straightforward.
|
|
|
|
@args:
|
|
format - a string representing the image format, "raw-xz"
|
|
@returns:
|
|
a dict with some metadata about the image
|
|
"""
|
|
newimg = os.path.join(self.workdir, self.imgname + '.raw.xz')
|
|
rawimg = os.path.join(self.workdir, self.imgname + '.raw')
|
|
cmd = ['/bin/cp', self.base_img.base_image.data, rawimg]
|
|
conlog = os.path.join(self.workdir,
|
|
'xz-cp-%s-%s.log' % (format, self.arch))
|
|
log_output(self.session, cmd[0], cmd, conlog, self.getUploadDir(),
|
|
logerror=1)
|
|
cmd = ['/usr/bin/xz', '-z', rawimg]
|
|
conlog = os.path.join(self.workdir,
|
|
'xz-%s-%s.log' % (format, self.arch))
|
|
log_output(self.session, cmd[0], cmd, conlog, self.getUploadDir(),
|
|
logerror=1)
|
|
return {'image': newimg}
|
|
|
|
|
|
def _buildTarGZ(self, format):
|
|
"""
|
|
Use tar and gzip to compress a raw disk image.
|
|
|
|
@args:
|
|
format - not used, we only handle tar-gz
|
|
@returns:
|
|
a dict with some metadata about the image
|
|
"""
|
|
orig = self.base_img.base_image.data
|
|
newimg = os.path.join(self.workdir, self.imgname + '.tar.gz')
|
|
|
|
# see also: https://cloud.google.com/compute/docs/creating-custom-image
|
|
|
|
# the image in the tarball must be named disk.raw
|
|
imgdir = os.path.dirname(orig)
|
|
rawimg = os.path.join(imgdir, 'disk.raw')
|
|
os.link(orig, rawimg)
|
|
|
|
# make the tarball
|
|
cmd = ['/bin/tar', '-Sczvf', newimg, 'disk.raw']
|
|
conlog = os.path.join(self.workdir, 'tar-gz-%s.log' % self.arch)
|
|
log_output(self.session, cmd[0], cmd, conlog, self.getUploadDir(),
|
|
logerror=1, cwd=imgdir)
|
|
|
|
# now that we've made the tarball, we don't need this hardlink
|
|
os.unlink(rawimg)
|
|
|
|
return {'image': newimg}
|
|
|
|
|
|
def _buildSquashfs(self, format):
|
|
"""
|
|
Use squashfs to wrap a raw disk image into liveimg compatible image.
|
|
This can be used by dracut for booting or anaconda to install.
|
|
|
|
@args:
|
|
format - a string representing the image format, "liveimg-squashfs"
|
|
@returns:
|
|
a dict with some metadata about the image
|
|
"""
|
|
newimg = os.path.join(self.workdir, self.imgname + '.squashfs')
|
|
fsimg = os.path.join(self.workdir, 'squashfs-root/LiveOS/rootfs.img')
|
|
|
|
os.makedirs(os.path.join(self.workdir, 'squashfs-root/LiveOS'))
|
|
|
|
cmd = ['/bin/dd', 'conv=sparse', 'bs=1M',
|
|
'skip=1', # FIXME Hack to strip the disklabel
|
|
'if=%s' % self.base_img.base_image.data,
|
|
'of=%s' % fsimg]
|
|
conlog = os.path.join(self.workdir,
|
|
'squashfs-dd-%s-%s.log' % (format, self.arch))
|
|
log_output(self.session, cmd[0], cmd, conlog, self.getUploadDir(),
|
|
logerror=1)
|
|
cmd = ['/usr/sbin/mksquashfs', os.path.join(self.workdir, 'squashfs-root'),
|
|
newimg, '-comp', 'xz', '-noappend']
|
|
conlog = os.path.join(self.workdir,
|
|
'squashfs-mksquashfs-%s-%s.log' % (format, self.arch))
|
|
log_output(self.session, cmd[0], cmd, conlog, self.getUploadDir(),
|
|
logerror=1)
|
|
return {'image': newimg}
|
|
|
|
def _buildOVA(self, format):
|
|
"""
|
|
Build an OVA target image. This is a format supported by RHEV and
|
|
vSphere
|
|
|
|
@args:
|
|
format - a string representing the image format, "rhevm-ova"
|
|
@returns:
|
|
a dict with some metadata about the image
|
|
"""
|
|
img_opts = {}
|
|
if self.opts.get('ova_option'):
|
|
img_opts = dict([o.split('=') for o in self.opts.get('ova_option')])
|
|
# As far as Image Factory is concerned, vagrant boxes are just another type of OVA
|
|
# We communicate the desire for vagrant-specific formatting by adding the *_ova_format
|
|
# options and turning the underlying format option back into one of the two target
|
|
# image types ('vsphere-ova' or 'rhevm-ova') that are used to generate the intermediate
|
|
# disk image
|
|
fixed_params = []
|
|
if format == 'vagrant-virtualbox':
|
|
format = 'vsphere-ova'
|
|
img_opts['vsphere_ova_format'] = 'vagrant-virtualbox'
|
|
fixed_params = [ 'vsphere_ova_format' ]
|
|
if format == 'vagrant-libvirt':
|
|
format = 'rhevm-ova'
|
|
img_opts['rhevm_ova_format'] = 'vagrant-libvirt'
|
|
fixed_params = [ 'rhevm_ova_format' ]
|
|
if format == 'vagrant-vmware-fusion':
|
|
format = 'vsphere-ova'
|
|
img_opts['vsphere_ova_format'] = 'vagrant-vmware-fusion'
|
|
# The initial disk image transform for VMWare Fusion/Workstation requires a "standard" VMDK
|
|
# not the stream oriented format used for VirtualBox or regular VMWare OVAs
|
|
img_opts['vsphere_vmdk_format'] = 'standard'
|
|
fixed_params = [ 'vsphere_ova_format', 'vsphere_vmdk_format' ]
|
|
if format == 'vagrant-hyperv':
|
|
format = 'hyperv-ova'
|
|
img_opts['hyperv_ova_format'] = 'hyperv-vagrant'
|
|
fixed_params = [ 'hyperv_ova_format' ]
|
|
targ = self._do_target_image(self.base_img.base_image.identifier,
|
|
format.replace('-ova', ''), img_opts=img_opts, fixed_params=fixed_params)
|
|
targ2 = self._do_target_image(targ.target_image.identifier, 'OVA',
|
|
img_opts=img_opts, fixed_params=fixed_params)
|
|
return {'image': targ2.target_image.data}
|
|
|
|
def _buildDocker(self, format):
|
|
"""
|
|
Build a base docker image. This image will be tagged with the NVR.A
|
|
automatically because we name it that way in the ImageFactory TDL.
|
|
|
|
@args:
|
|
format - the string "docker"
|
|
@returns:
|
|
a dict with some metadata about the image
|
|
"""
|
|
img_opts = {'compress': 'xz'}
|
|
targ = self._do_target_image(self.base_img.base_image.identifier,
|
|
'docker', img_opts=img_opts)
|
|
return {'image': targ.target_image.data}
|
|
|
|
def _do_target_image(self, base_id, image_type, img_opts=None, fixed_params=None):
|
|
"""
|
|
A generic method for building what ImageFactory calls "target images".
|
|
These are images based on a raw disk that was built before using the
|
|
_buildBase method.
|
|
|
|
@args:
|
|
base_id - a string ID of the image to build off of
|
|
image_type - a string representing the target type. ImageFactory
|
|
uses this to figure out what plugin to run
|
|
img_opts - a dict of additional options that specific to the target
|
|
type we pass in via image_type
|
|
fixed_params - a list of parameter keys that should not be overriden
|
|
by the --factory-parameter CLI
|
|
@returns:
|
|
A Builder() object from ImageFactory that contains information
|
|
about the image building include state and progress.
|
|
"""
|
|
# TODO: test the failure case where IF itself throws an exception
|
|
# ungracefully (missing a plugin for example)
|
|
# may need to still upload ozlog and remove the log handler
|
|
if img_opts is None:
|
|
img_opts = {}
|
|
if fixed_params is None:
|
|
fixed_params = []
|
|
self.logger.debug('img_opts_pre_merge: %s' % img_opts)
|
|
self._mergeFactoryParams(img_opts, fixed_params)
|
|
self.logger.debug('img_opts_post_merge: %s' % img_opts)
|
|
target = self.bd.builder_for_target_image(image_type,
|
|
image_id=base_id, template=None, parameters=img_opts)
|
|
target.target_thread.join()
|
|
self._checkImageState(target)
|
|
return target
|
|
|
|
def _buildConvert(self, format):
|
|
"""
|
|
Build an image by converting the format using qemu-img. This is method
|
|
enables a variety of formats like qcow, qcow2, vmdk, and vdi.
|
|
|
|
@args:
|
|
format - a string representing the image format, "qcow2"
|
|
@returns
|
|
a dict with some metadata about the image
|
|
"""
|
|
self.logger.debug('converting an image to "%s"' % format)
|
|
ofmt = format
|
|
if format == 'vpc':
|
|
ofmt = 'vhd'
|
|
newimg = os.path.join(self.workdir, self.imgname + '.%s' % ofmt)
|
|
cmd = ['/usr/bin/qemu-img', 'convert', '-f', 'raw', '-O',
|
|
format, self.base_img.base_image.data, newimg]
|
|
if format == 'qcow':
|
|
cmd.insert(2, '-c') # enable compression for qcow images
|
|
if format == 'qcow2':
|
|
# qemu-img changed its default behavior at some point to generate a
|
|
# v3 image when the requested output format is qcow2. We don't
|
|
# want koji to output different formats based on the verison of
|
|
# qemu-img that happens to be on the builder. Here we use a function
|
|
# inside of Image Factory that detects qemu-img behavior and adds
|
|
# the correct options to ensure original "v2" compatibility
|
|
cmd = qemu_convert_cmd(self.base_img.base_image.data, newimg, compress=True)
|
|
# Factory does not use a full path - for consistency, force that here
|
|
cmd[0] = '/usr/bin/qemu-img'
|
|
conlog = os.path.join(self.workdir,
|
|
'qemu-img-%s-%s.log' % (format, self.arch))
|
|
log_output(self.session, cmd[0], cmd, conlog,
|
|
self.getUploadDir(), logerror=1)
|
|
return {'image': newimg}
|
|
|
|
def handler(self, name, version, release, arch, target_info, build_tag, repo_info, inst_tree, opts=None):
|
|
|
|
if opts == None:
|
|
opts = {}
|
|
self.arch = arch
|
|
self.target_info = target_info
|
|
self.repo_info = repo_info
|
|
self.opts = opts
|
|
self.formats = self._format_deps(opts.get('format'))
|
|
|
|
# First, prepare the kickstart to use the repos we tell it
|
|
kspath = self.fetchKickstart()
|
|
ks = self.prepareKickstart(kspath, inst_tree)
|
|
kskoji = self.writeKickstart(ks,
|
|
os.path.join(self.workdir, 'koji-%s-%i-base.ks' %
|
|
(self.target_info['build_tag_name'], self.id)))
|
|
|
|
# auto-generate a TDL file and config dict for ImageFactory
|
|
self.imgname = '%s-%s-%s.%s' % (name, version, release, self.arch)
|
|
template = self.makeTemplate(self.imgname, inst_tree)
|
|
self.logger.debug('oz template: %s' % template)
|
|
config = self.makeConfig()
|
|
self.logger.debug('IF config object: %s' % config)
|
|
ApplicationConfiguration(configuration=config)
|
|
|
|
tdl_path = os.path.join(self.workdir, 'tdl-%s.xml' % self.arch)
|
|
tdl = open(tdl_path, 'w')
|
|
tdl.write(template)
|
|
tdl.close()
|
|
self.uploadFile(tdl_path)
|
|
|
|
# ImageFactory picks a port to the guest VM using a rolling integer.
|
|
# This is a problem for concurrency, so we override the port it picks
|
|
# here using the task ID. (not a perfect solution but good enough:
|
|
# the likelihood of image tasks clashing here is very small)
|
|
rm = ReservationManager()
|
|
rm._listen_port = rm.MIN_PORT + self.id % (rm.MAX_PORT - rm.MIN_PORT)
|
|
ozlogname = 'oz-%s.log' % self.arch
|
|
self.ozlog = os.path.join(self.workdir, ozlogname)
|
|
|
|
# invoke the image builds
|
|
images = self.do_images(ks, template, inst_tree)
|
|
images['raw']['tdl'] = os.path.basename(tdl_path),
|
|
|
|
# structure the results to pass back to the hub:
|
|
imgdata = {
|
|
'arch': self.arch,
|
|
'task_id': self.id,
|
|
'logs': [ozlogname],
|
|
'name': name,
|
|
'version': version,
|
|
'release': release,
|
|
'rpmlist': [],
|
|
'files': [os.path.basename(tdl_path),
|
|
os.path.basename(kspath),
|
|
os.path.basename(kskoji)]
|
|
}
|
|
# record the RPMs that were installed
|
|
if not opts.get('scratch'):
|
|
fields = ('name', 'version', 'release', 'arch', 'epoch', 'size',
|
|
'payloadhash', 'buildtime')
|
|
icicle = xml.dom.minidom.parseString(images['raw']['icicle'])
|
|
self.logger.debug('ICICLE: %s' % images['raw']['icicle'])
|
|
for p in icicle.getElementsByTagName('extra'):
|
|
bits = p.firstChild.nodeValue.split(',')
|
|
rpm = {
|
|
'name': bits[0],
|
|
'version': bits[1],
|
|
'release': bits[2],
|
|
'arch': bits[3],
|
|
# epoch is a special case, as usual
|
|
'size': int(bits[5]),
|
|
'payloadhash': bits[6],
|
|
'buildtime': int(bits[7])
|
|
}
|
|
if rpm['name'] in ['buildsys-build', 'gpg-pubkey']:
|
|
continue
|
|
if bits[4] == '(none)':
|
|
rpm['epoch'] = None
|
|
else:
|
|
rpm['epoch'] = int(bits[4])
|
|
imgdata['rpmlist'].append(rpm)
|
|
# TODO: hack to make this work for now, need to refactor
|
|
br = BuildRoot(self.session, self.options, build_tag, self.arch,
|
|
self.id, repo_id=self.repo_info['id'])
|
|
br.markExternalRPMs(imgdata['rpmlist'])
|
|
|
|
# upload the results
|
|
for format in (f for f in self.formats.keys() if self.formats[f]):
|
|
newimg = images[format]['image']
|
|
if ('ova' in format or format in ('raw-xz', 'liveimg-squashfs', 'tar-gz')):
|
|
newname = self.imgname + '.' + format.replace('-', '.')
|
|
elif 'vagrant' in format:
|
|
# This embeds the vagrant target and the ".box" format in the name
|
|
# Previously, based on filename, these looked like OVAs
|
|
# This was confusing to many people
|
|
newname = self.imgname + '.' + format + '.box'
|
|
elif format == 'docker':
|
|
newname = self.imgname + '.' + 'tar.xz'
|
|
elif format == 'vpc':
|
|
newname = self.imgname + '.' + 'vhd'
|
|
else:
|
|
newname = self.imgname + '.' + format
|
|
if format != 'docker':
|
|
lxml = images[format]['libvirt']
|
|
imgdata['files'].append(os.path.basename(lxml))
|
|
self.uploadFile(lxml)
|
|
imgdata['files'].append(os.path.basename(newname))
|
|
self.uploadFile(newimg, remoteName=newname)
|
|
|
|
# no need to delete anything since self.workdir will get scrubbed
|
|
return imgdata
|
|
|
|
class BuildIndirectionImageTask(OzImageTask):
|
|
Methods = ['indirectionimage']
|
|
|
|
# So, these are copied directly from the base image class
|
|
# Realistically, we want to inherit methods from both BuildImageTask
|
|
# and OzImageTask.
|
|
# TODO: refactor - my initial suggestion would be to have OzImageTask
|
|
# be a child of BuildImageTask
|
|
|
|
def initImageBuild(self, name, version, release, target_info, opts):
|
|
"""create a build object for this image build"""
|
|
pkg_cfg = self.session.getPackageConfig(target_info['dest_tag_name'],
|
|
name)
|
|
self.logger.debug("%r" % pkg_cfg)
|
|
if not opts.get('skip_tag') and not opts.get('scratch'):
|
|
# Make sure package is on the list for this tag
|
|
if pkg_cfg is None:
|
|
raise koji.BuildError, "package (image) %s not in list for tag %s" % (name, target_info['dest_tag_name'])
|
|
elif pkg_cfg['blocked']:
|
|
raise koji.BuildError, "package (image) %s is blocked for tag %s" % (name, target_info['dest_tag_name'])
|
|
return self.session.host.initImageBuild(self.id,
|
|
dict(name=name, version=version, release=release, epoch=0))
|
|
|
|
def getRelease(self, name, ver):
|
|
"""return the next available release number for an N-V"""
|
|
return self.session.getNextRelease(dict(name=name, version=ver))
|
|
|
|
# END inefficient base image task method copies
|
|
|
|
def fetchHubOrSCM(self, filepath, fileurl):
|
|
"""
|
|
Retrieve a file either from the hub or a remote scm
|
|
|
|
If fileurl is None we assume we are being asked to retrieve from
|
|
the hub and that filepath is relative to /mnt/koji/work.
|
|
if fileurl contains a value we assume a remote SCM.
|
|
|
|
If retrieving remote we assume that filepath is the file name and
|
|
fileurl is the path in the remote SCM where that file can be found.
|
|
|
|
@returns: absolute path to the retrieved file
|
|
"""
|
|
# TODO: A small change to the base image build code could allow this method
|
|
# to be shared between both tasks. I wanted this initial implementation
|
|
# to be entirely self contained. Revisit if anyone feels like a refactor.
|
|
self.logger.debug("filepath = %s" % filepath)
|
|
if fileurl:
|
|
scm = SCM(fileurl)
|
|
scm.assert_allowed(self.options.allowed_scms)
|
|
logfile = os.path.join(self.workdir, 'checkout.log')
|
|
scmsrcdir = scm.checkout(self.workdir, self.session,
|
|
self.getUploadDir(), logfile)
|
|
final_path = os.path.join(scmsrcdir, os.path.basename(filepath))
|
|
else:
|
|
tops = dict([(k, getattr(self.options, k)) for k in 'topurl','topdir'])
|
|
tops['tempdir'] = self.workdir
|
|
remote_fileobj = koji.openRemoteFile(filepath, **tops)
|
|
final_path = os.path.join(self.workdir, os.path.basename(filepath))
|
|
final_fileobj = open(final_path, 'w')
|
|
final_fileobj.write(remote_fileobj.read())
|
|
final_fileobj.close()
|
|
self.logger.debug('uploading retrieved file from here: %s' % final_path)
|
|
self.uploadFile(final_path) # upload the original ks file
|
|
return final_path # absolute path to the ks file
|
|
|
|
def handler(self, opts):
|
|
"""Governing task for building an image with two other images using Factory Indirection"""
|
|
# TODO: Add mode of operation where full build details are given for
|
|
# either base or utility or both, then spawn subtasks to do them first
|
|
def _task_to_image(task_id):
|
|
""" Take a task ID and turn it into an Image Factory Base Image object """
|
|
pim = PersistentImageManager.default_manager()
|
|
taskinfo = self.session.getTaskInfo(task_id)
|
|
taskstate = koji.TASK_STATES[taskinfo['state']].lower()
|
|
if taskstate != 'closed':
|
|
raise koji.BuildError("Input task (%d) must be in closed state - current state is (%s)" %
|
|
(task_id, taskstate))
|
|
taskmethod = taskinfo['method']
|
|
if taskmethod != "createImage":
|
|
raise koji.BuildError("Input task method must be 'createImage' - actual method (%s)" %
|
|
(taskmethod))
|
|
result = self.session.getTaskResult(task_id)
|
|
files = self.session.listTaskOutput(task_id)
|
|
|
|
# This approach works for both scratch and saved/formal images
|
|
# The downside is that we depend on the output file naming convention
|
|
def _match_name(inlist, namere):
|
|
for filename in inlist:
|
|
if re.search(namere, filename):
|
|
return filename
|
|
task_diskimage = _match_name(result['files'], ".*qcow2$")
|
|
task_tdl = _match_name(result['files'], "tdl.*xml")
|
|
|
|
task_dir = os.path.join(koji.pathinfo.work(),koji.pathinfo.taskrelpath(task_id))
|
|
diskimage_full = os.path.join(task_dir, task_diskimage)
|
|
tdl_full = os.path.join(task_dir, task_tdl)
|
|
|
|
if not (os.path.isfile(diskimage_full) and os.path.isfile(tdl_full)):
|
|
raise koji.BuildError("Missing TDL or qcow2 image for task (%d) - possible expired scratch build" % (task_id))
|
|
|
|
# The sequence to recreate a valid persistent image is as follows
|
|
# Create a new BaseImage object
|
|
factory_base_image = BaseImage()
|
|
# Add it to the persistence layer
|
|
pim.add_image(factory_base_image)
|
|
# Now replace the data and template with the files referenced above
|
|
# and mark it as a complete image
|
|
# Factory doesn't attempt to modify a disk image after it is COMPLETE so
|
|
# this will work safely on read-only NFS mounts
|
|
factory_base_image.data = diskimage_full
|
|
factory_base_image.template = open(tdl_full).read()
|
|
factory_base_image.status = 'COMPLETE'
|
|
# Now save it
|
|
pim.save_image(factory_base_image)
|
|
|
|
# We can now reference this object directly or via its UUID in persistent storage
|
|
return factory_base_image
|
|
|
|
def _nvr_to_image(nvr, arch):
|
|
""" Take a build ID or NVR plus arch and turn it into an Image Factory Base Image object """
|
|
pim = PersistentImageManager.default_manager()
|
|
build = self.session.getBuild(nvr)
|
|
if not build:
|
|
raise koji.BuildError("Could not find build for (%s)" % (nvr))
|
|
|
|
buildarchives = self.session.listArchives(build['id'])
|
|
if not buildarchives:
|
|
raise koji.BuildError("Could not retrieve archives for build (%s) from NVR (%s)" %
|
|
(build['id'], nvr))
|
|
|
|
buildfiles = [ x['filename'] for x in buildarchives ]
|
|
builddir = koji.pathinfo.imagebuild(build)
|
|
|
|
def _match_name(inlist, namere):
|
|
for filename in inlist:
|
|
if re.search(namere, filename):
|
|
return filename
|
|
|
|
build_diskimage = _match_name(buildfiles, ".*%s\.qcow2$" % (arch))
|
|
build_tdl = _match_name(buildfiles, "tdl.%s\.xml" % (arch))
|
|
|
|
diskimage_full = os.path.join(builddir, build_diskimage)
|
|
tdl_full = os.path.join(builddir, build_tdl)
|
|
|
|
if not (os.path.isfile(diskimage_full) and os.path.isfile(tdl_full)):
|
|
raise koji.BuildError("Missing TDL (%s) or qcow2 (%s) image for image (%s) - this should never happen" %
|
|
(build_tdl, build_diskimage, nvr))
|
|
|
|
# The sequence to recreate a valid persistent image is as follows
|
|
# Create a new BaseImage object
|
|
factory_base_image = BaseImage()
|
|
# Add it to the persistence layer
|
|
pim.add_image(factory_base_image)
|
|
# Now replace the data and template with the files referenced above
|
|
# and mark it as a complete image
|
|
# Factory doesn't attempt to modify a disk image after it is COMPLETE so
|
|
# this will work safely on read-only NFS mounts
|
|
factory_base_image.data = diskimage_full
|
|
factory_base_image.template = open(tdl_full).read()
|
|
factory_base_image.status = 'COMPLETE'
|
|
# Now save it
|
|
pim.save_image(factory_base_image)
|
|
|
|
# We can now reference this object directly or via its UUID in persistent storage
|
|
return factory_base_image
|
|
|
|
if opts == None:
|
|
opts = {}
|
|
self.opts = opts
|
|
|
|
config = self.makeConfig()
|
|
self.logger.debug('IF config object: %s' % config)
|
|
ApplicationConfiguration(configuration=config)
|
|
|
|
ozlogname = 'oz-indirection.log'
|
|
ozlog = os.path.join(self.workdir, ozlogname)
|
|
# END shared code
|
|
|
|
fhandler = logging.FileHandler(ozlog)
|
|
bd = BuildDispatcher()
|
|
tlog = logging.getLogger()
|
|
tlog.setLevel(logging.DEBUG)
|
|
tlog.addHandler(fhandler)
|
|
|
|
# TODO: Copy-paste from BaseImage - refactor
|
|
target_info = self.session.getBuildTarget(opts['target'], strict=True)
|
|
build_tag = target_info['build_tag']
|
|
repo_info = self.getRepo(build_tag)
|
|
|
|
name = opts['name']
|
|
version = opts['version']
|
|
release = opts['release']
|
|
|
|
# TODO: Another mostly copy-paste
|
|
if not release:
|
|
release = self.getRelease(name, version)
|
|
if '-' in version:
|
|
raise koji.ApplianceError('The Version may not have a hyphen')
|
|
if '-' in release:
|
|
raise koji.ApplianceError('The Release may not have a hyphen')
|
|
|
|
indirection_template = self.fetchHubOrSCM(opts.get('indirection_template'),
|
|
opts.get('indirection_template_url'))
|
|
|
|
self.logger.debug('Got indirection template %s' % (indirection_template))
|
|
|
|
try:
|
|
if opts['utility_image_build']:
|
|
utility_factory_image = _nvr_to_image(opts['utility_image_build'], opts['arch'])
|
|
else:
|
|
utility_factory_image = _task_to_image(int(opts['utility_image_task']))
|
|
|
|
if opts['base_image_build']:
|
|
base_factory_image = _nvr_to_image(opts['base_image_build'], opts['arch'])
|
|
else:
|
|
base_factory_image = _task_to_image(int(opts['base_image_task']))
|
|
except Exception, e:
|
|
self.logger.exception(e)
|
|
raise
|
|
|
|
# OK - We have a template and two input images - lets build
|
|
bld_info = None
|
|
if not opts['scratch']:
|
|
bld_info = self.initImageBuild(name, version, release,
|
|
target_info, opts)
|
|
|
|
try:
|
|
return self._do_indirection(opts, base_factory_image, utility_factory_image,
|
|
indirection_template, tlog, ozlog, fhandler,
|
|
bld_info, target_info, bd)
|
|
except:
|
|
if not opts.get('scratch'):
|
|
#scratch builds do not get imported
|
|
if bld_info:
|
|
self.session.host.failBuild(self.id, bld_info['id'])
|
|
# reraise the exception
|
|
raise
|
|
|
|
|
|
def _do_indirection(self, opts, base_factory_image, utility_factory_image,
|
|
indirection_template, tlog, ozlog, fhandler, bld_info,
|
|
target_info, bd):
|
|
# TODO: The next several lines are shared with the handler for other Factory tasks
|
|
# refactor in such a way that this can be a helper in OzImageTask
|
|
|
|
# ImageFactory picks a port to the guest VM using a rolling integer.
|
|
# This is a problem for concurrency, so we override the port it picks
|
|
# here using the task ID. (not a perfect solution but good enough:
|
|
# the likelihood of image tasks clashing here is very small)
|
|
rm = ReservationManager()
|
|
rm._listen_port = rm.MIN_PORT + self.id % (rm.MAX_PORT - rm.MIN_PORT)
|
|
|
|
utility_customizations = open(indirection_template).read()
|
|
results_loc = opts.get('results_loc', None)
|
|
if results_loc[0] != "/":
|
|
results_loc = "/" + results_loc
|
|
params = {'utility_image': str(utility_factory_image.identifier),
|
|
'utility_customizations': utility_customizations,
|
|
'results_location': results_loc }
|
|
random.seed() # necessary to ensure a unique mac address
|
|
try:
|
|
try:
|
|
# Embedded deep debug option - if template is just the string MOCK
|
|
# skip the actual build and create a mock target image instead
|
|
if utility_customizations.strip() == "MOCK":
|
|
target = Builder()
|
|
target_image = TargetImage()
|
|
pim = PersistentImageManager.default_manager()
|
|
pim.add_image(target_image)
|
|
target.target_image = target_image
|
|
open(target_image.data, "w").write("Mock build from task ID: %s" %
|
|
(str(self.id)))
|
|
target_image.status='COMPLETE'
|
|
else:
|
|
target = bd.builder_for_target_image('indirection',
|
|
image_id=base_factory_image.identifier,
|
|
parameters=params)
|
|
target.target_thread.join()
|
|
except Exception, e:
|
|
self.logger.debug("Exception encountered during target build")
|
|
self.logger.exception(e)
|
|
finally:
|
|
# upload log even if we failed to help diagnose an issue
|
|
tlog.removeHandler(fhandler)
|
|
self.uploadFile(ozlog)
|
|
self.logger.debug('Target image results: %s' % target.target_image.status)
|
|
|
|
if target.target_image.status == 'FAILED':
|
|
# TODO abort when a task is CANCELLED
|
|
if not self.session.checkUpload('', os.path.basename(ozlog)):
|
|
tlog.removeHandler(fhandler)
|
|
self.uploadFile(ozlog)
|
|
raise koji.ApplianceError('Image status is %s: %s' %
|
|
(target.target_image.status, target.target_image.status_detail))
|
|
|
|
self.uploadFile(target.target_image.data, remoteName=os.path.basename(results_loc))
|
|
|
|
myresults = { }
|
|
myresults['task_id'] = self.id
|
|
myresults['files'] = [ os.path.basename(results_loc) ]
|
|
myresults['logs'] = [ os.path.basename(ozlog) ]
|
|
myresults['arch'] = opts['arch']
|
|
# TODO: This should instead track the two input images: base and utility
|
|
myresults['rpmlist'] = [ ]
|
|
|
|
# This is compatible with some helper methods originally implemented for the base
|
|
# image build. In the original usage, the dict contains an entry per build arch
|
|
# TODO: If adding multiarch support, keep this in mind
|
|
results = { str(self.id): myresults }
|
|
self.logger.debug('Image Results for hub: %s' % results)
|
|
|
|
if opts['scratch']:
|
|
self.session.host.moveImageBuildToScratch(self.id, results)
|
|
else:
|
|
self.session.host.completeImageBuild(self.id, bld_info['id'],
|
|
results)
|
|
|
|
# tag it
|
|
if not opts.get('scratch') and not opts.get('skip_tag'):
|
|
tag_task_id = self.session.host.subtask(method='tagBuild',
|
|
arglist=[target_info['dest_tag'], bld_info['id'], False, None, True],
|
|
label='tag', parent=self.id, arch='noarch')
|
|
self.wait(tag_task_id)
|
|
|
|
# report results
|
|
report = ''
|
|
if opts.get('scratch'):
|
|
respath = ', '.join(
|
|
[os.path.join(koji.pathinfo.work(), koji.pathinfo.taskrelpath(tid)) for tid in [self.id] ])
|
|
report += 'Scratch '
|
|
else:
|
|
respath = koji.pathinfo.imagebuild(bld_info)
|
|
report += 'image build results in: %s' % respath
|
|
return report
|
|
|
|
|
|
class BuildSRPMFromSCMTask(BaseBuildTask):
|
|
|
|
Methods = ['buildSRPMFromSCM']
|
|
_taskWeight = 1.0
|
|
|
|
def spec_sanity_checks(self, filename):
|
|
spec = open(filename).read()
|
|
for tag in ("Packager", "Distribution", "Vendor"):
|
|
if re.match("%s:" % tag, spec, re.M):
|
|
raise koji.BuildError, "%s is not allowed to be set in spec file" % tag
|
|
for tag in ("packager", "distribution", "vendor"):
|
|
if re.match("%%define\s+%s\s+" % tag, spec, re.M):
|
|
raise koji.BuildError, "%s is not allowed to be defined in spec file" % tag
|
|
|
|
def patch_scm_source(self, sourcedir, logfile, opts):
|
|
# override if desired
|
|
pass
|
|
|
|
def checkHost(self, hostdata):
|
|
tag = self.params[1]
|
|
return self.checkHostArch(tag, hostdata)
|
|
|
|
def handler(self, url, build_tag, opts=None):
|
|
# will throw a BuildError if the url is invalid
|
|
scm = SCM(url)
|
|
scm.assert_allowed(self.options.allowed_scms)
|
|
|
|
if opts is None:
|
|
opts = {}
|
|
repo_id = opts.get('repo_id')
|
|
if not repo_id:
|
|
raise koji.BuildError, "A repo id must be provided"
|
|
|
|
repo_info = self.session.repoInfo(repo_id, strict=True)
|
|
event_id = repo_info['create_event']
|
|
build_tag = self.session.getTag(build_tag, strict=True, event=event_id)
|
|
|
|
# need DNS in the chroot because "make srpm" may need to contact
|
|
# a SCM or lookaside cache to retrieve the srpm contents
|
|
rootopts = {'install_group': 'srpm-build',
|
|
'setup_dns': True,
|
|
'repo_id': repo_id}
|
|
br_arch = self.find_arch('noarch', self.session.host.getHost(), self.session.getBuildConfig(build_tag['id'], event=event_id))
|
|
broot = BuildRoot(self.session, self.options, build_tag['id'], br_arch, self.id, **rootopts)
|
|
broot.workdir = self.workdir
|
|
|
|
self.logger.debug("Initializing buildroot")
|
|
broot.init()
|
|
|
|
# Setup files and directories for SRPM creation
|
|
# We can't put this under the mock homedir because that directory
|
|
# is completely blown away and recreated on every mock invocation
|
|
scmdir = broot.rootdir() + '/tmp/scmroot'
|
|
koji.ensuredir(scmdir)
|
|
logfile = self.workdir + '/checkout.log'
|
|
uploadpath = self.getUploadDir()
|
|
|
|
# Check out spec file, etc. from SCM
|
|
sourcedir = scm.checkout(scmdir, self.session, uploadpath, logfile)
|
|
# chown the sourcedir and everything under it to the mockuser
|
|
# so we can build the srpm as non-root
|
|
uid = pwd.getpwnam(self.options.mockuser)[2]
|
|
# rpmbuild seems to complain if it's running in the "mock" group but
|
|
# files are in a different group
|
|
gid = grp.getgrnam('mock')[2]
|
|
self.chownTree(scmdir, uid, gid)
|
|
|
|
# Hook for patching spec file in place
|
|
self.patch_scm_source(sourcedir, logfile, opts)
|
|
|
|
# Find and verify that there is only one spec file.
|
|
spec_files = glob.glob("%s/*.spec" % sourcedir)
|
|
if not spec_files and self.options.support_rpm_source_layout:
|
|
# also check SPECS dir
|
|
spec_files = glob.glob("%s/SPECS/*.spec" % sourcedir)
|
|
if len(spec_files) == 0:
|
|
raise koji.BuildError("No spec file found")
|
|
elif len(spec_files) > 1:
|
|
raise koji.BuildError("Multiple spec files found: %s" % spec_files)
|
|
spec_file = spec_files[0]
|
|
|
|
# Run spec file sanity checks. Any failures will throw a BuildError
|
|
self.spec_sanity_checks(spec_file)
|
|
|
|
#build srpm
|
|
self.logger.debug("Running srpm build")
|
|
broot.build_srpm(spec_file, sourcedir, scm.source_cmd)
|
|
|
|
srpms = glob.glob('%s/*.src.rpm' % broot.resultdir())
|
|
if len(srpms) == 0:
|
|
raise koji.BuildError, "No srpms found in %s" % sourcedir
|
|
elif len(srpms) > 1:
|
|
raise koji.BuildError, "Multiple srpms found in %s: %s" % (sourcedir, ", ".join(srpms))
|
|
else:
|
|
srpm = srpms[0]
|
|
|
|
# check srpm name
|
|
h = koji.get_rpm_header(srpm)
|
|
name = h[rpm.RPMTAG_NAME]
|
|
version = h[rpm.RPMTAG_VERSION]
|
|
release = h[rpm.RPMTAG_RELEASE]
|
|
srpm_name = "%(name)s-%(version)s-%(release)s.src.rpm" % locals()
|
|
if srpm_name != os.path.basename(srpm):
|
|
raise koji.BuildError, 'srpm name mismatch: %s != %s' % (srpm_name, os.path.basename(srpm))
|
|
|
|
#upload srpm and return
|
|
self.uploadFile(srpm)
|
|
|
|
brootid = broot.id
|
|
log_files = glob.glob('%s/*.log' % broot.resultdir())
|
|
|
|
broot.expire()
|
|
|
|
return {'srpm': "%s/%s" % (uploadpath, srpm_name),
|
|
'logs': ["%s/%s" % (uploadpath, os.path.basename(f))
|
|
for f in log_files],
|
|
'brootid': brootid,
|
|
}
|
|
|
|
class TagNotificationTask(BaseTaskHandler):
|
|
Methods = ['tagNotification']
|
|
|
|
_taskWeight = 0.1
|
|
|
|
message_templ = \
|
|
"""From: %(from_addr)s\r
|
|
Subject: %(nvr)s %(result)s %(operation)s by %(user_name)s\r
|
|
To: %(to_addrs)s\r
|
|
X-Koji-Package: %(pkg_name)s\r
|
|
X-Koji-NVR: %(nvr)s\r
|
|
X-Koji-User: %(user_name)s\r
|
|
X-Koji-Status: %(status)s\r
|
|
%(tag_headers)s\r
|
|
\r
|
|
Package: %(pkg_name)s\r
|
|
NVR: %(nvr)s\r
|
|
User: %(user_name)s\r
|
|
Status: %(status)s\r
|
|
%(operation_details)s\r
|
|
%(nvr)s %(result)s %(operation)s by %(user_name)s\r
|
|
%(failure_info)s\r
|
|
"""
|
|
|
|
def handler(self, recipients, is_successful, tag_info, from_info, build_info, user_info, ignore_success=None, failure_msg=''):
|
|
if len(recipients) == 0:
|
|
self.logger.debug('task %i: no recipients, not sending notifications', self.id)
|
|
return
|
|
|
|
if ignore_success and is_successful:
|
|
self.logger.debug('task %i: tag operation successful and ignore success is true, not sending notifications', self.id)
|
|
return
|
|
|
|
build = self.session.getBuild(build_info)
|
|
user = self.session.getUser(user_info)
|
|
pkg_name = build['package_name']
|
|
nvr = koji.buildLabel(build)
|
|
user_name = user['name']
|
|
|
|
from_addr = self.options.from_addr
|
|
to_addrs = ', '.join(recipients)
|
|
|
|
operation = '%(action)s'
|
|
operation_details = 'Tag Operation: %(action)s\r\n'
|
|
tag_headers = ''
|
|
if from_info:
|
|
from_tag = self.session.getTag(from_info)
|
|
from_tag_name = from_tag['name']
|
|
operation += ' from %s' % from_tag_name
|
|
operation_details += 'From Tag: %s\r\n' % from_tag_name
|
|
tag_headers += 'X-Koji-Tag: %s' % from_tag_name
|
|
action = 'untagged'
|
|
if tag_info:
|
|
tag = self.session.getTag(tag_info)
|
|
tag_name = tag['name']
|
|
operation += ' into %s' % tag_name
|
|
operation_details += 'Into Tag: %s\r\n' % tag_name
|
|
if tag_headers:
|
|
tag_headers += '\r\n'
|
|
tag_headers += 'X-Koji-Tag: %s' % tag_name
|
|
action = 'tagged'
|
|
if tag_info and from_info:
|
|
action = 'moved'
|
|
operation = operation % locals()
|
|
operation_details = operation_details % locals()
|
|
|
|
if is_successful:
|
|
result = 'successfully'
|
|
status = 'complete'
|
|
failure_info = ''
|
|
else:
|
|
result = 'unsuccessfully'
|
|
status = 'failed'
|
|
failure_info = "Operation failed with the error:\r\n %s\r\n" % failure_msg
|
|
|
|
message = self.message_templ % locals()
|
|
# ensure message is in UTF-8
|
|
message = koji.fixEncoding(message)
|
|
|
|
server = smtplib.SMTP(self.options.smtphost)
|
|
#server.set_debuglevel(True)
|
|
server.sendmail(from_addr, recipients, message)
|
|
server.quit()
|
|
|
|
return 'sent notification of tag operation %i to: %s' % (self.id, to_addrs)
|
|
|
|
class BuildNotificationTask(BaseTaskHandler):
|
|
Methods = ['buildNotification']
|
|
|
|
_taskWeight = 0.1
|
|
|
|
# XXX externalize these templates somewhere
|
|
subject_templ = """Package: %(build_nvr)s Tag: %(dest_tag)s Status: %(status)s Built by: %(build_owner)s"""
|
|
message_templ = \
|
|
"""From: %(from_addr)s\r
|
|
Subject: %(subject)s\r
|
|
To: %(to_addrs)s\r
|
|
X-Koji-Tag: %(dest_tag)s\r
|
|
X-Koji-Package: %(build_pkg_name)s\r
|
|
X-Koji-Builder: %(build_owner)s\r
|
|
X-Koji-Status: %(status)s\r
|
|
\r
|
|
Package: %(build_nvr)s\r
|
|
Tag: %(dest_tag)s\r
|
|
Status: %(status)s%(cancel_info)s\r
|
|
Built by: %(build_owner)s\r
|
|
ID: %(build_id)i\r
|
|
Started: %(creation_time)s\r
|
|
Finished: %(completion_time)s\r
|
|
%(changelog)s\r
|
|
%(failure)s\r
|
|
%(output)s\r
|
|
Task Info: %(weburl)s/taskinfo?taskID=%(task_id)i\r
|
|
Build Info: %(weburl)s/buildinfo?buildID=%(build_id)i\r
|
|
"""
|
|
|
|
def _getTaskData(self, task_id, data=None):
|
|
if not data:
|
|
data = {}
|
|
taskinfo = self.session.getTaskInfo(task_id)
|
|
|
|
if not taskinfo:
|
|
# invalid task_id
|
|
return data
|
|
|
|
if taskinfo['host_id']:
|
|
hostinfo = self.session.getHost(taskinfo['host_id'])
|
|
else:
|
|
hostinfo = None
|
|
|
|
result = None
|
|
try:
|
|
result = self.session.getTaskResult(task_id)
|
|
except:
|
|
excClass, result = sys.exc_info()[:2]
|
|
if hasattr(result, 'faultString'):
|
|
result = result.faultString
|
|
else:
|
|
result = '%s: %s' % (excClass.__name__, result)
|
|
result = result.strip()
|
|
# clear the exception, since we're just using
|
|
# it for display purposes
|
|
sys.exc_clear()
|
|
if not result:
|
|
result = 'Unknown'
|
|
|
|
files = self.session.listTaskOutput(task_id)
|
|
logs = [filename for filename in files if filename.endswith('.log')]
|
|
rpms = [filename for filename in files if filename.endswith('.rpm') and not filename.endswith('.src.rpm')]
|
|
srpms = [filename for filename in files if filename.endswith('.src.rpm')]
|
|
misc = [filename for filename in files if filename not in logs + rpms + srpms]
|
|
|
|
logs.sort()
|
|
rpms.sort()
|
|
misc.sort()
|
|
|
|
data[task_id] = {}
|
|
data[task_id]['id'] = taskinfo['id']
|
|
data[task_id]['method'] = taskinfo['method']
|
|
data[task_id]['arch'] = taskinfo['arch']
|
|
data[task_id]['build_arch'] = taskinfo['label']
|
|
data[task_id]['host'] = hostinfo and hostinfo['name'] or None
|
|
data[task_id]['state'] = koji.TASK_STATES[taskinfo['state']].lower()
|
|
data[task_id]['result'] = result
|
|
data[task_id]['request'] = self.session.getTaskRequest(task_id)
|
|
data[task_id]['logs'] = logs
|
|
data[task_id]['rpms'] = rpms
|
|
data[task_id]['srpms'] = srpms
|
|
data[task_id]['misc'] = misc
|
|
|
|
children = self.session.getTaskChildren(task_id)
|
|
for child in children:
|
|
data = self._getTaskData(child['id'], data)
|
|
return data
|
|
|
|
def handler(self, recipients, build, target, weburl):
|
|
if len(recipients) == 0:
|
|
self.logger.debug('task %i: no recipients, not sending notifications', self.id)
|
|
return
|
|
|
|
build_pkg_name = build['package_name']
|
|
build_pkg_evr = '%s%s-%s' % ((build['epoch'] and str(build['epoch']) + ':' or ''), build['version'], build['release'])
|
|
build_nvr = koji.buildLabel(build)
|
|
build_id = build['id']
|
|
build_owner = build['owner_name']
|
|
# target comes from session.py:_get_build_target()
|
|
dest_tag = None
|
|
if target is not None:
|
|
dest_tag = target['dest_tag_name']
|
|
status = koji.BUILD_STATES[build['state']].lower()
|
|
creation_time = koji.formatTimeLong(build['creation_time'])
|
|
completion_time = koji.formatTimeLong(build['completion_time'])
|
|
task_id = build['task_id']
|
|
|
|
task_data = self._getTaskData(task_id)
|
|
|
|
cancel_info = ''
|
|
failure_info = ''
|
|
if build['state'] == koji.BUILD_STATES['CANCELED']:
|
|
# The owner of the buildNotification task is the one
|
|
# who canceled the task, it turns out.
|
|
this_task = self.session.getTaskInfo(self.id)
|
|
if this_task['owner']:
|
|
canceler = self.session.getUser(this_task['owner'])
|
|
cancel_info = "\r\nCanceled by: %s" % canceler['name']
|
|
elif build['state'] == koji.BUILD_STATES['FAILED']:
|
|
failure_data = task_data[task_id]['result']
|
|
failed_hosts = ['%s (%s)' % (task['host'], task['arch']) for task in task_data.values() if task['host'] and task['state'] == 'failed']
|
|
failure_info = "\r\n%s (%d) failed on %s:\r\n %s" % (build_nvr, build_id,
|
|
', '.join(failed_hosts),
|
|
failure_data)
|
|
|
|
failure = failure_info or cancel_info or ''
|
|
|
|
tasks = {'failed' : [task for task in task_data.values() if task['state'] == 'failed'],
|
|
'canceled' : [task for task in task_data.values() if task['state'] == 'canceled'],
|
|
'closed' : [task for task in task_data.values() if task['state'] == 'closed']}
|
|
|
|
srpms = []
|
|
for taskinfo in task_data.values():
|
|
for srpmfile in taskinfo['srpms']:
|
|
srpms.append(srpmfile)
|
|
srpms = self.uniq(srpms)
|
|
srpms.sort()
|
|
|
|
if srpms:
|
|
output = "SRPMS:\r\n"
|
|
for srpm in srpms:
|
|
output += " %s" % srpm
|
|
output += "\r\n\r\n"
|
|
else:
|
|
output = ''
|
|
|
|
pathinfo = koji.PathInfo(topdir=self.options.topurl)
|
|
buildurl = pathinfo.build(build)
|
|
# list states here to make them go in the correct order
|
|
for task_state in ['failed', 'canceled', 'closed']:
|
|
if tasks[task_state]:
|
|
output += "%s tasks:\r\n" % task_state.capitalize()
|
|
output += "%s-------\r\n\r\n" % ("-" * len(task_state))
|
|
for task in tasks[task_state]:
|
|
output += "Task %s" % task['id']
|
|
if task['host']:
|
|
output += " on %s\r\n" % task['host']
|
|
else:
|
|
output += "\r\n"
|
|
output += "Task Type: %s\r\n" % koji.taskLabel(task)
|
|
for filetype in ['logs', 'rpms', 'misc']:
|
|
if task[filetype]:
|
|
output += "%s:\r\n" % filetype
|
|
for file in task[filetype]:
|
|
if filetype == 'rpms':
|
|
output += " %s\r\n" % '/'.join([buildurl, task['build_arch'], file])
|
|
elif filetype == 'logs':
|
|
if tasks[task_state] != 'closed':
|
|
output += " %s/getfile?taskID=%s&name=%s\r\n" % (weburl, task['id'], file)
|
|
else:
|
|
output += " %s\r\n" % '/'.join([buildurl, 'data', 'logs', task['build_arch'], file])
|
|
elif task[filetype] == 'misc':
|
|
output += " %s/getfile?taskID=%s&name=%s\r\n" % (weburl, task['id'], file)
|
|
output += "\r\n"
|
|
output += "\r\n"
|
|
|
|
changelog = koji.util.formatChangelog(self.session.getChangelogEntries(build_id, queryOpts={'limit': 3})).replace("\n","\r\n")
|
|
if changelog:
|
|
changelog = "Changelog:\r\n%s" % changelog
|
|
|
|
from_addr = self.options.from_addr
|
|
to_addrs = ', '.join(recipients)
|
|
subject = self.subject_templ % locals()
|
|
message = self.message_templ % locals()
|
|
# ensure message is in UTF-8
|
|
message = koji.fixEncoding(message)
|
|
|
|
server = smtplib.SMTP(self.options.smtphost)
|
|
# server.set_debuglevel(True)
|
|
server.sendmail(from_addr, recipients, message)
|
|
server.quit()
|
|
|
|
return 'sent notification of build %i to: %s' % (build_id, to_addrs)
|
|
|
|
def uniq(self, items):
|
|
"""Remove duplicates from the list of items, and sort the list."""
|
|
m = dict(zip(items, [1] * len(items)))
|
|
l = m.keys()
|
|
l.sort()
|
|
return l
|
|
|
|
|
|
class NewRepoTask(BaseTaskHandler):
|
|
Methods = ['newRepo']
|
|
_taskWeight = 0.1
|
|
|
|
def handler(self, tag, event=None, src=False, debuginfo=False):
|
|
tinfo = self.session.getTag(tag, strict=True, event=event)
|
|
kwargs = {}
|
|
if event is not None:
|
|
kwargs['event'] = event
|
|
if src:
|
|
kwargs['with_src'] = True
|
|
if debuginfo:
|
|
kwargs['with_debuginfo'] = True
|
|
repo_id, event_id = self.session.host.repoInit(tinfo['id'], **kwargs)
|
|
path = koji.pathinfo.repo(repo_id, tinfo['name'])
|
|
if not os.path.isdir(path):
|
|
raise koji.GenericError, "Repo directory missing: %s" % path
|
|
arches = []
|
|
for fn in os.listdir(path):
|
|
if fn != 'groups' and os.path.isfile("%s/%s/pkglist" % (path, fn)):
|
|
arches.append(fn)
|
|
#see if we can find a previous repo to update from
|
|
#only shadowbuild tags should start with SHADOWBUILD, their repos are auto
|
|
#expired. so lets get the most recent expired tag for newRepo shadowbuild tasks.
|
|
if tinfo['name'].startswith('SHADOWBUILD'):
|
|
oldrepo = self.session.getRepo(tinfo['id'], state=koji.REPO_EXPIRED)
|
|
else:
|
|
oldrepo = self.session.getRepo(tinfo['id'], state=koji.REPO_READY)
|
|
subtasks = {}
|
|
for arch in arches:
|
|
arglist = [repo_id, arch, oldrepo]
|
|
subtasks[arch] = self.session.host.subtask(method='createrepo',
|
|
arglist=arglist,
|
|
label=arch,
|
|
parent=self.id,
|
|
arch='noarch')
|
|
|
|
# gather subtask results
|
|
data = {}
|
|
if subtasks:
|
|
results = self.wait(subtasks.values(), all=True, failany=True)
|
|
for (arch, task_id) in subtasks.iteritems():
|
|
data[arch] = results[task_id]
|
|
self.logger.debug("DEBUG: %r : %r " % (arch,data[arch],))
|
|
|
|
# finalize
|
|
kwargs = {}
|
|
if event is not None:
|
|
kwargs['expire'] = True
|
|
self.session.host.repoDone(repo_id, data, **kwargs)
|
|
return repo_id, event_id
|
|
|
|
|
|
class CreaterepoTask(BaseTaskHandler):
|
|
|
|
Methods = ['createrepo']
|
|
_taskWeight = 1.5
|
|
|
|
def handler(self, repo_id, arch, oldrepo):
|
|
#arch is the arch of the repo, not the task
|
|
rinfo = self.session.repoInfo(repo_id, strict=True)
|
|
if rinfo['state'] != koji.REPO_INIT:
|
|
raise koji.GenericError, "Repo %(id)s not in INIT state (got %(state)s)" % rinfo
|
|
self.repo_id = rinfo['id']
|
|
self.pathinfo = koji.PathInfo(self.options.topdir)
|
|
toprepodir = self.pathinfo.repo(repo_id, rinfo['tag_name'])
|
|
self.repodir = '%s/%s' % (toprepodir, arch)
|
|
if not os.path.isdir(self.repodir):
|
|
raise koji.GenericError, "Repo directory missing: %s" % self.repodir
|
|
groupdata = os.path.join(toprepodir, 'groups', 'comps.xml')
|
|
#set up our output dir
|
|
self.outdir = '%s/repo' % self.workdir
|
|
self.datadir = '%s/repodata' % self.outdir
|
|
pkglist = os.path.join(self.repodir, 'pkglist')
|
|
if os.path.getsize(pkglist) == 0:
|
|
pkglist = None
|
|
self.create_local_repo(rinfo, arch, pkglist, groupdata, oldrepo)
|
|
|
|
external_repos = self.session.getExternalRepoList(rinfo['tag_id'], event=rinfo['create_event'])
|
|
if external_repos:
|
|
self.merge_repos(external_repos, arch, groupdata)
|
|
elif pkglist is None:
|
|
fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w')
|
|
fo.write("This repo is empty because its tag has no content for this arch\n")
|
|
fo.close()
|
|
|
|
uploadpath = self.getUploadDir()
|
|
files = []
|
|
for f in os.listdir(self.datadir):
|
|
files.append(f)
|
|
self.session.uploadWrapper('%s/%s' % (self.datadir, f), uploadpath, f)
|
|
|
|
return [uploadpath, files]
|
|
|
|
def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo):
|
|
koji.ensuredir(self.outdir)
|
|
if self.options.use_createrepo_c:
|
|
cmd = ['/usr/bin/createrepo_c']
|
|
else:
|
|
cmd = ['/usr/bin/createrepo']
|
|
cmd.extend(['-vd', '-o', self.outdir])
|
|
if pkglist is not None:
|
|
cmd.extend(['-i', pkglist])
|
|
if os.path.isfile(groupdata):
|
|
cmd.extend(['-g', groupdata])
|
|
#attempt to recycle repodata from last repo
|
|
if pkglist and oldrepo and self.options.createrepo_update:
|
|
oldpath = self.pathinfo.repo(oldrepo['id'], rinfo['tag_name'])
|
|
olddatadir = '%s/%s/repodata' % (oldpath, arch)
|
|
if not os.path.isdir(olddatadir):
|
|
self.logger.warn("old repodata is missing: %s" % olddatadir)
|
|
else:
|
|
shutil.copytree(olddatadir, self.datadir)
|
|
oldorigins = os.path.join(self.datadir, 'pkgorigins.gz')
|
|
if os.path.isfile(oldorigins):
|
|
# remove any previous origins file and rely on mergerepos
|
|
# to rewrite it (if we have external repos to merge)
|
|
os.unlink(oldorigins)
|
|
cmd.append('--update')
|
|
if self.options.createrepo_skip_stat:
|
|
cmd.append('--skip-stat')
|
|
# note: we can't easily use a cachedir because we do not have write
|
|
# permission. The good news is that with --update we won't need to
|
|
# be scanning many rpms.
|
|
if pkglist is None:
|
|
cmd.append(self.outdir)
|
|
else:
|
|
cmd.append(self.repodir)
|
|
|
|
logfile = '%s/createrepo.log' % self.workdir
|
|
status = log_output(self.session, cmd[0], cmd, logfile, self.getUploadDir(), logerror=True)
|
|
if not isSuccess(status):
|
|
raise koji.GenericError, 'failed to create repo: %s' \
|
|
% parseStatus(status, ' '.join(cmd))
|
|
|
|
def merge_repos(self, external_repos, arch, groupdata):
|
|
repos = []
|
|
localdir = '%s/repo_%s_premerge' % (self.workdir, self.repo_id)
|
|
os.rename(self.outdir, localdir)
|
|
koji.ensuredir(self.outdir)
|
|
repos.append('file://' + localdir + '/')
|
|
|
|
for repo in external_repos:
|
|
ext_url = repo['url']
|
|
# substitute $arch in the url with the arch of the repo we're generating
|
|
ext_url = ext_url.replace('$arch', arch)
|
|
repos.append(ext_url)
|
|
|
|
blocklist = self.repodir + '/blocklist'
|
|
if self.options.use_createrepo_c:
|
|
cmd = ['/usr/bin/mergerepo_c', '--koji']
|
|
else:
|
|
cmd = ['/usr/libexec/kojid/mergerepos']
|
|
cmd.extend(['--tempdir', self.workdir])
|
|
cmd.extend(['-a', arch, '-b', blocklist, '-o', self.outdir])
|
|
if os.path.isfile(groupdata):
|
|
cmd.extend(['-g', groupdata])
|
|
for repo in repos:
|
|
cmd.extend(['-r', repo])
|
|
|
|
logfile = '%s/mergerepos.log' % self.workdir
|
|
status = log_output(self.session, cmd[0], cmd, logfile, self.getUploadDir(), logerror=True)
|
|
if not isSuccess(status):
|
|
raise koji.GenericError, 'failed to merge repos: %s' \
|
|
% parseStatus(status, ' '.join(cmd))
|
|
|
|
class WaitrepoTask(BaseTaskHandler):
|
|
|
|
Methods = ['waitrepo']
|
|
#mostly just waiting
|
|
_taskWeight = 0.2
|
|
|
|
PAUSE = 60
|
|
# time in minutes before we fail this task
|
|
TIMEOUT = 120
|
|
|
|
def handler(self, tag, newer_than=None, nvrs=None):
|
|
"""Wait for a repo for the tag, subject to given conditions
|
|
|
|
newer_than: create_event timestamp should be newer than this
|
|
nvr: repo should contain this nvr (which may not exist at first)
|
|
|
|
Only one of the options may be specified. If neither is, then
|
|
the call will wait for the first ready repo.
|
|
|
|
Returns the repo info (from getRepo) of the chosen repo
|
|
"""
|
|
|
|
start = time.time()
|
|
|
|
taginfo = self.session.getTag(tag, strict=True)
|
|
targets = self.session.getBuildTargets(buildTagID=taginfo['id'])
|
|
if not targets:
|
|
raise koji.GenericError("No build target for tag: %s" % taginfo['name'])
|
|
|
|
if isinstance(newer_than, basestring) and newer_than.lower() == "now":
|
|
newer_than = start
|
|
if not isinstance(newer_than, (type(None), int, long, float)):
|
|
raise koji.GenericError, "Invalid value for newer_than: %s" % newer_than
|
|
|
|
if newer_than and nvrs:
|
|
raise koji.GenericError, "only one of (newer_than, nvrs) may be specified"
|
|
|
|
if not nvrs:
|
|
nvrs = []
|
|
builds = [koji.parse_NVR(nvr) for nvr in nvrs]
|
|
|
|
last_repo = None
|
|
|
|
while True:
|
|
repo = self.session.getRepo(taginfo['id'])
|
|
if repo and repo != last_repo:
|
|
if builds:
|
|
if koji.util.checkForBuilds(self.session, taginfo['id'], builds, repo['create_event']):
|
|
self.logger.debug("Successfully waited %s for %s to appear in the %s repo" % \
|
|
(koji.util.duration(start), koji.util.printList(nvrs), taginfo['name']))
|
|
return repo
|
|
elif newer_than:
|
|
if repo['create_ts'] > newer_than:
|
|
self.logger.debug("Successfully waited %s for a new %s repo" % \
|
|
(koji.util.duration(start), taginfo['name']))
|
|
return repo
|
|
else:
|
|
#no check requested -- return first ready repo
|
|
return repo
|
|
|
|
if (time.time() - start) > (self.TIMEOUT * 60.0):
|
|
if builds:
|
|
raise koji.GenericError, "Unsuccessfully waited %s for %s to appear in the %s repo" % \
|
|
(koji.util.duration(start), koji.util.printList(nvrs), taginfo['name'])
|
|
else:
|
|
raise koji.GenericError, "Unsuccessfully waited %s for a new %s repo" % \
|
|
(koji.util.duration(start), taginfo['name'])
|
|
|
|
time.sleep(self.PAUSE)
|
|
last_repo = repo
|
|
|
|
|
|
def get_options():
|
|
"""process options from command line and config file"""
|
|
# parse command line args
|
|
logger = logging.getLogger("koji.build")
|
|
parser = OptionParser()
|
|
parser.add_option("-c", "--config", dest="configFile",
|
|
help="use alternate configuration file", metavar="FILE",
|
|
default="/etc/kojid/kojid.conf")
|
|
parser.add_option("--user", help="specify user")
|
|
parser.add_option("--password", help="specify password")
|
|
parser.add_option("-f", "--fg", dest="daemon",
|
|
action="store_false", default=True,
|
|
help="run in foreground")
|
|
parser.add_option("--force-lock", action="store_true", default=False,
|
|
help="force lock for exclusive session")
|
|
parser.add_option("-v", "--verbose", action="store_true", default=False,
|
|
help="show verbose output")
|
|
parser.add_option("-d", "--debug", action="store_true", default=False,
|
|
help="show debug output")
|
|
parser.add_option("--debug-task", action="store_true", default=False,
|
|
help="enable debug output for tasks")
|
|
parser.add_option("--debug-xmlrpc", action="store_true", default=False,
|
|
help="show xmlrpc debug output")
|
|
parser.add_option("--debug-mock", action="store_true", default=False,
|
|
#obsolete option
|
|
help=SUPPRESS_HELP)
|
|
parser.add_option("--skip-main", action="store_true", default=False,
|
|
help="don't actually run main")
|
|
parser.add_option("--maxjobs", type='int', help="Specify maxjobs")
|
|
parser.add_option("--minspace", type='int', help="Specify minspace")
|
|
parser.add_option("--sleeptime", type='int', help="Specify the polling interval")
|
|
parser.add_option("--admin-emails", help="Address(es) to send error notices to")
|
|
parser.add_option("--topdir", help="Specify topdir")
|
|
parser.add_option("--topurl", help="Specify topurl")
|
|
parser.add_option("--workdir", help="Specify workdir")
|
|
parser.add_option("--pluginpath", help="Specify plugin search path")
|
|
parser.add_option("--plugin", action="append", help="Load specified plugin")
|
|
parser.add_option("--mockdir", help="Specify mockdir")
|
|
parser.add_option("--mockuser", help="User to run mock as")
|
|
parser.add_option("-s", "--server", help="url of XMLRPC server")
|
|
parser.add_option("--pkgurl", help=SUPPRESS_HELP)
|
|
(options, args) = parser.parse_args()
|
|
|
|
if args:
|
|
parser.error("incorrect number of arguments")
|
|
#not reached
|
|
assert False
|
|
|
|
# load local config
|
|
config = ConfigParser()
|
|
config.read(options.configFile)
|
|
for x in config.sections():
|
|
if x != 'kojid':
|
|
quit('invalid section found in config file: %s' % x)
|
|
defaults = {'sleeptime': 15,
|
|
'maxjobs': 10,
|
|
'literal_task_arches': '',
|
|
'minspace': 8192,
|
|
'admin_emails': None,
|
|
'log_level': None,
|
|
'topdir': '/mnt/koji',
|
|
'topurl': None,
|
|
'workdir': '/var/tmp/koji',
|
|
'pluginpath': '/usr/lib/koji-builder-plugins',
|
|
'mockdir': '/var/lib/mock',
|
|
'mockuser': 'kojibuilder',
|
|
'packager': 'Koji',
|
|
'vendor': 'Koji',
|
|
'distribution': 'Koji',
|
|
'mockhost': 'koji-linux-gnu',
|
|
'smtphost': 'example.com',
|
|
'from_addr': 'Koji Build System <buildsys@example.com>',
|
|
'krb_principal': None,
|
|
'host_principal_format': 'compile/%s@EXAMPLE.COM',
|
|
'keytab': '/etc/kojid/kojid.keytab',
|
|
'ccache': '/var/tmp/kojid.ccache',
|
|
'krbservice': 'host',
|
|
'krb_rdns': True,
|
|
'server': None,
|
|
'user': None,
|
|
'password': None,
|
|
'retry_interval': 60,
|
|
'max_retries': 120,
|
|
'offline_retry': True,
|
|
'offline_retry_interval': 120,
|
|
'keepalive' : True,
|
|
'timeout' : None,
|
|
'use_fast_upload': True,
|
|
'use_createrepo_c': False,
|
|
'createrepo_skip_stat': True,
|
|
'createrepo_update': True,
|
|
'pkgurl': None,
|
|
'allowed_scms': '',
|
|
'support_rpm_source_layout': True,
|
|
'yum_proxy': None,
|
|
'maven_repo_ignore': '*.md5 *.sha1 maven-metadata*.xml _maven.repositories '
|
|
'resolver-status.properties *.lastUpdated',
|
|
'failed_buildroot_lifetime' : 3600 * 4,
|
|
'rpmbuild_timeout' : 3600 * 24,
|
|
'cert': '/etc/kojid/client.crt',
|
|
'ca': '', # FIXME: Unused, remove in next major release
|
|
'serverca': '/etc/kojid/serverca.crt'}
|
|
if config.has_section('kojid'):
|
|
for name, value in config.items('kojid'):
|
|
if name in ['sleeptime', 'maxjobs', 'minspace', 'retry_interval',
|
|
'max_retries', 'offline_retry_interval', 'failed_buildroot_lifetime',
|
|
'timeout', 'rpmbuild_timeout',]:
|
|
try:
|
|
defaults[name] = int(value)
|
|
except ValueError:
|
|
quit("value for %s option must be a valid integer" % name)
|
|
elif name in ['offline_retry', 'use_createrepo_c', 'createrepo_skip_stat',
|
|
'createrepo_update', 'keepalive', 'use_fast_upload',
|
|
'support_rpm_source_layout', 'krb_rdns',
|
|
'build_arch_can_fail']:
|
|
defaults[name] = config.getboolean('kojid', name)
|
|
elif name in ['plugin', 'plugins']:
|
|
defaults['plugin'] = value.split()
|
|
elif name in defaults.keys():
|
|
defaults[name] = value
|
|
elif name.upper().startswith('RLIMIT_'):
|
|
defaults[name.upper()] = value
|
|
else:
|
|
quit("unknown config option: %s" % name)
|
|
for name, value in defaults.items():
|
|
if getattr(options, name, None) is None:
|
|
setattr(options, name, value)
|
|
|
|
#honor topdir
|
|
if options.topdir:
|
|
koji.BASEDIR = options.topdir
|
|
koji.pathinfo.topdir = options.topdir
|
|
|
|
#make sure workdir exists
|
|
if not os.path.exists(options.workdir):
|
|
koji.ensuredir(options.workdir)
|
|
|
|
if not options.server:
|
|
msg = "the server option is required"
|
|
logger.error(msg)
|
|
parser.error(msg)
|
|
|
|
if not options.topurl:
|
|
msg = "the topurl option is required"
|
|
logger.error(msg)
|
|
parser.error(msg)
|
|
|
|
topurls = options.topurl.split()
|
|
options.topurls = topurls
|
|
if len(topurls) > 1:
|
|
# XXX - fix the rest of the code so this is not necessary
|
|
options.topurl = topurls[0]
|
|
|
|
if options.pkgurl:
|
|
logger.warning("The pkgurl option is obsolete")
|
|
if options.debug_mock:
|
|
logger.warning("The debug-mock option is obsolete")
|
|
|
|
return options
|
|
|
|
def quit(msg=None, code=1):
|
|
if msg:
|
|
logging.getLogger("koji.build").error(msg)
|
|
sys.stderr.write('%s\n' % msg)
|
|
sys.stderr.flush()
|
|
sys.exit(code)
|
|
|
|
if __name__ == "__main__":
|
|
koji.add_file_logger("koji", "/var/log/kojid.log")
|
|
#note we're setting logging params for all of koji*
|
|
options = get_options()
|
|
if options.log_level:
|
|
lvl = getattr(logging, options.log_level, None)
|
|
if lvl is None:
|
|
quit("Invalid log level: %s" % options.log_level)
|
|
logging.getLogger("koji").setLevel(lvl)
|
|
else:
|
|
logging.getLogger("koji").setLevel(logging.WARN)
|
|
if options.debug:
|
|
logging.getLogger("koji").setLevel(logging.DEBUG)
|
|
elif options.verbose:
|
|
logging.getLogger("koji").setLevel(logging.INFO)
|
|
if options.debug_task:
|
|
logging.getLogger("koji.build.BaseTaskHandler").setLevel(logging.DEBUG)
|
|
if options.admin_emails:
|
|
koji.add_mail_logger("koji", options.admin_emails)
|
|
|
|
#build session options
|
|
session_opts = {}
|
|
for k in ('user', 'password', 'krbservice', 'krb_rdns', 'debug_xmlrpc', 'debug',
|
|
'retry_interval', 'max_retries', 'offline_retry', 'offline_retry_interval',
|
|
'keepalive', 'timeout', 'use_fast_upload',
|
|
):
|
|
v = getattr(options, k, None)
|
|
if v is not None:
|
|
session_opts[k] = v
|
|
#start a session and login
|
|
session = koji.ClientSession(options.server, session_opts)
|
|
if os.path.isfile(options.cert):
|
|
try:
|
|
# authenticate using SSL client certificates
|
|
session.ssl_login(options.cert, None,
|
|
options.serverca)
|
|
except koji.AuthError, e:
|
|
quit("Error: Unable to log in: %s" % e)
|
|
except xmlrpclib.ProtocolError:
|
|
quit("Error: Unable to connect to server %s" % (options.server))
|
|
elif options.user:
|
|
try:
|
|
# authenticate using user/password
|
|
session.login()
|
|
except koji.AuthError:
|
|
quit("Error: Unable to log in. Bad credentials?")
|
|
except xmlrpclib.ProtocolError:
|
|
quit("Error: Unable to connect to server %s" % (options.server))
|
|
elif sys.modules.has_key('krbV'):
|
|
krb_principal = options.krb_principal
|
|
if krb_principal is None:
|
|
krb_principal = options.host_principal_format % socket.getfqdn()
|
|
try:
|
|
session.krb_login(principal=krb_principal,
|
|
keytab=options.keytab,
|
|
ccache=options.ccache)
|
|
except krbV.Krb5Error, e:
|
|
quit("Kerberos authentication failed: '%s' (%s)" % (e.args[1], e.args[0]))
|
|
except socket.error, e:
|
|
quit("Could not connect to Kerberos authentication service: '%s'" % e.args[1])
|
|
else:
|
|
quit("No username/password supplied and Kerberos missing or not configured")
|
|
#make session exclusive
|
|
try:
|
|
session.exclusiveSession(force=options.force_lock)
|
|
except koji.AuthLockError:
|
|
quit("Error: Unable to get lock. Trying using --force-lock")
|
|
if not session.logged_in:
|
|
quit("Error: Unknown login error")
|
|
#make sure it works
|
|
try:
|
|
ret = session.echo("OK")
|
|
except xmlrpclib.ProtocolError:
|
|
quit("Error: Unable to connect to server %s" % (options.server))
|
|
if ret != ["OK"]:
|
|
quit("Error: incorrect server response: %r" % (ret))
|
|
|
|
# run main
|
|
if options.daemon:
|
|
#detach
|
|
koji.daemonize()
|
|
main(options, session)
|
|
# not reached
|
|
assert False
|
|
elif not options.skip_main:
|
|
koji.add_stderr_logger("koji")
|
|
main(options, session)
|