Windows builds working, including artifacts import
This commit is contained in:
parent
38f74ed5a0
commit
4aab93fe8f
8 changed files with 347 additions and 62 deletions
29
docs/schema-upgrade-1.4-1.5.sql
Normal file
29
docs/schema-upgrade-1.4-1.5.sql
Normal file
|
|
@ -0,0 +1,29 @@
|
||||||
|
-- upgrade script to migrate the Koji database schema
|
||||||
|
-- from version 1.4 to 1.5
|
||||||
|
|
||||||
|
BEGIN;
|
||||||
|
|
||||||
|
INSERT INTO channels (name) VALUES ('vm');
|
||||||
|
|
||||||
|
insert into archivetypes (name, description, extensions) values ('exe', 'Windows executable', 'exe');
|
||||||
|
insert into archivetypes (name, description, extensions) values ('dll', 'Windows dynamic link library', 'dll');
|
||||||
|
insert into archivetypes (name, description, extensions) values ('sys', 'Windows device driver', 'sys');
|
||||||
|
insert into archivetypes (name, description, extensions) values ('inf', 'Windows driver information file', 'inf');
|
||||||
|
insert into archivetypes (name, description, extensions) values ('cat', 'Windows catalog file', 'cat');
|
||||||
|
insert into archivetypes (name, description, extensions) values ('msi', 'Windows Installer package', 'msi');
|
||||||
|
insert into archivetypes (name, description, extensions) values ('pdb', 'Windows debug information', 'pdb');
|
||||||
|
|
||||||
|
-- flag to indicate that a build is a Windows build
|
||||||
|
CREATE TABLE win_builds (
|
||||||
|
build_id INTEGER NOT NULL PRIMARY KEY REFERENCES build(id),
|
||||||
|
platform TEXT NOT NULL
|
||||||
|
) WITHOUT OIDS;
|
||||||
|
|
||||||
|
-- Extended information about files built in Windows VMs
|
||||||
|
CREATE TABLE win_archives (
|
||||||
|
archive_id INTEGER NOT NULL PRIMARY KEY REFERENCES archiveinfo(id),
|
||||||
|
platforms TEXT NOT NULL,
|
||||||
|
flags TEXT
|
||||||
|
) WITHOUT OIDS;
|
||||||
|
|
||||||
|
COMMIT WORK;
|
||||||
|
|
@ -179,6 +179,7 @@ INSERT INTO channels (name) VALUES ('createrepo');
|
||||||
INSERT INTO channels (name) VALUES ('maven');
|
INSERT INTO channels (name) VALUES ('maven');
|
||||||
INSERT INTO channels (name) VALUES ('livecd');
|
INSERT INTO channels (name) VALUES ('livecd');
|
||||||
INSERT INTO channels (name) VALUES ('appliance');
|
INSERT INTO channels (name) VALUES ('appliance');
|
||||||
|
INSERT INTO channels (name) VALUES ('vm');
|
||||||
|
|
||||||
-- Here we track the build machines
|
-- Here we track the build machines
|
||||||
-- each host has an entry in the users table also
|
-- each host has an entry in the users table also
|
||||||
|
|
@ -668,6 +669,12 @@ CREATE TABLE maven_builds (
|
||||||
version TEXT NOT NULL
|
version TEXT NOT NULL
|
||||||
) WITHOUT OIDS;
|
) WITHOUT OIDS;
|
||||||
|
|
||||||
|
-- Windows-specific build information
|
||||||
|
CREATE TABLE win_builds (
|
||||||
|
build_id INTEGER NOT NULL PRIMARY KEY REFERENCES build(id),
|
||||||
|
platform TEXT NOT NULL
|
||||||
|
) WITHOUT OIDS;
|
||||||
|
|
||||||
-- Even though we call this archiveinfo, we can probably use it for
|
-- Even though we call this archiveinfo, we can probably use it for
|
||||||
-- any filetype output by a build process. In general they will be
|
-- any filetype output by a build process. In general they will be
|
||||||
-- archives (.zip, .jar, .tar.gz) but could also be installer executables (.exe)
|
-- archives (.zip, .jar, .tar.gz) but could also be installer executables (.exe)
|
||||||
|
|
@ -683,6 +690,13 @@ insert into archivetypes (name, description, extensions) values ('zip', 'Zip arc
|
||||||
insert into archivetypes (name, description, extensions) values ('pom', 'Maven Project Object Management files', 'pom');
|
insert into archivetypes (name, description, extensions) values ('pom', 'Maven Project Object Management files', 'pom');
|
||||||
insert into archivetypes (name, description, extensions) values ('tar', 'Tar files', 'tar tar.gz tar.bz2');
|
insert into archivetypes (name, description, extensions) values ('tar', 'Tar files', 'tar tar.gz tar.bz2');
|
||||||
insert into archivetypes (name, description, extensions) values ('xml', 'XML files', 'xml');
|
insert into archivetypes (name, description, extensions) values ('xml', 'XML files', 'xml');
|
||||||
|
insert into archivetypes (name, description, extensions) values ('exe', 'Windows executable', 'exe');
|
||||||
|
insert into archivetypes (name, description, extensions) values ('dll', 'Windows dynamic link library', 'dll');
|
||||||
|
insert into archivetypes (name, description, extensions) values ('sys', 'Windows device driver', 'sys');
|
||||||
|
insert into archivetypes (name, description, extensions) values ('inf', 'Windows driver information file', 'inf');
|
||||||
|
insert into archivetypes (name, description, extensions) values ('cat', 'Windows catalog file', 'cat');
|
||||||
|
insert into archivetypes (name, description, extensions) values ('msi', 'Windows Installer package', 'msi');
|
||||||
|
insert into archivetypes (name, description, extensions) values ('pdb', 'Windows debug information', 'pdb');
|
||||||
|
|
||||||
-- Do we want to enforce a constraint that a build can only generate one
|
-- Do we want to enforce a constraint that a build can only generate one
|
||||||
-- archive with a given name?
|
-- archive with a given name?
|
||||||
|
|
@ -715,4 +729,11 @@ CREATE TABLE buildroot_archives (
|
||||||
) WITHOUT OIDS;
|
) WITHOUT OIDS;
|
||||||
CREATE INDEX buildroot_archives_archive_idx ON buildroot_archives (archive_id);
|
CREATE INDEX buildroot_archives_archive_idx ON buildroot_archives (archive_id);
|
||||||
|
|
||||||
|
-- Extended information about files built in Windows VMs
|
||||||
|
CREATE TABLE win_archives (
|
||||||
|
archive_id INTEGER NOT NULL PRIMARY KEY REFERENCES archiveinfo(id),
|
||||||
|
platforms TEXT NOT NULL,
|
||||||
|
flags TEXT
|
||||||
|
) WITHOUT OIDS;
|
||||||
|
|
||||||
COMMIT WORK;
|
COMMIT WORK;
|
||||||
|
|
|
||||||
136
hub/kojihub.py
136
hub/kojihub.py
|
|
@ -3236,6 +3236,32 @@ def get_maven_build(buildInfo, strict=False):
|
||||||
WHERE build_id = %%(build_id)i""" % ', '.join(fields)
|
WHERE build_id = %%(build_id)i""" % ', '.join(fields)
|
||||||
return _singleRow(query, locals(), fields, strict)
|
return _singleRow(query, locals(), fields, strict)
|
||||||
|
|
||||||
|
def get_win_build(buildInfo, strict=False):
|
||||||
|
"""
|
||||||
|
Retrieve Windows-specific information about a build.
|
||||||
|
buildInfo can be either a string (n-v-r) or an integer
|
||||||
|
(build ID).
|
||||||
|
Returns a map containing the following keys:
|
||||||
|
|
||||||
|
build_id: id of the build (integer)
|
||||||
|
platform: the platform the build was performed on (string)
|
||||||
|
"""
|
||||||
|
fields = ('build_id', 'platform')
|
||||||
|
|
||||||
|
build_id = find_build_id(buildInfo)
|
||||||
|
if not build_id:
|
||||||
|
if strict:
|
||||||
|
raise koji.GenericError, 'No matching build found: %s' % buildInfo
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
query = QueryProcessor(tables=('win_builds',), columns=fields,
|
||||||
|
clauses=('build_id = %(build_id)i',),
|
||||||
|
values={'build_id': build_id})
|
||||||
|
result = query.executeOne()
|
||||||
|
if strict and not result:
|
||||||
|
raise koji.GenericError, 'no such Windows build: %s' % buildInfo
|
||||||
|
return result
|
||||||
|
|
||||||
def list_archives(buildID=None, buildrootID=None, componentBuildrootID=None, hostID=None, type=None,
|
def list_archives(buildID=None, buildrootID=None, componentBuildrootID=None, hostID=None, type=None,
|
||||||
filename=None, size=None, md5sum=None, typeInfo=None, queryOpts=None):
|
filename=None, size=None, md5sum=None, typeInfo=None, queryOpts=None):
|
||||||
"""
|
"""
|
||||||
|
|
@ -4230,7 +4256,25 @@ def new_maven_build(build, maven_info):
|
||||||
VALUES (%(build_id)i, %(group_id)s, %(artifact_id)s, %(version)s)"""
|
VALUES (%(build_id)i, %(group_id)s, %(artifact_id)s, %(version)s)"""
|
||||||
_dml(insert, maven_info)
|
_dml(insert, maven_info)
|
||||||
|
|
||||||
def import_archive(filepath, buildinfo, type, typeInfo, buildroot_id=None):
|
def new_win_build(build_id, win_info):
|
||||||
|
"""
|
||||||
|
Add Windows metadata to an existing build.
|
||||||
|
win_info must contain a 'platform' key.
|
||||||
|
"""
|
||||||
|
current = get_win_build(build_id, strict=False)
|
||||||
|
if current:
|
||||||
|
if current['platform'] != win_info['platform']:
|
||||||
|
update = UpdateProcessor('win_builds', clauses=['build_id=%(build_id)i'],
|
||||||
|
values={'build_id': build_id})
|
||||||
|
update.set(platform=win_info['platform'])
|
||||||
|
update.execute()
|
||||||
|
else:
|
||||||
|
insert = InsertProcessor('win_builds')
|
||||||
|
insert.set(build_id=build_id)
|
||||||
|
insert.set(platform=win_info['platform'])
|
||||||
|
insert.execute()
|
||||||
|
|
||||||
|
def import_archive(filepath, buildinfo, type, typeInfo, buildroot_id=None, destpath=None):
|
||||||
"""
|
"""
|
||||||
Import an archive file and associate it with a build. The archive can
|
Import an archive file and associate it with a build. The archive can
|
||||||
be any non-rpm filetype supported by Koji.
|
be any non-rpm filetype supported by Koji.
|
||||||
|
|
@ -4240,6 +4284,7 @@ def import_archive(filepath, buildinfo, type, typeInfo, buildroot_id=None):
|
||||||
type: type of the archive being imported. Currently supported archive types: maven
|
type: type of the archive being imported. Currently supported archive types: maven
|
||||||
typeInfo: dict of type-specific information
|
typeInfo: dict of type-specific information
|
||||||
buildroot_id: the id of the buildroot the archive was built in (may be null)
|
buildroot_id: the id of the buildroot the archive was built in (may be null)
|
||||||
|
destpath: the path relative to the destination directory that the file should be moved to (may be null)
|
||||||
"""
|
"""
|
||||||
if not os.path.exists(filepath):
|
if not os.path.exists(filepath):
|
||||||
raise koji.GenericError, 'no such file: %s' % filepath
|
raise koji.GenericError, 'no such file: %s' % filepath
|
||||||
|
|
@ -4283,6 +4328,17 @@ def import_archive(filepath, buildinfo, type, typeInfo, buildroot_id=None):
|
||||||
# move the file to it's final destination
|
# move the file to it's final destination
|
||||||
_import_archive_file(filepath, mavendir)
|
_import_archive_file(filepath, mavendir)
|
||||||
_generate_maven_metadata(maveninfo, mavendir)
|
_generate_maven_metadata(maveninfo, mavendir)
|
||||||
|
elif type == 'win':
|
||||||
|
insert = InsertProcessor('win_archives')
|
||||||
|
insert.set(archive_id=archive_id)
|
||||||
|
insert.set(platforms=' '.join(typeInfo['platforms']))
|
||||||
|
insert.set(flags=' '.join(typeInfo['flags']))
|
||||||
|
insert.execute()
|
||||||
|
wininfo = get_win_build(buildinfo, strict=True)
|
||||||
|
destdir = koji.pathinfo.winbuild(buildinfo, wininfo)
|
||||||
|
if destpath:
|
||||||
|
destdir = os.path.join(destdir, destpath)
|
||||||
|
_import_archive_file(filepath, destdir)
|
||||||
else:
|
else:
|
||||||
raise koji.BuildError, 'unsupported archive type: %s' % type
|
raise koji.BuildError, 'unsupported archive type: %s' % type
|
||||||
|
|
||||||
|
|
@ -8624,6 +8680,24 @@ class HostExports(object):
|
||||||
os.rename(fn,dest)
|
os.rename(fn,dest)
|
||||||
os.symlink(dest,fn)
|
os.symlink(dest,fn)
|
||||||
|
|
||||||
|
def moveWinBuildToScratch(self, task_id, results):
|
||||||
|
"Move a completed scratch build into place (not imported)"
|
||||||
|
if not context.opts.get('EnableWin'):
|
||||||
|
raise koji.GenericError, 'Windows support not enabled'
|
||||||
|
host = Host()
|
||||||
|
host.verify()
|
||||||
|
task = Task(task_id)
|
||||||
|
task.assertHost(host.id)
|
||||||
|
scratchdir = koji.pathinfo.scratch()
|
||||||
|
username = get_user(task.getOwner())['name']
|
||||||
|
destdir = "%s/%s/task_%s" % (scratchdir, username, task_id)
|
||||||
|
for relpath in results['output'].keys() + results['logs']:
|
||||||
|
filename = os.path.join(koji.pathinfo.task(results['task_id']), relpath)
|
||||||
|
dest = os.path.join(destdir, relpath)
|
||||||
|
koji.ensuredir(os.path.dirname(dest))
|
||||||
|
os.rename(filename, dest)
|
||||||
|
os.symlink(dest, filename)
|
||||||
|
|
||||||
def initBuild(self,data):
|
def initBuild(self,data):
|
||||||
"""Create a stub build entry.
|
"""Create a stub build entry.
|
||||||
|
|
||||||
|
|
@ -8775,6 +8849,66 @@ class HostExports(object):
|
||||||
|
|
||||||
_import_wrapper(task.id, build_info, rpm_results)
|
_import_wrapper(task.id, build_info, rpm_results)
|
||||||
|
|
||||||
|
def initWinBuild(self, task_id, build_info, win_info):
|
||||||
|
"""
|
||||||
|
Create a new in-progress Windows build.
|
||||||
|
"""
|
||||||
|
if not context.opts.get('EnableWin'):
|
||||||
|
raise koji.GenericError, 'Windows support not enabled'
|
||||||
|
host = Host()
|
||||||
|
host.verify()
|
||||||
|
#sanity checks
|
||||||
|
task = Task(task_id)
|
||||||
|
task.assertHost(host.id)
|
||||||
|
# build_info must contain name, version, and release
|
||||||
|
data = build_info.copy()
|
||||||
|
data['task_id'] = task_id
|
||||||
|
data['owner'] = task.getOwner()
|
||||||
|
data['state'] = koji.BUILD_STATES['BUILDING']
|
||||||
|
data['completion_time'] = None
|
||||||
|
build_id = new_build(data)
|
||||||
|
new_win_build(build_id, win_info)
|
||||||
|
return build_id
|
||||||
|
|
||||||
|
def completeWinBuild(self, task_id, build_id, results):
|
||||||
|
"""Complete a Windows build"""
|
||||||
|
if not context.opts.get('EnableWin'):
|
||||||
|
raise koji.GenericError, 'Windows support not enabled'
|
||||||
|
host = Host()
|
||||||
|
host.verify()
|
||||||
|
task = Task(task_id)
|
||||||
|
task.assertHost(host.id)
|
||||||
|
|
||||||
|
build_info = get_build(build_id, strict=True)
|
||||||
|
win_info = get_win_build(build_id, strict=True)
|
||||||
|
|
||||||
|
task_dir = koji.pathinfo.task(results['task_id'])
|
||||||
|
# import the build output
|
||||||
|
for relpath, metadata in results['output'].iteritems():
|
||||||
|
archivetype = get_archive_type(relpath)
|
||||||
|
if not archivetype:
|
||||||
|
# Unknown archive type, skip it
|
||||||
|
continue
|
||||||
|
filepath = os.path.join(task_dir, relpath)
|
||||||
|
import_archive(filepath, build_info, 'win', metadata,
|
||||||
|
destpath=os.path.dirname(relpath))
|
||||||
|
|
||||||
|
# move the logs to their final destination
|
||||||
|
for relpath in results['logs']:
|
||||||
|
import_build_log(os.path.join(task_dir, relpath),
|
||||||
|
build_info, subdir='win')
|
||||||
|
|
||||||
|
# update build state
|
||||||
|
st_complete = koji.BUILD_STATES['COMPLETE']
|
||||||
|
update = UpdateProcessor('build', clauses=['id=%(build_id)i'],
|
||||||
|
values={'build_id': build_id})
|
||||||
|
update.set(id=build_id, state=st_complete)
|
||||||
|
update.rawset(completion_time='now()')
|
||||||
|
update.execute()
|
||||||
|
|
||||||
|
# send email
|
||||||
|
build_notification(task_id, build_id)
|
||||||
|
|
||||||
def failBuild(self, task_id, build_id):
|
def failBuild(self, task_id, build_id):
|
||||||
"""Mark the build as failed. If the current state is not
|
"""Mark the build as failed. If the current state is not
|
||||||
'BUILDING', or the current competion_time is not null, a
|
'BUILDING', or the current competion_time is not null, a
|
||||||
|
|
|
||||||
|
|
@ -392,6 +392,7 @@ def load_config(req):
|
||||||
|
|
||||||
['MissingPolicyOk', 'boolean', True],
|
['MissingPolicyOk', 'boolean', True],
|
||||||
['EnableMaven', 'boolean', False],
|
['EnableMaven', 'boolean', False],
|
||||||
|
['EnableWin', 'boolean', False],
|
||||||
|
|
||||||
['LockOut', 'boolean', False],
|
['LockOut', 'boolean', False],
|
||||||
['ServerOffline', 'boolean', False],
|
['ServerOffline', 'boolean', False],
|
||||||
|
|
|
||||||
|
|
@ -1386,6 +1386,10 @@ class PathInfo(object):
|
||||||
release = build['release']
|
release = build['release']
|
||||||
return self.topdir + ("/maven2/%(group_path)s/%(artifact_id)s/%(version)s/%(release)s" % locals())
|
return self.topdir + ("/maven2/%(group_path)s/%(artifact_id)s/%(version)s/%(release)s" % locals())
|
||||||
|
|
||||||
|
def winbuild(self, build, wininfo):
|
||||||
|
"""Return the directory where the Windows build exists"""
|
||||||
|
return self.build(build) + '/win'
|
||||||
|
|
||||||
def mavenrepo(self, build, maveninfo):
|
def mavenrepo(self, build, maveninfo):
|
||||||
"""Return the directory where the Maven artifact exists in the per-tag Maven repo
|
"""Return the directory where the Maven artifact exists in the per-tag Maven repo
|
||||||
(/mnt/koji/repos/tag-name/repo-id/maven2/)"""
|
(/mnt/koji/repos/tag-name/repo-id/maven2/)"""
|
||||||
|
|
|
||||||
90
vm/kojikamid
90
vm/kojikamid
|
|
@ -273,31 +273,49 @@ class SCM(object):
|
||||||
|
|
||||||
class WindowsBuild(object):
|
class WindowsBuild(object):
|
||||||
|
|
||||||
def __init__(self, specpath, workdir):
|
def __init__(self, specpath, build_tag, workdir, opts):
|
||||||
"""constructor: check ini spec file syntax, set build properties"""
|
"""constructor: check ini spec file syntax, set build properties"""
|
||||||
buildconf = ConfigParser()
|
self.build_tag = build_tag
|
||||||
buildconf.read(specpath)
|
self.opts = opts
|
||||||
self.results = None
|
conf = ConfigParser()
|
||||||
|
conf.read(specpath)
|
||||||
|
|
||||||
|
# [naming] section
|
||||||
|
for entry in ('name', 'version', 'release', 'description'):
|
||||||
|
setattr(self, entry, conf.get('naming', entry))
|
||||||
|
|
||||||
|
# [building] section
|
||||||
|
self.platform = conf.get('building', 'platform')
|
||||||
|
# buildrequires and provides are multi-valued (space-separated)
|
||||||
|
for entry in ('buildrequires', 'provides'):
|
||||||
|
setattr(self, entry, conf.get('building', entry).split())
|
||||||
|
# execute is multi-valued (newline-separated)
|
||||||
|
self.execute = [e.strip() for e in conf.get('building', 'execute').split('\n')]
|
||||||
|
|
||||||
|
# [files] section
|
||||||
|
self.output = {}
|
||||||
|
for entry in conf.get('files', 'output').split('\n'):
|
||||||
|
tokens = entry.strip().split(':')
|
||||||
|
filename = tokens[0]
|
||||||
|
metadata = {}
|
||||||
|
metadata['platforms'] = tokens[1].split(',')
|
||||||
|
if len(tokens) > 2:
|
||||||
|
metadata['flags'] = tokens[2].split(',')
|
||||||
|
else:
|
||||||
|
metadata['flags'] = []
|
||||||
|
self.output[filename] = metadata
|
||||||
|
self.logs = [e.strip() for e in conf.get('files', 'logs').split('\n')]
|
||||||
|
|
||||||
# make sure we've got the right sections defined
|
|
||||||
goodsections = ('naming', 'building', 'files')
|
|
||||||
badsections = [s for s in buildconf.sections() if s not in goodsections]
|
|
||||||
if len(badsections) > 0:
|
|
||||||
raise BuildError, 'Unrecognized section(s) in ini: %s' % badsections
|
|
||||||
for section in goodsections:
|
|
||||||
if not buildconf.has_section(section):
|
|
||||||
raise BuildError, 'missing required section in ini: %s' % section
|
|
||||||
map(self.__dict__.update, (buildconf.items('naming'),))
|
|
||||||
for section in ('building', 'files'):
|
|
||||||
for name, value in buildconf.items(section):
|
|
||||||
value = value.split()
|
|
||||||
self.__setattr__(name, value)
|
|
||||||
self.workdir = workdir
|
self.workdir = workdir
|
||||||
|
|
||||||
def checkEnv(self):
|
def checkEnv(self):
|
||||||
"""Is this environment fit to build in, based on the spec file?"""
|
"""Is this environment fit to build in, based on the spec file?"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
def fetchBuildReqs(self):
|
||||||
|
"""Retrieve buildrequires listed in the spec file"""
|
||||||
|
pass
|
||||||
|
|
||||||
def build(self):
|
def build(self):
|
||||||
"""Do the build: run the execute line(s)"""
|
"""Do the build: run the execute line(s)"""
|
||||||
for cmd in self.execute:
|
for cmd in self.execute:
|
||||||
|
|
@ -309,28 +327,20 @@ class WindowsBuild(object):
|
||||||
"""check the build output for viruses"""
|
"""check the build output for viruses"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def gatherOutput(self):
|
def gatherResults(self):
|
||||||
"""gather information about the output from the build, return it"""
|
"""gather information about the output from the build, return it"""
|
||||||
if self.results != None: return self.results
|
return {'name': self.name, 'version': self.version, 'release': self.release,
|
||||||
# TODO: VM platform value in hash?
|
'description': self.description, 'platform': self.platform,
|
||||||
self.results = {}
|
'provides': self.provides,
|
||||||
for ftype in ('output', 'debug', 'log'):
|
'output': self.output, 'logs': self.logs}
|
||||||
files = getattr(self, ftype + 'files')
|
|
||||||
fdict = {}
|
|
||||||
if files:
|
|
||||||
for f in files:
|
|
||||||
if not os.path.exists(os.path.join(self.workdir, f)):
|
|
||||||
raise BuildError('Could not find %s after build' % f)
|
|
||||||
fdict[f] = {'platform': self.platform, 'debug': False, 'flags': ''}
|
|
||||||
self.results[ftype] = fdict
|
|
||||||
return self.results
|
|
||||||
|
|
||||||
def doAll(self):
|
def doAll(self):
|
||||||
"""helper function that runs the entire process"""
|
"""helper function that runs the entire process"""
|
||||||
self.checkEnv()
|
self.checkEnv()
|
||||||
|
self.fetchBuildReqs()
|
||||||
self.build()
|
self.build()
|
||||||
self.virusCheck()
|
self.virusCheck()
|
||||||
return self.gatherOutput()
|
return self.gatherResults()
|
||||||
|
|
||||||
def log(msg):
|
def log(msg):
|
||||||
print >> sys.stderr, '%s: %s' % (datetime.datetime.now().ctime(), msg)
|
print >> sys.stderr, '%s: %s' % (datetime.datetime.now().ctime(), msg)
|
||||||
|
|
@ -398,9 +408,8 @@ def upload_file(server, prefix, path):
|
||||||
|
|
||||||
def upload_results(server, codir, results):
|
def upload_results(server, codir, results):
|
||||||
"""upload the results of a build given the results dict"""
|
"""upload the results of a build given the results dict"""
|
||||||
for output_type in results.keys():
|
for filename in results['output'].keys() + results['logs']:
|
||||||
for output_file in results[output_type].keys():
|
upload_file(server, codir, filename)
|
||||||
upload_file(server, codir, output_file)
|
|
||||||
|
|
||||||
def get_mgmt_server():
|
def get_mgmt_server():
|
||||||
"""retrieve scmurls from kojivmd we'll use to build from"""
|
"""retrieve scmurls from kojivmd we'll use to build from"""
|
||||||
|
|
@ -445,7 +454,7 @@ def apply_patches(src_dir, patch_dir):
|
||||||
if ret:
|
if ret:
|
||||||
raise BuildError, 'error applying patches, output was: %s' % output
|
raise BuildError, 'error applying patches, output was: %s' % output
|
||||||
|
|
||||||
def run_build(workdir, source_url, task_opts):
|
def run_build(workdir, source_url, build_tag, task_opts):
|
||||||
"""run the build"""
|
"""run the build"""
|
||||||
src_scm = SCM(source_url)
|
src_scm = SCM(source_url)
|
||||||
src_dir = src_scm.checkout(os.path.join(workdir, 'source'))
|
src_dir = src_scm.checkout(os.path.join(workdir, 'source'))
|
||||||
|
|
@ -464,7 +473,7 @@ def run_build(workdir, source_url, task_opts):
|
||||||
raise BuildError, 'No .ini file found'
|
raise BuildError, 'No .ini file found'
|
||||||
elif len(specfile) > 1:
|
elif len(specfile) > 1:
|
||||||
raise BuildError, 'Multiple .ini files found'
|
raise BuildError, 'Multiple .ini files found'
|
||||||
winbld = WindowsBuild(os.path.join(spec_dir, specfile[0]), src_dir)
|
winbld = WindowsBuild(os.path.join(spec_dir, specfile[0]), build_tag, src_dir, task_opts)
|
||||||
return winbld.doAll(), src_dir
|
return winbld.doAll(), src_dir
|
||||||
|
|
||||||
def flunk(server):
|
def flunk(server):
|
||||||
|
|
@ -509,13 +518,14 @@ if __name__ == '__main__':
|
||||||
server = get_mgmt_server()
|
server = get_mgmt_server()
|
||||||
info = server.getTaskInfo()
|
info = server.getTaskInfo()
|
||||||
source_url = info[0]
|
source_url = info[0]
|
||||||
if len(info) > 1:
|
build_tag = info[1]
|
||||||
task_opts = info[1]
|
if len(info) > 2:
|
||||||
|
task_opts = info[2]
|
||||||
if not task_opts:
|
if not task_opts:
|
||||||
task_opts = {}
|
task_opts = {}
|
||||||
workdir = '/tmp/workdir'
|
workdir = '/tmp/workdir'
|
||||||
os.mkdir(workdir)
|
os.mkdir(workdir)
|
||||||
results, results_dir = run_build(workdir, source_url, task_opts)
|
results, results_dir = run_build(workdir, source_url, build_tag, task_opts)
|
||||||
if server is not None:
|
if server is not None:
|
||||||
upload_results(server, results_dir, results)
|
upload_results(server, results_dir, results)
|
||||||
server.closeTask(results)
|
server.closeTask(results)
|
||||||
|
|
|
||||||
113
vm/kojivmd
113
vm/kojivmd
|
|
@ -257,7 +257,97 @@ class TaskXMLRPCServer(DaemonXMLRPCServer):
|
||||||
self.register_function(task_handler.upload)
|
self.register_function(task_handler.upload)
|
||||||
self.register_function(task_handler.verifyChecksum)
|
self.register_function(task_handler.verifyChecksum)
|
||||||
|
|
||||||
class VMTask(BaseTaskHandler):
|
|
||||||
|
class WinBuildTask(BaseTaskHandler):
|
||||||
|
"""
|
||||||
|
Spawns a vmExec task to run a build, and imports the output.
|
||||||
|
"""
|
||||||
|
Methods = ['winbuild']
|
||||||
|
_taskWeight = 0.2
|
||||||
|
|
||||||
|
def handler(self, name, source_url, target, opts=None):
|
||||||
|
if not opts:
|
||||||
|
opts = {}
|
||||||
|
|
||||||
|
subopts = koji.util.dslice(opts, ['specfile', 'patches'],
|
||||||
|
strict=False)
|
||||||
|
# specfile and patches options are urls
|
||||||
|
# verify the urls before passing them to the VM
|
||||||
|
for url in [source_url] + subopts.values():
|
||||||
|
scm = SCM(url)
|
||||||
|
scm.assert_allowed(self.options.allowed_scms)
|
||||||
|
|
||||||
|
task_info = self.session.getTaskInfo(self.id)
|
||||||
|
target_info = self.session.getBuildTarget(target)
|
||||||
|
if not target_info:
|
||||||
|
raise koji.BuildError, 'unknown build target: %s' % target
|
||||||
|
dest_tag = self.session.getTag(target_info['dest_tag'], strict=True)
|
||||||
|
build_tag = self.session.getTag(target_info['build_tag'], strict=True)
|
||||||
|
repo_id = opts.get('repo_id')
|
||||||
|
if repo_id:
|
||||||
|
repo_info = session.repoInfo(repo_id)
|
||||||
|
if not repo_info:
|
||||||
|
raise koji.BuildError, 'invalid repo ID: %s' % repo_id
|
||||||
|
policy_data = {
|
||||||
|
'user_id' : task_info['owner'],
|
||||||
|
'source' : source_url,
|
||||||
|
'task_id' : self.id,
|
||||||
|
'build_tag' : build_tag['id'],
|
||||||
|
'skip_tag' : bool(opts.get('skip_tag')),
|
||||||
|
'target': target_info['id']
|
||||||
|
}
|
||||||
|
if not opts.get('skip_tag'):
|
||||||
|
policy_data['tag'] = dest_tag['id']
|
||||||
|
self.session.host.assertPolicy('build_from_repo_id', policy_data)
|
||||||
|
event_id = repo_info['create_event']
|
||||||
|
else:
|
||||||
|
event_id = self.session.getLastEvent()['id']
|
||||||
|
|
||||||
|
subopts['event_id'] = event_id
|
||||||
|
|
||||||
|
task_opts = koji.util.dslice(opts, ['timeout', 'cpus', 'mem'], strict=False)
|
||||||
|
task_id = self.session.host.subtask(method='vmExec',
|
||||||
|
arglist=[name, [source_url, build_tag['name'], subopts], task_opts],
|
||||||
|
label=name[:255],
|
||||||
|
parent=self.id)
|
||||||
|
results = self.wait(task_id)[task_id]
|
||||||
|
results['task_id'] = task_id
|
||||||
|
|
||||||
|
if opts.get('scratch'):
|
||||||
|
self.session.host.moveWinBuildToScratch(self.id, results)
|
||||||
|
else:
|
||||||
|
build_info = koji.util.dslice(results, ['name', 'version', 'release'])
|
||||||
|
pkg_cfg = self.session.getPackageConfig(dest_tag['id'], build_info['name'], event=event_id)
|
||||||
|
if not opts.get('skip_tag'):
|
||||||
|
# Make sure package is on the list for this tag
|
||||||
|
if pkg_cfg is None:
|
||||||
|
raise koji.BuildError, "package %s not in list for tag %s" \
|
||||||
|
% (build_info['name'], dest_tag['name'])
|
||||||
|
elif pkg_cfg['blocked']:
|
||||||
|
raise koji.BuildError, "package %s is blocked for tag %s" \
|
||||||
|
% (build_info['name'], dest_tag['name'])
|
||||||
|
|
||||||
|
# epoch is rpm-specific, so doesn't have a lot of relevance here
|
||||||
|
# but a value is required by the data model
|
||||||
|
build_info['epoch'] = None
|
||||||
|
build_id = self.session.host.initWinBuild(self.id, build_info,
|
||||||
|
koji.util.dslice(results, ['platform']))
|
||||||
|
try:
|
||||||
|
self.session.host.completeWinBuild(self.id, build_id, results)
|
||||||
|
except (SystemExit, ServerExit, KeyboardInterrupt):
|
||||||
|
raise
|
||||||
|
except:
|
||||||
|
self.session.host.failBuild(self.id, build_id)
|
||||||
|
raise
|
||||||
|
if not opts.get('skip_tag') and not opts.get('scratch'):
|
||||||
|
task_id = self.session.host.subtask(method='tagBuild',
|
||||||
|
arglist=[dest_tag['id'], build_id],
|
||||||
|
label='tag',
|
||||||
|
channel='default',
|
||||||
|
parent=self.id)
|
||||||
|
self.wait(task_id)
|
||||||
|
|
||||||
|
class VMExecTask(BaseTaskHandler):
|
||||||
"""
|
"""
|
||||||
Handles the startup, state-tracking, and shutdown of a VM
|
Handles the startup, state-tracking, and shutdown of a VM
|
||||||
for the purposes for executing a single task.
|
for the purposes for executing a single task.
|
||||||
|
|
@ -269,7 +359,7 @@ class VMTask(BaseTaskHandler):
|
||||||
QCOW2_EXT = '.qcow2'
|
QCOW2_EXT = '.qcow2'
|
||||||
|
|
||||||
def __init__(self, *args, **kw):
|
def __init__(self, *args, **kw):
|
||||||
super(VMTask, self).__init__(*args, **kw)
|
super(VMExecTask, self).__init__(*args, **kw)
|
||||||
self.task_manager = xmlrpclib.ServerProxy('http://%s:%s/' % (self.options.privaddr, self.options.portbase),
|
self.task_manager = xmlrpclib.ServerProxy('http://%s:%s/' % (self.options.privaddr, self.options.portbase),
|
||||||
allow_none=True)
|
allow_none=True)
|
||||||
self.port = None
|
self.port = None
|
||||||
|
|
@ -427,7 +517,7 @@ class VMTask(BaseTaskHandler):
|
||||||
thr.setDaemon(True)
|
thr.setDaemon(True)
|
||||||
thr.start()
|
thr.start()
|
||||||
|
|
||||||
def handler(self, name, source_url, opts=None):
|
def handler(self, name, task_info, opts=None):
|
||||||
"""
|
"""
|
||||||
Clone the VM named "name", and provide the data in "task_info" to it.
|
Clone the VM named "name", and provide the data in "task_info" to it.
|
||||||
Available options:
|
Available options:
|
||||||
|
|
@ -436,16 +526,9 @@ class VMTask(BaseTaskHandler):
|
||||||
"""
|
"""
|
||||||
if not opts:
|
if not opts:
|
||||||
opts = {}
|
opts = {}
|
||||||
timeout = opts.get('timeout', 720)
|
timeout = opts.get('timeout', 1440)
|
||||||
|
|
||||||
self.task_info = [source_url, koji.util.dslice(opts, ['specfile', 'patches'],
|
self.task_info = task_info
|
||||||
strict=False)]
|
|
||||||
# opts may contain specfile and patches entries, both of which are
|
|
||||||
# urls.
|
|
||||||
# Verify the urls before passing them to the VM.
|
|
||||||
for url in [source_url] + self.task_info[1].values():
|
|
||||||
scm = SCM(url)
|
|
||||||
scm.assert_allowed(self.options.allowed_scms)
|
|
||||||
|
|
||||||
conn = libvirt.open(None)
|
conn = libvirt.open(None)
|
||||||
clone_name = self.clone(conn, name)
|
clone_name = self.clone(conn, name)
|
||||||
|
|
@ -565,8 +648,8 @@ class VMTaskManager(TaskManager):
|
||||||
disks = []
|
disks = []
|
||||||
for node in nodelist:
|
for node in nodelist:
|
||||||
disk = node.prop('file')
|
disk = node.prop('file')
|
||||||
if os.path.basename(disk).startswith(VMTask.CLONE_PREFIX) and \
|
if os.path.basename(disk).startswith(VMExecTask.CLONE_PREFIX) and \
|
||||||
disk.endswith(VMTask.QCOW2_EXT):
|
disk.endswith(VMExecTask.QCOW2_EXT):
|
||||||
disks.append(disk)
|
disks.append(disk)
|
||||||
ctx.xpathFreeContext()
|
ctx.xpathFreeContext()
|
||||||
doc.freeDoc()
|
doc.freeDoc()
|
||||||
|
|
@ -646,7 +729,7 @@ class VMTaskManager(TaskManager):
|
||||||
for vm_name in vms:
|
for vm_name in vms:
|
||||||
if type(vm_name) == int:
|
if type(vm_name) == int:
|
||||||
vm_name = self.libvirt_conn.lookupByID(vm_name).name()
|
vm_name = self.libvirt_conn.lookupByID(vm_name).name()
|
||||||
if vm_name.startswith(VMTask.CLONE_PREFIX):
|
if vm_name.startswith(VMExecTask.CLONE_PREFIX):
|
||||||
self.cleanupVM(vm_name)
|
self.cleanupVM(vm_name)
|
||||||
|
|
||||||
def cleanupExpiredVMs(self):
|
def cleanupExpiredVMs(self):
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ import optparse
|
||||||
|
|
||||||
# cli/koji -c ~/.koji/config-mead call --python makeTask '"vmExec"' '["Win2k8-x86-vstudio-devel", ["wget -q -O /tmp/test-build.sh http://download.lab.bos.redhat.com/devel/mikeb/mead/debug/test-build.sh && chmod 755 /tmp/test-build.sh && /tmp/test-build.sh &> /tmp/output/build.log && echo build successful"], {"cpus": 2, "mem": 2048}]' --kwargs '{"channel": "vm"}'
|
# cli/koji -c ~/.koji/config-mead call --python makeTask '"vmExec"' '["Win2k8-x86-vstudio-devel", ["wget -q -O /tmp/test-build.sh http://download.lab.bos.redhat.com/devel/mikeb/mead/debug/test-build.sh && chmod 755 /tmp/test-build.sh && /tmp/test-build.sh &> /tmp/output/build.log && echo build successful"], {"cpus": 2, "mem": 2048}]' --kwargs '{"channel": "vm"}'
|
||||||
|
|
||||||
parser = optparse.OptionParser('%prog VM-NAME COMMAND-TO-RUN')
|
parser = optparse.OptionParser('%prog VM-NAME SCM-URL TARGET')
|
||||||
parser.add_option('--server', help='Koji hub')
|
parser.add_option('--server', help='Koji hub')
|
||||||
parser.add_option('--cert', help='Client certificate')
|
parser.add_option('--cert', help='Client certificate')
|
||||||
parser.add_option('--ca', help='Client CA')
|
parser.add_option('--ca', help='Client CA')
|
||||||
|
|
@ -17,14 +17,15 @@ parser.add_option('--mem', help='Amount of memory (in megabytes) to allocate to
|
||||||
parser.add_option('--channel', help='Channel to create the task in', default='vm')
|
parser.add_option('--channel', help='Channel to create the task in', default='vm')
|
||||||
parser.add_option('--specfile', help='Alternate SCM URL of the specfile')
|
parser.add_option('--specfile', help='Alternate SCM URL of the specfile')
|
||||||
parser.add_option('--patches', help='SCM URL of patches to apply before build')
|
parser.add_option('--patches', help='SCM URL of patches to apply before build')
|
||||||
|
parser.add_option('--scratch', help='Run a scratch build', action='store_true')
|
||||||
opts, args = parser.parse_args()
|
opts, args = parser.parse_args()
|
||||||
|
|
||||||
if len(args) < 2:
|
if len(args) < 3:
|
||||||
parser.error('You must specify a VM name and SCM URL')
|
parser.error('You must specify a VM name, a SCM URL, and a build target')
|
||||||
|
|
||||||
vm_name = args[0]
|
vm_name = args[0]
|
||||||
scm_url = args[1]
|
scm_url = args[1]
|
||||||
|
target = args[2]
|
||||||
|
|
||||||
session = koji.ClientSession(opts.server)
|
session = koji.ClientSession(opts.server)
|
||||||
session.ssl_login(opts.cert, opts.ca, opts.server_ca)
|
session.ssl_login(opts.cert, opts.ca, opts.server_ca)
|
||||||
|
|
@ -38,9 +39,11 @@ if opts.specfile:
|
||||||
task_opts['specfile'] = opts.specfile
|
task_opts['specfile'] = opts.specfile
|
||||||
if opts.patches:
|
if opts.patches:
|
||||||
task_opts['patches'] = opts.patches
|
task_opts['patches'] = opts.patches
|
||||||
|
if opts.scratch:
|
||||||
|
task_opts['scratch'] = True
|
||||||
|
|
||||||
params = [vm_name, scm_url, task_opts]
|
params = [vm_name, scm_url, target, task_opts]
|
||||||
|
|
||||||
task_id = session.makeTask('vmExec', params, channel=opts.channel)
|
task_id = session.makeTask('winbuild', params, channel=opts.channel)
|
||||||
|
|
||||||
print 'Created task %s' % task_id
|
print 'Created task %s' % task_id
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue