particle-os/debian-koji
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
debian-koji/vm/kojikamid
Mike Bonnet 39f1c9d355 - support for additional filetypes from Windows builds 
- handle cr/lf weirdness with git in Windows
2010-07-22 18:16:10 -04:00

576 lines
21 KiB
Python
Executable file
Raw Blame History

#!/usr/bin/python
# Koji daemon that runs in a Windows VM and executes commands associated
# with a task.
# Copyright (c) 2010 Red Hat
#
# Koji is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation;
# version 2.1 of the License.
#
# This software is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this software; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#
# Authors:
# Mike Bonnet <mikeb@redhat.com>
# To register this script as a service on Windows 2008 (with Cygwin 1.7.5 installed) run:
# kojiwind --install
# in a cygwin shell.
import datetime
from optparse import OptionParser
from ConfigParser import ConfigParser
import os
import subprocess
import sys
import tempfile
import time
import urlparse
import xmlrpclib
import base64
import hashlib
import traceback
MANAGER_PORT = 7000
##############################
# Begin heinous copy and paste
##############################
class GenericError(Exception):
"""Base class for our custom exceptions"""
faultCode = 1000
fromFault = False
def __str__(self):
try:
return str(self.args[0]['args'][0])
except:
try:
return str(self.args[0])
except:
return str(self.__dict__)
class BuildError(GenericError):
"""Raised when a build fails"""
faultCode = 1005
class SCM(object):
"SCM abstraction class"
types = { 'CVS': ('cvs://',),
'CVS+SSH': ('cvs+ssh://',),
'GIT': ('git://', 'git+http://', 'git+https://', 'git+rsync://'),
'GIT+SSH': ('git+ssh://',),
'SVN': ('svn://', 'svn+http://', 'svn+https://'),
'SVN+SSH': ('svn+ssh://',) }
def is_scm_url(url):
"""
Return True if the url appears to be a valid, accessible source location, False otherwise
"""
for schemes in SCM.types.values():
for scheme in schemes:
if url.startswith(scheme):
return True
else:
return False
is_scm_url = staticmethod(is_scm_url)
def __init__(self, url):
"""
Initialize the SCM object using the specified url.
The expected url format is:
scheme://[user@]host/path/to/repo?path/to/module#revision_or_tag_identifier
The initialized SCM object will have the following attributes:
- url (the unmodified url)
- scheme
- user (may be null)
- host
- repository
- module
- revision
- scmtype
The exact format of each attribute is SCM-specific, but the structure of the url
must conform to the template above, or an error will be raised.
"""
if not SCM.is_scm_url(url):
raise GenericError, 'Invalid SCM URL: %s' % url
self.url = url
scheme, user, host, path, query, fragment = self._parse_url()
self.scheme = scheme
self.user = user
self.host = host
self.repository = path
self.module = query
self.revision = fragment
for scmtype, schemes in SCM.types.items():
if self.scheme in schemes:
self.scmtype = scmtype
break
else:
# should never happen
raise GenericError, 'Invalid SCM URL: %s' % url
def _parse_url(self):
"""
Parse the SCM url into usable components.
Return the following tuple:
(scheme, user, host, path, query, fragment)
user may be None, everything else will have a value
"""
# get the url's scheme
scheme = self.url.split('://')[0] + '://'
# replace the scheme with http:// so that the urlparse works in all cases
dummyurl = self.url.replace(scheme, 'http://', 1)
dummyscheme, netloc, path, params, query, fragment = urlparse.urlparse(dummyurl)
user = None
userhost = netloc.split('@')
if len(userhost) == 2:
user = userhost[0]
if not user:
# Don't return an empty string
user = None
elif ':' in user:
raise GenericError, 'username:password format not supported: %s' % user
netloc = userhost[1]
elif len(userhost) > 2:
raise GenericError, 'Invalid username@hostname specified: %s' % netloc
# ensure that path and query do not end in /
if path.endswith('/'):
path = path[:-1]
if query.endswith('/'):
query = query[:-1]
# check for validity: params should be empty, query may be empty, everything else should be populated
if params or not (scheme and netloc and path and fragment):
raise GenericError, 'Unable to parse SCM URL: %s' % self.url
# return parsed values
return (scheme, user, netloc, path, query, fragment)
def checkout(self, scmdir, logfile):
"""
Checkout the module from SCM. Accepts the following parameters:
- scmdir: the working directory
Returns the directory that the module was checked-out into (a subdirectory of scmdir)
"""
# TODO: sanity check arguments
sourcedir = '%s/%s' % (scmdir, self.module)
update_checkout_cmd = None
update_checkout_dir = None
if self.scmtype == 'CVS':
pserver = ':pserver:%s@%s:%s' % ((self.user or 'anonymous'), self.host, self.repository)
module_checkout_cmd = ['cvs', '-d', pserver, 'checkout', '-r', self.revision, self.module]
elif self.scmtype == 'CVS+SSH':
if not self.user:
raise BuildError, 'No user specified for repository access scheme: %s' % self.scheme
cvsserver = ':ext:%s@%s:%s' % (self.user, self.host, self.repository)
module_checkout_cmd = ['cvs', '-d', cvsserver, 'checkout', '-r', self.revision, self.module]
elif self.scmtype == 'GIT':
scheme = self.scheme
if '+' in scheme:
scheme = scheme.split('+')[1]
gitrepo = '%s%s%s' % (scheme, self.host, self.repository)
checkout_path = os.path.basename(self.repository)
if self.repository.endswith('/.git'):
checkout_path = os.path.basename(self.repository[:-5])
elif self.repository.endswith('.git'):
checkout_path = os.path.basename(self.repository[:-4])
sourcedir = '%s/%s' % (scmdir, checkout_path)
module_checkout_cmd = ['git', 'clone', '-n', gitrepo, sourcedir]
update_checkout_cmd = ['git', 'reset', '--hard', self.revision]
update_checkout_dir = sourcedir
# self.module may be empty, in which case the specfile should be in the top-level directory
if self.module:
# Treat the module as a directory inside the git repository
sourcedir = '%s/%s' % (sourcedir, self.module)
elif self.scmtype == 'GIT+SSH':
if not self.user:
raise BuildError, 'No user specified for repository access scheme: %s' % self.scheme
gitrepo = 'git+ssh://%s@%s%s' % (self.user, self.host, self.repository)
checkout_path = os.path.basename(self.repository)
if self.repository.endswith('/.git'):
checkout_path = os.path.basename(self.repository[:-5])
elif self.repository.endswith('.git'):
checkout_path = os.path.basename(self.repository[:-4])
sourcedir = '%s/%s' % (scmdir, checkout_path)
module_checkout_cmd = ['git', 'clone', '-n', gitrepo, sourcedir]
update_checkout_cmd = ['git', 'reset', '--hard', self.revision]
update_checkout_dir = sourcedir
# self.module may be empty, in which case the specfile should be in the top-level directory
if self.module:
# Treat the module as a directory inside the git repository
sourcedir = '%s/%s' % (sourcedir, self.module)
elif self.scmtype == 'SVN':
scheme = self.scheme
if '+' in scheme:
scheme = scheme.split('+')[1]
svnserver = '%s%s%s' % (scheme, self.host, self.repository)
module_checkout_cmd = ['svn', 'checkout', '-r', self.revision, '%s/%s' % (svnserver, self.module), self.module]
elif self.scmtype == 'SVN+SSH':
if not self.user:
raise BuildError, 'No user specified for repository access scheme: %s' % self.scheme
svnserver = 'svn+ssh://%s@%s%s' % (self.user, self.host, self.repository)
module_checkout_cmd = ['svn', 'checkout', '-r', self.revision, '%s/%s' % (svnserver, self.module), self.module]
else:
raise BuildError, 'Unknown SCM type: %s' % self.scmtype
# perform checkouts
ret, output = run(module_checkout_cmd, chdir=scmdir, logfile=logfile)
log(output)
if ret:
raise BuildError, 'Error running %s checkout command "%s: %s"' % \
(self.scmtype, ' '.join(module_checkout_cmd), output)
if update_checkout_cmd:
# Currently only required for GIT checkouts
# Run the command in the directory the source was checked out into
if self.scmtype.startswith('GIT'):
run(['git', 'config', 'core.autocrlf', 'true'], chdir=update_checkout_dir, logfile=logfile, fatal=True)
run(['git', 'config', 'core.safecrlf', 'true'], chdir=update_checkout_dir, logfile=logfile, fatal=True)
ret, output = run(update_checkout_cmd, chdir=update_checkout_dir, logfile=logfile)
log(output)
if ret:
raise BuildError, 'Error running %s update command "%s": %s' % \
(self.scmtype, ' '.join(update_checkout_cmd), output)
return sourcedir
############################
# End heinous copy and paste
############################
class WindowsBuild(object):
def __init__(self, specpath, build_tag, workdir, opts, logfile):
"""constructor: check ini spec file syntax, set build properties"""
self.build_tag = build_tag
self.opts = opts
self.workdir = workdir
self.logfile = logfile
conf = ConfigParser()
conf.read(specpath)
# [naming] section
for entry in ('name', 'version', 'release', 'description'):
setattr(self, entry, conf.get('naming', entry))
# [building] section
self.platform = conf.get('building', 'platform')
# buildrequires and provides are multi-valued (space-separated)
for entry in ('buildrequires', 'provides'):
setattr(self, entry, [e for e in conf.get('building', entry).split() if e])
# execute is multi-valued (newline-separated)
self.execute = [e.strip() for e in conf.get('building', 'execute').split('\n') if e]
# [files] section
self.output = {}
for entry in conf.get('files', 'output').split('\n'):
if not entry:
continue
tokens = entry.strip().split(':')
filename = tokens[0]
metadata = {}
metadata['platforms'] = tokens[1].split(',')
if len(tokens) > 2:
metadata['flags'] = tokens[2].split(',')
else:
metadata['flags'] = []
self.output[filename] = metadata
self.logs = [e.strip() for e in conf.get('files', 'logs').split('\n') if e]
def checkEnv(self):
"""Is this environment fit to build in, based on the spec file?"""
pass
def fetchBuildReqs(self):
"""Retrieve buildrequires listed in the spec file"""
pass
def build(self):
"""Do the build: run the execute line(s)"""
tmpfd, tmpname = tempfile.mkstemp(prefix='koji-tmp.', dir='/tmp')
script = os.fdopen(tmpfd, 'w')
for cmd in self.execute:
script.write(cmd)
script.write('\n')
script.close()
cmd = ['/bin/bash', '-e', '-x', tmpname]
ret, output = run(cmd, chdir=self.workdir, logfile=self.logfile)
if ret:
raise BuildError, 'Build command failed, see %s for details' % \
os.path.basename(self.logfile.name)
def virusCheck(self):
"""check the build output for viruses"""
pass
def gatherResults(self):
"""gather information about the output from the build, return it"""
return {'name': self.name, 'version': self.version, 'release': self.release,
'description': self.description, 'platform': self.platform,
'provides': self.provides,
'output': self.output, 'logs': self.logs}
def doAll(self):
"""helper function that runs the entire process"""
self.checkEnv()
self.fetchBuildReqs()
self.build()
self.virusCheck()
return self.gatherResults()
def log(msg):
print >> sys.stderr, '%s: %s' % (datetime.datetime.now().ctime(), msg)
def run(cmd, chdir=None, logfile=None, fatal=False):
shell = False
if isinstance(cmd, (str, unicode)) and len(cmd.split()) > 1:
shell = True
olddir = None
if chdir:
olddir = os.getcwd()
os.chdir(chdir)
log('running command: %s' % cmd)
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
close_fds=True, shell=shell)
ret = proc.wait()
output = proc.stdout.read()
if logfile:
logfile.write('$ ')
if isinstance(cmd, (str, unicode)):
logfile.write(cmd)
else:
logfile.write(' '.join(cmd))
logfile.write('\n')
logfile.write(output)
logfile.write('\n')
if olddir:
os.chdir(olddir)
if ret and fatal:
raise BuildError, 'error running: %s, return code was %s, output was: %s' % (cmd, ret, output)
return ret, output
def find_net_info():
"""
Find the network gateway configured for this VM.
"""
ret, output = run(['ipconfig', '/all'])
if ret:
raise RuntimeError, 'error running ipconfig, output was: %s' % output
macaddr = None
gateway = None
for line in output.splitlines():
line = line.strip()
# take the first values we find
if line.startswith('Physical Address'):
if not macaddr:
macaddr = line.split()[-1]
# format it to be consistent with the libvirt MAC address
macaddr = macaddr.replace('-', ':').lower()
elif line.startswith('Default Gateway'):
if not gateway:
gateway = line.split()[-1]
# check that we have valid values
if macaddr and len(macaddr) != 17:
macaddr = None
if gateway and (len(gateway) < 7 or len(gateway) > 15):
gateway = None
return macaddr, gateway
def upload_file(server, prefix, path):
"""upload a single file to the vmd"""
fobj = file(os.path.join(prefix, path), 'r')
offset = 0
sum = hashlib.sha1()
while True:
data = fobj.read(131072)
if not data:
break
encoded = base64.b64encode(data)
server.upload(path, offset, encoded)
offset += len(data)
sum.update(data)
fobj.close()
server.verifyChecksum(path, sum.hexdigest(), 'sha1')
def upload_results(server, codir, results):
"""upload the results of a build given the results dict"""
for filename in results['output'].keys() + results['logs']:
upload_file(server, codir, filename)
def get_mgmt_server():
"""retrieve scmurls from kojivmd we'll use to build from"""
macaddr, gateway = find_net_info()
while not (macaddr and gateway):
# wait for the network connection to come up and get an address
time.sleep(5)
macaddr, gateway = find_net_info()
log('found MAC address %s, connecting to %s:%s' %
(macaddr, gateway, MANAGER_PORT))
server = xmlrpclib.ServerProxy('http://%s:%s/' %
(gateway, MANAGER_PORT), allow_none=True)
# we would set a timeout on the socket here, but that is apparently not
# supported by python/cygwin/Windows
task_port = server.getPort(macaddr)
log('found task-specific port %s' % task_port)
return xmlrpclib.ServerProxy('http://%s:%s/' % (gateway, task_port), allow_none=True)
def get_options():
"""handle usage and parse options"""
usage = """%prog [options]
Run Koji tasks assigned to a VM.
Run without any arguments to start this daemon.
"""
parser = OptionParser(usage=usage)
parser.add_option('-i', '--install', action='store_true', help='Install this daemon as a service', default=False)
parser.add_option('-u', '--uninstall', action='store_true', help='Uninstall this daemon if it was installed previously as a service', default=False)
parser.add_option('-s', '--scmurl', help='Forcibly specify an scmurl to checkout from')
(options, args) = parser.parse_args()
return options
def apply_patches(src_dir, patch_dir, logfile):
patches = [patch for patch in os.listdir(patch_dir) if \
os.path.isfile(os.path.join(patch_dir, patch)) and \
not patch.startswith('.')]
if not patches:
raise BuildError, 'no patches found at %s' % patch_dir
patches.sort()
for patch in patches:
cmd = ['/usr/bin/patch', '--verbose', '-d', src_dir, '-p1', '-i', os.path.join(patch_dir, patch)]
ret, output = run(cmd, logfile=logfile)
if ret:
raise BuildError, 'error applying patches, output was: %s' % output
def ensuredir(path):
if not os.path.isdir(path):
os.makedirs(path)
return path
def run_build(workdir, source_url, build_tag, task_opts, logfile):
"""run the build"""
src_scm = SCM(source_url)
src_dir = src_scm.checkout(ensuredir(os.path.join(workdir, 'source')), logfile)
if 'specfile' in task_opts:
spec_scm = SCM(task_opts['specfile'])
spec_dir = spec_scm.checkout(ensuredir(os.path.join(workdir, 'spec')), logfile)
else:
spec_dir = src_dir
if 'patches' in task_opts:
patch_scm = SCM(task_opts['patches'])
patch_dir = patch_scm.checkout(ensuredir(os.path.join(workdir, 'patches')), logfile)
apply_patches(src_dir, patch_dir, logfile)
specfile = [spec for spec in os.listdir(spec_dir) if spec.endswith('.ini')]
if len(specfile) == 0:
raise BuildError, 'No .ini file found'
elif len(specfile) > 1:
raise BuildError, 'Multiple .ini files found'
winbld = WindowsBuild(os.path.join(spec_dir, specfile[0]), build_tag, src_dir, task_opts, logfile)
return winbld.doAll(), src_dir
def flunk(server, logfile):
"""do the right thing when a build fails"""
exc_info = sys.exc_info()
tb = ''.join(traceback.format_exception(*exc_info))
if server is not None:
if logfile is not None:
try:
logfile.close()
upload_file(server, os.path.dirname(logfile.name),
os.path.basename(logfile.name))
except:
pass
server.failTask(tb)
log(tb)
sys.exit(1)
if __name__ == '__main__':
prog = os.path.basename(sys.argv[0])
opts = get_options()
if opts.install:
ret, output = run(['cygrunsrv', '--install', prog,
'--path', sys.executable, '--args', os.path.abspath(prog),
'--type', 'auto', '--dep', 'Dhcp',
'--disp', 'Koji Windows Daemon',
'--desc', 'Runs Koji tasks assigned to a VM'])
if ret:
print 'Error installing %s service, output was: %s' % (prog, output)
sys.exit(1)
else:
print 'Successfully installed the %s service' % prog
sys.exit(0)
elif opts.uninstall:
ret, output = run(['cygrunsrv', '--remove', prog])
if ret:
print 'Error removing the %s service, output was: %s' % (prog, output)
sys.exit(1)
else:
print 'Successfully removed the %s service' % prog
sys.exit(0)
server = None
logfile = None
try:
source_url = None
task_opts = None
if opts.scmurl:
source_url = opts.scmurl
else:
server = get_mgmt_server()
info = server.getTaskInfo()
source_url = info[0]
build_tag = info[1]
if len(info) > 2:
task_opts = info[2]
if not task_opts:
task_opts = {}
workdir = '/tmp/workdir'
os.mkdir(workdir)
logfile = file(os.path.join(workdir, 'build.log'), 'w')
results, results_dir = run_build(workdir, source_url, build_tag, task_opts, logfile)
logfile.close()
if server is not None:
upload_file(server, workdir, 'build.log')
upload_results(server, results_dir, results)
results['logs'].append('build.log')
server.closeTask(results)
log('Build results: %s' % results)
except:
flunk(server, logfile)
sys.exit(0)
Reference in a new issue View git blame Copy permalink