particle-os/debian-koji
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
debian-koji/vm/kojikamid
Jay Greguske e73cab0940 new revision of kojikamid
2010-07-15 16:47:53 -04:00

500 lines
18 KiB
Python
Executable file
Raw Blame History

#!/usr/bin/python
# Koji daemon that runs in a Windows VM and executes commands associated
# with a task.
# Copyright (c) 2010 Red Hat
#
# Koji is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation;
# version 2.1 of the License.
#
# This software is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this software; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#
# Authors:
# Mike Bonnet <mikeb@redhat.com>
# To register this script as a service on Windows 2008 (with Cygwin 1.7.5 installed) run:
# kojiwind --install
# in a cygwin shell.
import datetime
from optparse import OptionParser
from ConfigParser import ConfigParser
import os
import subprocess
import sys
import time
import urlparse
import xmlrpclib
import base64
import hashlib
import traceback
MANAGER_PORT = 7000
##############################
# Begin heinous copy and paste
##############################
class GenericError(Exception):
"""Base class for our custom exceptions"""
faultCode = 1000
fromFault = False
def __str__(self):
try:
return str(self.args[0]['args'][0])
except:
try:
return str(self.args[0])
except:
return str(self.__dict__)
class BuildError(GenericError):
"""Raised when a build fails"""
faultCode = 1005
class SCM(object):
"SCM abstraction class"
types = { 'CVS': ('cvs://',),
'CVS+SSH': ('cvs+ssh://',),
'GIT': ('git://', 'git+http://', 'git+https://', 'git+rsync://'),
'GIT+SSH': ('git+ssh://',),
'SVN': ('svn://', 'svn+http://', 'svn+https://'),
'SVN+SSH': ('svn+ssh://',) }
def is_scm_url(url):
"""
Return True if the url appears to be a valid, accessible source location, False otherwise
"""
for schemes in SCM.types.values():
for scheme in schemes:
if url.startswith(scheme):
return True
else:
return False
is_scm_url = staticmethod(is_scm_url)
def __init__(self, url):
"""
Initialize the SCM object using the specified url.
The expected url format is:
scheme://[user@]host/path/to/repo?path/to/module#revision_or_tag_identifier
The initialized SCM object will have the following attributes:
- url (the unmodified url)
- scheme
- user (may be null)
- host
- repository
- module
- revision
- scmtype
The exact format of each attribute is SCM-specific, but the structure of the url
must conform to the template above, or an error will be raised.
"""
if not SCM.is_scm_url(url):
raise GenericError, 'Invalid SCM URL: %s' % url
self.url = url
scheme, user, host, path, query, fragment = self._parse_url()
self.scheme = scheme
self.user = user
self.host = host
self.repository = path
self.module = query
self.revision = fragment
for scmtype, schemes in SCM.types.items():
if self.scheme in schemes:
self.scmtype = scmtype
break
else:
# should never happen
raise GenericError, 'Invalid SCM URL: %s' % url
def _parse_url(self):
"""
Parse the SCM url into usable components.
Return the following tuple:
(scheme, user, host, path, query, fragment)
user may be None, everything else will have a value
"""
# get the url's scheme
scheme = self.url.split('://')[0] + '://'
# replace the scheme with http:// so that the urlparse works in all cases
dummyurl = self.url.replace(scheme, 'http://', 1)
dummyscheme, netloc, path, params, query, fragment = urlparse.urlparse(dummyurl)
user = None
userhost = netloc.split('@')
if len(userhost) == 2:
user = userhost[0]
if not user:
# Don't return an empty string
user = None
elif ':' in user:
raise GenericError, 'username:password format not supported: %s' % user
netloc = userhost[1]
elif len(userhost) > 2:
raise GenericError, 'Invalid username@hostname specified: %s' % netloc
# ensure that path and query do not end in /
if path.endswith('/'):
path = path[:-1]
if query.endswith('/'):
query = query[:-1]
# check for validity: params should be empty, query may be empty, everything else should be populated
if params or not (scheme and netloc and path and fragment):
raise GenericError, 'Unable to parse SCM URL: %s' % self.url
# return parsed values
return (scheme, user, netloc, path, query, fragment)
def checkout(self, scmdir):
"""
Checkout the module from SCM. Accepts the following parameters:
- scmdir: the working directory
Returns the directory that the module was checked-out into (a subdirectory of scmdir)
"""
# TODO: sanity check arguments
sourcedir = '%s/%s' % (scmdir, self.module)
update_checkout_cmd = None
if self.scmtype == 'CVS':
pserver = ':pserver:%s@%s:%s' % ((self.user or 'anonymous'), self.host, self.repository)
module_checkout_cmd = ['cvs', '-d', pserver, 'checkout', '-r', self.revision, self.module]
elif self.scmtype == 'CVS+SSH':
if not self.user:
raise BuildError, 'No user specified for repository access scheme: %s' % self.scheme
cvsserver = ':ext:%s@%s:%s' % (self.user, self.host, self.repository)
module_checkout_cmd = ['cvs', '-d', cvsserver, 'checkout', '-r', self.revision, self.module]
elif self.scmtype == 'GIT':
scheme = self.scheme
if '+' in scheme:
scheme = scheme.split('+')[1]
gitrepo = '%s%s%s' % (scheme, self.host, self.repository)
checkout_path = os.path.basename(self.repository)
if self.repository.endswith('/.git'):
checkout_path = os.path.basename(self.repository[:-5])
elif self.repository.endswith('.git'):
checkout_path = os.path.basename(self.repository[:-4])
sourcedir = '%s/%s' % (scmdir, checkout_path)
module_checkout_cmd = ['git', 'clone', '-n', gitrepo, sourcedir]
update_checkout_cmd = ['git', 'reset', '--hard', self.revision]
# self.module may be empty, in which case the specfile should be in the top-level directory
if self.module:
# Treat the module as a directory inside the git repository
sourcedir = '%s/%s' % (sourcedir, self.module)
elif self.scmtype == 'GIT+SSH':
if not self.user:
raise BuildError, 'No user specified for repository access scheme: %s' % self.scheme
gitrepo = 'git+ssh://%s@%s%s' % (self.user, self.host, self.repository)
checkout_path = os.path.basename(self.repository)
if self.repository.endswith('/.git'):
checkout_path = os.path.basename(self.repository[:-5])
elif self.repository.endswith('.git'):
checkout_path = os.path.basename(self.repository[:-4])
sourcedir = '%s/%s' % (scmdir, checkout_path)
module_checkout_cmd = ['git', 'clone', '-n', gitrepo, sourcedir]
update_checkout_cmd = ['git', 'reset', '--hard', self.revision]
# self.module may be empty, in which case the specfile should be in the top-level directory
if self.module:
# Treat the module as a directory inside the git repository
sourcedir = '%s/%s' % (sourcedir, self.module)
elif self.scmtype == 'SVN':
scheme = self.scheme
if '+' in scheme:
scheme = scheme.split('+')[1]
svnserver = '%s%s%s' % (scheme, self.host, self.repository)
module_checkout_cmd = ['svn', 'checkout', '-r', self.revision, '%s/%s' % (svnserver, self.module), self.module]
elif self.scmtype == 'SVN+SSH':
if not self.user:
raise BuildError, 'No user specified for repository access scheme: %s' % self.scheme
svnserver = 'svn+ssh://%s@%s%s' % (self.user, self.host, self.repository)
module_checkout_cmd = ['svn', 'checkout', '-r', self.revision, '%s/%s' % (svnserver, self.module), self.module]
else:
raise BuildError, 'Unknown SCM type: %s' % self.scmtype
# perform checkouts
ret, output = run(module_checkout_cmd)
log(output)
if ret:
raise BuildError, 'Error running %s checkout command "%s: %s"' % \
(self.scmtype, ' '.join(module_checkout_cmd), output)
if update_checkout_cmd:
# Currently only required for GIT checkouts
# Run the command in the directory the source was checked out into
ret, output = run(update_checkout_cmd, chdir=sourcedir)
log(output)
if ret:
raise BuildError, 'Error running %s update command "%s": %s' % \
(self.scmtype, ' '.join(update_checkout_cmd), output)
return sourcedir
############################
# End heinous copy and paste
############################
class WindowsBuild(object):
def __init__(self, specpath, workdir):
"""constructor: check ini spec file syntax, set build properties"""
buildconf = ConfigParser()
buildconf.read(specpath)
self.results = None
# make sure we've got the right sections defined
goodsections = ('naming', 'building', 'files')
badsections = [s for s in buildconf.sections() if s not in goodsections]
if len(badsections) > 0:
raise BuildError, 'Unrecognized section(s) in ini: %s' % badsections
for section in goodsections:
if not buildconf.has_section(section):
raise BuildError, 'missing required section in ini: %s' % section
map(self.__dict__.update, (buildconf.items('naming'),))
for section in ('building', 'files'):
for name, value in buildconf.items(section):
value = value.split()
self.__setattr__(name, value)
self.workdir = workdir
def checkEnv(self):
"""Is this environment fit to build in, based on the spec file?"""
pass
def build(self):
"""Do the build: run the execute line(s)"""
for cmd in self.execute:
ret, output = run(os.path.join(self.workdir, cmd), chdir=self.workdir)
if ret:
raise BuildError, 'Build command failed: %s' % output
def virusCheck(self):
"""check the build output for viruses"""
pass
def gatherOutput(self):
"""gather information about the output from the build, return it"""
if self.results != None: return self.results
# TODO: VM platform value in hash?
self.results = {}
for ftype in ('output', 'debug', 'log'):
files = getattr(self, ftype + 'files')
fdict = {}
if files:
for f in files:
if not os.path.exists(os.path.join(self.workdir, f)):
raise BuildError('Could not find %s after build' % f)
fdict[f] = {'platform': self.platform, 'debug': False, 'flags': ''}
self.results[ftype] = fdict
return self.results
def doAll(self):
"""helper function that runs the entire process"""
self.checkEnv()
self.build()
self.virusCheck()
return self.gatherOutput()
def log(msg):
print >> sys.stderr, '%s: %s' % (datetime.datetime.now().ctime(), msg)
def run(cmd, chdir='.'):
shell = False
if isinstance(cmd, (str, unicode)) and len(cmd.split()) > 1:
shell = True
olddir = os.getcwd()
os.chdir(chdir)
log('running command: %s' % cmd)
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
close_fds=True, shell=shell)
ret = proc.wait()
output = proc.stdout.read()
os.chdir(olddir)
return ret, output
def find_net_info():
"""
Find the network gateway configured for this VM.
"""
ret, output = run(['ipconfig', '/all'])
if ret:
raise RuntimeError, 'error running ipconfig, output was: %s' % output
macaddr = None
gateway = None
for line in output.splitlines():
line = line.strip()
# take the first values we find
if line.startswith('Physical Address'):
if not macaddr:
macaddr = line.split()[-1]
# format it to be consistent with the libvirt MAC address
macaddr = macaddr.replace('-', ':').lower()
elif line.startswith('Default Gateway'):
if not gateway:
gateway = line.split()[-1]
# check that we have valid values
if macaddr and len(macaddr) != 17:
macaddr = None
if gateway and (len(gateway) < 7 or len(gateway) > 15):
gateway = None
return macaddr, gateway
def upload_file(server, prefix, path):
"""upload a single file to the vmd"""
fobj = file(os.path.join(prefix, path), 'r')
offset = 0
sum = hashlib.sha1()
while True:
data = fobj.read(131072)
if not data:
break
encoded = base64.b64encode(data)
server.upload(path, offset, encoded)
offset += len(data)
sum.update(data)
fobj.close()
server.verifyChecksum(path, sum.hexdigest(), 'sha1')
def upload_results(server, codir, results):
"""upload the results of a build given the results dict"""
for output_type in results.keys():
for output_file in results[output_type].keys():
upload_file(server, codir, output_file)
def get_mgmt_server():
"""retrieve scmurls from kojivmd we'll use to build from"""
macaddr, gateway = find_net_info()
while not (macaddr and gateway):
# wait for the network connection to come up and get an address
time.sleep(5)
macaddr, gateway = find_net_info()
log('found MAC address %s, connecting to %s:%s' %
(macaddr, gateway, MANAGER_PORT))
server = xmlrpclib.ServerProxy('http://%s:%s/' %
(gateway, MANAGER_PORT), allow_none=True)
# we would set a timeout on the socket here, but that is apparently not
# supported by python/cygwin/Windows
task_port = server.getPort(macaddr)
log('found task-specific port %s' % task_port)
return xmlrpclib.ServerProxy('http://%s:%s/' % (gateway, task_port), allow_none=True)
def get_options():
"""handle usage and parse options"""
usage = """%prog [options]
Run Koji tasks assigned to a VM.
Run without any arguments to start this daemon.
"""
parser = OptionParser(usage=usage)
parser.add_option('-i', '--install', action='store_true', help='Install this daemon as a service', default=False)
parser.add_option('-u', '--uninstall', action='store_true', help='Uninstall this daemon if it was installed previously as a service', default=False)
parser.add_option('-s', '--scmurl', action='append', help='Forcibly specify an scmurl to checkout from', default=[])
(options, args) = parser.parse_args()
return options
def run_build(workdir, scmurls):
"""run the build"""
if len(scmurls) == 2:
# should SCM.assert_allowed() be called on SCM objs?
specurl, srcurl = scmurls
spec_scm = SCM(specurl)
spec_dir = spec_scm.checkout(workdir)
src_scm = SCM(srcurl)
src_dir = src_scm.checkout(workdir)
elif len(scmurls) == 1:
specurl = scmurls[0]
spec_scm = SCM(specurl)
spec_dir = spec_scm.checkout(workdir)
src_scm, src_dir = spec_scm, spec_dir
else:
raise BuildError, 'Unexpected number of SCM URLs given: %s' % scmurls
specfile = [spec for spec in os.listdir(spec_dir) if spec.endswith('.ini')]
if len(specfile) != 1:
raise BuildError, 'Exactly one .ini file must be found in a check out'
winbld = WindowsBuild(os.path.join(spec_dir, specfile[0]), src_dir)
return winbld.doAll(), src_dir
def flunk(server_report, server=None):
"""do the right thing when a build fails"""
exc_info = sys.exc_info()
tb = ''.join(traceback.format_exception(*exc_info))
if server_report:
server.failTask(tb)
log(tb)
sys.exit(1)
if __name__ == '__main__':
prog = os.path.basename(sys.argv[0])
opts = get_options()
if opts.install:
ret, output = run(['cygrunsrv', '--install', prog,
'--path', sys.executable, '--args', os.path.abspath(prog),
'--type', 'auto', '--dep', 'Dhcp',
'--disp', 'Koji Windows Daemon',
'--desc', 'Runs Koji tasks assigned to a VM'])
if ret:
print 'Error installing %s service, output was: %s' % (prog, output)
sys.exit(1)
else:
print 'Successfully installed the %s service' % prog
sys.exit(0)
elif opts.uninstall:
ret, output = run(['cygrunsrv', '--remove', prog])
if ret:
print 'Error removing the %s service, output was: %s' % (prog, output)
sys.exit(1)
else:
print 'Successfully removed the %s service' % prog
sys.exit(0)
use_server, server = False, None
try:
if len(opts.scmurl) == 0:
use_server = True
server = get_mgmt_server()
scmurl = server.getTaskInfo()
opts.scmurl.append(scmurl)
workdir = '/tmp/workdir'
os.mkdir(workdir)
results, results_dir = run_build(workdir, opts.scmurl)
if use_server:
upload_results(server, results_dir, results)
server.closeTask(results)
log('Build results: %s' % results)
except:
flunk(use_server, server=server)
sys.exit(0)
Reference in a new issue View git blame Copy permalink