new revision of kojikamid
This commit is contained in:
Jay Greguske
parent
64cc01be89
commit
e73cab0940
1 changed files with 393 additions and 70 deletions
463
vm/kojikamid
463
vm/kojikamid
|
|
@ -26,10 +26,13 @@
|
|||
# in a cygwin shell.
|
||||
|
||||
import datetime
|
||||
from optparse import OptionParser
|
||||
from ConfigParser import ConfigParser
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urlparse
|
||||
import xmlrpclib
|
||||
import base64
|
||||
import hashlib
|
||||
|
|
@ -37,17 +40,310 @@ import traceback
|
|||
|
||||
MANAGER_PORT = 7000
|
||||
|
||||
##############################
|
||||
# Begin heinous copy and paste
|
||||
##############################
|
||||
|
||||
class GenericError(Exception):
|
||||
"""Base class for our custom exceptions"""
|
||||
faultCode = 1000
|
||||
fromFault = False
|
||||
def __str__(self):
|
||||
try:
|
||||
return str(self.args[0]['args'][0])
|
||||
except:
|
||||
try:
|
||||
return str(self.args[0])
|
||||
except:
|
||||
return str(self.__dict__)
|
||||
|
||||
class BuildError(GenericError):
|
||||
"""Raised when a build fails"""
|
||||
faultCode = 1005
|
||||
|
||||
class SCM(object):
|
||||
"SCM abstraction class"
|
||||
|
||||
types = { 'CVS': ('cvs://',),
|
||||
'CVS+SSH': ('cvs+ssh://',),
|
||||
'GIT': ('git://', 'git+http://', 'git+https://', 'git+rsync://'),
|
||||
'GIT+SSH': ('git+ssh://',),
|
||||
'SVN': ('svn://', 'svn+http://', 'svn+https://'),
|
||||
'SVN+SSH': ('svn+ssh://',) }
|
||||
|
||||
def is_scm_url(url):
|
||||
"""
|
||||
Return True if the url appears to be a valid, accessible source location, False otherwise
|
||||
"""
|
||||
for schemes in SCM.types.values():
|
||||
for scheme in schemes:
|
||||
if url.startswith(scheme):
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
is_scm_url = staticmethod(is_scm_url)
|
||||
|
||||
def __init__(self, url):
|
||||
"""
|
||||
Initialize the SCM object using the specified url.
|
||||
The expected url format is:
|
||||
|
||||
scheme://[user@]host/path/to/repo?path/to/module#revision_or_tag_identifier
|
||||
|
||||
The initialized SCM object will have the following attributes:
|
||||
- url (the unmodified url)
|
||||
- scheme
|
||||
- user (may be null)
|
||||
- host
|
||||
- repository
|
||||
- module
|
||||
- revision
|
||||
- scmtype
|
||||
|
||||
The exact format of each attribute is SCM-specific, but the structure of the url
|
||||
must conform to the template above, or an error will be raised.
|
||||
"""
|
||||
if not SCM.is_scm_url(url):
|
||||
raise GenericError, 'Invalid SCM URL: %s' % url
|
||||
|
||||
self.url = url
|
||||
scheme, user, host, path, query, fragment = self._parse_url()
|
||||
|
||||
self.scheme = scheme
|
||||
self.user = user
|
||||
self.host = host
|
||||
self.repository = path
|
||||
self.module = query
|
||||
self.revision = fragment
|
||||
|
||||
for scmtype, schemes in SCM.types.items():
|
||||
if self.scheme in schemes:
|
||||
self.scmtype = scmtype
|
||||
break
|
||||
else:
|
||||
# should never happen
|
||||
raise GenericError, 'Invalid SCM URL: %s' % url
|
||||
|
||||
def _parse_url(self):
|
||||
"""
|
||||
Parse the SCM url into usable components.
|
||||
Return the following tuple:
|
||||
|
||||
(scheme, user, host, path, query, fragment)
|
||||
|
||||
user may be None, everything else will have a value
|
||||
"""
|
||||
# get the url's scheme
|
||||
scheme = self.url.split('://')[0] + '://'
|
||||
|
||||
# replace the scheme with http:// so that the urlparse works in all cases
|
||||
dummyurl = self.url.replace(scheme, 'http://', 1)
|
||||
dummyscheme, netloc, path, params, query, fragment = urlparse.urlparse(dummyurl)
|
||||
|
||||
user = None
|
||||
userhost = netloc.split('@')
|
||||
if len(userhost) == 2:
|
||||
user = userhost[0]
|
||||
if not user:
|
||||
# Don't return an empty string
|
||||
user = None
|
||||
elif ':' in user:
|
||||
raise GenericError, 'username:password format not supported: %s' % user
|
||||
netloc = userhost[1]
|
||||
elif len(userhost) > 2:
|
||||
raise GenericError, 'Invalid username@hostname specified: %s' % netloc
|
||||
|
||||
# ensure that path and query do not end in /
|
||||
if path.endswith('/'):
|
||||
path = path[:-1]
|
||||
if query.endswith('/'):
|
||||
query = query[:-1]
|
||||
|
||||
# check for validity: params should be empty, query may be empty, everything else should be populated
|
||||
if params or not (scheme and netloc and path and fragment):
|
||||
raise GenericError, 'Unable to parse SCM URL: %s' % self.url
|
||||
|
||||
# return parsed values
|
||||
return (scheme, user, netloc, path, query, fragment)
|
||||
|
||||
def checkout(self, scmdir):
|
||||
"""
|
||||
Checkout the module from SCM. Accepts the following parameters:
|
||||
- scmdir: the working directory
|
||||
|
||||
Returns the directory that the module was checked-out into (a subdirectory of scmdir)
|
||||
"""
|
||||
# TODO: sanity check arguments
|
||||
sourcedir = '%s/%s' % (scmdir, self.module)
|
||||
|
||||
update_checkout_cmd = None
|
||||
|
||||
if self.scmtype == 'CVS':
|
||||
pserver = ':pserver:%s@%s:%s' % ((self.user or 'anonymous'), self.host, self.repository)
|
||||
module_checkout_cmd = ['cvs', '-d', pserver, 'checkout', '-r', self.revision, self.module]
|
||||
|
||||
elif self.scmtype == 'CVS+SSH':
|
||||
if not self.user:
|
||||
raise BuildError, 'No user specified for repository access scheme: %s' % self.scheme
|
||||
|
||||
cvsserver = ':ext:%s@%s:%s' % (self.user, self.host, self.repository)
|
||||
module_checkout_cmd = ['cvs', '-d', cvsserver, 'checkout', '-r', self.revision, self.module]
|
||||
|
||||
elif self.scmtype == 'GIT':
|
||||
scheme = self.scheme
|
||||
if '+' in scheme:
|
||||
scheme = scheme.split('+')[1]
|
||||
gitrepo = '%s%s%s' % (scheme, self.host, self.repository)
|
||||
checkout_path = os.path.basename(self.repository)
|
||||
if self.repository.endswith('/.git'):
|
||||
checkout_path = os.path.basename(self.repository[:-5])
|
||||
elif self.repository.endswith('.git'):
|
||||
checkout_path = os.path.basename(self.repository[:-4])
|
||||
|
||||
sourcedir = '%s/%s' % (scmdir, checkout_path)
|
||||
module_checkout_cmd = ['git', 'clone', '-n', gitrepo, sourcedir]
|
||||
update_checkout_cmd = ['git', 'reset', '--hard', self.revision]
|
||||
|
||||
# self.module may be empty, in which case the specfile should be in the top-level directory
|
||||
if self.module:
|
||||
# Treat the module as a directory inside the git repository
|
||||
sourcedir = '%s/%s' % (sourcedir, self.module)
|
||||
|
||||
elif self.scmtype == 'GIT+SSH':
|
||||
if not self.user:
|
||||
raise BuildError, 'No user specified for repository access scheme: %s' % self.scheme
|
||||
gitrepo = 'git+ssh://%s@%s%s' % (self.user, self.host, self.repository)
|
||||
checkout_path = os.path.basename(self.repository)
|
||||
if self.repository.endswith('/.git'):
|
||||
checkout_path = os.path.basename(self.repository[:-5])
|
||||
elif self.repository.endswith('.git'):
|
||||
checkout_path = os.path.basename(self.repository[:-4])
|
||||
|
||||
sourcedir = '%s/%s' % (scmdir, checkout_path)
|
||||
module_checkout_cmd = ['git', 'clone', '-n', gitrepo, sourcedir]
|
||||
update_checkout_cmd = ['git', 'reset', '--hard', self.revision]
|
||||
|
||||
# self.module may be empty, in which case the specfile should be in the top-level directory
|
||||
if self.module:
|
||||
# Treat the module as a directory inside the git repository
|
||||
sourcedir = '%s/%s' % (sourcedir, self.module)
|
||||
|
||||
elif self.scmtype == 'SVN':
|
||||
scheme = self.scheme
|
||||
if '+' in scheme:
|
||||
scheme = scheme.split('+')[1]
|
||||
|
||||
svnserver = '%s%s%s' % (scheme, self.host, self.repository)
|
||||
module_checkout_cmd = ['svn', 'checkout', '-r', self.revision, '%s/%s' % (svnserver, self.module), self.module]
|
||||
|
||||
elif self.scmtype == 'SVN+SSH':
|
||||
if not self.user:
|
||||
raise BuildError, 'No user specified for repository access scheme: %s' % self.scheme
|
||||
|
||||
svnserver = 'svn+ssh://%s@%s%s' % (self.user, self.host, self.repository)
|
||||
module_checkout_cmd = ['svn', 'checkout', '-r', self.revision, '%s/%s' % (svnserver, self.module), self.module]
|
||||
|
||||
else:
|
||||
raise BuildError, 'Unknown SCM type: %s' % self.scmtype
|
||||
|
||||
# perform checkouts
|
||||
ret, output = run(module_checkout_cmd)
|
||||
log(output)
|
||||
if ret:
|
||||
raise BuildError, 'Error running %s checkout command "%s: %s"' % \
|
||||
(self.scmtype, ' '.join(module_checkout_cmd), output)
|
||||
|
||||
if update_checkout_cmd:
|
||||
# Currently only required for GIT checkouts
|
||||
# Run the command in the directory the source was checked out into
|
||||
ret, output = run(update_checkout_cmd, chdir=sourcedir)
|
||||
log(output)
|
||||
if ret:
|
||||
raise BuildError, 'Error running %s update command "%s": %s' % \
|
||||
(self.scmtype, ' '.join(update_checkout_cmd), output)
|
||||
|
||||
return sourcedir
|
||||
|
||||
############################
|
||||
# End heinous copy and paste
|
||||
############################
|
||||
|
||||
class WindowsBuild(object):
|
||||
|
||||
def __init__(self, specpath, workdir):
|
||||
"""constructor: check ini spec file syntax, set build properties"""
|
||||
buildconf = ConfigParser()
|
||||
buildconf.read(specpath)
|
||||
self.results = None
|
||||
|
||||
# make sure we've got the right sections defined
|
||||
goodsections = ('naming', 'building', 'files')
|
||||
badsections = [s for s in buildconf.sections() if s not in goodsections]
|
||||
if len(badsections) > 0:
|
||||
raise BuildError, 'Unrecognized section(s) in ini: %s' % badsections
|
||||
for section in goodsections:
|
||||
if not buildconf.has_section(section):
|
||||
raise BuildError, 'missing required section in ini: %s' % section
|
||||
map(self.__dict__.update, (buildconf.items('naming'),))
|
||||
for section in ('building', 'files'):
|
||||
for name, value in buildconf.items(section):
|
||||
value = value.split()
|
||||
self.__setattr__(name, value)
|
||||
self.workdir = workdir
|
||||
|
||||
def checkEnv(self):
|
||||
"""Is this environment fit to build in, based on the spec file?"""
|
||||
pass
|
||||
|
||||
def build(self):
|
||||
"""Do the build: run the execute line(s)"""
|
||||
for cmd in self.execute:
|
||||
ret, output = run(os.path.join(self.workdir, cmd), chdir=self.workdir)
|
||||
if ret:
|
||||
raise BuildError, 'Build command failed: %s' % output
|
||||
|
||||
def virusCheck(self):
|
||||
"""check the build output for viruses"""
|
||||
pass
|
||||
|
||||
def gatherOutput(self):
|
||||
"""gather information about the output from the build, return it"""
|
||||
if self.results != None: return self.results
|
||||
# TODO: VM platform value in hash?
|
||||
self.results = {}
|
||||
for ftype in ('output', 'debug', 'log'):
|
||||
files = getattr(self, ftype + 'files')
|
||||
fdict = {}
|
||||
if files:
|
||||
for f in files:
|
||||
if not os.path.exists(os.path.join(self.workdir, f)):
|
||||
raise BuildError('Could not find %s after build' % f)
|
||||
fdict[f] = {'platform': self.platform, 'debug': False, 'flags': ''}
|
||||
self.results[ftype] = fdict
|
||||
return self.results
|
||||
|
||||
def doAll(self):
|
||||
"""helper function that runs the entire process"""
|
||||
self.checkEnv()
|
||||
self.build()
|
||||
self.virusCheck()
|
||||
return self.gatherOutput()
|
||||
|
||||
def log(msg):
|
||||
print >> sys.stderr, '%s: %s' % (datetime.datetime.now().ctime(), msg)
|
||||
|
||||
def run(cmd):
|
||||
def run(cmd, chdir='.'):
|
||||
shell = False
|
||||
if isinstance(cmd, (str, unicode)) and len(cmd.split()) > 1:
|
||||
shell = True
|
||||
olddir = os.getcwd()
|
||||
os.chdir(chdir)
|
||||
log('running command: %s' % cmd)
|
||||
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
close_fds=True, shell=shell)
|
||||
ret = proc.wait()
|
||||
output = proc.stdout.read()
|
||||
os.chdir(olddir)
|
||||
return ret, output
|
||||
|
||||
def find_net_info():
|
||||
|
|
@ -78,7 +374,8 @@ def find_net_info():
|
|||
gateway = None
|
||||
return macaddr, gateway
|
||||
|
||||
def uploadFile(server, prefix, path):
|
||||
def upload_file(server, prefix, path):
|
||||
"""upload a single file to the vmd"""
|
||||
fobj = file(os.path.join(prefix, path), 'r')
|
||||
offset = 0
|
||||
sum = hashlib.sha1()
|
||||
|
|
@ -93,85 +390,111 @@ def uploadFile(server, prefix, path):
|
|||
fobj.close()
|
||||
server.verifyChecksum(path, sum.hexdigest(), 'sha1')
|
||||
|
||||
def uploadDir(server, root):
|
||||
for dirpath, dirnames, filenames in os.walk(root):
|
||||
for filename in filenames:
|
||||
filepath = os.path.join(dirpath, filename)
|
||||
relpath = filepath[len(root) + 1:]
|
||||
uploadFile(server, root, relpath)
|
||||
def upload_results(server, codir, results):
|
||||
"""upload the results of a build given the results dict"""
|
||||
for output_type in results.keys():
|
||||
for output_file in results[output_type].keys():
|
||||
upload_file(server, codir, output_file)
|
||||
|
||||
def main():
|
||||
def get_mgmt_server():
|
||||
"""retrieve scmurls from kojivmd we'll use to build from"""
|
||||
macaddr, gateway = find_net_info()
|
||||
while not (macaddr and gateway):
|
||||
# wait for the network connection to come up and get an address
|
||||
time.sleep(5)
|
||||
macaddr, gateway = find_net_info()
|
||||
log('found MAC address %s, connecting to %s:%s' % (macaddr, gateway, MANAGER_PORT))
|
||||
server = xmlrpclib.ServerProxy('http://%s:%s/' % (gateway, MANAGER_PORT), allow_none=True)
|
||||
# we would set a timeout on the socket here, but that is apparently not supported
|
||||
# by python/cygwin/Windows
|
||||
log('found MAC address %s, connecting to %s:%s' %
|
||||
(macaddr, gateway, MANAGER_PORT))
|
||||
server = xmlrpclib.ServerProxy('http://%s:%s/' %
|
||||
(gateway, MANAGER_PORT), allow_none=True)
|
||||
# we would set a timeout on the socket here, but that is apparently not
|
||||
# supported by python/cygwin/Windows
|
||||
task_port = server.getPort(macaddr)
|
||||
log('found task-specific port %s' % task_port)
|
||||
server = xmlrpclib.ServerProxy('http://%s:%s/' % (gateway, task_port), allow_none=True)
|
||||
return xmlrpclib.ServerProxy('http://%s:%s/' % (gateway, task_port), allow_none=True)
|
||||
|
||||
ret = 1
|
||||
output = 'unknown error'
|
||||
exc_info = None
|
||||
def get_options():
|
||||
"""handle usage and parse options"""
|
||||
usage = """%prog [options]
|
||||
Run Koji tasks assigned to a VM.
|
||||
Run without any arguments to start this daemon.
|
||||
"""
|
||||
parser = OptionParser(usage=usage)
|
||||
parser.add_option('-i', '--install', action='store_true', help='Install this daemon as a service', default=False)
|
||||
parser.add_option('-u', '--uninstall', action='store_true', help='Uninstall this daemon if it was installed previously as a service', default=False)
|
||||
parser.add_option('-s', '--scmurl', action='append', help='Forcibly specify an scmurl to checkout from', default=[])
|
||||
(options, args) = parser.parse_args()
|
||||
return options
|
||||
|
||||
try:
|
||||
task_info = server.getTaskInfo()
|
||||
if task_info:
|
||||
cmd = task_info[0]
|
||||
os.mkdir('/tmp/output')
|
||||
log('running command: %s' % cmd)
|
||||
ret, output = run(cmd)
|
||||
else:
|
||||
ret = 1
|
||||
output = 'no command provided'
|
||||
uploadDir(server, '/tmp/output')
|
||||
except:
|
||||
exc_info = sys.exc_info()
|
||||
finally:
|
||||
if exc_info:
|
||||
tb = ''.join(traceback.format_exception(*exc_info))
|
||||
server.failTask(tb)
|
||||
elif ret:
|
||||
server.failTask('"%s" failed, return code was %s, output was %s' % (cmd, ret, output))
|
||||
else:
|
||||
server.closeTask(output)
|
||||
def run_build(workdir, scmurls):
|
||||
"""run the build"""
|
||||
if len(scmurls) == 2:
|
||||
# should SCM.assert_allowed() be called on SCM objs?
|
||||
specurl, srcurl = scmurls
|
||||
spec_scm = SCM(specurl)
|
||||
spec_dir = spec_scm.checkout(workdir)
|
||||
src_scm = SCM(srcurl)
|
||||
src_dir = src_scm.checkout(workdir)
|
||||
elif len(scmurls) == 1:
|
||||
specurl = scmurls[0]
|
||||
spec_scm = SCM(specurl)
|
||||
spec_dir = spec_scm.checkout(workdir)
|
||||
src_scm, src_dir = spec_scm, spec_dir
|
||||
else:
|
||||
raise BuildError, 'Unexpected number of SCM URLs given: %s' % scmurls
|
||||
|
||||
def usage():
|
||||
print '%s: Runs Koji tasks assigned to a VM'
|
||||
print ' run with no options to start the daemon'
|
||||
print
|
||||
print 'Options:'
|
||||
print ' --help show this help message and exit'
|
||||
print ' --install install this daemon as the "kojiwind" Windows service'
|
||||
print ' --uninstall uninstall the "kojiwind" Windows service'
|
||||
specfile = [spec for spec in os.listdir(spec_dir) if spec.endswith('.ini')]
|
||||
if len(specfile) != 1:
|
||||
raise BuildError, 'Exactly one .ini file must be found in a check out'
|
||||
winbld = WindowsBuild(os.path.join(spec_dir, specfile[0]), src_dir)
|
||||
return winbld.doAll(), src_dir
|
||||
|
||||
def flunk(server_report, server=None):
|
||||
"""do the right thing when a build fails"""
|
||||
exc_info = sys.exc_info()
|
||||
tb = ''.join(traceback.format_exception(*exc_info))
|
||||
if server_report:
|
||||
server.failTask(tb)
|
||||
log(tb)
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == '__main__':
|
||||
prog = os.path.abspath(sys.argv[0])
|
||||
if len(sys.argv) > 1:
|
||||
opt = sys.argv[1]
|
||||
if opt == '--install':
|
||||
ret, output = run(['cygrunsrv', '--install', 'kojiwind',
|
||||
'--path', sys.executable, '--args', prog,
|
||||
'--type', 'auto', '--dep', 'Dhcp',
|
||||
'--disp', 'Koji Windows Daemon',
|
||||
'--desc', 'Runs Koji tasks assigned to a VM'])
|
||||
if ret:
|
||||
print 'Error installing kojiwind service, output was: %s' % output
|
||||
sys.exit(1)
|
||||
else:
|
||||
print 'Successfully installed the kojiwind service'
|
||||
elif opt == '--uninstall':
|
||||
ret, output = run(['cygrunsrv', '--remove', 'kojiwind'])
|
||||
if ret:
|
||||
print 'Error removing the kojiwind service, output was: %s' % output
|
||||
sys.exit(1)
|
||||
else:
|
||||
print 'Successfully removed the kojiwind service'
|
||||
prog = os.path.basename(sys.argv[0])
|
||||
opts = get_options()
|
||||
if opts.install:
|
||||
ret, output = run(['cygrunsrv', '--install', prog,
|
||||
'--path', sys.executable, '--args', os.path.abspath(prog),
|
||||
'--type', 'auto', '--dep', 'Dhcp',
|
||||
'--disp', 'Koji Windows Daemon',
|
||||
'--desc', 'Runs Koji tasks assigned to a VM'])
|
||||
if ret:
|
||||
print 'Error installing %s service, output was: %s' % (prog, output)
|
||||
sys.exit(1)
|
||||
else:
|
||||
usage()
|
||||
else:
|
||||
main()
|
||||
print 'Successfully installed the %s service' % prog
|
||||
sys.exit(0)
|
||||
elif opts.uninstall:
|
||||
ret, output = run(['cygrunsrv', '--remove', prog])
|
||||
if ret:
|
||||
print 'Error removing the %s service, output was: %s' % (prog, output)
|
||||
sys.exit(1)
|
||||
else:
|
||||
print 'Successfully removed the %s service' % prog
|
||||
sys.exit(0)
|
||||
use_server, server = False, None
|
||||
try:
|
||||
if len(opts.scmurl) == 0:
|
||||
use_server = True
|
||||
server = get_mgmt_server()
|
||||
scmurl = server.getTaskInfo()
|
||||
opts.scmurl.append(scmurl)
|
||||
workdir = '/tmp/workdir'
|
||||
os.mkdir(workdir)
|
||||
results, results_dir = run_build(workdir, opts.scmurl)
|
||||
if use_server:
|
||||
upload_results(server, results_dir, results)
|
||||
server.closeTask(results)
|
||||
log('Build results: %s' % results)
|
||||
except:
|
||||
flunk(use_server, server=server)
|
||||
sys.exit(0)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue