diff --git a/vm/kojikamid b/vm/kojikamid index b296edbe..af2ae768 100755 --- a/vm/kojikamid +++ b/vm/kojikamid @@ -26,10 +26,13 @@ # in a cygwin shell. import datetime +from optparse import OptionParser +from ConfigParser import ConfigParser import os import subprocess import sys import time +import urlparse import xmlrpclib import base64 import hashlib @@ -37,17 +40,310 @@ import traceback MANAGER_PORT = 7000 +############################## +# Begin heinous copy and paste +############################## + +class GenericError(Exception): + """Base class for our custom exceptions""" + faultCode = 1000 + fromFault = False + def __str__(self): + try: + return str(self.args[0]['args'][0]) + except: + try: + return str(self.args[0]) + except: + return str(self.__dict__) + +class BuildError(GenericError): + """Raised when a build fails""" + faultCode = 1005 + +class SCM(object): + "SCM abstraction class" + + types = { 'CVS': ('cvs://',), + 'CVS+SSH': ('cvs+ssh://',), + 'GIT': ('git://', 'git+http://', 'git+https://', 'git+rsync://'), + 'GIT+SSH': ('git+ssh://',), + 'SVN': ('svn://', 'svn+http://', 'svn+https://'), + 'SVN+SSH': ('svn+ssh://',) } + + def is_scm_url(url): + """ + Return True if the url appears to be a valid, accessible source location, False otherwise + """ + for schemes in SCM.types.values(): + for scheme in schemes: + if url.startswith(scheme): + return True + else: + return False + is_scm_url = staticmethod(is_scm_url) + + def __init__(self, url): + """ + Initialize the SCM object using the specified url. + The expected url format is: + + scheme://[user@]host/path/to/repo?path/to/module#revision_or_tag_identifier + + The initialized SCM object will have the following attributes: + - url (the unmodified url) + - scheme + - user (may be null) + - host + - repository + - module + - revision + - scmtype + + The exact format of each attribute is SCM-specific, but the structure of the url + must conform to the template above, or an error will be raised. + """ + if not SCM.is_scm_url(url): + raise GenericError, 'Invalid SCM URL: %s' % url + + self.url = url + scheme, user, host, path, query, fragment = self._parse_url() + + self.scheme = scheme + self.user = user + self.host = host + self.repository = path + self.module = query + self.revision = fragment + + for scmtype, schemes in SCM.types.items(): + if self.scheme in schemes: + self.scmtype = scmtype + break + else: + # should never happen + raise GenericError, 'Invalid SCM URL: %s' % url + + def _parse_url(self): + """ + Parse the SCM url into usable components. + Return the following tuple: + + (scheme, user, host, path, query, fragment) + + user may be None, everything else will have a value + """ + # get the url's scheme + scheme = self.url.split('://')[0] + '://' + + # replace the scheme with http:// so that the urlparse works in all cases + dummyurl = self.url.replace(scheme, 'http://', 1) + dummyscheme, netloc, path, params, query, fragment = urlparse.urlparse(dummyurl) + + user = None + userhost = netloc.split('@') + if len(userhost) == 2: + user = userhost[0] + if not user: + # Don't return an empty string + user = None + elif ':' in user: + raise GenericError, 'username:password format not supported: %s' % user + netloc = userhost[1] + elif len(userhost) > 2: + raise GenericError, 'Invalid username@hostname specified: %s' % netloc + + # ensure that path and query do not end in / + if path.endswith('/'): + path = path[:-1] + if query.endswith('/'): + query = query[:-1] + + # check for validity: params should be empty, query may be empty, everything else should be populated + if params or not (scheme and netloc and path and fragment): + raise GenericError, 'Unable to parse SCM URL: %s' % self.url + + # return parsed values + return (scheme, user, netloc, path, query, fragment) + + def checkout(self, scmdir): + """ + Checkout the module from SCM. Accepts the following parameters: + - scmdir: the working directory + + Returns the directory that the module was checked-out into (a subdirectory of scmdir) + """ + # TODO: sanity check arguments + sourcedir = '%s/%s' % (scmdir, self.module) + + update_checkout_cmd = None + + if self.scmtype == 'CVS': + pserver = ':pserver:%s@%s:%s' % ((self.user or 'anonymous'), self.host, self.repository) + module_checkout_cmd = ['cvs', '-d', pserver, 'checkout', '-r', self.revision, self.module] + + elif self.scmtype == 'CVS+SSH': + if not self.user: + raise BuildError, 'No user specified for repository access scheme: %s' % self.scheme + + cvsserver = ':ext:%s@%s:%s' % (self.user, self.host, self.repository) + module_checkout_cmd = ['cvs', '-d', cvsserver, 'checkout', '-r', self.revision, self.module] + + elif self.scmtype == 'GIT': + scheme = self.scheme + if '+' in scheme: + scheme = scheme.split('+')[1] + gitrepo = '%s%s%s' % (scheme, self.host, self.repository) + checkout_path = os.path.basename(self.repository) + if self.repository.endswith('/.git'): + checkout_path = os.path.basename(self.repository[:-5]) + elif self.repository.endswith('.git'): + checkout_path = os.path.basename(self.repository[:-4]) + + sourcedir = '%s/%s' % (scmdir, checkout_path) + module_checkout_cmd = ['git', 'clone', '-n', gitrepo, sourcedir] + update_checkout_cmd = ['git', 'reset', '--hard', self.revision] + + # self.module may be empty, in which case the specfile should be in the top-level directory + if self.module: + # Treat the module as a directory inside the git repository + sourcedir = '%s/%s' % (sourcedir, self.module) + + elif self.scmtype == 'GIT+SSH': + if not self.user: + raise BuildError, 'No user specified for repository access scheme: %s' % self.scheme + gitrepo = 'git+ssh://%s@%s%s' % (self.user, self.host, self.repository) + checkout_path = os.path.basename(self.repository) + if self.repository.endswith('/.git'): + checkout_path = os.path.basename(self.repository[:-5]) + elif self.repository.endswith('.git'): + checkout_path = os.path.basename(self.repository[:-4]) + + sourcedir = '%s/%s' % (scmdir, checkout_path) + module_checkout_cmd = ['git', 'clone', '-n', gitrepo, sourcedir] + update_checkout_cmd = ['git', 'reset', '--hard', self.revision] + + # self.module may be empty, in which case the specfile should be in the top-level directory + if self.module: + # Treat the module as a directory inside the git repository + sourcedir = '%s/%s' % (sourcedir, self.module) + + elif self.scmtype == 'SVN': + scheme = self.scheme + if '+' in scheme: + scheme = scheme.split('+')[1] + + svnserver = '%s%s%s' % (scheme, self.host, self.repository) + module_checkout_cmd = ['svn', 'checkout', '-r', self.revision, '%s/%s' % (svnserver, self.module), self.module] + + elif self.scmtype == 'SVN+SSH': + if not self.user: + raise BuildError, 'No user specified for repository access scheme: %s' % self.scheme + + svnserver = 'svn+ssh://%s@%s%s' % (self.user, self.host, self.repository) + module_checkout_cmd = ['svn', 'checkout', '-r', self.revision, '%s/%s' % (svnserver, self.module), self.module] + + else: + raise BuildError, 'Unknown SCM type: %s' % self.scmtype + + # perform checkouts + ret, output = run(module_checkout_cmd) + log(output) + if ret: + raise BuildError, 'Error running %s checkout command "%s: %s"' % \ + (self.scmtype, ' '.join(module_checkout_cmd), output) + + if update_checkout_cmd: + # Currently only required for GIT checkouts + # Run the command in the directory the source was checked out into + ret, output = run(update_checkout_cmd, chdir=sourcedir) + log(output) + if ret: + raise BuildError, 'Error running %s update command "%s": %s' % \ + (self.scmtype, ' '.join(update_checkout_cmd), output) + + return sourcedir + +############################ +# End heinous copy and paste +############################ + +class WindowsBuild(object): + + def __init__(self, specpath, workdir): + """constructor: check ini spec file syntax, set build properties""" + buildconf = ConfigParser() + buildconf.read(specpath) + self.results = None + + # make sure we've got the right sections defined + goodsections = ('naming', 'building', 'files') + badsections = [s for s in buildconf.sections() if s not in goodsections] + if len(badsections) > 0: + raise BuildError, 'Unrecognized section(s) in ini: %s' % badsections + for section in goodsections: + if not buildconf.has_section(section): + raise BuildError, 'missing required section in ini: %s' % section + map(self.__dict__.update, (buildconf.items('naming'),)) + for section in ('building', 'files'): + for name, value in buildconf.items(section): + value = value.split() + self.__setattr__(name, value) + self.workdir = workdir + + def checkEnv(self): + """Is this environment fit to build in, based on the spec file?""" + pass + + def build(self): + """Do the build: run the execute line(s)""" + for cmd in self.execute: + ret, output = run(os.path.join(self.workdir, cmd), chdir=self.workdir) + if ret: + raise BuildError, 'Build command failed: %s' % output + + def virusCheck(self): + """check the build output for viruses""" + pass + + def gatherOutput(self): + """gather information about the output from the build, return it""" + if self.results != None: return self.results + # TODO: VM platform value in hash? + self.results = {} + for ftype in ('output', 'debug', 'log'): + files = getattr(self, ftype + 'files') + fdict = {} + if files: + for f in files: + if not os.path.exists(os.path.join(self.workdir, f)): + raise BuildError('Could not find %s after build' % f) + fdict[f] = {'platform': self.platform, 'debug': False, 'flags': ''} + self.results[ftype] = fdict + return self.results + + def doAll(self): + """helper function that runs the entire process""" + self.checkEnv() + self.build() + self.virusCheck() + return self.gatherOutput() + def log(msg): print >> sys.stderr, '%s: %s' % (datetime.datetime.now().ctime(), msg) -def run(cmd): +def run(cmd, chdir='.'): shell = False if isinstance(cmd, (str, unicode)) and len(cmd.split()) > 1: shell = True + olddir = os.getcwd() + os.chdir(chdir) + log('running command: %s' % cmd) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True, shell=shell) ret = proc.wait() output = proc.stdout.read() + os.chdir(olddir) return ret, output def find_net_info(): @@ -78,7 +374,8 @@ def find_net_info(): gateway = None return macaddr, gateway -def uploadFile(server, prefix, path): +def upload_file(server, prefix, path): + """upload a single file to the vmd""" fobj = file(os.path.join(prefix, path), 'r') offset = 0 sum = hashlib.sha1() @@ -93,85 +390,111 @@ def uploadFile(server, prefix, path): fobj.close() server.verifyChecksum(path, sum.hexdigest(), 'sha1') -def uploadDir(server, root): - for dirpath, dirnames, filenames in os.walk(root): - for filename in filenames: - filepath = os.path.join(dirpath, filename) - relpath = filepath[len(root) + 1:] - uploadFile(server, root, relpath) +def upload_results(server, codir, results): + """upload the results of a build given the results dict""" + for output_type in results.keys(): + for output_file in results[output_type].keys(): + upload_file(server, codir, output_file) -def main(): +def get_mgmt_server(): + """retrieve scmurls from kojivmd we'll use to build from""" macaddr, gateway = find_net_info() while not (macaddr and gateway): # wait for the network connection to come up and get an address time.sleep(5) macaddr, gateway = find_net_info() - log('found MAC address %s, connecting to %s:%s' % (macaddr, gateway, MANAGER_PORT)) - server = xmlrpclib.ServerProxy('http://%s:%s/' % (gateway, MANAGER_PORT), allow_none=True) - # we would set a timeout on the socket here, but that is apparently not supported - # by python/cygwin/Windows + log('found MAC address %s, connecting to %s:%s' % + (macaddr, gateway, MANAGER_PORT)) + server = xmlrpclib.ServerProxy('http://%s:%s/' % + (gateway, MANAGER_PORT), allow_none=True) + # we would set a timeout on the socket here, but that is apparently not + # supported by python/cygwin/Windows task_port = server.getPort(macaddr) log('found task-specific port %s' % task_port) - server = xmlrpclib.ServerProxy('http://%s:%s/' % (gateway, task_port), allow_none=True) + return xmlrpclib.ServerProxy('http://%s:%s/' % (gateway, task_port), allow_none=True) - ret = 1 - output = 'unknown error' - exc_info = None +def get_options(): + """handle usage and parse options""" + usage = """%prog [options] + Run Koji tasks assigned to a VM. + Run without any arguments to start this daemon. + """ + parser = OptionParser(usage=usage) + parser.add_option('-i', '--install', action='store_true', help='Install this daemon as a service', default=False) + parser.add_option('-u', '--uninstall', action='store_true', help='Uninstall this daemon if it was installed previously as a service', default=False) + parser.add_option('-s', '--scmurl', action='append', help='Forcibly specify an scmurl to checkout from', default=[]) + (options, args) = parser.parse_args() + return options - try: - task_info = server.getTaskInfo() - if task_info: - cmd = task_info[0] - os.mkdir('/tmp/output') - log('running command: %s' % cmd) - ret, output = run(cmd) - else: - ret = 1 - output = 'no command provided' - uploadDir(server, '/tmp/output') - except: - exc_info = sys.exc_info() - finally: - if exc_info: - tb = ''.join(traceback.format_exception(*exc_info)) - server.failTask(tb) - elif ret: - server.failTask('"%s" failed, return code was %s, output was %s' % (cmd, ret, output)) - else: - server.closeTask(output) +def run_build(workdir, scmurls): + """run the build""" + if len(scmurls) == 2: + # should SCM.assert_allowed() be called on SCM objs? + specurl, srcurl = scmurls + spec_scm = SCM(specurl) + spec_dir = spec_scm.checkout(workdir) + src_scm = SCM(srcurl) + src_dir = src_scm.checkout(workdir) + elif len(scmurls) == 1: + specurl = scmurls[0] + spec_scm = SCM(specurl) + spec_dir = spec_scm.checkout(workdir) + src_scm, src_dir = spec_scm, spec_dir + else: + raise BuildError, 'Unexpected number of SCM URLs given: %s' % scmurls -def usage(): - print '%s: Runs Koji tasks assigned to a VM' - print ' run with no options to start the daemon' - print - print 'Options:' - print ' --help show this help message and exit' - print ' --install install this daemon as the "kojiwind" Windows service' - print ' --uninstall uninstall the "kojiwind" Windows service' + specfile = [spec for spec in os.listdir(spec_dir) if spec.endswith('.ini')] + if len(specfile) != 1: + raise BuildError, 'Exactly one .ini file must be found in a check out' + winbld = WindowsBuild(os.path.join(spec_dir, specfile[0]), src_dir) + return winbld.doAll(), src_dir + +def flunk(server_report, server=None): + """do the right thing when a build fails""" + exc_info = sys.exc_info() + tb = ''.join(traceback.format_exception(*exc_info)) + if server_report: + server.failTask(tb) + log(tb) + sys.exit(1) if __name__ == '__main__': - prog = os.path.abspath(sys.argv[0]) - if len(sys.argv) > 1: - opt = sys.argv[1] - if opt == '--install': - ret, output = run(['cygrunsrv', '--install', 'kojiwind', - '--path', sys.executable, '--args', prog, - '--type', 'auto', '--dep', 'Dhcp', - '--disp', 'Koji Windows Daemon', - '--desc', 'Runs Koji tasks assigned to a VM']) - if ret: - print 'Error installing kojiwind service, output was: %s' % output - sys.exit(1) - else: - print 'Successfully installed the kojiwind service' - elif opt == '--uninstall': - ret, output = run(['cygrunsrv', '--remove', 'kojiwind']) - if ret: - print 'Error removing the kojiwind service, output was: %s' % output - sys.exit(1) - else: - print 'Successfully removed the kojiwind service' + prog = os.path.basename(sys.argv[0]) + opts = get_options() + if opts.install: + ret, output = run(['cygrunsrv', '--install', prog, + '--path', sys.executable, '--args', os.path.abspath(prog), + '--type', 'auto', '--dep', 'Dhcp', + '--disp', 'Koji Windows Daemon', + '--desc', 'Runs Koji tasks assigned to a VM']) + if ret: + print 'Error installing %s service, output was: %s' % (prog, output) + sys.exit(1) else: - usage() - else: - main() + print 'Successfully installed the %s service' % prog + sys.exit(0) + elif opts.uninstall: + ret, output = run(['cygrunsrv', '--remove', prog]) + if ret: + print 'Error removing the %s service, output was: %s' % (prog, output) + sys.exit(1) + else: + print 'Successfully removed the %s service' % prog + sys.exit(0) + use_server, server = False, None + try: + if len(opts.scmurl) == 0: + use_server = True + server = get_mgmt_server() + scmurl = server.getTaskInfo() + opts.scmurl.append(scmurl) + workdir = '/tmp/workdir' + os.mkdir(workdir) + results, results_dir = run_build(workdir, opts.scmurl) + if use_server: + upload_results(server, results_dir, results) + server.closeTask(results) + log('Build results: %s' % results) + except: + flunk(use_server, server=server) + sys.exit(0)