initial import of Koji VM code

vm/kojikamid

@@ -0,0 +1,177 @@
+#!/usr/bin/python
+
+# Koji daemon that runs in a Windows VM and executes commands associated
+# with a task.
+# Copyright (c) 2010 Red Hat
+#
+#    Koji is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU Lesser General Public
+#    License as published by the Free Software Foundation; 
+#    version 2.1 of the License.
+#
+#    This software is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#    Lesser General Public License for more details.
+#
+#    You should have received a copy of the GNU Lesser General Public
+#    License along with this software; if not, write to the Free Software
+#    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+#
+# Authors:
+#       Mike Bonnet <mikeb@redhat.com>
+
+# To register this script as a service on Windows 2008 (with Cygwin 1.7.5 installed) run:
+#   kojiwind --install
+# in a cygwin shell.
+
+import datetime
+import os
+import subprocess
+import sys
+import time
+import xmlrpclib
+import base64
+import hashlib
+import traceback
+
+MANAGER_PORT = 7000
+
+def log(msg):
+    print >> sys.stderr, '%s: %s' % (datetime.datetime.now().ctime(), msg)
+
+def run(cmd):
+    shell = False
+    if isinstance(cmd, (str, unicode)) and len(cmd.split()) > 1:
+        shell = True
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                            close_fds=True, shell=shell)
+    ret = proc.wait()
+    output = proc.stdout.read()
+    return ret, output
+
+def find_net_info():
+    """
+    Find the network gateway configured for this VM.
+    """
+    ret, output = run(['ipconfig', '/all'])
+    if ret:
+        raise RuntimeError, 'error running ipconfig, output was: %s' % output
+    macaddr = None
+    gateway = None
+    for line in output.splitlines():
+        line = line.strip()
+        # take the first values we find
+        if line.startswith('Physical Address'):
+            if not macaddr:
+                macaddr = line.split()[-1]
+                # format it to be consistent with the libvirt MAC address
+                macaddr = macaddr.replace('-', ':').lower()
+        elif line.startswith('Default Gateway'):
+            if not gateway:
+                gateway = line.split()[-1]
+
+    # check that we have valid values
+    if macaddr and len(macaddr) != 17:
+        macaddr = None
+    if gateway and (len(gateway) < 7 or len(gateway) > 15):
+        gateway = None
+    return macaddr, gateway
+
+def uploadFile(server, prefix, path):
+    fobj = file(os.path.join(prefix, path), 'r')
+    offset = 0
+    sum = hashlib.sha1()
+    while True:
+        data = fobj.read(131072)
+        if not data:
+            break
+        encoded = base64.b64encode(data)
+        server.upload(path, offset, encoded)
+        offset += len(data)
+        sum.update(data)
+    fobj.close()
+    server.verifyChecksum(path, sum.hexdigest(), 'sha1')
+
+def uploadDir(server, root):
+    for dirpath, dirnames, filenames in os.walk(root):
+        for filename in filenames:
+            filepath = os.path.join(dirpath, filename)
+            relpath = filepath[len(root) + 1:]
+            uploadFile(server, root, relpath)
+
+def main():
+    macaddr, gateway = find_net_info()
+    while not (macaddr and gateway):
+        # wait for the network connection to come up and get an address
+        time.sleep(5)
+        macaddr, gateway = find_net_info()
+    log('found MAC address %s, connecting to %s:%s' % (macaddr, gateway, MANAGER_PORT))
+    server = xmlrpclib.ServerProxy('http://%s:%s/' % (gateway, MANAGER_PORT), allow_none=True)
+    # we would set a timeout on the socket here, but that is apparently not supported
+    # by python/cygwin/Windows
+    task_port = server.getPort(macaddr)
+    log('found task-specific port %s' % task_port)
+    server = xmlrpclib.ServerProxy('http://%s:%s/' % (gateway, task_port), allow_none=True)
+
+    ret = 1
+    output = 'unknown error'
+    exc_info = None
+
+    try:
+        task_info = server.getTaskInfo()
+        if task_info:
+            cmd = task_info[0]
+            os.mkdir('/tmp/output')
+            log('running command: %s' % cmd)
+            ret, output = run(cmd)
+        else:
+            ret = 1
+            output = 'no command provided'
+        uploadDir(server, '/tmp/output')
+    except:
+        exc_info = sys.exc_info()
+    finally:
+        if exc_info:
+            tb = ''.join(traceback.format_exception(*exc_info))
+            server.failTask(tb)
+        elif ret:
+            server.failTask('"%s" failed, return code was %s, output was %s' % (cmd, ret, output))
+        else:
+            server.closeTask(output)
+
+def usage():
+    print '%s: Runs Koji tasks assigned to a VM'
+    print '    run with no options to start the daemon'
+    print
+    print 'Options:'
+    print '  --help       show this help message and exit'
+    print '  --install    install this daemon as the "kojiwind" Windows service'
+    print '  --uninstall  uninstall the "kojiwind" Windows service'
+
+if __name__ == '__main__':
+    prog = os.path.abspath(sys.argv[0])
+    if len(sys.argv) > 1:
+        opt = sys.argv[1]
+        if opt == '--install':
+            ret, output = run(['cygrunsrv', '--install', 'kojiwind',
+                               '--path', sys.executable, '--args', prog,
+                               '--type', 'auto', '--dep', 'Dhcp',
+                               '--disp', 'Koji Windows Daemon',
+                               '--desc', 'Runs Koji tasks assigned to a VM'])
+            if ret:
+                print 'Error installing kojiwind service, output was: %s' % output
+                sys.exit(1)
+            else:
+                print 'Successfully installed the kojiwind service'
+        elif opt == '--uninstall':
+            ret, output = run(['cygrunsrv', '--remove', 'kojiwind'])
+            if ret:
+                print 'Error removing the kojiwind service, output was: %s' % output
+                sys.exit(1)
+            else:
+                print 'Successfully removed the kojiwind service'
+        else:
+            usage()
+    else:
+        main()

vm/kojivmd.conf

@@ -0,0 +1,35 @@
+[kojivmd]
+; The number of seconds to sleep between tasks
+; sleeptime=15
+
+; The maximum number of jobs that kojivmd will handle at a time
+; maxjobs=10
+
+; Minimum amount of memory (in MBs) not allocated to a VM for kojivmd to take a new task
+; minmem=4096
+
+; The user the VM/emulator runs as (cloned disk images will be readable and writable by this user)
+; vmuser=qemu
+
+; The directory root for temporary storage
+; workdir=/tmp/koji
+
+; The URL for the xmlrpc server
+server=http://hub.example.com/kojihub
+
+; The mail host to use for sending email notifications
+smtphost=example.com
+
+; The From address used when sending email notifications
+from_addr=Koji Build System <buildsys@example.com>
+
+;configuration for SSL authentication
+
+;client certificate
+;cert = /etc/kojivmd/client.crt
+
+;certificate of the CA that issued the client certificate
+;ca = /etc/kojivmd/clientca.crt
+
+;certificate of the CA that issued the HTTP server certificate
+;serverca = /etc/kojivmd/serverca.crt

vm/run-vm-task

@@ -0,0 +1,40 @@
+#!/usr/bin/python
+
+import koji
+import optparse
+
+# cli/koji -c ~/.koji/config-mead call --python makeTask '"vmExec"' '["Win2k8-x86-vstudio-devel", ["wget -q -O /tmp/test-build.sh http://download.lab.bos.redhat.com/devel/mikeb/mead/debug/test-build.sh && chmod 755 /tmp/test-build.sh && /tmp/test-build.sh &> /tmp/output/build.log && echo build successful"], {"cpus": 2, "mem": 2048}]' --kwargs '{"channel": "vm"}'
+
+parser = optparse.OptionParser('%prog VM-NAME COMMAND-TO-RUN')
+parser.add_option('--server', help='Koji hub')
+parser.add_option('--cert', help='Client certificate')
+parser.add_option('--ca', help='Client CA')
+parser.add_option('--server-ca', help='Server CA')
+parser.add_option('--cpus', help='Number of virtual CPUs to allocate to the VM (optional)',
+                  type='int')
+parser.add_option('--mem', help='Amount of memory (in megabytes) to allocate to the VM (optional)',
+                  type='int')
+parser.add_option('--channel', help='Channel to create the task in', default='vm')
+
+opts, args = parser.parse_args()
+
+if len(args) < 2:
+    parser.error('You must specify a VM name and a command to run')
+
+vm_name = args[0]
+cmd = ' '.join(args[1:])
+
+session = koji.ClientSession(opts.server)
+session.ssl_login(opts.cert, opts.ca, opts.server_ca)
+
+task_opts = {}
+if opts.cpus:
+    task_opts['cpus'] = opts.cpus
+if opts.mem:
+    task_opts['mem'] = opts.mem
+
+params = [vm_name, [cmd], task_opts]
+
+task_id = session.makeTask('vmExec', params, channel=opts.channel)
+
+print 'Created task %s' % task_id