PR#200 Saving failed build trees

Merges #200

cli/koji

@@ -7203,6 +7203,70 @@ def handle_runroot(options, session, args):
     return
 
 
+def handle_save_failed_tree(options, session, args):
+    "Create tarball with whole buildtree"
+    usage = _("usage: %prog save-failed-tree [options] ID")
+    usage += _("\n(Specify the --help global option for a list of other help options)")
+    parser = OptionParser(usage=usage)
+    parser.add_option("-f", "--full", action="store_true", default=False,
+            help=_("Download whole tree, if not specified, only builddir will be downloaded"))
+    parser.add_option("-t", "--task", action="store_const", dest="mode",
+            const="task", default="task",
+            help=_("Treat ID as a task ID (the default)"))
+    parser.add_option("-r", "--buildroot", action="store_const", dest="mode",
+            const="buildroot",
+            help=_("Treat ID as a buildroot ID"))
+    parser.add_option("--quiet", action="store_true", default=options.quiet,
+                      help=_("Do not print the task information"))
+    parser.add_option("--nowait", action="store_true",
+                      help=_("Don't wait on build"))
+
+    (opts, args) = parser.parse_args(args)
+
+    if len(args) != 1:
+        parser.error(_("List exactly one task or buildroot ID"))
+
+    try:
+        id_val = int(args[0])
+    except ValueError:
+        parser.error(_("ID must be an integer"))
+
+    activate_session(session)
+
+    if opts.mode == "buildroot":
+        br_id = id_val
+    else:
+        brs = [b['id'] for b in session.listBuildroots(taskID=id_val)]
+        if not brs:
+            print(_("No buildroots for task %s") % id_val)
+            return 1
+        br_id = max(brs)
+        if len(brs) > 1:
+            print(_("Multiple buildroots for task. Choosing last one (%s)") % br_id)
+
+    try:
+        task_id = session.saveFailedTree(br_id, opts.full)
+    except koji.GenericError as e:
+        m = str(e)
+        if 'Invalid method' in m:
+            print(_("* The save_failed_tree plugin appears to not be "
+                    "installed on the koji hub.  Please contact the "
+                    "administrator."))
+            return 1
+        raise
+
+    if not opts.quiet:
+        print(_("Created task %s for buildroot %s") % (task_id, br_id))
+        print("Task info: %s/taskinfo?taskID=%s"
+                % (options.weburl, task_id))
+
+    if opts.nowait:
+        return
+    else:
+        session.logout()
+        watch_tasks(session, [task_id], quiet=opts.quiet)
+
+
 def handle_help(options, session, args):
     "[info] List available commands"
     usage = _("usage: %prog help <category> ...")

docs/source/index.rst

@@ -31,6 +31,7 @@ Contents
     server_bootstrap
     server_howto
     using_the_koji_build_system
+    plugins
     writing_a_plugin
     writing_koji_code
     content_generators

docs/source/plugins.rst

@@ -0,0 +1,68 @@
+=======
+Plugins
+=======
+
+Following plugins are available in default koji installation.
+
+Runroot
+=======
+
+Plugin for running any command in buildroot.
+
+Save Failed Tree Plugin
+=======================
+
+In some cases developers want to investigate exact environment in which their
+build failed. Reconstructing this environment via mock needn't end with
+exactly same structure (due to builder settings, etc.). In such case this
+plugin can be used to retrieve tarball with complete mock tree.
+
+Additional feature is that some paths from buildroot can be left out from
+tarball. Feature can be configured via
+`/etc/kojid/plugins/save_failed_tree.conf` file. Currently only field
+filters.paths is used and it consists of globs (standard python's fnmatch is
+used) separated by whitespaces.
+
+.. code-block:: ini
+
+  [filters]
+  paths = /etc/*.keytab /tmp/secret_data
+
+.. warning::
+  For security reasons, currently all ``/tmp/krb5cc*`` and ``/etc/*.keytab``
+  files are removed from tarball. If we found some other dangerous pieces,
+  they can be added to this blacklist.
+
+Special task method is created for achieving this which is called
+``SaveFailedTree``. This task can be created via CLI:
+``koji save-failed-tree <taskID>``. Additional options are:
+
+.. option:: --full
+
+   directs koji to create tarball with complete tree.
+
+.. option:: --nowait
+
+   exit immediately after creating task
+
+.. option:: --quiet
+
+   don't print any information to output
+
+After task finishes, one can find the tarball on relevant task web page (URL
+will be printed to stdout until ``--quiet`` is used.
+
+Plugin allow to save trees only for tasks defined in config
+``/etc/koji-hub/plugins/save_failed_tree.conf``. Option
+``allowed_methods`` contains list of comma-delimited names of tasks. Default
+configuration contains line: ``allowed_methods = buildArch``. Anybody
+is allowed to create this type of task (and download tarball).
+
+.. warning::
+  Don't forget that this type of task can generate huge amount of data, so use
+  it wisely.
+
+TODO
+----
+ * Separate volume/directory on hub
+ * garbage collector + policy for retaining generated tarballs

plugins/builder/save_failed_tree.conf

@@ -0,0 +1,5 @@
+[global]
+volume = DEFAULT
+
+[filters]
+paths = */tmp/krb5cc */etc/*.keytab

plugins/builder/save_failed_tree.py

@@ -0,0 +1,71 @@
+import fnmatch
+import os
+import tarfile
+import ConfigParser
+
+import koji
+import koji.tasks as tasks
+from __main__ import BuildRoot
+
+__all__ = ('SaveFailedTreeTask',)
+
+CONFIG_FILE = '/etc/kojid/plugins/save_failed_tree.conf'
+config = None
+
+
+def omit_paths(tarinfo):
+    if any([fnmatch.fnmatch(tarinfo.name, f) for f in config['path_filters']]):
+        return None
+    else:
+        return tarinfo
+
+
+def read_config():
+    global config
+    cp = ConfigParser.SafeConfigParser()
+    cp.read(CONFIG_FILE)
+    config = {
+        'path_filters': [],
+        'volume': None,
+    }
+    if cp.has_option('filters', 'paths'):
+        config['path_filters'] = cp.get('filters', 'paths').split()
+    if cp.has_option('general', 'volume'):
+        config['volume'] = cp.get('general', 'volume').strip()
+
+
+class SaveFailedTreeTask(tasks.BaseTaskHandler):
+    Methods = ['saveFailedTree']
+    _taskWeight = 3.0
+
+    def handler(self, buildrootID, full=False):
+        self.logger.debug("Saving buildroot %d [full=%s]", buildrootID, full)
+        read_config()
+
+        brinfo = self.session.getBuildroot(buildrootID)
+        if brinfo is None:
+            raise koji.GenericError("Nonexistent buildroot: %s" % buildrootID)
+        host_id = self.session.host.getHost()['id']
+        if brinfo['host_id'] != host_id:
+            raise koji.GenericError("Task is run on wrong builder")
+        broot = BuildRoot(self.session, self.options, brinfo['id'])
+        path = broot.rootdir()
+
+        if full:
+            self.logger.debug("Adding buildroot (full): %s" % path)
+        else:
+            path = os.path.join(path, 'builddir')
+            self.logger.debug("Adding buildroot: %s" % path)
+        if not os.path.exists(path):
+            raise koji.GenericError("Buildroot directory is missing: %s" % path)
+
+        tar_path = os.path.join(self.workdir, 'broot-%s.tar.gz' % buildrootID)
+        self.logger.debug("Creating buildroot archive %s", tar_path)
+        f = tarfile.open(tar_path, "w:gz")
+        f.add(path, filter=omit_paths)
+        f.close()
+
+        self.logger.debug("Uploading %s to hub", tar_path)
+        self.uploadFile(tar_path, volume=config['volume'])
+        os.unlink(tar_path)
+        self.logger.debug("Finished saving buildroot %s", buildrootID)

plugins/hub/save_failed_tree.conf

@@ -0,0 +1,7 @@
+# config file for the Koji save-failed-trees plugin
+
+[permissions]
+# task methods for whose can be triggered buildroot export
+# * can be used to allow everything. In such case it must be only component
+# on line. Otherwise multiple values are delimited by comma.
+allowed_methods = buildArch

plugins/hub/save_failed_tree.py

@@ -0,0 +1,54 @@
+import sys
+import ConfigParser
+import koji
+from koji.context import context
+from koji.plugin import export
+
+sys.path.insert(0, '/usr/share/koji-hub/')
+import kojihub
+
+__all__ = ('saveFailedTree',)
+
+CONFIG_FILE = '/etc/koji-hub/plugins/save_failed_tree.conf'
+config = None
+allowed_methods = None
+
+
+@export
+def saveFailedTree(buildrootID, full=False, **opts):
+    """Create saveFailedTree task
+
+    If arguments are invalid, error message is returned. Otherwise task id of
+    newly created task is returned."""
+    global config, allowed_methods
+
+    # let it raise errors
+    buildrootID = int(buildrootID)
+    full = bool(full)
+
+    # read configuration only once
+    if config is None:
+        config = ConfigParser.SafeConfigParser()
+        config.read(CONFIG_FILE)
+        allowed_methods = config.get('permissions', 'allowed_methods').split()
+        if len(allowed_methods) == 1 and allowed_methods[0] == '*':
+            allowed_methods = '*'
+
+    brinfo = kojihub.get_buildroot(buildrootID, strict=True)
+    taskID = brinfo['task_id']
+    task_info = kojihub.Task(taskID).getInfo()
+    if task_info['state'] != koji.TASK_STATES['FAILED']:
+        raise koji.PreBuildError("Task %s has not failed. Only failed tasks can upload their buildroots." % taskID)
+    elif allowed_methods != '*' and task_info['method'] not in allowed_methods:
+        raise koji.PreBuildError("Only %s tasks can upload their buildroots (Task %s is %s)." % \
+               (', '.join(allowed_methods), task_info['id'], task_info['method']))
+    elif task_info["owner"] != context.session.user_id and not context.session.assertPerm('admin'):
+        raise koji.ActionNotAllowed("Only owner of failed task or 'admin' can run this task.")
+    elif not kojihub.get_host(task_info['host_id'])['enabled']:
+        raise koji.PreBuildError("Host is disabled.")
+
+    args = koji.encode_args(buildrootID, full, **opts)
+    taskopts = {
+        'assign': brinfo['host_id'],
+    }
+    return kojihub.make_task('saveFailedTree', args, **taskopts)

tests/test_cli/data/list-commands.txt

@@ -118,6 +118,7 @@ miscellaneous commands:
         call                      Execute an arbitrary XML-RPC call
         import-comps              Import group/package information from a comps file
         moshimoshi                Introduce yourself
+        save-failed-tree          Create tarball with whole buildtree
 
 monitor commands:
         wait-repo                 Wait for a repo to be regenerated

tests/test_cli/test_save_failed_tree.py

@@ -0,0 +1,163 @@
+import StringIO
+import unittest
+import koji
+import mock
+
+import loadcli
+cli = loadcli.cli
+
+
+class TestSaveFailedTree(unittest.TestCase):
+    def setUp(self):
+        self.options = mock.MagicMock()
+        self.session = mock.MagicMock()
+        self.args = mock.MagicMock()
+        self.original_parser = cli.OptionParser
+        cli.OptionParser = mock.MagicMock()
+        self.parser = cli.OptionParser.return_value
+        cli.options = self.options  # globals!!!
+
+    def tearDown(self):
+        cli.OptionParser = self.original_parser
+
+    # Show long diffs in error output...
+    maxDiff = None
+
+    @mock.patch('koji_cli.activate_session')
+    def test_handle_save_failed_tree_simple(self, activate_session_mock):
+        # koji save-failed-tree 123456
+        task_id = 123456
+        broot_id = 321
+        arguments = [task_id]
+        options = mock.MagicMock()
+        options.full = False
+        options.nowait = True
+        self.parser.parse_args.return_value = [options, arguments]
+        self.session.getAPIVersion.return_value = koji.API_VERSION
+        self.session.listBuildroots.return_value = [{'id': 321}]
+
+        # Mock out the xmlrpc server
+        self.session.saveFailedTree.return_value = 123
+
+        # Run it and check immediate output
+        cli.handle_save_failed_tree(self.options, self.session, self.args)
+
+        # Finally, assert that things were called as we expected.
+        activate_session_mock.assert_called_once_with(self.session)
+        self.session.listBuildroots.assert_called_once_with(taskID=task_id)
+        self.session.saveFailedTree.assert_called_once_with(broot_id, options.full)
+
+    @mock.patch('koji_cli.activate_session')
+    def test_handle_save_failed_tree_buildroots(self, activate_session_mock):
+        # koji save-failed-tree --buildroot 123456
+        broot_id = 321
+        arguments = [broot_id]
+        options = mock.MagicMock()
+        options.full = False
+        options.nowait = True
+        options.mode = "buildroot"
+        self.parser.parse_args.return_value = [options, arguments]
+        self.session.getAPIVersion.return_value = koji.API_VERSION
+        self.session.listBuildroots.return_value = [{'id': 321}]
+
+        # Mock out the xmlrpc server
+        self.session.saveFailedTree.return_value = 123
+
+        # Run it and check immediate output
+        cli.handle_save_failed_tree(self.options, self.session, self.args)
+
+        # Finally, assert that things were called as we expected.
+        activate_session_mock.assert_called_once_with(self.session)
+        self.session.listBuildroots.assert_not_called()
+        self.session.saveFailedTree.assert_called_once_with(broot_id, options.full)
+
+
+    @mock.patch('koji_cli.activate_session')
+    def test_handle_save_failed_tree_full(self, activate_session_mock):
+        # koji save-failed-tree 123456 --full
+        task_id = 123456
+        broot_id = 321
+        arguments = [task_id]
+        options = mock.MagicMock()
+        options.full = True
+        options.nowait = True
+        self.parser.parse_args.return_value = [options, arguments]
+        self.session.getAPIVersion.return_value = koji.API_VERSION
+        self.session.listBuildroots.return_value = [{'id': 321}]
+
+        # Mock out the xmlrpc server
+        self.session.saveFailedTree.return_value = 123
+
+        # Run it and check immediate output
+        cli.handle_save_failed_tree(self.options, self.session, self.args)
+
+        # Finally, assert that things were called as we expected.
+        activate_session_mock.assert_called_once_with(self.session)
+        self.session.listBuildroots.assert_called_once_with(taskID=task_id)
+        self.session.saveFailedTree.assert_called_once_with(broot_id, options.full)
+
+    @mock.patch('koji_cli.activate_session')
+    @mock.patch('koji_cli.watch_tasks')
+    def test_handle_save_failed_tree_wait(self, watch_tasks_mock, activate_session_mock):
+        # koji save-failed-tree 123456 --full
+        task_id = 123456
+        broot_id = 321
+        arguments = [task_id]
+        options = mock.MagicMock()
+        options.full = True
+        options.nowait = False
+        options.quiet = False
+        self.parser.parse_args.return_value = [options, arguments]
+        self.session.getAPIVersion.return_value = koji.API_VERSION
+        self.session.listBuildroots.return_value = [{'id': 321}]
+
+        # Mock out the xmlrpc server
+        spawned_id = 123
+        self.session.saveFailedTree.return_value = spawned_id
+
+        # Run it and check immediate output
+        cli.handle_save_failed_tree(self.options, self.session, self.args)
+
+        # Finally, assert that things were called as we expected.
+        self.session.listBuildroots.assert_called_once_with(taskID=task_id)
+        self.session.saveFailedTree.assert_called_once_with(broot_id, options.full)
+        activate_session_mock.assert_called_once_with(self.session)
+        self.session.logout.assert_called_once_with()
+        watch_tasks_mock.assert_called_once_with(self.session, [spawned_id],
+                                                 quiet=options.quiet)
+
+    @mock.patch('sys.stdout', new_callable=StringIO.StringIO)
+    @mock.patch('koji_cli.activate_session')
+    @mock.patch('koji_cli.watch_tasks')
+    def test_handle_save_failed_tree_errors(self, watch_tasks_mock, activate_session_mock, stdout):
+        # koji save-failed-tree 123 456
+        arguments = [123, 456]
+        options = mock.MagicMock()
+        self.parser.parse_args.return_value = [options, arguments]
+        self.parser.error.side_effect = Exception()
+        self.session.getAPIVersion.return_value = koji.API_VERSION
+        self.session.listBuildroots.return_value = [{'id': 321}]
+
+        self.assertRaises(Exception, cli.handle_save_failed_tree,
+                          self.options, self.session, self.args)
+
+        arguments = ["text"]
+        self.parser.parse_args.return_value = [options, arguments]
+        self.assertRaises(Exception, cli.handle_save_failed_tree, self.options,
+                          self.session, self.args)
+        cli.logger = mock.MagicMock()
+
+        # plugin not installed
+        arguments = [123]
+        self.parser.parse_args.return_value = [options, arguments]
+        self.session.saveFailedTree.side_effect = koji.GenericError("Invalid method")
+        cli.handle_save_failed_tree(self.options, self.session, self.args)
+        actual = stdout.getvalue()
+        self.assertTrue('The save_failed_tree plugin appears to not be installed' in actual)
+
+        # Task which is not FAILED, disabled in config, wrong owner
+        self.session.saveFailedTree.side_effect = koji.PreBuildError('placeholder')
+        with self.assertRaises(koji.PreBuildError) as cm:
+            cli.handle_save_failed_tree(self.options, self.session, self.args)
+        e = cm.exception
+        self.assertEqual(e, self.session.saveFailedTree.side_effect)

tests/test_plugins/test_save_failed_tree_builder.py

@@ -0,0 +1,116 @@
+import mock
+import os
+import sys
+import unittest
+
+# alter pythonpath to not load hub plugin
+sys.path = [os.path.join(os.path.dirname(__file__), '../../plugins/builder')] + sys.path
+#raise(Exception(sys.path))
+
+import koji
+# inject builder data
+from tests.test_builder.loadkojid import kojid
+import __main__
+__main__.BuildRoot = kojid.BuildRoot
+
+from save_failed_tree import SaveFailedTreeTask
+
+class TestSaveFailedTree(unittest.TestCase):
+    def setUp(self):
+        self.session = mock.MagicMock()
+        self.session.host.getHost.return_value = {'id': 1}
+        options = mock.MagicMock()
+        options.workdir = '/tmp/nonexistentdirectory'
+        options.mockdir = '/tmp/mockdir'
+        options.name = 'name'
+        self.t = SaveFailedTreeTask(123, 'saveFailedTree', {}, self.session, options)
+
+    @mock.patch('os.unlink')
+    @mock.patch('tarfile.open')
+    def testNonExistentBuildroot(self, tarfile, os_unlink):
+        tfile = mock.MagicMock(name='tfile')
+        tarfile.return_value = tfile
+        self.session.getBuildroot.return_value = None
+
+        with self.assertRaises(koji.GenericError) as cm:
+            self.t.handler(1)
+        self.assertTrue('Nonexistent buildroot' in str(cm.exception))
+
+        tarfile.assert_not_called()
+        tfile.add.assert_not_called()
+        tfile.close.assert_not_called()
+        os_unlink.assert_not_called()
+
+    @mock.patch('os.path.exists')
+    @mock.patch('os.unlink')
+    @mock.patch('tarfile.open')
+    def testCorrect(self, tarfile, os_unlink, os_exists):
+        def getBuildroot(bid):
+            tmp = {
+                'tag_name': 'tag_name',
+                'repo_id': 'repo_id',
+                'host_id': 1,
+            }
+            if bid == 1:
+                tmp['id'] = 1
+                tmp['task_id'] = 1000
+                tmp['arch'] = 'x86_64'
+                tmp['tag_id'] = 5000
+            elif bid == 2:
+                tmp['id'] = 2
+                tmp['task_id'] = 1001
+                tmp['arch'] = 'i386'
+                tmp['tag_id'] = 5001
+            return tmp
+
+        self.session.getBuildroot.side_effect = getBuildroot
+        tfile = mock.MagicMock(name='tfile')
+        tfile.add = mock.MagicMock()
+        tarfile.return_value = tfile
+        os_exists.return_value = True
+
+        self.t.handler(1)
+
+        tarfile.assert_called_once_with(
+            '/tmp/nonexistentdirectory/tasks/123/123/broot-1.tar.gz',
+            'w:gz'
+        )
+
+        tfile.add.assert_called_once()
+        self.assertEqual(tfile.add.call_args_list[0][0][0], '/tmp/mockdir/tag_name-1-repo_id/root/builddir')
+        tfile.close.assert_called_once_with()
+        os_unlink.assert_called_once_with('/tmp/nonexistentdirectory/tasks/123/123/broot-1.tar.gz')
+
+    @mock.patch('os.unlink')
+    @mock.patch('tarfile.open')
+    def testWrongBuilder(self, tarfile, os_unlink):
+        def getBuildroot(bid):
+            tmp = {
+                'tag_name': 'tag_name',
+                'repo_id': 'repo_id',
+                'host_id': 2000,
+            }
+            if bid == 1:
+                tmp['id'] = 1
+                tmp['task_id'] = 1000
+                tmp['arch'] = 'x86_64'
+                tmp['tag_id'] = 5000
+            elif bid == 2:
+                tmp['id'] = 2
+                tmp['task_id'] = 1001
+                tmp['arch'] = 'i386'
+                tmp['tag_id'] = 5001
+            return tmp
+
+        self.session.getBuildroot.side_effect = getBuildroot
+        tfile = mock.MagicMock(name='tfile')
+        tarfile.return_value = tfile
+
+        with self.assertRaises(koji.GenericError):
+            self.t.handler(1)
+
+    def testFull(self):
+        pass
+
+    def testFailUpload(self):
+        pass