diff --git a/builder/kojid b/builder/kojid index b4e72b58..3178fd39 100755 --- a/builder/kojid +++ b/builder/kojid @@ -1102,6 +1102,8 @@ class BuildTask(BaseTaskHandler): % (data['name'], target_info['dest_tag_name'])) # TODO - more pre tests archlist = self.getArchList(build_tag, h, extra=extra_arches) + # pass draft option in + data['draft'] = opts.get('draft') # let the system know about the build we're attempting if not self.opts.get('scratch'): # scratch builds do not get imported @@ -2176,6 +2178,8 @@ class WrapperRPMTask(BaseBuildTask): data['extra'] = {'source': {'original_url': source['url']}} if opts.get('custom_user_metadata'): data['extra']['custom_user_metadata'] = opts['custom_user_metadata'] + # pass draft option in + data['draft'] = opts.get('draft') self.logger.info("Reading package config for %(name)s" % data) pkg_cfg = self.session.getPackageConfig(build_target['dest_tag'], data['name']) if not opts.get('skip_tag'): @@ -5249,6 +5253,7 @@ Subject: %(nvr)s %(result)s %(operation)s by %(user_name)s\r To: %(to_addrs)s\r X-Koji-Package: %(pkg_name)s\r X-Koji-NVR: %(nvr)s\r +X-Koji-Draft: %(draft)s\r X-Koji-User: %(user_name)s\r X-Koji-Status: %(status)s\r %(tag_headers)s\r @@ -5278,6 +5283,7 @@ Status: %(status)s\r user = self.session.getUser(user_info) pkg_name = build['package_name'] nvr = koji.buildLabel(build) + draft = build.get('draft', False) user_name = user['name'] from_addr = self.options.from_addr @@ -5349,6 +5355,7 @@ X-Koji-Tag: %(dest_tag)s\r X-Koji-Package: %(build_pkg_name)s\r X-Koji-Builder: %(build_owner)s\r X-Koji-Status: %(status)s\r +X-Koji-Draft: %(draft)s\r \r Package: %(build_nvr)s\r Tag: %(dest_tag)s\r @@ -5448,6 +5455,7 @@ Build Info: %(weburl)s/buildinfo?buildID=%(build_id)i\r build_nvr = koji.buildLabel(build) build_id = build['id'] build_owner = build['owner_name'] + draft = build.get('draft', False) # target comes from session.py:_get_build_target() dest_tag = None if target is not None: diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index 9c52358d..5da17703 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -580,6 +580,8 @@ def handle_build(options, session, args): parser.add_option("--custom-user-metadata", type="str", help="Provide a JSON string of custom metadata to be deserialized and " "stored under the build's extra.custom_user_metadata field") + parser.add_option("--draft", action="store_true", + help="Build draft build instead") (build_opts, args) = parser.parse_args(args) if len(args) != 2: parser.error("Exactly two arguments (a build target and a SCM URL or srpm file) are " @@ -588,6 +590,8 @@ def handle_build(options, session, args): parser.error("--no-/rebuild-srpm is only allowed for --scratch builds") if build_opts.arch_override and not build_opts.scratch: parser.error("--arch_override is only allowed for --scratch builds") + if build_opts.scratch and build_opts.draft: + parser.error("--scratch and --draft cannot be both specfied") custom_user_metadata = {} if build_opts.custom_user_metadata: try: @@ -618,7 +622,7 @@ def handle_build(options, session, args): if build_opts.arch_override: opts['arch_override'] = koji.parse_arches(build_opts.arch_override) for key in ('skip_tag', 'scratch', 'repo_id', 'fail_fast', 'wait_repo', 'wait_builds', - 'rebuild_srpm'): + 'rebuild_srpm', 'draft'): val = getattr(build_opts, key) if val is not None: opts[key] = val @@ -830,6 +834,8 @@ def handle_wrapper_rpm(options, session, args): parser.add_option("--nowait", action="store_false", dest="wait", help="Don't wait on build") parser.add_option("--background", action="store_true", help="Run the build at a lower priority") + parser.add_option("--draft", action="store_true", + help="Build draft build instead") (build_opts, args) = parser.parse_args(args) if build_opts.inis: @@ -839,6 +845,8 @@ def handle_wrapper_rpm(options, session, args): if len(args) < 3: parser.error("You must provide a build target, a build ID or NVR, " "and a SCM URL to a specfile fragment") + if build_opts.scratch and build_opts.draft: + parser.error("--scratch and --draft cannot be both specfied") activate_session(session, options) target = args[0] @@ -874,6 +882,8 @@ def handle_wrapper_rpm(options, session, args): opts['skip_tag'] = True if build_opts.scratch: opts['scratch'] = True + if build_opts.draft: + opts['draft'] = True task_id = session.wrapperRPM(build_id, url, target, priority, opts=opts) print("Created task: %d" % task_id) print("Task info: %s/taskinfo?taskID=%s" % (options.weburl, task_id)) @@ -1314,6 +1324,10 @@ def handle_import(goptions, session, args): parser.add_option("--test", action="store_true", help="Don't actually import") parser.add_option("--create-build", action="store_true", help="Auto-create builds as needed") parser.add_option("--src-epoch", help="When auto-creating builds, use this epoch") + parser.add_option("--create-draft", action="store_true", + help="Auto-create draft builds instead as needed") + parser.add_option("--draft-build", metavar='NVR|ID', + help="The target draft build to import to") (options, args) = parser.parse_args(args) if len(args) < 1: parser.error("At least one package must be specified") @@ -1324,6 +1338,35 @@ def handle_import(goptions, session, args): options.src_epoch = int(options.src_epoch) except (ValueError, TypeError): parser.error("Invalid value for epoch: %s" % options.src_epoch) + if options.create_draft: + print("Will create draft build instead if desired nvr doesn't exist") + options.create_build = True + draft_build = None + draft_target_nvr = None + if options.draft_build: + if options.create_build: + parser.error( + "To ensure no misuse, don't specify --draft-build while auto-creating." + ) + try: + draft_build = int(options.draft_build) + except ValueError: + draft_build = options.draft_build + draft_build = session.getBuild(draft_build) + if not draft_build: + error("No such build: %s" % options.draft_build) + if not draft_build.get('draft'): + error("%s is not a draft build" % draft_build['nvr']) + b_state = koji.BUILD_STATES[draft_build['state']] + if b_state != 'COMPLETE': + error("draft build %s is expected as COMPLETE, got %s" % (draft_build['nvr'], b_state)) + target_release = draft_build.get('extra', {}).get('draft', {}).get('target_release') + if not target_release: + error("Invalid draft build: %s," + " no draft.target_release found in extra" % draft_build['nvr']) + draft_target_nvr = "%s-%s-%s" % ( + draft_build['name'], draft_build['version'], target_release + ) activate_session(session, goptions) to_import = {} for path in args: @@ -1335,6 +1378,7 @@ def handle_import(goptions, session, args): else: nvr = "%(name)s-%(version)s-%(release)s" % koji.parse_NVRA(data['sourcerpm']) to_import.setdefault(nvr, []).append((path, data)) + builds_missing = False nvrs = sorted(to_list(to_import.keys())) for nvr in nvrs: @@ -1343,29 +1387,41 @@ def handle_import(goptions, session, args): if data['sourcepackage']: break else: + if nvr == draft_target_nvr: + continue + # when no srpm and create_draft + elif options.create_draft: + print("Missing srpm for draft build creating with target nvr: %s" % nvr) + builds_missing = True + continue # no srpm included, check for build binfo = session.getBuild(nvr) if not binfo: print("Missing build or srpm: %s" % nvr) builds_missing = True - if builds_missing and not options.create_build: + if builds_missing and (not options.create_build or options.create_draft): print("Aborting import") return # local function to help us out below - def do_import(path, data): + def do_import(path, data, draft_build=None): + draft = bool(draft_build) or options.create_draft rinfo = dict([(k, data[k]) for k in ('name', 'version', 'release', 'arch')]) - prev = session.getRPM(rinfo) - if prev and not prev.get('external_repo_id', 0): - if prev['payloadhash'] == koji.hex_string(data['sigmd5']): - print("RPM already imported: %s" % path) - else: - warn("md5sum mismatch for %s" % path) - warn(" A different rpm with the same name has already been imported") - warn(" Existing sigmd5 is %r, your import has %r" % ( - prev['payloadhash'], koji.hex_string(data['sigmd5']))) - print("Skipping import") - return + if draft_build or not options.create_draft: + opts = {} + if draft_build: + opts['build'] = draft_build + prev = session.getRPM(rinfo, **opts) + if prev and not prev.get('external_repo_id', 0): + if prev['payloadhash'] == koji.hex_string(data['sigmd5']): + print("RPM already imported: %s" % path) + else: + warn("md5sum mismatch for %s" % path) + warn(" A different rpm with the same name has already been imported") + warn(" Existing sigmd5 is %r, your import has %r" % ( + prev['payloadhash'], koji.hex_string(data['sigmd5']))) + print("Skipping import") + return if options.test: print("Test mode -- skipping import for %s" % path) return @@ -1381,41 +1437,69 @@ def handle_import(goptions, session, args): sys.stdout.write("importing %s... " % path) sys.stdout.flush() try: - session.importRPM(serverdir, os.path.basename(path)) + opts = {} + if draft_build: + opts['build'] = draft_build + if draft: + opts['draft'] = draft + rpminfo = session.importRPM(serverdir, os.path.basename(path), **opts) except koji.GenericError as e: + rpminfo = None print("\nError importing: %s" % str(e).splitlines()[-1]) sys.stdout.flush() else: print("done") sys.stdout.flush() + return rpminfo for nvr in nvrs: # check for existing build need_build = True - binfo = session.getBuild(nvr) - if binfo: - b_state = koji.BUILD_STATES[binfo['state']] - if b_state == 'COMPLETE': - need_build = False - elif b_state in ['FAILED', 'CANCELED']: - if not options.create_build: - print("Build %s state is %s. Skipping import" % (nvr, b_state)) + is_draft = True + if nvr == draft_target_nvr: + binfo = draft_build + need_build = False + elif options.create_draft: + binfo = None + need_build = True + else: + is_draft = False + binfo = session.getBuild(nvr) + if binfo: + b_state = koji.BUILD_STATES[binfo['state']] + if b_state == 'COMPLETE': + need_build = False + elif b_state in ['FAILED', 'CANCELED']: + if not options.create_build: + print("Build %s state is %s. Skipping import" % (nvr, b_state)) + continue + else: + print("Build %s exists with state=%s. Skipping import" % (nvr, b_state)) continue - else: - print("Build %s exists with state=%s. Skipping import" % (nvr, b_state)) - continue # import srpms first, if any for path, data in to_import[nvr]: if data['sourcepackage']: - if binfo and b_state != 'COMPLETE': + # we can not fix state for draft build by createEmptyBuild + if not is_draft and binfo and b_state != 'COMPLETE': # need to fix the state print("Creating empty build: %s" % nvr) b_data = koji.util.dslice(binfo, ['name', 'version', 'release']) b_data['epoch'] = data['epoch'] session.createEmptyBuild(**b_data) binfo = session.getBuild(nvr) - do_import(path, data) + dbld = binfo if is_draft else None + will_create = False + if options.create_draft and not dbld: + will_create = True + print("Will create draft build with target nvr: %s while importing" % nvr) + rpminfo = do_import(path, data, draft_build=dbld) + # only needed for draft build, but + # TODO: should be able to apply to regular import + if rpminfo and rpminfo.get('build', {}).get('draft'): + binfo = rpminfo['build'] + if will_create: + print("Draft build: %s created" % binfo['nvr']) need_build = False if need_build: @@ -1424,11 +1508,11 @@ def handle_import(goptions, session, args): if binfo: # should have caught this earlier, but just in case... b_state = koji.BUILD_STATES[binfo['state']] - print("Build %s state is %s. Skipping import" % (nvr, b_state)) + print("Build %s state is %s. Skipping import" % (binfo['nvr'], b_state)) continue else: print("No such build: %s (include matching srpm or use " - "--create-build option to add it)" % nvr) + "--create-build/--create-draft option to add it)" % nvr) continue else: # let's make a new build @@ -1439,17 +1523,22 @@ def handle_import(goptions, session, args): # pull epoch from first rpm data = to_import[nvr][0][1] b_data['epoch'] = data['epoch'] - if options.test: - print("Test mode -- would have created empty build: %s" % nvr) + if options.create_draft: + b_data['draft'] = True + b_display = "empty draft build with target nvr: %s" % nvr else: - print("Creating empty build: %s" % nvr) - session.createEmptyBuild(**b_data) - binfo = session.getBuild(nvr) + b_display = "empty build: %s" % nvr + if options.test: + print("Test mode -- would have created %s" % b_display) + else: + print("Creating %s" % b_display) + buildid = session.createEmptyBuild(**b_data) + binfo = session.getBuild(buildid) for path, data in to_import[nvr]: if data['sourcepackage']: continue - do_import(path, data) + do_import(path, data, draft_build=binfo if is_draft else None) def handle_import_cg(goptions, session, args): @@ -2727,11 +2816,15 @@ def anon_handle_list_tagged(goptions, session, args): parser.add_option("--ts", type='int', metavar="TIMESTAMP", help="query at last event before timestamp") parser.add_option("--repo", type='int', metavar="REPO#", help="query at event for a repo") + parser.add_option("--draft-only", action="store_true", help="Only list draft builds/rpms") + parser.add_option("--no-draft", action="store_true", help="Only list regular builds/rpms") (options, args) = parser.parse_args(args) if len(args) == 0: parser.error("A tag name must be specified") elif len(args) > 2: parser.error("Only one package name may be specified") + if options.no_draft and options.draft_only: + parser.error("--draft-only conflicts with --no-draft") ensure_connection(session, goptions) pathinfo = koji.PathInfo() package = None @@ -2753,6 +2846,10 @@ def anon_handle_list_tagged(goptions, session, args): options.rpms = True if options.type: opts['type'] = options.type + elif options.no_draft: + opts['draft'] = koji.FLAG_REGULAR_BUILD + elif options.draft_only: + opts['draft'] = koji.FLAG_DRAFT_BUILD event = koji.util.eventFromOpts(session, options) event_id = None if event: @@ -2798,7 +2895,9 @@ def anon_handle_list_tagged(goptions, session, args): fmt = "%(path)s" data = [x for x in data if 'path' in x] else: - fmt = "%(name)s-%(version)s-%(release)s.%(arch)s" + fmt = "%(name)s-%(version)s-%(release)s.%(arch)s%(draft_suffix)s" + for x in data: + x['draft_suffix'] = (' (#draft_%s)' % x['build_id']) if x.get('draft') else '' if options.sigs: fmt = "%(sigkey)s " + fmt else: @@ -2861,10 +2960,13 @@ def anon_handle_list_buildroot(goptions, session, args): fmt = "%(nvr)s.%(arch)s" order = sorted([(fmt % x, x) for x in list_rpms]) for nvra, rinfo in order: - if options.verbose and rinfo.get('is_update'): - print("%s [update]" % nvra) - else: - print(nvra) + line = nvra + if options.verbose: + if rinfo.get('draft'): + line += " (#draft_%s)" % rinfo['build_id'] + if rinfo.get('is_update'): + line += " [update]" + print(line) list_archives = session.listArchives(**opts) if list_archives: @@ -3403,6 +3505,8 @@ def anon_handle_list_builds(goptions, session, args): parser.add_option("--source", help="Only builds where the source field matches (glob pattern)") parser.add_option("--owner", help="List builds built by this owner") parser.add_option("--volume", help="List builds by volume ID") + parser.add_option("--draft-only", action="store_true", help="Only list draft builds") + parser.add_option("--no-draft", action="store_true", help="Only list regular builds") parser.add_option("-k", "--sort-key", action="append", metavar='FIELD', default=[], help="Sort the list by the named field. Allowed sort keys: " "build_id, owner_name, state") @@ -3419,6 +3523,12 @@ def anon_handle_list_builds(goptions, session, args): value = getattr(options, key) if value is not None: opts[key] = value + if options.no_draft and options.draft_only: + parser.error("--draft-only conflits with --no-draft") + elif options.no_draft: + opts['draft'] = koji.FLAG_REGULAR_BUILD + elif options.draft_only: + opts['draft'] = koji.FLAG_DRAFT_BUILD if options.cg: opts['cgID'] = options.cg if options.package: @@ -3520,13 +3630,24 @@ def anon_handle_rpminfo(goptions, session, args): parser = OptionParser(usage=get_usage_str(usage)) parser.add_option("--buildroots", action="store_true", help="show buildroots the rpm was used in") + parser.add_option("--build", metavar="NVR|ID", + help="show the rpm(s) in the build") + (options, args) = parser.parse_args(args) if len(args) < 1: parser.error("Please specify an RPM") + opts = {} + build = options.build + if options.build: + try: + build = int(build) + except ValueError: + pass + opts['build'] = build ensure_connection(session, goptions) error_hit = False for rpm in args: - info = session.getRPM(rpm) + info = session.getRPM(rpm, **opts) if info is None: warn("No such rpm: %s\n" % rpm) error_hit = True @@ -3536,24 +3657,29 @@ def anon_handle_rpminfo(goptions, session, args): else: info['epoch'] = str(info['epoch']) + ":" if not info.get('external_repo_id', 0): - buildinfo = session.getBuild(info['build_id']) - buildinfo['name'] = buildinfo['package_name'] - buildinfo['arch'] = 'src' - if buildinfo['epoch'] is None: - buildinfo['epoch'] = "" + if info['arch'] == 'src': + srpminfo = info.copy() else: - buildinfo['epoch'] = str(buildinfo['epoch']) + ":" + srpminfo = session.listRPMs(buildID=info['build_id'], arches='src')[0] + if srpminfo['epoch'] is None: + srpminfo['epoch'] = "" + else: + srpminfo['epoch'] = str(srpminfo['epoch']) + ":" + buildinfo = session.getBuild(info['build_id']) print("RPM: %(epoch)s%(name)s-%(version)s-%(release)s.%(arch)s [%(id)d]" % info) + if info.get('draft'): + print("Draft: YES") if info.get('external_repo_id'): repo = session.getExternalRepo(info['external_repo_id']) print("External Repository: %(name)s [%(id)i]" % repo) print("External Repository url: %(url)s" % repo) else: + print("Build: %(nvr)s [%(id)d]" % buildinfo) print("RPM Path: %s" % os.path.join(koji.pathinfo.build(buildinfo), koji.pathinfo.rpm(info))) - print("SRPM: %(epoch)s%(name)s-%(version)s-%(release)s [%(id)d]" % buildinfo) + print("SRPM: %(epoch)s%(name)s-%(version)s-%(release)s [%(id)d]" % srpminfo) print("SRPM Path: %s" % - os.path.join(koji.pathinfo.build(buildinfo), koji.pathinfo.rpm(buildinfo))) + os.path.join(koji.pathinfo.build(buildinfo), koji.pathinfo.rpm(srpminfo))) print("Built: %s" % time.strftime('%a, %d %b %Y %H:%M:%S %Z', time.localtime(info['buildtime']))) print("SIGMD5: %(payloadhash)s" % info) @@ -3563,6 +3689,7 @@ def anon_handle_rpminfo(goptions, session, args): headers=["license"]) if 'license' in headers: print("License: %(license)s" % headers) + # kept for backward compatibility print("Build ID: %(build_id)s" % info) if info['buildroot_id'] is None: print("No buildroot data available") @@ -3619,6 +3746,8 @@ def anon_handle_buildinfo(goptions, session, args): info['arch'] = 'src' info['state'] = koji.BUILD_STATES[info['state']] print("BUILD: %(name)s-%(version)s-%(release)s [%(id)d]" % info) + if info.get('draft'): + print("Draft: YES") print("State: %(state)s" % info) if info['state'] == 'BUILDING': print("Reserved by: %(cg_name)s" % info) diff --git a/cli/koji_cli/lib.py b/cli/koji_cli/lib.py index c605940f..b3a0812e 100644 --- a/cli/koji_cli/lib.py +++ b/cli/koji_cli/lib.py @@ -925,3 +925,10 @@ class DatetimeJSONEncoder(json.JSONEncoder): if isinstance(o, xmlrpc_client.DateTime): return str(o) return json.JSONEncoder.default(self, o) + + +def yesno(x): + if x: + return 'Y' + else: + return 'N' diff --git a/koji/__init__.py b/koji/__init__.py index 11f6cba4..b0b372e0 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -27,6 +27,7 @@ from __future__ import absolute_import, division import base64 import datetime import errno +import functools import hashlib import json import logging @@ -274,6 +275,7 @@ TAG_UPDATE_TYPES = Enum(( 'VOLUME_CHANGE', 'IMPORT', 'MANUAL', + 'DRAFT_PROMOTION', )) # BEGIN kojikamid dup # @@ -295,6 +297,44 @@ PRIO_DEFAULT = 20 DEFAULT_REQUEST_TIMEOUT = 60 * 60 * 12 DEFAULT_AUTH_TIMEOUT = 60 +# draft release format +DRAFT_RELEASE_FORMAT = '{release}#draft_{id}' + +FLAG_DRAFT_BUILD = 1 +FLAG_REGULAR_BUILD = 2 +FLAG_ALL_BUILD = FLAG_DRAFT_BUILD | FLAG_REGULAR_BUILD + +if six.PY3: + from enum import IntFlag + + # draft build bit FLAGs + class DRAFT_FLAG(IntFlag): + + DRAFT = FLAG_DRAFT_BUILD + REGULAR = FLAG_REGULAR_BUILD + ALL = FLAG_ALL_BUILD + + @classmethod + def _missing_(cls, value): + if not isinstance(value, int): + raise ValueError("%r is not a valid %s" % (value, cls.__name__)) + # diable member creation and negative integer + return cls._value2member_map_.get(value) + + def convert_draft_option(func=None, kw='draft'): + def wrapper(func): + @functools.wraps(func) + def convert(*args, **kwargs): + if kw in kwargs: + kwargs[kw] = DRAFT_FLAG(kwargs[kw]) + return func(*args, **kwargs) + return convert + if func is None: + return wrapper + else: + return wrapper(func) + + # BEGIN kojikamid dup # # Exceptions @@ -2363,6 +2403,39 @@ class PathInfo(object): return self.volumedir(build.get('volume_name')) + \ ("/packages/%(name)s/%(version)s/%(release)s" % build) + def to_buildinfo(self, path): + """Revert build dir path (/[/]packages///) + to build map with the data below: + + - name + - version + - release + - volume_name + """ + path = path.rstrip('/') + parts = path.rsplit('/', 4) + if len(parts) != 5: + raise GenericError("Invalid build path: %s" % path) + if parts[-4] != 'packages': + raise GenericError("Invalid build path: %s, packages subdir not found" % path) + nvr = parts[-3:] + rest = parts[0] + if not rest.startswith(self.topdir): + raise GenericError("Invalid build path: %s, topdir is not %r" % (path, self.topdir)) + vol_part = rest[len(self.topdir):] + if not vol_part: + vol = 'DEFAULT' + elif vol_part.startswith('/vol/'): + vol = vol_part[5:] + else: + raise GenericError( + "Invalid build path: %s, volume dir: %s is incorrect" % (path, vol_part) + ) + return {'name': nvr[0], + 'version': nvr[1], + 'release': nvr[2], + 'volume_name': vol} + def mavenbuild(self, build): """Return the directory where the Maven build exists in the global store (/mnt/koji/packages)""" @@ -2410,6 +2483,27 @@ class PathInfo(object): """Return the path (relative to build_dir) where an rpm belongs""" return "%(arch)s/%(name)s-%(version)s-%(release)s.%(arch)s.rpm" % rpminfo + def to_rpminfo(self, path, full=False): + """Revert rpm path in build dir to rpm nvra map with/without build dir""" + if path.endswith('/'): + raise GenericError("Invalid path: %s, cannot be a directory" % path) + parts = path.rsplit('/', 2) + if full: + if len(parts) != 3: + raise GenericError("No build dir in path: %s" % path) + return self.to_buildinfo(parts[-3]), self._to_rpminfo(parts[-2:]) + return self._to_rpminfo(parts) + + def _to_rpminfo(self, parts): + if len(parts) != 2 or not parts[0]: + raise GenericError("Invalid path: %s" % '/'.join(parts)) + rpminfo = parse_NVRA(parts[-1]) + if parts[0] != rpminfo['arch']: + raise GenericError( + 'mismatch between arch dir (%s) and arch (%s) in rpm' % (parts[0], rpminfo) + ) + return rpminfo + def signed(self, rpminfo, sigkey): """Return the path (relative to build dir) where a signed rpm lives""" return "data/signed/%s/" % sigkey + self.rpm(rpminfo) diff --git a/koji/plugin.py b/koji/plugin.py index 6306e978..89c0863b 100644 --- a/koji/plugin.py +++ b/koji/plugin.py @@ -60,6 +60,8 @@ callbacks = { 'postRepoInit': [], 'preRepoDone': [], 'postRepoDone': [], + 'preBuildPromote': [], + 'postBuildPromote': [], 'preCommit': [], 'postCommit': [], # builder diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 79ae4136..81d71e2e 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -55,6 +55,7 @@ import rpm from psycopg2._psycopg import IntegrityError import koji +from koji import DRAFT_FLAG, convert_draft_option import koji.plugin import koji.policy import koji.rpmdiff @@ -68,6 +69,7 @@ from koji.util import ( base64encode, decode_bytes, dslice, + encode_datetime, extract_build_task, joinpath, md5_constructor, @@ -1393,8 +1395,9 @@ def list_tags(build=None, package=None, perms=True, queryOpts=None, pattern=None return query.iterate() +@convert_draft_option def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, owner=None, - type=None, extra=False): + type=None, extra=False, *, draft=DRAFT_FLAG.ALL): """Returns a list of builds for specified tag :param int tag: tag ID @@ -1406,6 +1409,10 @@ def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, :param str type: restrict the list to builds of the given type. Currently the supported types are 'maven', 'win', 'image', or any custom content generator btypes. :param bool extra: Set to "True" to get the build extra info + :param DRAFT_FLAG draft: bit flag(enum.IntFlag) indicates + - DRAFT(1): draft only + - REGULAR(2): regular only + - ALL(3): both draft and regular builds :returns [dict]: list of buildinfo dicts """ # build - id pkg_id version release epoch @@ -1432,6 +1439,7 @@ def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, ('build.completion_time', 'completion_time'), ('build.start_time', 'start_time'), ('build.task_id', 'task_id'), + ('build.draft', 'draft'), ('users.id', 'owner_id'), ('users.name', 'owner_name'), ('events.id', 'creation_event_id'), ('events.time', 'creation_time'), ('volume.id', 'volume_id'), ('volume.name', 'volume_name'), @@ -1477,6 +1485,7 @@ def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, clauses.append('package.name = %(package)s') if owner: clauses.append('users.name = %(owner)s') + append_draft_clause(draft, clauses) queryOpts = {'order': '-create_event'} # latest first if extra: fields.append(('build.extra', 'extra')) @@ -1513,8 +1522,9 @@ def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, return builds +@convert_draft_option def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, latest=True, - rpmsigs=False, owner=None, type=None, extra=True): + rpmsigs=False, owner=None, type=None, extra=True, *, draft=DRAFT_FLAG.ALL): """Returns a list of rpms and builds for specified tag :param int|str tag: The tag name or ID to search @@ -1534,6 +1544,10 @@ def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, late :param str type: Filter by build type. Supported types are 'maven', 'win', and 'image'. :param bool extra: Set to "False" to skip the rpm extra info + :param DRAFT_FLAG draft: bit flag(enum.IntFlag) indicates + - DRAFT(1): draft only + - REGULAR(2): regular only + - ALL(3): both draft and regular builds :returns: a two-element list. The first element is the list of RPMs, and the second element is the list of builds. """ @@ -1544,7 +1558,7 @@ def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, late taglist += [link['parent_id'] for link in readFullInheritance(tag, event)] builds = readTaggedBuilds(tag, event=event, inherit=inherit, latest=latest, package=package, - owner=owner, type=type) + owner=owner, type=type, draft=draft) # index builds build_idx = dict([(b['build_id'], b) for b in builds]) @@ -1555,6 +1569,7 @@ def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, late ('rpminfo.arch', 'arch'), ('rpminfo.id', 'id'), ('rpminfo.epoch', 'epoch'), + ('rpminfo.draft', 'draft'), ('rpminfo.payloadhash', 'payloadhash'), ('rpminfo.size', 'size'), ('rpminfo.buildtime', 'buildtime'), @@ -1582,6 +1597,7 @@ def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, late clauses.append('rpminfo.arch IN %(arch)s') else: raise koji.GenericError('Invalid type for arch option: %s' % builtins.type(arch)) + append_draft_clause(draft, clauses, table='rpminfo') if extra: fields.append(('rpminfo.extra', 'extra')) @@ -4357,6 +4373,7 @@ def get_build(buildInfo, strict=False): release epoch nvr + draft: Whether the build is draft or not state task_id: ID of the task that kicked off the build owner_id: ID of the user who kicked off the build @@ -4392,7 +4409,9 @@ def get_build(buildInfo, strict=False): fields = (('build.id', 'id'), ('build.version', 'version'), ('build.release', 'release'), ('build.id', 'build_id'), - ('build.epoch', 'epoch'), ('build.state', 'state'), + ('build.epoch', 'epoch'), + ('build.draft', 'draft'), + ('build.state', 'state'), ('build.completion_time', 'completion_time'), ('build.start_time', 'start_time'), ('build.task_id', 'task_id'), @@ -4481,6 +4500,8 @@ def get_next_release(build_info, incr=1): This method searches the latest building, successful, or deleted build and returns the "next" release value for that version. + Note that draft builds are excluded while getting that latest build. + Examples: None becomes "1" @@ -4512,7 +4533,7 @@ def get_next_release(build_info, incr=1): query = QueryProcessor(tables=['build'], joins=['package ON build.pkg_id = package.id'], columns=['build.id', 'release'], clauses=['name = %(name)s', 'version = %(version)s', - 'state in %(states)s'], + 'state in %(states)s', 'NOT draft'], values=values, opts={'order': '-build.id', 'limit': 1}) result = query.executeOne() @@ -4575,10 +4596,10 @@ def _fix_rpm_row(row): _fix_archive_row = _fix_rpm_row -def get_rpm(rpminfo, strict=False, multi=False): +def get_rpm(rpminfo, strict=False, multi=False, build=None): """Get information about the specified RPM - rpminfo may be any one of the following: + rpminfo ma4666y be any one of the following: - a int ID - a string N-V-R.A - a string N-V-R.A@location @@ -4587,12 +4608,17 @@ def get_rpm(rpminfo, strict=False, multi=False): If specified, location should match the name of an external repo + If build is specfied, the rpm is limited to the build's + + build and location(not INTERNAL) is conflict because a rpm in + A map will be returned, with the following keys: - id - name - version - release - arch + - draft - epoch - payloadhash - size @@ -4620,6 +4646,7 @@ def get_rpm(rpminfo, strict=False, multi=False): ('release', 'release'), ('epoch', 'epoch'), ('arch', 'arch'), + ('draft', 'draft'), ('external_repo_id', 'external_repo_id'), ('external_repo.name', 'external_repo_name'), ('payloadhash', 'payloadhash'), @@ -4644,6 +4671,12 @@ def get_rpm(rpminfo, strict=False, multi=False): else: clauses.append("rpminfo.name=%(name)s AND version=%(version)s " "AND release=%(release)s AND arch=%(arch)s") + # build and non-INTERNAL location (and multi=True) conflict in theory, + # but we just do the query and return None. + if build: + # strict=True as we treate build not found as an input error + data['build_id'] = find_build_id(build, strict=True) + clauses.append("rpminfo.build_id = %(build_id)s") retry = False if 'location' in data: data['external_repo_id'] = get_external_repo_id(data['location'], strict=True) @@ -4675,8 +4708,9 @@ def get_rpm(rpminfo, strict=False, multi=False): return ret +@convert_draft_option def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID=None, hostID=None, - arches=None, queryOpts=None): + arches=None, queryOpts=None, *, draft=DRAFT_FLAG.ALL): """List RPMS. If buildID, imageID and/or buildrootID are specified, restrict the list of RPMs to only those RPMs that are part of that build, or were built in that buildroot. If componentBuildrootID is specified, @@ -4691,6 +4725,7 @@ def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID - nvr (synthesized for sorting purposes) - arch - epoch + - draft - payloadhash - size - buildtime @@ -4706,12 +4741,22 @@ def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID - is_update If no build has the given ID, or the build generated no RPMs, - an empty list is returned.""" + an empty list is returned. + + The option draft with a bit flag(enum.IntFlag) value is to filter rpm by that + rpm belongs to a draft build, a regular build or both (default). + - DRAFT(1): draft only + - REGULAR(2): regular only + - ALL(3): both draft and regular + """ + fields = [('rpminfo.id', 'id'), ('rpminfo.name', 'name'), ('rpminfo.version', 'version'), ('rpminfo.release', 'release'), ("rpminfo.name || '-' || rpminfo.version || '-' || rpminfo.release", 'nvr'), ('rpminfo.arch', 'arch'), - ('rpminfo.epoch', 'epoch'), ('rpminfo.payloadhash', 'payloadhash'), + ('rpminfo.epoch', 'epoch'), + ('rpminfo.draft', 'draft'), + ('rpminfo.payloadhash', 'payloadhash'), ('rpminfo.size', 'size'), ('rpminfo.buildtime', 'buildtime'), ('rpminfo.build_id', 'build_id'), ('rpminfo.buildroot_id', 'buildroot_id'), ('rpminfo.external_repo_id', 'external_repo_id'), @@ -4749,6 +4794,7 @@ def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID clauses.append('rpminfo.arch = %(arches)s') else: raise koji.GenericError('Invalid type for "arches" parameter: %s' % type(arches)) + append_draft_clause(draft, clauses) fields, aliases = zip(*fields) query = QueryProcessor(columns=fields, aliases=aliases, @@ -5959,7 +6005,7 @@ def check_volume_policy(data, strict=False, default=None): return None -def apply_volume_policy(build, strict=False): +def apply_volume_policy(build, strict=False, dry_run=False): """Apply volume policy, moving build as needed build should be the buildinfo returned by get_build() @@ -5967,12 +6013,16 @@ def apply_volume_policy(build, strict=False): The strict options determines what happens in the case of a bad policy. If strict is True, an exception will be raised. Otherwise, the existing volume we be retained. + + If dry_run is True, return the volume instead of doing the actual moving. """ policy_data = {'build': build} task_id = extract_build_task(build) if task_id: policy_data.update(policy_data_from_task(task_id)) volume = check_volume_policy(policy_data, strict=strict) + if dry_run: + return volume if volume is None: # just leave the build where it is return @@ -5985,6 +6035,10 @@ def apply_volume_policy(build, strict=False): def new_build(data, strict=False): """insert a new build entry + If the build to create is a draft, the release field is the target release + rather than its actual release with draft suffix. The draft suffix will be + generated here as #draft_. + If strict is specified, raise an exception, if build already exists. """ @@ -6005,8 +6059,11 @@ def new_build(data, strict=False): for f in ('version', 'release', 'epoch'): if f not in data: raise koji.GenericError("No %s value for build" % f) + extra = {} if 'extra' in data: try: + extra = data['extra'] + # backwards compatible for data in callback data['extra'] = json.dumps(data['extra']) except Exception: raise koji.GenericError("No such build extra data: %(extra)r" % data) @@ -6021,9 +6078,12 @@ def new_build(data, strict=False): data.setdefault('owner', context.session.user_id) data.setdefault('task_id', None) data.setdefault('volume_id', 0) + data.setdefault('draft', False) - # check for existing build - old_binfo = get_build(data) + old_binfo = None + if not data.get('draft'): + # check for existing build + old_binfo = get_build(data) if old_binfo: if strict: raise koji.GenericError(f'Existing build found: {old_binfo}') @@ -6034,12 +6094,30 @@ def new_build(data, strict=False): new=data['state'], info=data) # insert the new data - insert_data = dslice(data, ['pkg_id', 'version', 'release', 'epoch', 'state', 'volume_id', - 'task_id', 'owner', 'start_time', 'completion_time', 'source', - 'extra']) + insert_data = dslice(data, ['pkg_id', 'version', 'release', 'epoch', 'draft', 'state', + 'volume_id', 'task_id', 'owner', 'start_time', 'completion_time', + 'source', 'extra']) if 'cg_id' in data: insert_data['cg_id'] = data['cg_id'] data['id'] = insert_data['id'] = nextval('build_id_seq') + # handle draft suffix in release + if data.get('draft'): + target_release = data['release'] + data['release'] = insert_data['release'] = koji.DRAFT_RELEASE_FORMAT.format(**data) + # it's still possible to already have a build with the same nvr + draft_nvr = dslice(data, ['name', 'version', 'release']) + if find_build_id(draft_nvr): + raise koji.GenericError( + f"The build already exists: {draft_nvr}" + ) + # data.extra.draft should not contains anything + if extra.get('draft'): + raise koji.GenericError('"draft" found in input extra which is reserved already') + extra['draft'] = { + 'target_release': target_release, + 'promoted': False + } + insert_data['extra'] = json.dumps(extra) or None insert = InsertProcessor('build', data=insert_data) insert.execute() new_binfo = get_build(data['id'], strict=True) @@ -6157,7 +6235,7 @@ def check_noarch_rpms(basepath, rpms, logs=None): return result -def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None): +def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None, draft=False): """Import a build into the database (single transaction) Files must be uploaded and specified with path relative to the workdir @@ -6167,6 +6245,9 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None) brmap - dictionary mapping [s]rpms to buildroot ids task_id - associate the build with a task build_id - build is a finalization of existing entry + draft - If True and + - build_id: None, to create a draft build + Not None, the build_id must be a draft build with a valid release """ if brmap is None: brmap = {} @@ -6217,16 +6298,20 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None) build['volume_name'] = vol['name'] if build_id is None: + build['draft'] = draft build_id = new_build(build) binfo = get_build(build_id, strict=True) new_typed_build(binfo, 'rpm') else: # build_id was passed in - sanity check + build['id'] = build_id binfo = get_build(build_id, strict=True) st_complete = koji.BUILD_STATES['COMPLETE'] st_old = binfo['state'] koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=st_old, new=st_complete, info=binfo) + if draft: + build['release'] = koji.DRAFT_RELEASE_FORMAT.format(**build) for key in ('name', 'version', 'release', 'epoch', 'task_id'): if build[key] != binfo[key]: raise koji.GenericError( @@ -6266,11 +6351,17 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None) return binfo -def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None): +def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None, draft=False): """Import a single rpm into the database Designed to be called from import_build. """ + # draft option must be sane + if buildinfo and draft != buildinfo.get('draft', False): + raise koji.GenericError( + f'draft property: {buildinfo["draft"]} of build: {buildinfo["id"]} mismatch,' + f' {draft} is expected' + ) if not os.path.exists(fn): raise koji.GenericError("No such file: %s" % fn) @@ -6278,6 +6369,7 @@ def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None): hdr = koji.get_rpm_header(fn) rpminfo = koji.get_header_fields(hdr, ['name', 'version', 'release', 'epoch', 'sourcepackage', 'arch', 'buildtime', 'sourcerpm']) + rpminfo['draft'] = draft if rpminfo['sourcepackage'] == 1: rpminfo['arch'] = "src" @@ -6290,26 +6382,37 @@ def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None): if buildinfo is None: # figure it out for ourselves if rpminfo['sourcepackage'] == 1: - buildinfo = get_build(rpminfo, strict=False) + if not draft: + # we only check if nvr already exists for non-draft + buildinfo = get_build(rpminfo, strict=False) if not buildinfo: # create a new build build_id = new_build(rpminfo) # we add the rpm build type below buildinfo = get_build(build_id, strict=True) else: + if draft: + raise koji.GenericError( + f'Cannot import draft rpm: {basename} without specifying a build' + ) # figure it out from sourcerpm string buildinfo = get_build(koji.parse_NVRA(rpminfo['sourcerpm'])) if buildinfo is None: # XXX - handle case where package is not a source rpm # and we still need to create a new build raise koji.GenericError('No such build') - state = koji.BUILD_STATES[buildinfo['state']] - if state in ('FAILED', 'CANCELED', 'DELETED'): - nvr = "%(name)s-%(version)s-%(release)s" % buildinfo - raise koji.GenericError("Build is %s: %s" % (state, nvr)) elif not wrapper: # only enforce the srpm name matching the build for non-wrapper rpms - srpmname = "%(name)s-%(version)s-%(release)s.src.rpm" % buildinfo + nvrinfo = buildinfo.copy() + if draft: + # for draft build the release is buildinfo.extra.draft.target_release + target_release = (buildinfo.get('extra') or {}).get('draft', {}).get('target_release') + if not target_release: + raise koji.GenericError( + f'target release of draft build not found in extra of build: {buildinfo}' + ) + nvrinfo['release'] = target_release + srpmname = "%(name)s-%(version)s-%(release)s.src.rpm" % nvrinfo # either the sourcerpm field should match the build, or the filename # itself (for the srpm) if rpminfo['sourcepackage'] != 1: @@ -6320,6 +6423,11 @@ def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None): raise koji.GenericError("srpm mismatch for %s: %s (expected %s)" % (fn, basename, srpmname)) + state = koji.BUILD_STATES[buildinfo['state']] + if state in ('FAILED', 'CANCELED', 'DELETED'): + nvr = "%(name)s-%(version)s-%(release)s" % buildinfo + raise koji.GenericError("Build is %s: %s" % (state, nvr)) + # if we're adding an rpm to it, then this build is of rpm type # harmless if build already has this type new_typed_build(buildinfo, 'rpm') @@ -6336,6 +6444,11 @@ def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None): if fileinfo is not None: extra = fileinfo.get('extra') if extra is not None: + if draft: + # simply deny draft + raise koji.GenericError( + f'When importing draft rpm, cannot specify extra in fileinfo: {fileinfo}' + ) rpminfo['extra'] = json.dumps(extra) koji.plugin.run_callbacks('preImport', type='rpm', rpm=rpminfo, build=buildinfo, @@ -6391,6 +6504,7 @@ def cg_init_build(cg, data): other values will be ignored anyway (owner, state, ...) :return: dict with build_id and token """ + reject_draft(data) assert_cg(cg) cg_id = lookup_name('content_generator', cg, strict=True)['id'] data['owner'] = context.session.user_id @@ -6670,6 +6784,10 @@ class CG_Importer(object): if (koji.BUILD_STATES[buildinfo['state']] not in ('CANCELED', 'FAILED')): raise koji.GenericError("Build already exists: %r" % buildinfo) # note: the checks in recycle_build will also apply when we call new_build later + + if buildinfo: + reject_draft(buildinfo) + # gather needed data buildinfo = dslice(metadata['build'], ['name', 'version', 'release', 'extra', 'source']) if 'build_id' in metadata['build']: @@ -6896,6 +7014,8 @@ class CG_Importer(object): if 'id' in comp: # not in metadata spec, and will confuse get_rpm raise koji.GenericError("Unexpected 'id' field in component") + # rpm is no more unique with NVRA as draft build is introduced + # TODO: we should consider how to handle them once draft build is enabled for CG rinfo = get_rpm(comp, strict=False) if not rinfo: # XXX - this is a temporary workaround until we can better track external refs @@ -7286,6 +7406,7 @@ def merge_scratch(task_id): if not build: raise koji.ImportError('no such build: %(name)s-%(version)s-%(release)s' % build_nvr) + reject_draft(build, koji.ImportError(f"build to import is a draft build: {build['nvr']}")) if build['state'] != koji.BUILD_STATES['COMPLETE']: raise koji.ImportError('%s did not complete successfully' % build['nvr']) if not build['task_id']: @@ -7557,6 +7678,7 @@ def import_archive_internal(filepath, buildinfo, type, typeInfo, buildroot_id=No buildroot_id: the id of the buildroot the archive was built in (may be None) fileinfo: content generator metadata for file (may be None) """ + reject_draft(buildinfo) if fileinfo is None: fileinfo = {} @@ -8370,7 +8492,8 @@ def query_history(tables=None, **kwargs): return ret -def untagged_builds(name=None, queryOpts=None): +@convert_draft_option +def untagged_builds(name=None, queryOpts=None, *, draft=DRAFT_FLAG.ALL): """Returns the list of untagged builds""" st_complete = koji.BUILD_STATES['COMPLETE'] # following can be achieved with simple query but with @@ -8387,6 +8510,8 @@ def untagged_builds(name=None, queryOpts=None): if name is not None: clauses.append('package.name = %(name)s') + append_draft_clause(draft, clauses) + query = QueryProcessor(tables=['build', 'package'], columns=['build.id', 'package.name', 'build.version', 'build.release'], aliases=['id', 'name', 'version', 'release'], @@ -8671,6 +8796,7 @@ def _delete_build(binfo): update = UpdateProcessor('build', values=values, clauses=['id=%(build_id)i'], data={'state': st_deleted}) update.execute() + _clean_draft_link(binfo) # now clear the build dir builddir = koji.pathinfo.build(binfo) if os.path.exists(builddir): @@ -8686,9 +8812,12 @@ def reset_build(build): WARNING: this function is highly destructive. use with care. nulls task_id sets state to CANCELED + sets volume to DEFAULT clears all referenced data in other tables, including buildroot and archive component tables + draft and extra are kept + after reset, only the build table entry is left """ # Only an admin may do this @@ -8767,6 +8896,8 @@ def reset_build(build): update = UpdateProcessor('build', clauses=['id=%(id)s'], values={'id': binfo['id']}, data={'state': binfo['state'], 'task_id': None, 'volume_id': 0}) update.execute() + + _clean_draft_link(binfo) # now clear the build dir builddir = koji.pathinfo.build(binfo) if os.path.exists(builddir): @@ -9649,6 +9780,20 @@ class IsBuildOwnerTest(koji.policy.BaseSimpleTest): return False +class IsDraftTest(koji.policy.BaseSimpleTest): + """Check if the build is a draft build""" + name = "is_draft" + + def run(self, data): + if 'draft' in data: + return data['draft'] + if 'build' in data: + build = get_build(data['build']) + return build.get('draft', False) + # default... + return False + + class UserInGroupTest(koji.policy.BaseSimpleTest): """Check if user is in group(s) @@ -10798,23 +10943,24 @@ class RootExports(object): build = get_build(build, strict=True) return apply_volume_policy(build, strict) - def createEmptyBuild(self, name, version, release, epoch, owner=None): + def createEmptyBuild(self, name, version, release, epoch, owner=None, draft=False): """Creates empty build entry :param str name: build name :param str version: build version :param str release: release version :param str epoch: epoch version - :param userInfo: a str (Kerberos principal or name) or an int (user id) + :param owner: a str (Kerberos principal or name) or an int (user id) or a dict: - id: User's ID - name: User's name - krb_principal: Kerberos principal + :param bool draft: create a draft build or not :return: int build ID """ context.session.assertPerm('admin') data = {'name': name, 'version': version, 'release': release, - 'epoch': epoch} + 'epoch': epoch, 'draft': draft} if owner is not None: data['owner'] = owner return new_build(data) @@ -10836,7 +10982,9 @@ class RootExports(object): - artifact_id: Artifact's ID - version: version :raises: GenericError if type for build_info is not dict, when build isn`t existing. + :raises: GenericError if draft: True in buildinfo, when build isn't existing. :raises: GenericError if build info doesn't have mandatory keys. + :raises: GenericError if build is a draft, when it's existing. """ context.session.assertPerm('maven-import') if not context.opts.get('EnableMaven'): @@ -10845,11 +10993,13 @@ class RootExports(object): if not build: if not isinstance(build_info, dict): raise koji.GenericError('Invalid type for build_info: %s' % type(build_info)) + reject_draft(build_info) try: build_id = new_build(dslice(build_info, ('name', 'version', 'release', 'epoch'))) except KeyError as cm: raise koji.GenericError("Build info doesn't have mandatory %s key" % cm) build = get_build(build_id, strict=True) + reject_draft(build) new_maven_build(build, maven_info) def createWinBuild(self, build_info, win_info): @@ -10866,7 +11016,9 @@ class RootExports(object): :param dict win_info: - platform: build platform :raises: GenericError if type for build_info is not dict, when build isn`t existing. + :raises: GenericError if draft: True in buildinfo, when build isn't existing. :raises: GenericError if build info doesn't have mandatory keys. + :raises: GenericError if build is a draft, when it's existing. """ context.session.assertPerm('win-import') if not context.opts.get('EnableWin'): @@ -10875,11 +11027,13 @@ class RootExports(object): if not build: if not isinstance(build_info, dict): raise koji.GenericError('Invalid type for build_info: %s' % type(build_info)) + reject_draft(build_info) try: build_id = new_build(dslice(build_info, ('name', 'version', 'release', 'epoch'))) except KeyError as cm: raise koji.GenericError("Build info doesn't have mandatory %s key" % cm) build = get_build(build_id, strict=True) + reject_draft(build) new_win_build(build, win_info) def createImageBuild(self, build_info): @@ -10895,35 +11049,47 @@ class RootExports(object): - release: build release - epoch: build epoch :raises: GenericError if type for build_info is not dict, when build isn`t existing. + :raises: GenericError if draft: True in buildinfo, when build isn't existing. :raises: GenericError if build info doesn't have mandatory keys. + :raises: GenericError if build is a draft, when it's existing. """ context.session.assertPerm('image-import') build = get_build(build_info) if not build: if not isinstance(build_info, dict): raise koji.GenericError('Invalid type for build_info: %s' % type(build_info)) + reject_draft(build_info) try: build_id = new_build(dslice(build_info, ('name', 'version', 'release', 'epoch'))) except KeyError as cm: raise koji.GenericError("Build info doesn't have mandatory %s key" % cm) build = get_build(build_id, strict=True) + reject_draft(build) new_image_build(build) - def importRPM(self, path, basename): + def importRPM(self, path, basename, build=None, draft=False): """Import an RPM into the database. The file must be uploaded first. """ context.session.assertPerm('admin') + if build: + # can get unique nvr from rpm header when importing to regular build + if not draft: + raise koji.ParameterError( + "Only support specifying build when importing a draft rpm" + ) + build = get_build(build, strict=True) uploadpath = koji.pathinfo.work() fn = "%s/%s/%s" % (uploadpath, path, basename) if not os.path.exists(fn): raise koji.GenericError("No such file: %s" % fn) - rpminfo = import_rpm(fn) + rpminfo = import_rpm(fn, buildinfo=build, draft=draft) import_rpm_file(fn, rpminfo['build'], rpminfo) add_rpm_sig(rpminfo['id'], koji.rip_rpm_sighdr(fn)) for tag in list_tags(build=rpminfo['build_id']): set_tag_update(tag['id'], 'IMPORT') + return rpminfo def mergeScratch(self, task_id): """Import the rpms generated by a scratch build, and associate @@ -11709,38 +11875,46 @@ class RootExports(object): raise koji.GenericError("Finished task's priority can't be updated") task.setPriority(priority, recurse=recurse) + @convert_draft_option def listTagged(self, tag, event=None, inherit=False, prefix=None, latest=False, package=None, - owner=None, type=None, strict=True, extra=False): + owner=None, type=None, strict=True, extra=False, *, draft=DRAFT_FLAG.ALL): """List builds tagged with tag. :param int|str tag: tag name or ID number :param int event: event ID :param bool inherit: If inherit is True, follow the tag hierarchy and return - a list of tagged builds for all tags in the tree + a list of tagged builds for all tags in the tree :param str prefix: only builds whose package name starts with that prefix - :param bool|int latest: True for latest build per package, - N to get N latest builds per package. + :param bool|int latest: True for latest build per package, + N to get N latest builds per package. :param str package: only builds of the specified package :param owner: only builds of the specified owner :param str type: only builds of the given btype (such as maven or image) :param bool strict: If tag doesn't exist, an exception is raised, - unless strict is False in which case returns an empty list. + unless strict is False in which case returns an empty list. :param bool extra: Set to "True" to get the build extra info + :param DRAFT_FLAG draft: bit flag(enum.IntFlag) indicates + - DRAFT(1): draft only + - REGULAR(2): regular only + - ALL(3): both draft and regular builds """ # lookup tag id tag = get_tag(tag, strict=strict, event=event) if not tag: return [] results = readTaggedBuilds(tag['id'], event, inherit=inherit, latest=latest, - package=package, owner=owner, type=type, extra=extra) + package=package, owner=owner, type=type, extra=extra, + draft=draft) if prefix: prefix = prefix.lower() results = [build for build in results if build['package_name'].lower().startswith(prefix)] return results + @convert_draft_option def listTaggedRPMS(self, tag, event=None, inherit=False, latest=False, package=None, arch=None, - rpmsigs=False, owner=None, type=None, strict=True, extra=True): + rpmsigs=False, owner=None, type=None, strict=True, extra=True, + *, draft=DRAFT_FLAG.ALL): """List rpms and builds within tag. :param int|str tag: tag name or ID number @@ -11759,6 +11933,10 @@ class RootExports(object): :param bool strict: If tag doesn't exist, an exception is raised, unless strict is False in which case returns an empty list. :param bool extra: Set to "False" to skip the rpms extra info + :param DRAFT_FLAG draft: bit flag(enum.IntFlag) indicates + - DRAFT(1): draft only + - REGULAR(2): regular only + - ALL(3): both draft and regular """ # lookup tag id tag = get_tag(tag, strict=strict, event=event) @@ -11766,7 +11944,7 @@ class RootExports(object): return [] return readTaggedRPMS(tag['id'], event=event, inherit=inherit, latest=latest, package=package, arch=arch, rpmsigs=rpmsigs, owner=owner, - type=type, extra=extra) + type=type, extra=extra, draft=draft) def listTaggedArchives(self, tag, event=None, inherit=False, latest=False, package=None, type=None, strict=True, extra=True): @@ -11791,10 +11969,11 @@ class RootExports(object): return readTaggedArchives(tag['id'], event=event, inherit=inherit, latest=latest, package=package, type=type, extra=extra) + @convert_draft_option def listBuilds(self, packageID=None, userID=None, taskID=None, prefix=None, state=None, volumeID=None, source=None, createdBefore=None, createdAfter=None, completeBefore=None, completeAfter=None, type=None, typeInfo=None, - queryOpts=None, pattern=None, cgID=None): + queryOpts=None, pattern=None, cgID=None, *, draft=DRAFT_FLAG.ALL): """ Return a list of builds that match the given parameters @@ -11830,6 +12009,10 @@ class RootExports(object): fields are matched For type=win, the provided platform fields are matched + :param DRAFT_FLAG draft: bit flag(enum.IntFlag) indicates + - DRAFT(1): draft only + - REGULAR(2): regular only + - ALL(3): both draft and regular builds :returns: Returns a list of maps. Each map contains the following keys: @@ -11837,6 +12020,7 @@ class RootExports(object): - version - release - epoch + - draft - state - package_id - package_name @@ -11871,7 +12055,9 @@ class RootExports(object): """ fields = [('build.id', 'build_id'), ('build.version', 'version'), ('build.release', 'release'), - ('build.epoch', 'epoch'), ('build.state', 'state'), + ('build.epoch', 'epoch'), + ('build.draft', 'draft'), + ('build.state', 'state'), ('build.completion_time', 'completion_time'), ('build.start_time', 'start_time'), ('build.source', 'source'), @@ -11977,6 +12163,7 @@ class RootExports(object): btype_id = btype['id'] joins.append('build_types ON build.id = build_types.build_id ' 'AND btype_id = %(btype_id)s') + append_draft_clause(draft, clauses) query = QueryProcessor(columns=[pair[0] for pair in fields], aliases=[pair[1] for pair in fields], @@ -11986,7 +12173,8 @@ class RootExports(object): return query.iterate() - def getLatestBuilds(self, tag, event=None, package=None, type=None): + @convert_draft_option + def getLatestBuilds(self, tag, event=None, package=None, type=None, *, draft=DRAFT_FLAG.ALL): """List latest builds for tag (inheritance enabled, wrapper of readTaggedBuilds) :param int tag: tag ID @@ -11994,15 +12182,22 @@ class RootExports(object): :param int package: filter on package name :param str type: restrict the list to builds of the given type. Currently the supported types are 'maven', 'win', 'image', or any custom content generator btypes. + :param DRAFT_FLAG draft: bit flag(enum.IntFlag) indicates + - DRAFT(1): draft only + - REGULAR(2): regular only + - ALL(3): both draft and regular builds :returns [dict]: list of buildinfo dicts """ if not isinstance(tag, int): # lookup tag id tag = get_tag_id(tag, strict=True) - return readTaggedBuilds(tag, event, inherit=True, latest=True, package=package, type=type) + return readTaggedBuilds(tag, event, inherit=True, latest=True, package=package, type=type, + draft=draft) - def getLatestRPMS(self, tag, package=None, arch=None, event=None, rpmsigs=False, type=None): + @convert_draft_option + def getLatestRPMS(self, tag, package=None, arch=None, event=None, rpmsigs=False, type=None, + *, draft=DRAFT_FLAG.ALL): """List latest RPMS for tag (inheritance enabled, wrapper of readTaggedBuilds) :param int|str tag: The tag name or ID to search @@ -12015,6 +12210,10 @@ class RootExports(object): :param bool rpmsigs: query will return one record per rpm/signature combination :param str type: Filter by build type. Supported types are 'maven', 'win', and 'image'. + :param DRAFT_FLAG draft: bit flag(enum.IntFlag) indicates + - DRAFT(1): draft only + - REGULAR(2): regular only + - ALL(3): both draft and regular :returns: a two-element list. The first element is the list of RPMs, and the second element is the list of builds. """ @@ -12023,7 +12222,7 @@ class RootExports(object): # lookup tag id tag = get_tag_id(tag, strict=True) return readTaggedRPMS(tag, package=package, arch=arch, event=event, inherit=True, - latest=True, rpmsigs=rpmsigs, type=type) + latest=True, rpmsigs=rpmsigs, type=type, draft=draft) def getLatestMavenArchives(self, tag, event=None, inherit=True): """Return a list of the latest Maven archives in the tag, as of the given event @@ -12149,6 +12348,7 @@ class RootExports(object): - release - arch - epoch + - draft - payloadhash - size - buildtime @@ -13455,6 +13655,118 @@ class RootExports(object): koji.plugin.run_callbacks('postBuildStateChange', attribute='completion_ts', old=ts_old, new=ts, info=buildinfo) + def promoteBuild(self, build, strict=True, force=False): + """Promote a draft build to a regular build. + + - The build type is limited to rpm so far. + - The promoting action cannot be revoked. + - The release wil be changed to the target one, so build_id isn't changed + - buildpath will be changed as well and the old build path will symlink + to the new one, so both paths still will be existing until deleted. + + :param build: A build ID (int), a NVR (string), or a dict containing + "name", "version" and "release" of a draft build + :type build: int, str, dict + :param bool strict: Whether raising Error (default) + or depress it by return None + :param bool force: If False (default), Koji will check this + operation against the draft_promotion hub policy. If hub + policy does not allow the current user to promote the draft build, + then this method will raise an error (strict=True) or return None + (strict=False). + If True, then this method will bypass hub policy settings. + Only admin users can set force to True. + :returns: latest build info, or None if any failure with strict=False + :rtype: dict + """ + context.session.assertLogin() + user = self.getLoggedInUser() + + binfo = get_build(build, strict=strict) + if not binfo: + return None + if not binfo.get('draft'): + if strict: + raise koji.GenericError(f'Not a draft build: {binfo}') + else: + return None + old_release = binfo['release'] + extra = binfo.get('extra') or {} + # get target release in binfo.extra + draft_info = extra.get('draft', {}).copy() + target_release = draft_info.get('target_release') + if not target_release: + if strict: + raise koji.GenericError( + f'draft.target_release not found in extra of build: {binfo}' + ) + else: + return None + target_build = dslice(binfo, ['name', 'version']) + target_build['release'] = target_release + old_build = get_build(target_build) + if old_build: + if strict: + raise koji.GenericError(f'Target build already exists: {old_build}') + else: + return None + # policy checks + policy_data = { + 'build': binfo['id'], + 'target_release': target_release + } + assert_policy('draft_promotion', policy_data, force=force) + # volume check, deny it if volume is changed as it's only allowed for admin + # after building, see applyVolumePolicy + new_volume = apply_volume_policy(target_build, strict=False, dry_run=True) + if new_volume is not None and new_volume['id'] != binfo['volume_id']: + # probably we can just apply the volume change here + if strict: + raise koji.GenericError( + f'Denial as volume will be changed to {new_volume["name"]}' + ) + else: + return None + + koji.plugin.run_callbacks( + 'preBuildPromote', + draft_release=old_release, + target_release=target_release, + build=binfo, + user=user + ) + + # temp solution with python generated time, maybe better to be an event? + now = datetime.datetime.now() + + extra['draft'] = draft_info + draft_info['promoted'] = True + draft_info['old_release'] = old_release + draft_info['promotion_time'] = encode_datetime(now) + draft_info['promotion_ts'] = now.timestamp() + draft_info['promoter'] = user['name'] + extra = json.dumps(extra) + + update = UpdateProcessor('build', clauses=['id=%(id)i'], values=binfo) + update.set(draft=False, release=target_release, extra=extra) + update.execute() + + new_binfo = get_build(binfo['id'], strict=strict) + move_and_symlink(koji.pathinfo.build(binfo), koji.pathinfo.build(new_binfo)) + ensure_volume_symlink(new_binfo) + + for tag in list_tags(build=binfo['id']): + set_tag_update(tag['id'], 'DRAFT_PROMOTION') + + koji.plugin.run_callbacks( + 'postBuildPromote', + draft_release=old_release, + target_release=target_release, + build=new_binfo, + user=user + ) + return new_binfo + def count(self, methodName, *args, **kw): """Execute the XML-RPC method with the given name and count the results. A method return value of None will return O, a return value of type "list", "tuple", or @@ -14014,6 +14326,7 @@ class BuildRoot(object): ('epoch', 'epoch'), ('arch', 'arch'), ('build_id', 'build_id'), + ('draft', 'draft'), ('external_repo_id', 'external_repo_id'), ('external_repo.name', 'external_repo_name'), ) @@ -14040,7 +14353,7 @@ class BuildRoot(object): data = add_external_rpm(an_rpm, location, strict=False) # will add if missing, compare if not else: - data = get_rpm(an_rpm, strict=True) + data = get_rpm(an_rpm, strict=True, build=an_rpm.get('build', None)) rpm_id = data['id'] if update and rpm_id in current: # ignore duplicate packages for updates @@ -14604,14 +14917,14 @@ class HostExports(object): new_typed_build(binfo, 'rpm') return build_id - def completeBuild(self, task_id, build_id, srpm, rpms, brmap=None, logs=None): + def completeBuild(self, task_id, build_id, srpm, rpms, brmap=None, logs=None, draft=False): """Import final build contents into the database""" # sanity checks host = Host() host.verify() task = Task(task_id) task.assertHost(host.id) - result = import_build(srpm, rpms, brmap, task_id, build_id, logs=logs) + result = import_build(srpm, rpms, brmap, task_id, build_id, logs=logs, draft=False) build_notification(task_id, build_id) return result @@ -15684,3 +15997,64 @@ def create_rpm_checksum(rpm_id, sigkey, chsum_dict): insert.add_record(rpm_id=rpm_id, sigkey=sigkey, checksum=chsum, checksum_type=koji.CHECKSUM_TYPES[func]) insert.execute() + + +def reject_draft(buildinfo, error=None): + """block draft build + + TODO: remove this once draft build is open for all build types + + :param dict buildinfo: buildinfo dict + :param koji.GenericError error: the error raised if not a draft build, + defaults to None to raise the default "unsupported" error + :raises error: default or specified by input error when draft==True in buldinfo + """ + if buildinfo.get('draft'): + if error is None: + error = koji.GenericError("Draft build not supported") + raise error + + +def append_draft_clause(draft, clauses, table=None): + """append proper clause in build/rpm query for draft flags + + DRAFT=1: append "draft IS True" + REGULAR=2: append "draft IS NOT True" + ALL(DRAFT|REGULAR)=3: do nothing + + :param DRAFT_FLAGS draft: draft bit flag(s) + :param list clauses: clauses list to construct query by QueryProcessor, which the draft clause + to append + """ + if not isinstance(draft, (DRAFT_FLAG)): + raise koji.ParameterError(f'draft must be a DRAFT_FLAG, but got {draft}') + if not table: + table = '' + else: + table += '.' + if DRAFT_FLAG.ALL in draft: + return + if DRAFT_FLAG.DRAFT in draft: + clauses.append(f'{table}draft IS TRUE') + if DRAFT_FLAG.REGULAR in draft: + # null is included + clauses.append(f'{table}draft IS NOT TRUE') + + +def _clean_draft_link(promoted_build): + """remove the symlink of old builddir to prmoted builddir""" + draft_info = (promoted_build.get('extra') or {}).get('draft', {}) + if not (draft_info and draft_info.get('promoted')): + # skipped as it's not a promoted build. + return + old_release = draft_info.get('old_release') + if not old_release: + raise koji.GenericError( + f"extra.draft.old_release not found in build: {promoted_build['nvr']}" + ) + draft_buildinfo = dslice(promoted_build, ['name', 'version', 'volume_name']) + draft_buildinfo['release'] = old_release + draft_builddir = koji.pathinfo.build(draft_buildinfo) + # only unlink whe its a symlink. + if os.path.islink(draft_builddir): + os.unlink(draft_builddir) diff --git a/kojihub/kojixmlrpc.py b/kojihub/kojixmlrpc.py index 64ff8bca..36ec94b5 100644 --- a/kojihub/kojixmlrpc.py +++ b/kojihub/kojixmlrpc.py @@ -610,6 +610,11 @@ _default_policies = { 'priority': ''' all :: stay ''', + 'draft_promotion': ''' + has_perm draft-promoter :: allow + is_build_owner :: allow + all :: deny Only draft-promoter and build owner can do this via default policy + ''' } diff --git a/plugins/hub/protonmsg.py b/plugins/hub/protonmsg.py index 0489ff4e..c28aa4de 100644 --- a/plugins/hub/protonmsg.py +++ b/plugins/hub/protonmsg.py @@ -320,6 +320,22 @@ def prep_repo_done(cbtype, *args, **kws): queue_msg(address, props, kws) +@convert_datetime +@callback('postBuildPromote') +def prep_build_promote(cbtype, *args, **kws): + kws['build'] = _strip_extra(kws['build']) + address = 'build.promote' + props = {'type': cbtype[4:], + 'build_id': kws['build']['id'], + 'name': kws['build']['name'], + 'version': kws['build']['version'], + 'release': kws['build']['release'], + 'draft_release': kws['draft_release'], + 'target_release': kws['target_release'], + 'user': kws['user']['name']} + queue_msg(address, props, kws) + + def _send_msgs(urls, msgs, CONFIG): random.shuffle(urls) for url in urls: diff --git a/schemas/schema-upgrade-1.33-1.34.sql b/schemas/schema-upgrade-1.33-1.34.sql index ac028939..0250ad8d 100644 --- a/schemas/schema-upgrade-1.33-1.34.sql +++ b/schemas/schema-upgrade-1.33-1.34.sql @@ -2,6 +2,7 @@ -- from version 1.33 to 1.34 BEGIN; + -- scheduler tables CREATE TABLE scheduler_task_runs ( id SERIAL NOT NULL PRIMARY KEY, @@ -48,4 +49,27 @@ BEGIN; ) WITHOUT OIDS; INSERT INTO locks(name) VALUES('scheduler'); + + -- draft builds + INSERT INTO permissions (name, description) VALUES ('draft-promoter', 'The permission required in the default "draft_promotion" hub policy rule to promote draft build.'); + + ALTER TABLE build ADD COLUMN draft BOOLEAN NOT NULL DEFAULT 'false'; + ALTER TABLE build ADD CONSTRAINT draft_for_rpminfo UNIQUE (id, draft); + ALTER TABLE build ADD CONSTRAINT draft_release_sane CHECK + ((draft AND release ~ ('^.*#draft_' || id::TEXT || '$')) + OR NOT draft); + + ALTER TABLE rpminfo ADD COLUMN draft BOOLEAN; + ALTER TABLE rpminfo DROP CONSTRAINT rpminfo_build_id_fkey; + ALTER TABLE rpminfo ADD CONSTRAINT rpminfo_build_id_draft_fkey + FOREIGN KEY (build_id, draft) REFERENCES build(id, draft) + ON UPDATE CASCADE; + ALTER TABLE rpminfo DROP CONSTRAINT rpminfo_unique_nvra; + ALTER TABLE rpminfo ADD CONSTRAINT build_id_draft_external_repo_id_sane + CHECK ((draft IS NULL AND build_id IS NULL AND external_repo_id <> 0) + OR (draft IS NOT NULL AND build_id IS NOT NULL AND external_repo_id = 0)); + CREATE UNIQUE INDEX rpminfo_unique_nvra_not_draft + ON rpminfo(name,version,release,arch,external_repo_id) + WHERE draft IS NOT TRUE; + COMMIT; diff --git a/schemas/schema.sql b/schemas/schema.sql index a007b2cb..b03b891a 100644 --- a/schemas/schema.sql +++ b/schemas/schema.sql @@ -66,6 +66,7 @@ INSERT INTO permissions (name, description) VALUES ('tag', 'Manage packages in t INSERT INTO permissions (name, description) VALUES ('target', 'Add, edit, and remove targets.'); INSERT INTO permissions (name, description) VALUES ('win-admin', 'The default hub policy rule for "vm" requires this permission to trigger Windows builds.'); INSERT INTO permissions (name, description) VALUES ('win-import', 'Import win archives.'); +INSERT INTO permissions (name, description) VALUES ('draft-promoter', 'The permission required in the default "draft_promotion" hub policy rule to promote draft build.'); CREATE TABLE user_perms ( user_id INTEGER NOT NULL REFERENCES users(id), @@ -279,11 +280,12 @@ CREATE TABLE content_generator ( -- null, or may point to a deleted task. CREATE TABLE build ( id SERIAL NOT NULL PRIMARY KEY, - volume_id INTEGER NOT NULL REFERENCES volume (id), + volume_id INTEGER NOT NULL REFERENCES volume (id), pkg_id INTEGER NOT NULL REFERENCES package (id) DEFERRABLE, version TEXT NOT NULL, release TEXT NOT NULL, epoch INTEGER, + draft BOOLEAN NOT NULL DEFAULT 'false', source TEXT, create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), start_time TIMESTAMPTZ, @@ -294,8 +296,11 @@ CREATE TABLE build ( cg_id INTEGER REFERENCES content_generator(id), extra TEXT, CONSTRAINT build_pkg_ver_rel UNIQUE (pkg_id, version, release), + CONSTRAINT draft_for_rpminfo UNIQUE (id, draft), CONSTRAINT completion_sane CHECK ((state = 0 AND completion_time IS NULL) OR - (state != 0 AND completion_time IS NOT NULL)) + (state <> 0 AND completion_time IS NOT NULL)), + CONSTRAINT draft_release_sane CHECK ((draft AND release ~ ('^.*#draft_' || id::TEXT || '$')) OR + NOT draft) ) WITHOUT OIDS; CREATE INDEX build_by_pkg_id ON build (pkg_id); @@ -721,22 +726,28 @@ CREATE TABLE group_package_listing ( -- we don't store filename b/c filename should be N-V-R.A.rpm CREATE TABLE rpminfo ( id SERIAL NOT NULL PRIMARY KEY, - build_id INTEGER REFERENCES build (id), + build_id INTEGER, buildroot_id INTEGER REFERENCES buildroot (id), name TEXT NOT NULL, version TEXT NOT NULL, release TEXT NOT NULL, epoch INTEGER, arch VARCHAR(16) NOT NULL, + draft BOOLEAN, external_repo_id INTEGER NOT NULL REFERENCES external_repo(id), payloadhash TEXT NOT NULL, size BIGINT NOT NULL, buildtime BIGINT NOT NULL, metadata_only BOOLEAN NOT NULL DEFAULT FALSE, extra TEXT, - CONSTRAINT rpminfo_unique_nvra UNIQUE (name,version,release,arch,external_repo_id) + FOREIGN KEY (build_id, draft) REFERENCES build (id, draft) ON UPDATE CASCADE, + CONSTRAINT build_id_draft_external_repo_id_sane CHECK ( + (draft IS NULL AND build_id IS NULL AND external_repo_id <> 0) + OR (draft IS NOT NULL AND build_id IS NOT NULL AND external_repo_id = 0)) ) WITHOUT OIDS; CREATE INDEX rpminfo_build ON rpminfo(build_id); +CREATE UNIQUE INDEX rpminfo_unique_nvra_not_draft ON rpminfo(name,version,release,arch,external_repo_id) + WHERE draft IS NOT TRUE; -- index for default search method for rpms, PG11+ can benefit from new include method DO $$ DECLARE version integer; diff --git a/tests/test_builder/data/calls/build_notif_1/message.txt b/tests/test_builder/data/calls/build_notif_1/message.txt index 1693f6a0..e46f4894 100644 --- a/tests/test_builder/data/calls/build_notif_1/message.txt +++ b/tests/test_builder/data/calls/build_notif_1/message.txt @@ -5,6 +5,7 @@ X-Koji-Tag: f23 X-Koji-Package: sisu X-Koji-Builder: user X-Koji-Status: complete +X-Koji-Draft: False Package: sisu-0.3.0-0.2.M1.fc23 Tag: f23 diff --git a/tests/test_builder/data/calls/build_notif_1/params.json b/tests/test_builder/data/calls/build_notif_1/params.json index 64281ee6..12b2ebe3 100644 --- a/tests/test_builder/data/calls/build_notif_1/params.json +++ b/tests/test_builder/data/calls/build_notif_1/params.json @@ -22,7 +22,8 @@ "completion_ts": 1424271457.10787, "id": 612609, "volume_name": "DEFAULT", - "nvr": "sisu-0.3.0-0.2.M1.fc23" + "nvr": "sisu-0.3.0-0.2.M1.fc23", + "draft": false }, "target": { "dest_tag": 292, diff --git a/tests/test_cli/test_build.py b/tests/test_cli/test_build.py index fe539950..b66d391a 100644 --- a/tests/test_cli/test_build.py +++ b/tests/test_cli/test_build.py @@ -176,6 +176,7 @@ Options: Provide a JSON string of custom metadata to be deserialized and stored under the build's extra.custom_user_metadata field + --draft Build draft build instead """ % (self.progname, self.progname)) # Finally, assert that things were called as we expected. diff --git a/tests/test_cli/test_buildinfo.py b/tests/test_cli/test_buildinfo.py index e33b7c29..c0180f60 100644 --- a/tests/test_cli/test_buildinfo.py +++ b/tests/test_cli/test_buildinfo.py @@ -69,6 +69,37 @@ Volume: DEFAULT Task: 8 build (target, src) Finished: Thu, 04 Mar 2021 14:45:40 UTC Tags: +""" + anon_handle_buildinfo(self.options, self.session, [build]) + self.assert_console_message(stdout, expected_stdout) + self.session.listTags.assert_called_once_with(build) + self.session.getBuild.assert_called_once_with(build) + self.session.getTaskInfo.assert_called_once_with(self.buildinfo['task_id'], request=True) + self.session.getMavenBuild.assert_called_once_with(self.buildinfo['id']) + self.session.getWinBuild.assert_called_once_with(self.buildinfo['id']) + self.session.listRPMs.assert_called_once_with(buildID=self.buildinfo['id']) + self.assertEqual(self.session.listArchives.call_count, 4) + + @mock.patch('sys.stdout', new_callable=StringIO) + def test_buildinfo_draft(self, stdout): + build = 'test-build-1-1' + binfo = copy.deepcopy(self.buildinfo) + binfo['draft'] = True + self.session.getBuild.return_value = binfo + self.session.getTaskInfo.return_value = self.taskinfo + self.session.listTags.return_value = [] + self.session.getMavenBuild.return_value = None + self.session.getWinBuild.return_value = None + self.session.listArchives.return_value = [] + self.session.listRPMs.return_value = [] + expected_stdout = """BUILD: test-build-1-1 [1] +Draft: YES +State: COMPLETE +Built by: kojiadmin +Volume: DEFAULT +Task: 8 build (target, src) +Finished: Thu, 04 Mar 2021 14:45:40 UTC +Tags: """ anon_handle_buildinfo(self.options, self.session, [build]) self.assert_console_message(stdout, expected_stdout) diff --git a/tests/test_cli/test_import.py b/tests/test_cli/test_import.py index e91a5a76..5fbb5f9f 100644 --- a/tests/test_cli/test_import.py +++ b/tests/test_cli/test_import.py @@ -87,15 +87,22 @@ class TestImport(utils.CliTestCase): def __do_import_test(self, options, session, arguments, **kwargs): expected = kwargs.get('expected', None) rpm_header = kwargs.get('rpm_header', {}) + rpm_headers = kwargs.get('rpm_headers', []) + if not rpm_headers: + rpm_headers = [rpm_header] fake_srv_path = kwargs.get('srv_path', '/path/to/server/import') upload_rpm_mock = kwargs.get('upload_rpm_mock', session.uploadWrapper) + getrpm_called = kwargs.get('getrpm_called', True) + getrpm_calls = kwargs.get('getrpm_calls', []) + import_opts = kwargs.get('import_opts', {}) + import_rpm_calls = kwargs.get('import_rpm_calls', None) with mock.patch('koji.get_header_fields') as get_header_fields_mock: with mock.patch('koji_cli.commands.unique_path') as unique_path_mock: with mock.patch('koji_cli.commands.activate_session') as activate_session_mock: with mock.patch('sys.stdout', new_callable=six.StringIO) as stdout: with upload_rpm_mock: - get_header_fields_mock.return_value = rpm_header + get_header_fields_mock.side_effect = rpm_headers unique_path_mock.return_value = fake_srv_path handle_import(options, session, arguments) @@ -104,21 +111,33 @@ class TestImport(utils.CliTestCase): # check mock calls activate_session_mock.assert_called_with(session, options) - get_header_fields_mock.assert_called_with( - arguments[0], - ('name', 'version', 'release', 'epoch', - 'arch', 'sigmd5', 'sourcepackage', 'sourcerpm') - ) + + get_header_fields_calls = [ + mock.call(arguments[i], + ('name', 'version', 'release', 'epoch', + 'arch', 'sigmd5', 'sourcepackage', 'sourcerpm') + ) for i in range(len(rpm_headers) - 1) + ] - session.getRPM.assert_called_with( - dict((k, rpm_header.get(k, '')) - for k in ['release', 'version', 'arch', 'name']) - ) + get_header_fields_mock.assert_has_calls(get_header_fields_calls) + + if getrpm_calls: + session.getRPM.assert_has_calls(getrpm_calls) + elif getrpm_called: + session.getRPM.assert_called_with( + dict((k, rpm_header.get(k, '')) + for k in ['release', 'version', 'arch', 'name']) + ) + else: + session.getRPM.assert_not_called() unique_path_mock.assert_called_with('cli-import') upload_rpm_mock.assert_called_with(arguments[0], self.fake_srv_dir) - session.importRPM.assert_called_with( - self.fake_srv_dir, os.path.basename(arguments[0])) + if import_rpm_calls: + session.importRPM.assert_has_calls(import_rpm_calls) + else: + session.importRPM.assert_called_with( + self.fake_srv_dir, os.path.basename(arguments[0]), **import_opts) # reset for next test activate_session_mock.reset_mock() @@ -136,6 +155,7 @@ class TestImport(utils.CliTestCase): expected = kwargs.get('expected', None) expected_warn = kwargs.get('expected_warn', None) rpm_header = kwargs.get('rpm_header', {}) + getrpm_called = kwargs.get('getrpm_called', True) with mock.patch('koji.get_header_fields') as get_header_fields_mock: get_header_fields_mock.return_value = rpm_header @@ -152,11 +172,13 @@ class TestImport(utils.CliTestCase): ('name', 'version', 'release', 'epoch', 'arch', 'sigmd5', 'sourcepackage', 'sourcerpm') ) - - session.getRPM.assert_called_with( - dict((k, rpm_header.get(k, '')) - for k in ['release', 'version', 'arch', 'name']) - ) + if getrpm_called: + session.getRPM.assert_called_with( + dict((k, rpm_header.get(k, '')) + for k in ['release', 'version', 'arch', 'name']) + ) + else: + session.getRPM.assert_not_called() session.uploadWrapper.assert_not_called() session.importRPM.assert_not_called() @@ -677,6 +699,225 @@ class TestImport(utils.CliTestCase): activate_session=None) activate_session_mock.assert_not_called() + @mock.patch('sys.stdout', new_callable=six.StringIO) + @mock.patch('sys.stderr', new_callable=six.StringIO) + @mock.patch('koji_cli.commands.activate_session') + def test_handle_import_src_rpm_with_create_draft( + self, + activate_session_mock, + stderr, + stdout): + """Test handle_import SRPM import with creating draft build case.""" + arguments = ['/path/to/bash-4.4.12-5.fc26.rpm', '--create-draft'] + options = mock.MagicMock() + session = mock.MagicMock() + session.importRPM.return_value = {'build': {'nvr': 'a-draft-build', 'draft': True}} + + nvr = '%(name)s-%(version)s-%(release)s' % self.srpm_header + + # Case 1. import src rpm with --create-draft + # result: success + expected = "Will create draft build instead if desired nvr doesn't exist\n" + expected += "Will create draft build with target nvr: %s while importing\n" % nvr + expected += "uploading %s... done\n" % arguments[0] + expected += "importing %s... done\n" % arguments[0] + expected += "Draft build: a-draft-build created\n" + + self.__do_import_test( + options, session, arguments, + rpm_header=self.srpm_header, + expected=expected, + getrpm_called=False, + import_opts={'draft': True}) + + @mock.patch('sys.stdout', new_callable=six.StringIO) + @mock.patch('sys.stderr', new_callable=six.StringIO) + @mock.patch('koji_cli.commands.activate_session') + def test_handle_import_binary_rpm_with_create_draft( + self, + activate_session_mock, + stderr, + stdout): + """Test handle_import binary RPM import with creating draft build case.""" + arguments = ['/path/to/bash-4.4.12-5.fc26.rpm', '--create-draft'] + options = mock.MagicMock() + session = mock.MagicMock() + + nvr = '%(name)s-%(version)s-%(release)s' % self.rpm_header + + # Case 1. import bin rpm with --create-draft + # result: Aborting import + expected = "Will create draft build instead if desired nvr doesn't exist\n" + expected += "Missing srpm for draft build creating with target nvr: %s\n" % nvr + expected += "Aborting import\n" + self.__skip_import_test( + options, session, arguments, + rpm_header=self.rpm_header, + expected=expected, + getrpm_called=False) + + @mock.patch('sys.stdout', new_callable=six.StringIO) + @mock.patch('sys.stderr', new_callable=six.StringIO) + @mock.patch('koji_cli.commands.activate_session') + def test_handle_import_src_bin_rpm_with_create_draft( + self, + activate_session_mock, + stderr, + stdout): + """Test handle_import SRPM & RPM import with creating draft build case.""" + arguments = ['/path/to/bash-4.4.12-5.fc26.rpm', '/path/to/bash-4.4.12-5.fc26.src.rpm', + '--create-draft'] + options = mock.MagicMock() + session = mock.MagicMock() + session.getRPM.return_value = None + session.importRPM.return_value = {'build': {'nvr': 'a-draft-build', 'draft': True}} + + nvr = '%(name)s-%(version)s-%(release)s' % self.srpm_header + + # Case 1. import src & bin rpm with --create-draft + # result: success + expected = "Will create draft build instead if desired nvr doesn't exist\n" + expected += "Will create draft build with target nvr: %s while importing\n" % nvr + expected += "uploading %s... done\n" % arguments[1] + expected += "importing %s... done\n" % arguments[1] + expected += "Draft build: a-draft-build created\n" + expected += "uploading %s... done\n" % arguments[0] + expected += "importing %s... done\n" % arguments[0] + + self.__do_import_test( + options, session, arguments, + rpm_headers=[self.rpm_header, self.srpm_header], + expected=expected, + getrpm_calls=[ + mock.call( + {'name': 'bash', 'version': '4.4.12', 'release': '5.fc26', 'arch': 'x86_64'}, + build=session.importRPM.return_value['build'] + ) + ], + import_rpm_calls=[ + mock.call( + self.fake_srv_dir, os.path.basename(arguments[1]), draft=True + ), + mock.call( + self.fake_srv_dir, os.path.basename(arguments[0]), + build=session.importRPM.return_value['build'], draft=True + ) + ]) + + @mock.patch('sys.stdout', new_callable=six.StringIO) + @mock.patch('sys.stderr', new_callable=six.StringIO) + @mock.patch('koji_cli.commands.activate_session') + def test_handle_import_src_rpm_with_specified_draft_build( + self, + activate_session_mock, + stderr, + stdout): + """Test handle_import SRPM import with --draft-build case.""" + arguments = ['/path/to/bash-4.4.12-5.fc26.rpm', '--draft-build', 'a-draft-build'] + options = mock.MagicMock() + session = mock.MagicMock() + build = { + 'name': 'bash', + 'version': '4.4.12', + 'release': '5.fc26#draft_123', + 'nvr': 'bash-4.4.12-5.fc26', + 'draft': True, + 'extra': { + 'draft': { + 'target_release': '5.fc26' + } + }, + 'state': self.bstate['COMPLETE'] + } + session.getBuild.return_value = build + + # Case 1. import src rpm with --draft-build + # result: success + expected = "uploading %s... done\n" % arguments[0] + expected += "importing %s... done\n" % arguments[0] + + self.__do_import_test( + options, session, arguments, + rpm_header=self.srpm_header, + expected=expected, + getrpm_calls=[mock.call( + {'name': 'bash', 'version': '4.4.12', 'release': '5.fc26', 'arch': 'src'}, + build=build)], + import_opts={'build': build, 'draft': True}) + + @mock.patch('sys.stdout', new_callable=six.StringIO) + @mock.patch('sys.stderr', new_callable=six.StringIO) + @mock.patch('koji_cli.commands.activate_session') + def test_handle_import_binary_rpm_with_specified_draft_build( + self, + activate_session_mock, + stderr, + stdout): + """Test handle_import RPM import with --draft-build case.""" + arguments = ['/path/to/bash-4.4.12-5.fc26.rpm', '--draft-build', 'a-draft-build'] + options = mock.MagicMock() + session = mock.MagicMock() + session.getRPM.return_value = None + build = { + 'name': 'bash', + 'version': '4.4.12', + 'release': '5.fc26#draft_123', + 'nvr': 'bash-4.4.12-5.fc26', + 'draft': True, + 'extra': { + 'draft': { + 'target_release': '5.fc26' + } + }, + 'state': self.bstate['COMPLETE'] + } + session.getBuild.return_value = build + + # Case 1. import bin rpm with --draft-build + # result: success + expected = "uploading %s... done\n" % arguments[0] + expected += "importing %s... done\n" % arguments[0] + + self.__do_import_test( + options, session, arguments, + rpm_header=self.rpm_header, + expected=expected, + getrpm_calls=[mock.call( + {'name': 'bash', 'version': '4.4.12', 'release': '5.fc26', 'arch': 'x86_64'}, + build=build)], + import_opts={'build': build, 'draft': True}) + + def test_handle_import_specified_draft_build_invalid(self): + """Test handle_import RPM import with --draft-build case.""" + arguments = ['/path/to/bash-4.4.12-5.fc26.rpm', '--draft-build', '286'] + options = mock.MagicMock() + session = mock.MagicMock() + + cases = [ + # build, stderr + (None, "No such build: 286"), + ({'draft': False, 'nvr': 'a-bad-draft'}, "a-bad-draft is not a draft build"), + ({'draft': True, 'nvr': 'a-bad-draft', 'state': koji.BUILD_STATES['DELETED']}, + "draft build a-bad-draft is expected as COMPLETE, got DELETED"), + ({'draft': True, 'nvr': 'a-bad-draft', 'state': koji.BUILD_STATES['COMPLETE'], + 'extra': {'draft': {'no_target_release': 'omg'}}}, + "Invalid draft build: a-bad-draft, no draft.target_release found in extra") + ] + for build, expected in cases: + session.getBuild.return_value = build + # result: error + expected += "\n" + self.assert_system_exit( + handle_import, + options, + session, + arguments, + stderr=expected, + activate_session=None, + exit_code=1) + options.reset_mock() + session.reset_mock() + def test_handle_import_help(self): """Test handle_import function help message""" self.assert_help( @@ -691,6 +932,8 @@ Options: --create-build Auto-create builds as needed --src-epoch=SRC_EPOCH When auto-creating builds, use this epoch + --create-draft Auto-create draft builds instead as needed + --draft-build=NVR|ID The target draft build to import to """ % self.progname) diff --git a/tests/test_cli/test_list_builds.py b/tests/test_cli/test_list_builds.py index 0bc80353..4f0875a7 100644 --- a/tests/test_cli/test_list_builds.py +++ b/tests/test_cli/test_list_builds.py @@ -609,6 +609,8 @@ Options: pattern) --owner=OWNER List builds built by this owner --volume=VOLUME List builds by volume ID + --draft-only Only list draft builds + --no-draft Only list regular builds -k FIELD, --sort-key=FIELD Sort the list by the named field. Allowed sort keys: build_id, owner_name, state diff --git a/tests/test_cli/test_list_tagged.py b/tests/test_cli/test_list_tagged.py index b30dd5c3..3bccfe12 100644 --- a/tests/test_cli/test_list_tagged.py +++ b/tests/test_cli/test_list_tagged.py @@ -42,6 +42,15 @@ class TestCliListTagged(utils.CliTestCase): 'release': '1.el6', 'arch': 'x86_64', 'sigkey': 'sigkey', + 'extra': None}, + {'id': 102, + 'build_id': 2, + 'name': 'rpmA', + 'version': '0.0.1', + 'release': '2.el6', + 'arch': 'x86_64', + 'sigkey': 'sigkey', + 'draft': True, 'extra': None} ], [{'id': 1, 'name': 'packagename', @@ -50,6 +59,15 @@ class TestCliListTagged(utils.CliTestCase): 'nvr': 'n-v-r', 'tag_name': 'tag', 'owner_name': 'owner', + 'extra': 'extra-value-2'}, + {'id': 2, + 'name': 'packagename', + 'version': 'version', + 'release': '2.el6#draft_2', + 'nvr': 'n-v-r', + 'draft': True, + 'tag_name': 'tag', + 'owner_name': 'owner', 'extra': 'extra-value-2'}]] self.session.listTagged.return_value = [{'id': 1, 'name': 'packagename', @@ -77,13 +95,15 @@ Build Tag Built by ---------------------------------------- -------------------- ---------------- n-v-r tag owner """ - args = [self.tag, self.pkg, '--latest', '--inherit', '--event', str(self.event_id)] + args = [self.tag, self.pkg, '--no-draft', '--latest', '--inherit', + '--event', str(self.event_id)] anon_handle_list_tagged(self.options, self.session, args) self.ensure_connection_mock.assert_called_once_with(self.session, self.options) self.session.getTag.assert_called_once_with(self.tag, event=self.event_id) self.session.listTagged.assert_called_once_with( - self.tag, event=self.event_id, inherit=True, latest=True, package=self.pkg) + self.tag, event=self.event_id, inherit=True, latest=True, package=self.pkg, + draft=2) self.session.listTaggedRPMS.assert_not_called() self.assert_console_message(stdout, expected) @@ -94,14 +114,14 @@ n-v-r tag owner ---------------------------------------- -------------------- ---------------- /mnt/koji/packages/packagename/version/1.el6 tag owner """ - args = [self.tag, self.pkg, '--latest', '--inherit', '--paths'] + args = [self.tag, self.pkg, '--latest', '--inherit', '--paths', '--draft-only'] anon_handle_list_tagged(self.options, self.session, args) self.assert_console_message(stdout, expected) self.ensure_connection_mock.assert_called_once_with(self.session, self.options) self.session.getTag.assert_called_once_with(self.tag, event=None) self.session.listTagged.assert_called_once_with( - self.tag, inherit=True, latest=True, package=self.pkg) + self.tag, inherit=True, latest=True, package=self.pkg, draft=1) self.session.listTaggedRPMS.assert_not_called() @mock.patch('sys.stdout', new_callable=six.StringIO) @@ -109,6 +129,7 @@ n-v-r tag owner def test_list_tagged_rpms(self, event_from_opts_mock, stdout): expected = """sigkey rpmA-0.0.1-1.el6.noarch sigkey rpmA-0.0.1-1.el6.x86_64 +sigkey rpmA-0.0.1-2.el6.x86_64 (#draft_2) """ args = [self.tag, self.pkg, '--latest-n=3', '--rpms', '--sigs', '--arch=x86_64', '--arch=noarch'] @@ -129,6 +150,7 @@ sigkey rpmA-0.0.1-1.el6.x86_64 def test_list_tagged_rpms_paths(self, event_from_opts_mock, stdout, os_path_exists, isdir): expected = """/mnt/koji/packages/packagename/version/1.el6/noarch/rpmA-0.0.1-1.el6.noarch.rpm /mnt/koji/packages/packagename/version/1.el6/x86_64/rpmA-0.0.1-1.el6.x86_64.rpm +/mnt/koji/packages/packagename/version/2.el6#draft_2/x86_64/rpmA-0.0.1-2.el6.x86_64.rpm """ args = [self.tag, self.pkg, '--latest-n=3', '--rpms', '--arch=x86_64', '--paths'] @@ -233,6 +255,15 @@ n-v-r tag group self.session.listTaggedRPMS.assert_not_called() self.session.listTagged.assert_not_called() + def test_list_tagged_draft_opts_conflict(self): + self.assert_system_exit( + anon_handle_list_tagged, + self.options, self.session, ['--draft-only', '--no-draft', 'tag', 'pkg1'], + stderr=self.format_error_message("--draft-only conflicts with --no-draft"), + activate_session=None, + exit_code=2) + self.ensure_connection_mock.assert_not_called() + def test_list_tagged_tag_not_found(self): self.session.getTag.return_value = None self.assert_system_exit( @@ -267,4 +298,6 @@ Options: --event=EVENT# query at event --ts=TIMESTAMP query at last event before timestamp --repo=REPO# query at event for a repo + --draft-only Only list draft builds/rpms + --no-draft Only list regular builds/rpms """ % self.progname) diff --git a/tests/test_cli/test_rpminfo.py b/tests/test_cli/test_rpminfo.py index 3e2beb3a..8e7aebb0 100644 --- a/tests/test_cli/test_rpminfo.py +++ b/tests/test_cli/test_rpminfo.py @@ -53,6 +53,17 @@ class TestRpminfo(utils.CliTestCase): 'version': '1.1', 'payloadhash': 'b2b95550390e5f213fc25f33822425f7', 'size': 7030} + self.listrpminfos = [{'arch': 'src', + 'build_id': 1, + 'buildroot_id': 3, + 'buildtime': 1615877809, + 'epoch': 7, + 'id': 290, + 'name': 'test-rpm', + 'release': '11', + 'version': '1.1', + 'payloadhash': 'b2b95550390e5f213fc25f33822425f7', + 'size': 7030}] self.error_format = """Usage: %s rpminfo [options] [ ...] (Specify the --help global option for a list of other help options) @@ -74,9 +85,11 @@ class TestRpminfo(utils.CliTestCase): self.session.listBuildroots.return_value = [self.buildroot_info] self.session.getBuild.return_value = self.buildinfo self.session.getRPM.return_value = self.getrpminfo + self.session.listRPMs.return_value = self.listrpminfos expected_output = """RPM: 7:test-rpm-1.1-11.noarch [294] +Build: test-rpm-1.1-11 [1] RPM Path: /mnt/koji/packages/test-rpm/1.1/11/noarch/test-rpm-1.1-11.noarch.rpm -SRPM: 7:test-rpm-1.1-11 [1] +SRPM: 7:test-rpm-1.1-11 [290] SRPM Path: /mnt/koji/packages/test-rpm/1.1/11/src/test-rpm-1.1-11.src.rpm Built: Tue, 16 Mar 2021 06:56:49 UTC SIGMD5: b2b95550390e5f213fc25f33822425f7 @@ -98,6 +111,8 @@ Used in 1 buildroots: rpmID=self.getrpminfo['id']) self.session.getBuild.assert_called_once_with(self.getrpminfo['build_id']) self.session.getRPM.assert_called_once_with(rpm_nvra) + self.session.listRPMs.assert_called_once_with(buildID=self.getrpminfo['build_id'], + arches='src') def test_handle_rpminfo_non_exist_nvra(self): rpm_nvra = 'test-rpm-nvra.arch' @@ -119,9 +134,11 @@ Used in 1 buildroots: self.session.listBuildroots.return_value = [self.buildroot_info] self.session.getBuild.return_value = self.buildinfo self.session.getRPM.side_effect = [None, self.getrpminfo] + self.session.listRPMs.return_value = self.listrpminfos expected_output = """RPM: 7:test-rpm-1.1-11.noarch [294] +Build: test-rpm-1.1-11 [1] RPM Path: /mnt/koji/packages/test-rpm/1.1/11/noarch/test-rpm-1.1-11.noarch.rpm -SRPM: 7:test-rpm-1.1-11 [1] +SRPM: 7:test-rpm-1.1-11 [290] SRPM Path: /mnt/koji/packages/test-rpm/1.1/11/src/test-rpm-1.1-11.src.rpm Built: Tue, 16 Mar 2021 06:56:49 UTC SIGMD5: b2b95550390e5f213fc25f33822425f7 @@ -150,6 +167,45 @@ Used in 1 buildroots: rpmID=self.getrpminfo['id']) self.session.getBuild.assert_called_once_with(self.getrpminfo['build_id']) self.assertEqual(self.session.getRPM.call_count, 2) + self.session.listRPMs.assert_called_once_with(buildID=self.getrpminfo['build_id'], + arches='src') + + + @mock.patch('sys.stdout', new_callable=StringIO) + def test_handle_rpminfo_with_build(self, stdout): + rpm_nvra = 'test-rpm-1.1-11.noarch' + self.session.getBuildroot.return_value = self.buildroot_info + self.session.listBuildroots.return_value = [self.buildroot_info] + self.session.getBuild.return_value = self.buildinfo + self.session.getRPM.return_value = self.getrpminfo + self.session.listRPMs.return_value = self.listrpminfos + expected_output = """RPM: 7:test-rpm-1.1-11.noarch [294] +Build: test-rpm-1.1-11 [1] +RPM Path: /mnt/koji/packages/test-rpm/1.1/11/noarch/test-rpm-1.1-11.noarch.rpm +SRPM: 7:test-rpm-1.1-11 [290] +SRPM Path: /mnt/koji/packages/test-rpm/1.1/11/src/test-rpm-1.1-11.src.rpm +Built: Tue, 16 Mar 2021 06:56:49 UTC +SIGMD5: b2b95550390e5f213fc25f33822425f7 +Size: 7030 +Build ID: 1 +Buildroot: 3 (tag test-tag, arch x86_64, repo 2) +Build Host: kojibuilder +Build Task: 10 +Used in 1 buildroots: + id build tag arch build host + -------- ---------------------------- -------- ----------------------------- + 3 test-tag x86_64 kojibuilder +""" + + anon_handle_rpminfo(self.options, self.session, ['--buildroot', '--build', 'any', rpm_nvra]) + self.assert_console_message(stdout, expected_output) + self.session.getBuildroot.assert_called_once_with(self.getrpminfo['buildroot_id']) + self.session.listBuildroots.assert_called_once_with(queryOpts={'order': 'buildroot.id'}, + rpmID=self.getrpminfo['id']) + self.session.getBuild.assert_called_once_with(self.getrpminfo['build_id']) + self.session.getRPM.assert_called_once_with(rpm_nvra, build='any') + self.session.listRPMs.assert_called_once_with(buildID=self.getrpminfo['build_id'], + arches='src') def test_rpminfo_without_option(self): arguments = [] @@ -171,6 +227,7 @@ Used in 1 buildroots: (Specify the --help global option for a list of other help options) Options: - -h, --help show this help message and exit - --buildroots show buildroots the rpm was used in + -h, --help show this help message and exit + --buildroots show buildroots the rpm was used in + --build=NVR|ID show the rpm(s) in the build """ % self.progname) diff --git a/tests/test_cli/test_wrapper_rpm.py b/tests/test_cli/test_wrapper_rpm.py index bb40cb20..94fbd880 100644 --- a/tests/test_cli/test_wrapper_rpm.py +++ b/tests/test_cli/test_wrapper_rpm.py @@ -237,6 +237,7 @@ Options: --wait Wait on build, even if running in the background --nowait Don't wait on build --background Run the build at a lower priority + --draft Build draft build instead """ % self.progname) diff --git a/tests/test_hub/test_getRPM.py b/tests/test_hub/test_getRPM.py index 984c4e0c..1e195705 100644 --- a/tests/test_hub/test_getRPM.py +++ b/tests/test_hub/test_getRPM.py @@ -16,6 +16,10 @@ class TestGetRPM(DBQueryTestCase): self.exports = kojihub.RootExports() self.context = mock.patch('kojihub.kojihub.context').start() self.get_external_repo_id = mock.patch('kojihub.kojihub.get_external_repo_id').start() + self.find_build_id = mock.patch('kojihub.kojihub.find_build_id').start() + + def tearDown(self): + mock.patch.stopall() def test_wrong_type_rpminfo(self): rpminfo = ['test-user'] @@ -31,11 +35,10 @@ class TestGetRPM(DBQueryTestCase): self.assertEqual(len(self.queries), 1) query = self.queries[0] - str(query) self.assertEqual(query.tables, ['rpminfo']) columns = ['rpminfo.id', 'build_id', 'buildroot_id', 'rpminfo.name', 'version', 'release', - 'epoch', 'arch', 'external_repo_id', 'external_repo.name', 'payloadhash', - 'size', 'buildtime', 'metadata_only', 'extra'] + 'epoch', 'arch', 'draft', 'external_repo_id', 'external_repo.name', + 'payloadhash', 'size', 'buildtime', 'metadata_only', 'extra'] self.assertEqual(set(query.columns), set(columns)) self.assertEqual(query.clauses, ['external_repo_id = 0', "rpminfo.id=%(id)s"]) self.assertEqual(query.joins, @@ -50,11 +53,10 @@ class TestGetRPM(DBQueryTestCase): self.assertEqual(len(self.queries), 1) query = self.queries[0] - str(query) self.assertEqual(query.tables, ['rpminfo']) columns = ['rpminfo.id', 'build_id', 'buildroot_id', 'rpminfo.name', 'version', 'release', - 'epoch', 'arch', 'external_repo_id', 'external_repo.name', 'payloadhash', - 'size', 'buildtime', 'metadata_only', 'extra'] + 'epoch', 'arch', 'draft', 'external_repo_id', 'external_repo.name', + 'payloadhash', 'size', 'buildtime', 'metadata_only', 'extra'] self.assertEqual(set(query.columns), set(columns)) self.assertEqual(query.clauses, ["rpminfo.id=%(id)s"]) self.assertEqual(query.joins, @@ -70,11 +72,10 @@ class TestGetRPM(DBQueryTestCase): self.assertEqual(len(self.queries), 1) query = self.queries[0] - str(query) self.assertEqual(query.tables, ['rpminfo']) columns = ['rpminfo.id', 'build_id', 'buildroot_id', 'rpminfo.name', 'version', 'release', - 'epoch', 'arch', 'external_repo_id', 'external_repo.name', 'payloadhash', - 'size', 'buildtime', 'metadata_only', 'extra'] + 'epoch', 'arch', 'draft', 'external_repo_id', 'external_repo.name', + 'payloadhash', 'size', 'buildtime', 'metadata_only', 'extra'] self.assertEqual(set(query.columns), set(columns)) self.assertEqual(query.clauses, ["rpminfo.id=%(id)s"]) self.assertEqual(query.joins, @@ -87,11 +88,10 @@ class TestGetRPM(DBQueryTestCase): self.assertEqual(len(self.queries), 1) query = self.queries[0] - str(query) self.assertEqual(query.tables, ['rpminfo']) columns = ['rpminfo.id', 'build_id', 'buildroot_id', 'rpminfo.name', 'version', 'release', - 'epoch', 'arch', 'external_repo_id', 'external_repo.name', 'payloadhash', - 'size', 'buildtime', 'metadata_only', 'extra'] + 'epoch', 'arch', 'draft', 'external_repo_id', 'external_repo.name', + 'payloadhash', 'size', 'buildtime', 'metadata_only', 'extra'] self.assertEqual(set(query.columns), set(columns)) self.assertEqual(query.clauses, ["rpminfo.name=%(name)s AND version=%(version)s " "AND release=%(release)s AND arch=%(arch)s"]) @@ -110,17 +110,36 @@ class TestGetRPM(DBQueryTestCase): self.assertEqual(len(self.queries), 1) query = self.queries[0] - str(query) self.assertEqual(query.tables, ['rpminfo']) columns = ['rpminfo.id', 'build_id', 'buildroot_id', 'rpminfo.name', 'version', 'release', - 'epoch', 'arch', 'external_repo_id', 'external_repo.name', 'payloadhash', - 'size', 'buildtime', 'metadata_only', 'extra'] + 'epoch', 'arch', 'draft', 'external_repo_id', 'external_repo.name', + 'payloadhash', 'size', 'buildtime', 'metadata_only', 'extra'] self.assertEqual(set(query.columns), set(columns)) self.assertEqual(query.clauses, ["external_repo_id = %(external_repo_id)i", "rpminfo.id=%(id)s"]) self.assertEqual(query.joins, ['external_repo ON rpminfo.external_repo_id = external_repo.id']) self.assertEqual(query.values, rpminfo_data) + + def test_rpm_info_with_build(self): + rpminfo = {'id': 123, 'name': 'testrpm-1.23-4.x86_64.rpm', 'build_id': 101} + self.find_build_id.return_value = 101 + rpminfo_data = rpminfo.copy() + + kojihub.get_rpm(rpminfo, multi=True, build='any') + + self.assertEqual(len(self.queries), 1) + query = self.queries[0] + self.assertEqual(query.tables, ['rpminfo']) + columns = ['rpminfo.id', 'build_id', 'buildroot_id', 'rpminfo.name', 'version', 'release', + 'epoch', 'arch', 'draft', 'external_repo_id', 'external_repo.name', + 'payloadhash', 'size', 'buildtime', 'metadata_only', 'extra'] + self.assertEqual(set(query.columns), set(columns)) + self.assertEqual(query.clauses, + ["rpminfo.build_id = %(build_id)s", "rpminfo.id=%(id)s"]) + self.assertEqual(query.joins, + ['external_repo ON rpminfo.external_repo_id = external_repo.id']) + self.assertEqual(query.values, rpminfo_data) class TestGetRPMHeaders(unittest.TestCase): diff --git a/tests/test_hub/test_get_build.py b/tests/test_hub/test_get_build.py index 78c4ef56..ac4fe1eb 100644 --- a/tests/test_hub/test_get_build.py +++ b/tests/test_hub/test_get_build.py @@ -39,8 +39,8 @@ class TestGetBuild(DBQueryTestCase): self.assertEqual(query.columns, ['build.id', 'build.cg_id', 'build.completion_time', "date_part('epoch', build.completion_time)", 'events.id', 'events.time', - "date_part('epoch', events.time)", 'build.epoch', 'build.extra', - 'build.id', 'package.name', + "date_part('epoch', events.time)", 'build.draft', 'build.epoch', + 'build.extra', 'build.id', 'package.name', "package.name || '-' || build.version || '-' || build.release", 'users.id', 'users.name', 'package.id', 'package.name', 'build.release', 'build.source', 'build.start_time', @@ -64,8 +64,8 @@ class TestGetBuild(DBQueryTestCase): self.assertEqual(query.columns, ['build.id', 'build.cg_id', 'build.completion_time', "date_part('epoch', build.completion_time)", 'events.id', 'events.time', - "date_part('epoch', events.time)", 'build.epoch', 'build.extra', - 'build.id', 'package.name', + "date_part('epoch', events.time)", 'build.draft', 'build.epoch', + 'build.extra', 'build.id', 'package.name', "package.name || '-' || build.version || '-' || build.release", 'users.id', 'users.name', 'package.id', 'package.name', 'build.release', 'build.source', 'build.start_time', @@ -111,8 +111,8 @@ class TestGetBuild(DBQueryTestCase): self.assertEqual(query.columns, ['build.id', 'build.cg_id', 'build.completion_time', "date_part('epoch', build.completion_time)", 'events.id', 'events.time', - "date_part('epoch', events.time)", 'build.epoch', 'build.extra', - 'build.id', 'package.name', + "date_part('epoch', events.time)", 'build.draft', 'build.epoch', + 'build.extra', 'build.id', 'package.name', "package.name || '-' || build.version || '-' || build.release", 'users.id', 'users.name', 'package.id', 'package.name', 'build.release', 'build.source', 'build.start_time', @@ -138,8 +138,8 @@ class TestGetBuild(DBQueryTestCase): self.assertEqual(query.columns, ['build.id', 'build.cg_id', 'build.completion_time', "date_part('epoch', build.completion_time)", 'events.id', 'events.time', - "date_part('epoch', events.time)", 'build.epoch', 'build.extra', - 'build.id', 'package.name', + "date_part('epoch', events.time)", 'build.draft', 'build.epoch', + 'build.extra', 'build.id', 'package.name', "package.name || '-' || build.version || '-' || build.release", 'users.id', 'users.name', 'package.id', 'package.name', 'build.release', 'build.source', 'build.start_time', diff --git a/tests/test_hub/test_get_next_release.py b/tests/test_hub/test_get_next_release.py index 1ca0962d..457e1796 100644 --- a/tests/test_hub/test_get_next_release.py +++ b/tests/test_hub/test_get_next_release.py @@ -22,7 +22,8 @@ class TestGetNextRelease(DBQueryTestCase): self.assertEqual(query.tables, ['build']) self.assertEqual(query.joins, ['package ON build.pkg_id = package.id']) self.assertEqual(query.clauses, - ['name = %(name)s', 'state in %(states)s', 'version = %(version)s']) + ['NOT draft', 'name = %(name)s', 'state in %(states)s', + 'version = %(version)s']) self.assertEqual(query.values, {'name': self.binfo['name'], 'version': self.binfo['version'], 'states': (1, 2, 0) diff --git a/tests/test_hub/test_import_build.py b/tests/test_hub/test_import_build.py index 564075b8..a81d18f5 100644 --- a/tests/test_hub/test_import_build.py +++ b/tests/test_hub/test_import_build.py @@ -92,6 +92,7 @@ class TestImportBuild(unittest.TestCase): fields = [ 'completion_time', + 'draft', 'epoch', 'extra', 'id', @@ -123,6 +124,7 @@ class TestImportBuild(unittest.TestCase): 'release': 'release', 'pkg_id': mock.ANY, 'id': mock.ANY, + 'draft': False } self._dml.assert_called_once_with(statement, values) diff --git a/tests/test_hub/test_import_rpm.py b/tests/test_hub/test_import_rpm.py index 014bdbb4..25630e02 100644 --- a/tests/test_hub/test_import_rpm.py +++ b/tests/test_hub/test_import_rpm.py @@ -42,11 +42,12 @@ class TestImportRPM(unittest.TestCase): 1003: 'epoch', 1006: 'buildtime', 1022: 'arch', - 1044: 'name-version-release.arch', + 1044: 'name-version-release.src.rpm', 1106: 'sourcepackage', 261: 'payload hash', } self.get_build = mock.patch('kojihub.kojihub.get_build').start() + self.new_build = mock.patch('kojihub.kojihub.new_build').start() self.get_rpm_header = mock.patch('koji.get_rpm_header').start() self.new_typed_build = mock.patch('kojihub.kojihub.new_typed_build').start() self.nextval = mock.patch('kojihub.kojihub.nextval').start() @@ -65,6 +66,7 @@ class TestImportRPM(unittest.TestCase): kojihub.import_rpm("this does not exist") def test_import_rpm_failed_build(self): + self.os_path_basename.return_value = 'name-version-release.arch.rpm' self.get_rpm_header.return_value = self.rpm_header_retval self.get_build.return_value = { 'state': koji.BUILD_STATES['FAILED'], @@ -72,9 +74,11 @@ class TestImportRPM(unittest.TestCase): 'version': 'version', 'release': 'release', } - with self.assertRaises(koji.GenericError): + with self.assertRaises(koji.GenericError) as cm: kojihub.import_rpm(self.filename) + self.assertEqual("Build is FAILED: name-version-release", str(cm.exception)) self.assertEqual(len(self.inserts), 0) + def test_import_rpm_completed_build(self): self.os_path_basename.return_value = 'name-version-release.arch.rpm' @@ -94,6 +98,7 @@ class TestImportRPM(unittest.TestCase): 'name': 'name', 'arch': 'arch', 'buildtime': 'buildtime', + 'draft': False, 'payloadhash': '7061796c6f61642068617368', 'epoch': 'epoch', 'version': 'version', @@ -114,7 +119,7 @@ class TestImportRPM(unittest.TestCase): retval = copy.copy(self.rpm_header_retval) retval.update({ 'filename': 'name-version-release.arch.rpm', - 1044: 'name-version-release.src', + 1044: 'name-version-release.src.rpm.bad', 1022: 'src', 1106: 1, }) @@ -133,6 +138,7 @@ class TestImportRPM(unittest.TestCase): 'name': 'name', 'arch': 'src', 'buildtime': 'buildtime', + 'draft': False, 'payloadhash': '7061796c6f61642068617368', 'epoch': 'epoch', 'version': 'version', @@ -149,10 +155,9 @@ class TestImportRPM(unittest.TestCase): self.assertEqual(insert.rawdata, {}) def test_non_exist_file(self): - basename = 'rpm-1-34' self.os_path_exists.return_value = False with self.assertRaises(koji.GenericError) as cm: - kojihub.import_rpm(self.filename, basename) + kojihub.import_rpm(self.filename) self.assertEqual(f"No such file: {self.filename}", str(cm.exception)) self.assertEqual(len(self.inserts), 0) @@ -172,3 +177,211 @@ class TestImportRPM(unittest.TestCase): kojihub.import_rpm(self.src_filename) self.assertEqual("No such build", str(cm.exception)) self.assertEqual(len(self.inserts), 0) + + def test_import_draft_rpm_completed_build(self): + self.os_path_basename.return_value = 'name-version-release.arch.rpm' + self.get_rpm_header.return_value = self.rpm_header_retval + self.get_build.return_value = { + 'state': koji.BUILD_STATES['COMPLETE'], + 'name': 'name', + 'version': 'version', + 'release': 'release', + 'id': 12345, + } + self.nextval.return_value = 9876 + kojihub.import_rpm(self.filename) + + data = { + 'build_id': 12345, + 'name': 'name', + 'arch': 'arch', + 'buildtime': 'buildtime', + 'draft': False, + 'payloadhash': '7061796c6f61642068617368', + 'epoch': 'epoch', + 'version': 'version', + 'buildroot_id': None, + 'release': 'release', + 'external_repo_id': 0, + 'id': 9876, + 'size': 0, + } + self.assertEqual(len(self.inserts), 1) + insert = self.inserts[0] + self.assertEqual(insert.table, 'rpminfo') + self.assertEqual(insert.data, data) + self.assertEqual(insert.rawdata, {}) + + + def test_import_draft_conflict(self): + with self.assertRaises(koji.GenericError) as cm: + kojihub.import_rpm(self.filename, buildinfo={'id': 1024, 'draft': False}, draft=True) + self.assertEqual("draft property: False of build: 1024 mismatch, True is expected", + str(cm.exception)) + self.assertEqual(len(self.inserts), 0) + + def test_import_draft_rpm_without_buildinfo(self): + self.os_path_basename.return_value = 'name-version-release.arch.rpm' + self.get_rpm_header.return_value = self.rpm_header_retval + + with self.assertRaises(koji.GenericError) as cm: + kojihub.import_rpm(self.filename, draft=True) + self.assertEqual(f"Cannot import draft rpm: {self.os_path_basename.return_value}" + " without specifying a build", str(cm.exception)) + self.assertEqual(len(self.inserts), 0) + + def test_import_draft_rpm_non_extra_target_release(self): + self.os_path_basename.return_value = 'name-version-release.arch.rpm' + self.get_rpm_header.return_value = self.rpm_header_retval + + buildinfo = { + 'state': koji.BUILD_STATES['DELETED'], + 'name': 'name', + 'version': 'version', + 'release': 'release', + 'id': 12345, + 'draft': True + } + + with self.assertRaises(koji.GenericError) as cm: + kojihub.import_rpm(self.filename, buildinfo=buildinfo, draft=True) + self.assertEqual( + f'target release of draft build not found in extra of build: {buildinfo}', + str(cm.exception) + ) + self.assertEqual(len(self.inserts), 0) + + def test_import_draft_rpm_valid(self): + self.os_path_basename.return_value = 'name-version-release.arch.rpm' + self.get_rpm_header.return_value = self.rpm_header_retval + + buildinfo = { + 'state': koji.BUILD_STATES['COMPLETE'], + 'name': 'name', + 'version': 'version', + 'release': 'release', + 'id': 12345, + 'draft': True, + 'extra': { + 'draft': { + 'target_release': 'release' + } + } + } + self.nextval.return_value = 9876 + kojihub.import_rpm(self.filename, buildinfo=buildinfo, draft=True) + data = { + 'build_id': 12345, + 'name': 'name', + 'arch': 'arch', + 'buildtime': 'buildtime', + 'draft': True, + 'payloadhash': '7061796c6f61642068617368', + 'epoch': 'epoch', + 'version': 'version', + 'buildroot_id': None, + 'release': 'release', + 'external_repo_id': 0, + 'id': 9876, + 'size': 0, + } + self.assertEqual(len(self.inserts), 1) + insert = self.inserts[0] + self.assertEqual(insert.table, 'rpminfo') + self.assertEqual(insert.data, data) + self.assertEqual(insert.rawdata, {}) + + def test_import_draft_srpm_with_buildinfo(self): + self.os_path_basename.return_value = 'name-version-release.src.rpm' + retval = copy.copy(self.rpm_header_retval) + retval.update({ + 'filename': 'name-version-release.src.rpm', + 1044: 'name-version-release.src.rpm.bad', + 1022: 'src', + 1106: 1, + }) + self.get_rpm_header.return_value = retval + buildinfo = { + 'state': koji.BUILD_STATES['COMPLETE'], + 'name': 'name', + 'version': 'version', + 'release': 'release', + 'id': 12345, + 'draft': True, + 'extra': { + 'draft': { + 'target_release': 'release' + } + } + } + self.nextval.return_value = 9876 + kojihub.import_rpm(self.src_filename, buildinfo=buildinfo, draft=True) + data = { + 'build_id': 12345, + 'name': 'name', + 'arch': 'src', + 'buildtime': 'buildtime', + 'draft': True, + 'payloadhash': '7061796c6f61642068617368', + 'epoch': 'epoch', + 'version': 'version', + 'buildroot_id': None, + 'release': 'release', + 'external_repo_id': 0, + 'id': 9876, + 'size': 0, + } + self.assertEqual(len(self.inserts), 1) + insert = self.inserts[0] + self.assertEqual(insert.table, 'rpminfo') + self.assertEqual(insert.data, data) + self.assertEqual(insert.rawdata, {}) + + def test_import_draft_srpm_without_buildinfo(self): + self.os_path_basename.return_value = 'name-version-release.src.rpm' + retval = copy.copy(self.rpm_header_retval) + retval.update({ + 'filename': 'name-version-release.src.rpm', + 1044: 'name-version-release.src.rpm.bad', + 1022: 'src', + 1106: 1, + }) + self.get_rpm_header.return_value = retval + self.get_build.return_value = { + 'state': koji.BUILD_STATES['COMPLETE'], + 'name': 'name', + 'version': 'version', + 'release': 'release', + 'id': 5566, + 'draft': True, + 'extra': { + 'draft': { + 'target_release': 'release' + } + } + } + self.new_build.return_value = 5566 + self.nextval.return_value = 9876 + kojihub.import_rpm(self.src_filename, draft=True) + data = { + 'build_id': 5566, + 'name': 'name', + 'arch': 'src', + 'buildtime': 'buildtime', + 'draft': True, + 'payloadhash': '7061796c6f61642068617368', + 'epoch': 'epoch', + 'version': 'version', + 'buildroot_id': None, + 'release': 'release', + 'external_repo_id': 0, + 'id': 9876, + 'size': 0, + } + self.assertEqual(len(self.inserts), 1) + insert = self.inserts[0] + self.assertEqual(insert.table, 'rpminfo') + self.assertEqual(insert.data, data) + self.assertEqual(insert.rawdata, {}) + self.get_build.assert_called_once_with(5566, strict=True) + self.assertEqual(self.get_build.call_count, 1) \ No newline at end of file diff --git a/tests/test_hub/test_list_builds.py b/tests/test_hub/test_list_builds.py index b54015ad..fd80bacc 100644 --- a/tests/test_hub/test_list_builds.py +++ b/tests/test_hub/test_list_builds.py @@ -17,6 +17,32 @@ class TestListBuilds(unittest.TestCase): return query def setUp(self): + + # defaults + self.tables= ['build'] + self.columns = [ + 'build.id', 'build.completion_time', + "date_part('epoch', build.completion_time)", + 'events.id', 'events.time', + "date_part('epoch', events.time)", + 'build.draft', + 'build.epoch', + 'build.extra', 'package.name', + "package.name || '-' || build.version || '-' || " + "build.release", 'users.id', 'users.name', 'package.id', + 'package.name', 'build.release', 'build.source', + 'build.start_time', "date_part('epoch', build.start_time)", + 'build.state', 'build.task_id', 'build.version', + 'volume.id', 'volume.name' + ] + self.clauses = ['package.id = %(packageID)i'] + self.joins = [ + 'LEFT JOIN events ON build.create_event = events.id', + 'LEFT JOIN package ON build.pkg_id = package.id', + 'LEFT JOIN volume ON build.volume_id = volume.id', + 'LEFT JOIN users ON build.owner = users.id' + ] + self.maxDiff = None self.exports = kojihub.RootExports() self.query_executeOne = mock.MagicMock() @@ -41,7 +67,8 @@ class TestListBuilds(unittest.TestCase): 'task_id': 879, 'version': '11', 'volume_id': 0, - 'volume_name': 'DEFAULT'}] + 'volume_name': 'DEFAULT', + 'draft': False},] def test_wrong_package(self): package = 'test-package' @@ -58,26 +85,27 @@ class TestListBuilds(unittest.TestCase): self.assertEqual(len(self.queries), 1) args, kwargs = self.QueryProcessor.call_args qp = QP(**kwargs) - self.assertEqual(qp.tables, ['build']) - self.assertEqual(qp.columns, ['build.id', 'build.completion_time', - "date_part('epoch', build.completion_time)", - 'events.id', 'events.time', - "date_part('epoch', events.time)", 'build.epoch', - 'build.extra', 'package.name', - "package.name || '-' || build.version || '-' || " - "build.release", 'users.id', 'users.name', 'package.id', - 'package.name', 'build.release', 'build.source', - 'build.start_time', "date_part('epoch', build.start_time)", - 'build.state', 'build.task_id', 'build.version', - 'volume.id', 'volume.name']) - self.assertEqual(qp.clauses, ['package.id = %(packageID)i']) - self.assertEqual(qp.joins, ['LEFT JOIN events ON build.create_event = events.id', - 'LEFT JOIN package ON build.pkg_id = package.id', - 'LEFT JOIN volume ON build.volume_id = volume.id', - 'LEFT JOIN users ON build.owner = users.id']) + self.assertEqual(qp.tables, self.tables) + self.assertEqual(qp.columns, self.columns) + self.assertEqual(qp.clauses, self.clauses) + self.assertEqual(qp.joins, self.joins) def test_wrong_user(self): user = 'test-user' self.get_user.return_value = None rv = self.exports.listBuilds(userID=user) self.assertEqual(rv, []) + + def test_draft(self): + package = 'test-package' + package_id = 1 + self.get_package_id.return_value = package_id + self.query_executeOne.return_value = None + self.exports.listBuilds(packageID=package, draft=1) + self.assertEqual(len(self.queries), 1) + args, kwargs = self.QueryProcessor.call_args + qp = QP(**kwargs) + self.assertEqual(qp.tables, self.tables) + self.assertEqual(qp.columns, self.columns) + self.assertEqual(qp.clauses, ['draft IS TRUE'] + self.clauses) + self.assertEqual(qp.joins, self.joins) diff --git a/tests/test_hub/test_new_build.py b/tests/test_hub/test_new_build.py index ab4a270f..758da47e 100644 --- a/tests/test_hub/test_new_build.py +++ b/tests/test_hub/test_new_build.py @@ -2,6 +2,7 @@ import mock import unittest import koji +from koji.util import dslice import kojihub IP = kojihub.InsertProcessor @@ -23,6 +24,7 @@ class TestNewBuild(unittest.TestCase): self.get_build = mock.patch('kojihub.kojihub.get_build').start() self.recycle_build = mock.patch('kojihub.kojihub.recycle_build').start() self.context = mock.patch('kojihub.kojihub.context').start() + self.find_build_id = mock.patch('kojihub.kojihub.find_build_id').start() def tearDown(self): mock.patch.stopall() @@ -64,6 +66,7 @@ class TestNewBuild(unittest.TestCase): 'start_time': 'NOW', 'state': 1, 'task_id': None, + 'draft': False, 'version': 'test_version', 'volume_id': 0 }) @@ -156,3 +159,48 @@ class TestNewBuild(unittest.TestCase): self.assertEqual(len(self.inserts), 0) self.assertEqual("No such build extra data: %(extra)r" % data, str(cm.exception)) + + def test_draft(self): + data = { + 'owner': 123456, + 'name': 'test_name', + 'version': 'test_version', + 'release': 'test_release', + 'epoch': 'test_epoch', + 'draft': True + } + insert_data = { + 'completion_time': 'NOW', + 'epoch': 'test_epoch', + 'extra': '{"draft": {"target_release": "test_release", "promoted": false}}', + 'id': 108, + 'owner': 123, + 'pkg_id': 54, + 'release': 'test_release#draft_108', + 'source': None, + 'start_time': 'NOW', + 'state': 1, + 'task_id': None, + 'draft': True, + 'version': 'test_version', + 'volume_id': 0 + } + self.nextval.return_value = 108 + self.new_package.return_value = 54 + self.get_user.return_value = {'id': 123} + self.find_build_id.return_value = None + + kojihub.new_build(data) + + self.assertEqual(len(self.inserts), 1) + insert = self.inserts[0] + self.assertEqual(insert.table, 'build') + self.assertEqual(insert.data, insert_data) + self.get_build.assert_called_once_with(108, strict=True) + self.assertEqual(self.get_build.call_count, 1) + self.find_build_id.assert_called_once_with( + { + 'name': 'test_name', + 'version': 'test_version', + 'release': 'test_release#draft_108' + }) diff --git a/tests/test_hub/test_promote_build.py b/tests/test_hub/test_promote_build.py new file mode 100644 index 00000000..8dc58cfa --- /dev/null +++ b/tests/test_hub/test_promote_build.py @@ -0,0 +1,179 @@ +import datetime +import json +import mock +import unittest +import koji +import kojihub + + +UP = kojihub.UpdateProcessor + + +class TestPromoteBuild(unittest.TestCase): + + def getUpdate(self, *args, **kwargs): + update = UP(*args, **kwargs) + update.execute = mock.MagicMock() + self.updates.append(update) + return update + + def setUp(self): + self.exports = kojihub.RootExports() + self.UpdateProcessor = mock.patch('kojihub.kojihub.UpdateProcessor', + side_effect=self.getUpdate).start() + self.updates = [] + self.context = mock.patch('kojihub.kojihub.context').start() + self.context.session.assertLogin = mock.MagicMock() + self.user = {'id': 1, 'name': 'jdoe'} + self.getLoggedInUser = mock.patch.object(self.exports, 'getLoggedInUser', + return_value=self.user).start() + self.get_build = mock.patch('kojihub.kojihub.get_build').start() + self.assert_policy = mock.patch('kojihub.kojihub.assert_policy').start() + self.apply_volume_policy = mock.patch('kojihub.kojihub.apply_volume_policy', + return_value=None).start() + self.move_and_symlink = mock.patch('kojihub.kojihub.move_and_symlink').start() + self.ensure_volume_symlink = mock.patch('kojihub.kojihub.ensure_volume_symlink').start() + self.list_tags = mock.patch('kojihub.kojihub.list_tags', + return_value=[{'id': 101}]).start() + self.set_tag_update = mock.patch('kojihub.kojihub.set_tag_update').start() + self.encode_datetime = mock.patch('kojihub.kojihub.encode_datetime', return_value='NOW').start() + self._now = datetime.datetime.now() + self._datetime = mock.patch('kojihub.kojihub.datetime.datetime').start() + self.now = self._datetime.now = mock.MagicMock(return_value=self._now) + + self.draft_build = { + 'id': 1, + 'name': 'foo', + 'version': 'bar', + 'release': 'dftrel_1', + 'extra': { + 'draft': { + 'promoted': False, + 'target_release': 'tgtrel_1' + }}, + 'draft': True, + 'volume_id': 99, + 'volume_name': 'X' + } + + self.new_build = { + # no check on the info + 'id': 1, + 'name': 'foo', + 'version': 'bar', + 'release': 'tgtrel_1', + 'volume_name': 'X' + } + + def tearDown(self): + mock.patch.stopall() + + def test_promote_build_valid(self): + self.get_build.side_effect = [ + self.draft_build, + None, + self.new_build + ] + + extra = json.dumps( + { + 'draft': { + 'promoted': True, + 'target_release': 'tgtrel_1', + 'old_release': 'dftrel_1', + 'promotion_time': 'NOW', + 'promotion_ts': self._now.timestamp(), + 'promoter': self.user['name'] + } + } + ) + + ret = self.exports.promoteBuild('a-draft-build', strict=True) + self.assertEqual(ret, self.new_build) + self.assertEqual(len(self.updates), 1) + update = self.updates[0] + self.assertEqual(update.table, 'build') + self.assertEqual(update.values, self.draft_build) + self.assertEqual(update.data, {'draft': False, + 'release': 'tgtrel_1', + 'extra': extra}) + self.assertEqual(update.rawdata, {}) + self.assertEqual(update.clauses, ['id=%(id)i']) + + def test_promote_build_not_draft(self): + self.get_build.return_value = {'draft': False} + + with self.assertRaises(koji.GenericError) as cm: + self.exports.promoteBuild('a-regular-build', strict=True) + self.assertEqual(str(cm.exception), "Not a draft build: {'draft': False}") + self.assertEqual(len(self.updates), 0) + + ret = self.exports.promoteBuild('a-regular-build', strict=False) + self.assertIsNone(ret) + self.assertEqual(len(self.updates), 0) + + def test_promote_build_no_target_release(self): + draft = { + 'id': 1, + 'name': 'foo', + 'version': 'bar', + 'release': 'dftrel_1', + 'extra': { + 'draft': { + 'promoted': False + # no target_release + }}, + 'draft': True, + 'volume_id': 99, + 'volume_name': 'X' + } + + self.get_build.return_value = draft + + with self.assertRaises(koji.GenericError) as cm: + self.exports.promoteBuild('a-regular-build', strict=True) + self.assertEqual(str(cm.exception), f"draft.target_release not found in extra of build: {draft}") + self.assertEqual(len(self.updates), 0) + + ret = self.exports.promoteBuild('a-regular-build', strict=False) + self.assertIsNone(ret) + self.assertEqual(len(self.updates), 0) + + def test_promote_build_target_build_exists(self): + old = { + 'id': 'any' + } + self.get_build.side_effect = [self.draft_build, old] + + with self.assertRaises(koji.GenericError) as cm: + self.exports.promoteBuild('a-regular-build', strict=True) + self.assertEqual(str(cm.exception), f"Target build already exists: {old}") + self.assertEqual(len(self.updates), 0) + self.get_build.assert_called_with({ + 'name': 'foo', + 'version': 'bar', + 'release': 'tgtrel_1' + }) + + self.get_build.reset_mock() + self.get_build.side_effect = [self.draft_build, old] + ret = self.exports.promoteBuild('a-regular-build', strict=False) + self.assertIsNone(ret) + self.assertEqual(len(self.updates), 0) + + def test_promote_build_volume_changed(self): + self.get_build.side_effect = [self.draft_build, None] + self.apply_volume_policy.return_value = { + 'id': 100, + 'name': 'Y' + } + with self.assertRaises(koji.GenericError) as cm: + self.exports.promoteBuild('a-regular-build', strict=True) + self.assertEqual(str(cm.exception), f"Denial as volume will be changed to Y") + self.assertEqual(len(self.updates), 0) + + self.get_build.reset_mock() + self.get_build.side_effect = [self.draft_build, None] + ret = self.exports.promoteBuild('a-regular-build', strict=False) + self.assertIsNone(ret) + self.assertEqual(len(self.updates), 0) diff --git a/tests/test_hub/test_read_tagged_builds.py b/tests/test_hub/test_read_tagged_builds.py index 96240f66..df827523 100644 --- a/tests/test_hub/test_read_tagged_builds.py +++ b/tests/test_hub/test_read_tagged_builds.py @@ -31,7 +31,7 @@ class TestReadTaggedBuilds(unittest.TestCase): self.tag_name = 'test-tag' self.columns = ['tag.id', 'tag.name', 'build.id', 'build.version', 'build.release', 'build.epoch', 'build.state', 'build.completion_time', 'build.start_time', - 'build.task_id', 'users.id', 'users.name', 'events.id', 'events.time', + 'build.task_id', 'build.draft', 'users.id', 'users.name', 'events.id', 'events.time', 'volume.id', 'volume.name', 'package.id', 'package.name', 'package.name || \'-\' || build.version || \'-\' || build.release', 'tag_listing.create_event'] @@ -40,6 +40,7 @@ class TestReadTaggedBuilds(unittest.TestCase): ('build.release', 'release'), ('build.epoch', 'epoch'), ('build.state', 'state'), ('build.completion_time', 'completion_time'), ('build.start_time', 'start_time'), ('build.task_id', 'task_id'), + ('build.draft', 'draft'), ('users.id', 'owner_id'), ('users.name', 'owner_name'), ('events.id', 'creation_event_id'), ('events.time', 'creation_time'), ('volume.id', 'volume_id'), ('volume.name', 'volume_name'), @@ -54,7 +55,7 @@ class TestReadTaggedBuilds(unittest.TestCase): 'volume ON volume.id = build.volume_id', 'users ON users.id = build.owner', ] self.aliases = ['tag_id', 'tag_name', 'id', 'build_id', 'version', 'release', 'epoch', - 'state', 'completion_time', 'start_time', 'task_id', 'owner_id', + 'state', 'completion_time', 'start_time', 'task_id', 'draft', 'owner_id', 'owner_name', 'creation_event_id', 'creation_time', 'volume_id', 'volume_name', 'package_id', 'package_name', 'name', 'nvr', 'create_event'] self.clauses = ['(tag_listing.active = TRUE)', @@ -83,7 +84,7 @@ class TestReadTaggedBuilds(unittest.TestCase): 'package': None, 'packages': self.package_list, 'queryOpts': {'order': '-create_event'}, 'st_complete': 1, 'tables': self.tables, 'tag': self.tag_name, 'tagid': self.tag_name, 'taglist': [self.tag_name], - 'type': None + 'type': None, 'draft': 3 } self.assertEqual(query.tables, self.tables) self.assertEqual(query.joins, self.joins) @@ -119,7 +120,7 @@ class TestReadTaggedBuilds(unittest.TestCase): 'package': self.pkg_name, 'packages': self.package_list, 'queryOpts': {'order': '-create_event'}, 'st_complete': 1, 'tables': self.tables, 'tag': self.tag_name, 'tagid': self.tag_name, 'taglist': [self.tag_name], - 'type': 'maven'} + 'type': 'maven', 'draft': 3} self.assertEqual(query.tables, self.tables) self.assertEqual(query.joins, joins) self.assertEqual(set(query.columns), set(columns)) @@ -148,7 +149,7 @@ class TestReadTaggedBuilds(unittest.TestCase): 'package': None, 'packages': self.package_list, 'queryOpts': {'order': '-create_event'}, 'st_complete': 1, 'tables': self.tables, 'tag': self.tag_name, 'tagid': self.tag_name, 'taglist': [self.tag_name], - 'type': 'win'} + 'type': 'win', 'draft': 3} self.assertEqual(query.tables, self.tables) self.assertEqual(query.joins, joins) self.assertEqual(set(query.columns), set(columns)) @@ -177,7 +178,7 @@ class TestReadTaggedBuilds(unittest.TestCase): 'package': None, 'packages': self.package_list, 'queryOpts': {'order': '-create_event'}, 'st_complete': 1, 'tables': self.tables, 'tag': self.tag_name, 'tagid': self.tag_name, 'taglist': [self.tag_name], - 'type': 'image'} + 'type': 'image', 'draft': 3} self.assertEqual(query.tables, self.tables) self.assertEqual(query.joins, joins) self.assertEqual(set(query.columns), set(columns)) @@ -212,10 +213,35 @@ class TestReadTaggedBuilds(unittest.TestCase): 'package': None, 'packages': self.package_list, 'queryOpts': {'order': '-create_event'}, 'st_complete': 1, 'tables': self.tables, 'tag': self.tag_name, 'tagid': self.tag_name, 'taglist': [self.tag_name], - 'type': type} + 'type': type, 'draft': 3} self.assertEqual(query.tables, self.tables) self.assertEqual(query.joins, joins) self.assertEqual(set(query.columns), set(self.columns)) self.assertEqual(set(query.aliases), set(self.aliases)) self.assertEqual(set(query.clauses), set(self.clauses)) self.assertEqual(query.values, values) + + def test_get_tagged_builds_draft(self): + self.readPackageList.return_value = self.package_list + kojihub.readTaggedBuilds(self.tag_name, draft=koji.DRAFT_FLAG.DRAFT) + + self.assertEqual(len(self.queries), 1) + query = self.queries[0] + + clauses = copy.deepcopy(self.clauses) + clauses.extend(['draft IS TRUE']) + + values = {'clauses': clauses, 'event': None, 'extra': False, 'fields': self.fields, + 'inherit': False, 'joins': self.joins, 'latest': False, 'owner': None, + 'package': None, 'packages': self.package_list, + 'queryOpts': {'order': '-create_event'}, 'st_complete': 1, 'tables': self.tables, + 'tag': self.tag_name, 'tagid': self.tag_name, 'taglist': [self.tag_name], + 'type': None, 'draft': koji.DRAFT_FLAG.DRAFT + } + + self.assertEqual(query.tables, self.tables) + self.assertEqual(query.joins, self.joins) + self.assertEqual(set(query.columns), set(self.columns)) + self.assertEqual(set(query.aliases), set(self.aliases)) + self.assertEqual(set(query.clauses), set(clauses)) + self.assertEqual(query.values, values) diff --git a/tests/test_hub/test_read_tagged_rpms.py b/tests/test_hub/test_read_tagged_rpms.py index 057ceb3b..1a32aec7 100644 --- a/tests/test_hub/test_read_tagged_rpms.py +++ b/tests/test_hub/test_read_tagged_rpms.py @@ -29,12 +29,13 @@ class TestReadTaggedRPMS(unittest.TestCase): self.readTaggedBuilds = mock.patch('kojihub.kojihub.readTaggedBuilds').start() self.tag_name = 'test-tag' self.columns = ['rpminfo.name', 'rpminfo.version', 'rpminfo.release', 'rpminfo.arch', - 'rpminfo.id', 'rpminfo.epoch', 'rpminfo.payloadhash', 'rpminfo.size', - 'rpminfo.buildtime', 'rpminfo.buildroot_id', 'rpminfo.build_id', - 'rpminfo.metadata_only'] + 'rpminfo.id', 'rpminfo.epoch', 'rpminfo.draft', 'rpminfo.payloadhash', + 'rpminfo.size', 'rpminfo.buildtime', 'rpminfo.buildroot_id', + 'rpminfo.build_id', 'rpminfo.metadata_only'] self.joins = ['tag_listing ON rpminfo.build_id = tag_listing.build_id'] - self.aliases = ['name', 'version', 'release', 'arch', 'id', 'epoch', 'payloadhash', - 'size', 'buildtime', 'buildroot_id', 'build_id', 'metadata_only'] + self.aliases = ['name', 'version', 'release', 'arch', 'id', 'epoch', 'draft', + 'payloadhash', 'size', 'buildtime', 'buildroot_id', 'build_id', + 'metadata_only'] self.clauses = ['(tag_listing.active = TRUE)', 'tag_id=%(tagid)s'] self.tables = ['rpminfo'] @@ -101,3 +102,20 @@ class TestReadTaggedRPMS(unittest.TestCase): self.assertEqual(set(query.aliases), set(aliases)) self.assertEqual(set(query.clauses), set(clauses)) self.assertEqual(query.values, values) + + def test_get_tagged_rpms_draft(self): + self.readTaggedBuilds.return_value = self.build_list + kojihub.readTaggedRPMS(self.tag_name, draft=2, extra=False) + + self.assertEqual(len(self.queries), 1) + query = self.queries[0] + + clauses = copy.deepcopy(self.clauses) + clauses.extend(['rpminfo.draft IS NOT TRUE']) + + self.assertEqual(query.tables, self.tables) + self.assertEqual(set(query.columns), set(self.columns)) + self.assertEqual(set(query.joins), set(self.joins)) + self.assertEqual(set(query.aliases), set(self.aliases)) + self.assertEqual(set(query.clauses), set(clauses)) + self.assertEqual(query.values, {}) \ No newline at end of file diff --git a/tests/test_hub/test_reset_build.py b/tests/test_hub/test_reset_build.py index 13d31b2e..06fe49df 100644 --- a/tests/test_hub/test_reset_build.py +++ b/tests/test_hub/test_reset_build.py @@ -44,6 +44,9 @@ class TestResetBuild(unittest.TestCase): self.get_build = mock.patch('kojihub.kojihub.get_build').start() self.context = mock.patch('kojihub.kojihub.context').start() self.context.session.assertPerm = mock.MagicMock() + # don't remove anything unexpected + self.rmtree = mock.patch('koji.util.rmtree').start() + self.unlink = mock.patch('os.unlink').start() self.build_id = 3 self.binfo = {'id': 3, 'state': koji.BUILD_STATES['COMPLETE'], 'name': 'test_nvr', 'nvr': 'test_nvr-3.3-20.el8', 'version': '3.3', 'release': '20',