From e1f928af6b032dfa3cb841e47e69747eed63d7a1 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 9 Apr 2015 15:29:16 -0400 Subject: [PATCH 001/109] cg br schema pass 1 --- docs/schema.sql | 40 ++++++++++++++++++++++++++++++++++++++-- hub/kojihub.py | 2 +- koji/__init__.py | 5 +++++ 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/docs/schema.sql b/docs/schema.sql index c087d566..1a45bce9 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -464,16 +464,52 @@ create table tag_external_repos ( -- here we track the buildroots on the machines CREATE TABLE buildroot ( id SERIAL NOT NULL PRIMARY KEY, + br_type INTEGER NOT NULL +) WITHOUT OIDS; + +CREATE TABLE standard_buildroot ( + buildroot_id INTEGER NOT NULL PRIMARY KEY REFERENCES buildroot(id), host_id INTEGER NOT NULL REFERENCES host(id), repo_id INTEGER NOT NULL REFERENCES repo (id), arch VARCHAR(16) NOT NULL, task_id INTEGER NOT NULL REFERENCES task (id), create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), retire_event INTEGER, - state INTEGER, - dirtyness INTEGER + state INTEGER ) WITHOUT OIDS; +CREATE TABLE buildroot_host_info ( + buildroot_id INTEGER NOT NULL PRIMARY KEY REFERENCES buildroot(id), + os TEXT NOT NULL, + arch TEXT NOT NULL +) WITHOUT OIDS; + +CREATE TABLE buildroot_cg_info ( + buildroot_id INTEGER NOT NULL PRIMARY KEY REFERENCES buildroot(id), + name TEXT NOT NULL, + version TEXT NOT NULL +) WITHOUT OIDS; + +CREATE TABLE buildroot_container_info ( + buildroot_id INTEGER NOT NULL PRIMARY KEY REFERENCES buildroot(id), + ctype TEXT NOT NULL, + arch TEXT NOT NULL +) WITHOUT OIDS; + +CREATE TABLE buildroot_tools_info ( + buildroot_id INTEGER NOT NULL REFERENCES buildroot(id), + tool TEXT NOT NULL, + version TEXT NOT NULL, + PRIMARY KEY (buildroot_id, tool) +) WITHOUT OIDS; + +CREATE TABLE buildroot_extra_info ( + buildroot_id INTEGER NOT NULL PRIMARY KEY REFERENCES buildroot(id), + info TEXT NOT NULL +-- ^ we'll eventually make this a json type when we require newer postgres +) WITHOUT OIDS; + + -- track spun images (livecds, installation, VMs...) CREATE TABLE image_builds ( build_id INTEGER NOT NULL PRIMARY KEY REFERENCES build(id) diff --git a/hub/kojihub.py b/hub/kojihub.py index 9decf7cf..773c0f35 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4100,7 +4100,7 @@ def query_buildroots(hostID=None, tagID=None, state=None, rpmID=None, archiveID= queryOpts - query options """ fields = [('buildroot.id', 'id'), ('buildroot.arch', 'arch'), ('buildroot.state', 'state'), - ('buildroot.dirtyness', 'dirtyness'), ('buildroot.task_id', 'task_id'), + ('buildroot.task_id', 'task_id'), ('host.id', 'host_id'), ('host.name', 'host_name'), ('repo.id', 'repo_id'), ('repo.state', 'repo_state'), ('tag.id', 'tag_id'), ('tag.name', 'tag_name'), diff --git a/koji/__init__.py b/koji/__init__.py index d133ed19..42b91fef 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -210,6 +210,11 @@ BR_STATES = Enum(( 'EXPIRED', )) +BR_TYPES = Enum(( + 'STANDARD', + 'EXTERNAL', +)) + TAG_UPDATE_TYPES = Enum(( 'VOLUME_CHANGE', 'IMPORT', From c36adb8eb56bf0860929e149666e04acc2e2f151 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 9 Apr 2015 19:56:36 -0400 Subject: [PATCH 002/109] merge some of the new buildroot tables, refactor buildroot_extra_info --- docs/schema.sql | 55 ++++++++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/docs/schema.sql b/docs/schema.sql index 1a45bce9..57364a1c 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -461,41 +461,51 @@ create table tag_external_repos ( UNIQUE (tag_id, external_repo_id, active) ); + +-- data for content generators +CREATE TABLE content_generator ( + id SERIAL PRIMARY KEY, + name TEXT +) WITHOUT OIDS; + +CREATE TABLE cg_users ( + cg_id INTEGER NOT NULL REFERENCES content_generator (id), + user_id INTEGER NOT NULL REFERENCES users (id), + PRIMARY KEY (cg_id, user_id) +-- XXX: should we version this? +) WITHOUT OIDS; + + -- here we track the buildroots on the machines CREATE TABLE buildroot ( id SERIAL NOT NULL PRIMARY KEY, br_type INTEGER NOT NULL + cg_id INTEGER REFERENCES content_generator (id), + cg_version version TEXT, + --^ XXX should cg_version be integer? array? + CONSTRAINT cg_sane CHECK ( + (cg_id IS NULL AND cg_version IS NULL) + OR (cg_id IS NOT NULL AND cg_version IS NOT NULL)), + container_type TEXT, + -- XXX should container_type be lookup table or perhaps a lib Enum? + container_arch TEXT, + CONSTRAINT cg_sane CHECK ( + (container_type IS NULL AND container_arch IS NULL) + OR (container_type IS NOT NULL AND container_arch IS NOT NULL)), + host_os TEXT, + host_arch TEXT ) WITHOUT OIDS; CREATE TABLE standard_buildroot ( buildroot_id INTEGER NOT NULL PRIMARY KEY REFERENCES buildroot(id), host_id INTEGER NOT NULL REFERENCES host(id), repo_id INTEGER NOT NULL REFERENCES repo (id), - arch VARCHAR(16) NOT NULL, task_id INTEGER NOT NULL REFERENCES task (id), create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), retire_event INTEGER, state INTEGER ) WITHOUT OIDS; -CREATE TABLE buildroot_host_info ( - buildroot_id INTEGER NOT NULL PRIMARY KEY REFERENCES buildroot(id), - os TEXT NOT NULL, - arch TEXT NOT NULL -) WITHOUT OIDS; - -CREATE TABLE buildroot_cg_info ( - buildroot_id INTEGER NOT NULL PRIMARY KEY REFERENCES buildroot(id), - name TEXT NOT NULL, - version TEXT NOT NULL -) WITHOUT OIDS; - -CREATE TABLE buildroot_container_info ( - buildroot_id INTEGER NOT NULL PRIMARY KEY REFERENCES buildroot(id), - ctype TEXT NOT NULL, - arch TEXT NOT NULL -) WITHOUT OIDS; - CREATE TABLE buildroot_tools_info ( buildroot_id INTEGER NOT NULL REFERENCES buildroot(id), tool TEXT NOT NULL, @@ -505,8 +515,11 @@ CREATE TABLE buildroot_tools_info ( CREATE TABLE buildroot_extra_info ( buildroot_id INTEGER NOT NULL PRIMARY KEY REFERENCES buildroot(id), - info TEXT NOT NULL --- ^ we'll eventually make this a json type when we require newer postgres + key TEXT NOT NULL, + value TEXT NOT NULL, + v_type INTEGER, + -- XXX is it worth having the v_type field? + PRIMARY KEY (buildroot_id, key) ) WITHOUT OIDS; From fec4b3b09b24362633cf2b6023b4026bd9f35716 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 16 Apr 2015 14:39:07 -0400 Subject: [PATCH 003/109] image_archive_listing; add drops for new tables; --- docs/schema.sql | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/schema.sql b/docs/schema.sql index 57364a1c..3facb9c5 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -7,6 +7,7 @@ DROP TABLE log_messages; DROP TABLE buildroot_listing; DROP TABLE image_listing; +DROP TABLE image_archive_listing; DROP TABLE rpminfo; DROP TABLE image_builds; @@ -20,6 +21,9 @@ DROP TABLE groups; DROP TABLE tag_listing; DROP TABLE tag_packages; +DROP TABLE buildroot_extra_info; +DROP TABLE buildroot_tools_info; +DROP TABLE standard_buildroot; DROP TABLE buildroot; DROP TABLE repo; @@ -40,6 +44,9 @@ DROP TABLE host; DROP TABLE channels; DROP TABLE package; +DROP TABLE cg_users; +DROP TABLE content_generator; + DROP TABLE user_groups; DROP TABLE user_perms; DROP TABLE permissions; @@ -824,6 +831,15 @@ CREATE TABLE image_listing ( ) WITHOUT OIDS; CREATE INDEX image_listing_rpms on image_listing(rpm_id); +-- track the archive contents of an image +CREATE TABLE image_archive_listing ( + image_id INTEGER NOT NULL REFERENCES image_archives(archive_id), + archive_id INTEGER NOT NULL REFERENCES archiveinfo(id), + UNIQUE (image_id, archive_id) +) WITHOUT OIDS; +CREATE INDEX image_listing_archives on image_archive_listing(archive_id); + + CREATE TABLE buildroot_archives ( buildroot_id INTEGER NOT NULL REFERENCES buildroot (id), archive_id INTEGER NOT NULL REFERENCES archiveinfo (id), From be066043001db15f381578b6fab8578eb2f81ea3 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 16 Apr 2015 15:12:30 -0400 Subject: [PATCH 004/109] include image listings in build_references check --- hub/kojihub.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/hub/kojihub.py b/hub/kojihub.py index 773c0f35..1e22c938 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -5678,6 +5678,18 @@ def build_references(build_id, limit=None): break ret['rpms'] = idx.values() + ret['images'] = [] + # find images that contain the build rpms + fields = ['image_id'] + clauses = ['image_listing.rpm_id = %(rpm_id)s'] + # TODO: join in other tables to provide something more than image id + query = QueryProcessor(columns=fields, tables=['image_listing'], clauses=clauses, + opts={'asList': True}) + for (rpm_id,) in rpm_ids: + query.values = {'rpm_id': rpm_id} + image_ids = [i[0] for i in query.execute()] + ret['images'].extend(image_ids) + # find archives whose buildroots we were in q = """SELECT id FROM archiveinfo WHERE build_id = %(build_id)i""" archive_ids = _fetchMulti(q, locals()) @@ -5699,6 +5711,17 @@ def build_references(build_id, limit=None): break ret['archives'] = idx.values() + # find images that contain the build archives + fields = ['image_id'] + clauses = ['image_archive_listing.archive_id = %(archive_id)s'] + # TODO: join in other tables to provide something more than image id + query = QueryProcessor(columns=fields, tables=['image_archive_listing'], clauses=clauses, + opts={'asList': True}) + for (archive_id,) in archive_ids: + query.values = {'archive_id': archive_id} + image_ids = [i[0] for i in query.execute()] + ret['images'].extend(image_ids) + # find timestamp of most recent use in a buildroot q = """SELECT buildroot.create_event FROM buildroot_listing From cce59398be32cdf0d36a4dec977a1196e7dea9ab Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 16 Apr 2015 15:32:43 -0400 Subject: [PATCH 005/109] fix repo_references() for cgen schema --- hub/kojihub.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 1e22c938..71a57ebf 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2395,12 +2395,19 @@ def repo_expire_older(tag_id, event_id): def repo_references(repo_id): """Return a list of buildroots that reference the repo""" - fields = ('id', 'host_id', 'create_event', 'state') - q = """SELECT %s FROM buildroot WHERE repo_id=%%(repo_id)s - AND retire_event IS NULL""" % ','.join(fields) + fields = { + 'buildroot_id': 'id', + 'host_id': 'host_id', + 'create_event': 'create_event', + 'state': 'state'} + fields, aliases = zip(*fields.items()) + values = {'repo_id': repo_id} + clauses = ['repo_id=%(repo_id)s', 'retire_event IS NULL'] + query = QueryProcessor(fields=fields, aliases=aliases, tables=['standard_buildroot'], + clauses=clauses, values=values, queryOpts = {'asList':True}) #check results for bad states ret = [] - for data in _multiRow(q, locals(), fields): + for data in query.execute(): if data['state'] == koji.BR_STATES['EXPIRED']: log_error("Error: buildroot %(id)s expired, but has no retire_event" % data) continue From 5ff5a760d9a5377e74c523daeb1c5acee193d4d0 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 16 Apr 2015 15:38:55 -0400 Subject: [PATCH 006/109] add imageID arg to list_archives() --- hub/kojihub.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 71a57ebf..b0d5987c 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -3534,7 +3534,7 @@ def get_image_build(buildInfo, strict=False): raise koji.GenericError, 'no such image build: %s' % buildInfo return result -def list_archives(buildID=None, buildrootID=None, componentBuildrootID=None, hostID=None, type=None, +def list_archives(buildID=None, buildrootID=None, imageID=None, componentBuildrootID=None, hostID=None, type=None, filename=None, size=None, checksum=None, typeInfo=None, queryOpts=None): """ Retrieve information about archives. @@ -3626,6 +3626,9 @@ def list_archives(buildID=None, buildrootID=None, componentBuildrootID=None, hos values['component_buildroot_id'] = componentBuildrootID fields.append(['buildroot_archives.buildroot_id', 'component_buildroot_id']) fields.append(['buildroot_archives.project_dep', 'project']) + if imageID != None: + clauses.append('image_archive_listing.image_id = %(imageID)i') + joins.append('image_archive_listing ON archiveinfo.id = image_archive_listing.archive_id') if hostID is not None: joins.append('buildroot on archiveinfo.buildroot_id = buildroot.id') clauses.append('buildroot.host_id = %(host_id)i') From 6401cb7a2766bdb63d65f203303f4a66ca73a139 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 16 Apr 2015 16:00:02 -0400 Subject: [PATCH 007/109] update query_buildroots() for cgen --- hub/kojihub.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index b0d5987c..c7380808 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4109,8 +4109,11 @@ def query_buildroots(hostID=None, tagID=None, state=None, rpmID=None, archiveID= buildrootID - only the specified buildroot queryOpts - query options """ - fields = [('buildroot.id', 'id'), ('buildroot.arch', 'arch'), ('buildroot.state', 'state'), - ('buildroot.task_id', 'task_id'), + fields = [('buildroot.id', 'id'), + ('buildroot.host_arch', 'host_arch'), + ('buildroot.host_arch', 'arch'), #alias for back compat + ('standard_buildroot.state', 'state'), + ('standard_buildroot.task_id', 'task_id'), ('host.id', 'host_id'), ('host.name', 'host_name'), ('repo.id', 'repo_id'), ('repo.state', 'repo_state'), ('tag.id', 'tag_id'), ('tag.name', 'tag_name'), @@ -4121,11 +4124,12 @@ def query_buildroots(hostID=None, tagID=None, state=None, rpmID=None, archiveID= ('repo_create.id', 'repo_create_event_id'), ('repo_create.time', 'repo_create_event_time')] tables = ['buildroot'] - joins=['host ON host.id = buildroot.host_id', - 'repo ON repo.id = buildroot.repo_id', + joins=['LEFT OUTER JOIN standard_buildroot ON buildroot_id = buildroot.id', + 'LEFT OUTER JOIN host ON host.id = standard_buildroot.host_id', + 'LEFT OUTER JOIN repo ON repo.id = standard_buildroot.repo_id', 'tag ON tag.id = repo.tag_id', - 'events AS create_events ON create_events.id = buildroot.create_event', - 'LEFT OUTER JOIN events AS retire_events ON buildroot.retire_event = retire_events.id', + 'LEFT OUTER JOIN events AS create_events ON create_events.id = standard_buildroot.create_event', + 'LEFT OUTER JOIN events AS retire_events ON standard_buildroot.retire_event = retire_events.id', 'events AS repo_create ON repo_create.id = repo.create_event'] clauses = [] @@ -4140,9 +4144,9 @@ def query_buildroots(hostID=None, tagID=None, state=None, rpmID=None, archiveID= clauses.append('tag.id = %(tagID)i') if state != None: if isinstance(state, list) or isinstance(state, tuple): - clauses.append('buildroot.state IN %(state)s') + clauses.append('standard_buildroot.state IN %(state)s') else: - clauses.append('buildroot.state = %(state)i') + clauses.append('standard_buildroot.state = %(state)i') if rpmID != None: joins.insert(0, 'buildroot_listing ON buildroot.id = buildroot_listing.buildroot_id') fields.append(('buildroot_listing.is_update', 'is_update')) @@ -4151,7 +4155,7 @@ def query_buildroots(hostID=None, tagID=None, state=None, rpmID=None, archiveID= joins.append('buildroot_archives ON buildroot.id = buildroot_archives.buildroot_id') clauses.append('buildroot_archives.archive_id = %(archiveID)i') if taskID != None: - clauses.append('buildroot.task_id = %(taskID)i') + clauses.append('standard_buildroot.task_id = %(taskID)i') query = QueryProcessor(columns=[f[0] for f in fields], aliases=[f[1] for f in fields], tables=tables, joins=joins, clauses=clauses, values=locals(), From 6c781b6b845ebf8787117b2ab42e6b68a89d2c4b Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 16 Apr 2015 16:36:44 -0400 Subject: [PATCH 008/109] fix constraint name; drop buildroot_extra_info.v_type; drop some obsolete comments --- docs/schema.sql | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/schema.sql b/docs/schema.sql index 3facb9c5..91f260be 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -489,14 +489,12 @@ CREATE TABLE buildroot ( br_type INTEGER NOT NULL cg_id INTEGER REFERENCES content_generator (id), cg_version version TEXT, - --^ XXX should cg_version be integer? array? CONSTRAINT cg_sane CHECK ( (cg_id IS NULL AND cg_version IS NULL) OR (cg_id IS NOT NULL AND cg_version IS NOT NULL)), container_type TEXT, - -- XXX should container_type be lookup table or perhaps a lib Enum? container_arch TEXT, - CONSTRAINT cg_sane CHECK ( + CONSTRAINT container_sane CHECK ( (container_type IS NULL AND container_arch IS NULL) OR (container_type IS NOT NULL AND container_arch IS NOT NULL)), host_os TEXT, @@ -524,8 +522,6 @@ CREATE TABLE buildroot_extra_info ( buildroot_id INTEGER NOT NULL PRIMARY KEY REFERENCES buildroot(id), key TEXT NOT NULL, value TEXT NOT NULL, - v_type INTEGER, - -- XXX is it worth having the v_type field? PRIMARY KEY (buildroot_id, key) ) WITHOUT OIDS; From bd4aa00fddfcb676d35ddc0694eb10c6d791b8f6 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 16 Apr 2015 16:43:15 -0400 Subject: [PATCH 009/109] adapt list_rpms() for cgen schema --- hub/kojihub.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index c7380808..f2fc00e4 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -3453,8 +3453,8 @@ def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID joins.append('image_listing ON rpminfo.id = image_listing.rpm_id') if hostID != None: - joins.append('buildroot ON rpminfo.buildroot_id = buildroot.id') - clauses.append('buildroot.host_id = %(hostID)i') + joins.append('standard_buildroot ON rpminfo.buildroot_id = standard_buildroot.id') + clauses.append('standard_buildroot.host_id = %(hostID)i') if arches != None: if isinstance(arches, list) or isinstance(arches, tuple): clauses.append('rpminfo.arch IN %(arches)s') From 9b3a7deea7813c7e29c5ec89ce9f55437c781f03 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 16 Apr 2015 16:45:37 -0400 Subject: [PATCH 010/109] adapt list_archives() for cgen schema --- hub/kojihub.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index f2fc00e4..ae2aa9a5 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -3630,10 +3630,10 @@ def list_archives(buildID=None, buildrootID=None, imageID=None, componentBuildro clauses.append('image_archive_listing.image_id = %(imageID)i') joins.append('image_archive_listing ON archiveinfo.id = image_archive_listing.archive_id') if hostID is not None: - joins.append('buildroot on archiveinfo.buildroot_id = buildroot.id') - clauses.append('buildroot.host_id = %(host_id)i') + joins.append('standard_buildroot on archiveinfo.buildroot_id = standard_buildroot.id') + clauses.append('standard_buildroot.host_id = %(host_id)i') values['host_id'] = hostID - fields.append(['buildroot.host_id', 'host_id']) + fields.append(['standard_buildroot.host_id', 'host_id']) if filename is not None: clauses.append('filename = %(filename)s') values['filename'] = filename From ec79d023600edc9f173a9554312a615f19d1e883 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 16 Apr 2015 18:14:09 -0400 Subject: [PATCH 011/109] more cgen porting --- hub/kojihub.py | 65 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index ae2aa9a5..cd9e0b4d 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4933,7 +4933,7 @@ def import_old_image(old, name, version): # the old schema did not track buildroot directly, so we have to infer # by task id. # If the task had multiple buildroots, we chose the latest - query = QueryProcessor(columns=['id'], tables=['buildroot'], + query = QueryProcessor(columns=['buildroot_id'], tables=['standard_buildroot'], clauses=['task_id=%(task_id)i'], values=old, opts={'order': '-id', 'limit': 1}) br_id = query.singleValue(strict=False) @@ -5737,15 +5737,17 @@ def build_references(build_id, limit=None): ret['images'].extend(image_ids) # find timestamp of most recent use in a buildroot - q = """SELECT buildroot.create_event - FROM buildroot_listing - JOIN buildroot ON buildroot_listing.buildroot_id = buildroot.id - WHERE buildroot_listing.rpm_id = %(rpm_id)s - ORDER BY buildroot.create_event DESC - LIMIT 1""" + query = QueryProcessor( + fields=['standard_buildroot.create_event'] + tables=['buildroot_listing'], + joins=['standard_buildroot ON buildroot_listing.buildroot_id = buildroot.id'], + clauses=['buildroot_listing.rpm_id = %(rpm_id)s'], + opts={'order': '-standard_buildroot.create_event', 'limit': 1}) + ) event_id = -1 for (rpm_id,) in rpm_ids: - tmp_id = _singleValue(q, locals(), strict=False) + query.values={'rpm_id': rpm_id} + tmp_id = query.singleValue(strict=False) if tmp_id is not None and tmp_id > event_id: event_id = tmp_id if event_id == -1: @@ -5807,6 +5809,10 @@ def delete_build(build, strict=True, min_ref_age=604800): if strict: raise koji.GenericError, "Cannot delete build, used in archive buildroots: %s" % refs['archives'] return False + if refs['images']: + if strict: + raise koji.GenericError, "Cannot delete build, used in images: %r" % refs['images'] + return False if refs['last_used']: age = time.time() - refs['last_used'] if age < min_ref_age: @@ -9386,21 +9392,38 @@ class BuildRoot(object): self.load(id) def load(self,id): - fields = ('id', 'host_id', 'repo_id', 'arch', 'task_id', - 'create_event', 'retire_event', 'state') - q = """SELECT %s FROM buildroot WHERE id=%%(id)i""" % (",".join(fields)) - data = _singleRow(q,locals(),fields,strict=False) - if data == None: + fields = { + 'buildroot_id': 'id', + 'host_id': 'host_id', + 'repo_id': 'repo_id', + 'buildroot.container_arch': 'arch', + 'task_id': 'task_id', + 'create_event': 'create_event', + 'retire_event': 'retire_event', + 'state': 'state'} + fields, aliases = zip(*fields) + query = QueryProcessor(fields = fields, aliases=aliases, + tables=['standard_buildroot'], + joins=['buildroot ON buildroot.id=standard_buildroot.buildroot_id'], + clauses=['id=%(id)s'], opts={'asList':True}) + data = query.execute() + if not data: raise koji.GenericError, 'no buildroot with ID: %i' % id self.id = id - self.data = data + self.data = data[0] - def new(self, host, repo, arch, task_id=None): + def new(self, host, repo, arch, task_id=None, ctype='chroot'): state = koji.BR_STATES['INIT'] - id = _singleValue("SELECT nextval('buildroot_id_seq')", strict=True) - q = """INSERT INTO buildroot(id,host_id,repo_id,arch,state,task_id) - VALUES (%(id)i,%(host)i,%(repo)i,%(arch)s,%(state)i,%(task_id)s)""" - _dml(q,locals()) + br_id = _singleValue("SELECT nextval('buildroot_id_seq')", strict=True) + insert = InsertProcessor('buildroot', data={'id': br_id}) + insert.set(container_arch=arch, container_type=ctype) + insert.set(br_type=koji.BR_TYPES['standard']) + insert.execute() + # and now the other table + insert = InsertProcessor('standard_buildroot') + insert.set(buildroot_id=br_id) + insert.set(host_id=host, repo_id=repo, task_id=task_id, state=state) + insert.execute() self.load(id) return self.id @@ -9434,7 +9457,7 @@ class BuildRoot(object): if state == koji.BR_STATES['INIT']: #we do not re-init buildroots raise koji.GenericError, "Cannot change buildroot state to INIT" - q = """SELECT state,retire_event FROM buildroot WHERE id=%(id)s FOR UPDATE""" + q = """SELECT state,retire_event FROM standard_buildroot WHERE id=%(id)s FOR UPDATE""" lstate,retire_event = _fetchSingle(q,locals(),strict=True) if koji.BR_STATES[lstate] == 'EXPIRED': #we will quietly ignore a request to expire an expired buildroot @@ -9446,7 +9469,7 @@ class BuildRoot(object): set = "state=%(state)s" if koji.BR_STATES[state] == 'EXPIRED': set += ",retire_event=get_event()" - update = """UPDATE buildroot SET %s WHERE id=%%(id)s""" % set + update = """UPDATE standard_buildroot SET %s WHERE id=%%(id)s""" % set _dml(update,locals()) self.data['state'] = state From 391a6194cc8a2ffb4630a645be9920fd3c46f556 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 16 Apr 2015 18:18:24 -0400 Subject: [PATCH 012/109] fix arch field in query_buildroots --- hub/kojihub.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index cd9e0b4d..39bc6d0f 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4110,8 +4110,9 @@ def query_buildroots(hostID=None, tagID=None, state=None, rpmID=None, archiveID= queryOpts - query options """ fields = [('buildroot.id', 'id'), - ('buildroot.host_arch', 'host_arch'), - ('buildroot.host_arch', 'arch'), #alias for back compat + ('buildroot.container_arch', 'container_arch'), + ('buildroot.container_arch', 'arch'), #alias for back compat + ('buildroot.container_type', 'container_type'), ('standard_buildroot.state', 'state'), ('standard_buildroot.task_id', 'task_id'), ('host.id', 'host_id'), ('host.name', 'host_name'), From 764509854307340f35848df4dd500e9c9cfb3e9d Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 27 Apr 2015 15:42:16 -0400 Subject: [PATCH 013/109] schema-update-cgen.sql; fix buildroot_extra_info primary key --- docs/schema-update-cgen.sql | 58 +++++++++++++++++++++++++++++++++++++ docs/schema.sql | 2 +- 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 docs/schema-update-cgen.sql diff --git a/docs/schema-update-cgen.sql b/docs/schema-update-cgen.sql new file mode 100644 index 00000000..78dabde8 --- /dev/null +++ b/docs/schema-update-cgen.sql @@ -0,0 +1,58 @@ +BEGIN; + +-- New tables +CREATE TABLE content_generator ( + id SERIAL PRIMARY KEY, + name TEXT +) WITHOUT OIDS; + +CREATE TABLE cg_users ( + cg_id INTEGER NOT NULL REFERENCES content_generator (id), + user_id INTEGER NOT NULL REFERENCES users (id), + PRIMARY KEY (cg_id, user_id) +-- XXX: should we version this? +) WITHOUT OIDS; + +CREATE TABLE buildroot_tools_info ( + buildroot_id INTEGER NOT NULL REFERENCES buildroot(id), + tool TEXT NOT NULL, + version TEXT NOT NULL, + PRIMARY KEY (buildroot_id, tool) +) WITHOUT OIDS; + +CREATE TABLE buildroot_extra_info ( + buildroot_id INTEGER NOT NULL REFERENCES buildroot(id), + key TEXT NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY (buildroot_id, key) +) WITHOUT OIDS; + + +CREATE TABLE image_archive_listing ( + image_id INTEGER NOT NULL REFERENCES image_archives(archive_id), + archive_id INTEGER NOT NULL REFERENCES archiveinfo(id), + UNIQUE (image_id, archive_id) +) WITHOUT OIDS; +CREATE INDEX image_listing_archives on image_archive_listing(archive_id); + + +CREATE TABLE standard_buildroot ( + buildroot_id INTEGER NOT NULL PRIMARY KEY REFERENCES buildroot(id), + host_id INTEGER NOT NULL REFERENCES host(id), + repo_id INTEGER NOT NULL REFERENCES repo (id), + task_id INTEGER NOT NULL REFERENCES task (id), + create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + retire_event INTEGER, + state INTEGER +) WITHOUT OIDS; + +-- the more complicated stuff + +INSERT INTO + standard_buildroot(buildroot_id, host_id, repo_id, task_id, create_event, retire_event, state) +SELECT id, host_id, repo_id, task_id, create_event, retire_event, state from buildroot; + + + + + diff --git a/docs/schema.sql b/docs/schema.sql index 91f260be..f96c4327 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -519,7 +519,7 @@ CREATE TABLE buildroot_tools_info ( ) WITHOUT OIDS; CREATE TABLE buildroot_extra_info ( - buildroot_id INTEGER NOT NULL PRIMARY KEY REFERENCES buildroot(id), + buildroot_id INTEGER NOT NULL REFERENCES buildroot(id), key TEXT NOT NULL, value TEXT NOT NULL, PRIMARY KEY (buildroot_id, key) From 1f4ca498bdd3b7d81d02a341255543b60ccf8395 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 27 Apr 2015 16:47:51 -0400 Subject: [PATCH 014/109] Getting cgen schema update script to working state --- docs/schema-update-cgen.sql | 70 ++++++++++++++++++++++++++++++------- docs/schema.sql | 2 +- 2 files changed, 59 insertions(+), 13 deletions(-) diff --git a/docs/schema-update-cgen.sql b/docs/schema-update-cgen.sql index 78dabde8..1d259a12 100644 --- a/docs/schema-update-cgen.sql +++ b/docs/schema-update-cgen.sql @@ -1,6 +1,9 @@ BEGIN; -- New tables + +SELECT now(), 'Creating new tables' as msg; + CREATE TABLE content_generator ( id SERIAL PRIMARY KEY, name TEXT @@ -36,23 +39,66 @@ CREATE TABLE image_archive_listing ( CREATE INDEX image_listing_archives on image_archive_listing(archive_id); -CREATE TABLE standard_buildroot ( - buildroot_id INTEGER NOT NULL PRIMARY KEY REFERENCES buildroot(id), - host_id INTEGER NOT NULL REFERENCES host(id), - repo_id INTEGER NOT NULL REFERENCES repo (id), - task_id INTEGER NOT NULL REFERENCES task (id), - create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), - retire_event INTEGER, - state INTEGER -) WITHOUT OIDS; +--CREATE TABLE standard_buildroot ( +-- buildroot_id INTEGER NOT NULL PRIMARY KEY REFERENCES buildroot(id), +-- host_id INTEGER NOT NULL REFERENCES host(id), +-- repo_id INTEGER NOT NULL REFERENCES repo (id), +-- task_id INTEGER NOT NULL REFERENCES task (id), +-- create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), +-- retire_event INTEGER, +-- state INTEGER +--) WITHOUT OIDS; -- the more complicated stuff -INSERT INTO - standard_buildroot(buildroot_id, host_id, repo_id, task_id, create_event, retire_event, state) -SELECT id, host_id, repo_id, task_id, create_event, retire_event, state from buildroot; +SELECT now(), 'Copying buildroot to standard_buildroot' as msg; +-- CREATE TABLE standard_buildroot AS TABLE buildroot; +CREATE TABLE standard_buildroot AS SELECT id,host_id,repo_id,task_id,create_event,retire_event,state from buildroot; + +SELECT now(), 'Fixing up standard_buildroot table' as msg; +-- ALTER TABLE standard_buildroot DROP COLUMN dirtyness; +-- ALTER TABLE standard_buildroot DROP COLUMN arch; +ALTER TABLE standard_buildroot RENAME id TO buildroot_id; +ALTER TABLE standard_buildroot ALTER COLUMN buildroot_id SET NOT NULL; +ALTER TABLE standard_buildroot ALTER COLUMN host_id SET NOT NULL; +ALTER TABLE standard_buildroot ALTER COLUMN repo_id SET NOT NULL; +ALTER TABLE standard_buildroot ALTER COLUMN task_id SET NOT NULL; +ALTER TABLE standard_buildroot ALTER COLUMN create_event SET NOT NULL; +ALTER TABLE standard_buildroot ALTER COLUMN create_event SET DEFAULT get_event(); +SELECT now(), 'Fixing up standard_buildroot table, foreign key constraints' as msg; +ALTER TABLE standard_buildroot ADD CONSTRAINT brfk FOREIGN KEY (buildroot_id) REFERENCES buildroot(id); +ALTER TABLE standard_buildroot ADD CONSTRAINT hfk FOREIGN KEY (host_id) REFERENCES host(id); +ALTER TABLE standard_buildroot ADD CONSTRAINT rfk FOREIGN KEY (repo_id) REFERENCES repo(id); +ALTER TABLE standard_buildroot ADD CONSTRAINT tfk FOREIGN KEY (task_id) REFERENCES task(id); +ALTER TABLE standard_buildroot ADD CONSTRAINT efk FOREIGN KEY (create_event) REFERENCES events(id) ; +SELECT now(), 'Fixing up standard_buildroot table, primary key' as msg; +ALTER TABLE standard_buildroot ADD PRIMARY KEY (buildroot_id); + + +SELECT now(), 'Altering buildroot table (dropping columns)' as msg; +ALTER TABLE buildroot DROP COLUMN host_id; +ALTER TABLE buildroot DROP COLUMN repo_id; +ALTER TABLE buildroot DROP COLUMN task_id; +ALTER TABLE buildroot DROP COLUMN create_event; +ALTER TABLE buildroot DROP COLUMN retire_event; +ALTER TABLE buildroot DROP COLUMN state; +ALTER TABLE buildroot DROP COLUMN dirtyness; + +SELECT now(), 'Altering buildroot table (adding columns)' as msg; +ALTER TABLE buildroot ADD COLUMN br_type INTEGER NOT NULL DEFAULT 0; +ALTER TABLE buildroot ADD COLUMN cg_id INTEGER REFERENCES content_generator (id); +ALTER TABLE buildroot ADD COLUMN cg_version TEXT; +ALTER TABLE buildroot ADD COLUMN container_type TEXT; +ALTER TABLE buildroot ADD COLUMN host_os TEXT; + +SELECT now(), 'Altering buildroot table (altering columns)' as msg; +ALTER TABLE buildroot RENAME arch TO container_arch; +ALTER TABLE buildroot ALTER COLUMN container_arch TYPE TEXT; +ALTER TABLE buildroot ALTER COLUMN br_type DROP DEFAULT; +-- TODO +ROLLBACK; -- XXX diff --git a/docs/schema.sql b/docs/schema.sql index f96c4327..9f7ab2e2 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -488,7 +488,7 @@ CREATE TABLE buildroot ( id SERIAL NOT NULL PRIMARY KEY, br_type INTEGER NOT NULL cg_id INTEGER REFERENCES content_generator (id), - cg_version version TEXT, + cg_version TEXT, CONSTRAINT cg_sane CHECK ( (cg_id IS NULL AND cg_version IS NULL) OR (cg_id IS NOT NULL AND cg_version IS NOT NULL)), From 1ccd2209f405aba09f392d56bec51181d2812fbc Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 27 Apr 2015 21:19:37 -0400 Subject: [PATCH 015/109] clean up cgen schema update script --- docs/schema-update-cgen.sql | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/docs/schema-update-cgen.sql b/docs/schema-update-cgen.sql index 1d259a12..f7a0d26a 100644 --- a/docs/schema-update-cgen.sql +++ b/docs/schema-update-cgen.sql @@ -13,7 +13,6 @@ CREATE TABLE cg_users ( cg_id INTEGER NOT NULL REFERENCES content_generator (id), user_id INTEGER NOT NULL REFERENCES users (id), PRIMARY KEY (cg_id, user_id) --- XXX: should we version this? ) WITHOUT OIDS; CREATE TABLE buildroot_tools_info ( @@ -39,25 +38,14 @@ CREATE TABLE image_archive_listing ( CREATE INDEX image_listing_archives on image_archive_listing(archive_id); ---CREATE TABLE standard_buildroot ( --- buildroot_id INTEGER NOT NULL PRIMARY KEY REFERENCES buildroot(id), --- host_id INTEGER NOT NULL REFERENCES host(id), --- repo_id INTEGER NOT NULL REFERENCES repo (id), --- task_id INTEGER NOT NULL REFERENCES task (id), --- create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), --- retire_event INTEGER, --- state INTEGER ---) WITHOUT OIDS; - -- the more complicated stuff SELECT now(), 'Copying buildroot to standard_buildroot' as msg; --- CREATE TABLE standard_buildroot AS TABLE buildroot; CREATE TABLE standard_buildroot AS SELECT id,host_id,repo_id,task_id,create_event,retire_event,state from buildroot; +-- doing it this way and fixing up after is *much* faster than creating the empty table +-- and using insert..select to populate SELECT now(), 'Fixing up standard_buildroot table' as msg; --- ALTER TABLE standard_buildroot DROP COLUMN dirtyness; --- ALTER TABLE standard_buildroot DROP COLUMN arch; ALTER TABLE standard_buildroot RENAME id TO buildroot_id; ALTER TABLE standard_buildroot ALTER COLUMN buildroot_id SET NOT NULL; ALTER TABLE standard_buildroot ALTER COLUMN host_id SET NOT NULL; @@ -96,9 +84,5 @@ ALTER TABLE buildroot RENAME arch TO container_arch; ALTER TABLE buildroot ALTER COLUMN container_arch TYPE TEXT; ALTER TABLE buildroot ALTER COLUMN br_type DROP DEFAULT; - - --- TODO - -ROLLBACK; -- XXX +COMMIT; From 9199fa3e54723b20093a4c2092a543fc9e749a12 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 27 Apr 2015 21:36:33 -0400 Subject: [PATCH 016/109] typos --- hub/kojihub.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 39bc6d0f..1584fa0c 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -5739,12 +5739,11 @@ def build_references(build_id, limit=None): # find timestamp of most recent use in a buildroot query = QueryProcessor( - fields=['standard_buildroot.create_event'] + fields=['standard_buildroot.create_event'], tables=['buildroot_listing'], joins=['standard_buildroot ON buildroot_listing.buildroot_id = buildroot.id'], clauses=['buildroot_listing.rpm_id = %(rpm_id)s'], opts={'order': '-standard_buildroot.create_event', 'limit': 1}) - ) event_id = -1 for (rpm_id,) in rpm_ids: query.values={'rpm_id': rpm_id} From 9867818a3220e5b68bbe463ce47f61590976fc37 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 4 May 2015 10:31:11 -0400 Subject: [PATCH 017/109] fix several small mistakes in the cgen modifications --- hub/kojihub.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 1584fa0c..937cd995 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2403,7 +2403,7 @@ def repo_references(repo_id): fields, aliases = zip(*fields.items()) values = {'repo_id': repo_id} clauses = ['repo_id=%(repo_id)s', 'retire_event IS NULL'] - query = QueryProcessor(fields=fields, aliases=aliases, tables=['standard_buildroot'], + query = QueryProcessor(columns=fields, aliases=aliases, tables=['standard_buildroot'], clauses=clauses, values=values, queryOpts = {'asList':True}) #check results for bad states ret = [] @@ -5739,9 +5739,9 @@ def build_references(build_id, limit=None): # find timestamp of most recent use in a buildroot query = QueryProcessor( - fields=['standard_buildroot.create_event'], + columns=['standard_buildroot.create_event'], tables=['buildroot_listing'], - joins=['standard_buildroot ON buildroot_listing.buildroot_id = buildroot.id'], + joins=['standard_buildroot ON buildroot_listing.buildroot_id = standard_buildroot.buildroot_id'], clauses=['buildroot_listing.rpm_id = %(rpm_id)s'], opts={'order': '-standard_buildroot.create_event', 'limit': 1}) event_id = -1 @@ -9401,11 +9401,12 @@ class BuildRoot(object): 'create_event': 'create_event', 'retire_event': 'retire_event', 'state': 'state'} - fields, aliases = zip(*fields) - query = QueryProcessor(fields = fields, aliases=aliases, + fields, aliases = zip(*fields.items()) + query = QueryProcessor(columns=fields, aliases=aliases, tables=['standard_buildroot'], joins=['buildroot ON buildroot.id=standard_buildroot.buildroot_id'], - clauses=['id=%(id)s'], opts={'asList':True}) + values={'id': id}, + clauses=['id=%(id)s']) data = query.execute() if not data: raise koji.GenericError, 'no buildroot with ID: %i' % id @@ -9417,14 +9418,14 @@ class BuildRoot(object): br_id = _singleValue("SELECT nextval('buildroot_id_seq')", strict=True) insert = InsertProcessor('buildroot', data={'id': br_id}) insert.set(container_arch=arch, container_type=ctype) - insert.set(br_type=koji.BR_TYPES['standard']) + insert.set(br_type=koji.BR_TYPES['STANDARD']) insert.execute() # and now the other table insert = InsertProcessor('standard_buildroot') insert.set(buildroot_id=br_id) insert.set(host_id=host, repo_id=repo, task_id=task_id, state=state) insert.execute() - self.load(id) + self.load(br_id) return self.id def verifyTask(self,task_id): @@ -9457,7 +9458,7 @@ class BuildRoot(object): if state == koji.BR_STATES['INIT']: #we do not re-init buildroots raise koji.GenericError, "Cannot change buildroot state to INIT" - q = """SELECT state,retire_event FROM standard_buildroot WHERE id=%(id)s FOR UPDATE""" + q = """SELECT state,retire_event FROM standard_buildroot WHERE buildroot_id=%(id)s FOR UPDATE""" lstate,retire_event = _fetchSingle(q,locals(),strict=True) if koji.BR_STATES[lstate] == 'EXPIRED': #we will quietly ignore a request to expire an expired buildroot @@ -9469,7 +9470,7 @@ class BuildRoot(object): set = "state=%(state)s" if koji.BR_STATES[state] == 'EXPIRED': set += ",retire_event=get_event()" - update = """UPDATE standard_buildroot SET %s WHERE id=%%(id)s""" % set + update = """UPDATE standard_buildroot SET %s WHERE buildroot_id=%%(id)s""" % set _dml(update,locals()) self.data['state'] = state From d70477696a928c0c7b1a1b206ab8fa56f040f451 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 4 May 2015 17:08:29 -0400 Subject: [PATCH 018/109] More query fixes --- hub/kojihub.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 937cd995..2c0eebd5 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -5756,11 +5756,11 @@ def build_references(build_id, limit=None): q = """SELECT EXTRACT(EPOCH FROM get_event_time(%(event_id)i))""" ret['last_used'] = _singleValue(q, locals()) - q = """SELECT buildroot.create_event + q = """SELECT standard_buildroot.create_event FROM buildroot_archives - JOIN buildroot ON buildroot_archives.buildroot_id = buildroot.id + JOIN standard_buildroot ON buildroot_archives.buildroot_id = standard_buildroot.buildroot_id WHERE buildroot_archives.archive_id = %(archive_id)i - ORDER BY buildroot.create_event DESC + ORDER BY standard_buildroot.create_event DESC LIMIT 1""" event_id = -1 for (archive_id,) in archive_ids: @@ -9406,7 +9406,7 @@ class BuildRoot(object): tables=['standard_buildroot'], joins=['buildroot ON buildroot.id=standard_buildroot.buildroot_id'], values={'id': id}, - clauses=['id=%(id)s']) + clauses=['buildroot_id=%(id)s']) data = query.execute() if not data: raise koji.GenericError, 'no buildroot with ID: %i' % id From f64914a0e5da5ed5e0564277ecb10e49d99aa395 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 18 Jun 2015 23:49:58 -0400 Subject: [PATCH 019/109] cg_import stub --- hub/kojihub.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/hub/kojihub.py b/hub/kojihub.py index 32169005..4c4c123c 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4562,6 +4562,58 @@ def import_rpm(fn,buildinfo=None,brootid=None,wrapper=False): rpminfo['brootid'] = brootid return rpminfo + +def cg_import(metadata, files): + """ Import build from a content generator""" + + # TODO: assert cg access + + metaver = metadata['metadata_version'] + if metaver != 0: + raise koji.GenericError("Unknown metadata version: %r", metaver) + + # TODO: pre import callback + + # build entry + buildinfo = get_build(metadata['build'], strict=False) + if buildinfo: + # TODO : allow in some cases + raise koji.GenericError("Build already exists: %r", buildinfo) + else: + # create a new build + # TODO: how do we deal with epoch? + build_id = new_build(rpminfo) + buildinfo = get_build(build_id, strict=True) + + # buildroots + brmap = {} + for brdata in metadata['buildroots']: + brfakeid = brdata['id'] + if brfakeid in brmap: + raise koji.GenericError("Duplicate buildroot id in metadata: %r", brfakeid) + brmap[brfakeid] = import_buildroot(brdata) + + # outputs + for fileinfo in metadata['output']: + brinfo = brmap.get(fileinfo['buildroot_id']) + if not brinfo: + raise koji.GenericError("Missing buildroot metadata for id %(buildroot_id)r", + fileinfo) + # TODO map info to files entry (determine fn) + if fileinfo['type'] == 'rpm': + rpminfo = import_rpm(fn, buildinfo, brinfo) + import_rpm_file(fn, buildinfo, rpminfo) + add_rpm_sig(rpminfo['id'], koji.rip_rpm_sighdr(fn)) + elif fileinfo['type'] == 'log': + # TODO: determine subdir + import_build_log(fn, buildinfo, subdir=None) + else: + # TODO support other types + raise koji.GenericError("Unsupported file type in import: %r", fileinfo['type']) + + # TODO: post import callback + + def add_external_rpm(rpminfo, external_repo, strict=True): """Add an external rpm entry to the rpminfo table From 29f1c568b3ff9c75016e37212947bc2f12d6be0a Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 24 Aug 2015 13:00:54 -0400 Subject: [PATCH 020/109] provide CGImport call --- hub/kojihub.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hub/kojihub.py b/hub/kojihub.py index 4c4c123c..e282a09c 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -7776,6 +7776,10 @@ class RootExports(object): fullpath = '%s/%s' % (koji.pathinfo.work(), filepath) import_archive(fullpath, buildinfo, type, typeInfo) + def CGImport(self, metadata, files): + context.session.assertPerm('admin') # TODO: fix access check + return cg_import(metadata, files) + untaggedBuilds = staticmethod(untagged_builds) tagHistory = staticmethod(tag_history) queryHistory = staticmethod(query_history) From 019786d9696a5ed4dcaa965a8c0095b0ffbe8b7f Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 27 Aug 2015 15:45:37 -0400 Subject: [PATCH 021/109] cg_export --- hub/kojihub.py | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/hub/kojihub.py b/hub/kojihub.py index e282a09c..4a6ef2bd 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4614,6 +4614,92 @@ def cg_import(metadata, files): # TODO: post import callback + +def cg_export(build): + """Return CG metadata and file refs for a given build""" + + binfo = get_build(build, strict=True) + # TODO: handle multiple build types + metadata = { + 'metadata_version' : 0, + 'build' : {}, + 'buildroots' : {}, + 'output' : {}, + } + metadata['build'] = dslice(binfo, ['name', 'version', 'release']) + # TODO: get source from task if possible + # TODO: get start_time and end_time from task and or build + + metadata['output'] = [] + brmap = {} + + # gather rpms + for rpminfo in list_rpms(buildID=binfo['id']): + data = {} + data['filename'] = koji.pathinfo.rpm(rpminfo) # XXX [?] + data['filesize'] = rpminfo['size'] + data['arch'] = rpminfo['arch'] + data['checksum'] = rpminfo['payloadhash'] + data['checksum_type'] = 'sigmd5' + data['type'] = 'rpm' + br_id = rpminfo['buildroot_id'] + if br_id is None: + data['buildroot_id'] = None + else: + brmap.setdefault(br_id, len(brmap)) + data['buildroot_id'] = brmap[br_id] + metadata['output'].append(data) + + # archives too + for archiveinfo in list_archives(buildID=binfo['id']): + data = {} + data['filename'] = archiveinfo['filename'] + data['filesize'] = archiveinfo['size'] + data['arch'] = None #XXX + data['checksum'] = archiveinfo['checksum'] + data['checksum_type'] = koji.CHECKSUM_TYPES(archiveinfo['checksum_type']) + data['type'] = archiveinfo['type_name'] + br_id = archiveinfo['buildroot_id'] + if br_id is None: + data['buildroot_id'] = None + else: + brmap.setdefault(br_id, len(brmap)) + data['buildroot_id'] = brmap[br_id] + metadata['output'].append(data) + + # gather buildroot info + metadata['buildroots'] = [] + for br_id in brmap: + # host(os, arch), cg(name, version), container(type, arch), tools([name, version]) + # rpms([n,v,r,e,a,md5,sig]), archives([fn, sz, csum, sumtype]) + # extra + brinfo = get_buildroot(br_id) + data = {} + data['id'] = brmap[br_id] + data['container'] = {'type': 'mock', 'arch': brinfo['arch']} + data['host'] = {'os': 'unknown', 'arch': brinfo['arch']} #XXX + data['tools'] = [] + data['component_rpms'] = [] + for rpminfo in list_rpms(buildrootID=br_id): + info = dslice(rpminfo, ['name', 'version', 'release', 'epoch', 'arch']) + info['sigmd5'] = rpminfo['payloadhash'] + info['sig'] = None + data['component_rpms'].append(info) + data['component_archives'] = [] + for archiveinfo in list_archives(buildrootID=br_id): + info = dslice(archiveinfo, ['filename', 'checksum']) + info['filesize'] = info['size'] + info['checksum_type'] = koji.CHECKSUM_TYPES(archiveinfo['checksum_type']) + data['component_archives'].append(info) + data['extra'] = dslice(brinfo, ['host_id', 'host_name', 'repo_id', 'task_id']) + data['extra']['orig_id'] = brinfo['id'] + + #TODO also return paths to all output files relative to topdir + + #TODO logs + return metadata + + def add_external_rpm(rpminfo, external_repo, strict=True): """Add an external rpm entry to the rpminfo table @@ -7780,6 +7866,8 @@ class RootExports(object): context.session.assertPerm('admin') # TODO: fix access check return cg_import(metadata, files) + CGExport = staticmethod(cg_export) + untaggedBuilds = staticmethod(untagged_builds) tagHistory = staticmethod(tag_history) queryHistory = staticmethod(query_history) From c2caf04c5e01932ea3fafa2cec10fbcd25e81c32 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Fri, 28 Aug 2015 12:54:18 -0400 Subject: [PATCH 022/109] fix ambiguous buildroot_id column ref --- hub/kojihub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 4a6ef2bd..331b0619 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4128,7 +4128,7 @@ def query_buildroots(hostID=None, tagID=None, state=None, rpmID=None, archiveID= ('repo_create.id', 'repo_create_event_id'), ('repo_create.time', 'repo_create_event_time')] tables = ['buildroot'] - joins=['LEFT OUTER JOIN standard_buildroot ON buildroot_id = buildroot.id', + joins=['LEFT OUTER JOIN standard_buildroot ON standard_buildroot.buildroot_id = buildroot.id', 'LEFT OUTER JOIN host ON host.id = standard_buildroot.host_id', 'LEFT OUTER JOIN repo ON repo.id = standard_buildroot.repo_id', 'tag ON tag.id = repo.tag_id', From 28abac4958da83e7600139395cabac3e8fe4d114 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Fri, 28 Aug 2015 12:58:11 -0400 Subject: [PATCH 023/109] cg_export fixes --- hub/kojihub.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 331b0619..2f3d3900 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4618,14 +4618,10 @@ def cg_import(metadata, files): def cg_export(build): """Return CG metadata and file refs for a given build""" + metadata = {'metadata_version' : 0} + binfo = get_build(build, strict=True) # TODO: handle multiple build types - metadata = { - 'metadata_version' : 0, - 'build' : {}, - 'buildroots' : {}, - 'output' : {}, - } metadata['build'] = dslice(binfo, ['name', 'version', 'release']) # TODO: get source from task if possible # TODO: get start_time and end_time from task and or build @@ -4693,6 +4689,7 @@ def cg_export(build): data['component_archives'].append(info) data['extra'] = dslice(brinfo, ['host_id', 'host_name', 'repo_id', 'task_id']) data['extra']['orig_id'] = brinfo['id'] + metadata['buildroots'].append(data) #TODO also return paths to all output files relative to topdir From ba80fc1141946daf27529c35baa9bb289c97464c Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Fri, 28 Aug 2015 13:00:19 -0400 Subject: [PATCH 024/109] fix br components data in cg_export --- hub/kojihub.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 2f3d3900..ce870a11 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4676,13 +4676,13 @@ def cg_export(build): data['host'] = {'os': 'unknown', 'arch': brinfo['arch']} #XXX data['tools'] = [] data['component_rpms'] = [] - for rpminfo in list_rpms(buildrootID=br_id): + for rpminfo in list_rpms(componentBuildrootID=br_id): info = dslice(rpminfo, ['name', 'version', 'release', 'epoch', 'arch']) info['sigmd5'] = rpminfo['payloadhash'] info['sig'] = None data['component_rpms'].append(info) data['component_archives'] = [] - for archiveinfo in list_archives(buildrootID=br_id): + for archiveinfo in list_archives(componentBuildrootID=br_id): info = dslice(archiveinfo, ['filename', 'checksum']) info['filesize'] = info['size'] info['checksum_type'] = koji.CHECKSUM_TYPES(archiveinfo['checksum_type']) From 1d22b65e218a826728ffcda036f8babddfc4b19d Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Fri, 28 Aug 2015 13:50:41 -0400 Subject: [PATCH 025/109] include new br info in queries --- hub/kojihub.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/hub/kojihub.py b/hub/kojihub.py index ce870a11..3c2e954e 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4113,9 +4113,15 @@ def query_buildroots(hostID=None, tagID=None, state=None, rpmID=None, archiveID= queryOpts - query options """ fields = [('buildroot.id', 'id'), + ('buildroot.br_type', 'br_type'), + ('buildroot.cg_id', 'cg_id'), + ('content_generator.name', 'cg_name'), + ('buildroot.cg_version', 'cg_version'), ('buildroot.container_arch', 'container_arch'), ('buildroot.container_arch', 'arch'), #alias for back compat ('buildroot.container_type', 'container_type'), + ('buildroot.host_os', 'host_os'), + ('buildroot.host_arch', 'host_arch'), ('standard_buildroot.state', 'state'), ('standard_buildroot.task_id', 'task_id'), ('host.id', 'host_id'), ('host.name', 'host_name'), @@ -4129,6 +4135,7 @@ def query_buildroots(hostID=None, tagID=None, state=None, rpmID=None, archiveID= tables = ['buildroot'] joins=['LEFT OUTER JOIN standard_buildroot ON standard_buildroot.buildroot_id = buildroot.id', + 'LEFT OUTER JOIN content_generator ON buildroot.cg_id = content_generator.id', 'LEFT OUTER JOIN host ON host.id = standard_buildroot.host_id', 'LEFT OUTER JOIN repo ON repo.id = standard_buildroot.repo_id', 'tag ON tag.id = repo.tag_id', From 77e26b7e19e95de9759307cbe21cac56628e1505 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Wed, 26 Aug 2015 14:31:37 -0400 Subject: [PATCH 026/109] cgen cli stub --- cli/koji | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/cli/koji b/cli/koji index 908d829e..9ab54e96 100755 --- a/cli/koji +++ b/cli/koji @@ -33,6 +33,13 @@ try: import ast except ImportError: ast = None +try: + import json +except ImportError: + try: + import simplejson as json + except ImportError: + json = None import ConfigParser import base64 import errno @@ -1632,6 +1639,34 @@ def handle_import(options, session, args): do_import(path, data) +def handle_import_cg(options, session, args): + "[admin] Import external builds with rich metadata" + usage = _("usage: %prog import-cg [options] metadata_file files_dir") + usage += _("\n(Specify the --help global option for a list of other help options)") + parser = OptionParser(usage=usage) + parser.add_option("--link", action="store_true", help=_("Attempt to hardlink instead of uploading")) + parser.add_option("--test", action="store_true", help=_("Don't actually import")) + (options, args) = parser.parse_args(args) + if len(args) < 2: + parser.error(_("Please specify metadata and at least one file")) + assert False + if json is None: + parser.error(_("Unable to find json module")) + assert False + activate_session(session) + metadata = json.load(file(args[0], 'r')) + if 'output' not in metadata: + print _("Metadata contains no output") + sys.exit(1) + + # TODO: determine upload path + for info in metadata['output']: + # TODO: upload file + pass + + session.CGImport(metadata, upload_path) + + def handle_import_comps(options, session, args): "Import group/package information from a comps file" usage = _("usage: %prog import-comps [options] ") From 91c877d3598c283a5aabb70c94b9341ca0758664 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 21 Sep 2015 14:47:56 -0400 Subject: [PATCH 027/109] add/remove user from cg --- cli/koji | 8 ++++ docs/schema-update-cgen.sql | 18 ++++++-- docs/schema.sql | 14 ++++++- hub/kojihub.py | 83 ++++++++++++++++++++++++++++++++----- 4 files changed, 107 insertions(+), 16 deletions(-) diff --git a/cli/koji b/cli/koji index 9ab54e96..a06e8c8c 100755 --- a/cli/koji +++ b/cli/koji @@ -3867,6 +3867,13 @@ def _print_histline(entry, **kwargs): fmt = "user %(user.name)s added to group %(group.name)s" else: fmt = "user %(user.name)s removed from group %(group.name)s" + elif table == 'cg_users': + if edit: + fmt = "user %(user.name)s re-added to content generator %(content_generator.name)s" + elif create: + fmt = "user %(user.name)s added to content generator %(content_generator.name)s" + else: + fmt = "user %(user.name)s removed from content generator %(content_generator.name)s" elif table == 'tag_packages': if edit: fmt = "package list entry for %(package.name)s in %(tag.name)s updated" @@ -3995,6 +4002,7 @@ def _print_histline(entry, **kwargs): _table_keys = { 'user_perms' : ['user_id', 'perm_id'], 'user_groups' : ['user_id', 'group_id'], + 'cg_users' : ['user_id', 'cg_id'], 'tag_inheritance' : ['tag_id', 'parent_id'], 'tag_config' : ['tag_id'], 'build_target_config' : ['build_target_id'], diff --git a/docs/schema-update-cgen.sql b/docs/schema-update-cgen.sql index f7a0d26a..2e435eee 100644 --- a/docs/schema-update-cgen.sql +++ b/docs/schema-update-cgen.sql @@ -10,11 +10,22 @@ CREATE TABLE content_generator ( ) WITHOUT OIDS; CREATE TABLE cg_users ( - cg_id INTEGER NOT NULL REFERENCES content_generator (id), - user_id INTEGER NOT NULL REFERENCES users (id), - PRIMARY KEY (cg_id, user_id) + cg_id INTEGER NOT NULL REFERENCES content_generator (id), + user_id INTEGER NOT NULL REFERENCES users (id), +-- versioned + create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + revoke_event INTEGER REFERENCES events(id), + creator_id INTEGER NOT NULL REFERENCES users(id), + revoker_id INTEGER REFERENCES users(id), + active BOOLEAN DEFAULT 'true' CHECK (active), + CONSTRAINT active_revoke_sane CHECK ( + (active IS NULL AND revoke_event IS NOT NULL AND revoker_id IS NOT NULL) + OR (active IS NOT NULL AND revoke_event IS NULL AND revoker_id IS NULL)), + PRIMARY KEY (create_event, cg_id, user_id), + UNIQUE (cg_id, user_id, active) ) WITHOUT OIDS; + CREATE TABLE buildroot_tools_info ( buildroot_id INTEGER NOT NULL REFERENCES buildroot(id), tool TEXT NOT NULL, @@ -78,6 +89,7 @@ ALTER TABLE buildroot ADD COLUMN cg_id INTEGER REFERENCES content_generator (id) ALTER TABLE buildroot ADD COLUMN cg_version TEXT; ALTER TABLE buildroot ADD COLUMN container_type TEXT; ALTER TABLE buildroot ADD COLUMN host_os TEXT; +ALTER TABLE buildroot ADD COLUMN host_arch TEXT; SELECT now(), 'Altering buildroot table (altering columns)' as msg; ALTER TABLE buildroot RENAME arch TO container_arch; diff --git a/docs/schema.sql b/docs/schema.sql index 12b9f39f..fdf0c51a 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -476,11 +476,21 @@ CREATE TABLE content_generator ( name TEXT ) WITHOUT OIDS; + CREATE TABLE cg_users ( cg_id INTEGER NOT NULL REFERENCES content_generator (id), user_id INTEGER NOT NULL REFERENCES users (id), - PRIMARY KEY (cg_id, user_id) --- XXX: should we version this? +-- versioned - see earlier description of versioning + create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + revoke_event INTEGER REFERENCES events(id), + creator_id INTEGER NOT NULL REFERENCES users(id), + revoker_id INTEGER REFERENCES users(id), + active BOOLEAN DEFAULT 'true' CHECK (active), + CONSTRAINT active_revoke_sane CHECK ( + (active IS NULL AND revoke_event IS NOT NULL AND revoker_id IS NOT NULL) + OR (active IS NOT NULL AND revoke_event IS NULL AND revoker_id IS NULL)), + PRIMARY KEY (create_event, cg_id, user_id), + UNIQUE (cg_id, user_id, active) ) WITHOUT OIDS; diff --git a/hub/kojihub.py b/hub/kojihub.py index 3c2e954e..1cb70ab0 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -3164,27 +3164,38 @@ def get_external_repo_list(tag_info, event=None): seen_repos[tag_repo['external_repo_id']] = 1 return repos -def get_user(userInfo=None,strict=False): - """Return information about a user. userInfo may be either a str - (Kerberos principal) or an int (user id). A map will be returned with the - following keys: + +def get_user(userInfo=None, strict=False): + """Return information about a user. + + userInfo may be either a str (Kerberos principal or name) or an int (user id). + + A map will be returned with the following keys: id: user id name: user name status: user status (int), may be null usertype: user type (int), 0 person, 1 for host, may be null - krb_principal: the user's Kerberos principal""" + krb_principal: the user's Kerberos principal + """ if userInfo is None: userInfo = context.session.user_id - #will still be None if not logged in - fields = ('id', 'name', 'status', 'usertype', 'krb_principal') - q = """SELECT %s FROM users WHERE""" % ', '.join(fields) + if userInfo is None: + # not logged in + raise koji.GenericError("No user provided") + fields = ['id', 'name', 'status', 'usertype', 'krb_principal'] + #fields, aliases = zip(*fields.items()) + data = {'info' : userInfo} if isinstance(userInfo, int) or isinstance(userInfo, long): - q += """ id = %(userInfo)i""" + clauses = ['id = %(info)i'] elif isinstance(userInfo, str): - q += """ (krb_principal = %(userInfo)s or name = %(userInfo)s)""" + clauses = ['krb_principal = %(info)s OR name = %(info)s'] else: raise koji.GenericError, 'invalid type for userInfo: %s' % type(userInfo) - return _singleRow(q,locals(),fields,strict=strict) + query = QueryProcessor(tables=['users'], columns=fields, + clauses=clauses, values=data) + user = query.executeOne() + return user + def find_build_id(X, strict=False): if isinstance(X,int) or isinstance(X,long): @@ -5615,6 +5626,7 @@ def query_history(tables=None, **kwargs): table_fields = { 'user_perms' : ['user_id', 'perm_id'], 'user_groups' : ['user_id', 'group_id'], + 'cg_users' : ['user_id', 'cg_id'], 'tag_inheritance' : ['tag_id', 'parent_id', 'priority', 'maxdepth', 'intransitive', 'noconfig', 'pkg_filter'], 'tag_config' : ['tag_id', 'arches', 'perm_id', 'locked', 'maven_support', 'maven_include_all'], 'build_target_config' : ['build_target_id', 'build_tag', 'dest_tag'], @@ -5632,6 +5644,7 @@ def query_history(tables=None, **kwargs): #field : [table, join-alias, alias] 'user_id' : ['users', 'users', 'user'], 'perm_id' : ['permissions', 'permission'], + 'cg_id' : ['content_generator'], #group_id is overloaded (special case below) 'tag_id' : ['tag'], 'parent_id' : ['tag', 'parent'], @@ -6417,6 +6430,33 @@ def set_user_status(user, status): raise koji.GenericError, 'invalid user ID: %i' % user_id +def add_user_to_cg(user, cg): + """Associate a user with a content generator""" + + context.session.assertPerm('admin') + user = get_user(user, strict=True) + cg = lookup_name('content_generator', cg, strict=True) + ins = InsertProcessor('cg_users') + ins.set(cg_id=cg['id'], user_id=user['id']) + ins.make_create() + if ins.dup_check(): + raise koji.GenericError("User already associated with content generator") + ins.execute() + + +def remove_user_from_cg(user, cg): + """De-associate a user with a content generator""" + + context.session.assertPerm('admin') + user = get_user(user, strict=True) + cg = lookup_name('content_generator', cg, strict=True) + data = {'user_id': user['id'], 'cg_id' : cg['id']} + update = UpdateProcessor('cg_users', values=data, + clauses=["user_id = %(user_id)i", "cg_id = %(cg_id)i"]) + update.make_revoke() + update.execute() + + def get_event(): """Get an event id for this transaction @@ -6487,6 +6527,24 @@ class InsertProcessor(object): self.data['create_event'] = event_id self.data['creator_id'] = user_id + def dup_check(self): + """Check to see if the insert duplicates an existing row""" + if self.rawdata: + logger.warning("Can't perform duplicate check") + return None + data = self.data.copy() + if 'create_event' in self.data: + # versioned table + data['active'] = True + del data['create_event'] + del data['creator_id'] + clauses = ["%s = %%(%s)s" % (k, k) for k in data] + query = QueryProcessor(columns=data.keys(), tables=[self.table], + clauses=clauses, values=data) + if query.execute(): + return True + return False + def execute(self): return _dml(str(self), self.data) @@ -8915,6 +8973,9 @@ class RootExports(object): raise koji.GenericError, 'unknown user: %s' % username set_user_status(user, koji.USER_STATUS['BLOCKED']) + addUserToCG = staticmethod(add_user_to_cg) + removeUserFromCG = staticmethod(remove_user_from_cg) + #group management calls newGroup = staticmethod(new_group) addGroupMember = staticmethod(add_group_member) From a09a898cf08f62fa5f7e9fa7cb51e01341648cd1 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 21 Sep 2015 16:58:54 -0400 Subject: [PATCH 028/109] assert cg access --- hub/kojihub.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 2fcbd466..7ddfcdf8 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4632,12 +4632,19 @@ def import_rpm(fn,buildinfo=None,brootid=None,wrapper=False): def cg_import(metadata, files): """ Import build from a content generator""" - # TODO: assert cg access - metaver = metadata['metadata_version'] if metaver != 0: raise koji.GenericError("Unknown metadata version: %r", metaver) + # assert cg access + cgs = set() + for brdata in metadata['buildroots']: + cginfo = brdata['content_generator'] + cg = lookup_name('content_generator', cginfo['name'], strict=True) + cgs.add(cg['id']) + for cg_id in cgs: + assert_cg(cg_id) + # TODO: pre import callback # build entry @@ -6506,6 +6513,20 @@ def remove_user_from_cg(user, cg): update.execute() +def assert_cg(cg, user=None): + cg = lookup_name('content_generator', cg, strict=True) + if user is None: + if context.session.logged_in: + raise koji.AuthError("Not logged in") + user = context.session.user_id + user = get_user(user, strict=True) + clauses = ['active = TRUE', 'user_id = %(user_id)s', 'cg_id = %(cg_id)s'] + data = {'user_id' : user['id'], 'cg_id' : cg['id']} + query = QueryProcessor(tables='cg_users', fields=['cg_id'], clauses=clauses, values=data) + if not query.execute(): + raise koji.AuthError("Content generator access required (%s)" % cg['name']) + + def get_event(): """Get an event id for this transaction From c31ad75740141631ea1edfdd1a62869def502ea8 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 21 Sep 2015 18:35:30 -0400 Subject: [PATCH 029/109] partial work --- hub/kojihub.py | 82 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 63 insertions(+), 19 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 7ddfcdf8..e0573ff0 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4647,6 +4647,10 @@ def cg_import(metadata, files): # TODO: pre import callback + # TODO: basic metadata sanity check (use jsonschema?) + + # TODO: policy hooks + # build entry buildinfo = get_build(metadata['build'], strict=False) if buildinfo: @@ -4654,8 +4658,10 @@ def cg_import(metadata, files): raise koji.GenericError("Build already exists: %r", buildinfo) else: # create a new build - # TODO: how do we deal with epoch? - build_id = new_build(rpminfo) + buildinfo = dslice(metadata['build'], ['name', 'version', 'release']) + # epoch is not in the metadata spec, but we allow it to be specified + buildinfo['epoch'] = metadata['build'].get('epoch', None) + build_id = new_build(buildinfo) buildinfo = get_build(build_id, strict=True) # buildroots @@ -4664,7 +4670,7 @@ def cg_import(metadata, files): brfakeid = brdata['id'] if brfakeid in brmap: raise koji.GenericError("Duplicate buildroot id in metadata: %r", brfakeid) - brmap[brfakeid] = import_buildroot(brdata) + brmap[brfakeid] = cg_import_buildroot(brdata) # outputs for fileinfo in metadata['output']: @@ -4687,6 +4693,24 @@ def cg_import(metadata, files): # TODO: post import callback +def cg_import_buildroot(brdata): + """Import the given buildroot data""" + + # buildroot entry + buildroot_id = + + # standard buildroot entry (if applicable) + + # buildroot_listing + + # buildroot_archives + + # buildroot_tools_info + + # buildroot_extra_info + + return buildroot_id + def cg_export(build): """Return CG metadata and file refs for a given build""" @@ -9820,27 +9844,47 @@ class BuildRoot(object): #load buildroot data self.load(id) - def load(self,id): - fields = { - 'buildroot_id': 'id', - 'host_id': 'host_id', - 'repo_id': 'repo_id', - 'buildroot.container_arch': 'arch', - 'task_id': 'task_id', - 'create_event': 'create_event', - 'retire_event': 'retire_event', - 'state': 'state'} - fields, aliases = zip(*fields.items()) - query = QueryProcessor(columns=fields, aliases=aliases, - tables=['standard_buildroot'], - joins=['buildroot ON buildroot.id=standard_buildroot.buildroot_id'], - values={'id': id}, - clauses=['buildroot_id=%(id)s']) + def load(self, id): + fields = [ + 'id', + 'br_type', + 'cg_id', + 'cg_version', + 'container_type', + 'container_arch', + 'host_os', + 'host_arch', + ] + query = QueryProcessor(columns=fields, tables=['buildroot'], + values={'id': id}, clauses=['id=%(id)s']) data = query.execute() if not data: raise koji.GenericError, 'no buildroot with ID: %i' % id self.id = id self.data = data[0] + if data.br_type == koji.BR_TYPES['STANDARD']: + self._load_standard() + else: + self.is_standard = False + + def _load_standard(self): + fields = [ + 'host_id', + 'repo_id', + 'task_id', + 'create_event', + 'retire_event', + 'state', + ] + query = QueryProcessor(columns=fields, tables=['standard_buildroot'], + values={'id': self.id}, clauses=['buildroot_id=%(id)s']) + data = query.execute() + if not data: + raise koji.GenericError, 'Not a standard buildroot: %i' % self.id + self.data.update(data) + # arch for compat + self.data['arch'] = self.data['container_arch'] + self.is_standard = True def new(self, host, repo, arch, task_id=None, ctype='chroot'): state = koji.BR_STATES['INIT'] From 0842e53abfc80771f939d0aea69f0a98d779fb78 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 21 Sep 2015 21:20:03 -0400 Subject: [PATCH 030/109] BuildRoot.cg_new() and related updates --- hub/kojihub.py | 128 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 84 insertions(+), 44 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index e0573ff0..be8452a5 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4697,7 +4697,7 @@ def cg_import_buildroot(brdata): """Import the given buildroot data""" # buildroot entry - buildroot_id = + buildroot_id = 'FOO' # standard buildroot entry (if applicable) @@ -9901,50 +9901,82 @@ class BuildRoot(object): self.load(br_id) return self.id - def verifyTask(self,task_id): + def cg_new(self, data): + """New content generator buildroot""" + + fields = [ + 'br_type', + 'cg_id', + 'cg_version', + 'container_type', + 'container_arch', + 'host_os', + 'host_arch', + ] + data.setdefault('br_type', koji.BR_STATES['EXTERNAL']) + data = dslice(data, fields) + for key in fields: + if key not in data: + raise koji.GenericError("Buildroot field %s not specified" % key) + br_id = _singleValue("SELECT nextval('buildroot_id_seq')", strict=True) + insert = InsertProcessor('buildroot') + insert.set(id = br_id, **data) + insert.execute() + + def assertStandard(self): if self.id is None: raise koji.GenericError, "buildroot not specified" + if not self.is_standard: + raise koji.GenericError('Not a standard buildroot: %s' % self.id) + + def verifyTask(self, task_id): + self.assertStandard() return (task_id == self.data['task_id']) - def assertTask(self,task_id): + def assertTask(self, task_id): + self.assertStandard() if not self.verifyTask(task_id): raise koji.ActionNotAllowed, 'Task %s does not have lock on buildroot %s' \ %(task_id,self.id) - def verifyHost(self,host_id): - if self.id is None: - raise koji.GenericError, "buildroot not specified" + def verifyHost(self, host_id): + self.assertStandard() return (host_id == self.data['host_id']) - def assertHost(self,host_id): + def assertHost(self, host_id): + self.assertStandard() if not self.verifyHost(host_id): raise koji.ActionNotAllowed, "Host %s not owner of buildroot %s" \ % (host_id,self.id) - def setState(self,state): - if self.id is None: - raise koji.GenericError, "buildroot not specified" - id = self.id - if isinstance(state,str): + def setState(self, state): + self.assertStandard() + if isinstance(state, str): state = koji.BR_STATES[state] #sanity checks if state == koji.BR_STATES['INIT']: #we do not re-init buildroots raise koji.GenericError, "Cannot change buildroot state to INIT" - q = """SELECT state,retire_event FROM standard_buildroot WHERE buildroot_id=%(id)s FOR UPDATE""" - lstate,retire_event = _fetchSingle(q,locals(),strict=True) - if koji.BR_STATES[lstate] == 'EXPIRED': + query = QueryProcessor(fields=['state', 'retire_event'], values=self.data, + tables=['standard_buildroot'], clauses=['buildroot_id=%(id)s'], + opts={'rowlock':True}) + row = query.executeOne() + if not row: + raise koji.GenericError("Unable to get state for buildroot %s" % self.id) + lstate,retire_event = ro + if koji.BR_STATES[row['state']] == 'EXPIRED': #we will quietly ignore a request to expire an expired buildroot #otherwise this is an error - if state == lstate: + if state == 'EXPIRED': return else: - raise koji.GenericError, "buildroot %i is EXPIRED" % id - set = "state=%(state)s" + raise koji.GenericError, "buildroot %i is EXPIRED" % self.id + update = UpdateProcessor('standard_buildroot', clauses=['buildroot_id=%(id)s'], + values=self.data) + update.set(state=state) if koji.BR_STATES[state] == 'EXPIRED': - set += ",retire_event=get_event()" - update = """UPDATE standard_buildroot SET %s WHERE buildroot_id=%%(id)s""" % set - _dml(update,locals()) + update.rawset(retire_event='get_event()') + update.execute() self.data['state'] = state def getList(self): @@ -9970,15 +10002,13 @@ class BuildRoot(object): values=locals()) return query.execute() - def _setList(self,rpmlist,update=False): + def _setList(self, rpmlist, update=False): """Set or update the list of rpms in a buildroot""" + if self.id is None: raise koji.GenericError, "buildroot not specified" - brootid = self.id if update: - current = dict([(r['rpm_id'],1) for r in self.getList()]) - q = """INSERT INTO buildroot_listing (buildroot_id,rpm_id,is_update) - VALUES (%(brootid)s,%(rpm_id)s,%(update)s)""" + current = set([r['rpm_id'] for r in self.getList()]) rpm_ids = [] for an_rpm in rpmlist: location = an_rpm.get('location') @@ -9988,24 +10018,30 @@ class BuildRoot(object): else: data = get_rpm(an_rpm, strict=True) rpm_id = data['id'] - if update and current.has_key(rpm_id): + if update and rpm_id in current: #ignore duplicate packages for updates continue rpm_ids.append(rpm_id) #we sort to try to avoid deadlock issues rpm_ids.sort() - for rpm_id in rpm_ids: - _dml(q, locals()) - def setList(self,rpmlist): + # actually do the inserts + insert = InsertProcessor('buildroot_listing') + insert.set(buildroot_id=self.id, is_update=bool(update)) + for rpm_id in rpm_ids: + insert.set(rpm_id=rpm_id) + insert.execute() + + def setList(self, rpmlist): """Set the initial list of rpms in a buildroot""" - if self.data['state'] != koji.BR_STATES['INIT']: + + if self.is_standard and self.data['state'] != koji.BR_STATES['INIT']: raise koji.GenericError, "buildroot %(id)s in wrong state %(state)s" % self.data self._setList(rpmlist,update=False) - def updateList(self,rpmlist): + def updateList(self, rpmlist): """Update the list of packages in a buildroot""" - if self.data['state'] != koji.BR_STATES['BUILDING']: + if self.is_standard and self.data['state'] != koji.BR_STATES['BUILDING']: raise koji.GenericError, "buildroot %(id)s in wrong state %(state)s" % self.data self._setList(rpmlist,update=True) @@ -10033,21 +10069,25 @@ class BuildRoot(object): def updateArchiveList(self, archives, project=False): """Update the list of archives in a buildroot. - If project is True, the archives are project dependencies. If False, they dependencies required to setup the - build environment.""" - if not (context.opts.get('EnableMaven') or context.opts.get('EnableWin')): - raise koji.GenericError, "non-rpm support is not enabled" - if self.data['state'] != koji.BR_STATES['BUILDING']: - raise koji.GenericError, "buildroot %(id)s in wrong state %(state)s" % self.data + + If project is True, the archives are project dependencies. + If False, they dependencies required to setup the build environment. + """ + + if self.is_standard: + if not (context.opts.get('EnableMaven') or context.opts.get('EnableWin')): + raise koji.GenericError, "non-rpm support is not enabled" + if self.data['state'] != koji.BR_STATES['BUILDING']: + raise koji.GenericError, "buildroot %(id)s in wrong state %(state)s" % self.data archives = set([r['id'] for r in archives]) current = set([r['id'] for r in self.getArchiveList()]) new_archives = archives.difference(current) - insert = """INSERT INTO buildroot_archives (buildroot_id, archive_id, project_dep) - VALUES - (%(broot_id)i, %(archive_id)i, %(project)s)""" - broot_id = self.id + insert = InsertProcessor('buildroot_archives') + insert.set(buildroot_id=self.id, project_dep=bool(project)) for archive_id in sorted(new_archives): - _dml(insert, locals()) + insert.set(archive_id=archive_id) + insert.execute() + class Host(object): From 850db82881a9b7e542e8f79bc2c8f6301d35b8d4 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 22 Sep 2015 12:33:58 -0400 Subject: [PATCH 031/109] fix int encoding in list_rpms --- hub/kojihub.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index be8452a5..1910f0f9 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -3522,12 +3522,18 @@ def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID else: raise koji.GenericError, 'invalid type for "arches" parameter: %s' % type(arches) - query = QueryProcessor(columns=[f[0] for f in fields], aliases=[f[1] for f in fields], + fields, aliases = zip(*fields) + query = QueryProcessor(columns=fields, aliases=aliases, tables=['rpminfo'], joins=joins, clauses=clauses, values=locals(), opts=queryOpts) data = query.execute() - for row in data: - row['size'] = koji.encode_int(row['size']) + if not (queryOpts and queryOpts.get('countOnly')): + if queryOpts and 'asList' in queryOpts: + key = aliases.index('size') + else: + key = 'size' + for row in data: + row[key] = koji.encode_int(row[key]) return data def get_maven_build(buildInfo, strict=False): From ce4e19739bae3b913ec8c35a55c97d1993aa41d3 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 22 Sep 2015 15:32:29 -0400 Subject: [PATCH 032/109] fix superclass ref --- hub/kojixmlrpc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hub/kojixmlrpc.py b/hub/kojixmlrpc.py index efb99a60..bcaff895 100644 --- a/hub/kojixmlrpc.py +++ b/hub/kojixmlrpc.py @@ -598,7 +598,8 @@ class HubFormatter(logging.Formatter): record.session_id = None record.callnum = None record.user_name = None - return logging.Formatter.format(self, record) + return super(HubFormatter, self).format(record) + def setup_logging1(): """Set up basic logging, before options are loaded""" From 27aa25cc264aae7ca03ba9ad9e4f69529a9ccdf3 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 22 Sep 2015 15:50:30 -0400 Subject: [PATCH 033/109] fix var name --- hub/kojihub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 1910f0f9..0000ec03 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -9868,7 +9868,7 @@ class BuildRoot(object): raise koji.GenericError, 'no buildroot with ID: %i' % id self.id = id self.data = data[0] - if data.br_type == koji.BR_TYPES['STANDARD']: + if self.data.br_type == koji.BR_TYPES['STANDARD']: self._load_standard() else: self.is_standard = False From 844bdc7495b49ad2fff6b525e964227f90262d3c Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 22 Sep 2015 15:53:28 -0400 Subject: [PATCH 034/109] fix data ref --- hub/kojihub.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 0000ec03..bb55c6f2 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -9863,12 +9863,12 @@ class BuildRoot(object): ] query = QueryProcessor(columns=fields, tables=['buildroot'], values={'id': id}, clauses=['id=%(id)s']) - data = query.execute() + data = query.executeOne() if not data: raise koji.GenericError, 'no buildroot with ID: %i' % id self.id = id - self.data = data[0] - if self.data.br_type == koji.BR_TYPES['STANDARD']: + self.data = data + if data['br_type'] == koji.BR_TYPES['STANDARD']: self._load_standard() else: self.is_standard = False @@ -9884,7 +9884,7 @@ class BuildRoot(object): ] query = QueryProcessor(columns=fields, tables=['standard_buildroot'], values={'id': self.id}, clauses=['buildroot_id=%(id)s']) - data = query.execute() + data = query.executeOne() if not data: raise koji.GenericError, 'Not a standard buildroot: %i' % self.id self.data.update(data) From 7d45a642a3877b369c111a1b75f12bc03b35a05c Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 22 Sep 2015 16:28:18 -0400 Subject: [PATCH 035/109] use cg_new, fix typos --- hub/kojihub.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index bb55c6f2..d5619187 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4699,15 +4699,27 @@ def cg_import(metadata, files): # TODO: post import callback -def cg_import_buildroot(brdata): +def cg_import_buildroot(cg_id, brdata): """Import the given buildroot data""" # buildroot entry - buildroot_id = 'FOO' + brinfo = { + 'cg_id' : cg_id, + 'cg_version' : brdata['content_generator']['version'], + 'container_type' : brdata['container']['type'], + 'container_arch' : brdata['container']['arch'], + 'host_os' : brdata['host']['os'], + 'host_arch' : brdata['host']['arch'], + } + br = BuildRoot() + br.cg_new(brinfo) # standard buildroot entry (if applicable) + # ??? # buildroot_listing + rpms = [r for r in brdata['components'] if r['type'] == 'rpm'] + # ... # buildroot_archives @@ -6552,7 +6564,7 @@ def assert_cg(cg, user=None): user = get_user(user, strict=True) clauses = ['active = TRUE', 'user_id = %(user_id)s', 'cg_id = %(cg_id)s'] data = {'user_id' : user['id'], 'cg_id' : cg['id']} - query = QueryProcessor(tables='cg_users', fields=['cg_id'], clauses=clauses, values=data) + query = QueryProcessor(tables='cg_users', columns=['cg_id'], clauses=clauses, values=data) if not query.execute(): raise koji.AuthError("Content generator access required (%s)" % cg['name']) @@ -9963,13 +9975,13 @@ class BuildRoot(object): if state == koji.BR_STATES['INIT']: #we do not re-init buildroots raise koji.GenericError, "Cannot change buildroot state to INIT" - query = QueryProcessor(fields=['state', 'retire_event'], values=self.data, + query = QueryProcessor(columns=['state', 'retire_event'], values=self.data, tables=['standard_buildroot'], clauses=['buildroot_id=%(id)s'], opts={'rowlock':True}) row = query.executeOne() if not row: raise koji.GenericError("Unable to get state for buildroot %s" % self.id) - lstate,retire_event = ro + lstate, retire_event = row if koji.BR_STATES[row['state']] == 'EXPIRED': #we will quietly ignore a request to expire an expired buildroot #otherwise this is an error From 16f367bce0c57815ecbf7f40c96a5ca1440515b9 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 22 Sep 2015 17:17:14 -0400 Subject: [PATCH 036/109] work on handling components from cg --- hub/kojihub.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index d5619187..b4e5e9a0 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4718,16 +4718,42 @@ def cg_import_buildroot(cg_id, brdata): # ??? # buildroot_listing - rpms = [r for r in brdata['components'] if r['type'] == 'rpm'] - # ... + rpmlist = [] + for comp in brdata['components']: + if comp['type'] != 'rpm': + continue + # TODO: do we allow inclusion of external rpms? + if 'location' in comp: + raise koji.GenericError("External rpms not allowed") + if 'id' in comp: + # not in metadata spec, and will confuse get_rpm + raise koji.GenericError("Unexpected 'id' field in component") + rinfo = get_rpm(comp, strict=True) + if rinfo['payloadhash'] != comp['sigmd5']: + nvr = "%(name)s-%(version)s-%(release)s" % rinfo + raise koji.GenericError("md5sum mismatch for %s: %s != %s" + % (nvr, comp['sigmd5'], rinfo['payloadhash'])) + # TODO - should we check the signature field? + rpmlist.append(rinfo) + br.setList(rpmlist) # buildroot_archives + archives = [] + for comp in brdata['components']: + if comp['type'] != 'rpm': + continue + # hmm, how do we look up archives? + # updateMavenBuildRootList does seriously wild stuff + # only unique field in archiveinfo is id + # checksum/checksum_type only works if they match + # at the moment, we only have md5 entries in archiveinfo + # buildroot_tools_info # buildroot_extra_info - return buildroot_id + return brinfo def cg_export(build): From 70946260b44447c8f174f6d8e2f1d1f4a16881c7 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 22 Sep 2015 19:47:38 -0400 Subject: [PATCH 037/109] assert buildroots are used --- hub/kojihub.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hub/kojihub.py b/hub/kojihub.py index b4e5e9a0..fdd636f8 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4671,9 +4671,12 @@ def cg_import(metadata, files): buildinfo = get_build(build_id, strict=True) # buildroots + br_used = set([f['buildroot_id'] for f in metadata['output']]) brmap = {} for brdata in metadata['buildroots']: brfakeid = brdata['id'] + if brfakeid not in br_used: + raise koji.GenericError("Buildroot id not used in output: %r", brfakeid) if brfakeid in brmap: raise koji.GenericError("Duplicate buildroot id in metadata: %r", brfakeid) brmap[brfakeid] = cg_import_buildroot(brdata) From abdf78d05277a2ddbf56b96125daa765f4508a60 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Wed, 23 Sep 2015 10:19:19 -0400 Subject: [PATCH 038/109] set tools/extra for br --- hub/kojihub.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index fdd636f8..7b2ebfe0 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4751,10 +4751,11 @@ def cg_import_buildroot(cg_id, brdata): # checksum/checksum_type only works if they match # at the moment, we only have md5 entries in archiveinfo - # buildroot_tools_info + br.setTools(brdata['tools']) # buildroot_extra_info + br.setExtra(brdata['extra']) return brinfo @@ -10135,6 +10136,26 @@ class BuildRoot(object): insert.set(archive_id=archive_id) insert.execute() + def setTools(self, tools): + """Set tools info for buildroot""" + + insert = InsertProcessor('buildroot_tools_info') + insert.set(buildroot_id=self.id) + for tool in tools: + insert.set(tool=tool['name']) + insert.set(version=tool['version']) + insert.execute() + + def setExtra(self, extra): + """Set extra info for buildroot""" + + insert = InsertProcessor('buildroot_extra_info') + insert.set(buildroot_id=self.id) + for key in extra: + insert.set(key=key) + insert.set(value=json.dumps(extra[key])) + insert.execute() + class Host(object): From 6a0526cc2d63e6d603acba8f4679d5795e559224 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Wed, 23 Sep 2015 11:35:30 -0400 Subject: [PATCH 039/109] cg archive listing --- hub/kojihub.py | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 7b2ebfe0..8cf4af06 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4717,14 +4717,23 @@ def cg_import_buildroot(cg_id, brdata): br = BuildRoot() br.cg_new(brinfo) - # standard buildroot entry (if applicable) + # TODO: standard buildroot entry (if applicable) # ??? + #split components + rpms = [] + files = [] + for comp in brdata['components']: + if comp['type'] == 'rpm': + rpms.append(comp) + elif comp['type'] == 'file': + files.append(comp) + else: + raise koji.GenericError("Unknown component type: %(type)s" % comp) + # buildroot_listing rpmlist = [] - for comp in brdata['components']: - if comp['type'] != 'rpm': - continue + for comp in rpms: # TODO: do we allow inclusion of external rpms? if 'location' in comp: raise koji.GenericError("External rpms not allowed") @@ -4742,15 +4751,29 @@ def cg_import_buildroot(cg_id, brdata): # buildroot_archives archives = [] - for comp in brdata['components']: - if comp['type'] != 'rpm': - continue + for comp in files: # hmm, how do we look up archives? # updateMavenBuildRootList does seriously wild stuff # only unique field in archiveinfo is id - # checksum/checksum_type only works if they match + # checksum/checksum_type only works if type matches # at the moment, we only have md5 entries in archiveinfo + type_mismatches = 0 + for archive in list_archives(filename=comp['filename'], size=comp['filesize']): + if archive['checksum_type'] != comp['checksum_type'] + type_mismatches += 1 + continue + if archive['checksum'] == comp['checksum']: + archives.appen(archive) + break + else: + logger.error("Failed to match archive %(filename)s (size %(filesize)s, sum %(checksum)s", comp) + if type_mismatches: + logger.error("Match failed with %i type mismatches", type_mismatches) + # TODO: allow external archives [??] + raise koji.GenericError("No match: %(filename)s (size %(filesize)s, sum %(checksum)s" % comp) + br.updateArchiveList(archives) + # buildroot_tools_info br.setTools(brdata['tools']) From 584e579982086870d6cef81cdd5bf33c9dc0ca2e Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Wed, 23 Sep 2015 15:32:21 -0400 Subject: [PATCH 040/109] stubs for cg output import handlers --- hub/kojihub.py | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 8cf4af06..ab3e1962 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4687,17 +4687,15 @@ def cg_import(metadata, files): if not brinfo: raise koji.GenericError("Missing buildroot metadata for id %(buildroot_id)r", fileinfo) + # TODO map info to files entry (determine fn) + if fileinfo['type'] == 'rpm': - rpminfo = import_rpm(fn, buildinfo, brinfo) - import_rpm_file(fn, buildinfo, rpminfo) - add_rpm_sig(rpminfo['id'], koji.rip_rpm_sighdr(fn)) + cg_import_rpm(buildinfo, brinfo, fileinfo) elif fileinfo['type'] == 'log': - # TODO: determine subdir - import_build_log(fn, buildinfo, subdir=None) + cg_import_log(buildinfo, fileinfo) else: - # TODO support other types - raise koji.GenericError("Unsupported file type in import: %r", fileinfo['type']) + cg_import_archive(buildinfo, brinfo, fileinfo) # TODO: post import callback @@ -4760,7 +4758,7 @@ def cg_import_buildroot(cg_id, brdata): type_mismatches = 0 for archive in list_archives(filename=comp['filename'], size=comp['filesize']): - if archive['checksum_type'] != comp['checksum_type'] + if archive['checksum_type'] != comp['checksum_type']: type_mismatches += 1 continue if archive['checksum'] == comp['checksum']: @@ -4783,6 +4781,30 @@ def cg_import_buildroot(cg_id, brdata): return brinfo +def cg_import_rpm(buildinfo, brinfo, fileinfo): + rpminfo = import_rpm(fn, buildinfo, brinfo) + # TODO - handle fileinfo['extra'] + import_rpm_file(fn, buildinfo, rpminfo) + add_rpm_sig(rpminfo['id'], koji.rip_rpm_sighdr(fn)) + + +def cg_import_log(buildinfo, fileinfo): + # TODO: determine subdir + import_build_log(fn, buildinfo, subdir=None) + + +def cg_import_archive(buildinfo, brinfo, fileinfo): + typeinfo = get_archive_type(type_name = fileinfo['type']) + # XXX ^ is this sane? + if typeinfo is None: + # XXX should we support types we don't know about? + raise koji.GenericError("Unsupported file type in import: %r", fileinfo['type']) + + # TODO - determine archive import type (maven/win/image) + # Do we need to add another archive import type? + + + def cg_export(build): """Return CG metadata and file refs for a given build""" From f0d112551cff18dfb50d16498ee62dc251986bb6 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 28 Sep 2015 19:54:49 -0400 Subject: [PATCH 041/109] work on cg import cli --- cli/koji | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/cli/koji b/cli/koji index 36bb5c2a..31047465 100755 --- a/cli/koji +++ b/cli/koji @@ -1651,7 +1651,7 @@ def handle_import_cg(options, session, args): parser.add_option("--test", action="store_true", help=_("Don't actually import")) (options, args) = parser.parse_args(args) if len(args) < 2: - parser.error(_("Please specify metadata and at least one file")) + parser.error(_("Please specify metadata files directory")) assert False if json is None: parser.error(_("Unable to find json module")) @@ -1661,13 +1661,29 @@ def handle_import_cg(options, session, args): if 'output' not in metadata: print _("Metadata contains no output") sys.exit(1) + localdir = args[1] # TODO: determine upload path + to_upload = [] for info in metadata['output']: - # TODO: upload file - pass + localpath = os.path.join(localdir, info.get('relpath', ''), info['filename']) + if not os.path.exists(localpath): + parser.error(_("No such file: %s") % localpath) + to_upload.append([localpath, info]) - session.CGImport(metadata, upload_path) + # get upload path + # XXX - need a better way + serverdir = _unique_path('cli-import') + + for localpath, info in to_upload: + relpath = os.path.join(serverdir, info.get('relpath', '')) + if options.link: + linked_upload(path, serverdir) + else: + session.uploadWrapper(localpath, serverdir) + # TODO - progress callback + + session.CGImport(metadata, serverdir) def handle_import_comps(options, session, args): From ce3c401936631201408aab7a522876da3dea1d11 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 28 Sep 2015 20:00:23 -0400 Subject: [PATCH 042/109] fix login check --- hub/kojihub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index ab3e1962..eb5a9c65 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -6633,7 +6633,7 @@ def remove_user_from_cg(user, cg): def assert_cg(cg, user=None): cg = lookup_name('content_generator', cg, strict=True) if user is None: - if context.session.logged_in: + if not context.session.logged_in: raise koji.AuthError("Not logged in") user = context.session.user_id user = get_user(user, strict=True) From d8dfe3f2d6966479d5ff5b19c73ddab0fa825ee3 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 28 Sep 2015 20:02:59 -0400 Subject: [PATCH 043/109] fix table list --- hub/kojihub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index eb5a9c65..ebfa0439 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -6639,7 +6639,7 @@ def assert_cg(cg, user=None): user = get_user(user, strict=True) clauses = ['active = TRUE', 'user_id = %(user_id)s', 'cg_id = %(cg_id)s'] data = {'user_id' : user['id'], 'cg_id' : cg['id']} - query = QueryProcessor(tables='cg_users', columns=['cg_id'], clauses=clauses, values=data) + query = QueryProcessor(tables=['cg_users'], columns=['cg_id'], clauses=clauses, values=data) if not query.execute(): raise koji.AuthError("Content generator access required (%s)" % cg['name']) From 3af152d21cbb9c22ecf1147d36fc37202e549d19 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 28 Sep 2015 22:26:13 -0400 Subject: [PATCH 044/109] fix cg_id ref --- hub/kojihub.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index ebfa0439..ea6075e1 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4648,6 +4648,7 @@ def cg_import(metadata, files): cginfo = brdata['content_generator'] cg = lookup_name('content_generator', cginfo['name'], strict=True) cgs.add(cg['id']) + brdata['cg_id'] = cg['id'] for cg_id in cgs: assert_cg(cg_id) @@ -4700,12 +4701,12 @@ def cg_import(metadata, files): # TODO: post import callback -def cg_import_buildroot(cg_id, brdata): +def cg_import_buildroot(brdata): """Import the given buildroot data""" # buildroot entry brinfo = { - 'cg_id' : cg_id, + 'cg_id' : brdata['cg_id'], 'cg_version' : brdata['content_generator']['version'], 'container_type' : brdata['container']['type'], 'container_arch' : brdata['container']['arch'], From b21bd65b9feb1a0361262599767218e01422c639 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 28 Sep 2015 22:29:06 -0400 Subject: [PATCH 045/109] typo --- hub/kojihub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index ea6075e1..636a4db9 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -10007,7 +10007,7 @@ class BuildRoot(object): 'host_os', 'host_arch', ] - data.setdefault('br_type', koji.BR_STATES['EXTERNAL']) + data.setdefault('br_type', koji.BR_TYPES['EXTERNAL']) data = dslice(data, fields) for key in fields: if key not in data: From 26c3f184f5801c3582fb569436eb1f77b76dcb3c Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 28 Sep 2015 22:42:33 -0400 Subject: [PATCH 046/109] fix br init --- hub/kojihub.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hub/kojihub.py b/hub/kojihub.py index 636a4db9..03739abb 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -10016,6 +10016,8 @@ class BuildRoot(object): insert = InsertProcessor('buildroot') insert.set(id = br_id, **data) insert.execute() + self.load(br_id) + return self.id def assertStandard(self): if self.id is None: From aca5eae2b084a285492a9b27edd831c7663ab294 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 28 Sep 2015 22:45:12 -0400 Subject: [PATCH 047/109] let extra be optional --- hub/kojihub.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 03739abb..0dd4f5b7 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4777,7 +4777,8 @@ def cg_import_buildroot(brdata): br.setTools(brdata['tools']) # buildroot_extra_info - br.setExtra(brdata['extra']) + if 'extra' in brdata: + br.setExtra(brdata['extra']) return brinfo From c1fc975ee1ee7a5ae6e53cb92f5d890d927c00e0 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 28 Sep 2015 22:51:12 -0400 Subject: [PATCH 048/109] fix file path on hub --- hub/kojihub.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 0dd4f5b7..d81e0b5c 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4635,7 +4635,7 @@ def import_rpm(fn,buildinfo=None,brootid=None,wrapper=False): return rpminfo -def cg_import(metadata, files): +def cg_import(metadata, directory): """ Import build from a content generator""" metaver = metadata['metadata_version'] @@ -4684,6 +4684,9 @@ def cg_import(metadata, files): # outputs for fileinfo in metadata['output']: + workdir = koji.pathinfo.work() + path = os.path.join(workdir, directory, fileinfo.get('relpath', ''), fileinfo['filename']) + fileinfo['hub.path'] = path brinfo = brmap.get(fileinfo['buildroot_id']) if not brinfo: raise koji.GenericError("Missing buildroot metadata for id %(buildroot_id)r", @@ -4784,6 +4787,7 @@ def cg_import_buildroot(brdata): def cg_import_rpm(buildinfo, brinfo, fileinfo): + fn = fileinfo['hub.path'] rpminfo = import_rpm(fn, buildinfo, brinfo) # TODO - handle fileinfo['extra'] import_rpm_file(fn, buildinfo, rpminfo) From 53642d2031dde96135af89412a8434f6405cd784 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 28 Sep 2015 23:06:13 -0400 Subject: [PATCH 049/109] fix upload path --- cli/koji | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cli/koji b/cli/koji index 31047465..04a58d71 100755 --- a/cli/koji +++ b/cli/koji @@ -1663,7 +1663,6 @@ def handle_import_cg(options, session, args): sys.exit(1) localdir = args[1] - # TODO: determine upload path to_upload = [] for info in metadata['output']: localpath = os.path.join(localdir, info.get('relpath', ''), info['filename']) @@ -1678,9 +1677,9 @@ def handle_import_cg(options, session, args): for localpath, info in to_upload: relpath = os.path.join(serverdir, info.get('relpath', '')) if options.link: - linked_upload(path, serverdir) + linked_upload(path, relpath) else: - session.uploadWrapper(localpath, serverdir) + session.uploadWrapper(localpath, relpath) # TODO - progress callback session.CGImport(metadata, serverdir) From 6d7b4f0348801fe595473b3b82e6e96e62c22563 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 28 Sep 2015 23:32:36 -0400 Subject: [PATCH 050/109] fix br ref --- hub/kojihub.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index d81e0b5c..40844c3f 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4783,12 +4783,12 @@ def cg_import_buildroot(brdata): if 'extra' in brdata: br.setExtra(brdata['extra']) - return brinfo + return br def cg_import_rpm(buildinfo, brinfo, fileinfo): fn = fileinfo['hub.path'] - rpminfo = import_rpm(fn, buildinfo, brinfo) + rpminfo = import_rpm(fn, buildinfo, brinfo.id) # TODO - handle fileinfo['extra'] import_rpm_file(fn, buildinfo, rpminfo) add_rpm_sig(rpminfo['id'], koji.rip_rpm_sighdr(fn)) From ff141e9d989a17a79d6e751654193cf1aa773325 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 28 Sep 2015 23:36:17 -0400 Subject: [PATCH 051/109] fix log path ref --- hub/kojihub.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hub/kojihub.py b/hub/kojihub.py index 40844c3f..d2decef8 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4796,6 +4796,7 @@ def cg_import_rpm(buildinfo, brinfo, fileinfo): def cg_import_log(buildinfo, fileinfo): # TODO: determine subdir + fn = fileinfo['hub.path'] import_build_log(fn, buildinfo, subdir=None) From c212a9b29aa5ccb49085dd844249a7b12bcfee59 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 29 Sep 2015 00:02:43 -0400 Subject: [PATCH 052/109] buildroots might not have tags or repos now --- hub/kojihub.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index d2decef8..0b71d75a 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4203,10 +4203,10 @@ def query_buildroots(hostID=None, tagID=None, state=None, rpmID=None, archiveID= 'LEFT OUTER JOIN content_generator ON buildroot.cg_id = content_generator.id', 'LEFT OUTER JOIN host ON host.id = standard_buildroot.host_id', 'LEFT OUTER JOIN repo ON repo.id = standard_buildroot.repo_id', - 'tag ON tag.id = repo.tag_id', + 'LEFT OUTER JOIN tag ON tag.id = repo.tag_id', 'LEFT OUTER JOIN events AS create_events ON create_events.id = standard_buildroot.create_event', 'LEFT OUTER JOIN events AS retire_events ON standard_buildroot.retire_event = retire_events.id', - 'events AS repo_create ON repo_create.id = repo.create_event'] + 'LEFT OUTER JOIN events AS repo_create ON repo_create.id = repo.create_event'] clauses = [] if buildrootID != None: From b63e8e1eae831f599ba10d4807410d00a847f2cd Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 29 Sep 2015 11:31:48 -0400 Subject: [PATCH 053/109] attempt to fix web display --- www/kojiweb/buildrootinfo_cg.chtml | 41 ++++++++++++++++++++++++++++++ www/kojiweb/index.py | 15 +++++++---- 2 files changed, 51 insertions(+), 5 deletions(-) create mode 100644 www/kojiweb/buildrootinfo_cg.chtml diff --git a/www/kojiweb/buildrootinfo_cg.chtml b/www/kojiweb/buildrootinfo_cg.chtml new file mode 100644 index 00000000..7485813a --- /dev/null +++ b/www/kojiweb/buildrootinfo_cg.chtml @@ -0,0 +1,41 @@ +#import koji +#from kojiweb import util + +#include "includes/header.chtml" + +

Information for external buildroot $buildroot.cg_name:$buildroot.id

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ID$buildroot.id
Host OS$buildroot.host_os
Host Arch$buildroot.host_arch
Content Generator$buildroot.cg_name ($buildroot.cg_version)
Container Type$buildroot.container_type
Container Arch$buildroot.container_arch
Component RPMs
Built RPMs
Component Archives
Built Archives
+ +#include "includes/footer.chtml" diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index fdbbc279..fc4b6a3f 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -1623,17 +1623,22 @@ def buildrootinfo(environ, buildrootID, builtStart=None, builtOrder=None, compon buildrootID = int(buildrootID) buildroot = server.getBuildroot(buildrootID) - values['title'] = '%(tag_name)s-%(id)i-%(repo_id)i' % buildroot + ' | Buildroot Info' - if buildroot == None: raise koji.GenericError, 'unknown buildroot ID: %i' % buildrootID - task = server.getTaskInfo(buildroot['task_id'], request=True) + elif buildroot['br_type'] == koji.BR_TYPES['STANDARD']: + template = 'buildrootinfo.chtml' + values['title'] = '%(tag_name)s-%(id)i-%(repo_id)i | Buildroot Info' % buildroot + values['task'] = server.getTaskInfo(buildroot['task_id'], request=True) + + else: + template = 'buildrootinfo_cg.chtml' + values['title'] = '%(cg_name)s:%(id)i | Buildroot Info' % buildroot + # TODO - fetch tools and extras info values['buildroot'] = buildroot - values['task'] = task - return _genHTML(environ, 'buildrootinfo.chtml') + return _genHTML(environ, template) def rpmlist(environ, type, buildrootID=None, imageID=None, start=None, order='nvr'): """ From f6b37caa90401de26d8937153edd8cfb982f4ec7 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 29 Sep 2015 12:03:20 -0400 Subject: [PATCH 054/109] fix buildroot title display on rpminfo page --- www/kojiweb/index.py | 4 ++++ www/kojiweb/rpminfo.chtml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index fc4b6a3f..76f5b2d8 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -1341,6 +1341,10 @@ def rpminfo(environ, rpmID, fileOrder='name', fileStart=None, buildrootOrder='-i builtInRoot = None if rpm['buildroot_id'] != None: builtInRoot = server.getBuildroot(rpm['buildroot_id']) + if builtInRoot['br_type'] == koji.BR_TYPES['STANDARD']: + builtInRoot['_display'] = '%(tag_name)s-%(id)i-%(repo_id)i' % builtInRoot + else: + builtInRoot['_display'] = '%(cg_name)s:%(id)i' % builtInRoot if rpm['external_repo_id'] == 0: values['requires'] = server.getRPMDeps(rpm['id'], koji.DEP_REQUIRE) values['requires'].sort(_sortbyname) diff --git a/www/kojiweb/rpminfo.chtml b/www/kojiweb/rpminfo.chtml index 83e4e4d9..3590eb4e 100644 --- a/www/kojiweb/rpminfo.chtml +++ b/www/kojiweb/rpminfo.chtml @@ -65,7 +65,7 @@ #if $builtInRoot - Buildroot$builtInRoot.tag_name-$builtInRoot.id-$builtInRoot.repo_id + Buildroot$builtInRoot._display #end if #if $rpm.external_repo_id == 0 From 1920f98e2f15ed74d058a32e55bb80ca93b5c5b7 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 29 Sep 2015 15:22:26 -0400 Subject: [PATCH 055/109] obsolete comment --- hub/kojihub.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 0b71d75a..0f313387 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4692,8 +4692,6 @@ def cg_import(metadata, directory): raise koji.GenericError("Missing buildroot metadata for id %(buildroot_id)r", fileinfo) - # TODO map info to files entry (determine fn) - if fileinfo['type'] == 'rpm': cg_import_rpm(buildinfo, brinfo, fileinfo) elif fileinfo['type'] == 'log': From 4b0438c8f606e0298328fd9ac03a3fe893187d05 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Wed, 30 Sep 2015 14:00:35 -0400 Subject: [PATCH 056/109] cli for managing cg users --- cli/koji | 45 +++++++++++++++++++++++++++++++++++++++++++++ hub/kojihub.py | 7 +++++-- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/cli/koji b/cli/koji index 04a58d71..1c72f9b4 100755 --- a/cli/koji +++ b/cli/koji @@ -2520,6 +2520,51 @@ def handle_revoke_permission(options, session, args): for user in users: session.revokePermission(user['name'], perm) + +def handle_add_user_to_cg(options, session, args): + "[admin] Add a user to a content generator" + usage = _("usage: %prog grant-permission ") + usage += _("\n(Specify the --help global option for a list of other help options)") + parser = OptionParser(usage=usage) + parser.add_option("--new", action="store_true", help=_("Create a new content generator")) + (options, args) = parser.parse_args(args) + if len(args) != 2: + parser.error(_("Please specify a user and content generator")) + assert False + activate_session(session) + user = args[0] + cg = args[1] + user = session.getUser(user) + if user is None: + parser.error(_("No such user: %s" % n)) + assert False + users.append(user) + kwargs = {} + if options.new: + kwargs['create'] = True + session.addUserToCG(user['name'], cg, **kwargs) + + +def handle_remove_user_from_cg(options, session, args): + "[admin] Remove a user from a content generator" + usage = _("usage: %prog remove-user-from-cg ") + usage += _("\n(Specify the --help global option for a list of other help options)") + parser = OptionParser(usage=usage) + (options, args) = parser.parse_args(args) + if len(args) != 2: + parser.error(_("Please specify a user and content generator")) + assert False + activate_session(session) + user = args[0] + cg = args[1] + user = session.getUser(user) + if user is None: + parser.error(_("No such user: %s" % n)) + assert False + users.append(user) + session.removeUserFromCG(user['name'], cg) + + def anon_handle_latest_build(options, session, args): "Print the latest builds for a tag" usage = _("usage: %prog latest-build [options] tag package [package...]") diff --git a/hub/kojihub.py b/hub/kojihub.py index 0f313387..682865c9 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -6608,12 +6608,15 @@ def set_user_status(user, status): raise koji.GenericError, 'invalid user ID: %i' % user_id -def add_user_to_cg(user, cg): +def add_user_to_cg(user, cg, create=False): """Associate a user with a content generator""" context.session.assertPerm('admin') user = get_user(user, strict=True) - cg = lookup_name('content_generator', cg, strict=True) + if create: + cg = lookup_name('content_generator', cg, create=True) + else: + cg = lookup_name('content_generator', cg, strict=True) ins = InsertProcessor('cg_users') ins.set(cg_id=cg['id'], user_id=user['id']) ins.make_create() From bdfcde8604ec934926fc818fe8f69c43731facc5 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 5 Oct 2015 16:18:34 -0400 Subject: [PATCH 057/109] adjust cg access terminology --- cli/koji | 12 ++++++------ hub/kojihub.py | 14 +++++++------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cli/koji b/cli/koji index 1c72f9b4..39144686 100755 --- a/cli/koji +++ b/cli/koji @@ -2521,9 +2521,9 @@ def handle_revoke_permission(options, session, args): session.revokePermission(user['name'], perm) -def handle_add_user_to_cg(options, session, args): +def handle_grant_cg_access(options, session, args): "[admin] Add a user to a content generator" - usage = _("usage: %prog grant-permission ") + usage = _("usage: %prog grant-cg-access ") usage += _("\n(Specify the --help global option for a list of other help options)") parser = OptionParser(usage=usage) parser.add_option("--new", action="store_true", help=_("Create a new content generator")) @@ -2542,12 +2542,12 @@ def handle_add_user_to_cg(options, session, args): kwargs = {} if options.new: kwargs['create'] = True - session.addUserToCG(user['name'], cg, **kwargs) + session.grantCGAccess(user['name'], cg, **kwargs) -def handle_remove_user_from_cg(options, session, args): +def handle_revoke_cg_access(options, session, args): "[admin] Remove a user from a content generator" - usage = _("usage: %prog remove-user-from-cg ") + usage = _("usage: %prog revoke-cg-access ") usage += _("\n(Specify the --help global option for a list of other help options)") parser = OptionParser(usage=usage) (options, args) = parser.parse_args(args) @@ -2562,7 +2562,7 @@ def handle_remove_user_from_cg(options, session, args): parser.error(_("No such user: %s" % n)) assert False users.append(user) - session.removeUserFromCG(user['name'], cg) + session.revokeCGAccess(user['name'], cg) def anon_handle_latest_build(options, session, args): diff --git a/hub/kojihub.py b/hub/kojihub.py index 682865c9..390190ff 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -6608,8 +6608,8 @@ def set_user_status(user, status): raise koji.GenericError, 'invalid user ID: %i' % user_id -def add_user_to_cg(user, cg, create=False): - """Associate a user with a content generator""" +def grant_cg_access(user, cg, create=False): + """Grant user access to act as the given content generator""" context.session.assertPerm('admin') user = get_user(user, strict=True) @@ -6621,12 +6621,12 @@ def add_user_to_cg(user, cg, create=False): ins.set(cg_id=cg['id'], user_id=user['id']) ins.make_create() if ins.dup_check(): - raise koji.GenericError("User already associated with content generator") + raise koji.GenericError("User already has access to content generator %(name)s" % cg) ins.execute() -def remove_user_from_cg(user, cg): - """De-associate a user with a content generator""" +def revoke_cg_access(user, cg): + """Revoke a user's access to act as the given content generator""" context.session.assertPerm('admin') user = get_user(user, strict=True) @@ -9168,8 +9168,8 @@ class RootExports(object): raise koji.GenericError, 'unknown user: %s' % username set_user_status(user, koji.USER_STATUS['BLOCKED']) - addUserToCG = staticmethod(add_user_to_cg) - removeUserFromCG = staticmethod(remove_user_from_cg) + grantCGAccess = staticmethod(grant_cg_access) + revokeCGAccess = staticmethod(revoke_cg_access) #group management calls newGroup = staticmethod(new_group) From e36e5fef1c44faf582b18ef0f2533b127e8e47e4 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 5 Oct 2015 16:33:19 -0400 Subject: [PATCH 058/109] more extra data in schema --- docs/schema-update-cgen.sql | 24 ++++++++++++++++++++++++ docs/schema.sql | 27 +++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/docs/schema-update-cgen.sql b/docs/schema-update-cgen.sql index 2e435eee..70fe89b6 100644 --- a/docs/schema-update-cgen.sql +++ b/docs/schema-update-cgen.sql @@ -41,6 +41,30 @@ CREATE TABLE buildroot_extra_info ( ) WITHOUT OIDS; +CREATE TABLE build_extra_info ( + build_id INTEGER NOT NULL REFERENCES build(id), + key TEXT NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY (build_id, key) +) WITHOUT OIDS; + + +CREATE TABLE rpminfo_extra_info ( + rpm_id INTEGER NOT NULL REFERENCES rpminfo(id), + key TEXT NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY (rpminfo_id, key) +) WITHOUT OIDS; + + +CREATE TABLE archiveinfo_extra_info ( + archive_id INTEGER NOT NULL REFERENCES archiveinfo(id), + key TEXT NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY (archiveinfo_id, key) +) WITHOUT OIDS; + + CREATE TABLE image_archive_listing ( image_id INTEGER NOT NULL REFERENCES image_archives(archive_id), archive_id INTEGER NOT NULL REFERENCES archiveinfo(id), diff --git a/docs/schema.sql b/docs/schema.sql index 8bc2c290..2fa1505c 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -303,6 +303,15 @@ CREATE TABLE build ( CREATE INDEX build_by_pkg_id ON build (pkg_id); CREATE INDEX build_completion ON build(completion_time); + +CREATE TABLE build_extra_info ( + build_id INTEGER NOT NULL REFERENCES build(id), + key TEXT NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY (build_id, key) +) WITHOUT OIDS; + + -- Note: some of these CREATEs may seem a little out of order. This is done to keep -- the references sane. @@ -701,6 +710,15 @@ CREATE TABLE rpminfo ( ) WITHOUT OIDS; CREATE INDEX rpminfo_build ON rpminfo(build_id); + +CREATE TABLE rpminfo_extra_info ( + rpm_id INTEGER NOT NULL REFERENCES rpminfo(id), + key TEXT NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY (rpminfo_id, key) +) WITHOUT OIDS; + + -- sighash is the checksum of the signature header CREATE TABLE rpmsigs ( rpm_id INTEGER NOT NULL REFERENCES rpminfo (id), @@ -836,6 +854,15 @@ CREATE INDEX archiveinfo_buildroot_idx on archiveinfo (buildroot_id); CREATE INDEX archiveinfo_type_idx on archiveinfo (type_id); CREATE INDEX archiveinfo_filename_idx on archiveinfo(filename); + +CREATE TABLE archiveinfo_extra_info ( + archive_id INTEGER NOT NULL REFERENCES archiveinfo(id), + key TEXT NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY (archiveinfo_id, key) +) WITHOUT OIDS; + + CREATE TABLE maven_archives ( archive_id INTEGER NOT NULL PRIMARY KEY REFERENCES archiveinfo(id), group_id TEXT NOT NULL, From ccd486f53499dfd4f3282927621c38ebdf5c7b19 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 5 Oct 2015 23:07:24 -0400 Subject: [PATCH 059/109] Revert "more extra data in schema" This reverts commit e36e5fef1c44faf582b18ef0f2533b127e8e47e4. --- docs/schema-update-cgen.sql | 24 ------------------------ docs/schema.sql | 27 --------------------------- 2 files changed, 51 deletions(-) diff --git a/docs/schema-update-cgen.sql b/docs/schema-update-cgen.sql index 70fe89b6..2e435eee 100644 --- a/docs/schema-update-cgen.sql +++ b/docs/schema-update-cgen.sql @@ -41,30 +41,6 @@ CREATE TABLE buildroot_extra_info ( ) WITHOUT OIDS; -CREATE TABLE build_extra_info ( - build_id INTEGER NOT NULL REFERENCES build(id), - key TEXT NOT NULL, - value TEXT NOT NULL, - PRIMARY KEY (build_id, key) -) WITHOUT OIDS; - - -CREATE TABLE rpminfo_extra_info ( - rpm_id INTEGER NOT NULL REFERENCES rpminfo(id), - key TEXT NOT NULL, - value TEXT NOT NULL, - PRIMARY KEY (rpminfo_id, key) -) WITHOUT OIDS; - - -CREATE TABLE archiveinfo_extra_info ( - archive_id INTEGER NOT NULL REFERENCES archiveinfo(id), - key TEXT NOT NULL, - value TEXT NOT NULL, - PRIMARY KEY (archiveinfo_id, key) -) WITHOUT OIDS; - - CREATE TABLE image_archive_listing ( image_id INTEGER NOT NULL REFERENCES image_archives(archive_id), archive_id INTEGER NOT NULL REFERENCES archiveinfo(id), diff --git a/docs/schema.sql b/docs/schema.sql index 2fa1505c..8bc2c290 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -303,15 +303,6 @@ CREATE TABLE build ( CREATE INDEX build_by_pkg_id ON build (pkg_id); CREATE INDEX build_completion ON build(completion_time); - -CREATE TABLE build_extra_info ( - build_id INTEGER NOT NULL REFERENCES build(id), - key TEXT NOT NULL, - value TEXT NOT NULL, - PRIMARY KEY (build_id, key) -) WITHOUT OIDS; - - -- Note: some of these CREATEs may seem a little out of order. This is done to keep -- the references sane. @@ -710,15 +701,6 @@ CREATE TABLE rpminfo ( ) WITHOUT OIDS; CREATE INDEX rpminfo_build ON rpminfo(build_id); - -CREATE TABLE rpminfo_extra_info ( - rpm_id INTEGER NOT NULL REFERENCES rpminfo(id), - key TEXT NOT NULL, - value TEXT NOT NULL, - PRIMARY KEY (rpminfo_id, key) -) WITHOUT OIDS; - - -- sighash is the checksum of the signature header CREATE TABLE rpmsigs ( rpm_id INTEGER NOT NULL REFERENCES rpminfo (id), @@ -854,15 +836,6 @@ CREATE INDEX archiveinfo_buildroot_idx on archiveinfo (buildroot_id); CREATE INDEX archiveinfo_type_idx on archiveinfo (type_id); CREATE INDEX archiveinfo_filename_idx on archiveinfo(filename); - -CREATE TABLE archiveinfo_extra_info ( - archive_id INTEGER NOT NULL REFERENCES archiveinfo(id), - key TEXT NOT NULL, - value TEXT NOT NULL, - PRIMARY KEY (archiveinfo_id, key) -) WITHOUT OIDS; - - CREATE TABLE maven_archives ( archive_id INTEGER NOT NULL PRIMARY KEY REFERENCES archiveinfo(id), group_id TEXT NOT NULL, From 36e56a72e3165a304b83545c38de7608e24d8994 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 5 Oct 2015 23:23:15 -0400 Subject: [PATCH 060/109] more extra data. just use a json text field for it. --- docs/schema-update-cgen.sql | 16 +++++++++------- docs/schema.sql | 16 ++++++---------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/docs/schema-update-cgen.sql b/docs/schema-update-cgen.sql index 2e435eee..588076ec 100644 --- a/docs/schema-update-cgen.sql +++ b/docs/schema-update-cgen.sql @@ -33,13 +33,6 @@ CREATE TABLE buildroot_tools_info ( PRIMARY KEY (buildroot_id, tool) ) WITHOUT OIDS; -CREATE TABLE buildroot_extra_info ( - buildroot_id INTEGER NOT NULL REFERENCES buildroot(id), - key TEXT NOT NULL, - value TEXT NOT NULL, - PRIMARY KEY (buildroot_id, key) -) WITHOUT OIDS; - CREATE TABLE image_archive_listing ( image_id INTEGER NOT NULL REFERENCES image_archives(archive_id), @@ -49,6 +42,14 @@ CREATE TABLE image_archive_listing ( CREATE INDEX image_listing_archives on image_archive_listing(archive_id); +-- new columns -- + +select now(), 'Adding new columns' as msg; +ALTER TABLE build ADD COLUMN extra TEXT; +ALTER TABLE rpminfo ADD COLUMN extra TEXT; +ALTER TABLE archiveinfo ADD COLUMN extra TEXT; + + -- the more complicated stuff SELECT now(), 'Copying buildroot to standard_buildroot' as msg; @@ -90,6 +91,7 @@ ALTER TABLE buildroot ADD COLUMN cg_version TEXT; ALTER TABLE buildroot ADD COLUMN container_type TEXT; ALTER TABLE buildroot ADD COLUMN host_os TEXT; ALTER TABLE buildroot ADD COLUMN host_arch TEXT; +ALTER TABLE buildroot ADD COLUMN extra TEXT; SELECT now(), 'Altering buildroot table (altering columns)' as msg; ALTER TABLE buildroot RENAME arch TO container_arch; diff --git a/docs/schema.sql b/docs/schema.sql index 8bc2c290..bd9b4203 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -21,7 +21,6 @@ DROP TABLE groups; DROP TABLE tag_listing; DROP TABLE tag_packages; -DROP TABLE buildroot_extra_info; DROP TABLE buildroot_tools_info; DROP TABLE standard_buildroot; DROP TABLE buildroot; @@ -295,6 +294,7 @@ CREATE TABLE build ( state INTEGER NOT NULL, task_id INTEGER REFERENCES task (id), owner INTEGER NOT NULL REFERENCES users (id), + extra TEXT, CONSTRAINT build_pkg_ver_rel UNIQUE (pkg_id, version, release), CONSTRAINT completion_sane CHECK ((state = 0 AND completion_time IS NULL) OR (state != 0 AND completion_time IS NOT NULL)) @@ -525,7 +525,8 @@ CREATE TABLE buildroot ( (container_type IS NULL AND container_arch IS NULL) OR (container_type IS NOT NULL AND container_arch IS NOT NULL)), host_os TEXT, - host_arch TEXT + host_arch TEXT, + extra TEXT ) WITHOUT OIDS; CREATE TABLE standard_buildroot ( @@ -545,13 +546,6 @@ CREATE TABLE buildroot_tools_info ( PRIMARY KEY (buildroot_id, tool) ) WITHOUT OIDS; -CREATE TABLE buildroot_extra_info ( - buildroot_id INTEGER NOT NULL REFERENCES buildroot(id), - key TEXT NOT NULL, - value TEXT NOT NULL, - PRIMARY KEY (buildroot_id, key) -) WITHOUT OIDS; - -- track spun images (livecds, installation, VMs...) CREATE TABLE image_builds ( @@ -697,6 +691,7 @@ CREATE TABLE rpminfo ( payloadhash TEXT NOT NULL, size BIGINT NOT NULL, buildtime BIGINT NOT NULL, + extra TEXT, CONSTRAINT rpminfo_unique_nvra UNIQUE (name,version,release,arch,external_repo_id) ) WITHOUT OIDS; CREATE INDEX rpminfo_build ON rpminfo(build_id); @@ -829,7 +824,8 @@ CREATE TABLE archiveinfo ( filename TEXT NOT NULL, size BIGINT NOT NULL, checksum TEXT NOT NULL, - checksum_type INTEGER NOT NULL + checksum_type INTEGER NOT NULL, + extra TEXT ) WITHOUT OIDS; CREATE INDEX archiveinfo_build_idx ON archiveinfo (build_id); CREATE INDEX archiveinfo_buildroot_idx on archiveinfo (buildroot_id); From 25648afced0307a9f4c3627bca4df59383ec87a1 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 5 Oct 2015 23:56:49 -0400 Subject: [PATCH 061/109] adjust handling of buildroot extra. handle extra data in new_build --- hub/kojihub.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 390190ff..de17ea4d 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4407,6 +4407,13 @@ def new_build(data): for f in ('version','release','epoch'): if not data.has_key(f): raise koji.GenericError, "No %s value for build" % f + if 'extra' in data: + try: + data['extra'] = json.loads(data['extra']) + except Exception: + raise koji.GenericError("Invalid build extra data: %(extra)r", data) + else: + data['extra'] = None #provide a few default values data.setdefault('state',koji.BUILD_STATES['COMPLETE']) data.setdefault('completion_time', 'NOW') @@ -4445,7 +4452,7 @@ def new_build(data): koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=None, new=data['state'], info=data) #insert the new data insert_data = dslice(data, ['pkg_id', 'version', 'release', 'epoch', 'state', 'volume_id', - 'task_id', 'owner', 'completion_time']) + 'task_id', 'owner', 'completion_time', 'extra']) data['id'] = insert_data['id'] = _singleValue("SELECT nextval('build_id_seq')") insert = InsertProcessor('build', data=insert_data) insert.execute() @@ -4713,6 +4720,7 @@ def cg_import_buildroot(brdata): 'container_arch' : brdata['container']['arch'], 'host_os' : brdata['host']['os'], 'host_arch' : brdata['host']['arch'], + 'extra' : brdata.get('extra'), } br = BuildRoot() br.cg_new(brinfo) @@ -4777,10 +4785,6 @@ def cg_import_buildroot(brdata): # buildroot_tools_info br.setTools(brdata['tools']) - # buildroot_extra_info - if 'extra' in brdata: - br.setExtra(brdata['extra']) - return br @@ -9955,10 +9959,16 @@ class BuildRoot(object): 'container_arch', 'host_os', 'host_arch', + 'extra', ] query = QueryProcessor(columns=fields, tables=['buildroot'], values={'id': id}, clauses=['id=%(id)s']) data = query.executeOne() + if data['extra'] != None: + try: + data['extra'] = json.loads(data['extra']) + except Exception: + raise koji.GenericError("Invalid buildroot extra data: %(extra)r", data) if not data: raise koji.GenericError, 'no buildroot with ID: %i' % id self.id = id @@ -10013,12 +10023,15 @@ class BuildRoot(object): 'container_arch', 'host_os', 'host_arch', + 'extra', ] data.setdefault('br_type', koji.BR_TYPES['EXTERNAL']) data = dslice(data, fields) for key in fields: if key not in data: raise koji.GenericError("Buildroot field %s not specified" % key) + if data['extra'] is not None: + data['extra'] = json.dumps(data['extra']), br_id = _singleValue("SELECT nextval('buildroot_id_seq')", strict=True) insert = InsertProcessor('buildroot') insert.set(id = br_id, **data) @@ -10201,16 +10214,6 @@ class BuildRoot(object): insert.set(version=tool['version']) insert.execute() - def setExtra(self, extra): - """Set extra info for buildroot""" - - insert = InsertProcessor('buildroot_extra_info') - insert.set(buildroot_id=self.id) - for key in extra: - insert.set(key=key) - insert.set(value=json.dumps(extra[key])) - insert.execute() - class Host(object): From 17edca1e24e6b1bbaedd12ef89452282ddf10395 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 6 Oct 2015 00:15:40 -0400 Subject: [PATCH 062/109] return extra data in get_build --- hub/kojihub.py | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index de17ea4d..9b577236 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -3268,14 +3268,18 @@ def find_build_id(X, strict=False): return None return r[0] + def get_build(buildInfo, strict=False): - """Return information about a build. buildID may be either - a int ID, a string NVR, or a map containing 'name', 'version' - and 'release. A map will be returned containing the following - keys: + """Return information about a build. + + buildID may be either a int ID, a string NVR, or a map containing + 'name', 'version' and 'release. + + A map will be returned containing the following keys: id: build ID package_id: ID of the package built package_name: name of the package built + name: same as package_name version release epoch @@ -3291,6 +3295,7 @@ def get_build(buildInfo, strict=False): creation_ts: time the build was created (epoch) completion_time: time the build was completed (may be null) completion_ts: time the build was completed (epoch, may be null) + extra: dictionary with extra data about the build If there is no build matching the buildInfo given, and strict is specified, raise an error. Otherwise return None. @@ -3307,18 +3312,18 @@ def get_build(buildInfo, strict=False): ("package.name || '-' || build.version || '-' || build.release", 'nvr'), ('EXTRACT(EPOCH FROM events.time)','creation_ts'), ('EXTRACT(EPOCH FROM build.completion_time)','completion_ts'), - ('users.id', 'owner_id'), ('users.name', 'owner_name')) - query = """SELECT %s - FROM build - JOIN events ON build.create_event = events.id - JOIN package on build.pkg_id = package.id - JOIN volume on build.volume_id = volume.id - JOIN users on build.owner = users.id - WHERE build.id = %%(buildID)i""" % ', '.join([pair[0] for pair in fields]) - - c = context.cnx.cursor() - c.execute(query, locals()) - result = c.fetchone() + ('users.id', 'owner_id'), ('users.name', 'owner_name'), + ('build.extra', 'extra')) + fields, aliases = zip(*fields) + joins = ['events ON build.create_event = events.id', + 'package on build.pkg_id = package.id', + 'volume on build.volume_id = volume.id', + 'users on build.owner = users.id', + ] + clauses = ['build.id = %(buildID)i'] + query = QueryProcessor(columns=fields, aliases=aliases, values=locals(), + tables=['build'], joins=joins, clauses=clauses) + result = query.executeOne() if not result: if strict: @@ -3326,8 +3331,10 @@ def get_build(buildInfo, strict=False): else: return None else: - ret = dict(zip([pair[1] for pair in fields], result)) - return ret + if result['extra'] is not None: + result['extra'] = json.loads(result['extra']) + return result + def get_next_release(build_info): """find the last successful or deleted build of this N-V""" From 8ed6efaa0a823b566de0de9d2badda1da204a7a5 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 13 Oct 2015 17:36:40 -0400 Subject: [PATCH 063/109] pull maven/win/image build info from metadata build.extra --- hub/kojihub.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/hub/kojihub.py b/hub/kojihub.py index 9b577236..02fb162f 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4685,6 +4685,19 @@ def cg_import(metadata, directory): build_id = new_build(buildinfo) buildinfo = get_build(build_id, strict=True) + # handle special build types + b_extra = metadata['build'].get('extra', {}) + if b_extra.get('is_maven'): + keys = ['group_id', 'artifact_id', 'version'] + maven_info = dict([(k, b_extra['maven.%s' % k]) for k in keys]) + new_maven_build(buildinfo, maven_info) + if b_extra.get('is_win'): + win_info = {'platform' : b_extra['maven.platform']} + new_win_build(buildinfo, win_info) + if b_extra.get('is_image'): + # no extra info tracked at build level + new_image_build(buildinfo) + # buildroots br_used = set([f['buildroot_id'] for f in metadata['output']]) brmap = {} From 2e05c196b02d8eb50adddc70ee8a7046d880cda7 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 13 Oct 2015 19:18:02 -0400 Subject: [PATCH 064/109] rework cg_import_archive a bit --- hub/kojihub.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 02fb162f..af2afbf0 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4823,15 +4823,23 @@ def cg_import_log(buildinfo, fileinfo): def cg_import_archive(buildinfo, brinfo, fileinfo): - typeinfo = get_archive_type(type_name = fileinfo['type']) - # XXX ^ is this sane? - if typeinfo is None: - # XXX should we support types we don't know about? - raise koji.GenericError("Unsupported file type in import: %r", fileinfo['type']) + fn = fileinfo['hub.path'] - # TODO - determine archive import type (maven/win/image) - # Do we need to add another archive import type? + # determine archive import type (maven/win/image/other) + extra = fileinfo.get('extra', {}) + legacy_types = ['maven', 'win', 'image'] + l_type = None + type_info = None + for name in legacy_types: + key = '%s_info' % name + if key in extra: + if l_type is not None: + raise koji.GenericError("Output file has multiple archive types: %s", fn) + l_type = name + type_info = extra[key] + # TODO: teach import_archive to handle extra + import_archive(fn, buildinfo, l_type, type_info, brinfo['id']) def cg_export(build): From 298609a9616d6838b63fc5fd22fed1d196e3bdd2 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 13 Oct 2015 19:38:20 -0400 Subject: [PATCH 065/109] cg_import does its own access check now --- hub/kojihub.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index af2afbf0..35d7a104 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -8151,10 +8151,7 @@ class RootExports(object): fullpath = '%s/%s' % (koji.pathinfo.work(), filepath) import_archive(fullpath, buildinfo, type, typeInfo) - def CGImport(self, metadata, files): - context.session.assertPerm('admin') # TODO: fix access check - return cg_import(metadata, files) - + CGImport = staticmethod(cg_import) CGExport = staticmethod(cg_export) untaggedBuilds = staticmethod(untagged_builds) From 94794bcc942d3e9329226f52bc3566a9c63b5882 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Wed, 14 Oct 2015 14:59:13 -0400 Subject: [PATCH 066/109] tweak debug messages in schema update script --- docs/schema-update-cgen.sql | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/schema-update-cgen.sql b/docs/schema-update-cgen.sql index 588076ec..320dd80f 100644 --- a/docs/schema-update-cgen.sql +++ b/docs/schema-update-cgen.sql @@ -2,7 +2,7 @@ BEGIN; -- New tables -SELECT now(), 'Creating new tables' as msg; +SELECT statement_timestamp(), 'Creating new tables' as msg; CREATE TABLE content_generator ( id SERIAL PRIMARY KEY, @@ -44,7 +44,7 @@ CREATE INDEX image_listing_archives on image_archive_listing(archive_id); -- new columns -- -select now(), 'Adding new columns' as msg; +select statement_timestamp(), 'Adding new columns' as msg; ALTER TABLE build ADD COLUMN extra TEXT; ALTER TABLE rpminfo ADD COLUMN extra TEXT; ALTER TABLE archiveinfo ADD COLUMN extra TEXT; @@ -52,12 +52,12 @@ ALTER TABLE archiveinfo ADD COLUMN extra TEXT; -- the more complicated stuff -SELECT now(), 'Copying buildroot to standard_buildroot' as msg; +SELECT statement_timestamp(), 'Copying buildroot to standard_buildroot' as msg; CREATE TABLE standard_buildroot AS SELECT id,host_id,repo_id,task_id,create_event,retire_event,state from buildroot; -- doing it this way and fixing up after is *much* faster than creating the empty table -- and using insert..select to populate -SELECT now(), 'Fixing up standard_buildroot table' as msg; +SELECT statement_timestamp(), 'Fixing up standard_buildroot table' as msg; ALTER TABLE standard_buildroot RENAME id TO buildroot_id; ALTER TABLE standard_buildroot ALTER COLUMN buildroot_id SET NOT NULL; ALTER TABLE standard_buildroot ALTER COLUMN host_id SET NOT NULL; @@ -65,17 +65,17 @@ ALTER TABLE standard_buildroot ALTER COLUMN repo_id SET NOT NULL; ALTER TABLE standard_buildroot ALTER COLUMN task_id SET NOT NULL; ALTER TABLE standard_buildroot ALTER COLUMN create_event SET NOT NULL; ALTER TABLE standard_buildroot ALTER COLUMN create_event SET DEFAULT get_event(); -SELECT now(), 'Fixing up standard_buildroot table, foreign key constraints' as msg; +SELECT statement_timestamp(), 'Fixing up standard_buildroot table, foreign key constraints' as msg; ALTER TABLE standard_buildroot ADD CONSTRAINT brfk FOREIGN KEY (buildroot_id) REFERENCES buildroot(id); ALTER TABLE standard_buildroot ADD CONSTRAINT hfk FOREIGN KEY (host_id) REFERENCES host(id); ALTER TABLE standard_buildroot ADD CONSTRAINT rfk FOREIGN KEY (repo_id) REFERENCES repo(id); ALTER TABLE standard_buildroot ADD CONSTRAINT tfk FOREIGN KEY (task_id) REFERENCES task(id); ALTER TABLE standard_buildroot ADD CONSTRAINT efk FOREIGN KEY (create_event) REFERENCES events(id) ; -SELECT now(), 'Fixing up standard_buildroot table, primary key' as msg; +SELECT statement_timestamp(), 'Fixing up standard_buildroot table, primary key' as msg; ALTER TABLE standard_buildroot ADD PRIMARY KEY (buildroot_id); -SELECT now(), 'Altering buildroot table (dropping columns)' as msg; +SELECT statement_timestamp(), 'Altering buildroot table (dropping columns)' as msg; ALTER TABLE buildroot DROP COLUMN host_id; ALTER TABLE buildroot DROP COLUMN repo_id; ALTER TABLE buildroot DROP COLUMN task_id; @@ -84,7 +84,7 @@ ALTER TABLE buildroot DROP COLUMN retire_event; ALTER TABLE buildroot DROP COLUMN state; ALTER TABLE buildroot DROP COLUMN dirtyness; -SELECT now(), 'Altering buildroot table (adding columns)' as msg; +SELECT statement_timestamp(), 'Altering buildroot table (adding columns)' as msg; ALTER TABLE buildroot ADD COLUMN br_type INTEGER NOT NULL DEFAULT 0; ALTER TABLE buildroot ADD COLUMN cg_id INTEGER REFERENCES content_generator (id); ALTER TABLE buildroot ADD COLUMN cg_version TEXT; @@ -93,7 +93,7 @@ ALTER TABLE buildroot ADD COLUMN host_os TEXT; ALTER TABLE buildroot ADD COLUMN host_arch TEXT; ALTER TABLE buildroot ADD COLUMN extra TEXT; -SELECT now(), 'Altering buildroot table (altering columns)' as msg; +SELECT statement_timestamp(), 'Altering buildroot table (altering columns)' as msg; ALTER TABLE buildroot RENAME arch TO container_arch; ALTER TABLE buildroot ALTER COLUMN container_arch TYPE TEXT; ALTER TABLE buildroot ALTER COLUMN br_type DROP DEFAULT; From c19f9a60a8cce9df424185a70fa83e3dfecc0bde Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Wed, 14 Oct 2015 16:45:26 -0400 Subject: [PATCH 067/109] Revert "fix superclass ref" This reverts commit ce4e19739bae3b913ec8c35a55c97d1993aa41d3. --- hub/kojixmlrpc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hub/kojixmlrpc.py b/hub/kojixmlrpc.py index bcaff895..efb99a60 100644 --- a/hub/kojixmlrpc.py +++ b/hub/kojixmlrpc.py @@ -598,8 +598,7 @@ class HubFormatter(logging.Formatter): record.session_id = None record.callnum = None record.user_name = None - return super(HubFormatter, self).format(record) - + return logging.Formatter.format(self, record) def setup_logging1(): """Set up basic logging, before options are loaded""" From b82f702784ba7238878878daca33acb085991dd1 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Fri, 16 Oct 2015 12:48:15 -0400 Subject: [PATCH 068/109] tweak metadata format for maven/win/image extra data --- hub/kojihub.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 35d7a104..f12d7faf 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4687,14 +4687,11 @@ def cg_import(metadata, directory): # handle special build types b_extra = metadata['build'].get('extra', {}) - if b_extra.get('is_maven'): - keys = ['group_id', 'artifact_id', 'version'] - maven_info = dict([(k, b_extra['maven.%s' % k]) for k in keys]) - new_maven_build(buildinfo, maven_info) - if b_extra.get('is_win'): - win_info = {'platform' : b_extra['maven.platform']} - new_win_build(buildinfo, win_info) - if b_extra.get('is_image'): + if 'maven' in b_extra: + new_maven_build(buildinfo, b_extra['maven']) + if 'win' in b_extra: + new_win_build(buildinfo, b_extra['win']) + if 'image' in b_extra: # no extra info tracked at build level new_image_build(buildinfo) @@ -4830,12 +4827,11 @@ def cg_import_archive(buildinfo, brinfo, fileinfo): legacy_types = ['maven', 'win', 'image'] l_type = None type_info = None - for name in legacy_types: - key = '%s_info' % name + for key in legacy_types: if key in extra: if l_type is not None: raise koji.GenericError("Output file has multiple archive types: %s", fn) - l_type = name + l_type = key type_info = extra[key] # TODO: teach import_archive to handle extra From 445a70b5cb46e9d81edfd30aa89d68a01bfa51c3 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Fri, 16 Oct 2015 12:56:15 -0400 Subject: [PATCH 069/109] fix up mistakes in cg cli handlers --- cli/koji | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/cli/koji b/cli/koji index 39144686..076c4084 100755 --- a/cli/koji +++ b/cli/koji @@ -1677,7 +1677,7 @@ def handle_import_cg(options, session, args): for localpath, info in to_upload: relpath = os.path.join(serverdir, info.get('relpath', '')) if options.link: - linked_upload(path, relpath) + linked_upload(localpath, relpath) else: session.uploadWrapper(localpath, relpath) # TODO - progress callback @@ -2534,15 +2534,14 @@ def handle_grant_cg_access(options, session, args): activate_session(session) user = args[0] cg = args[1] - user = session.getUser(user) - if user is None: - parser.error(_("No such user: %s" % n)) + uinfo = session.getUser(user) + if uinfo is None: + parser.error(_("No such user: %s" % user)) assert False - users.append(user) kwargs = {} if options.new: kwargs['create'] = True - session.grantCGAccess(user['name'], cg, **kwargs) + session.grantCGAccess(uinfo['name'], cg, **kwargs) def handle_revoke_cg_access(options, session, args): @@ -2557,12 +2556,11 @@ def handle_revoke_cg_access(options, session, args): activate_session(session) user = args[0] cg = args[1] - user = session.getUser(user) - if user is None: - parser.error(_("No such user: %s" % n)) + uinfo = session.getUser(user) + if uinfo is None: + parser.error(_("No such user: %s" % user)) assert False - users.append(user) - session.revokeCGAccess(user['name'], cg) + session.revokeCGAccess(uinfo['name'], cg) def anon_handle_latest_build(options, session, args): From d932f7b74a3726a786046cd01cfe7eb5841975d4 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Fri, 16 Oct 2015 15:38:20 -0400 Subject: [PATCH 070/109] fix some exception lines --- hub/kojihub.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index f12d7faf..03e312f3 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4654,7 +4654,7 @@ def cg_import(metadata, directory): metaver = metadata['metadata_version'] if metaver != 0: - raise koji.GenericError("Unknown metadata version: %r", metaver) + raise koji.GenericError("Unknown metadata version: %r" % metaver) # assert cg access cgs = set() @@ -4676,7 +4676,7 @@ def cg_import(metadata, directory): buildinfo = get_build(metadata['build'], strict=False) if buildinfo: # TODO : allow in some cases - raise koji.GenericError("Build already exists: %r", buildinfo) + raise koji.GenericError("Build already exists: %r" % buildinfo) else: # create a new build buildinfo = dslice(metadata['build'], ['name', 'version', 'release']) @@ -4701,9 +4701,9 @@ def cg_import(metadata, directory): for brdata in metadata['buildroots']: brfakeid = brdata['id'] if brfakeid not in br_used: - raise koji.GenericError("Buildroot id not used in output: %r", brfakeid) + raise koji.GenericError("Buildroot id not used in output: %r" % brfakeid) if brfakeid in brmap: - raise koji.GenericError("Duplicate buildroot id in metadata: %r", brfakeid) + raise koji.GenericError("Duplicate buildroot id in metadata: %r" % brfakeid) brmap[brfakeid] = cg_import_buildroot(brdata) # outputs @@ -4713,8 +4713,7 @@ def cg_import(metadata, directory): fileinfo['hub.path'] = path brinfo = brmap.get(fileinfo['buildroot_id']) if not brinfo: - raise koji.GenericError("Missing buildroot metadata for id %(buildroot_id)r", - fileinfo) + raise koji.GenericError("Missing buildroot metadata for id %(buildroot_id)r" % fileinfo) if fileinfo['type'] == 'rpm': cg_import_rpm(buildinfo, brinfo, fileinfo) @@ -4830,7 +4829,7 @@ def cg_import_archive(buildinfo, brinfo, fileinfo): for key in legacy_types: if key in extra: if l_type is not None: - raise koji.GenericError("Output file has multiple archive types: %s", fn) + raise koji.GenericError("Output file has multiple archive types: %s" % fn) l_type = key type_info = extra[key] From 2c77d3666711413a9529b31b7ebd091a42a7faa7 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 20 Oct 2015 15:55:43 -0400 Subject: [PATCH 071/109] first stab at support metadata only cg imports, plus a few fixes --- cli/koji | 2 + hub/kojihub.py | 124 +++++++++++++++++++++++++++++++++++-------------- 2 files changed, 92 insertions(+), 34 deletions(-) diff --git a/cli/koji b/cli/koji index 076c4084..ab31996b 100755 --- a/cli/koji +++ b/cli/koji @@ -1665,6 +1665,8 @@ def handle_import_cg(options, session, args): to_upload = [] for info in metadata['output']: + if info.get('metadata_only', False): + continue localpath = os.path.join(localdir, info.get('relpath', ''), info['filename']) if not os.path.exists(localpath): parser.error(_("No such file: %s") % localpath) diff --git a/hub/kojihub.py b/hub/kojihub.py index 03e312f3..ecb9e37d 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2833,7 +2833,7 @@ def get_tag_extra(tagInfo, event=None): value = json.loads(value) except Exception: # this should not happen - raise koji.GenericError("Invalid tag extra data: %s : %r", key, value) + raise koji.GenericError("Invalid tag extra data: %s : %r" % (key, value)) result[key] = value return result @@ -4418,7 +4418,7 @@ def new_build(data): try: data['extra'] = json.loads(data['extra']) except Exception: - raise koji.GenericError("Invalid build extra data: %(extra)r", data) + raise koji.GenericError("Invalid build extra data: %(extra)r" % data) else: data['extra'] = None #provide a few default values @@ -4708,6 +4708,9 @@ def cg_import(metadata, directory): # outputs for fileinfo in metadata['output']: + if fileinfo.get('metadata_only', False): + if not metadata['build'].get('metadata_only'): + raise koji.GenericError('Build must be marked metadata-only to include metadata-only outputs') workdir = koji.pathinfo.work() path = os.path.join(workdir, directory, fileinfo.get('relpath', ''), fileinfo['filename']) fileinfo['hub.path'] = path @@ -4805,6 +4808,9 @@ def cg_import_buildroot(brdata): def cg_import_rpm(buildinfo, brinfo, fileinfo): + if fileinfo.get('metadata_only', False): + raise koji.GenericError('Metadata-only imports are not supported for rpms') + # TODO - support for rpms too fn = fileinfo['hub.path'] rpminfo = import_rpm(fn, buildinfo, brinfo.id) # TODO - handle fileinfo['extra'] @@ -4813,6 +4819,9 @@ def cg_import_rpm(buildinfo, brinfo, fileinfo): def cg_import_log(buildinfo, fileinfo): + if fileinfo.get('metadata_only', False): + # logs are not currently tracked, so this is a no op + return # TODO: determine subdir fn = fileinfo['hub.path'] import_build_log(fn, buildinfo, subdir=None) @@ -4834,7 +4843,7 @@ def cg_import_archive(buildinfo, brinfo, fileinfo): type_info = extra[key] # TODO: teach import_archive to handle extra - import_archive(fn, buildinfo, l_type, type_info, brinfo['id']) + import_archive_internal(fn, buildinfo, l_type, type_info, brinfo.id, fileinfo) def cg_export(build): @@ -5455,40 +5464,83 @@ def import_old_image(old, name, version): return binfo + def import_archive(filepath, buildinfo, type, typeInfo, buildroot_id=None): """ Import an archive file and associate it with a build. The archive can be any non-rpm filetype supported by Koji. + This wraps import_archive_internal and limits options + """ + + return import_archive_internal(filepath, buildinfo, type, typeInfo, buildroot_id=None) + + +def import_archive_internal(filepath, buildinfo, type, typeInfo, buildroot_id=None, fileinfo=None): + """ + Import an archive file and associate it with a build. The archive can + be any non-rpm filetype supported by Koji. + filepath: full path to the archive file buildinfo: dict of information about the build to associate the archive with (as returned by getBuild()) type: type of the archive being imported. Currently supported archive types: maven, win, image typeInfo: dict of type-specific information - buildroot_id: the id of the buildroot the archive was built in (may be null) + buildroot_id: the id of the buildroot the archive was built in (may be None) + fileinfo: content generator metadata for file (may be None) """ - if not os.path.exists(filepath): + + if fileinfo is None: + fileinfo = {} + metadata_only = fileinfo.get('metadata_only', False) + + if metadata_only: + filepath = None + elif not os.path.exists(filepath): raise koji.GenericError, 'no such file: %s' % filepath archiveinfo = {'buildroot_id': buildroot_id} - filename = koji.fixEncoding(os.path.basename(filepath)) - archiveinfo['filename'] = filename + archiveinfo['build_id'] = buildinfo['id'] + if metadata_only: + filename = koji.fixEncoding(fileinfo['filename']) + archiveinfo['filename'] = filename + archiveinfo['size'] = fileinfo['filesize'] + archiveinfo['checksum'] = fileinfo['checksum'] + if fileinfo['checksum_type'] != 'md5': + # XXX + # until we change the way we handle checksums, we have to limit this to md5 + raise koji.GenericError("Unsupported checksum type: %(checksum_type)s" % fileinfo) + archiveinfo['checksum_type'] = koji.CHECKSUM_TYPES[fileinfo['checksum_type']] + else: + filename = koji.fixEncoding(os.path.basename(filepath)) + archiveinfo['filename'] = filename + archiveinfo['size'] = os.path.getsize(filepath) + archivefp = file(filepath) + m = md5_constructor() + while True: + contents = archivefp.read(8192) + if not contents: + break + m.update(contents) + archivefp.close() + archiveinfo['checksum'] = m.hexdigest() + archiveinfo['checksum_type'] = koji.CHECKSUM_TYPES['md5'] + if fileinfo: + # check against metadata + if archiveinfo['size'] != fileinfo['filesize']: + raise koji.GenericError("File size mismatch for %s: %s != %s" % + (filename, archiveinfo['size'], fileinfo['filesize'])) + if fileinfo['checksum_type'] != 'md5': + # XXX + # until we change the way we handle checksums, we have to limit this to md5 + raise koji.GenericError("Unsupported checksum type: %(checksum_type)s" % fileinfo) + if archiveinfo['checksum'] != fileinfo['checksum']: + raise koji.GenericError("File checksum mismatch for %s: %s != %s" % + (filename, archiveinfo['checksum'], fileinfo['checksum'])) archivetype = get_archive_type(filename, strict=True) archiveinfo['type_id'] = archivetype['id'] - archiveinfo['build_id'] = buildinfo['id'] - archiveinfo['size'] = os.path.getsize(filepath) - archivefp = file(filepath) - m = md5_constructor() - while True: - contents = archivefp.read(8192) - if not contents: - break - m.update(contents) - archivefp.close() - archiveinfo['checksum'] = m.hexdigest() - archiveinfo['checksum_type'] = koji.CHECKSUM_TYPES['md5'] koji.plugin.run_callbacks('preImport', type='archive', archive=archiveinfo, build=buildinfo, - build_type=type, filepath=filepath) + build_type=type, filepath=filepath, fileinfo=fileinfo) # XXX verify that the buildroot is associated with a task that's associated with the build archive_id = _singleValue("SELECT nextval('archiveinfo_id_seq')", strict=True) @@ -5499,7 +5551,7 @@ def import_archive(filepath, buildinfo, type, typeInfo, buildroot_id=None): if type == 'maven': maveninfo = get_maven_build(buildinfo, strict=True) - if archivetype['name'] == 'pom': + if archivetype['name'] == 'pom' and not metadata_only: pom_info = koji.parse_pom(filepath) pom_maveninfo = koji.pom_to_maven_info(pom_info) # sanity check: Maven info from pom must match the user-supplied typeInfo @@ -5515,11 +5567,12 @@ def import_archive(filepath, buildinfo, type, typeInfo, buildroot_id=None): insert.set(archive_id=archive_id) insert.execute() - # move the file to it's final destination - mavendir = os.path.join(koji.pathinfo.mavenbuild(buildinfo), - koji.pathinfo.mavenrepo(typeInfo)) - _import_archive_file(filepath, mavendir) - _generate_maven_metadata(mavendir) + if not metadata_only: + # move the file to it's final destination + mavendir = os.path.join(koji.pathinfo.mavenbuild(buildinfo), + koji.pathinfo.mavenrepo(typeInfo)) + _import_archive_file(filepath, mavendir) + _generate_maven_metadata(mavendir) elif type == 'win': wininfo = get_win_build(buildinfo, strict=True) @@ -5534,26 +5587,29 @@ def import_archive(filepath, buildinfo, type, typeInfo, buildroot_id=None): insert.set(flags=' '.join(typeInfo['flags'])) insert.execute() - destdir = koji.pathinfo.winbuild(buildinfo) - if relpath: - destdir = os.path.join(destdir, relpath) - _import_archive_file(filepath, destdir) + if not metadata_only: + destdir = koji.pathinfo.winbuild(buildinfo) + if relpath: + destdir = os.path.join(destdir, relpath) + _import_archive_file(filepath, destdir) elif type == 'image': insert = InsertProcessor('image_archives') insert.set(archive_id=archive_id) insert.set(arch=typeInfo['arch']) insert.execute() - imgdir = os.path.join(koji.pathinfo.imagebuild(buildinfo)) - _import_archive_file(filepath, imgdir) + if not metadata_only: + imgdir = os.path.join(koji.pathinfo.imagebuild(buildinfo)) + _import_archive_file(filepath, imgdir) # import log files? else: raise koji.BuildError, 'unsupported archive type: %s' % type archiveinfo = get_archive(archive_id, strict=True) koji.plugin.run_callbacks('postImport', type='archive', archive=archiveinfo, build=buildinfo, - build_type=type, filepath=filepath) + build_type=type, filepath=filepath, fileinfo=fileinfo) return archiveinfo + def _import_archive_file(filepath, destdir): """ Move the file to it's final location on the filesystem. @@ -9988,7 +10044,7 @@ class BuildRoot(object): try: data['extra'] = json.loads(data['extra']) except Exception: - raise koji.GenericError("Invalid buildroot extra data: %(extra)r", data) + raise koji.GenericError("Invalid buildroot extra data: %(extra)r" % data) if not data: raise koji.GenericError, 'no buildroot with ID: %i' % id self.id = id From d8865cdd51571d66648319d6cfcddd1c680cc57f Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Wed, 21 Oct 2015 10:41:01 -0400 Subject: [PATCH 072/109] fix buildroot display on archiveinfo page --- www/kojiweb/archiveinfo.chtml | 2 +- www/kojiweb/index.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/www/kojiweb/archiveinfo.chtml b/www/kojiweb/archiveinfo.chtml index ffda9cff..966668bb 100644 --- a/www/kojiweb/archiveinfo.chtml +++ b/www/kojiweb/archiveinfo.chtml @@ -126,7 +126,7 @@ #for $buildroot in $buildroots - $buildroot.tag_name-$buildroot.id-$buildroot.repo_id + $buildroot._display $util.formatTime($buildroot.create_event_time) $util.imageTag($util.brStateName($buildroot.state)) diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 76f5b2d8..f67f895b 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -1395,6 +1395,12 @@ def archiveinfo(environ, archiveID, fileOrder='name', fileStart=None, buildrootO start=buildrootStart, dataName='buildroots', prefix='buildroot', order=buildrootOrder) + for br in buildroots: + if br['br_type'] == koji.BR_TYPES['STANDARD']: + br['_display'] = '%(tag_name)s-%(id)i-%(repo_id)i' % br + else: + br['_display'] = '%(cg_name)s:%(id)i' % br + values['title'] = archive['filename'] + ' | Archive Info' values['archiveID'] = archive['id'] From e50b5e04d7d9ddbaefce6903641d5f7d268e168f Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Wed, 21 Oct 2015 11:20:15 -0400 Subject: [PATCH 073/109] move buildroot label display code into util function --- www/kojiweb/archiveinfo.chtml | 4 ++-- www/kojiweb/index.py | 13 +------------ www/kojiweb/rpminfo.chtml | 4 ++-- www/lib/kojiweb/util.py | 8 ++++++++ 4 files changed, 13 insertions(+), 16 deletions(-) diff --git a/www/kojiweb/archiveinfo.chtml b/www/kojiweb/archiveinfo.chtml index 966668bb..a1c49fd3 100644 --- a/www/kojiweb/archiveinfo.chtml +++ b/www/kojiweb/archiveinfo.chtml @@ -51,7 +51,7 @@ #end if #if $builtInRoot - Buildroot$builtInRoot.tag_name-$builtInRoot.id-$builtInRoot.repo_id + Buildroot$util.brLabel($builtInRoot) #end if #if $files @@ -126,7 +126,7 @@ #for $buildroot in $buildroots - $buildroot._display + $util.brLabel($buildroot) $util.formatTime($buildroot.create_event_time) $util.imageTag($util.brStateName($buildroot.state)) diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index f67f895b..89930ca5 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -1341,10 +1341,6 @@ def rpminfo(environ, rpmID, fileOrder='name', fileStart=None, buildrootOrder='-i builtInRoot = None if rpm['buildroot_id'] != None: builtInRoot = server.getBuildroot(rpm['buildroot_id']) - if builtInRoot['br_type'] == koji.BR_TYPES['STANDARD']: - builtInRoot['_display'] = '%(tag_name)s-%(id)i-%(repo_id)i' % builtInRoot - else: - builtInRoot['_display'] = '%(cg_name)s:%(id)i' % builtInRoot if rpm['external_repo_id'] == 0: values['requires'] = server.getRPMDeps(rpm['id'], koji.DEP_REQUIRE) values['requires'].sort(_sortbyname) @@ -1395,12 +1391,6 @@ def archiveinfo(environ, archiveID, fileOrder='name', fileStart=None, buildrootO start=buildrootStart, dataName='buildroots', prefix='buildroot', order=buildrootOrder) - for br in buildroots: - if br['br_type'] == koji.BR_TYPES['STANDARD']: - br['_display'] = '%(tag_name)s-%(id)i-%(repo_id)i' % br - else: - br['_display'] = '%(cg_name)s:%(id)i' % br - values['title'] = archive['filename'] + ' | Archive Info' values['archiveID'] = archive['id'] @@ -1638,14 +1628,13 @@ def buildrootinfo(environ, buildrootID, builtStart=None, builtOrder=None, compon elif buildroot['br_type'] == koji.BR_TYPES['STANDARD']: template = 'buildrootinfo.chtml' - values['title'] = '%(tag_name)s-%(id)i-%(repo_id)i | Buildroot Info' % buildroot values['task'] = server.getTaskInfo(buildroot['task_id'], request=True) else: template = 'buildrootinfo_cg.chtml' - values['title'] = '%(cg_name)s:%(id)i | Buildroot Info' % buildroot # TODO - fetch tools and extras info + values['title'] = '%s | Buildroot Info' % kojiweb.util.brLabel(buildroot) values['buildroot'] = buildroot return _genHTML(environ, template) diff --git a/www/kojiweb/rpminfo.chtml b/www/kojiweb/rpminfo.chtml index 3590eb4e..177cfa5a 100644 --- a/www/kojiweb/rpminfo.chtml +++ b/www/kojiweb/rpminfo.chtml @@ -65,7 +65,7 @@ #if $builtInRoot - Buildroot$builtInRoot._display + Buildroot$util.brLabel($builtInRoot) #end if #if $rpm.external_repo_id == 0 @@ -208,7 +208,7 @@ #for $buildroot in $buildroots - $buildroot.tag_name-$buildroot.id-$buildroot.repo_id + $util.brLabel($buildroot) $util.formatTime($buildroot.create_event_time) $util.imageTag($util.brStateName($buildroot.state)) diff --git a/www/lib/kojiweb/util.py b/www/lib/kojiweb/util.py index 4779e132..bf4fac07 100644 --- a/www/lib/kojiweb/util.py +++ b/www/lib/kojiweb/util.py @@ -377,6 +377,14 @@ def brStateName(stateID): """Convert a numeric buildroot state into a readable name.""" return koji.BR_STATES[stateID].lower() + +def brLabel(brinfo): + if brinfo['br_type'] == koji.BR_TYPES['STANDARD']: + return '%(tag_name)s-%(id)i-%(repo_id)i' % brinfo + else: + return '%(cg_name)s:%(id)i' % brinfo + + def repoStateName(stateID): """Convert a numeric repository state into a readable name.""" if stateID == koji.REPO_INIT: From da3a5b5c09c7bf255c5804b57832fe920a9f4dda Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Wed, 21 Oct 2015 11:34:08 -0400 Subject: [PATCH 074/109] use util.brLabel when appropriate --- www/kojiweb/archivelist.chtml | 4 ++-- www/kojiweb/buildrootinfo.chtml | 2 +- www/kojiweb/buildrootinfo_cg.chtml | 2 +- www/kojiweb/rpmlist.chtml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/www/kojiweb/archivelist.chtml b/www/kojiweb/archivelist.chtml index 4696e4e5..cb5e0c19 100644 --- a/www/kojiweb/archivelist.chtml +++ b/www/kojiweb/archivelist.chtml @@ -3,9 +3,9 @@ #include "includes/header.chtml" #if $type == 'component' -

Component Archives of buildroot $buildroot.tag_name-$buildroot.id-$buildroot.repo_id

+

Component Archives of buildroot $util.brLabel($buildroot)

#else -

Archives built in buildroot $buildroot.tag_name-$buildroot.id-$buildroot.repo_id

+

Archives built in buildroot $util.brLabel($buildroot)

#end if diff --git a/www/kojiweb/buildrootinfo.chtml b/www/kojiweb/buildrootinfo.chtml index 987e7873..83ca9670 100644 --- a/www/kojiweb/buildrootinfo.chtml +++ b/www/kojiweb/buildrootinfo.chtml @@ -3,7 +3,7 @@ #include "includes/header.chtml" -

Information for buildroot $buildroot.tag_name-$buildroot.id-$buildroot.repo_id

+

Information for buildroot $util.brLabel($buildroot)

diff --git a/www/kojiweb/buildrootinfo_cg.chtml b/www/kojiweb/buildrootinfo_cg.chtml index 7485813a..2690de47 100644 --- a/www/kojiweb/buildrootinfo_cg.chtml +++ b/www/kojiweb/buildrootinfo_cg.chtml @@ -3,7 +3,7 @@ #include "includes/header.chtml" -

Information for external buildroot $buildroot.cg_name:$buildroot.id

+

Information for external buildroot $util.brLabel($buildroot)

diff --git a/www/kojiweb/rpmlist.chtml b/www/kojiweb/rpmlist.chtml index 71f4b322..0e8e5543 100644 --- a/www/kojiweb/rpmlist.chtml +++ b/www/kojiweb/rpmlist.chtml @@ -21,11 +21,11 @@ colspan="2" #slurp #end def #if $type == 'component' -

Component RPMs of buildroot $buildroot.tag_name-$buildroot.id-$buildroot.repo_id

+

Component RPMs of buildroot $util.brLabel($buildroot)

#elif $type == 'image'

RPMs installed in $image.filename

#else -

RPMs built in buildroot $buildroot.tag_name-$buildroot.id-$buildroot.repo_id

+

RPMs built in buildroot $util.brLabel($buildroot)

#end if
From 95f118272772d4ea22f9190dfbda3ec56ba2b211 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Wed, 21 Oct 2015 17:20:44 -0400 Subject: [PATCH 075/109] show upload progress in import-cg --- cli/koji | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cli/koji b/cli/koji index ab31996b..8c201fd6 100755 --- a/cli/koji +++ b/cli/koji @@ -1647,6 +1647,8 @@ def handle_import_cg(options, session, args): usage = _("usage: %prog import-cg [options] metadata_file files_dir") usage += _("\n(Specify the --help global option for a list of other help options)") parser = OptionParser(usage=usage) + parser.add_option("--noprogress", action="store_true", + help=_("Do not display progress of the upload")) parser.add_option("--link", action="store_true", help=_("Attempt to hardlink instead of uploading")) parser.add_option("--test", action="store_true", help=_("Don't actually import")) (options, args) = parser.parse_args(args) @@ -1678,11 +1680,17 @@ def handle_import_cg(options, session, args): for localpath, info in to_upload: relpath = os.path.join(serverdir, info.get('relpath', '')) + if _running_in_bg() or options.noprogress: + callback = None + else: + callback = _progress_callback if options.link: linked_upload(localpath, relpath) else: - session.uploadWrapper(localpath, relpath) - # TODO - progress callback + print "Uploading %s" % localpath + session.uploadWrapper(localpath, relpath, callback=callback) + if callback: + print session.CGImport(metadata, serverdir) From b8ac6cd80b87226172a62fe3cefec564faa02198 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 22 Oct 2015 16:18:34 -0400 Subject: [PATCH 076/109] import image components --- hub/kojihub.py | 126 +++++++++++++++++++++++++++++-------------------- 1 file changed, 74 insertions(+), 52 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index ecb9e37d..3c068ae7 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4747,58 +4747,9 @@ def cg_import_buildroot(brdata): # TODO: standard buildroot entry (if applicable) # ??? - #split components - rpms = [] - files = [] - for comp in brdata['components']: - if comp['type'] == 'rpm': - rpms.append(comp) - elif comp['type'] == 'file': - files.append(comp) - else: - raise koji.GenericError("Unknown component type: %(type)s" % comp) - - # buildroot_listing - rpmlist = [] - for comp in rpms: - # TODO: do we allow inclusion of external rpms? - if 'location' in comp: - raise koji.GenericError("External rpms not allowed") - if 'id' in comp: - # not in metadata spec, and will confuse get_rpm - raise koji.GenericError("Unexpected 'id' field in component") - rinfo = get_rpm(comp, strict=True) - if rinfo['payloadhash'] != comp['sigmd5']: - nvr = "%(name)s-%(version)s-%(release)s" % rinfo - raise koji.GenericError("md5sum mismatch for %s: %s != %s" - % (nvr, comp['sigmd5'], rinfo['payloadhash'])) - # TODO - should we check the signature field? - rpmlist.append(rinfo) + #buildroot components + rpmlist, archives = cg_match_components(brdata['components']) br.setList(rpmlist) - - # buildroot_archives - archives = [] - for comp in files: - # hmm, how do we look up archives? - # updateMavenBuildRootList does seriously wild stuff - # only unique field in archiveinfo is id - # checksum/checksum_type only works if type matches - # at the moment, we only have md5 entries in archiveinfo - - type_mismatches = 0 - for archive in list_archives(filename=comp['filename'], size=comp['filesize']): - if archive['checksum_type'] != comp['checksum_type']: - type_mismatches += 1 - continue - if archive['checksum'] == comp['checksum']: - archives.appen(archive) - break - else: - logger.error("Failed to match archive %(filename)s (size %(filesize)s, sum %(checksum)s", comp) - if type_mismatches: - logger.error("Match failed with %i type mismatches", type_mismatches) - # TODO: allow external archives [??] - raise koji.GenericError("No match: %(filename)s (size %(filesize)s, sum %(checksum)s" % comp) br.updateArchiveList(archives) # buildroot_tools_info @@ -4807,6 +4758,57 @@ def cg_import_buildroot(brdata): return br +def cg_match_components(components): + rpms = [] + files = [] + for comp in components: + if comp['type'] == 'rpm': + rpms.append(cg_match_rpm(comp)) + elif comp['type'] == 'file': + files.append(cg_match_file(comp)) + else: + raise koji.GenericError("Unknown component type: %(type)s" % comp) + return rpms, files + + +def cg_match_rpm(comp): + # TODO: do we allow inclusion of external rpms? + if 'location' in comp: + raise koji.GenericError("External rpms not allowed") + if 'id' in comp: + # not in metadata spec, and will confuse get_rpm + raise koji.GenericError("Unexpected 'id' field in component") + rinfo = get_rpm(comp, strict=True) + if rinfo['payloadhash'] != comp['sigmd5']: + nvr = "%(name)s-%(version)s-%(release)s" % rinfo + raise koji.GenericError("md5sum mismatch for %s: %s != %s" + % (nvr, comp['sigmd5'], rinfo['payloadhash'])) + # TODO - should we check the signature field? + return rinfo + + +def cg_match_file(comp): + # hmm, how do we look up archives? + # updateMavenBuildRootList does seriously wild stuff + # only unique field in archiveinfo is id + # checksum/checksum_type only works if type matches + # at the moment, we only have md5 entries in archiveinfo + + type_mismatches = 0 + for archive in list_archives(filename=comp['filename'], size=comp['filesize']): + if archive['checksum_type'] != comp['checksum_type']: + type_mismatches += 1 + continue + if archive['checksum'] == comp['checksum']: + return archive + #else + logger.error("Failed to match archive %(filename)s (size %(filesize)s, sum %(checksum)s", comp) + if type_mismatches: + logger.error("Match failed with %i type mismatches", type_mismatches) + # TODO: allow external archives [??] + raise koji.GenericError("No match: %(filename)s (size %(filesize)s, sum %(checksum)s" % comp) + + def cg_import_rpm(buildinfo, brinfo, fileinfo): if fileinfo.get('metadata_only', False): raise koji.GenericError('Metadata-only imports are not supported for rpms') @@ -4843,7 +4845,27 @@ def cg_import_archive(buildinfo, brinfo, fileinfo): type_info = extra[key] # TODO: teach import_archive to handle extra - import_archive_internal(fn, buildinfo, l_type, type_info, brinfo.id, fileinfo) + archiveinfo = import_archive_internal(fn, buildinfo, l_type, type_info, brinfo.id, fileinfo) + + if l_type == 'image': + components = fileinfo.get('components', []) + cg_import_components(archiveinfo['id'], components) + + +def cg_import_components(image_id, components): + rpmlist, archives = cg_match_components(components) + + insert = InsertProcessor('image_listing') + insert.set(image_id=image_id) + for rpminfo in rpmlist: + insert.set(rpm_id=rpminfo['id']) + insert.execute() + + insert = InsertProcessor('image_archive_listing') + insert.set(image_id=image_id) + for archiveinfo in archives: + insert.set(archive_id=archiveinfo['id']) + insert.execute() def cg_export(build): From 2e8c3ba31895c36220d0bf788cb8c6afa0404d29 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 27 Oct 2015 15:32:08 -0400 Subject: [PATCH 077/109] pass build extra along --- hub/kojihub.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 3c068ae7..27241400 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4416,7 +4416,7 @@ def new_build(data): raise koji.GenericError, "No %s value for build" % f if 'extra' in data: try: - data['extra'] = json.loads(data['extra']) + data['extra'] = json.dumps(data['extra']) except Exception: raise koji.GenericError("Invalid build extra data: %(extra)r" % data) else: @@ -4679,7 +4679,7 @@ def cg_import(metadata, directory): raise koji.GenericError("Build already exists: %r" % buildinfo) else: # create a new build - buildinfo = dslice(metadata['build'], ['name', 'version', 'release']) + buildinfo = dslice(metadata['build'], ['name', 'version', 'release', 'extra']) # epoch is not in the metadata spec, but we allow it to be specified buildinfo['epoch'] = metadata['build'].get('epoch', None) build_id = new_build(buildinfo) From 15419c26ebae12ca2177c1015a29a2151e2baf6b Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 29 Oct 2015 22:39:42 -0400 Subject: [PATCH 078/109] handle extra info in cg archive outputs --- hub/kojihub.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 27241400..20a53aa7 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4844,7 +4844,6 @@ def cg_import_archive(buildinfo, brinfo, fileinfo): l_type = key type_info = extra[key] - # TODO: teach import_archive to handle extra archiveinfo = import_archive_internal(fn, buildinfo, l_type, type_info, brinfo.id, fileinfo) if l_type == 'image': @@ -5561,6 +5560,11 @@ def import_archive_internal(filepath, buildinfo, type, typeInfo, buildroot_id=No archivetype = get_archive_type(filename, strict=True) archiveinfo['type_id'] = archivetype['id'] + # cg extra data + extra = fileinfo.get(extra, None) + if extra is not None: + archiveinfo['extra'] = json.dumps(extra) + koji.plugin.run_callbacks('preImport', type='archive', archive=archiveinfo, build=buildinfo, build_type=type, filepath=filepath, fileinfo=fileinfo) From 8a996d5033e059d1cae1b2f814998e329bc12819 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Fri, 30 Oct 2015 10:05:47 -0400 Subject: [PATCH 079/109] handle extra in cg_import_rpm --- hub/kojihub.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 20a53aa7..20324869 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4570,7 +4570,8 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None) task_id=task_id, build_id=build_id, build=binfo, logs=logs) return binfo -def import_rpm(fn,buildinfo=None,brootid=None,wrapper=False): + +def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None): """Import a single rpm into the database Designed to be called from import_build. @@ -4631,8 +4632,14 @@ def import_rpm(fn,buildinfo=None,brootid=None,wrapper=False): rpminfo['buildroot_id'] = brootid rpminfo['external_repo_id'] = 0 + # handle cg extra info + if fileinfo is not None: + extra = fileinfo.get('extra') + if extra is not None: + rpminfo['extra'] = json.dumps(extra) + koji.plugin.run_callbacks('preImport', type='rpm', rpm=rpminfo, build=buildinfo, - filepath=fn) + filepath=fn, fileinfo=fileinfo) data = rpminfo.copy() del data['sourcepackage'] @@ -4641,7 +4648,7 @@ def import_rpm(fn,buildinfo=None,brootid=None,wrapper=False): insert.execute() koji.plugin.run_callbacks('postImport', type='rpm', rpm=rpminfo, build=buildinfo, - filepath=fn) + filepath=fn, fileinfo=fileinfo) #extra fields for return rpminfo['build'] = buildinfo @@ -4814,8 +4821,7 @@ def cg_import_rpm(buildinfo, brinfo, fileinfo): raise koji.GenericError('Metadata-only imports are not supported for rpms') # TODO - support for rpms too fn = fileinfo['hub.path'] - rpminfo = import_rpm(fn, buildinfo, brinfo.id) - # TODO - handle fileinfo['extra'] + rpminfo = import_rpm(fn, buildinfo, brinfo.id, fileinfo=fileinfo) import_rpm_file(fn, buildinfo, rpminfo) add_rpm_sig(rpminfo['id'], koji.rip_rpm_sighdr(fn)) From 05ea5ea24a2442eb4f1a0153ef0eea01f9468a97 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Fri, 30 Oct 2015 12:18:53 -0400 Subject: [PATCH 080/109] return extra info in get_rpm, support transforms in QueryProcessor --- hub/kojihub.py | 67 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 56 insertions(+), 11 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 20324869..1ecd81d3 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -3360,6 +3360,13 @@ def get_next_release(build_info): raise koji.BuildError, 'Unable to increment release value: %s' % release return release + +def _fix_rpm_row(row): + row['size'] = koji.encode_int(row['size']) + row['extra'] = parse_json(row['extra'], desc='rpm extra') + return row + + def get_rpm(rpminfo, strict=False, multi=False): """Get information about the specified RPM @@ -3386,6 +3393,7 @@ def get_rpm(rpminfo, strict=False, multi=False): - buildroot_id - external_repo_id - external_repo_name + - extra If there is no RPM with the given ID, None is returned, unless strict is True in which case an exception is raised @@ -3408,6 +3416,7 @@ def get_rpm(rpminfo, strict=False, multi=False): ('payloadhash', 'payloadhash'), ('size', 'size'), ('buildtime', 'buildtime'), + ('extra', 'extra'), ) # we can look up by id or NVRA data = None @@ -3439,15 +3448,11 @@ def get_rpm(rpminfo, strict=False, multi=False): query = QueryProcessor(columns=[f[0] for f in fields], aliases=[f[1] for f in fields], tables=['rpminfo'], joins=joins, clauses=clauses, - values=data) + values=data, transform=_fix_rpm_row) if multi: - data = query.execute() - for row in data: - row['size'] = koji.encode_int(row['size']) - return data + return query.execute() ret = query.executeOne() if ret: - ret['size'] = koji.encode_int(ret['size']) return ret if retry: #at this point we have just an NVRA with no internal match. Open it up to externals @@ -3457,9 +3462,9 @@ def get_rpm(rpminfo, strict=False, multi=False): if strict: raise koji.GenericError, "No such rpm: %r" % data return None - ret['size'] = koji.encode_int(ret['size']) return ret + def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID=None, hostID=None, arches=None, queryOpts=None): """List RPMS. If buildID, imageID and/or buildrootID are specified, restrict the list of RPMs to only those RPMs that are part of that @@ -6782,6 +6787,20 @@ def get_event(): return event_id +def parse_json(value, desc=None, errstr=None): + if value is None: + return value + try: + return json.loads(value) + except Exception: + if errstr is None: + if desc is None: + errstr = "Invalid json data for %s" % desc + else + errstr = "Invalid json data" + raise koji.GenericError("%s: %r" % (errstr, value)) + + class InsertProcessor(object): """Build an insert statement @@ -6945,6 +6964,8 @@ class QueryProcessor(object): - clauses: a list of where clauses in the form 'table1.col1 OPER table2.col2-or-variable'; each clause will be surrounded by parentheses and all will be AND'ed together - values: the map that will be used to replace any substitution expressions in the query + - transform: a function that will be called on each row (not compatible with + countOnly or singleValue) - opts: a map of query options; currently supported options are: countOnly: if True, return an integer indicating how many results would have been returned, rather than the actual query results @@ -6959,7 +6980,8 @@ class QueryProcessor(object): iterchunksize = 1000 def __init__(self, columns=None, aliases=None, tables=None, - joins=None, clauses=None, values=None, opts=None): + joins=None, clauses=None, values=None, transform=None, + opts=None): self.columns = columns self.aliases = aliases if columns and aliases: @@ -6976,6 +6998,7 @@ class QueryProcessor(object): self.values = values else: self.values = {} + self.transform=transform if opts: self.opts = opts else: @@ -7083,16 +7106,29 @@ SELECT %(col_str)s return '' def singleValue(self, strict=True): + # self.transform not applied here return _singleValue(str(self), self.values, strict=strict) + def execute(self): query = str(self) if self.opts.get('countOnly'): return _singleValue(query, self.values, strict=True) elif self.opts.get('asList'): - return _fetchMulti(query, self.values) + if self.transform is None: + return _fetchMulti(query, self.values) + else: + # if we're transforming, generate the dicts so the transform can modify + fields = self.aliases or self.columns + data = _multiRow(query, self.values, fields) + data = [self.transform(row) for row in data] + # and then convert back to lists + data = [ [row[f] for f in fields] for row in data] else: - return _multiRow(query, self.values, (self.aliases or self.columns)) + data = _multiRow(query, self.values, (self.aliases or self.columns)) + if self.transform is not None: + data = [self.transform(row) for row in data] + return data def iterate(self): @@ -7120,9 +7156,18 @@ SELECT %(col_str)s query = "FETCH %i FROM %s" % (chunksize, cname) while True: if as_list: - buf = _fetchMulti(query, {}) + if self.transform is None: + buf = _fetchMulti(query, {}) + else: + # if we're transforming, generate the dicts so the transform can modify + buf = _multiRow(query, self.values, fields) + buf = [self.transform(row) for row in data] + # and then convert back to lists + buf = [ [row[f] for f in fields] for row in data] else: buf = _multiRow(query, {}, fields) + if self.transform is not None: + buf = [self.transform(buf) for row in data] if not buf: break for row in buf: From 906712f223ec17cf895d0d9c5ccc8aa303ab757f Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Fri, 30 Oct 2015 12:42:59 -0400 Subject: [PATCH 081/109] return rpm extra data more places, plus fix some typos from earlier commits --- hub/kojihub.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 1ecd81d3..8a43b117 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -1255,7 +1255,9 @@ def readTaggedRPMS(tag, package=None, arch=None, event=None,inherit=False,latest ('rpminfo.size', 'size'), ('rpminfo.buildtime', 'buildtime'), ('rpminfo.buildroot_id', 'buildroot_id'), - ('rpminfo.build_id', 'build_id')] + ('rpminfo.build_id', 'build_id'), + ('rpminfo.extra', 'extra'), + ] tables = ['rpminfo'] joins = ['tag_listing ON rpminfo.build_id = tag_listing.build_id'] clauses = [eventCondition(event), 'tag_id=%(tagid)s'] @@ -1279,7 +1281,7 @@ def readTaggedRPMS(tag, package=None, arch=None, event=None,inherit=False,latest fields, aliases = zip(*fields) query = QueryProcessor(tables=tables, joins=joins, clauses=clauses, - columns=fields, aliases=aliases, values=data) + columns=fields, aliases=aliases, values=data, transform=_fix_rpm_row) # unique constraints ensure that each of these queries will not report # duplicate rpminfo entries, BUT since we make the query multiple times, @@ -3503,6 +3505,7 @@ def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID ('rpminfo.build_id', 'build_id'), ('rpminfo.buildroot_id', 'buildroot_id'), ('rpminfo.external_repo_id', 'external_repo_id'), ('external_repo.name', 'external_repo_name'), + ('rpminfo.extra', 'extra'), ] joins = ['external_repo ON rpminfo.external_repo_id = external_repo.id'] clauses = [] @@ -3537,17 +3540,11 @@ def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID fields, aliases = zip(*fields) query = QueryProcessor(columns=fields, aliases=aliases, tables=['rpminfo'], joins=joins, clauses=clauses, - values=locals(), opts=queryOpts) + values=locals(), transform=_fix_rpm_row, opts=queryOpts) data = query.execute() - if not (queryOpts and queryOpts.get('countOnly')): - if queryOpts and 'asList' in queryOpts: - key = aliases.index('size') - else: - key = 'size' - for row in data: - row[key] = koji.encode_int(row[key]) return data + def get_maven_build(buildInfo, strict=False): """ Retrieve Maven-specific information about a build. @@ -6796,7 +6793,7 @@ def parse_json(value, desc=None, errstr=None): if errstr is None: if desc is None: errstr = "Invalid json data for %s" % desc - else + else: errstr = "Invalid json data" raise koji.GenericError("%s: %r" % (errstr, value)) @@ -7161,13 +7158,13 @@ SELECT %(col_str)s else: # if we're transforming, generate the dicts so the transform can modify buf = _multiRow(query, self.values, fields) - buf = [self.transform(row) for row in data] + buf = [self.transform(row) for row in buf] # and then convert back to lists - buf = [ [row[f] for f in fields] for row in data] + buf = [ [row[f] for f in fields] for row in buf] else: buf = _multiRow(query, {}, fields) if self.transform is not None: - buf = [self.transform(buf) for row in data] + buf = [self.transform(row) for row in buf] if not buf: break for row in buf: From 8d349ecde25edb22cd33c7872b9824fd481f88da Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Sun, 1 Nov 2015 12:27:52 -0500 Subject: [PATCH 082/109] more centralization of json parsing --- hub/kojihub.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 8a43b117..d567a2eb 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2831,11 +2831,7 @@ def get_tag_extra(tagInfo, event=None): opts={'asList': True}) result = {} for key, value in query.execute(): - try: - value = json.loads(value) - except Exception: - # this should not happen - raise koji.GenericError("Invalid tag extra data: %s : %r" % (key, value)) + value = parse_json(value, errstr="Invalid tag extra data: %s" % key) result[key] = value return result @@ -3324,6 +3320,7 @@ def get_build(buildInfo, strict=False): ] clauses = ['build.id = %(buildID)i'] query = QueryProcessor(columns=fields, aliases=aliases, values=locals(), + transform=_fix_extra_field, tables=['build'], joins=joins, clauses=clauses) result = query.executeOne() @@ -3333,8 +3330,6 @@ def get_build(buildInfo, strict=False): else: return None else: - if result['extra'] is not None: - result['extra'] = json.loads(result['extra']) return result @@ -6798,6 +6793,11 @@ def parse_json(value, desc=None, errstr=None): raise koji.GenericError("%s: %r" % (errstr, value)) +def _fix_extra_field(row): + row['extra'] = parse_json(row['extra'], errstr='Invalid extra data') + return row + + class InsertProcessor(object): """Build an insert statement @@ -10112,13 +10112,9 @@ class BuildRoot(object): 'extra', ] query = QueryProcessor(columns=fields, tables=['buildroot'], + transform=_fix_extra_field, values={'id': id}, clauses=['id=%(id)s']) data = query.executeOne() - if data['extra'] != None: - try: - data['extra'] = json.loads(data['extra']) - except Exception: - raise koji.GenericError("Invalid buildroot extra data: %(extra)r" % data) if not data: raise koji.GenericError, 'no buildroot with ID: %i' % id self.id = id From 1aaaee7dd0e497131ebd95d687627c8b33079ab0 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Sun, 1 Nov 2015 16:00:09 -0500 Subject: [PATCH 083/109] provide extra data in several archive queries --- hub/kojihub.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index d567a2eb..19edf306 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -1346,6 +1346,7 @@ def readTaggedArchives(tag, package=None, event=None, inherit=False, latest=True ('archiveinfo.size', 'size'), ('archiveinfo.checksum', 'checksum'), ('archiveinfo.checksum_type', 'checksum_type'), + ('archiveinfo.extra', 'extra'), ] tables = ['archiveinfo'] joins = ['tag_listing ON archiveinfo.build_id = tag_listing.build_id'] @@ -1370,6 +1371,7 @@ def readTaggedArchives(tag, package=None, event=None, inherit=False, latest=True raise koji.GenericError, 'unsupported archive type: %s' % type query = QueryProcessor(tables=tables, joins=joins, clauses=clauses, + transform=_fix_archive_row, columns=[pair[0] for pair in fields], aliases=[pair[1] for pair in fields]) @@ -2109,6 +2111,7 @@ def maven_tag_archives(tag_id, event_id=None, inherit=True): ('archiveinfo.filename', 'filename'), ('archiveinfo.size', 'size'), ('archiveinfo.checksum', 'checksum'), ('archiveinfo.checksum_type', 'checksum_type'), + ('archiveinfo.extra', 'extra'), ('maven_archives.group_id', 'group_id'), ('maven_archives.artifact_id', 'artifact_id'), ('maven_archives.version', 'version'), @@ -2124,6 +2127,7 @@ def maven_tag_archives(tag_id, event_id=None, inherit=True): order = '-tag_event' query = QueryProcessor(tables=tables, joins=joins, clauses=clauses, opts={'order': order}, + transform=_fix_archive_row, columns=[f[0] for f in fields], aliases=[f[1] for f in fields]) included = {} @@ -3359,10 +3363,15 @@ def get_next_release(build_info): def _fix_rpm_row(row): - row['size'] = koji.encode_int(row['size']) - row['extra'] = parse_json(row['extra'], desc='rpm extra') + if 'size'in row: + row['size'] = koji.encode_int(row['size']) + if 'extra' in row: + row['extra'] = parse_json(row['extra'], desc='rpm extra') return row +#alias for now, may change in the future +_fix_archive_row = _fix_rpm_row + def get_rpm(rpminfo, strict=False, multi=False): """Get information about the specified RPM @@ -3680,6 +3689,7 @@ def list_archives(buildID=None, buildrootID=None, imageID=None, componentBuildro ('archiveinfo.size', 'size'), ('archiveinfo.checksum', 'checksum'), ('archiveinfo.checksum_type', 'checksum_type'), + ('archiveinfo.extra', 'extra'), ('archivetypes.name', 'type_name'), ('archivetypes.description', 'type_description'), ('archivetypes.extensions', 'type_extensions'), @@ -3761,16 +3771,11 @@ def list_archives(buildID=None, buildrootID=None, imageID=None, componentBuildro columns, aliases = zip(*fields) ret = QueryProcessor(tables=tables, columns=columns, aliases=aliases, joins=joins, + transform=_fix_archive_row, clauses=clauses, values=values, opts=queryOpts).execute() - if not (queryOpts and queryOpts.get('countOnly')): - if queryOpts and 'asList' in queryOpts: - key = aliases.index('size') - else: - key = 'size' - for row in ret: - row[key] = koji.encode_int(row[key]) return ret + def get_archive(archive_id, strict=False): """ Get information about the archive with the given ID. Returns a map @@ -3796,14 +3801,12 @@ def get_archive(archive_id, strict=False): If the archive is part of an image build, and it is the image file that contains the root partitioning ('/'), there will be a additional fields: - rootid arch """ - fields = ('id', 'type_id', 'build_id', 'buildroot_id', 'filename', 'size', 'checksum', 'checksum_type') - select = """SELECT %s FROM archiveinfo - WHERE id = %%(archive_id)i""" % ', '.join(fields) - archive = _singleRow(select, locals(), fields, strict=strict) + fields = ('id', 'type_id', 'build_id', 'buildroot_id', 'filename', 'size', 'checksum', 'checksum_type', 'extra') + archive = QueryProcessor(tables=['archiveinfo'], columns=fields, transform=_fix_archive_row, + clauses=['id=%(archive_id)s'], values=locals()).executeOne() if not archive: # strict is taken care of by _singleRow() return None @@ -3819,7 +3822,6 @@ def get_archive(archive_id, strict=False): if image_info: del image_info['archive_id'] archive.update(image_info) - archive['size'] = koji.encode_int(archive['size']) return archive def get_maven_archive(archive_id, strict=False): @@ -4191,6 +4193,7 @@ def query_buildroots(hostID=None, tagID=None, state=None, rpmID=None, archiveID= ('buildroot.container_type', 'container_type'), ('buildroot.host_os', 'host_os'), ('buildroot.host_arch', 'host_arch'), + ('buildroot.extra', 'extra'), ('standard_buildroot.state', 'state'), ('standard_buildroot.task_id', 'task_id'), ('host.id', 'host_id'), ('host.name', 'host_name'), @@ -4239,6 +4242,7 @@ def query_buildroots(hostID=None, tagID=None, state=None, rpmID=None, archiveID= query = QueryProcessor(columns=[f[0] for f in fields], aliases=[f[1] for f in fields], tables=tables, joins=joins, clauses=clauses, values=locals(), + transform=_fix_extra_field, opts=queryOpts) return query.execute() From e71ee4bad348d73dac67dfad12e29414ec183777 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Sun, 1 Nov 2015 16:47:38 -0500 Subject: [PATCH 084/109] show extra data on buildinfo page --- www/kojiweb/buildinfo.chtml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/www/kojiweb/buildinfo.chtml b/www/kojiweb/buildinfo.chtml index 77cb2497..59059db1 100644 --- a/www/kojiweb/buildinfo.chtml +++ b/www/kojiweb/buildinfo.chtml @@ -1,5 +1,6 @@ #import koji #import koji.util +#from pprint import pformat #from kojiweb import util #include "includes/header.chtml" @@ -77,6 +78,11 @@ #end if + #if 'extra' in $build + + + + #end if #end if + #if $archive.get('extra') + + + + #end if #if $files diff --git a/www/kojiweb/buildinfo.chtml b/www/kojiweb/buildinfo.chtml index 59059db1..0a08d9af 100644 --- a/www/kojiweb/buildinfo.chtml +++ b/www/kojiweb/buildinfo.chtml @@ -78,7 +78,7 @@ #end if - #if 'extra' in $build + #if $build.get('extra') diff --git a/www/kojiweb/buildrootinfo.chtml b/www/kojiweb/buildrootinfo.chtml index 83ca9670..e04781a5 100644 --- a/www/kojiweb/buildrootinfo.chtml +++ b/www/kojiweb/buildrootinfo.chtml @@ -1,5 +1,6 @@ #import koji #from kojiweb import util +#from pprint import pformat #include "includes/header.chtml" @@ -39,6 +40,11 @@ + #if $buildroot.get('extra') + + + + #end if diff --git a/www/kojiweb/buildrootinfo_cg.chtml b/www/kojiweb/buildrootinfo_cg.chtml index 2690de47..8d2933dd 100644 --- a/www/kojiweb/buildrootinfo_cg.chtml +++ b/www/kojiweb/buildrootinfo_cg.chtml @@ -1,5 +1,6 @@ #import koji #from kojiweb import util +#from pprint import pformat #include "includes/header.chtml" @@ -24,6 +25,11 @@ + #if $buildroot.get('extra') + + + + #end if diff --git a/www/kojiweb/rpminfo.chtml b/www/kojiweb/rpminfo.chtml index 177cfa5a..586279ef 100644 --- a/www/kojiweb/rpminfo.chtml +++ b/www/kojiweb/rpminfo.chtml @@ -1,5 +1,6 @@ #import koji #from kojiweb import util +#from pprint import pformat #import time #import urllib @@ -68,6 +69,11 @@ #end if + #if $rpm.get('extra') + + + + #end if #if $rpm.external_repo_id == 0 From 66c31817817b2e479c4a24757ed307a240d03eba Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Sun, 1 Nov 2015 17:56:29 -0500 Subject: [PATCH 086/109] show extra data in cli. fix cli rpminfo for cg brs --- cli/koji | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/cli/koji b/cli/koji index 8c201fd6..13de24f1 100755 --- a/cli/koji +++ b/cli/koji @@ -3171,9 +3171,16 @@ def anon_handle_rpminfo(options, session, args): print "No buildroot data available" else: br_info = session.getBuildroot(info['buildroot_id']) - print "Buildroot: %(id)i (tag %(tag_name)s, arch %(arch)s, repo %(repo_id)i)" % br_info - print "Build Host: %(host_name)s" % br_info - print "Build Task: %(task_id)i" % br_info + if br_info['br_type'] == koji.BR_TYPES['STANDARD']: + print "Buildroot: %(id)i (tag %(tag_name)s, arch %(arch)s, repo %(repo_id)i)" % br_info + print "Build Host: %(host_name)s" % br_info + print "Build Task: %(task_id)i" % br_info + else: + print "Content generator: %(cg_name)s" % br_info + print "Buildroot: %(id)i" % br_info + print "Build Host OS: %(host_os)s (%(host_arch)s)" % br_info + if info.get('extra'): + print "Extra: %(extra)r" % info if options.buildroots: br_list = session.listBuildroots(rpmID=info['id'], queryOpts={'order':'buildroot.id'}) print "Used in %i buildroots:" % len(br_list) @@ -3229,6 +3236,8 @@ def anon_handle_buildinfo(options, session, args): if win_info: print "Windows build platform: %s" % win_info['platform'] print "Tags: %s" % ' '.join(taglist) + if info.get('extra'): + print "Extra: %(extra)r" % info maven_archives = session.listArchives(buildID=info['id'], type='maven') if maven_archives: print "Maven archives:" From f16683d523830b098eef0c0db33e170c80f6ea1e Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 2 Nov 2015 12:14:02 -0500 Subject: [PATCH 087/109] remove broken cg_export stub --- hub/kojihub.py | 83 -------------------------------------------------- 1 file changed, 83 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 3e51bc9e..580f2d72 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4874,88 +4874,6 @@ def cg_import_components(image_id, components): insert.execute() -def cg_export(build): - """Return CG metadata and file refs for a given build""" - - metadata = {'metadata_version' : 0} - - binfo = get_build(build, strict=True) - # TODO: handle multiple build types - metadata['build'] = dslice(binfo, ['name', 'version', 'release']) - # TODO: get source from task if possible - # TODO: get start_time and end_time from task and or build - - metadata['output'] = [] - brmap = {} - - # gather rpms - for rpminfo in list_rpms(buildID=binfo['id']): - data = {} - data['filename'] = koji.pathinfo.rpm(rpminfo) # XXX [?] - data['filesize'] = rpminfo['size'] - data['arch'] = rpminfo['arch'] - data['checksum'] = rpminfo['payloadhash'] - data['checksum_type'] = 'sigmd5' - data['type'] = 'rpm' - br_id = rpminfo['buildroot_id'] - if br_id is None: - data['buildroot_id'] = None - else: - brmap.setdefault(br_id, len(brmap)) - data['buildroot_id'] = brmap[br_id] - metadata['output'].append(data) - - # archives too - for archiveinfo in list_archives(buildID=binfo['id']): - data = {} - data['filename'] = archiveinfo['filename'] - data['filesize'] = archiveinfo['size'] - data['arch'] = None #XXX - data['checksum'] = archiveinfo['checksum'] - data['checksum_type'] = koji.CHECKSUM_TYPES(archiveinfo['checksum_type']) - data['type'] = archiveinfo['type_name'] - br_id = archiveinfo['buildroot_id'] - if br_id is None: - data['buildroot_id'] = None - else: - brmap.setdefault(br_id, len(brmap)) - data['buildroot_id'] = brmap[br_id] - metadata['output'].append(data) - - # gather buildroot info - metadata['buildroots'] = [] - for br_id in brmap: - # host(os, arch), cg(name, version), container(type, arch), tools([name, version]) - # rpms([n,v,r,e,a,md5,sig]), archives([fn, sz, csum, sumtype]) - # extra - brinfo = get_buildroot(br_id) - data = {} - data['id'] = brmap[br_id] - data['container'] = {'type': 'mock', 'arch': brinfo['arch']} - data['host'] = {'os': 'unknown', 'arch': brinfo['arch']} #XXX - data['tools'] = [] - data['component_rpms'] = [] - for rpminfo in list_rpms(componentBuildrootID=br_id): - info = dslice(rpminfo, ['name', 'version', 'release', 'epoch', 'arch']) - info['sigmd5'] = rpminfo['payloadhash'] - info['sig'] = None - data['component_rpms'].append(info) - data['component_archives'] = [] - for archiveinfo in list_archives(componentBuildrootID=br_id): - info = dslice(archiveinfo, ['filename', 'checksum']) - info['filesize'] = info['size'] - info['checksum_type'] = koji.CHECKSUM_TYPES(archiveinfo['checksum_type']) - data['component_archives'].append(info) - data['extra'] = dslice(brinfo, ['host_id', 'host_name', 'repo_id', 'task_id']) - data['extra']['orig_id'] = brinfo['id'] - metadata['buildroots'].append(data) - - #TODO also return paths to all output files relative to topdir - - #TODO logs - return metadata - - def add_external_rpm(rpminfo, external_repo, strict=True): """Add an external rpm entry to the rpminfo table @@ -8281,7 +8199,6 @@ class RootExports(object): import_archive(fullpath, buildinfo, type, typeInfo) CGImport = staticmethod(cg_import) - CGExport = staticmethod(cg_export) untaggedBuilds = staticmethod(untagged_builds) tagHistory = staticmethod(tag_history) From 3019974148d35e034360b24a5144bd0732cc8bf8 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 2 Nov 2015 16:01:47 -0500 Subject: [PATCH 088/109] handle json metadata as an upload --- hub/kojihub.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 580f2d72..203cba24 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4658,7 +4658,15 @@ def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None): def cg_import(metadata, directory): - """ Import build from a content generator""" + """Import build from a content generator + + metadata can be one of the following + - json encoded string representing the metadata + - a dictionary (parsed metadata) + - a filename containing the metadata + """ + + metadata = cg_get_metadata(metadata, directory) metaver = metadata['metadata_version'] if metaver != 0: @@ -4736,6 +4744,28 @@ def cg_import(metadata, directory): # TODO: post import callback +def cg_get_metadata(metadata, directory): + """Get the metadata from the args""" + + if isinstance(metadata, dict): + return metadata + if metadata is None: + #default to looking for uploaded file + metadata = 'metadata.json' + if not isinstance(metadata, (str, unicode)): + raise koji.GenericError("Invalid metadata value: %r" % metadata) + if metadata.endswith('.json'): + # handle uploaded metadata + workdir = koji.pathinfo.work() + path = os.path.join(workdir, directory, metadata) + if not os.path.exists(path): + raise koji.GenericError("No such file: %s" % metadata) + fo = open(path, 'r') + metadata = fo.read() + fo.close() + return parse_json(metadata, desc='metadata') + + def cg_import_buildroot(brdata): """Import the given buildroot data""" From c0e753894be615e6c52a480a6421ff55839f2ddd Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 3 Nov 2015 12:38:56 -0500 Subject: [PATCH 089/109] fix typo --- hub/kojihub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 203cba24..91069f0e 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -5516,7 +5516,7 @@ def import_archive_internal(filepath, buildinfo, type, typeInfo, buildroot_id=No archiveinfo['type_id'] = archivetype['id'] # cg extra data - extra = fileinfo.get(extra, None) + extra = fileinfo.get('extra', None) if extra is not None: archiveinfo['extra'] = json.dumps(extra) From 704f39e24ece31a3f5551b5fd4cd79ef657eb2e7 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 3 Nov 2015 19:39:51 -0500 Subject: [PATCH 090/109] combine cg import code into a class --- hub/kojihub.py | 439 +++++++++++++++++++++++++------------------------ 1 file changed, 224 insertions(+), 215 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 91069f0e..10281c7b 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4666,242 +4666,251 @@ def cg_import(metadata, directory): - a filename containing the metadata """ - metadata = cg_get_metadata(metadata, directory) + importer = CG_Importer() + importer.do_import(metadata, directory) - metaver = metadata['metadata_version'] - if metaver != 0: - raise koji.GenericError("Unknown metadata version: %r" % metaver) - # assert cg access - cgs = set() - for brdata in metadata['buildroots']: - cginfo = brdata['content_generator'] - cg = lookup_name('content_generator', cginfo['name'], strict=True) - cgs.add(cg['id']) - brdata['cg_id'] = cg['id'] - for cg_id in cgs: - assert_cg(cg_id) +class CG_Importer(object): - # TODO: pre import callback - # TODO: basic metadata sanity check (use jsonschema?) + def do_import(self, metadata, directory): - # TODO: policy hooks + metadata = self.get_metadata(metadata, directory) - # build entry - buildinfo = get_build(metadata['build'], strict=False) - if buildinfo: - # TODO : allow in some cases - raise koji.GenericError("Build already exists: %r" % buildinfo) - else: - # create a new build - buildinfo = dslice(metadata['build'], ['name', 'version', 'release', 'extra']) - # epoch is not in the metadata spec, but we allow it to be specified - buildinfo['epoch'] = metadata['build'].get('epoch', None) - build_id = new_build(buildinfo) - buildinfo = get_build(build_id, strict=True) + metaver = metadata['metadata_version'] + if metaver != 0: + raise koji.GenericError("Unknown metadata version: %r" % metaver) - # handle special build types - b_extra = metadata['build'].get('extra', {}) - if 'maven' in b_extra: - new_maven_build(buildinfo, b_extra['maven']) - if 'win' in b_extra: - new_win_build(buildinfo, b_extra['win']) - if 'image' in b_extra: - # no extra info tracked at build level - new_image_build(buildinfo) + # assert cg access + cgs = set() + for brdata in metadata['buildroots']: + cginfo = brdata['content_generator'] + cg = lookup_name('content_generator', cginfo['name'], strict=True) + cgs.add(cg['id']) + brdata['cg_id'] = cg['id'] + for cg_id in cgs: + assert_cg(cg_id) - # buildroots - br_used = set([f['buildroot_id'] for f in metadata['output']]) - brmap = {} - for brdata in metadata['buildroots']: - brfakeid = brdata['id'] - if brfakeid not in br_used: - raise koji.GenericError("Buildroot id not used in output: %r" % brfakeid) - if brfakeid in brmap: - raise koji.GenericError("Duplicate buildroot id in metadata: %r" % brfakeid) - brmap[brfakeid] = cg_import_buildroot(brdata) + # TODO: pre import callback - # outputs - for fileinfo in metadata['output']: + # TODO: basic metadata sanity check (use jsonschema?) + + # TODO: policy hooks + + # build entry + buildinfo = get_build(metadata['build'], strict=False) + if buildinfo: + # TODO : allow in some cases + raise koji.GenericError("Build already exists: %r" % buildinfo) + else: + # create a new build + buildinfo = dslice(metadata['build'], ['name', 'version', 'release', 'extra']) + # epoch is not in the metadata spec, but we allow it to be specified + buildinfo['epoch'] = metadata['build'].get('epoch', None) + build_id = new_build(buildinfo) + buildinfo = get_build(build_id, strict=True) + + # handle special build types + b_extra = metadata['build'].get('extra', {}) + if 'maven' in b_extra: + new_maven_build(buildinfo, b_extra['maven']) + if 'win' in b_extra: + new_win_build(buildinfo, b_extra['win']) + if 'image' in b_extra: + # no extra info tracked at build level + new_image_build(buildinfo) + + # buildroots + br_used = set([f['buildroot_id'] for f in metadata['output']]) + brmap = {} + for brdata in metadata['buildroots']: + brfakeid = brdata['id'] + if brfakeid not in br_used: + raise koji.GenericError("Buildroot id not used in output: %r" % brfakeid) + if brfakeid in brmap: + raise koji.GenericError("Duplicate buildroot id in metadata: %r" % brfakeid) + brmap[brfakeid] = self.import_buildroot(brdata) + + # outputs + for fileinfo in metadata['output']: + if fileinfo.get('metadata_only', False): + if not metadata['build'].get('metadata_only'): + raise koji.GenericError('Build must be marked metadata-only to include metadata-only outputs') + workdir = koji.pathinfo.work() + path = os.path.join(workdir, directory, fileinfo.get('relpath', ''), fileinfo['filename']) + fileinfo['hub.path'] = path + brinfo = brmap.get(fileinfo['buildroot_id']) + if not brinfo: + raise koji.GenericError("Missing buildroot metadata for id %(buildroot_id)r" % fileinfo) + + if fileinfo['type'] == 'rpm': + self.import_rpm(buildinfo, brinfo, fileinfo) + elif fileinfo['type'] == 'log': + self.import_log(buildinfo, fileinfo) + else: + self.import_archive(buildinfo, brinfo, fileinfo) + + # TODO: post import callback + + + def get_metadata(self, metadata, directory): + """Get the metadata from the args""" + + if isinstance(metadata, dict): + return metadata + if metadata is None: + #default to looking for uploaded file + metadata = 'metadata.json' + if not isinstance(metadata, (str, unicode)): + raise koji.GenericError("Invalid metadata value: %r" % metadata) + if metadata.endswith('.json'): + # handle uploaded metadata + workdir = koji.pathinfo.work() + path = os.path.join(workdir, directory, metadata) + if not os.path.exists(path): + raise koji.GenericError("No such file: %s" % metadata) + fo = open(path, 'r') + metadata = fo.read() + fo.close() + return parse_json(metadata, desc='metadata') + + + def import_buildroot(self, brdata): + """Import the given buildroot data""" + + # buildroot entry + brinfo = { + 'cg_id' : brdata['cg_id'], + 'cg_version' : brdata['content_generator']['version'], + 'container_type' : brdata['container']['type'], + 'container_arch' : brdata['container']['arch'], + 'host_os' : brdata['host']['os'], + 'host_arch' : brdata['host']['arch'], + 'extra' : brdata.get('extra'), + } + br = BuildRoot() + br.cg_new(brinfo) + + # TODO: standard buildroot entry (if applicable) + # ??? + + #buildroot components + rpmlist, archives = self.match_components(brdata['components']) + br.setList(rpmlist) + br.updateArchiveList(archives) + + # buildroot_tools_info + br.setTools(brdata['tools']) + + return br + + + def match_components(self, components): + rpms = [] + files = [] + for comp in components: + if comp['type'] == 'rpm': + rpms.append(self.match_rpm(comp)) + elif comp['type'] == 'file': + files.append(self.match_file(comp)) + else: + raise koji.GenericError("Unknown component type: %(type)s" % comp) + return rpms, files + + + def match_rpm(self, comp): + # TODO: do we allow inclusion of external rpms? + if 'location' in comp: + raise koji.GenericError("External rpms not allowed") + if 'id' in comp: + # not in metadata spec, and will confuse get_rpm + raise koji.GenericError("Unexpected 'id' field in component") + rinfo = get_rpm(comp, strict=True) + if rinfo['payloadhash'] != comp['sigmd5']: + nvr = "%(name)s-%(version)s-%(release)s" % rinfo + raise koji.GenericError("md5sum mismatch for %s: %s != %s" + % (nvr, comp['sigmd5'], rinfo['payloadhash'])) + # TODO - should we check the signature field? + return rinfo + + + def match_file(self, comp): + # hmm, how do we look up archives? + # updateMavenBuildRootList does seriously wild stuff + # only unique field in archiveinfo is id + # checksum/checksum_type only works if type matches + # at the moment, we only have md5 entries in archiveinfo + + type_mismatches = 0 + for archive in list_archives(filename=comp['filename'], size=comp['filesize']): + if archive['checksum_type'] != comp['checksum_type']: + type_mismatches += 1 + continue + if archive['checksum'] == comp['checksum']: + return archive + #else + logger.error("Failed to match archive %(filename)s (size %(filesize)s, sum %(checksum)s", comp) + if type_mismatches: + logger.error("Match failed with %i type mismatches", type_mismatches) + # TODO: allow external archives [??] + raise koji.GenericError("No match: %(filename)s (size %(filesize)s, sum %(checksum)s" % comp) + + + def import_rpm(self, buildinfo, brinfo, fileinfo): if fileinfo.get('metadata_only', False): - if not metadata['build'].get('metadata_only'): - raise koji.GenericError('Build must be marked metadata-only to include metadata-only outputs') - workdir = koji.pathinfo.work() - path = os.path.join(workdir, directory, fileinfo.get('relpath', ''), fileinfo['filename']) - fileinfo['hub.path'] = path - brinfo = brmap.get(fileinfo['buildroot_id']) - if not brinfo: - raise koji.GenericError("Missing buildroot metadata for id %(buildroot_id)r" % fileinfo) - - if fileinfo['type'] == 'rpm': - cg_import_rpm(buildinfo, brinfo, fileinfo) - elif fileinfo['type'] == 'log': - cg_import_log(buildinfo, fileinfo) - else: - cg_import_archive(buildinfo, brinfo, fileinfo) - - # TODO: post import callback + raise koji.GenericError('Metadata-only imports are not supported for rpms') + # TODO - support for rpms too + fn = fileinfo['hub.path'] + rpminfo = import_rpm(fn, buildinfo, brinfo.id, fileinfo=fileinfo) + import_rpm_file(fn, buildinfo, rpminfo) + add_rpm_sig(rpminfo['id'], koji.rip_rpm_sighdr(fn)) -def cg_get_metadata(metadata, directory): - """Get the metadata from the args""" - - if isinstance(metadata, dict): - return metadata - if metadata is None: - #default to looking for uploaded file - metadata = 'metadata.json' - if not isinstance(metadata, (str, unicode)): - raise koji.GenericError("Invalid metadata value: %r" % metadata) - if metadata.endswith('.json'): - # handle uploaded metadata - workdir = koji.pathinfo.work() - path = os.path.join(workdir, directory, metadata) - if not os.path.exists(path): - raise koji.GenericError("No such file: %s" % metadata) - fo = open(path, 'r') - metadata = fo.read() - fo.close() - return parse_json(metadata, desc='metadata') + def import_log(self, buildinfo, fileinfo): + if fileinfo.get('metadata_only', False): + # logs are not currently tracked, so this is a no op + return + # TODO: determine subdir + fn = fileinfo['hub.path'] + import_build_log(fn, buildinfo, subdir=None) -def cg_import_buildroot(brdata): - """Import the given buildroot data""" + def import_archive(self, buildinfo, brinfo, fileinfo): + fn = fileinfo['hub.path'] - # buildroot entry - brinfo = { - 'cg_id' : brdata['cg_id'], - 'cg_version' : brdata['content_generator']['version'], - 'container_type' : brdata['container']['type'], - 'container_arch' : brdata['container']['arch'], - 'host_os' : brdata['host']['os'], - 'host_arch' : brdata['host']['arch'], - 'extra' : brdata.get('extra'), - } - br = BuildRoot() - br.cg_new(brinfo) + # determine archive import type (maven/win/image/other) + extra = fileinfo.get('extra', {}) + legacy_types = ['maven', 'win', 'image'] + l_type = None + type_info = None + for key in legacy_types: + if key in extra: + if l_type is not None: + raise koji.GenericError("Output file has multiple archive types: %s" % fn) + l_type = key + type_info = extra[key] - # TODO: standard buildroot entry (if applicable) - # ??? + archiveinfo = import_archive_internal(fn, buildinfo, l_type, type_info, brinfo.id, fileinfo) - #buildroot components - rpmlist, archives = cg_match_components(brdata['components']) - br.setList(rpmlist) - br.updateArchiveList(archives) - - # buildroot_tools_info - br.setTools(brdata['tools']) - - return br + if l_type == 'image': + components = fileinfo.get('components', []) + self.import_components(archiveinfo['id'], components) -def cg_match_components(components): - rpms = [] - files = [] - for comp in components: - if comp['type'] == 'rpm': - rpms.append(cg_match_rpm(comp)) - elif comp['type'] == 'file': - files.append(cg_match_file(comp)) - else: - raise koji.GenericError("Unknown component type: %(type)s" % comp) - return rpms, files + def import_components(self, image_id, components): + rpmlist, archives = self.match_components(components) + insert = InsertProcessor('image_listing') + insert.set(image_id=image_id) + for rpminfo in rpmlist: + insert.set(rpm_id=rpminfo['id']) + insert.execute() -def cg_match_rpm(comp): - # TODO: do we allow inclusion of external rpms? - if 'location' in comp: - raise koji.GenericError("External rpms not allowed") - if 'id' in comp: - # not in metadata spec, and will confuse get_rpm - raise koji.GenericError("Unexpected 'id' field in component") - rinfo = get_rpm(comp, strict=True) - if rinfo['payloadhash'] != comp['sigmd5']: - nvr = "%(name)s-%(version)s-%(release)s" % rinfo - raise koji.GenericError("md5sum mismatch for %s: %s != %s" - % (nvr, comp['sigmd5'], rinfo['payloadhash'])) - # TODO - should we check the signature field? - return rinfo - - -def cg_match_file(comp): - # hmm, how do we look up archives? - # updateMavenBuildRootList does seriously wild stuff - # only unique field in archiveinfo is id - # checksum/checksum_type only works if type matches - # at the moment, we only have md5 entries in archiveinfo - - type_mismatches = 0 - for archive in list_archives(filename=comp['filename'], size=comp['filesize']): - if archive['checksum_type'] != comp['checksum_type']: - type_mismatches += 1 - continue - if archive['checksum'] == comp['checksum']: - return archive - #else - logger.error("Failed to match archive %(filename)s (size %(filesize)s, sum %(checksum)s", comp) - if type_mismatches: - logger.error("Match failed with %i type mismatches", type_mismatches) - # TODO: allow external archives [??] - raise koji.GenericError("No match: %(filename)s (size %(filesize)s, sum %(checksum)s" % comp) - - -def cg_import_rpm(buildinfo, brinfo, fileinfo): - if fileinfo.get('metadata_only', False): - raise koji.GenericError('Metadata-only imports are not supported for rpms') - # TODO - support for rpms too - fn = fileinfo['hub.path'] - rpminfo = import_rpm(fn, buildinfo, brinfo.id, fileinfo=fileinfo) - import_rpm_file(fn, buildinfo, rpminfo) - add_rpm_sig(rpminfo['id'], koji.rip_rpm_sighdr(fn)) - - -def cg_import_log(buildinfo, fileinfo): - if fileinfo.get('metadata_only', False): - # logs are not currently tracked, so this is a no op - return - # TODO: determine subdir - fn = fileinfo['hub.path'] - import_build_log(fn, buildinfo, subdir=None) - - -def cg_import_archive(buildinfo, brinfo, fileinfo): - fn = fileinfo['hub.path'] - - # determine archive import type (maven/win/image/other) - extra = fileinfo.get('extra', {}) - legacy_types = ['maven', 'win', 'image'] - l_type = None - type_info = None - for key in legacy_types: - if key in extra: - if l_type is not None: - raise koji.GenericError("Output file has multiple archive types: %s" % fn) - l_type = key - type_info = extra[key] - - archiveinfo = import_archive_internal(fn, buildinfo, l_type, type_info, brinfo.id, fileinfo) - - if l_type == 'image': - components = fileinfo.get('components', []) - cg_import_components(archiveinfo['id'], components) - - -def cg_import_components(image_id, components): - rpmlist, archives = cg_match_components(components) - - insert = InsertProcessor('image_listing') - insert.set(image_id=image_id) - for rpminfo in rpmlist: - insert.set(rpm_id=rpminfo['id']) - insert.execute() - - insert = InsertProcessor('image_archive_listing') - insert.set(image_id=image_id) - for archiveinfo in archives: - insert.set(archive_id=archiveinfo['id']) - insert.execute() + insert = InsertProcessor('image_archive_listing') + insert.set(image_id=image_id) + for archiveinfo in archives: + insert.set(archive_id=archiveinfo['id']) + insert.execute() def add_external_rpm(rpminfo, external_repo, strict=True): From 928e69f61ec972bf13ef665c5b8288714e700718 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Wed, 4 Nov 2015 14:20:01 -0500 Subject: [PATCH 091/109] set completion_time for cg builds --- hub/kojihub.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hub/kojihub.py b/hub/kojihub.py index 10281c7b..638fc401 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4707,6 +4707,8 @@ class CG_Importer(object): buildinfo = dslice(metadata['build'], ['name', 'version', 'release', 'extra']) # epoch is not in the metadata spec, but we allow it to be specified buildinfo['epoch'] = metadata['build'].get('epoch', None) + buildinfo['completion_time'] = \ + datetime.datetime.fromtimestamp(metadata['build']['end_time']).isoformat(' ') build_id = new_build(buildinfo) buildinfo = get_build(build_id, strict=True) From 90d7b3635320703a37bafc8aa1c8f71add36a92c Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Wed, 4 Nov 2015 14:34:07 -0500 Subject: [PATCH 092/109] add start_time to build table --- docs/schema-update-cgen.sql | 1 + docs/schema.sql | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/schema-update-cgen.sql b/docs/schema-update-cgen.sql index 320dd80f..81a6f38b 100644 --- a/docs/schema-update-cgen.sql +++ b/docs/schema-update-cgen.sql @@ -45,6 +45,7 @@ CREATE INDEX image_listing_archives on image_archive_listing(archive_id); -- new columns -- select statement_timestamp(), 'Adding new columns' as msg; +ALTER TABLE build ADD COLUMN start_time TIMESTAMP; ALTER TABLE build ADD COLUMN extra TEXT; ALTER TABLE rpminfo ADD COLUMN extra TEXT; ALTER TABLE archiveinfo ADD COLUMN extra TEXT; diff --git a/docs/schema.sql b/docs/schema.sql index bd9b4203..b2a64ec4 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -290,6 +290,7 @@ CREATE TABLE build ( release TEXT NOT NULL, epoch INTEGER, create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), + start_time TIMESTAMP, completion_time TIMESTAMP, state INTEGER NOT NULL, task_id INTEGER REFERENCES task (id), From c80a7aa9aed7bd86fa416a5ecd615940ca6054ca Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Wed, 4 Nov 2015 16:25:33 -0500 Subject: [PATCH 093/109] support for build.start_time field --- hub/kojihub.py | 48 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 638fc401..110bb249 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -1152,6 +1152,7 @@ def readTaggedBuilds(tag,event=None,inherit=False,latest=False,package=None,owne fields = [('tag.id', 'tag_id'), ('tag.name', 'tag_name'), ('build.id', 'id'), ('build.id', 'build_id'), ('build.version', 'version'), ('build.release', 'release'), ('build.epoch', 'epoch'), ('build.state', 'state'), ('build.completion_time', 'completion_time'), + ('build.start_time', 'start_time'), ('build.task_id','task_id'), ('events.id', 'creation_event_id'), ('events.time', 'creation_time'), ('volume.id', 'volume_id'), ('volume.name', 'volume_name'), @@ -3295,6 +3296,8 @@ def get_build(buildInfo, strict=False): creation_event_id: id of the create_event creation_time: time the build was created (text) creation_ts: time the build was created (epoch) + start_time: time the build was started (may be null) + start_ts: time the build was started (epoch, may be null) completion_time: time the build was completed (may be null) completion_ts: time the build was completed (epoch, may be null) extra: dictionary with extra data about the build @@ -3308,11 +3311,13 @@ def get_build(buildInfo, strict=False): fields = (('build.id', 'id'), ('build.version', 'version'), ('build.release', 'release'), ('build.epoch', 'epoch'), ('build.state', 'state'), ('build.completion_time', 'completion_time'), + ('build.start_time', 'start_time'), ('build.task_id', 'task_id'), ('events.id', 'creation_event_id'), ('events.time', 'creation_time'), ('package.id', 'package_id'), ('package.name', 'package_name'), ('package.name', 'name'), ('volume.id', 'volume_id'), ('volume.name', 'volume_name'), ("package.name || '-' || build.version || '-' || build.release", 'nvr'), ('EXTRACT(EPOCH FROM events.time)','creation_ts'), + ('EXTRACT(EPOCH FROM build.start_time)', 'start_ts'), ('EXTRACT(EPOCH FROM build.completion_time)','completion_ts'), ('users.id', 'owner_id'), ('users.name', 'owner_name'), ('build.extra', 'extra')) @@ -4403,7 +4408,10 @@ def change_build_volume(build, volume, strict=True): def new_build(data): """insert a new build entry""" + data = data.copy() + + # basic sanity checks if 'pkg_id' in data: data['name'] = lookup_package(data['pkg_id'], strict=True)['name'] else: @@ -4422,45 +4430,50 @@ def new_build(data): raise koji.GenericError("Invalid build extra data: %(extra)r" % data) else: data['extra'] = None + #provide a few default values data.setdefault('state',koji.BUILD_STATES['COMPLETE']) + data.setdefault('start_time', 'NOW') data.setdefault('completion_time', 'NOW') data.setdefault('owner',context.session.user_id) data.setdefault('task_id',None) data.setdefault('volume_id', 0) + #check for existing build - # TODO - table lock? - q="""SELECT id,state,task_id FROM build - WHERE pkg_id=%(pkg_id)d AND version=%(version)s AND release=%(release)s - FOR UPDATE""" - row = _fetchSingle(q, data) + query = QueryProcessor( + tables=['build'], columns=['id', 'state', 'task_id'], + clauses=['pkg_id=%(pkg_id)s', 'version=%(version)s', + 'release=%(release)s'], + values=data, opts={'rowlock':True}) + row = query.executeOne() if row: - id, state, task_id = row - data['id'] = id + build_id, state, task_id = row + data['id'] = build_id koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=state, new=data['state'], info=data) st_desc = koji.BUILD_STATES[state] if st_desc == 'BUILDING': # check to see if this is the controlling task if data['state'] == state and data.get('task_id','') == task_id: #the controlling task must have restarted (and called initBuild again) - return id + return build_id raise koji.GenericError, "Build already in progress (task %d)" % task_id # TODO? - reclaim 'stale' builds (state=BUILDING and task_id inactive) if st_desc in ('FAILED','CANCELED'): #should be ok to replace - update = """UPDATE build SET state=%(state)i,task_id=%(task_id)s, - owner=%(owner)s,completion_time=%(completion_time)s,create_event=get_event() - WHERE id = %(id)i""" - _dml(update, data) + update = UpdateProcessor('build', clauses=['id=%(id)s'], values=data) + update.set(**dslice(data, ['owner', 'start_time', 'completion_time'])) + update.rawset(create_event='get_event()') + update.execute() koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=state, new=data['state'], info=data) - return id + return build_id raise koji.GenericError, "Build already exists (id=%d, state=%s): %r" \ - % (id, st_desc, data) + % (build_id, st_desc, data) else: koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=None, new=data['state'], info=data) + #insert the new data insert_data = dslice(data, ['pkg_id', 'version', 'release', 'epoch', 'state', 'volume_id', - 'task_id', 'owner', 'completion_time', 'extra']) + 'task_id', 'owner', 'start_time', 'completion_time', 'extra']) data['id'] = insert_data['id'] = _singleValue("SELECT nextval('build_id_seq')") insert = InsertProcessor('build', data=insert_data) insert.execute() @@ -4468,6 +4481,7 @@ def new_build(data): #return build_id return data['id'] + def check_noarch_rpms(basepath, rpms): """ If rpms contains any noarch rpms with identical names, @@ -8759,6 +8773,8 @@ class RootExports(object): - creation_event_id - creation_time - creation_ts + - start_time + - start_ts - completion_time - completion_ts - task_id @@ -8773,8 +8789,10 @@ class RootExports(object): """ fields = [('build.id', 'build_id'), ('build.version', 'version'), ('build.release', 'release'), ('build.epoch', 'epoch'), ('build.state', 'state'), ('build.completion_time', 'completion_time'), + ('build.start_time', 'start_time'), ('events.id', 'creation_event_id'), ('events.time', 'creation_time'), ('build.task_id', 'task_id'), ('EXTRACT(EPOCH FROM events.time)','creation_ts'), + ('EXTRACT(EPOCH FROM build.start_time)','start_ts'), ('EXTRACT(EPOCH FROM build.completion_time)','completion_ts'), ('package.id', 'package_id'), ('package.name', 'package_name'), ('package.name', 'name'), ('volume.id', 'volume_id'), ('volume.name', 'volume_name'), From 8eb55ee35fc2e0d30ae90cefcd6679c77c2f2a9c Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Wed, 4 Nov 2015 16:30:22 -0500 Subject: [PATCH 094/109] use build.start_time in webui is set --- www/kojiweb/index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 89930ca5..512f9652 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -1182,7 +1182,7 @@ def buildinfo(environ, buildID): if not values.has_key(field): values[field] = None - values['start_time'] = build['creation_time'] + values['start_time'] = build.get('start_time') or build['creation_time'] # the build start time is not accurate for maven and win builds, get it from the # task start time instead if mavenbuild or winbuild: From 26fb75920bc86663af996519132afd0f93e45f1d Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Wed, 4 Nov 2015 16:40:55 -0500 Subject: [PATCH 095/109] set build.start_time in cg_import --- hub/kojihub.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hub/kojihub.py b/hub/kojihub.py index 110bb249..a365f5a5 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4721,6 +4721,8 @@ class CG_Importer(object): buildinfo = dslice(metadata['build'], ['name', 'version', 'release', 'extra']) # epoch is not in the metadata spec, but we allow it to be specified buildinfo['epoch'] = metadata['build'].get('epoch', None) + buildinfo['start_time'] = \ + datetime.datetime.fromtimestamp(metadata['build']['start_time']).isoformat(' ') buildinfo['completion_time'] = \ datetime.datetime.fromtimestamp(metadata['build']['end_time']).isoformat(' ') build_id = new_build(buildinfo) From 470c2de57cecbb9a986e9943b5d2b2d655cbb2e7 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 5 Nov 2015 16:37:51 -0500 Subject: [PATCH 096/109] record raw metadata --- hub/kojihub.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index a365f5a5..ab655c35 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4727,6 +4727,7 @@ class CG_Importer(object): datetime.datetime.fromtimestamp(metadata['build']['end_time']).isoformat(' ') build_id = new_build(buildinfo) buildinfo = get_build(build_id, strict=True) + self.buildinfo = buildinfo # handle special build types b_extra = metadata['build'].get('extra', {}) @@ -4768,6 +4769,9 @@ class CG_Importer(object): else: self.import_archive(buildinfo, brinfo, fileinfo) + # also import metadata file + self.import_metadata() + # TODO: post import callback @@ -4775,6 +4779,12 @@ class CG_Importer(object): """Get the metadata from the args""" if isinstance(metadata, dict): + self.metadata = metadata + try: + self.raw_metadata = json.dumps(metadata, indent=2) + except Exception: + logger.exception("Cannot encode supplied metadata") + raise koji.GenericError("Invalid metada, cannot encode: %r" % metadata) return metadata if metadata is None: #default to looking for uploaded file @@ -4790,7 +4800,24 @@ class CG_Importer(object): fo = open(path, 'r') metadata = fo.read() fo.close() - return parse_json(metadata, desc='metadata') + self.raw_metadata = metadata + self.metadata = parse_json(metadata, desc='metadata') + return self.metadata + + + def import_metadata(self): + """Import the raw metadata""" + + # TODO - eventually, import this as an archive, but for now we just write it to disk + # because there are complications + # - no buildroot (could confuse policies checking that builds were built sanely + # - doesn't fit with current major archive categories + path = os.path.join(koji.pathinfo.build(self.buildinfo), 'metadata.json') + fo = open(path, 'w') + try: + fo.write(self.raw_metadata) + finally: + fo.close() def import_buildroot(self, brdata): From f5d2ad14ccd595af3f8fcb25610054b6adcb5e55 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Wed, 11 Nov 2015 00:40:14 -0500 Subject: [PATCH 097/109] pre/postImport callbacks for content generators --- hub/kojihub.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index ab655c35..a78d7bbd 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4705,7 +4705,8 @@ class CG_Importer(object): for cg_id in cgs: assert_cg(cg_id) - # TODO: pre import callback + koji.plugin.run_callbacks('preImport', type='cg', metadata=metadata, + directory=directory, importer=self) # TODO: basic metadata sanity check (use jsonschema?) @@ -4772,7 +4773,8 @@ class CG_Importer(object): # also import metadata file self.import_metadata() - # TODO: post import callback + koji.plugin.run_callbacks('postImport', type='cg', metadata=metadata, + directory=directory, importer=self, buildinfo=buildinfo) def get_metadata(self, metadata, directory): From 3faa31501c156f296f6571ec3e12e7261b9559a6 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Wed, 11 Nov 2015 00:41:13 -0500 Subject: [PATCH 098/109] temporary workaround for external component refs in cg_import --- hub/kojihub.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index a78d7bbd..57d5f937 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4857,9 +4857,13 @@ class CG_Importer(object): files = [] for comp in components: if comp['type'] == 'rpm': - rpms.append(self.match_rpm(comp)) + match = self.match_rpm(comp) + if match: + rpms.append(match) elif comp['type'] == 'file': - files.append(self.match_file(comp)) + match = self.match_file(comp) + if match: + files.append(match) else: raise koji.GenericError("Unknown component type: %(type)s" % comp) return rpms, files @@ -4872,11 +4876,17 @@ class CG_Importer(object): if 'id' in comp: # not in metadata spec, and will confuse get_rpm raise koji.GenericError("Unexpected 'id' field in component") - rinfo = get_rpm(comp, strict=True) + rinfo = get_rpm(comp, strict=False) + if not rinfo: + # XXX - this is a temporary workaround until we can better track external refs + logger.warning("IGNORING unmatched rpm component: %r", comp) + return None if rinfo['payloadhash'] != comp['sigmd5']: nvr = "%(name)s-%(version)s-%(release)s" % rinfo - raise koji.GenericError("md5sum mismatch for %s: %s != %s" - % (nvr, comp['sigmd5'], rinfo['payloadhash'])) + # XXX - this is a temporary workaround until we can better track external refs + logger.warning("IGNORING rpm component (md5 mismatch): %r", comp) + #raise koji.GenericError("md5sum mismatch for %s: %s != %s" + # % (nvr, comp['sigmd5'], rinfo['payloadhash'])) # TODO - should we check the signature field? return rinfo @@ -4899,8 +4909,11 @@ class CG_Importer(object): logger.error("Failed to match archive %(filename)s (size %(filesize)s, sum %(checksum)s", comp) if type_mismatches: logger.error("Match failed with %i type mismatches", type_mismatches) - # TODO: allow external archives [??] - raise koji.GenericError("No match: %(filename)s (size %(filesize)s, sum %(checksum)s" % comp) + # TODO: allow external archives + # XXX - this is a temporary workaround until we can better track external refs + logger.warning("IGNORING unmatched archive: %r", comp) + return None + #raise koji.GenericError("No match: %(filename)s (size %(filesize)s, sum %(checksum)s" % comp) def import_rpm(self, buildinfo, brinfo, fileinfo): From 910051e783387b1e56b44a91ce8a2531916ff6d8 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Wed, 11 Nov 2015 00:56:39 -0500 Subject: [PATCH 099/109] make sure timestamps are floats to keep datetime happy --- hub/kojihub.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 57d5f937..7d9b2304 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4723,9 +4723,9 @@ class CG_Importer(object): # epoch is not in the metadata spec, but we allow it to be specified buildinfo['epoch'] = metadata['build'].get('epoch', None) buildinfo['start_time'] = \ - datetime.datetime.fromtimestamp(metadata['build']['start_time']).isoformat(' ') + datetime.datetime.fromtimestamp(float(metadata['build']['start_time'])).isoformat(' ') buildinfo['completion_time'] = \ - datetime.datetime.fromtimestamp(metadata['build']['end_time']).isoformat(' ') + datetime.datetime.fromtimestamp(float(metadata['build']['end_time'])).isoformat(' ') build_id = new_build(buildinfo) buildinfo = get_build(build_id, strict=True) self.buildinfo = buildinfo From d8baff3b639693d8a59a14b68268ccf33f8fea79 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 12 Nov 2015 14:48:07 -0500 Subject: [PATCH 100/109] fix a small typo --- hub/kojihub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 7d9b2304..65c6e34b 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -3368,7 +3368,7 @@ def get_next_release(build_info): def _fix_rpm_row(row): - if 'size'in row: + if 'size' in row: row['size'] = koji.encode_int(row['size']) if 'extra' in row: row['extra'] = parse_json(row['extra'], desc='rpm extra') From a0036c44895658a44a7f971be9f1d696ee02ad7a Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 12 Nov 2015 15:57:30 -0500 Subject: [PATCH 101/109] refactoring cg importer to facilitate policy hooks --- hub/kojihub.py | 99 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 60 insertions(+), 39 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 65c6e34b..9515a1c8 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4686,6 +4686,8 @@ def cg_import(metadata, directory): class CG_Importer(object): + def __init__(self): + self.buildinfo = None def do_import(self, metadata, directory): @@ -4695,15 +4697,7 @@ class CG_Importer(object): if metaver != 0: raise koji.GenericError("Unknown metadata version: %r" % metaver) - # assert cg access - cgs = set() - for brdata in metadata['buildroots']: - cginfo = brdata['content_generator'] - cg = lookup_name('content_generator', cginfo['name'], strict=True) - cgs.add(cg['id']) - brdata['cg_id'] = cg['id'] - for cg_id in cgs: - assert_cg(cg_id) + self.assert_cg_access() koji.plugin.run_callbacks('preImport', type='cg', metadata=metadata, directory=directory, importer=self) @@ -4713,32 +4707,8 @@ class CG_Importer(object): # TODO: policy hooks # build entry - buildinfo = get_build(metadata['build'], strict=False) - if buildinfo: - # TODO : allow in some cases - raise koji.GenericError("Build already exists: %r" % buildinfo) - else: - # create a new build - buildinfo = dslice(metadata['build'], ['name', 'version', 'release', 'extra']) - # epoch is not in the metadata spec, but we allow it to be specified - buildinfo['epoch'] = metadata['build'].get('epoch', None) - buildinfo['start_time'] = \ - datetime.datetime.fromtimestamp(float(metadata['build']['start_time'])).isoformat(' ') - buildinfo['completion_time'] = \ - datetime.datetime.fromtimestamp(float(metadata['build']['end_time'])).isoformat(' ') - build_id = new_build(buildinfo) - buildinfo = get_build(build_id, strict=True) - self.buildinfo = buildinfo - - # handle special build types - b_extra = metadata['build'].get('extra', {}) - if 'maven' in b_extra: - new_maven_build(buildinfo, b_extra['maven']) - if 'win' in b_extra: - new_win_build(buildinfo, b_extra['win']) - if 'image' in b_extra: - # no extra info tracked at build level - new_image_build(buildinfo) + self.prep_build() + self.get_build() # buildroots br_used = set([f['buildroot_id'] for f in metadata['output']]) @@ -4764,17 +4734,17 @@ class CG_Importer(object): raise koji.GenericError("Missing buildroot metadata for id %(buildroot_id)r" % fileinfo) if fileinfo['type'] == 'rpm': - self.import_rpm(buildinfo, brinfo, fileinfo) + self.import_rpm(self.buildinfo, brinfo, fileinfo) elif fileinfo['type'] == 'log': - self.import_log(buildinfo, fileinfo) + self.import_log(self.buildinfo, fileinfo) else: - self.import_archive(buildinfo, brinfo, fileinfo) + self.import_archive(self.buildinfo, brinfo, fileinfo) # also import metadata file self.import_metadata() koji.plugin.run_callbacks('postImport', type='cg', metadata=metadata, - directory=directory, importer=self, buildinfo=buildinfo) + directory=directory, importer=self, buildinfo=self.buildinfo) def get_metadata(self, metadata, directory): @@ -4807,6 +4777,57 @@ class CG_Importer(object): return self.metadata + def assert_cg_access(self): + """Check that user has access for all referenced content generators""" + + cgs = set() + for brdata in self.metadata['buildroots']: + cginfo = brdata['content_generator'] + cg = lookup_name('content_generator', cginfo['name'], strict=True) + cgs.add(cg['id']) + brdata['cg_id'] = cg['id'] + for cg_id in cgs: + assert_cg(cg_id) + self.cgs = cgs + + + def prep_build(self): + metadata = self.metadata + buildinfo = get_build(metadata['build'], strict=False) + if buildinfo: + # TODO : allow in some cases + raise koji.GenericError("Build already exists: %r" % buildinfo) + else: + # gather needed data + buildinfo = dslice(metadata['build'], ['name', 'version', 'release', 'extra']) + # epoch is not in the metadata spec, but we allow it to be specified + buildinfo['epoch'] = metadata['build'].get('epoch', None) + buildinfo['start_time'] = \ + datetime.datetime.fromtimestamp(float(metadata['build']['start_time'])).isoformat(' ') + buildinfo['completion_time'] = \ + datetime.datetime.fromtimestamp(float(metadata['build']['end_time'])).isoformat(' ') + self.buildinfo = buildinfo + return buildinfo + + + def get_build(self): + build_id = new_build(self.buildinfo) + buildinfo = get_build(build_id, strict=True) + + # handle special build types + b_extra = self.metadata['build'].get('extra', {}) + if 'maven' in b_extra: + new_maven_build(buildinfo, b_extra['maven']) + if 'win' in b_extra: + new_win_build(buildinfo, b_extra['win']) + if 'image' in b_extra: + # no extra info tracked at build level + new_image_build(buildinfo) + + self.buildinfo = buildinfo + return buildinfo + + def import_metadata(self): """Import the raw metadata""" From 03b43c1c37743679381bf2b2d6733da977becd89 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 12 Nov 2015 17:16:24 -0500 Subject: [PATCH 102/109] don't pass importer class to callback. could confuse plugins --- hub/kojihub.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 9515a1c8..9d0355e5 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4700,7 +4700,7 @@ class CG_Importer(object): self.assert_cg_access() koji.plugin.run_callbacks('preImport', type='cg', metadata=metadata, - directory=directory, importer=self) + directory=directory) # TODO: basic metadata sanity check (use jsonschema?) @@ -4744,7 +4744,7 @@ class CG_Importer(object): self.import_metadata() koji.plugin.run_callbacks('postImport', type='cg', metadata=metadata, - directory=directory, importer=self, buildinfo=self.buildinfo) + directory=directory, buildinfo=self.buildinfo) def get_metadata(self, metadata, directory): From 6be5d30b13ff03c7e3cb9a59ec1052d8d2a6930c Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 12 Nov 2015 18:01:48 -0500 Subject: [PATCH 103/109] more refactor --- hub/kojihub.py | 161 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 107 insertions(+), 54 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 9d0355e5..478fff97 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4688,10 +4688,12 @@ class CG_Importer(object): def __init__(self): self.buildinfo = None + self.metadata_only = False def do_import(self, metadata, directory): metadata = self.get_metadata(metadata, directory) + self.directory = directory metaver = metadata['metadata_version'] if metaver != 0: @@ -4710,35 +4712,12 @@ class CG_Importer(object): self.prep_build() self.get_build() - # buildroots - br_used = set([f['buildroot_id'] for f in metadata['output']]) - brmap = {} - for brdata in metadata['buildroots']: - brfakeid = brdata['id'] - if brfakeid not in br_used: - raise koji.GenericError("Buildroot id not used in output: %r" % brfakeid) - if brfakeid in brmap: - raise koji.GenericError("Duplicate buildroot id in metadata: %r" % brfakeid) - brmap[brfakeid] = self.import_buildroot(brdata) + self.prep_brs() + self.import_brs() # outputs - for fileinfo in metadata['output']: - if fileinfo.get('metadata_only', False): - if not metadata['build'].get('metadata_only'): - raise koji.GenericError('Build must be marked metadata-only to include metadata-only outputs') - workdir = koji.pathinfo.work() - path = os.path.join(workdir, directory, fileinfo.get('relpath', ''), fileinfo['filename']) - fileinfo['hub.path'] = path - brinfo = brmap.get(fileinfo['buildroot_id']) - if not brinfo: - raise koji.GenericError("Missing buildroot metadata for id %(buildroot_id)r" % fileinfo) - - if fileinfo['type'] == 'rpm': - self.import_rpm(self.buildinfo, brinfo, fileinfo) - elif fileinfo['type'] == 'log': - self.import_log(self.buildinfo, fileinfo) - else: - self.import_archive(self.buildinfo, brinfo, fileinfo) + self.prep_outputs() + self.import_outputs() # also import metadata file self.import_metadata() @@ -4843,10 +4822,30 @@ class CG_Importer(object): fo.close() - def import_buildroot(self, brdata): - """Import the given buildroot data""" + def prep_brs(self): + metadata = self.metadata + br_used = set([f['buildroot_id'] for f in metadata['output']]) + br_idx = {} + for brdata in metadata['buildroots']: + brfakeid = brdata['id'] + if brfakeid not in br_used: + raise koji.GenericError("Buildroot id not used in output: %r" % brfakeid) + if brfakeid in br_idx: + raise koji.GenericError("Duplicate buildroot id in metadata: %r" % brfakeid) + br_idx[brfakeid] = self.prep_buildroot(brdata) + self.br_prep = br_idx - # buildroot entry + + def import_brs(self): + brmap = {} + for brfakeid in self.br_prep: + entry = self.br_prep[brfakeid] + brmap[brfakeid] = self.import_buildroot(entry) + self.brmap = brmap + + + def prep_buildroot(self, brdata): + ret = {} brinfo = { 'cg_id' : brdata['cg_id'], 'cg_version' : brdata['content_generator']['version'], @@ -4856,19 +4855,29 @@ class CG_Importer(object): 'host_arch' : brdata['host']['arch'], 'extra' : brdata.get('extra'), } - br = BuildRoot() - br.cg_new(brinfo) + rpmlist, archives = self.match_components(brdata['components']) + ret = { + 'brinfo' : brinfo, + 'rpmlist' : rpmlist, + 'archives' : archives, + 'tools' : brdata['tools'], + } + return ret - # TODO: standard buildroot entry (if applicable) - # ??? + + def import_buildroot(self, entry): + """Import the prepared buildroot data""" + + # buildroot entry + br = BuildRoot() + br.cg_new(entry['brinfo']) #buildroot components - rpmlist, archives = self.match_components(brdata['components']) - br.setList(rpmlist) - br.updateArchiveList(archives) + br.setList(entry['rpmlist']) + br.updateArchiveList(entry['archives']) # buildroot_tools_info - br.setTools(brdata['tools']) + br.setTools(entry['tools']) return br @@ -4937,6 +4946,60 @@ class CG_Importer(object): #raise koji.GenericError("No match: %(filename)s (size %(filesize)s, sum %(checksum)s" % comp) + def prep_outputs(self): + metadata = self.metadata + outputs = [] + for fileinfo in metadata['output']: + fileinfo = fileinfo.copy() # [!] + if fileinfo.get('metadata_only', False): + self.metadata_only = True + workdir = koji.pathinfo.work() + path = os.path.join(workdir, self.directory, fileinfo.get('relpath', ''), fileinfo['filename']) + fileinfo['hub.path'] = path + if fileinfo['buildroot_id'] not in self.br_prep: + raise koji.GenericError("Missing buildroot metadata for id %(buildroot_id)r" % fileinfo) + if fileinfo['type'] not in ['rpm', 'log']: + self.prep_archive(fileinfo) + outputs.append(fileinfo) + self.prepped_outputs = outputs + + + def import_outputs(self): + for fileinfo in self.prepped_outputs: + brinfo = self.brmap.get(fileinfo['buildroot_id']) + if not brinfo: + # should not happen + logger.error("No buildroot mapping for file: %r", fileinfo) + raise koji.GenericError("Unable to map buildroot %(buildroot_id)s" % fileinfo) + if fileinfo['type'] == 'rpm': + self.import_rpm(self.buildinfo, brinfo, fileinfo) + elif fileinfo['type'] == 'log': + self.import_log(self.buildinfo, fileinfo) + else: + self.import_archive(self.buildinfo, brinfo, fileinfo) + + + def prep_archive(self, fileinfo): + # determine archive import type (maven/win/image/other) + extra = fileinfo.get('extra', {}) + legacy_types = ['maven', 'win', 'image'] + l_type = None + type_info = None + for key in legacy_types: + if key in extra: + if l_type is not None: + raise koji.GenericError("Output file has multiple archive types: %s" % fn) + l_type = key + type_info = extra[key] + fileinfo['hub.l_type'] = l_type + fileinfo['hub.type_info'] = type_info + + rpmlist, archives = self.match_components(components) + # TODO - note presence of external components + fileinfo['hub.rpmlist'] = rpmlist + fileinfo['hub.archives'] = archives + + def import_rpm(self, buildinfo, brinfo, fileinfo): if fileinfo.get('metadata_only', False): raise koji.GenericError('Metadata-only imports are not supported for rpms') @@ -4958,28 +5021,18 @@ class CG_Importer(object): def import_archive(self, buildinfo, brinfo, fileinfo): fn = fileinfo['hub.path'] - - # determine archive import type (maven/win/image/other) - extra = fileinfo.get('extra', {}) - legacy_types = ['maven', 'win', 'image'] - l_type = None - type_info = None - for key in legacy_types: - if key in extra: - if l_type is not None: - raise koji.GenericError("Output file has multiple archive types: %s" % fn) - l_type = key - type_info = extra[key] + l_type = fileinfo['hub.l_type'] + type_info = fileinfo['hub.type_info'] archiveinfo = import_archive_internal(fn, buildinfo, l_type, type_info, brinfo.id, fileinfo) if l_type == 'image': - components = fileinfo.get('components', []) - self.import_components(archiveinfo['id'], components) + self.import_components(archiveinfo['id'], fileinfo) - def import_components(self, image_id, components): - rpmlist, archives = self.match_components(components) + def import_components(self, image_id, fileinfo): + rpmlist = fileinfo['hub.rpmlist'] + archives = fileinfo['hub.archives'] insert = InsertProcessor('image_listing') insert.set(image_id=image_id) From fe4982c70cbbfbe23dea1e826a837fbb2fbec2a8 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Fri, 13 Nov 2015 17:14:40 -0500 Subject: [PATCH 104/109] run prep steps early --- hub/kojihub.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 478fff97..630812fb 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4699,27 +4699,24 @@ class CG_Importer(object): if metaver != 0: raise koji.GenericError("Unknown metadata version: %r" % metaver) + # TODO: basic metadata sanity check (use jsonschema?) + self.assert_cg_access() + # prepare data for import + self.prep_build() + self.prep_brs() + self.prep_outputs() + + # TODO: policy hooks + koji.plugin.run_callbacks('preImport', type='cg', metadata=metadata, directory=directory) - # TODO: basic metadata sanity check (use jsonschema?) - - # TODO: policy hooks - - # build entry - self.prep_build() + # finalize import self.get_build() - - self.prep_brs() self.import_brs() - - # outputs - self.prep_outputs() self.import_outputs() - - # also import metadata file self.import_metadata() koji.plugin.run_callbacks('postImport', type='cg', metadata=metadata, From 34a400652e8f73dbdfba47b2a0aa17cd661a37fe Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 16 Nov 2015 14:54:34 -0500 Subject: [PATCH 105/109] cg_import policy stub --- hub/kojihub.py | 13 ++++++++++++- hub/kojixmlrpc.py | 5 ++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 630812fb..57699099 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4708,7 +4708,7 @@ class CG_Importer(object): self.prep_brs() self.prep_outputs() - # TODO: policy hooks + self.assert_policy() koji.plugin.run_callbacks('preImport', type='cg', metadata=metadata, directory=directory) @@ -4767,6 +4767,17 @@ class CG_Importer(object): self.cgs = cgs + def assert_policy(self): + policy_data = { + 'package': self.buildinfo['name'], + 'source': self.buildinfo.get('source'), + 'metadata_only': self.metadata_only, + 'cg_list' : list(self.cgs), + # TODO: provide more data + } + assert_policy('cg_import', policy_data) + + def prep_build(self): metadata = self.metadata buildinfo = get_build(metadata['build'], strict=False) diff --git a/hub/kojixmlrpc.py b/hub/kojixmlrpc.py index efb99a60..a3059f3c 100644 --- a/hub/kojixmlrpc.py +++ b/hub/kojixmlrpc.py @@ -546,7 +546,10 @@ _default_policies = { 'vm' : ''' has_perm admin win-admin :: allow all :: deny - ''' + ''', + 'cg_import' :''' + all :: allow + ''', } def get_policy(opts, plugins): From 353927e00b55713afb77cac11bfbe0b3a27b557a Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 16 Nov 2015 15:04:08 -0500 Subject: [PATCH 106/109] fix refactor issue --- hub/kojihub.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 57699099..01df71b5 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -5002,10 +5002,12 @@ class CG_Importer(object): fileinfo['hub.l_type'] = l_type fileinfo['hub.type_info'] = type_info - rpmlist, archives = self.match_components(components) - # TODO - note presence of external components - fileinfo['hub.rpmlist'] = rpmlist - fileinfo['hub.archives'] = archives + if l_type == 'image': + components = fileinfo.get('components', []) + rpmlist, archives = self.match_components(components) + # TODO - note presence of external components + fileinfo['hub.rpmlist'] = rpmlist + fileinfo['hub.archives'] = archives def import_rpm(self, buildinfo, brinfo, fileinfo): From f40b39cd68b280dd3ff7945a53beecab2691a6c8 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 16 Nov 2015 15:16:36 -0500 Subject: [PATCH 107/109] ensure builddir when writing cg metadata file --- hub/kojihub.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 01df71b5..886cb15a 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4822,7 +4822,9 @@ class CG_Importer(object): # because there are complications # - no buildroot (could confuse policies checking that builds were built sanely # - doesn't fit with current major archive categories - path = os.path.join(koji.pathinfo.build(self.buildinfo), 'metadata.json') + builddir = koji.pathinfo.build(self.buildinfo) + koji.ensuredir(builddir) + path = os.path.join(builddir, 'metadata.json') fo = open(path, 'w') try: fo.write(self.raw_metadata) From 6c6bef96303e5659ceecf098017883fdff7efafe Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 16 Nov 2015 15:29:30 -0500 Subject: [PATCH 108/109] record and report source and metadata_only fields --- docs/schema-update-cgen.sql | 3 +++ docs/schema.sql | 3 +++ hub/kojihub.py | 18 ++++++++++++++++-- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/schema-update-cgen.sql b/docs/schema-update-cgen.sql index 81a6f38b..0f9afde4 100644 --- a/docs/schema-update-cgen.sql +++ b/docs/schema-update-cgen.sql @@ -46,8 +46,11 @@ CREATE INDEX image_listing_archives on image_archive_listing(archive_id); select statement_timestamp(), 'Adding new columns' as msg; ALTER TABLE build ADD COLUMN start_time TIMESTAMP; +ALTER TABLE build ADD COLUMN source TEXT; ALTER TABLE build ADD COLUMN extra TEXT; +ALTER TABLE rpminfo ADD COLUMN metadata_only BOOLEAN NOT NULL DEFAULT FALSE; ALTER TABLE rpminfo ADD COLUMN extra TEXT; +ALTER TABLE archiveinfo ADD COLUMN metadata_only BOOLEAN NOT NULL DEFAULT FALSE; ALTER TABLE archiveinfo ADD COLUMN extra TEXT; diff --git a/docs/schema.sql b/docs/schema.sql index b2a64ec4..f78bd5dd 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -289,6 +289,7 @@ CREATE TABLE build ( version TEXT NOT NULL, release TEXT NOT NULL, epoch INTEGER, + source TEXT, create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), start_time TIMESTAMP, completion_time TIMESTAMP, @@ -692,6 +693,7 @@ CREATE TABLE rpminfo ( payloadhash TEXT NOT NULL, size BIGINT NOT NULL, buildtime BIGINT NOT NULL, + metadata_only BOOLEAN NOT NULL DEFAULT FALSE, extra TEXT, CONSTRAINT rpminfo_unique_nvra UNIQUE (name,version,release,arch,external_repo_id) ) WITHOUT OIDS; @@ -826,6 +828,7 @@ CREATE TABLE archiveinfo ( size BIGINT NOT NULL, checksum TEXT NOT NULL, checksum_type INTEGER NOT NULL, + metadata_only BOOLEAN NOT NULL DEFAULT FALSE, extra TEXT ) WITHOUT OIDS; CREATE INDEX archiveinfo_build_idx ON archiveinfo (build_id); diff --git a/hub/kojihub.py b/hub/kojihub.py index 886cb15a..e0a8d20e 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -1257,6 +1257,7 @@ def readTaggedRPMS(tag, package=None, arch=None, event=None,inherit=False,latest ('rpminfo.buildtime', 'buildtime'), ('rpminfo.buildroot_id', 'buildroot_id'), ('rpminfo.build_id', 'build_id'), + ('rpminfo.metadata_only', 'metadata_only'), ('rpminfo.extra', 'extra'), ] tables = ['rpminfo'] @@ -1347,6 +1348,7 @@ def readTaggedArchives(tag, package=None, event=None, inherit=False, latest=True ('archiveinfo.size', 'size'), ('archiveinfo.checksum', 'checksum'), ('archiveinfo.checksum_type', 'checksum_type'), + ('archiveinfo.metadata_only', 'metadata_only'), ('archiveinfo.extra', 'extra'), ] tables = ['archiveinfo'] @@ -2112,6 +2114,7 @@ def maven_tag_archives(tag_id, event_id=None, inherit=True): ('archiveinfo.filename', 'filename'), ('archiveinfo.size', 'size'), ('archiveinfo.checksum', 'checksum'), ('archiveinfo.checksum_type', 'checksum_type'), + ('archiveinfo.metadata_only', 'metadata_only'), ('archiveinfo.extra', 'extra'), ('maven_archives.group_id', 'group_id'), ('maven_archives.artifact_id', 'artifact_id'), @@ -3300,6 +3303,7 @@ def get_build(buildInfo, strict=False): start_ts: time the build was started (epoch, may be null) completion_time: time the build was completed (may be null) completion_ts: time the build was completed (epoch, may be null) + source: the SCM URL of the sources used in the build extra: dictionary with extra data about the build If there is no build matching the buildInfo given, and strict is specified, @@ -3320,6 +3324,7 @@ def get_build(buildInfo, strict=False): ('EXTRACT(EPOCH FROM build.start_time)', 'start_ts'), ('EXTRACT(EPOCH FROM build.completion_time)','completion_ts'), ('users.id', 'owner_id'), ('users.name', 'owner_name'), + ('build.source', 'source'), ('build.extra', 'extra')) fields, aliases = zip(*fields) joins = ['events ON build.create_event = events.id', @@ -3404,6 +3409,7 @@ def get_rpm(rpminfo, strict=False, multi=False): - buildroot_id - external_repo_id - external_repo_name + - metadata_only - extra If there is no RPM with the given ID, None is returned, unless strict @@ -3427,6 +3433,7 @@ def get_rpm(rpminfo, strict=False, multi=False): ('payloadhash', 'payloadhash'), ('size', 'size'), ('buildtime', 'buildtime'), + ('metadata_only', 'metadata_only'), ('extra', 'extra'), ) # we can look up by id or NVRA @@ -3498,6 +3505,8 @@ def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID - buildroot_id - external_repo_id - external_repo_name + - metadata_only + - extra If componentBuildrootID is specified, two additional keys will be included: - component_buildroot_id @@ -3514,6 +3523,7 @@ def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID ('rpminfo.build_id', 'build_id'), ('rpminfo.buildroot_id', 'buildroot_id'), ('rpminfo.external_repo_id', 'external_repo_id'), ('external_repo.name', 'external_repo_name'), + ('rpminfo.metadata_only', 'metadata_only'), ('rpminfo.extra', 'extra'), ] joins = ['external_repo ON rpminfo.external_repo_id = external_repo.id'] @@ -3694,6 +3704,7 @@ def list_archives(buildID=None, buildrootID=None, imageID=None, componentBuildro ('archiveinfo.size', 'size'), ('archiveinfo.checksum', 'checksum'), ('archiveinfo.checksum_type', 'checksum_type'), + ('archiveinfo.metadata_only', 'metadata_only'), ('archiveinfo.extra', 'extra'), ('archivetypes.name', 'type_name'), ('archivetypes.description', 'type_description'), @@ -3809,7 +3820,8 @@ def get_archive(archive_id, strict=False): rootid arch """ - fields = ('id', 'type_id', 'build_id', 'buildroot_id', 'filename', 'size', 'checksum', 'checksum_type', 'extra') + fields = ('id', 'type_id', 'build_id', 'buildroot_id', 'filename', 'size', + 'checksum', 'checksum_type', 'metadata_only', 'extra') archive = QueryProcessor(tables=['archiveinfo'], columns=fields, transform=_fix_archive_row, clauses=['id=%(archive_id)s'], values=locals()).executeOne() if not archive: @@ -4435,6 +4447,7 @@ def new_build(data): data.setdefault('state',koji.BUILD_STATES['COMPLETE']) data.setdefault('start_time', 'NOW') data.setdefault('completion_time', 'NOW') + data.setdefault('source', None) data.setdefault('owner',context.session.user_id) data.setdefault('task_id',None) data.setdefault('volume_id', 0) @@ -4473,7 +4486,7 @@ def new_build(data): #insert the new data insert_data = dslice(data, ['pkg_id', 'version', 'release', 'epoch', 'state', 'volume_id', - 'task_id', 'owner', 'start_time', 'completion_time', 'extra']) + 'task_id', 'owner', 'start_time', 'completion_time', 'source', 'extra']) data['id'] = insert_data['id'] = _singleValue("SELECT nextval('build_id_seq')") insert = InsertProcessor('build', data=insert_data) insert.execute() @@ -5641,6 +5654,7 @@ def import_archive_internal(filepath, buildinfo, type, typeInfo, buildroot_id=No # until we change the way we handle checksums, we have to limit this to md5 raise koji.GenericError("Unsupported checksum type: %(checksum_type)s" % fileinfo) archiveinfo['checksum_type'] = koji.CHECKSUM_TYPES[fileinfo['checksum_type']] + archiveinfo['metadata_only'] = True else: filename = koji.fixEncoding(os.path.basename(filepath)) archiveinfo['filename'] = filename From 3e663b855619c9d854e2062ad03a0bfd7431ce22 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 16 Nov 2015 16:36:56 -0500 Subject: [PATCH 109/109] Better web display for metadata_only imports --- www/kojiweb/archiveinfo.chtml | 5 +++++ www/kojiweb/buildinfo.chtml | 14 ++++++++------ www/kojiweb/index.py | 14 ++++++++++++-- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/www/kojiweb/archiveinfo.chtml b/www/kojiweb/archiveinfo.chtml index fe656ef3..ee2d3f42 100644 --- a/www/kojiweb/archiveinfo.chtml +++ b/www/kojiweb/archiveinfo.chtml @@ -19,6 +19,11 @@ #end if + #if $archive.metadata_only + + + + #end if diff --git a/www/kojiweb/buildinfo.chtml b/www/kojiweb/buildinfo.chtml index 0a08d9af..73b85dd8 100644 --- a/www/kojiweb/buildinfo.chtml +++ b/www/kojiweb/buildinfo.chtml @@ -111,7 +111,11 @@ #set $rpmpath = $pathinfo.rpm($rpm) + #if $rpm.metadata_only + + #else + #end if #end for #end if @@ -175,12 +179,10 @@ diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 512f9652..a5505d1a 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -1098,7 +1098,18 @@ def buildinfo(environ, buildID): archivetype = None archives = server.listArchives(build['id'], type=archivetype, queryOpts={'order': 'filename'}) archivesByExt = {} + topurl = environ['koji.options']['KojiFilesURL'] + pathinfo = koji.PathInfo(topdir=topurl) for archive in archives: + if mavenbuild: + archive['display'] = archive['filename'] + archive['dl_url'] = '/'.join([pathinfo.mavenbuild(build), pathinfo.mavenfile(archive)]) + elif winbuild: + archive['display'] = pathinfo.winfile(archive) + archive['dl_url'] = '/'.join([pathinfo.winbuild(build), pathinfo.winfile(archive)]) + elif imagebuild: + archive['display'] = archive['filename'] + archive['dl_url'] = '/'.join([pathinfo.imagebuild(build), archive['filename']]) archivesByExt.setdefault(os.path.splitext(archive['filename'])[1][1:], []).append(archive) rpmsByArch = {} @@ -1197,8 +1208,7 @@ def buildinfo(environ, buildID): else: values['estCompletion'] = None - topurl = environ['koji.options']['KojiFilesURL'] - values['pathinfo'] = koji.PathInfo(topdir=topurl) + values['pathinfo'] = pathinfo return _genHTML(environ, 'buildinfo.chtml') def builds(environ, userID=None, tagID=None, packageID=None, state=None, order='-build_id', start=None, prefix=None, inherited='1', latest='1', type=None):
Task$koji.taskLabel($task)
Extra$util.escapeHTML($pformat($build.extra))
Tags From a6b1981074468004b1cad4ebd84b3924a3157f61 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Sun, 1 Nov 2015 17:38:43 -0500 Subject: [PATCH 085/109] show extra data in web ui --- www/kojiweb/archiveinfo.chtml | 6 ++++++ www/kojiweb/buildinfo.chtml | 2 +- www/kojiweb/buildrootinfo.chtml | 6 ++++++ www/kojiweb/buildrootinfo_cg.chtml | 6 ++++++ www/kojiweb/rpminfo.chtml | 6 ++++++ 5 files changed, 25 insertions(+), 1 deletion(-) diff --git a/www/kojiweb/archiveinfo.chtml b/www/kojiweb/archiveinfo.chtml index a1c49fd3..fe656ef3 100644 --- a/www/kojiweb/archiveinfo.chtml +++ b/www/kojiweb/archiveinfo.chtml @@ -1,5 +1,6 @@ #import koji #from kojiweb import util +#from pprint import pformat #import urllib #attr _PASSTHROUGH = ['archiveID', 'fileOrder', 'fileStart', 'buildrootOrder', 'buildrootStart'] @@ -54,6 +55,11 @@ Buildroot$util.brLabel($builtInRoot)
Extra$util.escapeHTML($pformat($archive.extra))
FilesTask$koji.taskLabel($task)
Extra$util.escapeHTML($pformat($build.extra))
Repo Created$util.formatTimeLong($buildroot.repo_create_event_time)
Extra$util.escapeHTML($pformat($buildroot.extra))
Component RPMs
Container Arch$buildroot.container_arch
Extra$util.escapeHTML($pformat($buildroot.extra))
Component RPMs
Buildroot$util.brLabel($builtInRoot)
Extra$util.escapeHTML($pformat($rpm.extra))
ProvidesFile Name$archive.filename
Metadata onlyTrue (file not imported)
File Type$archive_type.description
$rpmfile (info) (metadata only)$rpmfile (info) (download)
- #if $mavenbuild - $archive.filename (info) (download) - #elif $winbuild - $pathinfo.winfile($archive) (info) (download) - #elif $imagebuild - $archive.filename (info) (download) + #if $archive.metadata_only + $archive.display (info) + #else + $archive.display (info) (download) #end if