particle-os/debian-koji
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

PR#1464: API for reserving NVRs for content generators

Browse source 
Merges #1464
https://pagure.io/koji/pull-request/1464

Fixes: #1463
https://pagure.io/koji/issue/1463
[RFE] Predeclare nvr for content generators
This commit is contained in:
Mike McLean 2019-07-16 10:45:40 -04:00
commit bf394684ef
9 changed files with 242 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

5
cli/koji_cli/commands.py
View file

@ -1292,6 +1292,7 @@ def handle_import_cg(goptions, session, args):
help=_("Do not display progress of the upload"))
parser.add_option("--link", action="store_true", help=_("Attempt to hardlink instead of uploading"))
parser.add_option("--test", action="store_true", help=_("Don't actually import"))
parser.add_option("--token", action="store", default=None, help=_("Build reservation token"))
(options, args) = parser.parse_args(args)
if len(args) < 2:
parser.error(_("Please specify metadata files directory"))
@ -1336,7 +1337,7 @@ def handle_import_cg(goptions, session, args):
if callback:
print('')
session.CGImport(metadata, serverdir)
session.CGImport(metadata, serverdir, options.token)
def handle_import_comps(goptions, session, args):
@ -3154,6 +3155,8 @@ def anon_handle_buildinfo(goptions, session, args):
info['state'] = koji.BUILD_STATES[info['state']]
print("BUILD: %(name)s-%(version)s-%(release)s [%(id)d]" % info)
print("State: %(state)s" % info)
if info['state'] == 'BUILDING':
print("Reserved by: %(reserved_by_name)s" % info)
print("Built by: %(owner_name)s" % info)
source = info.get('source')
if source is not None:

11
docs/schema-upgrade-1.17-1.18.sql
View file

@ -23,4 +23,15 @@ insert into archivetypes (name, description, extensions) values ('qcow2-compress
-- add better index for sessions
CREATE INDEX sessions_expired ON sessions(expired);
-- table for content generator build reservations
CREATE TABLE build_reservations (
build_id INTEGER NOT NULL REFERENCES build(id),
token VARCHAR(64),
created TIMESTAMP NOT NULL,
PRIMARY KEY (build_id)
) WITHOUT OIDS;
CREATE INDEX build_reservations_created ON build_reservations(created);
ALTER TABLE build ADD COLUMN cg_id INTEGER REFERENCES content_generator(id);
COMMIT;

22
docs/schema.sql
View file

@ -249,6 +249,12 @@ CREATE TABLE volume (
INSERT INTO volume (id, name) VALUES (0, 'DEFAULT');
-- data for content generators
CREATE TABLE content_generator (
id SERIAL PRIMARY KEY,
name TEXT
) WITHOUT OIDS;
-- here we track the built packages
-- this is at the srpm level, since builds are by srpm
-- see rpminfo for isolated packages
@ -269,6 +275,7 @@ CREATE TABLE build (
state INTEGER NOT NULL,
task_id INTEGER REFERENCES task (id),
owner INTEGER NOT NULL REFERENCES users (id),
cg_id INTEGER REFERENCES content_generator(id),
extra TEXT,
CONSTRAINT build_pkg_ver_rel UNIQUE (pkg_id, version, release),
CONSTRAINT completion_sane CHECK ((state = 0 AND completion_time IS NULL) OR
@ -483,14 +490,6 @@ create table tag_external_repos (
UNIQUE (tag_id, external_repo_id, active)
);
-- data for content generators
CREATE TABLE content_generator (
id SERIAL PRIMARY KEY,
name TEXT
) WITHOUT OIDS;
CREATE TABLE cg_users (
cg_id INTEGER NOT NULL REFERENCES content_generator (id),
user_id INTEGER NOT NULL REFERENCES users (id),
@ -507,6 +506,13 @@ CREATE TABLE cg_users (
UNIQUE (cg_id, user_id, active)
) WITHOUT OIDS;
CREATE TABLE build_reservations (
build_id INTEGER NOT NULL REFERENCES build(id),
token VARCHAR(64),
created TIMESTAMP NOT NULL,
PRIMARY KEY (build_id)
) WITHOUT OIDS;
CREATE INDEX build_reservations_created ON build_reservations(created);
-- here we track the buildroots on the machines
CREATE TABLE buildroot (

1
docs/source/content_generator_metadata.rst
View file

@ -44,6 +44,7 @@ The build map contains the following entries:
epoch.
- owner: The owner of the build task in username format. This field
is optional.
- build_id: Reserved build ID. This field is optional.
- extra: A map of extra metadata associated with the build, which
must include one of:

27
docs/source/content_generators.rst
View file

@ -114,3 +114,30 @@ Metadata
Metadata will be provided by the Content Generator as a JSON file. There
is a proposal of the :doc:`Content Generator
Metadata <content_generator_metadata>` format available for review.
API
===
Relevant API calls for Content Generator are:
- ``CGImport(metadata, directory, token=None)``: This is basic integration point
of Content Generator with koji. It is supplied with metadata as json encoded
string or dict or filename of metadata described in previous chapter and
directory with all uploaded content referenced from metadata. These files
needs to be uploaded before ``CGImport`` is called.
Optionally, ``token`` can be specified in case, that build ID reservation was
done before.
- ``CGInitBuild(cg, data)``: It can be helpful in many cases to reserve NVR for
future build before Content Generator evend starts building. Especially, if
there is some CI or other workflow competing for same NVRs. This call creates
special ``token`` which can be used to claim specific build (ID + NVR). Such
claimed build will be displayed as BUILDING and can be used by ``CGImport``
call later.
As an input are here Content Generator name and `data` which is basically
dictionary with name/version/release/epoch keys. Call will return a dict
containing ``token`` and ``build_id``. ``token`` would be used in subsequent
call of ``CGImport`` while ``build_id`` needs to be part of metadata (as item
in ``build`` key).

200
hub/kojihub.py
View file

@ -44,6 +44,11 @@ import time
import traceback
import six.moves.xmlrpc_client
import zipfile
try:
# py 3.6+
import secrets
except ImportError:
import random
import rpm
import six
@ -3703,6 +3708,8 @@ def get_build(buildInfo, strict=False):
completion_ts: time the build was completed (epoch, may be null)
source: the SCM URL of the sources used in the build
extra: dictionary with extra data about the build
cg_id: ID of CG which reserved or imported this build
cg_name: name of CG which reserved or imported this build
If there is no build matching the buildInfo given, and strict is specified,
raise an error. Otherwise return None.
@ -3726,6 +3733,7 @@ def get_build(buildInfo, strict=False):
('EXTRACT(EPOCH FROM build.start_time)', 'start_ts'),
('EXTRACT(EPOCH FROM build.completion_time)', 'completion_ts'),
('users.id', 'owner_id'), ('users.name', 'owner_name'),
('build.cg_id', 'cg_id'),
('build.source', 'source'),
('build.extra', 'extra'))
fields, aliases = zip(*fields)
@ -3745,8 +3753,11 @@ def get_build(buildInfo, strict=False):
raise koji.GenericError('No matching build found: %s' % buildInfo)
else:
return None
if result['cg_id']:
result['cg_name'] = lookup_name('content_generator', result['cg_id'], strict=True)['name']
else:
return result
result['cg_name'] = None
return result
def get_build_logs(build):
@ -5182,8 +5193,11 @@ def apply_volume_policy(build, strict=False):
_set_build_volume(build, volume, strict=True)
def new_build(data):
"""insert a new build entry"""
def new_build(data, strict=False):
"""insert a new build entry
If strict is specified, raise an exception, if build already exists.
"""
data = data.copy()
@ -5222,15 +5236,18 @@ def new_build(data):
#check for existing build
old_binfo = get_build(data)
if old_binfo:
if strict:
raise koji.GenericError('Existing build found: %s' % data)
recycle_build(old_binfo, data)
# Raises exception if there is a problem
return old_binfo['id']
#else
koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=None, new=data['state'], info=data)
#insert the new data
insert_data = dslice(data, ['pkg_id', 'version', 'release', 'epoch', 'state', 'volume_id',
'task_id', 'owner', 'start_time', 'completion_time', 'source', 'extra'])
if 'cg_id' in data:
insert_data['cg_id'] = data['cg_id']
data['id'] = insert_data['id'] = _singleValue("SELECT nextval('build_id_seq')")
insert = InsertProcessor('build', data=insert_data)
insert.execute()
@ -5295,7 +5312,7 @@ def recycle_build(old, data):
update = UpdateProcessor('build', clauses=['id=%(id)s'], values=data)
update.set(**dslice(data,
['state', 'task_id', 'owner', 'start_time', 'completion_time',
'epoch', 'source', 'extra', 'volume_id']))
'epoch', 'source', 'extra', 'volume_id', 'cg_id']))
update.rawset(create_event='get_event()')
update.execute()
builddir = koji.pathinfo.build(data)
@ -5534,7 +5551,58 @@ def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None):
return rpminfo
def cg_import(metadata, directory):
def generate_token(nbytes=32):
"""
Generate random hex-string token of length 2 * nbytes
"""
if secrets:
return secrets.token_hex(nbytes=nbytes)
else:
values = ['%02x' % random.randint(0, 256) for x in range(nbytes)]
return ''.join(values)
def get_reservation_token(build_id):
query = QueryProcessor(
tables=['build_reservations'],
columns=['build_id', 'token'],
clauses=['build_id = %(build_id)d'],
values=locals(),
)
return query.executeOne()
def clear_reservation(build_id):
'''Remove reservation entry for build'''
delete = "DELETE FROM build_reservations WHERE build_id = %(build_id)i"
_dml(delete, {'build_id': build_id})
def cg_init_build(cg, data):
"""Create (reserve) a build_id for given data.
If build already exists, init_build will raise GenericError
"""
assert_cg(cg)
cg_id = lookup_name('content_generator', cg, strict=True)['id']
data['owner'] = context.session.user_id
data['state'] = koji.BUILD_STATES['BUILDING']
data['completion_time'] = None
data['cg_id'] = cg_id
# CGs shouldn't have to worry about epoch
data.setdefault('epoch', None)
build_id = new_build(data, strict=True)
# store token
token = generate_token()
insert = InsertProcessor(table='build_reservations')
insert.set(build_id=build_id, token=token)
insert.rawset(created='NOW()')
insert.execute()
return {'build_id': build_id, 'token': token}
def cg_import(metadata, directory, token=None):
"""Import build from a content generator
metadata can be one of the following
@ -5544,7 +5612,7 @@ def cg_import(metadata, directory):
"""
importer = CG_Importer()
return importer.do_import(metadata, directory)
return importer.do_import(metadata, directory, token)
class CG_Importer(object):
@ -5553,8 +5621,7 @@ class CG_Importer(object):
self.buildinfo = None
self.metadata_only = False
def do_import(self, metadata, directory):
def do_import(self, metadata, directory, token=None):
metadata = self.get_metadata(metadata, directory)
self.directory = directory
@ -5567,7 +5634,7 @@ class CG_Importer(object):
self.assert_cg_access()
# prepare data for import
self.prep_build()
self.prep_build(token)
self.prep_brs()
self.prep_outputs()
@ -5579,7 +5646,7 @@ class CG_Importer(object):
directory=directory)
# finalize import
self.get_build()
self.get_build(token)
self.import_brs()
try:
self.import_outputs()
@ -5677,26 +5744,54 @@ class CG_Importer(object):
raise koji.GenericError("Destination directory already exists: %s" % path)
def prep_build(self):
def prep_build(self, token=None):
metadata = self.metadata
buildinfo = get_build(metadata['build'], strict=False)
if buildinfo:
# TODO : allow in some cases
raise koji.GenericError("Build already exists: %r" % buildinfo)
if metadata['build'].get('build_id'):
if len(self.cgs) != 1:
raise koji.GenericError("Reserved builds can handle only single content generator.")
cg_id = list(self.cgs)[0]
build_id = metadata['build']['build_id']
buildinfo = get_build(build_id, strict=True)
build_token = get_reservation_token(build_id)
if not build_token or build_token['token'] != token:
raise koji.GenericError("Token doesn't match build ID %s" % build_id)
if buildinfo['cg_id'] != cg_id:
raise koji.GenericError('Build ID %s is not reserved by this CG' % build_id)
if buildinfo['state'] != koji.BUILD_STATES['BUILDING']:
raise koji.GenericError('Build ID %s is not in BUILDING state' % build_id)
if buildinfo['name'] != metadata['build']['name'] or \
buildinfo['version'] != metadata['build']['version'] or \
buildinfo['release'] != metadata['build']['release']:
raise koji.GenericError("Build (%i) NVR is different" % build_id)
if ('epoch' in metadata['build'] and
buildinfo['epoch'] != metadata['build']['epoch']):
raise koji.GenericError("Build (%i) epoch is different"
% build_id)
elif token is not None:
raise koji.GenericError('Reservation token given, but no build_id '
'in metadata')
else:
# gather needed data
buildinfo = dslice(metadata['build'], ['name', 'version', 'release', 'extra', 'source'])
# epoch is not in the metadata spec, but we allow it to be specified
buildinfo['epoch'] = metadata['build'].get('epoch', None)
buildinfo['start_time'] = \
datetime.datetime.fromtimestamp(float(metadata['build']['start_time'])).isoformat(' ')
buildinfo['completion_time'] = \
datetime.datetime.fromtimestamp(float(metadata['build']['end_time'])).isoformat(' ')
owner = metadata['build'].get('owner', None)
if owner:
if not isinstance(owner, six.string_types):
raise koji.GenericError("Invalid owner format (expected username): %s" % owner)
buildinfo['owner'] = get_user(owner, strict=True)['id']
buildinfo = get_build(metadata['build'], strict=False)
if buildinfo and not metadata['build'].get('build_id'):
# TODO : allow in some cases
raise koji.GenericError("Build already exists: %r" % buildinfo)
# gather needed data
buildinfo = dslice(metadata['build'], ['name', 'version', 'release', 'extra', 'source'])
if 'build_id' in metadata['build']:
buildinfo['build_id'] = metadata['build']['build_id']
# epoch is not in the metadata spec, but we allow it to be specified
buildinfo['epoch'] = metadata['build'].get('epoch', None)
buildinfo['start_time'] = \
datetime.datetime.fromtimestamp(float(metadata['build']['start_time'])).isoformat(' ')
buildinfo['completion_time'] = \
datetime.datetime.fromtimestamp(float(metadata['build']['end_time'])).isoformat(' ')
owner = metadata['build'].get('owner', None)
if owner:
if not isinstance(owner, six.string_types):
raise koji.GenericError("Invalid owner format (expected username): %s" % owner)
buildinfo['owner'] = get_user(owner, strict=True)['id']
self.buildinfo = buildinfo
koji.check_NVR(buildinfo, strict=True)
@ -5722,10 +5817,25 @@ class CG_Importer(object):
return buildinfo
def get_build(self):
build_id = new_build(self.buildinfo)
buildinfo = get_build(build_id, strict=True)
def get_build(self, token=None):
try:
binfo = dslice(self.buildinfo, ('name', 'version', 'release'))
buildinfo = get_build(binfo, strict=True)
build_token = get_reservation_token(buildinfo['build_id'])
if len(self.cgs) != 1:
raise koji.GenericError("Reserved builds can handle only single content generator.")
cg_id = list(self.cgs)[0]
if buildinfo.get('task_id') or \
buildinfo['state'] != koji.BUILD_STATES['BUILDING'] or \
not build_token or \
buildinfo['cg_id'] != cg_id or \
build_token['token'] != token:
raise koji.GenericError("Build is not reserved")
buildinfo['extra'] = self.buildinfo['extra']
build_id = buildinfo['build_id']
except Exception:
build_id = new_build(self.buildinfo)
buildinfo = get_build(build_id, strict=True)
# handle special build types
for btype in self.typeinfo:
tinfo = self.typeinfo[btype]
@ -5745,6 +5855,24 @@ class CG_Importer(object):
if [o for o in self.prepped_outputs if o['type'] == 'rpm']:
new_typed_build(buildinfo, 'rpm')
# update build state
if buildinfo.get('extra'):
extra = json.dumps(buildinfo['extra'])
else:
extra = None
owner = get_user(self.buildinfo['owner'], strict=True)['id']
source = self.buildinfo.get('source')
st_complete = koji.BUILD_STATES['COMPLETE']
st_old = buildinfo['state']
koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=st_old, new=st_complete, info=buildinfo)
update = UpdateProcessor('build', clauses=['id=%(id)s'], values=buildinfo)
update.set(state=st_complete, extra=extra, owner=owner, source=source)
update.rawset(completion_time='NOW()')
update.execute()
buildinfo = get_build(build_id, strict=True)
clear_reservation(build_id)
koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=st_old, new=st_complete, info=buildinfo)
self.buildinfo = buildinfo
return buildinfo
@ -7504,6 +7632,11 @@ def cancel_build(build_id, cancel_task=True):
build_notification(task_id, build_id)
if cancel_task:
Task(task_id).cancelFull(strict=False)
# remove possible CG reservations
delete = "DELETE FROM build_reservations WHERE build_id = %(build_id)i"
_dml(delete, {'build_id': build_id})
build = get_build(build_id, strict=True)
koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=st_old, new=st_canceled, info=build)
return True
@ -9519,6 +9652,7 @@ class RootExports(object):
fullpath = '%s/%s' % (koji.pathinfo.work(), filepath)
import_archive(fullpath, buildinfo, type, typeInfo)
CGInitBuild = staticmethod(cg_init_build)
CGImport = staticmethod(cg_import)
untaggedBuilds = staticmethod(untagged_builds)

15
tests/test_cli/test_import_cg.py
View file

@ -95,7 +95,7 @@ class TestImportCG(utils.CliTestCase):
self.assert_console_message(stdout, expected)
linked_upload_mock.assert_not_called()
session.uploadWrapper.assert_has_calls(calls)
session.CGImport.assert_called_with(metadata, fake_srv_path)
session.CGImport.assert_called_with(metadata, fake_srv_path, None)
# Case 2, running in fg, progress off
with mock.patch(utils.get_builtin_open()):
@ -105,7 +105,7 @@ class TestImportCG(utils.CliTestCase):
self.assert_console_message(stdout, expected)
linked_upload_mock.assert_not_called()
session.uploadWrapper.assert_has_calls(calls)
session.CGImport.assert_called_with(metadata, fake_srv_path)
session.CGImport.assert_called_with(metadata, fake_srv_path, None)
# reset mocks
linked_upload_mock.reset_mock()
@ -129,7 +129,7 @@ class TestImportCG(utils.CliTestCase):
linked_upload_mock.assert_has_calls(calls)
session.uploadWrapper.assert_not_called()
session.CGImport.assert_called_with(metadata, fake_srv_path)
session.CGImport.assert_called_with(metadata, fake_srv_path, None)
# make sure there is no message on output
self.assert_console_message(stdout, '')
@ -213,10 +213,11 @@ class TestImportCG(utils.CliTestCase):
(Specify the --help global option for a list of other help options)
Options:
-h, --help show this help message and exit
--noprogress Do not display progress of the upload
--link Attempt to hardlink instead of uploading
--test Don't actually import
-h, --help show this help message and exit
--noprogress Do not display progress of the upload
--link Attempt to hardlink instead of uploading
--test Don't actually import
--token=TOKEN Build reservation token
""" % self.progname)

5
tests/test_hub/test_recycle_build.py
View file

@ -43,6 +43,7 @@ class TestRecycleBuild():
'version': '3.2.6',
'source': None,
'extra': None,
'cg_id': None,
'volume_id': 0,
'volume_name': 'DEFAULT'}
new = {'state': 0,
@ -56,6 +57,7 @@ class TestRecycleBuild():
'owner': 2,
'source': None,
'extra': None,
'cg_id': None,
'volume_id': 0}
def test_recycle_building(self):
@ -90,7 +92,10 @@ class TestRecycleBuild():
old['task_id'] = 99
new['task_id'] = 137
query = self.QueryProcessor.return_value
# for all checks
query.execute.return_value = []
# for getBuild
query.executeOne.return_value = old
self.run_pass(old, new)
def run_pass(self, old, new):

5
www/kojiweb/buildinfo.chtml
View file

@ -82,6 +82,11 @@
<th>Completed</th><td>$util.formatTimeLong($build.completion_time)</td>
</tr>
#end if
#if $build.cg_id
<tr>
<th>Content generator</th><td>$build.cg_name</td>
</tr>
#end if
#if $task
<tr>
<th>Task</th><td><a href="taskinfo?taskID=$task.id" class="task$util.taskState($task.state)">$koji.taskLabel($task)</a></td>