particle-os/debian-koji
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

use nl-delimited json to dump rpm data

Browse source
This commit is contained in:
Mike McLean 2023-12-05 15:48:30 -05:00 • committed by Yu Ming Zhu
parent 71898e43f4
commit fc08fc0f74
1 changed files with 15 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

49
kojihub/kojihub.py
View file

@ -2717,20 +2717,6 @@ def repo_init(tag, task_id=None, with_src=False, with_debuginfo=False, event=Non
# Note: the repo_include_all option is not recommended for common use # Note: the repo_include_all option is not recommended for common use
# see https://pagure.io/koji/issue/588 for background # see https://pagure.io/koji/issue/588 for background
rpms, builds = readTaggedRPMS(tag_id, event=event_id, inherit=True, latest=latest) rpms, builds = readTaggedRPMS(tag_id, event=event_id, inherit=True, latest=latest)
# we will iterate rpms twice, so covert it from generator to list
rpms = list(rpms)
# Since draft builds are introduced, we cannot guarantee the uniqueness of NVRA
# don't allow duplicate nvras in tag for repo generation.
rpm_mappings = {}
for r in rpms:
nvra = "{name}-{version}-{release}.{arch}".format(**r)
dup_rpms = rpm_mappings.setdefault(nvra, [])
dup_rpms.append(r['id'])
dups = {nvra: d for nvra, d in rpm_mappings.items() if len(d) > 1}
if dups:
raise koji.GenericError(
f"duplicate nvras found while repo#{repo_id} initiation: {dups}"
)
groups = readTagGroups(tag_id, event=event_id, inherit=True) groups = readTagGroups(tag_id, event=event_id, inherit=True)
blocks = [pkg for pkg in readPackageList(tag_id, event=event_id, inherit=True, blocks = [pkg for pkg in readPackageList(tag_id, event=event_id, inherit=True,
@ -2766,9 +2752,9 @@ def repo_init(tag, task_id=None, with_src=False, with_debuginfo=False, event=Non
for build in builds: for build in builds:
relpath = relpathinfo.build(build) relpath = relpathinfo.build(build)
builddirs[build['id']] = relpath.lstrip('/') builddirs[build['id']] = relpath.lstrip('/')
# generate pkglist files # generate pkglist and rpmlist files
pkglist = {} pkglist = {}
content_jsons = {} rpmlist = {}
for repoarch in repo_arches: for repoarch in repo_arches:
archdir = joinpath(repodir, repoarch) archdir = joinpath(repodir, repoarch)
koji.ensuredir(archdir) koji.ensuredir(archdir)
@ -2777,48 +2763,43 @@ def repo_init(tag, task_id=None, with_src=False, with_debuginfo=False, event=Non
top_link = joinpath(archdir, 'toplink') top_link = joinpath(archdir, 'toplink')
os.symlink(top_relpath, top_link) os.symlink(top_relpath, top_link)
pkglist[repoarch] = open(joinpath(archdir, 'pkglist'), 'wt', encoding='utf-8') pkglist[repoarch] = open(joinpath(archdir, 'pkglist'), 'wt', encoding='utf-8')
content_jsons[repoarch] = (joinpath(archdir, 'content.json'), {}) rpmlist[repoarch] = open(joinpath(archdir, 'rpmlist.jsonl'), 'wt', encoding='utf-8')
# NOTE - rpms is now an iterator # NOTE - rpms is a generator
for rpminfo in rpms: for rpminfo in rpms:
if not with_debuginfo and koji.is_debuginfo(rpminfo['name']): if not with_debuginfo and koji.is_debuginfo(rpminfo['name']):
continue continue
relpath = "%s/%s\n" % (builddirs[rpminfo['build_id']], relpathinfo.rpm(rpminfo)) relpath = "%s/%s\n" % (builddirs[rpminfo['build_id']], relpathinfo.rpm(rpminfo))
content = dslice( rpm_json = json.dumps(rpminfo, indent=None)
rpminfo, # must be one line for nl-delimited json
(
'id', 'build_id', 'name', 'version', 'release', 'epoch',
'arch', 'payloadhash', 'size', 'buildtime'
),
strict=True
)
content['filepath'] = relpath
nvra = "{name}-{version}-{release}.{arch}".format(**content)
arch = rpminfo['arch'] arch = rpminfo['arch']
if arch == 'src': if arch == 'src':
if with_src: if with_src:
for repoarch in repo_arches: for repoarch in repo_arches:
pkglist[repoarch].write(relpath) pkglist[repoarch].write(relpath)
content_jsons[repoarch][1][nvra] = content rpmlist[repoarch].write(rpm_json)
rpmlist[repoarch].write('\n')
if with_separate_src: if with_separate_src:
pkglist[arch].write(relpath) pkglist[arch].write(relpath)
content_jsons[repoarch][1][nvra] = content rpmlist[arch].write(rpm_json)
rpmlist[arch].write('\n')
elif arch == 'noarch': elif arch == 'noarch':
for repoarch in repo_arches: for repoarch in repo_arches:
if repoarch == 'src': if repoarch == 'src':
continue continue
pkglist[repoarch].write(relpath) pkglist[repoarch].write(relpath)
content_jsons[repoarch][1][nvra] = content rpmlist[repoarch].write(rpm_json)
rpmlist[repoarch].write('\n')
else: else:
repoarch = koji.canonArch(arch) repoarch = koji.canonArch(arch)
if repoarch not in repo_arches: if repoarch not in repo_arches:
# Do not create a repo for arches not in the arch list for this tag # Do not create a repo for arches not in the arch list for this tag
continue continue
pkglist[repoarch].write(relpath) pkglist[repoarch].write(relpath)
content_jsons[repoarch][1][nvra] = content rpmlist[repoarch].write(rpm_json)
rpmlist[repoarch].write('\n')
for repoarch in repo_arches: for repoarch in repo_arches:
pkglist[repoarch].close() pkglist[repoarch].close()
with open(content_jsons[repoarch][0], 'wt', encoding='utf-8') as fp: rpmlist[repoarch].close()
json.dump(content_jsons[repoarch][1], fp, indent=2)
# write blocked package lists # write blocked package lists
for repoarch in repo_arches: for repoarch in repo_arches: