From 18f5a29ead532d6f4d92b15d9155625428505b90 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Mon, 14 Sep 2020 16:11:00 +0200 Subject: [PATCH] packaging --- builder/kojid | 219 ++++++-------------------------------------------- koji.spec | 6 +- setup.py | 11 +-- tox.ini | 4 - 4 files changed, 32 insertions(+), 208 deletions(-) diff --git a/builder/kojid b/builder/kojid index e6fe53e4..a1f81624 100755 --- a/builder/kojid +++ b/builder/kojid @@ -26,6 +26,7 @@ from __future__ import absolute_import, division import copy import glob import grp +import io import json import logging import logging.handlers @@ -48,6 +49,8 @@ from gzip import GzipFile from optparse import SUPPRESS_HELP, OptionParser import Cheetah.Template +import dnf +import librepo import requests import rpm import six @@ -79,26 +82,6 @@ except ImportError: # pragma: no cover except ImportError: # pragma: no cover reqgssapi = None -try: - import librepo - import io -except ImportError: - librepo = None - -try: - import dnf -except ImportError: - dnf = None - -try: - # yum - from yum import repoMDObject - import yum.packages - import yum.Errors - yum_available = True -except ImportError: - yum_available = False - # imports for LiveCD, LiveMedia, and Appliance handler try: import pykickstart.parser as ksparser @@ -803,42 +786,26 @@ class BuildRoot(object): opts = dict([(k, getattr(self.options, k)) for k in ('topurl', 'topdir')]) opts['tempdir'] = self.options.workdir - # prefer librepo - if librepo is not None: - repo_url = os.path.join(repodir, self.br_arch) - # repo_url can start with '/', don't use os.path.join - if self.options.topurl: - repo_url = '%s/%s' % (self.options.topurl, repo_url) - elif self.options.topdir: - repo_url = '%s/%s' % (self.options.topdir, repo_url) - logging.error(repo_url) - tmpdir = os.path.join(self.tmpdir(), 'librepo-markExternalRPMs') - koji.ensuredir(tmpdir) - h = librepo.Handle() - r = librepo.Result() - h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO) - h.setopt(librepo.LRO_URLS, [repo_url]) - h.setopt(librepo.LRO_DESTDIR, tmpdir) - # We are using this just to find out location of 'origin', - # we don't even need to download it since we use openRemoteFile - h.setopt(librepo.LRO_YUMDLIST, []) - h.perform(r) - pkgorigins = r.getinfo(librepo.LRR_YUM_REPOMD)['origin']['location_href'] - koji.util.rmtree(tmpdir) - elif yum_available: - # XXX - cheap hack to get relative paths - repomdpath = os.path.join(repodir, self.br_arch, 'repodata', 'repomd.xml') - with koji.openRemoteFile(repomdpath, **opts) as fo: - try: - repodata = repoMDObject.RepoMD('ourrepo', fo) - except Exception: - raise koji.BuildError("Unable to parse repomd.xml file for %s" % - os.path.join(repodir, self.br_arch)) - data = repodata.getData('origin') - pkgorigins = data.location[1] - else: - # shouldn't occur - raise koji.GenericError("install librepo or yum") + repo_url = os.path.join(repodir, self.br_arch) + # repo_url can start with '/', don't use os.path.join + if self.options.topurl: + repo_url = '%s/%s' % (self.options.topurl, repo_url) + elif self.options.topdir: + repo_url = '%s/%s' % (self.options.topdir, repo_url) + logging.error(repo_url) + tmpdir = os.path.join(self.tmpdir(), 'librepo-markExternalRPMs') + koji.ensuredir(tmpdir) + h = librepo.Handle() + r = librepo.Result() + h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO) + h.setopt(librepo.LRO_URLS, [repo_url]) + h.setopt(librepo.LRO_DESTDIR, tmpdir) + # We are using this just to find out location of 'origin', + # we don't even need to download it since we use openRemoteFile + h.setopt(librepo.LRO_YUMDLIST, []) + h.perform(r) + pkgorigins = r.getinfo(librepo.LRR_YUM_REPOMD)['origin']['location_href'] + koji.util.rmtree(tmpdir) relpath = os.path.join(repodir, self.br_arch, pkgorigins) with koji.openRemoteFile(relpath, **opts) as fo: @@ -5754,10 +5721,7 @@ class createDistRepoTask(BaseTaskHandler): self.uploadpath = self.getUploadDir() self.get_rpms(tag, arch, keys, opts) if opts['multilib'] and koji.arch.isMultiLibArch(arch): - if dnf is not None: - self.do_multilib_dnf(arch, self.archmap[arch], opts['multilib']) - else: - self.do_multilib_yum(arch, self.archmap[arch], opts['multilib']) + self.do_multilib(arch, self.archmap[arch], opts['multilib']) self.split_pkgs(opts) self.write_kojipkgs() self.write_pkglist() @@ -5884,7 +5848,7 @@ class createDistRepoTask(BaseTaskHandler): raise koji.GenericError('failed to create repo: %s' % parseStatus(status, ' '.join(cmd))) - def do_multilib_dnf(self, arch, ml_arch, conf): + def do_multilib(self, arch, ml_arch, conf): repodir = koji.pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name']) mldir = os.path.join(repodir, koji.canonArch(ml_arch)) ml_true = set() # multilib packages we need to include before depsolve @@ -6010,139 +5974,6 @@ enabled=1 rpminfo['_multilib'] = True self.kojipkgs[bnp] = rpminfo - def do_multilib_yum(self, arch, ml_arch, conf): - repodir = koji.pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name']) - mldir = os.path.join(repodir, koji.canonArch(ml_arch)) - ml_true = set() # multilib packages we need to include before depsolve - ml_conf = os.path.join(koji.pathinfo.work(), conf) - - # read pkgs data from multilib repo - ml_pkgfile = os.path.join(mldir, 'kojipkgs') - ml_pkgs = json.load(open(ml_pkgfile, 'r')) - - # step 1: figure out which packages are multilib (should already exist) - mlm = multilib.DevelMultilibMethod(ml_conf) - fs_missing = set() - for bnp in self.kojipkgs: - rpminfo = self.kojipkgs[bnp] - ppath = rpminfo['_pkgpath'] - po = yum.packages.YumLocalPackage(filename=ppath) - if mlm.select(po): - # we need a multilib package to be included - ml_bnp = bnp.replace(arch, self.archmap[arch]) - ml_path = os.path.join(mldir, ml_bnp[0].lower(), ml_bnp) - # ^ XXX - should actually generate this - if ml_bnp not in ml_pkgs: - # not in our multilib repo - self.logger.error('%s (multilib) is not on the filesystem' % ml_path) - fs_missing.add(ml_path) - # we defer failure so can report all the missing deps - continue - ml_true.add(ml_path) - - # step 2: set up architectures for yum configuration - self.logger.info("Resolving multilib for %s using method devel" % arch) - yumbase = yum.YumBase() - yumbase.verbose_logger.setLevel(logging.ERROR) - yumdir = os.path.join(self.workdir, 'yum') - # TODO: unwind this arch mess - archlist = (arch, 'noarch') - transaction_arch = arch - archlist = archlist + self.compat[self.biarch[arch]] - best_compat = self.compat[self.biarch[arch]][0] - if koji.arch.archDifference(best_compat, arch) > 0: - transaction_arch = best_compat - if hasattr(koji.arch, 'ArchStorage'): - yumbase.preconf.arch = transaction_arch - else: - koji.arch.canonArch = transaction_arch - - yconfig = """ -[main] -debuglevel=2 -pkgpolicy=newest -exactarch=1 -gpgcheck=0 -reposdir=/dev/null -cachedir=/yumcache -installroot=%s -logfile=/yum.log - -[koji-%s] -name=koji multilib task -baseurl=file://%s -enabled=1 - -""" % (yumdir, self.id, mldir) - os.makedirs(os.path.join(yumdir, "yumcache")) - os.makedirs(os.path.join(yumdir, 'var/lib/rpm')) - - # step 3: proceed with yum config and set up - yconfig_path = os.path.join(yumdir, 'yum.conf-koji-%s' % arch) - with open(yconfig_path, 'w') as f: - f.write(yconfig) - self.session.uploadWrapper(yconfig_path, self.uploadpath, - os.path.basename(yconfig_path)) - yumbase.doConfigSetup(fn=yconfig_path) - yumbase.conf.cache = 0 - yumbase.doRepoSetup() - yumbase.doTsSetup() - yumbase.doRpmDBSetup() - # we trust Koji's files, so skip verifying sigs and digests - yumbase.ts.pushVSFlags( - (rpm._RPMVSF_NOSIGNATURES | rpm._RPMVSF_NODIGESTS)) - yumbase.doSackSetup(archlist=archlist, thisrepo='koji-%s' % arch) - yumbase.doSackFilelistPopulate() - for pkg in ml_true: - # TODO: store packages by first letter - # ppath = os.path.join(pkgdir, pkg.name[0].lower(), pname) - po = yum.packages.YumLocalPackage(filename=pkg) - yumbase.tsInfo.addInstall(po) - - # step 4: execute yum transaction to get dependencies - self.logger.info("Resolving depenencies for arch %s" % arch) - rc, errors = yumbase.resolveDeps() - ml_needed = {} - for tspkg in yumbase.tsInfo.getMembers(): - bnp = os.path.basename(tspkg.po.localPkg()) - dep_path = os.path.join(mldir, bnp[0].lower(), bnp) - ml_needed[dep_path] = tspkg - self.logger.debug("added %s" % dep_path) - if not os.path.exists(dep_path): - self.logger.error('%s (multilib dep) not on filesystem' % dep_path) - fs_missing.add(dep_path) - self.logger.info('yum return code: %s' % rc) - if not rc: - self.logger.error('yum depsolve was unsuccessful') - raise koji.GenericError(errors) - if len(fs_missing) > 0: - missing_log = os.path.join(self.workdir, 'missing_multilib.log') - with open(missing_log, 'w') as outfile: - outfile.write('The following multilib files were missing:\n') - for ml_path in fs_missing: - outfile.write(ml_path) - outfile.write('\n') - self.session.uploadWrapper(missing_log, self.uploadpath) - raise koji.GenericError('multilib packages missing. ' - 'See missing_multilib.log') - - # step 5: update kojipkgs - for dep_path in ml_needed: - tspkg = ml_needed[dep_path] - bnp = os.path.basename(dep_path) - if bnp in self.kojipkgs: - # we expect duplication with noarch, but not other arches - if tspkg.arch != 'noarch': - self.logger.warning("Multilib duplicate: %s", bnp) - continue - rpminfo = ml_pkgs[bnp].copy() - # fix _pkgpath, which comes from another task and could be wrong - # for us - # TODO: would be better if we could use the proper path here - rpminfo['_pkgpath'] = dep_path - rpminfo['_multilib'] = True - self.kojipkgs[bnp] = rpminfo - def pick_key(self, keys, avail_keys): best = None best_idx = None diff --git a/koji.spec b/koji.spec index a5f3d789..42ff5417 100644 --- a/koji.spec +++ b/koji.spec @@ -18,9 +18,13 @@ # and no python2 on rhel8+ %define py2_support 0 %else -%if 0%{?rhel} +%if 0%{?rhel} >= 7 # No python3 for older rhel %define py3_support 0 +%else +# don't build anything for rhel6 +%define py2_support 0 +%define py3_support 0 %endif %endif diff --git a/setup.py b/setup.py index 94f31b32..b34c811d 100644 --- a/setup.py +++ b/setup.py @@ -13,6 +13,7 @@ def get_install_requires(): requires = [ 'python-dateutil', + 'pyOpenSSL', 'requests', 'requests-gssapi', 'six', @@ -20,13 +21,6 @@ def get_install_requires(): # 'rpm-py-installer', # it is optional feature # 'rpm', ] - # since pyOpenSSL-18.0.0, py26 support is dropped - # see https://pagure.io/koji/issue/1060 - if sys.version_info[0] == 2 and sys.version_info[1] < 7: - requires.append('pyOpenSSL<18.0.0') - else: - requires.append('pyOpenSSL') - return requires @@ -51,7 +45,6 @@ setup( "Intended Audience :: Developers", "License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)", "Natural Language :: English", - "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Operating System :: POSIX :: Linux", @@ -74,6 +67,6 @@ setup( 'util/koji-sweep-db', 'util/kojira', ], - python_requires='>=2.6', + python_requires='>=2.7', install_requires=get_install_requires(), ) diff --git a/tox.ini b/tox.ini index c188442b..81638095 100644 --- a/tox.ini +++ b/tox.ini @@ -4,10 +4,6 @@ envlist = flake8,py2,py3 [testenv:flake8] deps = flake8 -# On EL6, pip would get us a flake8 that doesn't work with python2.6. The next -# two lines let us use an RPM-based version from EPEL if it is installed -sitepackages = true -whitelist_externals = flake8 # These two lines just speed things up by avoiding unnecessary setup skip_install=true usedevelop=true