diff --git a/builder/kojid b/builder/kojid index 0b24df07..7c4d792f 100755 --- a/builder/kojid +++ b/builder/kojid @@ -1862,16 +1862,9 @@ class BuildTask(BaseTaskHandler): for fn in result['rpms']: rpms.append(fn) brmap[fn] = brootid - existing_rpm = [] - for fn in result['noarch_rpms']: - noarch_rpm = fn.split('/')[-1] - if noarch_rpm not in existing_rpm: - rpms.append(fn) - existing_rpm.append(noarch_rpm) - brmap[fn] = brootid for fn in result['logs']: logs.setdefault(arch,[]).append(fn) - if len(result['srpms']) == 1: + if result['srpms']: if built_srpm: raise koji.BuildError, "multiple builds returned a srpm. task %i" % self.id else: @@ -1963,7 +1956,6 @@ class BuildArchTask(BaseTaskHandler): rpm_files = [] srpm_files = [] log_files = [] - noarch_rpm_files = [] unexpected = [] for f in os.listdir(resultdir): # files here should have one of two extensions: .log and .rpm @@ -1971,14 +1963,11 @@ class BuildArchTask(BaseTaskHandler): log_files.append(f) elif f[-8:] == ".src.rpm": srpm_files.append(f) - elif f[-11:] == ".noarch.rpm": - noarch_rpm_files.append(f) elif f[-4:] == ".rpm": rpm_files.append(f) else: unexpected.append(f) self.logger.debug("rpms: %r" % rpm_files) - self.logger.debug("noarch rpms: %r" % noarch_rpm_files) self.logger.debug("srpms: %r" % srpm_files) self.logger.debug("logs: %r" % log_files) self.logger.debug("unexpected: %r" % unexpected) @@ -1987,8 +1976,6 @@ class BuildArchTask(BaseTaskHandler): uploadpath = broot.getUploadPath() for f in rpm_files: self.uploadFile("%s/%s" % (resultdir,f)) - for f in noarch_rpm_files: - self.uploadFile("%s/%s" % (resultdir,f)) self.logger.debug("keep srpm %i %s %s" % (self.id, keep_srpm, opts)) if keep_srpm: if len(srpm_files) == 0: @@ -2001,14 +1988,10 @@ class BuildArchTask(BaseTaskHandler): self.logger.debug("uploading %s/%s to %s" % (resultdir,srpm_files[0], uploadpath)) self.uploadFile("%s/%s" % (resultdir,srpm_files[0])) - if not rpm_files == []: + if rpm_files: ret['rpms'] = [ "%s/%s" % (uploadpath,f) for f in rpm_files ] else: ret['rpms'] = [] - if not noarch_rpm_files == []: - ret['noarch_rpms'] = [ "%s/%s" % (uploadpath,f) for f in noarch_rpm_files ] - else: - ret['noarch_rpms'] = [] if keep_srpm: ret['srpms'] = [ "%s/%s" % (uploadpath,f) for f in srpm_files ] else: diff --git a/hub/Makefile b/hub/Makefile index c0e42f3c..ccd84db9 100644 --- a/hub/Makefile +++ b/hub/Makefile @@ -1,5 +1,6 @@ PYTHON=python PACKAGE = $(shell basename `pwd`) +LIBEXECFILES = rpmdiff PYFILES = $(wildcard *.py) PYVER := $(shell $(PYTHON) -c 'import sys; print "%.3s" %(sys.version)') PYSYSDIR := $(shell $(PYTHON) -c 'import sys; print sys.prefix') @@ -22,6 +23,9 @@ install: exit 1; \ fi + mkdir -p $(DESTDIR)/usr/libexec/koji-hub + install -p -m 755 $(LIBEXECFILES) $(DESTDIR)/usr/libexec/koji-hub + mkdir -p $(DESTDIR)/etc/httpd/conf.d install -p -m 644 httpd.conf $(DESTDIR)/etc/httpd/conf.d/kojihub.conf diff --git a/hub/kojihub.py b/hub/kojihub.py index 8bc8d95a..9f8d3aa5 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -36,6 +36,7 @@ import fnmatch import md5 import os import pgdb +import popen2 import random import re import rpm @@ -3086,6 +3087,33 @@ def new_build(data): q="""SELECT currval('build_id_seq')""" return _singleValue(q) +def check_noarch_rpms(basepath, rpms): + """ + If rpms contains any noarch rpms with identical names, + run rpmdiff against the duplicate rpms. + Return the list of rpms with any duplicate entries removed (only + the first entry will be retained). + """ + result = [] + noarch_rpms = {} + for relpath in rpms: + if relpath.endswith('.noarch.rpm'): + filename = os.path.basename(relpath) + if noarch_rpms.has_key(filename): + # duplicate found, add it to the duplicate list + # but not the result list + noarch_rpms[filename].append(relpath) + else: + noarch_rpms[filename] = [relpath] + result.append(relpath) + else: + result.append(relpath) + + for noarch_list in noarch_rpms.values(): + rpmdiff(basepath, noarch_list) + + return result + def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None): """Import a build into the database (single transaction) @@ -3106,6 +3134,8 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None) if not os.path.exists(fn): raise koji.GenericError, "no such file: %s" % fn + rpms = check_noarch_rpms(uploadpath, rpms) + #verify buildroot ids from brmap found = {} for br_id in brmap.values(): @@ -4532,6 +4562,27 @@ def assert_policy(name, data, default='deny'): """ check_policy(name, data, default=default, strict=True) +def rpmdiff(basepath, rpmlist): + "Diff the first rpm in the list against the rest of the rpms." + if len(rpmlist) < 2: + return + first_rpm = rpmlist[0] + for other_rpm in rpmlist[1:]: + # ignore differences in file size, md5sum, and mtime + # (files may have been generated at build time and contain + # embedded dates or other insignificant differences) + proc = popen2.Popen4(['/usr/libexec/koji-hub/rpmdiff', + '--ignore', 'S', '--ignore', '5', + '--ignore', 'T', + os.path.join(basepath, first_rpm), + os.path.join(basepath, other_rpm)]) + proc.tochild.close() + output = proc.fromchild.read() + status = proc.wait() + if os.WIFSIGNALED(status) or \ + (os.WEXITSTATUS(status) != 0): + raise koji.BuildError, 'mismatch when analyzing %s, rpmdiff output was:\n%s' % \ + (os.path.basename(first_rpm), output) # # XMLRPC Methods @@ -6955,6 +7006,9 @@ class HostExports(object): fn = "%s/%s" % (uploadpath,relpath) if not os.path.exists(fn): raise koji.GenericError, "no such file: %s" % fn + + rpms = check_noarch_rpms(uploadpath, rpms) + #figure out storage location # //task_ scratchdir = koji.pathinfo.scratch() diff --git a/hub/rpmdiff b/hub/rpmdiff new file mode 100755 index 00000000..3497d2db --- /dev/null +++ b/hub/rpmdiff @@ -0,0 +1,247 @@ +#!/usr/bin/python +# +# Copyright (C) 2006 Mandriva; 2009 Red Hat, Inc. +# Authors: Frederic Lepied, Florian Festi +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library General Public License as published by +# the Free Software Foundation; version 2 only +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU Library General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +# This library and program is heavily based on rpmdiff from the rpmlint package +# It was modified to be used as standalone library for the Koji project. + +import rpm +import os +import itertools + +import sys, getopt + + +class Rpmdiff: + + # constants + + TAGS = ( rpm.RPMTAG_NAME, rpm.RPMTAG_SUMMARY, + rpm.RPMTAG_DESCRIPTION, rpm.RPMTAG_GROUP, + rpm.RPMTAG_LICENSE, rpm.RPMTAG_URL, + rpm.RPMTAG_PREIN, rpm.RPMTAG_POSTIN, + rpm.RPMTAG_PREUN, rpm.RPMTAG_POSTUN) + + PRCO = ( 'REQUIRES', 'PROVIDES', 'CONFLICTS', 'OBSOLETES') + + #{fname : (size, mode, mtime, flags, dev, inode, + # nlink, state, vflags, user, group, digest)} + __FILEIDX = [ ['S', 0], + ['M', 1], + ['5', 11], + ['D', 4], + ['N', 6], + ['L', 7], + ['V', 8], + ['U', 9], + ['G', 10], + ['F', 3], + ['T', 2] ] + + try: + if rpm.RPMSENSE_SCRIPT_PRE: + PREREQ_FLAG=rpm.RPMSENSE_PREREQ|rpm.RPMSENSE_SCRIPT_PRE|\ + rpm.RPMSENSE_SCRIPT_POST|rpm.RPMSENSE_SCRIPT_PREUN|\ + rpm.RPMSENSE_SCRIPT_POSTUN + except AttributeError: + try: + PREREQ_FLAG=rpm.RPMSENSE_PREREQ + except: + #(proyvind): This seems ugly, but then again so does + # this whole check as well. + PREREQ_FLAG=False + + DEPFORMAT = '%-11s%s %s %s %s' + FORMAT = '%-11s%s' + + ADDED = 'added' + REMOVED = 'removed' + + # code starts here + + def __init__(self, old, new, ignore=None): + self.result = [] + self.ignore = ignore + if self.ignore is None: + self.ignore = [] + + FILEIDX = self.__FILEIDX + for tag in self.ignore: + for entry in FILEIDX: + if tag == entry[0]: + entry[1] = None + break + + old = self.__load_pkg(old) + new = self.__load_pkg(new) + + # Compare single tags + for tag in self.TAGS: + old_tag = old[tag] + new_tag = new[tag] + if old_tag != new_tag: + tagname = rpm.tagnames[tag] + if old_tag == None: + self.__add(self.FORMAT, (self.ADDED, tagname)) + elif new_tag == None: + self.__add(self.FORMAT, (self.REMOVED, tagname)) + else: + self.__add(self.FORMAT, ('S.5.....', tagname)) + + # compare Provides, Requires, ... + for tag in self.PRCO: + self.__comparePRCOs(old, new, tag) + + # compare the files + + old_files_dict = self.__fileIteratorToDict(old.fiFromHeader()) + new_files_dict = self.__fileIteratorToDict(new.fiFromHeader()) + files = list(set(itertools.chain(old_files_dict.iterkeys(), + new_files_dict.iterkeys()))) + files.sort() + + for f in files: + diff = 0 + + old_file = old_files_dict.get(f) + new_file = new_files_dict.get(f) + + if not old_file: + self.__add(self.FORMAT, (self.ADDED, f)) + elif not new_file: + self.__add(self.FORMAT, (self.REMOVED, f)) + else: + format = '' + for entry in FILEIDX: + if entry[1] != None and \ + old_file[entry[1]] != new_file[entry[1]]: + format = format + entry[0] + diff = 1 + else: + format = format + '.' + if diff: + self.__add(self.FORMAT, (format, f)) + + # return a report of the differences + def textdiff(self): + return '\n'.join((format % data for format, data in self.result)) + + # do the two rpms differ + def differs(self): + return bool(self.result) + + # add one differing item + def __add(self, format, data): + self.result.append((format, data)) + + # load a package from a file or from the installed ones + def __load_pkg(self, filename): + ts = rpm.ts() + f = os.open(filename, os.O_RDONLY) + hdr = ts.hdrFromFdno(f) + os.close(f) + return hdr + + # output the right string according to RPMSENSE_* const + def sense2str(self, sense): + s = "" + for tag, char in ((rpm.RPMSENSE_LESS, "<"), + (rpm.RPMSENSE_GREATER, ">"), + (rpm.RPMSENSE_EQUAL, "=")): + if sense & tag: + s += char + return s + + # compare Provides, Requires, Conflicts, Obsoletes + def __comparePRCOs(self, old, new, name): + oldflags = old[name[:-1]+'FLAGS'] + newflags = new[name[:-1]+'FLAGS'] + # fix buggy rpm binding not returning list for single entries + if not isinstance(oldflags, list): oldflags = [ oldflags ] + if not isinstance(newflags, list): newflags = [ newflags ] + + o = zip(old[name], oldflags, old[name[:-1]+'VERSION']) + n = zip(new[name], newflags, new[name[:-1]+'VERSION']) + + if name == 'PROVIDES': # filter our self provide + oldNV = (old['name'], rpm.RPMSENSE_EQUAL, + "%s-%s" % (old['version'], old['release'])) + newNV = (new['name'], rpm.RPMSENSE_EQUAL, + "%s-%s" % (new['version'], new['release'])) + o = [entry for entry in o if entry != oldNV] + n = [entry for entry in n if entry != newNV] + + for oldentry in o: + if not oldentry in n: + if name == 'REQUIRES' and oldentry[1] & self.PREREQ_FLAG: + tagname = 'PREREQ' + else: + tagname = name + self.__add(self.DEPFORMAT, + (self.REMOVED, tagname, oldentry[0], + self.sense2str(oldentry[1]), oldentry[2])) + for newentry in n: + if not newentry in o: + if name == 'REQUIRES' and newentry[1] & self.PREREQ_FLAG: + tagname = 'PREREQ' + else: + tagname = name + self.__add(self.DEPFORMAT, + (self.ADDED, tagname, newentry[0], + self.sense2str(newentry[1]), newentry[2])) + + def __fileIteratorToDict(self, fi): + result = {} + for filedata in fi: + result[filedata[0]] = filedata[1:] + return result + +def _usage(exit=1): + print "Usage: %s [] " % sys.argv[0] + print "Options:" + print " -h, --help Output this message and exit" + print " -i, --ignore Tag to ignore when calculating differences" + print " (may be used multiple times)" + print " Valid values are: SM5DNLVUGFT" + sys.exit(exit) + +def main(): + + ignore_tags = [] + try: + opts, args = getopt.getopt(sys.argv[1:], "hi:", ["help", "ignore="]) + except getopt.GetoptError, e: + print "Error: %s" % e + _usage() + + for option, argument in opts: + if option in ("-h", "--help"): + _usage(0) + if option in ("-i", "--ignore"): + ignore_tags.append(argument) + + if len(args) != 2: + _usage() + + d = Rpmdiff(args[0], args[1], ignore=ignore_tags) + print d.textdiff() + sys.exit(int(d.differs())) + +if __name__ == '__main__': + main() + +# rpmdiff ends here diff --git a/koji.spec b/koji.spec index 0169b804..fb112e6e 100644 --- a/koji.spec +++ b/koji.spec @@ -109,6 +109,8 @@ rm -rf $RPM_BUILD_ROOT %files hub %defattr(-,root,root) %{_datadir}/koji-hub +%dir %{_libexecdir}/koji-hub +%{_libexecdir}/koji-hub/rpmdiff %config(noreplace) /etc/httpd/conf.d/kojihub.conf %config(noreplace) /etc/koji-hub/hub.conf