particle-os/debian-koji
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

run rpmdiff against noarch subpackages

Browse source
This commit is contained in:
Mike Bonnet 2009-02-06 18:06:38 -05:00
parent 55761eacbd
commit f8e875afc0
5 changed files with 309 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

21
builder/kojid
View file

@ -1862,16 +1862,9 @@ class BuildTask(BaseTaskHandler):
for fn in result['rpms']:
rpms.append(fn)
brmap[fn] = brootid
existing_rpm = []
for fn in result['noarch_rpms']:
noarch_rpm = fn.split('/')[-1]
if noarch_rpm not in existing_rpm:
rpms.append(fn)
existing_rpm.append(noarch_rpm)
brmap[fn] = brootid
for fn in result['logs']:
logs.setdefault(arch,[]).append(fn)
if len(result['srpms']) == 1:
if result['srpms']:
if built_srpm:
raise koji.BuildError, "multiple builds returned a srpm. task %i" % self.id
else:
@ -1963,7 +1956,6 @@ class BuildArchTask(BaseTaskHandler):
rpm_files = []
srpm_files = []
log_files = []
noarch_rpm_files = []
unexpected = []
for f in os.listdir(resultdir):
# files here should have one of two extensions: .log and .rpm
@ -1971,14 +1963,11 @@ class BuildArchTask(BaseTaskHandler):
log_files.append(f)
elif f[-8:] == ".src.rpm":
srpm_files.append(f)
elif f[-11:] == ".noarch.rpm":
noarch_rpm_files.append(f)
elif f[-4:] == ".rpm":
rpm_files.append(f)
else:
unexpected.append(f)
self.logger.debug("rpms: %r" % rpm_files)
self.logger.debug("noarch rpms: %r" % noarch_rpm_files)
self.logger.debug("srpms: %r" % srpm_files)
self.logger.debug("logs: %r" % log_files)
self.logger.debug("unexpected: %r" % unexpected)
@ -1987,8 +1976,6 @@ class BuildArchTask(BaseTaskHandler):
uploadpath = broot.getUploadPath()
for f in rpm_files:
self.uploadFile("%s/%s" % (resultdir,f))
for f in noarch_rpm_files:
self.uploadFile("%s/%s" % (resultdir,f))
self.logger.debug("keep srpm %i %s %s" % (self.id, keep_srpm, opts))
if keep_srpm:
if len(srpm_files) == 0:
@ -2001,14 +1988,10 @@ class BuildArchTask(BaseTaskHandler):
self.logger.debug("uploading %s/%s to %s" % (resultdir,srpm_files[0], uploadpath))
self.uploadFile("%s/%s" % (resultdir,srpm_files[0]))
if not rpm_files == []:
if rpm_files:
ret['rpms'] = [ "%s/%s" % (uploadpath,f) for f in rpm_files ]
else:
ret['rpms'] = []
if not noarch_rpm_files == []:
ret['noarch_rpms'] = [ "%s/%s" % (uploadpath,f) for f in noarch_rpm_files ]
else:
ret['noarch_rpms'] = []
if keep_srpm:
ret['srpms'] = [ "%s/%s" % (uploadpath,f) for f in srpm_files ]
else:

4
hub/Makefile
View file

@ -1,5 +1,6 @@
PYTHON=python
PACKAGE = $(shell basename `pwd`)
LIBEXECFILES = rpmdiff
PYFILES = $(wildcard *.py)
PYVER := $(shell $(PYTHON) -c 'import sys; print "%.3s" %(sys.version)')
PYSYSDIR := $(shell $(PYTHON) -c 'import sys; print sys.prefix')
@ -22,6 +23,9 @@ install:
exit 1; \
fi
mkdir -p $(DESTDIR)/usr/libexec/koji-hub
install -p -m 755 $(LIBEXECFILES) $(DESTDIR)/usr/libexec/koji-hub
mkdir -p $(DESTDIR)/etc/httpd/conf.d
install -p -m 644 httpd.conf $(DESTDIR)/etc/httpd/conf.d/kojihub.conf

54
hub/kojihub.py
View file

@ -36,6 +36,7 @@ import fnmatch
import md5
import os
import pgdb
import popen2
import random
import re
import rpm
@ -3086,6 +3087,33 @@ def new_build(data):
q="""SELECT currval('build_id_seq')"""
return _singleValue(q)
def check_noarch_rpms(basepath, rpms):
"""
If rpms contains any noarch rpms with identical names,
run rpmdiff against the duplicate rpms.
Return the list of rpms with any duplicate entries removed (only
the first entry will be retained).
"""
result = []
noarch_rpms = {}
for relpath in rpms:
if relpath.endswith('.noarch.rpm'):
filename = os.path.basename(relpath)
if noarch_rpms.has_key(filename):
# duplicate found, add it to the duplicate list
# but not the result list
noarch_rpms[filename].append(relpath)
else:
noarch_rpms[filename] = [relpath]
result.append(relpath)
else:
result.append(relpath)
for noarch_list in noarch_rpms.values():
rpmdiff(basepath, noarch_list)
return result
def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None):
"""Import a build into the database (single transaction)
@ -3106,6 +3134,8 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None)
if not os.path.exists(fn):
raise koji.GenericError, "no such file: %s" % fn
rpms = check_noarch_rpms(uploadpath, rpms)
#verify buildroot ids from brmap
found = {}
for br_id in brmap.values():
@ -4532,6 +4562,27 @@ def assert_policy(name, data, default='deny'):
"""
check_policy(name, data, default=default, strict=True)
def rpmdiff(basepath, rpmlist):
"Diff the first rpm in the list against the rest of the rpms."
if len(rpmlist) < 2:
return
first_rpm = rpmlist[0]
for other_rpm in rpmlist[1:]:
# ignore differences in file size, md5sum, and mtime
# (files may have been generated at build time and contain
# embedded dates or other insignificant differences)
proc = popen2.Popen4(['/usr/libexec/koji-hub/rpmdiff',
'--ignore', 'S', '--ignore', '5',
'--ignore', 'T',
os.path.join(basepath, first_rpm),
os.path.join(basepath, other_rpm)])
proc.tochild.close()
output = proc.fromchild.read()
status = proc.wait()
if os.WIFSIGNALED(status) or \
(os.WEXITSTATUS(status) != 0):
raise koji.BuildError, 'mismatch when analyzing %s, rpmdiff output was:\n%s' % \
(os.path.basename(first_rpm), output)
#
# XMLRPC Methods
@ -6955,6 +7006,9 @@ class HostExports(object):
fn = "%s/%s" % (uploadpath,relpath)
if not os.path.exists(fn):
raise koji.GenericError, "no such file: %s" % fn
rpms = check_noarch_rpms(uploadpath, rpms)
#figure out storage location
# <scratchdir>/<username>/task_<id>
scratchdir = koji.pathinfo.scratch()

247
hub/rpmdiff Executable file
View file

@ -0,0 +1,247 @@
#!/usr/bin/python
#
# Copyright (C) 2006 Mandriva; 2009 Red Hat, Inc.
# Authors: Frederic Lepied, Florian Festi
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Library General Public License as published by
# the Free Software Foundation; version 2 only
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# This library and program is heavily based on rpmdiff from the rpmlint package
# It was modified to be used as standalone library for the Koji project.
import rpm
import os
import itertools
import sys, getopt
class Rpmdiff:
# constants
TAGS = ( rpm.RPMTAG_NAME, rpm.RPMTAG_SUMMARY,
rpm.RPMTAG_DESCRIPTION, rpm.RPMTAG_GROUP,
rpm.RPMTAG_LICENSE, rpm.RPMTAG_URL,
rpm.RPMTAG_PREIN, rpm.RPMTAG_POSTIN,
rpm.RPMTAG_PREUN, rpm.RPMTAG_POSTUN)
PRCO = ( 'REQUIRES', 'PROVIDES', 'CONFLICTS', 'OBSOLETES')
#{fname : (size, mode, mtime, flags, dev, inode,
# nlink, state, vflags, user, group, digest)}
__FILEIDX = [ ['S', 0],
['M', 1],
['5', 11],
['D', 4],
['N', 6],
['L', 7],
['V', 8],
['U', 9],
['G', 10],
['F', 3],
['T', 2] ]
try:
if rpm.RPMSENSE_SCRIPT_PRE:
PREREQ_FLAG=rpm.RPMSENSE_PREREQ|rpm.RPMSENSE_SCRIPT_PRE|\
rpm.RPMSENSE_SCRIPT_POST|rpm.RPMSENSE_SCRIPT_PREUN|\
rpm.RPMSENSE_SCRIPT_POSTUN
except AttributeError:
try:
PREREQ_FLAG=rpm.RPMSENSE_PREREQ
except:
#(proyvind): This seems ugly, but then again so does
# this whole check as well.
PREREQ_FLAG=False
DEPFORMAT = '%-11s%s %s %s %s'
FORMAT = '%-11s%s'
ADDED = 'added'
REMOVED = 'removed'
# code starts here
def __init__(self, old, new, ignore=None):
self.result = []
self.ignore = ignore
if self.ignore is None:
self.ignore = []
FILEIDX = self.__FILEIDX
for tag in self.ignore:
for entry in FILEIDX:
if tag == entry[0]:
entry[1] = None
break
old = self.__load_pkg(old)
new = self.__load_pkg(new)
# Compare single tags
for tag in self.TAGS:
old_tag = old[tag]
new_tag = new[tag]
if old_tag != new_tag:
tagname = rpm.tagnames[tag]
if old_tag == None:
self.__add(self.FORMAT, (self.ADDED, tagname))
elif new_tag == None:
self.__add(self.FORMAT, (self.REMOVED, tagname))
else:
self.__add(self.FORMAT, ('S.5.....', tagname))
# compare Provides, Requires, ...
for tag in self.PRCO:
self.__comparePRCOs(old, new, tag)
# compare the files
old_files_dict = self.__fileIteratorToDict(old.fiFromHeader())
new_files_dict = self.__fileIteratorToDict(new.fiFromHeader())
files = list(set(itertools.chain(old_files_dict.iterkeys(),
new_files_dict.iterkeys())))
files.sort()
for f in files:
diff = 0
old_file = old_files_dict.get(f)
new_file = new_files_dict.get(f)
if not old_file:
self.__add(self.FORMAT, (self.ADDED, f))
elif not new_file:
self.__add(self.FORMAT, (self.REMOVED, f))
else:
format = ''
for entry in FILEIDX:
if entry[1] != None and \
old_file[entry[1]] != new_file[entry[1]]:
format = format + entry[0]
diff = 1
else:
format = format + '.'
if diff:
self.__add(self.FORMAT, (format, f))
# return a report of the differences
def textdiff(self):
return '\n'.join((format % data for format, data in self.result))
# do the two rpms differ
def differs(self):
return bool(self.result)
# add one differing item
def __add(self, format, data):
self.result.append((format, data))
# load a package from a file or from the installed ones
def __load_pkg(self, filename):
ts = rpm.ts()
f = os.open(filename, os.O_RDONLY)
hdr = ts.hdrFromFdno(f)
os.close(f)
return hdr
# output the right string according to RPMSENSE_* const
def sense2str(self, sense):
s = ""
for tag, char in ((rpm.RPMSENSE_LESS, "<"),
(rpm.RPMSENSE_GREATER, ">"),
(rpm.RPMSENSE_EQUAL, "=")):
if sense & tag:
s += char
return s
# compare Provides, Requires, Conflicts, Obsoletes
def __comparePRCOs(self, old, new, name):
oldflags = old[name[:-1]+'FLAGS']
newflags = new[name[:-1]+'FLAGS']
# fix buggy rpm binding not returning list for single entries
if not isinstance(oldflags, list): oldflags = [ oldflags ]
if not isinstance(newflags, list): newflags = [ newflags ]
o = zip(old[name], oldflags, old[name[:-1]+'VERSION'])
n = zip(new[name], newflags, new[name[:-1]+'VERSION'])
if name == 'PROVIDES': # filter our self provide
oldNV = (old['name'], rpm.RPMSENSE_EQUAL,
"%s-%s" % (old['version'], old['release']))
newNV = (new['name'], rpm.RPMSENSE_EQUAL,
"%s-%s" % (new['version'], new['release']))
o = [entry for entry in o if entry != oldNV]
n = [entry for entry in n if entry != newNV]
for oldentry in o:
if not oldentry in n:
if name == 'REQUIRES' and oldentry[1] & self.PREREQ_FLAG:
tagname = 'PREREQ'
else:
tagname = name
self.__add(self.DEPFORMAT,
(self.REMOVED, tagname, oldentry[0],
self.sense2str(oldentry[1]), oldentry[2]))
for newentry in n:
if not newentry in o:
if name == 'REQUIRES' and newentry[1] & self.PREREQ_FLAG:
tagname = 'PREREQ'
else:
tagname = name
self.__add(self.DEPFORMAT,
(self.ADDED, tagname, newentry[0],
self.sense2str(newentry[1]), newentry[2]))
def __fileIteratorToDict(self, fi):
result = {}
for filedata in fi:
result[filedata[0]] = filedata[1:]
return result
def _usage(exit=1):
print "Usage: %s [<options>] <old package> <new package>" % sys.argv[0]
print "Options:"
print " -h, --help Output this message and exit"
print " -i, --ignore Tag to ignore when calculating differences"
print " (may be used multiple times)"
print " Valid values are: SM5DNLVUGFT"
sys.exit(exit)
def main():
ignore_tags = []
try:
opts, args = getopt.getopt(sys.argv[1:], "hi:", ["help", "ignore="])
except getopt.GetoptError, e:
print "Error: %s" % e
_usage()
for option, argument in opts:
if option in ("-h", "--help"):
_usage(0)
if option in ("-i", "--ignore"):
ignore_tags.append(argument)
if len(args) != 2:
_usage()
d = Rpmdiff(args[0], args[1], ignore=ignore_tags)
print d.textdiff()
sys.exit(int(d.differs()))
if __name__ == '__main__':
main()
# rpmdiff ends here

2
koji.spec
View file

@ -109,6 +109,8 @@ rm -rf $RPM_BUILD_ROOT
%files hub
%defattr(-,root,root)
%{_datadir}/koji-hub
%dir %{_libexecdir}/koji-hub
%{_libexecdir}/koji-hub/rpmdiff
%config(noreplace) /etc/httpd/conf.d/kojihub.conf
%config(noreplace) /etc/koji-hub/hub.conf