use the filesize and md5sum to track files in the build environment that don't have an associated .pom
This commit is contained in:
Mike Bonnet
parent
091a726196
commit
c4f1d2ca4d
3 changed files with 79 additions and 23 deletions
|
|
@ -515,8 +515,10 @@ class BuildRoot(object):
|
|||
raise koji.BuildrootError, "could not init mock buildroot, %s" % self._mockResult(rv)
|
||||
session.host.setBuildRootList(self.id,self.getPackageList())
|
||||
|
||||
def _mockResult(self, rv):
|
||||
if os.WIFEXITED(rv) and os.WEXITSTATUS(rv) == 1:
|
||||
def _mockResult(self, rv, logfile=None):
|
||||
if logfile:
|
||||
pass
|
||||
elif os.WIFEXITED(rv) and os.WEXITSTATUS(rv) == 1:
|
||||
logfile = 'build.log'
|
||||
else:
|
||||
logfile = 'root.log'
|
||||
|
|
@ -579,7 +581,7 @@ class BuildRoot(object):
|
|||
# Found a pom file, process it
|
||||
try:
|
||||
pom_info = koji.parse_pom(path=os.path.join(path, repofile))
|
||||
maven_files.append(repofile)
|
||||
maven_files.append({'filename': repofile})
|
||||
except:
|
||||
self.logger.error('Could not parse pom: %s' % repofile)
|
||||
self.logger.error(''.join(traceback.format_exception(*sys.exc_info())))
|
||||
|
|
@ -591,11 +593,30 @@ class BuildRoot(object):
|
|||
# Maven throws these metadata files around all over the place, ignore them
|
||||
continue
|
||||
else:
|
||||
maven_files.append(repofile)
|
||||
maven_files.append({'filename': repofile})
|
||||
if pom_info:
|
||||
packages.append({'pom': pom_info, 'files': maven_files})
|
||||
elif maven_files:
|
||||
self.logger.error('Found files in repo with no associated .pom: %s' % ', '.join(maven_files))
|
||||
# get the filesize and md5sum of the files so the hub might be able to identify them
|
||||
# if the hub can't identify them, it'll throw an error (we don't want untracked files
|
||||
# in the build environment)
|
||||
orphan_info = []
|
||||
for orphan in maven_files:
|
||||
orphan_path = os.path.join(path, orphan['filename'])
|
||||
orphan_size = os.stat(orphan_path).st_size
|
||||
sum = md5.new()
|
||||
orphan_fd = file(orphan_path)
|
||||
while True:
|
||||
contents = orphan_fd.read(65536)
|
||||
if contents:
|
||||
sum.update(contents)
|
||||
else:
|
||||
break
|
||||
orphan_fd.close()
|
||||
orphan_sum = sum.hexdigest()
|
||||
orphan_info.append({'filename': orphan['filename'], 'size': orphan_size,
|
||||
'md5sum': orphan_sum})
|
||||
packages.append({'pom': None, 'files': orphan_info})
|
||||
|
||||
return packages
|
||||
|
||||
|
|
@ -610,7 +631,7 @@ class BuildRoot(object):
|
|||
rv = self.mock(cmd + ['dependency:resolve-plugins'])
|
||||
if rv:
|
||||
self.expire()
|
||||
raise koji.BuildrootError, 'error resolving plugin dependencies, %s' % self._mockResult(rv)
|
||||
raise koji.BuildrootError, 'error resolving plugin dependencies, %s' % self._mockResult(rv, logfile='root.log')
|
||||
|
||||
plugin_deps = self.getMavenPackageList(repodir)
|
||||
session.host.updateMavenBuildRootList(self.id, self.task_id, plugin_deps, project=False)
|
||||
|
|
@ -625,7 +646,7 @@ class BuildRoot(object):
|
|||
project=True)
|
||||
if rv:
|
||||
self.expire()
|
||||
raise koji.BuildrootError, 'error building Maven package, %s' % self._mockResult(rv)
|
||||
raise koji.BuildrootError, 'error building Maven package, %s' % self._mockResult(rv, logfile='root.log')
|
||||
|
||||
def scrub(self):
|
||||
"Non-mock implementation of clean"
|
||||
|
|
|
|||
|
|
@ -644,6 +644,7 @@ CREATE TABLE archiveinfo (
|
|||
CREATE INDEX archiveinfo_build_idx ON archiveinfo (build_id);
|
||||
CREATE INDEX archiveinfo_buildroot_idx on archiveinfo (buildroot_id);
|
||||
CREATE INDEX archiveinfo_type_idx on archiveinfo (type_id);
|
||||
CREATE INDEX archiveinfo_filename_idx on archiveinfo(filename);
|
||||
|
||||
CREATE TABLE maven_archives (
|
||||
archive_id INTEGER NOT NULL PRIMARY KEY REFERENCES archiveinfo(id),
|
||||
|
|
|
|||
|
|
@ -2732,7 +2732,8 @@ def get_maven_build(buildInfo=None, mavenInfo=None, strict=False):
|
|||
else:
|
||||
raise koji.GenericError, 'either buildInfo or mavenInfo must be specified'
|
||||
|
||||
def list_archives(buildID=None, buildrootID=None, componentBuildrootID=None, hostID=None, type=None, queryOpts=None):
|
||||
def list_archives(buildID=None, buildrootID=None, componentBuildrootID=None, hostID=None, type=None,
|
||||
filename=None, size=None, md5sum=None, queryOpts=None):
|
||||
"""
|
||||
Retrieve information about archives.
|
||||
If buildID is not null it will restrict the list to archives built by the build with that ID.
|
||||
|
|
@ -2740,6 +2741,7 @@ def list_archives(buildID=None, buildrootID=None, componentBuildrootID=None, hos
|
|||
If componentBuildrootID is not null it will restrict the list to archives that were present in the
|
||||
buildroot with that ID.
|
||||
If hostID is not null it will restrict the list to archives built on the host with that ID.
|
||||
If filename, size, and/or md5sum are not null it will filter the results to entries matching the provided values.
|
||||
|
||||
Returns a list of maps containing the following keys:
|
||||
|
||||
|
|
@ -2803,6 +2805,15 @@ def list_archives(buildID=None, buildrootID=None, componentBuildrootID=None, hos
|
|||
values['host_id'] = hostID
|
||||
columns.append('buildroot.host_id')
|
||||
aliases.append('host_id')
|
||||
if filename is not None:
|
||||
clauses.append('filename = %(filename)s')
|
||||
values['filename'] = filename
|
||||
if size is not None:
|
||||
clauses.append('size = %(size)i')
|
||||
values['size'] = size
|
||||
if md5sum is not None:
|
||||
clauses.append('md5sum = %(md5sum)s')
|
||||
values['md5sum'] = md5sum
|
||||
|
||||
if type is None:
|
||||
pass
|
||||
|
|
@ -7178,32 +7189,55 @@ class HostExports(object):
|
|||
for entry in mavenlist:
|
||||
pom = entry['pom']
|
||||
files = entry['files']
|
||||
maven_info = {'group_id': pom['groupId'],
|
||||
'artifact_id': pom['artifactId'],
|
||||
'version': pom['version']}
|
||||
for filename in files:
|
||||
archiveinfos = find_maven_archives(maven_info, filename)
|
||||
if pom:
|
||||
maven_info = {'group_id': pom['groupId'],
|
||||
'artifact_id': pom['artifactId'],
|
||||
'version': pom['version']}
|
||||
for fileinfo in files:
|
||||
filename = fileinfo['filename']
|
||||
if pom:
|
||||
archiveinfos = find_maven_archives(maven_info, filename)
|
||||
else:
|
||||
archiveinfos = list_archives(filename=filename, size=fileinfo['size'],
|
||||
md5sum=fileinfo['md5sum'])
|
||||
if archiveinfos:
|
||||
if len(archiveinfos) == 1:
|
||||
archives.append(archiveinfos[0])
|
||||
else:
|
||||
raise koji.BuildrootError, 'multiple matches for %s in %s; archive IDs: %s' % \
|
||||
(filename, koji.mavenLabel(maven_info), ', '.join([str(a['id']) for a in archiveinfos]))
|
||||
raise koji.BuildrootError, 'multiple matches for %s; archive IDs: %s' % \
|
||||
(filename, ', '.join([str(a['id']) for a in archiveinfos]))
|
||||
else:
|
||||
# check if it's in the ignore list
|
||||
for ignore_entry in ignore:
|
||||
ignore_pom = ignore_entry['pom']
|
||||
ignore_files = ignore_entry['files']
|
||||
if pom['groupId'] == ignore_pom['groupId'] and \
|
||||
pom['artifactId'] == ignore_pom['artifactId'] and \
|
||||
pom['version'] == ignore_pom['version'] and \
|
||||
filename in ignore_files:
|
||||
# artifact is in the ignore list, don't bail out
|
||||
break
|
||||
if pom:
|
||||
if ignore_pom:
|
||||
if pom['groupId'] == ignore_pom['groupId'] and \
|
||||
pom['artifactId'] == ignore_pom['artifactId'] and \
|
||||
pom['version'] == ignore_pom['version'] and \
|
||||
filename in [e['filename'] for e in ignore_files]:
|
||||
# artifact is in the ignore list, don't bail out
|
||||
break
|
||||
else:
|
||||
found = False
|
||||
if not ignore_pom:
|
||||
for ignore_file in ignore_files:
|
||||
if filename == ignore_file['filename'] and \
|
||||
fileinfo['size'] == ignore_file['size'] and \
|
||||
fileinfo['md5sum'] == ignore_file['md5sum']:
|
||||
found = True
|
||||
break
|
||||
if found:
|
||||
break
|
||||
else:
|
||||
# artifact was not in the ignore list, raise an error
|
||||
raise koji.BuildrootError, 'unknown file in buildroot: %s; groupId: %s, artifactId: %s, version: %s' % \
|
||||
(filename, pom['groupId'], pom['artifactId'], pom['version'])
|
||||
if pom:
|
||||
raise koji.BuildrootError, 'unknown file in buildroot: %s; groupId: %s, artifactId: %s, version: %s' % \
|
||||
(filename, pom['groupId'], pom['artifactId'], pom['version'])
|
||||
else:
|
||||
raise koji.BuildrootError, 'unknown file in buildroot: %s; size: %s, md5sum: %s' % \
|
||||
(filename, fileinfo['size'], fileinfo['md5sum'])
|
||||
|
||||
return br.updateArchiveList(archives, project)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue