particle-os/debian-koji
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Make it possible to rebuild a Maven project without changing the version

Browse source 
- synthesize the release value by incrementing the release of the previous successful build with the same name and version
 - add an extra level to the storage hierarchy for Maven builds so they can be differentiated by release
 - create the per-tag Maven repos by symlinking into the storage hierarchy, rather than copying files
 - significant refactoring of Maven build environment population and tracking to differentiate between builds of the same version
This commit is contained in:
Mike Bonnet 2008-11-13 23:49:14 -05:00
parent d402a7d433
commit 6e80a7b226
4 changed files with 177 additions and 131 deletions
Download patch file Download diff file Expand all files Collapse all files

73
builder/kojid
View file

@ -594,54 +594,37 @@ class BuildRoot(object):
to satisfy build requirements.
Each member of the list is a dictionary containing the following fields:
- pom: a dict of Maven POM info containing the groupId, artifactId, and version fields
- maven_info: a dict of Maven info containing the groupId, artifactId, and version fields
- files: a list of files associated with that POM
"""
packages = []
for path, dirs, files in os.walk(repodir):
pom_info = None
relpath = path[len(repodir) + 1:]
maven_files = []
for repofile in files:
if repofile.endswith('.pom'):
# Found a pom file, process it
try:
pom_info = koji.parse_pom(path=os.path.join(path, repofile))
maven_files.append({'filename': repofile})
except:
self.logger.error('Could not parse pom: %s' % repofile)
self.logger.error(''.join(traceback.format_exception(*sys.exc_info())))
continue
elif os.path.splitext(repofile)[1] in ('.md5', '.sha1'):
if os.path.splitext(repofile)[1] in ('.md5', '.sha1'):
# generated by the Koji, not tracked
continue
elif repofile.startswith('maven-metadata') and repofile.endswith('.xml'):
elif fnmatch(repofile, 'maven-metadata*.xml'):
# Maven throws these metadata files around all over the place, ignore them
continue
elif fnmatch(repofile, 'koji-task*.zip'):
# special-case the archives of the sources and patches, since we drop them in
# root of the output directory
continue
else:
maven_files.append({'filename': repofile})
if pom_info:
packages.append({'pom': pom_info, 'files': maven_files})
elif maven_files:
# get the filesize and md5sum of the files so the hub might be able to identify them
# if the hub can't identify them, it'll throw an error (we don't want untracked files
# in the build environment)
orphan_info = []
for orphan in maven_files:
orphan_path = os.path.join(path, orphan['filename'])
orphan_size = os.stat(orphan_path).st_size
sum = md5.new()
orphan_fd = file(orphan_path)
while True:
contents = orphan_fd.read(65536)
if contents:
sum.update(contents)
else:
break
orphan_fd.close()
orphan_sum = sum.hexdigest()
orphan_info.append({'filename': orphan['filename'], 'size': orphan_size,
'md5sum': orphan_sum})
packages.append({'pom': None, 'files': orphan_info})
maven_files.append({'path': relpath, 'filename': repofile,
'size': os.path.getsize(os.path.join(path, repofile))})
if maven_files:
path_comps = relpath.split('/')
if len(path_comps) < 3:
raise koji.BuildrootError, 'Files found in unexpected path in local Maven repo: %s, files: %s' % \
(relpath, ', '.join([f['filename'] for f in maven_files]))
# extract the Maven info from the path within the local repo
maven_info = {'version': path_comps[-1],
'artifact_id': path_comps[-2],
'group_id': '.'.join(path_comps[:-2])}
packages.append({'maven_info': maven_info, 'files': maven_files})
return packages
@ -658,16 +641,16 @@ class BuildRoot(object):
self.expire()
raise koji.BuildrootError, 'error resolving plugin dependencies, %s' % self._mockResult(rv, logfile='root.log')
plugin_deps = self.getMavenPackageList(repodir)
session.host.updateMavenBuildRootList(self.id, self.task_id, plugin_deps, project=False)
session.host.updateMavenBuildRootList(self.id, self.task_id,
self.getMavenPackageList(repodir), project=False)
cmd.extend(['deploy', '-DaltDeploymentRepository=koji-output::default::file://%s' % outputdir[len(self.rootdir()):]])
rv = self.mock(cmd, skip_setarch=True)
# ignore plugin dependencies (we've already recorded them) and
# plugin dependencies will be ignored
# newly-built archives we find in the repo (we'll import them soon)
session.host.updateMavenBuildRootList(self.id, self.task_id, self.getMavenPackageList(repodir),
ignore=(plugin_deps + self.getMavenPackageList(outputdir)),
ignore=self.getMavenPackageList(outputdir),
project=True)
if rv:
self.expire()
@ -2124,7 +2107,7 @@ class MavenTask(BaseTaskHandler):
raise koji.BuildError, "package %s is blocked for tag %s" \
% (build_info['name'], dest_tag['name'])
self.build_id = session.host.initMavenBuild(self.id, build_info, maven_info)
self.build_id, build_info = session.host.initMavenBuild(self.id, build_info, maven_info)
try:
rpm_results = None
@ -2208,6 +2191,7 @@ class BuildMavenTask(BaseTaskHandler):
outputdir = '%s/maven/output' % buildroot.rootdir()
repodir = '%s/maven/repo' % buildroot.rootdir()
patchdir = '%s/maven/patches' % buildroot.rootdir()
koji.ensuredir(scmdir)
koji.ensuredir(outputdir)
koji.ensuredir(repodir)
@ -2372,9 +2356,9 @@ class WrapperRPMTask(BaseTaskHandler):
# get the list of files from the build instead of the task, because the task output directory may
# have already been cleaned up
artifacts = session.listArchives(buildID=build['id'], type='maven')
build_artifacts = session.listArchives(buildID=build['id'], type='maven')
for artifact in artifacts:
for artifact in build_artifacts:
artifact_name = artifact['filename']
base, ext = os.path.splitext(artifact_name)
if ext == '.log':
@ -2407,6 +2391,7 @@ class WrapperRPMTask(BaseTaskHandler):
if pom:
pom_info = koji.parse_pom(contents=pom)
pom_nvr = koji.pom_to_nvr(pom_info)
pom_nvr['release'] = 'scratch'
self.copy_fields(pom_nvr, values, 'epoch', 'name', 'version', 'release')
break
else:

3
docs/schema.sql
View file

@ -613,8 +613,7 @@ CREATE TABLE maven_builds (
build_id INTEGER NOT NULL PRIMARY KEY REFERENCES build(id),
group_id TEXT NOT NULL,
artifact_id TEXT NOT NULL,
version TEXT NOT NULL,
CONSTRAINT maven_builds_un UNIQUE (group_id, artifact_id, version)
version TEXT NOT NULL
) WITHOUT OIDS;
-- Even though we call this archiveinfo, we can probably use it for

221
hub/kojihub.py
View file

@ -1782,6 +1782,48 @@ def get_task_descendents(task, childMap=None, request=False):
get_task_descendents(Task(child['id']), childMap, request)
return childMap
def maven_tag_packages(taginfo, event_id):
"""
Get Maven builds associated with the given tag, following inheritance.
For any parent tags where 'maven_include_all' is true, include all tagged
builds, not just the latest. If there are multiple releases of the same
Maven groupId-artifactId-version, only take the latest release.
"""
logger = logging.getLogger("koji.hub.repo_init")
if not taginfo['maven_support']:
return []
tag_id = taginfo['id']
# Get the latest Maven builds using the normal build resolution logic
builds = readTaggedBuilds(tag_id, event=event_id, inherit=True, latest=True, maven_only=True)
taglist = [tag_id]
taglist += [t['parent_id'] for t in readFullInheritance(tag_id, event=event_id)]
# Check if any tag in the inheritance hierarchy have maven_include_all == True
# If so, pull in all packages directly tagged into that tag as well
for maven_tag_id in taglist:
maven_tag = get_tag(maven_tag_id, strict=True)
if maven_tag['maven_include_all']:
logger.info('Including all packages in %s' % maven_tag['name'])
builds.extend(readTaggedBuilds(maven_tag['id'], event=event_id, inherit=False, latest=False, maven_only=True))
seen = {}
results = []
# Since a Maven repo structure only has room for one version of a given groupId-artifactId, keep the
# first version found via the inheritance/tag-date mechanism, and skip all the rest
for build in builds:
maven_info = {'group_id': build['maven_group_id'],
'artifact_id': build['maven_artifact_id'],
'version': build['maven_version']}
maven_label = koji.mavenLabel(maven_info)
if seen.has_key(maven_label):
logger.info('Skipping duplicate Maven package: %s, build ID: %i' % (maven_label, build['id']))
continue
else:
results.append(build)
seen[maven_label] = True
return results
def repo_init(tag, with_src=False, with_debuginfo=False, event=None):
"""Create a new repo entry in the INIT state, return full repo data
@ -1845,19 +1887,7 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None):
fo.close()
if tinfo['maven_support']:
# Get the latest Maven builds using the normal build resolution logic
maven_builds = dict([(build['id'], build) for build in \
readTaggedBuilds(tag_id, event=event_id, inherit=True, latest=True, maven_only=True)])
taglist = [tag_id]
taglist += [t['parent_id'] for t in readFullInheritance(tag_id, event=event_id)]
# Check if any tag in the inheritance hierarchy have maven_include_all == True
# If so, pull in all packages directly tagged into that tag as well
for maven_tag_id in taglist:
maven_tag = get_tag(maven_tag_id, strict=True)
if maven_tag['maven_include_all']:
logger.info('Including all packages in %s' % maven_tag['name'])
maven_builds.update([(build['id'], build) for build in \
readTaggedBuilds(maven_tag['id'], event=event_id, inherit=False, latest=False, maven_only=True)])
maven_builds = dict([(build['id'], build) for build in maven_tag_packages(tinfo, event_id)])
# commit the transaction now so we don't hold locks in the database while we're creating
# links on the filesystem (which can take a long time)
@ -1915,15 +1945,12 @@ def _populate_maven_repodir(buildinfo, maveninfo, repodir, artifact_dirs):
if not os.path.isdir(srcdir):
# srcdir doesn't exist, so there's nothing to do
return
destdir = maven_pi.mavenbuild(buildinfo, maveninfo)
koji.ensuredir(destdir)
# link all files in srcdir to destdir, including metadata files
for srcfile in os.listdir(srcdir):
srcpath = '%s/%s' % (srcdir, srcfile)
destpath = '%s/%s' % (destdir, srcfile)
if os.path.isfile(srcpath):
os.link(srcpath, destpath)
artifact_dirs.setdefault(os.path.dirname(destdir), []).append(maveninfo)
# trim the release from the Maven storage dir structure
destdir = os.path.dirname(maven_pi.mavenbuild(buildinfo, maveninfo))
koji.ensuredir(os.path.dirname(destdir))
if not os.path.exists(destdir):
os.symlink(srcdir, destdir)
artifact_dirs.setdefault(os.path.dirname(destdir), []).append(maveninfo)
def _write_maven_repo_metadata(destdir, artifacts):
# Sort the list so that the highest version number comes last.
@ -3642,8 +3669,9 @@ def new_maven_build(build, maven_info):
"""
maven_info = maven_info.copy()
maven_nvr = koji.maven_info_to_nvr(maven_info)
maven_nvr['release'] = build['release']
for field in ('name', 'version', 'release', 'epoch'):
for field in ('name', 'version', 'epoch'):
if build[field] != maven_nvr[field]:
raise koji.BuildError, '%s mismatch (build: %s, maven: %s)' % \
(field, build[field], maven_nvr[field])
@ -6907,8 +6935,8 @@ class BuildRoot(object):
tables = ('archiveinfo',)
joins = ('buildroot_archives ON archiveinfo.id = buildroot_archives.archive_id',)
clauses = ('buildroot_archives.buildroot_id = %(id)i',)
columns = ('id', 'type_id', 'build_id', 'archiveinfo.buildroot_id', 'filename', 'size', 'md5sum')
aliases = ('id', 'type_id', 'build_id', 'buildroot_id', 'filename', 'size', 'md5sum')
columns = ('id', 'type_id', 'build_id', 'archiveinfo.buildroot_id', 'filename', 'size', 'md5sum', 'project_dep')
aliases = ('id', 'type_id', 'build_id', 'buildroot_id', 'filename', 'size', 'md5sum', 'project_dep')
query = QueryProcessor(tables=tables, columns=columns,
joins=joins, clauses=clauses,
values=self.data,
@ -7320,21 +7348,47 @@ class HostExports(object):
return result
def initMavenBuild(self, task_id, build_info, maven_info):
"""Create a new in-progress Maven build"""
"""Create a new in-progress Maven build
Synthesize the release number by taking the (integer) release of the
last successful build and incrementing it."""
host = Host()
host.verify()
task = Task(task_id)
task.assertHost(host.id)
build_info['task_id'] = task_id
build_info['owner'] = task.getOwner()
build_info['state'] = koji.BUILD_STATES['BUILDING']
build_info['completion_time'] = None
build_id = new_build(build_info)
# find the last successful build of this N-V
values = {'name': build_info['name'],
'version': build_info['version'],
'state': koji.BUILD_STATES['COMPLETE']}
query = QueryProcessor(tables=['build'], joins=['package ON build.pkg_id = package.id'],
columns=['build.id', 'release'],
clauses=['name = %(name)s', 'version = %(version)s',
'state = %(state)s'],
values=values,
opts={'order': '-build.id'})
result = query.executeOne()
release = None
if result:
release = result['release']
if not release:
release = '1'
elif release.isdigit():
release = str(int(release) + 1)
else:
raise koji.BuildError, 'Invalid release value for a Maven build: %s' % release
build_info['release'] = release
data = build_info.copy()
data['task_id'] = task_id
data['owner'] = task.getOwner()
data['state'] = koji.BUILD_STATES['BUILDING']
data['completion_time'] = None
build_id = new_build(data)
build_info['id'] = build_id
new_maven_build(build_info, maven_info)
return build_id
return build_id, build_info
def completeMavenBuild(self, task_id, build_id, maven_results, rpm_results):
"""Complete the Maven build."""
@ -7566,61 +7620,66 @@ class HostExports(object):
br = BuildRoot(brootid)
br.assertHost(host.id)
br.assertTask(task_id)
repo = repo_info(br.data['repo_id'], strict=True)
tag = get_tag(repo['tag_id'], strict=True)
tag_builds = maven_tag_packages(tag, repo['create_event'])
archives_by_label = {}
for build in tag_builds:
maven_info = {'group_id': build['maven_group_id'],
'artifact_id': build['maven_artifact_id'],
'version': build['maven_version']}
maven_label = koji.mavenLabel(maven_info)
if archives_by_label.has_key(maven_label):
# A previous build has already claimed this groupId-artifactId-version, and thus
# the spot in the filesystem. This build never made it into the build environment.
continue
newly_added_versions = [maven_label]
archives_by_label[maven_label] = {}
for archive in list_archives(buildID=build['id'], type='maven'):
archive_label = koji.mavenLabel(archive)
if archive_label in newly_added_versions:
archives_by_label[archive_label][archive['filename']] = archive
elif not archives_by_label.has_key(archive_label):
# These archives have a different label than their parent build, but that label
# has not been processed yet. These archives may have made their way into the
# build environment.
newly_added_versions.append(archive_label)
archives_by_label[archive_label] = {}
archives_by_label[archive_label][archive['filename']] = archive
else:
# We've already added entries for archives with this label, but associated
# with another build. These archives never made it into the build environment.
continue
if not ignore:
ignore = []
ignore_by_label = dict([(koji.mavenLabel(entry['maven_info']), entry['files']) for entry in ignore])
archives = []
for entry in mavenlist:
pom = entry['pom']
files = entry['files']
if pom:
maven_info = {'group_id': pom['groupId'],
'artifact_id': pom['artifactId'],
'version': pom['version']}
for fileinfo in files:
filename = fileinfo['filename']
if pom:
archiveinfos = find_maven_archives(maven_info, filename)
maven_info = entry['maven_info']
maven_label = koji.mavenLabel(maven_info)
tag_archives = archives_by_label.get(maven_label, {})
for fileinfo in entry['files']:
tag_archive = tag_archives.get(fileinfo['filename'])
if tag_archive and fileinfo['size'] == tag_archive['size']:
archives.append(tag_archive)
else:
archiveinfos = list_archives(filename=filename, size=fileinfo['size'],
md5sum=fileinfo['md5sum'])
if archiveinfos:
if len(archiveinfos) == 1:
archives.append(archiveinfos[0])
# check the ignore list
ignore = False
ignore_archives = ignore_by_label.get(maven_label, [])
for ignore_archive in ignore_archives:
if ignore_archive['filename'] == fileinfo['filename'] and \
ignore_archive['size'] == fileinfo['size']:
ignore = True
break
if ignore:
continue
else:
raise koji.BuildrootError, 'multiple matches for %s; archive IDs: %s' % \
(filename, ', '.join([str(a['id']) for a in archiveinfos]))
else:
# check if it's in the ignore list
for ignore_entry in ignore:
ignore_pom = ignore_entry['pom']
ignore_files = ignore_entry['files']
if pom:
if ignore_pom:
if pom['groupId'] == ignore_pom['groupId'] and \
pom['artifactId'] == ignore_pom['artifactId'] and \
pom['version'] == ignore_pom['version'] and \
filename in [e['filename'] for e in ignore_files]:
# artifact is in the ignore list, don't bail out
break
else:
found = False
if not ignore_pom:
for ignore_file in ignore_files:
if filename == ignore_file['filename'] and \
fileinfo['size'] == ignore_file['size'] and \
fileinfo['md5sum'] == ignore_file['md5sum']:
found = True
break
if found:
break
else:
# artifact was not in the ignore list, raise an error
if pom:
raise koji.BuildrootError, 'unknown file in buildroot: %s; groupId: %s, artifactId: %s, version: %s' % \
(filename, pom['groupId'], pom['artifactId'], pom['version'])
else:
raise koji.BuildrootError, 'unknown file in buildroot: %s; size: %s, md5sum: %s' % \
(filename, fileinfo['size'], fileinfo['md5sum'])
raise koji.BuildrootError, 'Unknown file in build environment: %s, size: %s' % \
('%s/%s' % (fileinfo['path'], fileinfo['filename']), fileinfo['size'])
return br.updateArchiveList(archives, project)

11
koji/__init__.py
View file

@ -806,7 +806,7 @@ def pom_to_nvr(pominfo):
The pom-to-nvr mapping is as follows:
- name: groupId + '-' + artifactId
- version: version.replace('-', '_')
- release: 1
- release: None
- epoch: None
"""
maveninfo = pom_to_maven_info(pominfo)
@ -831,10 +831,12 @@ def maven_info_to_nvr(maveninfo):
"""
Convert the maveninfo to NVR-compatible format.
Uses the same mapping as pom_to_nvr().
The release cannot be determined from Maven metadata, and will
be set to None.
"""
nvr = {'name': maveninfo['group_id'] + '-' + maveninfo['artifact_id'],
'version': maveninfo['version'].replace('-', '_'),
'release': '1',
'release': None,
'epoch': None}
# for backwards-compatibility
nvr['package_name'] = nvr['name']
@ -1213,10 +1215,11 @@ class PathInfo(object):
def mavenbuild(self, build, maveninfo):
"""Return the directory where a maven build belongs"""
group_id = maveninfo['group_id'].replace('.', '/')
group_path = maveninfo['group_id'].replace('.', '/')
artifact_id = maveninfo['artifact_id']
version = maveninfo['version']
return self.topdir + ("/maven2/%(group_id)s/%(artifact_id)s/%(version)s" % locals())
release = build['release']
return self.topdir + ("/maven2/%(group_path)s/%(artifact_id)s/%(version)s/%(release)s" % locals())
def archive(self, build):
"""Return the directory where the archive belongs"""