diff --git a/docs/schema-upgrade-1.29-1.30.sql b/docs/schema-upgrade-1.29-1.30.sql new file mode 100644 index 00000000..483b1b1d --- /dev/null +++ b/docs/schema-upgrade-1.29-1.30.sql @@ -0,0 +1,13 @@ +-- upgrade script to migrate the Koji database schema +-- from version 1.29 to 1.30 + + +BEGIN; + +ALTER TABLE archivetypes ADD COLUMN compression_type TEXT; + +UPDATE archivetypes set compression_type='zip' WHERE name = 'jar'; +UPDATE archivetypes set compression_type='zip' WHERE name = 'zip'; +UPDATE archivetypes set compression_type='tar' WHERE name = 'tar'; + +COMMIT; diff --git a/docs/schema.sql b/docs/schema.sql index 860bffcb..41a20e95 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -802,13 +802,14 @@ CREATE TABLE archivetypes ( id SERIAL NOT NULL PRIMARY KEY, name TEXT NOT NULL UNIQUE, description TEXT NOT NULL, - extensions TEXT NOT NULL + extensions TEXT NOT NULL, + compression_type TEXT ) WITHOUT OIDS; -INSERT INTO archivetypes (name, description, extensions) VALUES ('jar', 'Jar file', 'jar war rar ear sar jdocbook jdocbook-style'); -INSERT INTO archivetypes (name, description, extensions) VALUES ('zip', 'Zip file', 'zip'); +INSERT INTO archivetypes (name, description, extensions, compression_type) VALUES ('jar', 'Jar file', 'jar war rar ear sar jdocbook jdocbook-style', 'zip'); +INSERT INTO archivetypes (name, description, extensions, compression_type) VALUES ('zip', 'Zip file', 'zip', 'zip'); INSERT INTO archivetypes (name, description, extensions) VALUES ('pom', 'Maven Project Object Management file', 'pom'); -INSERT INTO archivetypes (name, description, extensions) VALUES ('tar', 'Tar file', 'tar tar.gz tar.bz2 tar.xz tgz'); +INSERT INTO archivetypes (name, description, extensions, compression_type) VALUES ('tar', 'Tar file', 'tar tar.gz tar.bz2 tar.xz tgz', 'tar'); INSERT INTO archivetypes (name, description, extensions) VALUES ('xml', 'XML file', 'xml'); INSERT INTO archivetypes (name, description, extensions) VALUES ('xmlcompressed', 'Compressed XML file', 'xml.gz xml.bz2 xml.xz'); INSERT INTO archivetypes (name, description, extensions) VALUES ('xsd', 'XML Schema Definition', 'xsd'); diff --git a/hub/kojihub.py b/hub/kojihub.py index 5789236b..9642b5e3 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -5031,6 +5031,7 @@ def list_archives(buildID=None, buildrootID=None, componentBuildrootID=None, hos ('archivetypes.name', 'type_name'), ('archivetypes.description', 'type_description'), ('archivetypes.extensions', 'type_extensions'), + ('archivetypes.compression_type', 'compression_type') ] clauses = [] @@ -5337,9 +5338,9 @@ def list_archive_files(archive_id, queryOpts=None, strict=False): # should not happen raise koji.GenericError("Missing build type info for archive %s" % archive_id) - if archive_type['name'] in ('zip', 'jar'): + if archive_type['compression_type'] == 'zip': filelist = _get_zipfile_list(archive_id, file_path) - elif archive_type['name'] == 'tar': + elif archive_type['compression_type'] == 'tar': filelist = _get_tarball_list(archive_id, file_path) else: # TODO: support other archive types @@ -7391,21 +7392,23 @@ def merge_scratch(task_id): def get_archive_types(): """Return a list of all supported archive types.""" - select = """SELECT id, name, description, extensions FROM archivetypes + select = """SELECT id, name, description, extensions, compression_type FROM archivetypes ORDER BY id""" - return _multiRow(select, {}, ('id', 'name', 'description', 'extensions')) + return _multiRow(select, {}, ('id', 'name', 'description', 'extensions', 'compression_type')) def _get_archive_type_by_name(name, strict=True): - select = """SELECT id, name, description, extensions FROM archivetypes + select = """SELECT id, name, description, extensions, compression_type FROM archivetypes WHERE name = %(name)s""" - return _singleRow(select, locals(), ('id', 'name', 'description', 'extensions'), strict) + return _singleRow(select, locals(), + ('id', 'name', 'description', 'extensions', 'compression_type'), strict) def _get_archive_type_by_id(type_id, strict=False): - select = """SELECT id, name, description, extensions FROM archivetypes + select = """SELECT id, name, description, extensions, compression_type FROM archivetypes WHERE id = %(type_id)i""" - return _singleRow(select, locals(), ('id', 'name', 'description', 'extensions'), strict) + return _singleRow(select, locals(), + ('id', 'name', 'description', 'extensions', 'compression_type'), strict) def get_archive_type(filename=None, type_name=None, type_id=None, strict=False): @@ -7424,7 +7427,7 @@ def get_archive_type(filename=None, type_name=None, type_id=None, strict=False): parts = filename.split('.') query = QueryProcessor( tables=['archivetypes'], - columns=['id', 'name', 'description', 'extensions'], + columns=['id', 'name', 'description', 'extensions', 'compression_type'], clauses=['extensions ~* %(pattern)s'], ) for start in range(len(parts) - 1, -1, -1): @@ -7444,7 +7447,7 @@ def get_archive_type(filename=None, type_name=None, type_id=None, strict=False): return None -def add_archive_type(name, description, extensions): +def add_archive_type(name, description, extensions, compression_type=None): """ Add new archive type. @@ -7459,21 +7462,25 @@ def add_archive_type(name, description, extensions): verify_name_internal(name) convert_value(description, cast=str, check_only=True) convert_value(extensions, cast=str, check_only=True) + convert_value(compression_type, cast=str, none_allowed=True, check_only=True) + if compression_type not in ['zip', 'tar', None]: + raise koji.GenericError(f"Unsupported compression type {compression_type}") data = {'name': name, 'description': description, 'extensions': extensions, + 'compression_type': compression_type, } if get_archive_type(type_name=name): - raise koji.GenericError("archivetype %s already exists" % name) + raise koji.GenericError(f"archivetype {name} already exists") # No invalid or duplicate extensions for ext in extensions.split(' '): if not ext.replace('.', '').isalnum(): - raise koji.GenericError('No such %s file extension' % ext) + raise koji.GenericError(f'No such {ext} file extension') select = r"""SELECT id FROM archivetypes WHERE extensions ~* E'(\\s|^)%s(\\s|$)'""" % ext results = _multiRow(select, {}, ('id',)) if len(results) > 0: - raise koji.GenericError('file extension %s already exists' % ext) + raise koji.GenericError(f'file extension {ext} already exists') insert = InsertProcessor('archivetypes', data=data) insert.execute() diff --git a/tests/test_hub/test_add_archivetype.py b/tests/test_hub/test_add_archivetype.py index 6081c420..41dd944a 100644 --- a/tests/test_hub/test_add_archivetype.py +++ b/tests/test_hub/test_add_archivetype.py @@ -34,7 +34,7 @@ class TestAddArchiveType(unittest.TestCase): self.inserts.append(insert) return insert - def test_add_archive_type_valid(self): + def test_add_archive_type_valid_empty_compression_type(self): self.verify_name_internal.return_value = None self.get_archive_type.return_value = None kojihub.add_archive_type('deb', 'Debian package', 'deb') @@ -44,7 +44,23 @@ class TestAddArchiveType(unittest.TestCase): self.assertEqual(insert.table, 'archivetypes') self.assertEqual(insert.data, {'name': 'deb', 'description': 'Debian package', - 'extensions': 'deb'}) + 'extensions': 'deb', + 'compression_type': None}) + self.assertEqual(insert.rawdata, {}) + self.context.session.assertPerm.assert_called_with('admin') + + def test_add_archive_type_valid_with_compression_type(self): + self.verify_name_internal.return_value = None + self.get_archive_type.return_value = None + kojihub.add_archive_type('jar', 'Jar package', 'jar', 'zip') + + self.assertEqual(len(self.inserts), 1) + insert = self.inserts[0] + self.assertEqual(insert.table, 'archivetypes') + self.assertEqual(insert.data, {'name': 'jar', + 'description': 'Jar package', + 'extensions': 'jar', + 'compression_type': 'zip'}) self.assertEqual(insert.rawdata, {}) self.context.session.assertPerm.assert_called_with('admin') diff --git a/tests/test_hub/test_list_archive_files.py b/tests/test_hub/test_list_archive_files.py index 87018e6c..cb046979 100644 --- a/tests/test_hub/test_list_archive_files.py +++ b/tests/test_hub/test_list_archive_files.py @@ -6,7 +6,7 @@ import kojihub GET_ARCHIVE_RV = {'id': 1, 'build_id': 2, 'type_id': 3, 'btype': 'btype', 'filename': 'somearchive.zip'} -GET_ARCHIVE_TYPE_RV = {'id': 3, 'name': 'zip'} +GET_ARCHIVE_TYPE_RV = {'id': 3, 'name': 'zip', 'compression_type': 'zip'} GET_BUILD_RV = {'id': 2, 'name': 'somebuild', 'version': '1.2.3', 'release': '1.el6', 'volume_name': 'archive_vol'} @@ -22,119 +22,165 @@ class TestListArchiveFiles(unittest.TestCase): return_value=GET_ARCHIVE_TYPE_RV.copy()).start() self.mm.get_archive = mock.patch('kojihub.get_archive', return_value=GET_ARCHIVE_RV.copy()).start() + self.mm.get_maven_archive = mock.patch('kojihub.get_maven_archive').start() + self.mm.get_win_archive = mock.patch('kojihub.get_win_archive').start() + self.mm.get_image_archive = mock.patch('kojihub.get_image_archive').start() + self.mm.get_zipfile_list = mock.patch('kojihub._get_zipfile_list').start() + self.mm.get_tarball_list = mock.patch('kojihub._get_tarball_list').start() + self.maven_archive = {'archive_id': 1, 'group_id': 'gid', 'artifact_id': 'aid', + 'version': '1.0.0'} + self.win_archive = {'archive_id': 1, 'relpath': 'rpath', 'platform': 'all', + 'version': 'src'} + self.image_archive = {'archive_id': 1, 'arch': 'noarch'} + self.zipfile_list = [ + {'archive_id': 1, 'name': 'file1', 'size': 4096, 'mtime': 1000}, + {'archive_id': 1, 'name': 'file2', 'size': 512000, 'mtime': 103420}, + ] + self.tarball_list = [ + {'archive_id': 1, 'name': 'file1', 'size': 4096, 'mtime': 1000, 'mode': '0755', + 'user': 1000, 'group': 1000}, + {'archive_id': 1, 'name': 'file2', 'size': 512000, 'mtime': 103420, 'mode': '0644', + 'user': 1001, 'group': 1001} + ] + self.archive_id = 1 def tearDown(self): mock.patch.stopall() def test_simple(self): - rv = kojihub.list_archive_files(1) - self.mm.get_archive.assert_called_once_with(1, strict=True) + self.mm.get_zipfile_list.return_value = [] + rv = kojihub.list_archive_files(self.archive_id) + self.mm.get_archive.assert_called_once_with(self.archive_id, strict=True) self.mm.get_archive_type.assert_called_once_with(type_id=3, strict=True) self.mm.get_build.assert_called_once_with(2, strict=True) + self.mm.get_zipfile_list.assert_called_once_with( + 1, + '/mnt/koji/vol/archive_vol/packages/somebuild/1.2.3/1.el6/files/btype/somearchive.zip') + self.mm.get_maven_archive.assert_not_called() + self.mm.get_win_archive.assert_not_called() + self.mm.get_image_archive.assert_not_called() + self.mm.get_tarball_list.assert_not_called() self.assertListEqual(rv, []) - @mock.patch('kojihub.get_maven_archive', - return_value={'archive_id': 1, - 'group_id': 'gid', - 'artifact_id': 'aid', - 'version': '1.0.0'}) - @mock.patch('kojihub._get_zipfile_list', return_value=[]) - def test_simple_strict_empty(self, get_zipfile_list, get_maven_archive): - rv = kojihub.list_archive_files(1, strict=True) + def test_simple_strict_empty(self): + self.mm.get_zipfile_list.return_value = [] + rv = kojihub.list_archive_files(self.archive_id, strict=True) + self.mm.get_archive.assert_called_once_with(self.archive_id, strict=True) + self.mm.get_archive_type.assert_called_once_with(type_id=3, + strict=True) + self.mm.get_build.assert_called_once_with(2, strict=True) + self.mm.get_zipfile_list.assert_called_once_with( + 1, + '/mnt/koji/vol/archive_vol/packages/somebuild/1.2.3/1.el6/files/btype/somearchive.zip') + self.mm.get_maven_archive.assert_not_called() + self.mm.get_win_archive.assert_not_called() + self.mm.get_image_archive.assert_not_called() + self.mm.get_tarball_list.assert_not_called() self.assertListEqual(rv, []) def test_simple_strict_missing_btype(self): self.mm.get_archive.return_value['btype'] = None with self.assertRaises(koji.GenericError) as cm: - kojihub.list_archive_files(1, strict=True) - self.assertEqual(cm.exception.args[0][:18], "Missing build type") + kojihub.list_archive_files(self.archive_id, strict=True) + self.assertEqual(cm.exception.args[0], + f'Missing build type info for archive {self.archive_id}') + self.mm.get_archive.assert_called_once_with(self.archive_id, strict=True) + self.mm.get_archive_type.assert_called_once_with(type_id=3, + strict=True) + self.mm.get_build.assert_called_once_with(2, strict=True) + self.mm.get_maven_archive.assert_not_called() + self.mm.get_win_archive.assert_not_called() + self.mm.get_image_archive.assert_not_called() + self.mm.get_zipfile_list.assert_not_called() + self.mm.get_tarball_list.assert_not_called() - @mock.patch('kojihub.get_maven_archive', - return_value={'archive_id': 1, - 'group_id': 'gid', - 'artifact_id': 'aid', - 'version': '1.0.0'}) - def test_simple_strict_bad_archive_type(self, get_maven_archive): - self.mm.get_archive_type.return_value = {'id': 9, 'name': 'txt'} + def test_simple_strict_bad_archive_type_with_strict(self): + self.mm.get_archive_type.return_value = {'id': 9, 'name': 'txt', 'compression_type': ''} with self.assertRaises(koji.GenericError) as cm: - kojihub.list_archive_files(1, strict=True) + kojihub.list_archive_files(self.archive_id, strict=True) self.assertEqual(cm.exception.args[0], "Unsupported archive type: txt") + self.mm.get_archive.assert_called_once_with(self.archive_id, strict=True) + self.mm.get_archive_type.assert_called_once_with(type_id=3, + strict=True) + self.mm.get_build.assert_called_once_with(2, strict=True) + self.mm.get_maven_archive.assert_not_called() + self.mm.get_win_archive.assert_not_called() + self.mm.get_image_archive.assert_not_called() + self.mm.get_zipfile_list.assert_not_called() + self.mm.get_tarball_list.assert_not_called() - @mock.patch('kojihub.get_maven_archive', - return_value={'archive_id': 1, - 'group_id': 'gid', - 'artifact_id': 'aid', - 'version': '1.0.0'}) - @mock.patch('kojihub._get_zipfile_list', return_value=[{'archive_id': 1, - 'name': 'file1', - 'size': 4096, - 'mtime': 1000}, - {'archive_id': 1, - 'name': 'file2', - 'size': 512000, - 'mtime': 103420}, - ]) - def test_maven_archive(self, get_zipfile_list, get_maven_archive): + def test_simple_strict_bad_archive_type_without_strict(self): + self.mm.get_archive_type.return_value = {'id': 9, 'name': 'txt', 'compression_type': ''} + rv = kojihub.list_archive_files(1) + self.mm.get_archive.assert_called_once_with(self.archive_id, strict=True) + self.mm.get_archive_type.assert_called_once_with(type_id=3, + strict=True) + self.mm.get_build.assert_called_once_with(2, strict=True) + self.mm.get_win_archive.assert_not_called() + self.mm.get_image_archive.assert_not_called() + self.mm.get_zipfile_list.assert_not_called() + self.mm.get_tarball_list.assert_not_called() + self.mm.get_maven_archive.assert_not_called() + self.assertListEqual(rv, []) + + def test_maven_archive(self): + self.mm.get_maven_archive.return_value = self.maven_archive + self.mm.get_zipfile_list.return_value = self.zipfile_list self.mm.get_archive.return_value['btype'] = 'maven' rv = kojihub.list_archive_files(1) - get_maven_archive.assert_called_once_with(1, strict=True) - get_zipfile_list.assert_called_once_with(1, - '/mnt/koji/vol/archive_vol/packages' - '/somebuild/1.2.3/1.el6/maven/gid/aid/1.0.0/somearchive.zip') + self.mm.get_maven_archive.assert_called_once_with(1, strict=True) + self.mm.get_zipfile_list.assert_called_once_with( + 1, + '/mnt/koji/vol/archive_vol/packages/somebuild/1.2.3/1.el6/maven/gid/aid/' + '1.0.0/somearchive.zip') + self.mm.get_archive.assert_called_once_with(self.archive_id, strict=True) + self.mm.get_archive_type.assert_called_once_with(type_id=3, + strict=True) + self.mm.get_build.assert_called_once_with(2, strict=True) + self.mm.get_win_archive.assert_not_called() + self.mm.get_image_archive.assert_not_called() self.assertListEqual(rv, [ {'archive_id': 1, 'mtime': 1000, 'name': 'file1', 'size': 4096}, {'archive_id': 1, 'mtime': 103420, 'name': 'file2', 'size': 512000}]) - @mock.patch('kojihub.get_win_archive', return_value={'archive_id': 1, - 'relpath': 'rpath', - 'platform': 'all', - 'version': 'src'}) - @mock.patch('kojihub._get_zipfile_list', return_value=[{'archive_id': 1, - 'name': 'file1', - 'size': 4096, - 'mtime': 1000}, - {'archive_id': 1, - 'name': 'file2', - 'size': 512000, - 'mtime': 103420}, - ]) - def test_win_archive(self, get_zipfile_list, get_win_archive): + def test_win_archive(self): + self.mm.get_win_archive.return_value = self.win_archive + self.mm.get_zipfile_list.return_value = self.zipfile_list self.mm.get_archive.return_value['btype'] = 'win' rv = kojihub.list_archive_files(1) - get_win_archive.assert_called_once_with(1, strict=True) - get_zipfile_list.assert_called_once_with(1, - '/mnt/koji/vol/archive_vol/packages' - '/somebuild/1.2.3/1.el6/win/rpath/somearchive.zip') + self.mm.get_win_archive.assert_called_once_with(1, strict=True) + self.mm.get_zipfile_list.assert_called_once_with( + 1, + '/mnt/koji/vol/archive_vol/packages/somebuild/1.2.3/1.el6/win/rpath/somearchive.zip') + self.mm.get_archive.assert_called_once_with(self.archive_id, strict=True) + self.mm.get_archive_type.assert_called_once_with(type_id=3, + strict=True) + self.mm.get_build.assert_called_once_with(2, strict=True) + self.mm.get_image_archive.assert_not_called() + self.mm.get_tarball_list.assert_not_called() + self.mm.get_maven_archive.assert_not_called() self.assertListEqual(rv, [ {'archive_id': 1, 'mtime': 1000, 'name': 'file1', 'size': 4096}, {'archive_id': 1, 'mtime': 103420, 'name': 'file2', 'size': 512000}]) - @mock.patch('kojihub.get_image_archive', return_value={'archive_id': 1, - 'arch': 'noarch'}) - @mock.patch('kojihub._get_tarball_list', return_value=[{'archive_id': 1, - 'name': 'file1', - 'size': 4096, - 'mtime': 1000, - 'mode': '0755', - 'user': 1000, - 'group': 1000}, - {'archive_id': 1, - 'name': 'file2', - 'size': 512000, - 'mtime': 103420, - 'mode': '0644', - 'user': 1001, - 'group': 1001} - ]) - def test_image_archive(self, get_tarball_list, get_image_archive): - self.mm.get_archive_type.return_value = {'id': 3, 'name': 'tar'} + def test_image_archive(self): + self.mm.get_image_archive.return_value = self.image_archive + self.mm.get_tarball_list.return_value = self.tarball_list + self.mm.get_archive_type.return_value = {'id': 3, 'name': 'tar', 'compression_type': 'tar'} self.mm.get_archive.return_value['btype'] = 'image' rv = kojihub.list_archive_files(1, queryOpts={'countOnly': True}) - get_image_archive.assert_called_once_with(1, strict=True) - get_tarball_list.assert_called_once_with(1, - '/mnt/koji/vol/archive_vol/packages' - '/somebuild/1.2.3/1.el6/images/somearchive.zip') + self.mm.get_image_archive.assert_called_once_with(1, strict=True) + self.mm.get_tarball_list.assert_called_once_with( + 1, + '/mnt/koji/vol/archive_vol/packages/somebuild/1.2.3/1.el6/images/somearchive.zip') + self.mm.get_archive.assert_called_once_with(self.archive_id, strict=True) + self.mm.get_archive_type.assert_called_once_with(type_id=3, + strict=True) + self.mm.get_build.assert_called_once_with(2, strict=True) + self.mm.get_win_archive.assert_not_called() + self.mm.get_zipfile_list.assert_not_called() + self.mm.get_maven_archive.assert_not_called() self.assertEqual(rv, 2)