From cb5515aca482b69f8298f1765858c8df875ed4bf Mon Sep 17 00:00:00 2001 From: Jana Cupova Date: Wed, 24 Aug 2022 10:30:14 +0200 Subject: [PATCH] Order download tasks and download only first srpm Fixes: https://pagure.io/koji/issue/3461 --- cli/koji_cli/commands.py | 16 ++++++++++++++-- tests/test_cli/test_download_task.py | 19 +++++++++---------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index f0e0ca2d..b3476833 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -6932,6 +6932,7 @@ def anon_handle_download_task(options, session, args): list_tasks = [base_task] if not suboptions.parentonly: list_tasks.extend(session.getTaskChildren(base_task_id)) + list_tasks = sorted(list_tasks, key=lambda k: k['id']) required_tasks = {} for task in list_tasks: @@ -7020,6 +7021,7 @@ def anon_handle_download_task(options, session, args): # perform the download number = 0 pathinfo = koji.PathInfo(topdir=suboptions.topurl) + srpm_downloaded = [] for (task, filename, volume, new_filename, task_id) in downloads: if suboptions.dirpertask: koji.ensuredir(task_id) @@ -7034,8 +7036,18 @@ def anon_handle_download_task(options, session, args): if '..' in filename: error('Invalid file name: %s' % filename) url = '%s/%s/%s' % (pathinfo.work(volume), pathinfo.taskrelpath(task["id"]), filename) - download_file(url, new_filename, quiet=suboptions.quiet, noprogress=suboptions.noprogress, - size=len(downloads), num=number) + if not new_filename.endswith('src.rpm'): + download_file(url, new_filename, quiet=suboptions.quiet, + noprogress=suboptions.noprogress, size=len(downloads), num=number) + else: + if (new_filename, volume) not in srpm_downloaded: + download_file(url, new_filename, quiet=suboptions.quiet, + noprogress=suboptions.noprogress, size=len(downloads), num=number) + srpm_downloaded.append((new_filename, volume)) + else: + if not suboptions.quiet: + print("Downloading [%d/%d] %s" % (number, len(downloads), new_filename)) + print("File %s already downloaded, skipping" % new_filename) def anon_handle_wait_repo(options, session, args): diff --git a/tests/test_cli/test_download_task.py b/tests/test_cli/test_download_task.py index 21baf8e8..fc1b5448 100644 --- a/tests/test_cli/test_download_task.py +++ b/tests/test_cli/test_download_task.py @@ -53,7 +53,7 @@ class TestDownloadTask(utils.CliTestCase): self.ensure_connection = mock.patch('koji_cli.commands.ensure_connection').start() self.stdout = mock.patch('sys.stdout', new_callable=six.StringIO).start() self.stderr = mock.patch('sys.stderr', new_callable=six.StringIO).start() - self.parent_task_id = 123333 + self.parent_task_id = 123 self.parent_task_info = {'id': self.parent_task_id, 'method': 'buildArch', 'arch': 'taskarch', 'state': 2, 'parent': None} self.error_format = """Usage: %s download-task @@ -78,7 +78,7 @@ Default behavior without --all option downloads .rpm files only for build and bu 'somelog.log': ['DEFAULT', 'vol1']} calls = self.gen_calls(self.list_task_output_all_volumes.return_value, - 'https://topurl/%swork/tasks/3333/123333/%s', + 'https://topurl/%swork/tasks/123/123/%s', ['somelog.log']) # Run it and check immediate output @@ -159,7 +159,7 @@ Default behavior without --all option downloads .rpm files only for build and bu self.session.getTaskInfo.assert_called_once_with(self.parent_task_id) self.session.getTaskChildren.assert_called_once_with(self.parent_task_id) self.assertEqual(self.list_task_output_all_volumes.mock_calls, [ - call(self.session, 123333), + call(self.session, 123), call(self.session, 22222), call(self.session, 33333), call(self.session, 44444), @@ -259,7 +259,7 @@ Default behavior without --all option downloads .rpm files only for build and bu self.assert_system_exit( anon_handle_download_task, self.options, self.session, args, - stderr="Task 123333 has not finished yet.\n", + stderr="Task 123 has not finished yet.\n", stdout='', activate_session=None, exit_code=1) @@ -422,7 +422,7 @@ Options: self.session.getTaskInfo.assert_called_once_with(self.parent_task_id) self.session.getTaskChildren.assert_called_once_with(self.parent_task_id) self.assertEqual(self.list_task_output_all_volumes.mock_calls, [ - call(self.session, 123333), + call(self.session, 123), call(self.session, 22222), call(self.session, 33333), call(self.session, 44444), @@ -483,7 +483,7 @@ Options: self.session.getTaskInfo.assert_called_once_with(self.parent_task_id) self.session.getTaskChildren.assert_called_once_with(self.parent_task_id) self.assertEqual(self.list_task_output_all_volumes.mock_calls, [ - call(self.session, 123333), + call(self.session, 123), call(self.session, 22222), call(self.session, 33333), call(self.session, 44444), @@ -758,7 +758,7 @@ Options: } calls = self.gen_calls(self.list_task_output_all_volumes.return_value, - 'https://topurl/%swork/tasks/3333/123333/%s', + 'https://topurl/%swork/tasks/123/123/%s', ['somelog.log', 'somefile.json']) # Run it and check immediate output @@ -811,7 +811,8 @@ Options: rv = anon_handle_download_task(self.options, self.session, args) actual = self.stdout.getvalue() - expected = '' + expected = 'Downloading [3/9] somerpm.src.rpm\n' \ + 'File somerpm.src.rpm already downloaded, skipping\n' self.assertMultiLineEqual(actual, expected) # Finally, assert that things were called as we expected. self.ensure_connection.assert_called_once_with(self.session, self.options) @@ -828,8 +829,6 @@ Options: 'somerpm.src.rpm', quiet=None, noprogress=None, size=9, num=1), call('https://topurl/work/tasks/2222/22222/somerpm.noarch.rpm', 'somerpm.noarch.rpm', quiet=None, noprogress=None, size=9, num=2), - call('https://topurl/work/tasks/3333/33333/somerpm.src.rpm', - 'somerpm.src.rpm', quiet=None, noprogress=None, size=9, num=3), call('https://topurl/work/tasks/3333/33333/somerpm.x86_64.rpm', 'somerpm.x86_64.rpm', quiet=None, noprogress=None, size=9, num=4), call('https://topurl/vol/vol2/work/tasks/3333/33333/somerpm.x86_64.rpm',