From c76e97ddc9cc4ddd42834467dca4c61bca4ea699 Mon Sep 17 00:00:00 2001 From: Tomas Hozza Date: Fri, 29 Jul 2022 17:28:43 +0200 Subject: [PATCH] Support specifying upload options for image builds The upload options are expected to be provided as a JSON file. The same options will be used for all image type and architecture combinations, similarly as it is done for ostree options. Extend unit tests to cover the newly added functionality. --- plugins/builder/osbuild.py | 9 ++++ plugins/cli/osbuild.py | 7 +++ plugins/hub/osbuild.py | 94 ++++++++++++++++++++++++++++++++++++++ test/unit/test_builder.py | 41 +++++++++++++++++ 4 files changed, 151 insertions(+) diff --git a/plugins/builder/osbuild.py b/plugins/builder/osbuild.py index 3b608bc..45736b9 100644 --- a/plugins/builder/osbuild.py +++ b/plugins/builder/osbuild.py @@ -116,6 +116,7 @@ class ImageRequest: self.image_type = image_type self.repositories = repos self.ostree: Optional[OSTreeOptions] = None + self.upload_options: Optional[Dict] = None def as_dict(self): arch = self.architecture @@ -128,6 +129,8 @@ class ImageRequest: } if self.ostree: res["ostree"] = self.ostree.as_dict(self.architecture) + if self.upload_options: + res["upload_options"] = self.upload_options return res @@ -688,6 +691,12 @@ class OSBuildImage(BaseTaskHandler): for ireq in ireqs: ireq.ostree = ostree + # Cloud upload options + upload_options = opts.get("upload_options") + if upload_options: + for ireq in ireqs: + ireq.upload_options = upload_options + self.logger.debug("Creating compose: %s (%s)\n koji: %s\n images: %s", nvr, distro, self.koji_url, str([i.as_dict() for i in ireqs])) diff --git a/plugins/cli/osbuild.py b/plugins/cli/osbuild.py index 6fc1f77..ea637d2 100755 --- a/plugins/cli/osbuild.py +++ b/plugins/cli/osbuild.py @@ -49,6 +49,8 @@ def parse_args(argv): parser.add_option("--customizations", type=str, default=None, dest="customizations", help="Additional customizations to pass to Composer (json file)") + parser.add_option("--upload-options", type=str, default=None, dest="upload_options", + help="Cloud target upload options (json file)") parser.add_option("--nowait", action="store_false", dest="wait", help="Don't wait on image creation") parser.add_option("--ostree-parent", type=str, dest="ostree_parent", @@ -140,6 +142,11 @@ def handle_osbuild_image(options, session, argv): with open(args.customizations, "r", encoding="utf-8") as f: opts["customizations"] = json.load(f) + # cloud upload options handling + if args.upload_options: + with open(args.upload_options, "r", encoding="utf-8") as f: + opts["upload_options"] = json.load(f) + # Do some early checks to be able to give quick feedback check_target(session, target) diff --git a/plugins/hub/osbuild.py b/plugins/hub/osbuild.py index 615b73e..5a8635f 100644 --- a/plugins/hub/osbuild.py +++ b/plugins/hub/osbuild.py @@ -94,6 +94,15 @@ OSBUILD_IMAGE_SCHEMA = { "type": "object", "$ref": "#/definitions/ostree" }, + "upload_options": { + "oneOf": [ + {"$ref": "#/definitions/AWSEC2UploadOptions"}, + {"$ref": "#/definitions/AWSS3UploadOptions"}, + {"$ref": "#/definitions/GCPUploadOptions"}, + {"$ref": "#/definitions/AzureUploadOptions"}, + {"$ref": "#/definitions/ContainerUploadOptions"} + ], + }, "repo": { "type": "array", "description": "Repositories", @@ -113,6 +122,91 @@ OSBUILD_IMAGE_SCHEMA = { "description": "Omit tagging the result" } } + }, + "AWSEC2UploadOptions": { + "type": "object", + "additionalProperties": False, + "required": ["region", "share_with_accounts"], + "properties": { + "region": { + "type": "string", + }, + "snapshot_name": { + "type": "string", + }, + "share_with_accounts": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "AWSS3UploadOptions": { + "type": "object", + "additionalProperties": False, + "required": ["region"], + "properties": { + "region": { + "type": "string" + } + } + }, + "AzureUploadOptions": { + "type": "object", + "additionalProperties": False, + "required": ["tenant_id", "subscription_id", "resource_group", "location"], + "properties": { + "tenant_id": { + "type": "string" + }, + "subscription_id": { + "type": "string" + }, + "resource_group": { + "type": "string" + }, + "location": { + "type": "string" + }, + "image_name": { + "type": "string", + } + } + }, + "GCPUploadOptions": { + "type": "object", + "additionalProperties": False, + "required": ["region", "bucket"], + "properties": { + "region": { + "type": "string" + }, + "bucket": { + "type": "string" + }, + "image_name": { + "type": "string", + }, + "share_with_accounts": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "ContainerUploadOptions": { + "type": "object", + "additionalProperties": False, + "properties": { + "name": { + "type": "string" + }, + "tag": { + "type": "string" + } + } } } } diff --git a/test/unit/test_builder.py b/test/unit/test_builder.py index 0c5d2ba..381c31e 100644 --- a/test/unit/test_builder.py +++ b/test/unit/test_builder.py @@ -1199,6 +1199,9 @@ class TestBuilderPlugin(PluginTest): # pylint: disable=too-many-public-methods for ir in ireqs: arch = ir["architecture"] + # Piggyback on this test case to test that no upload_options + # are set, if they were not provided in the args. + self.assertIsNone(ir.get("upload_options")) repos = ir["repositories"] assert len(repos) == 3 @@ -1209,6 +1212,44 @@ class TestBuilderPlugin(PluginTest): # pylint: disable=too-many-public-methods ps = r.get("package_sets") assert ps and ps == ["a", "b", "c", "d"] + @httpretty.activate + def test_compose_upload_options_global(self): + # Check we properly handle compose requests with global upload options + session = self.mock_session() + handler = self.make_handler(session=session) + + arches = ["x86_64", "aarch64"] + upload_options = { + "region": "us-east-1", + "share_with_accounts": ["123456789"] + } + args = ["name", "version", "distro", + "image_type", + "fedora-candidate", + arches, + {"upload_options": upload_options}] + + url = self.plugin.DEFAULT_COMPOSER_URL + composer = MockComposer(url, architectures=arches) + composer.httpretty_register() + + res = handler.handler(*args) + assert res, "invalid compose result" + compose_id = res["composer"]["id"] + compose = composer.composes.get(compose_id) + self.assertIsNotNone(compose) + + ireqs = compose["request"]["image_requests"] + + # Check we got all the requested architectures + ireq_arches = [i["architecture"] for i in ireqs] + diff = set(arches) ^ set(ireq_arches) + self.assertEqual(diff, set()) + + for ir in ireqs: + uo = ir["upload_options"] + self.assertEqual(uo, upload_options) + @httpretty.activate def test_compose_status_retry(self): compose_id = "43e57e63-ab32-4a8d-854d-3bbc117fdce3"