From ba6918f945619f2633e5ba11ea05c40da0d5f1d6 Mon Sep 17 00:00:00 2001 From: Tom Gundersen Date: Thu, 29 Aug 2019 15:31:35 +0200 Subject: [PATCH] osbuild: allow additional an additional build-pipeline to be prepended The best practice for creating a pipeline should be to include at least one level of build-pipelines. This makes sure that the tools used to generate the target image are well-defined. In principle one could add several layers, though in pracite, one would hope that the envinment used to build the buildroot does not affect the final image (and as we anyway cannot recurr indefinitely, we fall back to simply using the host system in this case). This only makes sense, if the contents of the host system truly does not affect the generated image, and as such we do not include any information about the host when computing the hash that identifies a pipeline. In fact, any image could be used in its place, as long as the required tools are present. This commit takes advantage of that fact. Rather than run a pipeline with the host as the build root, take a second pipeline to generate the buildroot, but do not include this when computing the pipeline id (so it is different from simply editing the original JSON). This is necessary so we can use the same pipelines on significantly different host systems (run with different --bulid-pipeline arguments). In particular, it allows our test pipelines that generate f30 images to be run unmodified on Travis (which runs Ubuntu). Signed-off-by: Tom Gundersen --- osbuild/__main__.py | 7 +++++++ osbuild/pipeline.py | 6 ++++++ test/__main__.py | 5 +++++ test/integration_tests/build.py | 8 +++++--- test/integration_tests/test_case.py | 3 ++- 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/osbuild/__main__.py b/osbuild/__main__.py index 64d6d660..04508d07 100755 --- a/osbuild/__main__.py +++ b/osbuild/__main__.py @@ -14,6 +14,8 @@ def main(): parser = argparse.ArgumentParser(description="Build operating system images") parser.add_argument("pipeline_path", metavar="PIPELINE", help="json file containing the pipeline that should be built") + parser.add_argument("--build-pipeline", metavar="PIPELINE", type=os.path.abspath, + help="json file containing the pipeline to create a build environment") parser.add_argument("--store", metavar="DIRECTORY", type=os.path.abspath, default=".osbuild/store", help="the directory where intermediary os trees are stored") @@ -27,6 +29,11 @@ def main(): with open(args.pipeline_path) as f: pipeline = osbuild.load(json.load(f)) + if args.build_pipeline: + with open(args.build_pipeline) as f: + build = osbuild.load(json.load(f)) + pipeline.prepend_build_pipeline(build) + try: pipeline.run(args.output_dir, args.store, interactive=True, libdir=args.libdir) except KeyboardInterrupt: diff --git a/osbuild/pipeline.py b/osbuild/pipeline.py index 07d78ea0..0b3a3921 100644 --- a/osbuild/pipeline.py +++ b/osbuild/pipeline.py @@ -151,6 +151,12 @@ class Pipeline: def set_assembler(self, name, options=None): self.assembler = Assembler(name, options or {}) + def prepend_build_pipeline(self, build): + pipeline = self + while pipeline.build: + pipeline = pipeline.build + pipeline.build = build + def description(self): description = {} if self.build: diff --git a/test/__main__.py b/test/__main__.py index 6bf20172..9d9c7674 100644 --- a/test/__main__.py +++ b/test/__main__.py @@ -31,6 +31,8 @@ def test_firewall(extract_dir): if __name__ == '__main__': parser = argparse.ArgumentParser(description='Run integration tests') parser.add_argument('--list', dest='list', action='store_true', help='list test cases') + parser.add_argument('--build-pipeline', dest='build_pipeline', metavar='PIPELINE', + type=os.path.abspath, help='the build pipeline to run tests in') parser.add_argument('--case', dest='specific_case', metavar='TEST_CASE', help='run single test case') args = parser.parse_args() @@ -41,6 +43,7 @@ if __name__ == '__main__': f30_boot = IntegrationTestCase( name="f30-boot", pipeline="f30-boot.json", + build_pipeline=args.build_pipeline, output_image="f30-boot.qcow2", test_cases=[test_is_system_running], type=IntegrationTestType.BOOT_WITH_QEMU @@ -48,6 +51,7 @@ if __name__ == '__main__': timezone = IntegrationTestCase( name="timezone", pipeline="timezone.json", + build_pipeline=args.build_pipeline, output_image="timezone.tar.xz", test_cases=[test_timezone], type=IntegrationTestType.EXTRACT @@ -55,6 +59,7 @@ if __name__ == '__main__': firewall = IntegrationTestCase( name="firewall", pipeline="firewall.json", + build_pipeline=args.build_pipeline, output_image="firewall.tar.xz", test_cases=[test_firewall], type=IntegrationTestType.EXTRACT diff --git a/test/integration_tests/build.py b/test/integration_tests/build.py index 09a572b0..f8802086 100644 --- a/test/integration_tests/build.py +++ b/test/integration_tests/build.py @@ -5,8 +5,10 @@ import sys from .config import * -def run_osbuild(pipeline: str, check=True): +def run_osbuild(pipeline: str, build_pipeline: str, check=True): cmd = OSBUILD + ["--store", OBJECTS, "-o", OUTPUT_DIR, pipeline] + if build_pipeline: + cmd += ["--build-pipeline", build_pipeline] logging.info(f"Running osbuild: {cmd}") osbuild = subprocess.run(cmd, capture_output=True) if osbuild.returncode != 0: @@ -21,5 +23,5 @@ def run_osbuild(pipeline: str, check=True): return osbuild.returncode -def build_testing_image(pipeline_full_path): - run_osbuild(pipeline_full_path) +def build_testing_image(pipeline_full_path, build_pipeline_full_path): + run_osbuild(pipeline_full_path, build_pipeline_full_path) diff --git a/test/integration_tests/test_case.py b/test/integration_tests/test_case.py index bc2bb2da..e6f03970 100644 --- a/test/integration_tests/test_case.py +++ b/test/integration_tests/test_case.py @@ -16,12 +16,13 @@ class IntegrationTestType(Enum): class IntegrationTestCase: name: str pipeline: str + build_pipeline: str output_image: str test_cases: List[Callable[[Any], None]] type: IntegrationTestType def run(self): - run_osbuild(rel_path(f"pipelines/{self.pipeline}")) + run_osbuild(rel_path(f"pipelines/{self.pipeline}"), self.build_pipeline) if self.type == IntegrationTestType.BOOT_WITH_QEMU: self.run_and_test() else: