osbuild: store outputs in objectstore
Treat outputs like we treat trees: store them in the object store. This simplifies using osbuild and allows returning a cached version if one is available. This makes the `--output` parameter redundant. Remove it.
This commit is contained in:
Lars Karlitski
parent
cb173f7d3c
commit
83475cc9f4
9 changed files with 27 additions and 34 deletions
|
|
@ -21,8 +21,8 @@ jobs:
|
|||
- name: pipeline-noop
|
||||
before_install: sudo apt-get install -y systemd-container
|
||||
script:
|
||||
- sudo env "PATH=$PATH" python3 -m osbuild --libdir . --output . samples/noop.json
|
||||
- sudo env "PATH=$PATH" python3 -m osbuild --libdir . --output . samples/noop.json
|
||||
- sudo env "PATH=$PATH" python3 -m osbuild --libdir . samples/noop.json
|
||||
- sudo env "PATH=$PATH" python3 -m osbuild --libdir . samples/noop.json
|
||||
- name: f30-boot
|
||||
before_install: sudo apt-get install -y systemd-container yum qemu-kvm
|
||||
script: sudo env "PATH=$PATH" python3 -m test --case f30-boot --build-pipeline samples/build-from-yum.json
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ The above pipeline has no base and produces a qcow2 image.
|
|||
|
||||
```
|
||||
usage: python3 -m osbuild [-h] [--build-pipeline PIPELINE] [--store DIRECTORY]
|
||||
[-l DIRECTORY] -o DIRECTORY
|
||||
[-l DIRECTORY]
|
||||
PIPELINE
|
||||
|
||||
Build operating system images
|
||||
|
|
@ -87,11 +87,6 @@ optional arguments:
|
|||
-l DIRECTORY, --libdir DIRECTORY
|
||||
the directory containing stages, assemblers, and the
|
||||
osbuild library
|
||||
|
||||
required named arguments:
|
||||
-o DIRECTORY, --output DIRECTORY
|
||||
provide the empty DIRECTORY as output argument to the
|
||||
last stage
|
||||
```
|
||||
|
||||
### Running example
|
||||
|
|
@ -99,7 +94,7 @@ required named arguments:
|
|||
You can build basic qcow2 image of Fedora 30 by running a following command:
|
||||
|
||||
```
|
||||
sudo python3 -m osbuild -o output --libdir . samples/base-qcow2.json
|
||||
sudo python3 -m osbuild --libdir . samples/base-qcow2.json
|
||||
```
|
||||
|
||||
- Root rights are required because osbuild heavily relies on creating
|
||||
|
|
|
|||
|
|
@ -23,9 +23,6 @@ def main():
|
|||
help="the directory containing stages, assemblers, and the osbuild library")
|
||||
parser.add_argument("--json", action="store_true",
|
||||
help="output results in JSON format")
|
||||
requiredNamed = parser.add_argument_group('required named arguments')
|
||||
requiredNamed.add_argument("-o", "--output", dest="output_dir", metavar="DIRECTORY", type=os.path.abspath,
|
||||
help="provide the empty DIRECTORY as output argument to the last stage", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
with open(args.pipeline_path) as f:
|
||||
|
|
@ -37,7 +34,7 @@ def main():
|
|||
pipeline.prepend_build_pipeline(build)
|
||||
|
||||
try:
|
||||
pipeline.run(args.output_dir, args.store, interactive=not args.json, libdir=args.libdir)
|
||||
pipeline.run(args.store, interactive=not args.json, libdir=args.libdir)
|
||||
except KeyboardInterrupt:
|
||||
print()
|
||||
print(f"{RESET}{BOLD}{RED}Aborted{RESET}")
|
||||
|
|
|
|||
|
|
@ -111,7 +111,7 @@ class Assembler:
|
|||
def run(self, tree, build_tree, output_dir=None, interactive=False, check=True, libdir=None):
|
||||
with buildroot.BuildRoot(build_tree) as build_root:
|
||||
if interactive:
|
||||
print_header(f"Assembling: {self.name}", self.options)
|
||||
print_header(f"Assembler {self.name}: {self.id}", self.options)
|
||||
|
||||
args = {
|
||||
"tree": "/run/osbuild/tree",
|
||||
|
|
@ -195,11 +195,11 @@ class Pipeline:
|
|||
finally:
|
||||
subprocess.run(["umount", "--lazy", tmp], check=True)
|
||||
|
||||
def run(self, output_dir, store, interactive=False, check=True, libdir=None):
|
||||
def run(self, store, interactive=False, check=True, libdir=None):
|
||||
os.makedirs("/run/osbuild", exist_ok=True)
|
||||
object_store = objectstore.ObjectStore(store)
|
||||
if self.build:
|
||||
if not self.build.run(None, store, interactive, check, libdir):
|
||||
if not self.build.run(store, interactive, check, libdir):
|
||||
return False
|
||||
|
||||
with self.get_buildtree(object_store) as build_tree:
|
||||
|
|
@ -228,8 +228,9 @@ class Pipeline:
|
|||
libdir=libdir):
|
||||
return False
|
||||
|
||||
if self.assembler:
|
||||
with object_store.get(self.tree_id) as tree:
|
||||
if self.assembler and not object_store.contains(self.output_id):
|
||||
with object_store.get(self.tree_id) as tree, \
|
||||
object_store.new(self.output_id) as output_dir:
|
||||
if not self.assembler.run(tree,
|
||||
build_tree,
|
||||
output_dir=output_dir,
|
||||
|
|
|
|||
|
|
@ -43,7 +43,6 @@ if __name__ == '__main__':
|
|||
args = parser.parse_args()
|
||||
|
||||
logging.info(f"Using {OBJECTS} for objects storage.")
|
||||
logging.info(f"Using {OUTPUT_DIR} for output images storage.")
|
||||
logging.info(f"Using {OSBUILD} for building images.")
|
||||
|
||||
f30_boot = IntegrationTestCase(
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import json
|
||||
import logging
|
||||
import subprocess
|
||||
import sys
|
||||
|
|
@ -5,8 +6,8 @@ import sys
|
|||
from .config import *
|
||||
|
||||
|
||||
def run_osbuild(pipeline: str, build_pipeline: str, check=True):
|
||||
cmd = OSBUILD + ["--store", OBJECTS, "-o", OUTPUT_DIR, pipeline]
|
||||
def run_osbuild(pipeline: str, build_pipeline: str):
|
||||
cmd = OSBUILD + ["--json", "--store", OBJECTS, pipeline]
|
||||
if build_pipeline:
|
||||
cmd += ["--build-pipeline", build_pipeline]
|
||||
logging.info(f"Running osbuild: {cmd}")
|
||||
|
|
@ -17,10 +18,10 @@ def run_osbuild(pipeline: str, build_pipeline: str, check=True):
|
|||
print(osbuild.stderr.decode())
|
||||
print(f"{BOLD}STDOUT{RESET}")
|
||||
print(osbuild.stdout.decode())
|
||||
if check:
|
||||
sys.exit(1)
|
||||
sys.exit(1)
|
||||
|
||||
return osbuild.returncode
|
||||
result = json.loads(osbuild.stdout.decode())
|
||||
return result["tree_id"], result.get("output_id")
|
||||
|
||||
|
||||
def build_testing_image(pipeline_full_path, build_pipeline_full_path):
|
||||
|
|
|
|||
|
|
@ -7,5 +7,4 @@ RESET = "\033[0m"
|
|||
BOLD = "\033[1m"
|
||||
RED = "\033[31m"
|
||||
OBJECTS = os.environ.get("OBJECTS", ".osbuild-test")
|
||||
OUTPUT_DIR = os.environ.get("OUTPUT_DIR", "output-test")
|
||||
OSBUILD = os.environ.get("OSBUILD", "python3 -m osbuild --libdir .").split(' ')
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ def run_image(file_name: str):
|
|||
silence = ["-nographic", "-monitor", "none", "-serial", "none"]
|
||||
serial = ["-chardev", "stdio,id=stdio", "-device", "virtio-serial", "-device", "virtserialport,chardev=stdio"]
|
||||
cmd = ["qemu-system-x86_64", "-m", "1024", "-snapshot"] + \
|
||||
acceleration + silence + serial + [f"{OUTPUT_DIR}/{file_name}"]
|
||||
acceleration + silence + serial + [file_name]
|
||||
logging.info(f"Booting image: {cmd}")
|
||||
return subprocess.run(cmd, capture_output=True, timeout=EXPECTED_TIME_TO_BOOT, encoding="utf-8", check=True)
|
||||
|
||||
|
|
@ -19,7 +19,7 @@ def run_image(file_name: str):
|
|||
@contextlib.contextmanager
|
||||
def extract_image(file_name: str):
|
||||
extract_dir = tempfile.mkdtemp(prefix="osbuild-")
|
||||
archive = path.join(os.getcwd(), OUTPUT_DIR, file_name)
|
||||
archive = path.join(os.getcwd(), file_name)
|
||||
subprocess.run(["tar", "xf", archive], cwd=extract_dir, check=True)
|
||||
try:
|
||||
yield extract_dir
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from typing import List, Callable, Any
|
|||
from . import evaluate_test, rel_path
|
||||
from .build import run_osbuild
|
||||
from .run import run_image, extract_image
|
||||
from test.integration_tests.config import *
|
||||
|
||||
|
||||
class IntegrationTestType(Enum):
|
||||
|
|
@ -22,18 +23,18 @@ class IntegrationTestCase:
|
|||
type: IntegrationTestType
|
||||
|
||||
def run(self):
|
||||
run_osbuild(rel_path(f"pipelines/{self.pipeline}"), self.build_pipeline)
|
||||
tree_id, output_id = run_osbuild(rel_path(f"pipelines/{self.pipeline}"), self.build_pipeline)
|
||||
if self.type == IntegrationTestType.BOOT_WITH_QEMU:
|
||||
self.run_and_test()
|
||||
self.run_and_test(output_id)
|
||||
else:
|
||||
self.extract_and_test()
|
||||
self.extract_and_test(output_id)
|
||||
|
||||
def run_and_test(self):
|
||||
r = run_image(self.output_image)
|
||||
def run_and_test(self, output_id):
|
||||
r = run_image(f"{OBJECTS}/refs/{output_id}/{self.output_image}")
|
||||
for test in self.test_cases:
|
||||
evaluate_test(test, r.stdout)
|
||||
|
||||
def extract_and_test(self):
|
||||
with extract_image(self.output_image) as fstree:
|
||||
def extract_and_test(self, output_id):
|
||||
with extract_image(f"{OBJECTS}/refs/{output_id}/{self.output_image}") as fstree:
|
||||
for test in self.test_cases:
|
||||
evaluate_test(lambda: test(fstree), name=test.__name__)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue