particle-os/debian-forge
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

osbuild: auto-detect best available runner

Browse source 
Use the new `Index.detect_runner` method that will give us the best
available runner for a requested one. To do so a new `pipeline.Runner`
class is introduced that stores the `meta.RunnerInfo` class for the
specific runner and the original name that was requested.
In the manifest loading and describing functions of the formats, use
`Index.detect_runner` to get the `RunnerInfo` for a requested runner
and then wrap it in a `pipeline.Runner` object, which is then passed
to the `Manifest.add_pipeline` method.
See also commit "meta: ability to auto-detect runner".
Adjust all test.
This commit is contained in:
Christian Kellner 2022-10-01 11:11:07 +02:00
parent 0554ac652b
commit 5bdc8d030c
8 changed files with 71 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

11
osbuild/formats/v1.py
View file

@ -11,7 +11,7 @@ from typing import Any, Dict
from osbuild.meta import Index, ValidationResult
from ..pipeline import BuildResult, Manifest, Pipeline, detect_host_runner
from ..pipeline import BuildResult, Manifest, Pipeline, Runner
VERSION = "1"
@ -32,7 +32,7 @@ def describe(manifest: Manifest, *, with_id=False) -> Dict[str, Any]:
build = manifest[pipeline.build]
description["build"] = {
"pipeline": describe_pipeline(build),
"runner": pipeline.runner
"runner": pipeline.runner.name
}
if pipeline.stages:
@ -91,7 +91,10 @@ def load_build(description: Dict, index: Index, manifest: Manifest, n: int):
else:
build_pipeline = None
return build_pipeline, description["runner"]
runner_name = description["runner"]
runner_info = index.detect_runner(runner_name)
return build_pipeline, Runner(runner_info, runner_name)
def load_stage(description: Dict, index: Index, pipeline: Pipeline):
@ -146,7 +149,7 @@ def load_pipeline(description: Dict, index: Index, manifest: Manifest, n: int =
if build:
build_pipeline, runner = load_build(build, index, manifest, n)
else:
build_pipeline, runner = None, detect_host_runner()
build_pipeline, runner = None, Runner(index.detect_host_runner())
# the "main" pipeline is called `tree`, since it is building the
# tree that will later be used by the `assembler`. Nested build

17
osbuild/formats/v2.py
View file

@ -7,7 +7,7 @@ from typing import Any, Dict
from osbuild.meta import Index, ModuleInfo, ValidationResult
from ..inputs import Input
from ..pipeline import Manifest, Pipeline, Stage, detect_host_runner
from ..pipeline import Manifest, Pipeline, Runner, Stage
from ..sources import Source
VERSION = "2"
@ -130,7 +130,7 @@ def describe(manifest: Manifest, *, with_id=False) -> Dict:
runner = runners.get(p.id)
if runner:
desc["runner"] = runner
desc["runner"] = runner.name
stages = [
describe_stage(stage)
@ -325,13 +325,22 @@ def load_stage(description: Dict, index: Index, pipeline: Pipeline, manifest: Ma
def load_pipeline(description: Dict, index: Index, manifest: Manifest, source_refs: set):
name = description["name"]
build = description.get("build")
runner = description.get("runner")
source_epoch = description.get("source-epoch")
if build and build.startswith("name:"):
target = resolve_ref(build, manifest)
build = target
# NB: The runner mapping will later be changed in `load`.
# The host runner here is just to always have a Runner
# (instead of a Optional[Runner]) to make mypy happy
runner_name = description.get("runner")
runner = None
if runner_name:
runner = Runner(index.detect_runner(runner_name), runner_name)
else:
runner = Runner(index.detect_host_runner())
pl = manifest.add_pipeline(name, runner, build, source_epoch)
for desc in description.get("stages", []):
@ -365,7 +374,7 @@ def load(description: Dict, index: Index) -> Manifest:
# go through the pipelines and fix things up
pipelines = manifest.pipelines.values()
host_runner = detect_host_runner()
host_runner = Runner(index.detect_host_runner())
runners = {
pl.id: pl.runner for pl in pipelines
}

18
osbuild/pipeline.py
View file

@ -145,7 +145,7 @@ class Stage:
def run(self, tree, runner, build_tree, store, monitor, libdir, timeout=None):
with contextlib.ExitStack() as cm:
build_root = buildroot.BuildRoot(build_tree, runner, libdir, store.tmp)
build_root = buildroot.BuildRoot(build_tree, runner.path, libdir, store.tmp)
cm.enter_context(build_root)
# if we have a build root, then also bind-mount the boot
@ -239,8 +239,22 @@ class Stage:
return BuildResult(self, r.returncode, r.output, api.metadata, api.error)
class Runner:
def __init__(self, info, name: Optional[str] = None) -> None:
self.info = info # `meta.RunnerInfo`
self.name = name or os.path.basename(info.path)
@property
def path(self):
return self.info.path
@property
def exec(self):
return os.path.basename(self.info.path)
class Pipeline:
def __init__(self, name: str, runner=None, build=None, source_epoch=None):
def __init__(self, name: str, runner: Runner, build=None, source_epoch=None):
self.name = name
self.build = build
self.runner = runner

33
test/mod/test_buildroot.py
View file

@ -9,9 +9,9 @@ from tempfile import TemporaryDirectory
import pytest
import osbuild.meta
from osbuild.buildroot import BuildRoot
from osbuild.monitor import LogMonitor, NullMonitor
from osbuild.pipeline import detect_host_runner
from osbuild.util import linux
from ..test import TestBase
@ -23,9 +23,15 @@ def tempdir_fixture():
yield tmp
@pytest.fixture(name="runner")
def runner_fixture():
meta = osbuild.meta.Index(os.curdir)
runner = meta.detect_host_runner()
return runner.path
@pytest.mark.skipif(not TestBase.can_bind_mount(), reason="root only")
def test_basic(tempdir):
runner = detect_host_runner()
def test_basic(tempdir, runner):
libdir = os.path.abspath(os.curdir)
var = pathlib.Path(tempdir, "var")
var.mkdir()
@ -69,8 +75,7 @@ def test_runner_fail(tempdir):
@pytest.mark.skipif(not TestBase.can_bind_mount(), reason="root only")
def test_output(tempdir):
runner = detect_host_runner()
def test_output(tempdir, runner):
libdir = os.path.abspath(os.curdir)
var = pathlib.Path(tempdir, "var")
var.mkdir()
@ -88,8 +93,7 @@ def test_output(tempdir):
@pytest.mark.skipif(not TestBase.have_test_data(), reason="no test-data access")
@pytest.mark.skipif(not TestBase.can_bind_mount(), reason="root only")
def test_bind_mounts(tempdir):
runner = detect_host_runner()
def test_bind_mounts(tempdir, runner):
libdir = os.path.abspath(os.curdir)
var = pathlib.Path(tempdir, "var")
var.mkdir()
@ -122,12 +126,11 @@ def test_bind_mounts(tempdir):
@pytest.mark.skipif(not TestBase.have_test_data(), reason="no test-data access")
@pytest.mark.skipif(not os.path.exists("/sys/fs/selinux"), reason="no SELinux")
def test_selinuxfs_ro(tempdir):
def test_selinuxfs_ro(tempdir, runner):
# /sys/fs/selinux must never be writable in the container
# because RPM and other tools must not assume the policy
# of the host is the valid policy
runner = detect_host_runner()
libdir = os.path.abspath(os.curdir)
var = pathlib.Path(tempdir, "var")
var.mkdir()
@ -148,8 +151,7 @@ def test_selinuxfs_ro(tempdir):
@pytest.mark.skipif(not TestBase.can_bind_mount(), reason="root only")
def test_proc_overrides(tempdir):
runner = detect_host_runner()
def test_proc_overrides(tempdir, runner):
libdir = os.path.abspath(os.curdir)
var = pathlib.Path(tempdir, "var")
var.mkdir()
@ -167,8 +169,7 @@ def test_proc_overrides(tempdir):
@pytest.mark.skipif(not TestBase.can_bind_mount(), reason="root only")
def test_timeout(tempdir):
runner = detect_host_runner()
def test_timeout(tempdir, runner):
libdir = os.path.abspath(os.curdir)
var = pathlib.Path(tempdir, "var")
var.mkdir()
@ -187,8 +188,7 @@ def test_timeout(tempdir):
@pytest.mark.skipif(not TestBase.can_bind_mount(), reason="root only")
def test_env_isolation(tempdir):
runner = detect_host_runner()
def test_env_isolation(tempdir, runner):
libdir = os.path.abspath(os.curdir)
var = pathlib.Path(tempdir, "var")
var.mkdir()
@ -229,8 +229,7 @@ def test_env_isolation(tempdir):
@pytest.mark.skipif(not TestBase.can_bind_mount(), reason="root only")
def test_caps(tempdir):
runner = detect_host_runner()
def test_caps(tempdir, runner):
libdir = os.path.abspath(os.curdir)
var = pathlib.Path(tempdir, "var")
var.mkdir()

7
test/mod/test_fmt_v1.py
View file

@ -141,7 +141,7 @@ class TestFormatV1(unittest.TestCase):
pl = manifest["build"]
have = pl.stages[0]
want = build["pipeline"]["stages"][0]
self.assertEqual(pl.runner, runner)
self.assertEqual(pl.runner.name, runner)
check_stage(have, want)
runner = build["runner"]
@ -150,14 +150,14 @@ class TestFormatV1(unittest.TestCase):
pl = manifest["tree"]
have = pl.stages[0]
want = description["pipeline"]["stages"][0]
self.assertEqual(pl.runner, runner)
self.assertEqual(pl.runner.name, runner)
check_stage(have, want)
# the assembler pipeline
pl = manifest["assembler"]
have = pl.stages[0]
want = description["pipeline"]["assembler"]
self.assertEqual(pl.runner, runner)
self.assertEqual(pl.runner.name, runner)
check_stage(have, want)
def test_describe(self):
@ -211,6 +211,7 @@ class TestFormatV1(unittest.TestCase):
self.assertIsNotNone(res)
result = fmt.output(manifest, res)
print(result)
self.assertIsNotNone(result)
self.assertIn("success", result)
self.assertFalse(result["success"])

4
test/mod/test_fmt_v2.py
View file

@ -212,13 +212,13 @@ class TestFormatV2(unittest.TestCase):
self.assertIsNotNone(tree)
self.assertIsNotNone(tree.build)
self.assertEqual(tree.build, build.id)
self.assertEqual(tree.runner, "org.osbuild.linux")
self.assertEqual(tree.runner.name, "org.osbuild.linux")
assembler = manifest["assembler"]
self.assertIsNotNone(assembler)
self.assertIsNotNone(assembler.build)
self.assertEqual(assembler.build, build.id)
self.assertEqual(assembler.runner, "org.osbuild.linux")
self.assertEqual(assembler.runner.name, "org.osbuild.linux")
def test_format_info(self):
index = self.index

6
test/mod/test_monitor.py
View file

@ -13,7 +13,7 @@ import osbuild
import osbuild.meta
from osbuild.monitor import LogMonitor
from osbuild.objectstore import ObjectStore
from osbuild.pipeline import detect_host_runner
from osbuild.pipeline import Runner
from .. import test
@ -56,7 +56,7 @@ class TestMonitor(unittest.TestCase):
# Checks the basic functioning of the LogMonitor
index = osbuild.meta.Index(os.curdir)
runner = detect_host_runner()
runner = Runner(index.detect_host_runner())
pipeline = osbuild.Pipeline("pipeline", runner=runner)
info = index.get_module_info("Stage", "org.osbuild.noop")
pipeline.add_stage(info, {
@ -84,8 +84,8 @@ class TestMonitor(unittest.TestCase):
@unittest.skipUnless(test.TestBase.can_bind_mount(), "root-only")
def test_monitor_integration(self):
# Checks the monitoring API is called properly from the pipeline
runner = detect_host_runner()
index = osbuild.meta.Index(os.curdir)
runner = Runner(index.detect_host_runner())
pipeline = osbuild.Pipeline("pipeline", runner=runner)
noop_info = index.get_module_info("Stage", "org.osbuild.noop")

18
test/mod/test_osbuild.py
View file

@ -13,7 +13,7 @@ import osbuild
import osbuild.meta
from osbuild.monitor import NullMonitor
from osbuild.objectstore import ObjectStore
from osbuild.pipeline import Manifest, detect_host_runner
from osbuild.pipeline import Manifest, Runner
from .. import test
@ -43,7 +43,7 @@ class TestDescriptions(unittest.TestCase):
data = pathlib.Path(tmpdir, "data")
storedir = pathlib.Path(tmpdir, "store")
root = pathlib.Path("/")
runner = detect_host_runner()
runner = Runner(index.detect_host_runner())
monitor = NullMonitor(sys.stderr.fileno())
libdir = os.path.abspath(os.curdir)
store = ObjectStore(storedir)
@ -124,19 +124,21 @@ class TestDescriptions(unittest.TestCase):
# pylint: disable=too-many-statements
def test_on_demand(self):
index = osbuild.meta.Index(os.curdir)
host_runner = Runner(index.detect_host_runner())
runner = Runner(index.detect_runner("org.osbuild.linux"))
manifest = Manifest()
noop = index.get_module_info("Stage", "org.osbuild.noop")
noip = index.get_module_info("Input", "org.osbuild.noop")
# the shared build pipeline
build = manifest.add_pipeline("build", None, None)
build = manifest.add_pipeline("build", host_runner, None)
build.add_stage(noop, {"option": 1})
# a pipeline simulating some intermediate artefact
# that other pipeline need as dependency
dep = manifest.add_pipeline("dep",
"org.osbuild.linux",
runner,
build.id)
dep.add_stage(noop, {"option": 2})
@ -146,7 +148,7 @@ class TestDescriptions(unittest.TestCase):
# not be built unless explicitly requested
# has an input that depends on `dep`
ul = manifest.add_pipeline("unlinked",
"org.osbuild.linux",
runner,
build.id)
stage = ul.add_stage(noop, {"option": 3})
@ -155,14 +157,14 @@ class TestDescriptions(unittest.TestCase):
# the main os root file system
rootfs = manifest.add_pipeline("rootfs",
"org.osbuild.inux",
runner,
build.id)
stage = rootfs.add_stage(noop, {"option": 4})
# the main raw image artefact, depending on "dep" and
# "rootfs"
image = manifest.add_pipeline("image",
"org.osbuild.inux",
runner,
build.id)
stage = image.add_stage(noop, {"option": 5})
@ -177,7 +179,7 @@ class TestDescriptions(unittest.TestCase):
# some compression of the image, like a qcow2
qcow2 = manifest.add_pipeline("qcow2",
"org.osbuild.inux",
runner,
build.id)
stage = qcow2.add_stage(noop, {"option": 7})
ip = stage.add_input("image", noip, "org.osbuild.pipeline")