osbuild: added a configuable timeout for package installation

Also added new command line option for setting the timeout in milliseconds

osbuild/buildroot.py

@@ -10,9 +10,11 @@ import importlib
 import importlib.util
 import io
 import os
+import select
 import stat
 import subprocess
 import tempfile
+import time
 
 
 __all__ = [
@@ -166,7 +168,7 @@ class BuildRoot(contextlib.AbstractContextManager):
         if self._exitstack:
             self._exitstack.enter_context(api)
 
-    def run(self, argv, monitor, binds=None, readonly_binds=None):
+    def run(self, argv, monitor, stage_timeout=None, binds=None, readonly_binds=None):
         """Runs a command in the buildroot.
 
         Takes the command and arguments, as well as bind mounts to mirror
@@ -281,10 +283,12 @@ class BuildRoot(contextlib.AbstractContextManager):
                                 close_fds=True)
 
         data = io.StringIO()
-
+        start = time.monotonic()
-        fd = proc.stdout.fileno()
+        READ_ONLY = select.POLLIN | select.POLLPRI | select.POLLHUP | select.POLLERR
+        poller = select.poll()
+        poller.register(proc.stdout.fileno(), READ_ONLY)
         while True:
-            buf = os.read(fd, 32768)
+            buf = self.read_with_timeout(proc, poller, start, stage_timeout)
             if not buf:
                 break
 
@@ -292,6 +296,7 @@ class BuildRoot(contextlib.AbstractContextManager):
             data.write(txt)
             monitor.log(txt)
 
+        poller.unregister(proc.stdout.fileno())
         buf, _ = proc.communicate()
         txt = buf.decode("utf-8")
         monitor.log(txt)
@@ -300,3 +305,27 @@ class BuildRoot(contextlib.AbstractContextManager):
         data.close()
 
         return CompletedBuild(proc, output)
+
+    @classmethod
+    def read_with_timeout(cls, proc, poller, start, stage_timeout):
+        fd = proc.stdout.fileno()
+        if stage_timeout is None:
+            return os.read(fd, 32768)
+
+        # convert stage_timeout to milliseconds
+        remaining = (stage_timeout * 1000) - (time.monotonic() - start)
+        if remaining <= 0:
+            proc.terminate()
+            raise TimeoutError
+
+        buf = None
+        events = poller.poll(remaining)
+        if not events:
+            proc.terminate()
+            raise TimeoutError
+        for fd, flag in events:
+            if flag & (select.POLLIN | select.POLLPRI):
+                buf = os.read(fd, 32768)
+            if flag & (select.POLLERR | select.POLLHUP):
+                proc.terminate()
+        return buf

osbuild/main_cli.py

@@ -80,6 +80,8 @@ def parse_arguments(sys_argv):
                         help="directory where result objects are stored")
     parser.add_argument("--inspect", action="store_true",
                         help="return the manifest in JSON format including all the ids")
+    parser.add_argument("--stage-timeout", type=int, default=None,
+                        help="set the timeout in seconds for building an image")
 
     return parser.parse_args(sys_argv[1:])
 
@@ -143,6 +145,7 @@ def osbuild_cli():
 
     try:
         with ObjectStore(args.store) as object_store:
+            stage_timeout = args.stage_timeout
 
             pipelines = manifest.depsolve(object_store, exports)
 
@@ -152,7 +155,8 @@ def osbuild_cli():
                 object_store,
                 pipelines,
                 monitor,
-                args.libdir
+                args.libdir,
+                stage_timeout=stage_timeout
             )
 
             if r["success"] and exports:

osbuild/pipeline.py

@@ -114,7 +114,7 @@ class Stage:
         with open(location, "w", encoding="utf-8") as fp:
             json.dump(args, fp)
 
-    def run(self, tree, runner, build_tree, store, monitor, libdir):
+    def run(self, tree, runner, build_tree, store, monitor, libdir, stage_timeout=None):
         with contextlib.ExitStack() as cm:
 
             build_root = buildroot.BuildRoot(build_tree, runner, libdir, store.tmp)
@@ -195,6 +195,7 @@ class Stage:
 
             r = build_root.run([f"/run/osbuild/bin/{self.name}"],
                                monitor,
+                               stage_timeout=stage_timeout,
                                binds=binds,
                                readonly_binds=ro_binds)
 
@@ -232,7 +233,7 @@ class Pipeline:
             self.assembler.base = stage.id
         return stage
 
-    def build_stages(self, object_store, monitor, libdir):
+    def build_stages(self, object_store, monitor, libdir, stage_timeout=None):
         results = {"success": True}
 
         # We need a build tree for the stages below, which is either
@@ -290,7 +291,8 @@ class Pipeline:
                               build_path,
                               object_store,
                               monitor,
-                              libdir)
+                              libdir,
+                              stage_timeout)
 
                 monitor.result(r)
 
@@ -309,7 +311,7 @@ class Pipeline:
 
         return results, build_tree, tree
 
-    def run(self, store, monitor, libdir):
+    def run(self, store, monitor, libdir,stage_timeout=None):
         results = {"success": True}
 
         monitor.begin(self)
@@ -322,7 +324,7 @@ class Pipeline:
         obj = store.get(self.id)
 
         if not obj:
-            results, _, obj = self.build_stages(store, monitor, libdir)
+            results, _, obj = self.build_stages(store, monitor, libdir, stage_timeout)
 
             if not results["success"]:
                 return results
@@ -405,11 +407,11 @@ class Manifest:
 
         return list(map(lambda x: x.name, reversed(build.values())))
 
-    def build(self, store, pipelines, monitor, libdir):
+    def build(self, store, pipelines, monitor, libdir, stage_timeout=None):
         results = {"success": True}
 
         for pl in map(self.get, pipelines):
-            res = pl.run(store, monitor, libdir)
+            res = pl.run(store, monitor, libdir, stage_timeout)
             results[pl.id] = res
             if not res["success"]:
                 results["success"] = False