From c90b587dccf5f82e5732b7a264866c3535c066c2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20Budai?= <ondrej@budai.cz>
Date: Mon, 26 Jun 2023 18:37:52 +0200
Subject: [PATCH] inputs: Move arguments for InputService.map to a temporary
 file

Prior this commit, the arguments for the input service were passed inline.
However, jsoncomm uses the SOCK_SEQPACKET socket type underneath that has
a fixed maximum packet size. On my system, it's 212960 bytes. Unfortunately,
that's not enough for big inputs (e.g. when building packages with a lot
of rpms).

This commit moves all arguments to a temporary file. Then, just a file
descriptor is sent. Thus, we are now able to send arbitrarily sized args
for inputs, making osbuild work even for large image builds.
---
 osbuild/inputs.py   | 24 +++++++++++++++++++-----
 osbuild/pipeline.py |  2 +-
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/osbuild/inputs.py b/osbuild/inputs.py
index 0cb12da6..5e845df7 100644
--- a/osbuild/inputs.py
+++ b/osbuild/inputs.py
@@ -17,15 +17,17 @@ osbuild is the path. The input options are just passed to the
 """
 
 import abc
+import contextlib
 import hashlib
 import json
 import os
+import tempfile
 from typing import Any, Dict, Optional, Tuple
 
 from osbuild import host
 from osbuild.util.types import PathLike
 
-from .objectstore import StoreClient, StoreServer
+from .objectstore import ObjectStore, StoreClient, StoreServer
 
 
 class Input:
@@ -66,7 +68,7 @@ class InputManager:
         self.root = root
         self.inputs: Dict[str, Input] = {}
 
-    def map(self, ip: Input) -> Tuple[str, Dict]:
+    def map(self, ip: Input, store: ObjectStore) -> Tuple[str, Dict]:
 
         target = os.path.join(self.root, ip.name)
         os.makedirs(target)
@@ -87,8 +89,10 @@ class InputManager:
             }
         }
 
-        client = self.service_manager.start(f"input/{ip.name}", ip.info.path)
-        reply = client.call("map", args)
+        with make_args_file(store.tmp, args) as fd:
+            fds = [fd]
+            client = self.service_manager.start(f"input/{ip.name}", ip.info.path)
+            reply, _ = client.call_with_fds("map", {}, fds)
 
         path = reply["path"]
 
@@ -102,6 +106,14 @@ class InputManager:
         return reply
 
 
+@contextlib.contextmanager
+def make_args_file(tmp, args):
+    with tempfile.TemporaryFile("w+", dir=tmp, encoding="utf-8") as f:
+        json.dump(args, f)
+        f.seek(0)
+        yield f.fileno()
+
+
 class InputService(host.Service):
     """Input host service"""
 
@@ -115,8 +127,10 @@ class InputService(host.Service):
     def stop(self):
         self.unmap()
 
-    def dispatch(self, method: str, args, _fds):
+    def dispatch(self, method: str, _, _fds):
         if method == "map":
+            with os.fdopen(_fds.steal(0)) as f:
+                args = json.load(f)
             store = StoreClient(connect_to=args["api"]["store"])
             r = self.map(store,
                          args["origin"],
diff --git a/osbuild/pipeline.py b/osbuild/pipeline.py
index 59b2f020..8013df0d 100644
--- a/osbuild/pipeline.py
+++ b/osbuild/pipeline.py
@@ -209,7 +209,7 @@ class Stage:
 
             ipmgr = InputManager(mgr, storeapi, inputs_tmpdir)
             for key, ip in self.inputs.items():
-                data = ipmgr.map(ip)
+                data = ipmgr.map(ip, store)
                 inputs[key] = data
 
             devmgr = DeviceManager(mgr, build_root.dev, tree)