inputs: Move arguments for InputService.map to a temporary file

Prior this commit, the arguments for the input service were passed inline.
However, jsoncomm uses the SOCK_SEQPACKET socket type underneath that has
a fixed maximum packet size. On my system, it's 212960 bytes. Unfortunately,
that's not enough for big inputs (e.g. when building packages with a lot
of rpms).

This commit moves all arguments to a temporary file. Then, just a file
descriptor is sent. Thus, we are now able to send arbitrarily sized args
for inputs, making osbuild work even for large image builds.
This commit is contained in:
Ondřej Budai 2023-06-26 18:37:52 +02:00 committed by Achilleas Koutsou
parent 8c29e8144d
commit c90b587dcc
2 changed files with 20 additions and 6 deletions

View file

@ -17,15 +17,17 @@ osbuild is the path. The input options are just passed to the
"""
import abc
import contextlib
import hashlib
import json
import os
import tempfile
from typing import Any, Dict, Optional, Tuple
from osbuild import host
from osbuild.util.types import PathLike
from .objectstore import StoreClient, StoreServer
from .objectstore import ObjectStore, StoreClient, StoreServer
class Input:
@ -66,7 +68,7 @@ class InputManager:
self.root = root
self.inputs: Dict[str, Input] = {}
def map(self, ip: Input) -> Tuple[str, Dict]:
def map(self, ip: Input, store: ObjectStore) -> Tuple[str, Dict]:
target = os.path.join(self.root, ip.name)
os.makedirs(target)
@ -87,8 +89,10 @@ class InputManager:
}
}
client = self.service_manager.start(f"input/{ip.name}", ip.info.path)
reply = client.call("map", args)
with make_args_file(store.tmp, args) as fd:
fds = [fd]
client = self.service_manager.start(f"input/{ip.name}", ip.info.path)
reply, _ = client.call_with_fds("map", {}, fds)
path = reply["path"]
@ -102,6 +106,14 @@ class InputManager:
return reply
@contextlib.contextmanager
def make_args_file(tmp, args):
with tempfile.TemporaryFile("w+", dir=tmp, encoding="utf-8") as f:
json.dump(args, f)
f.seek(0)
yield f.fileno()
class InputService(host.Service):
"""Input host service"""
@ -115,8 +127,10 @@ class InputService(host.Service):
def stop(self):
self.unmap()
def dispatch(self, method: str, args, _fds):
def dispatch(self, method: str, _, _fds):
if method == "map":
with os.fdopen(_fds.steal(0)) as f:
args = json.load(f)
store = StoreClient(connect_to=args["api"]["store"])
r = self.map(store,
args["origin"],