particle-os/debian-forge
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

osbuild: make inputs map() function use fd for reply as well

Browse source 
We recently hit the issue that `osbuild` crashed with:
```
Unable to decode response body "Traceback (most recent call last):
  File \"/usr/bin/osbuild\", line 33, in <module>
    sys.exit(load_entry_point('osbuild==124', 'console_scripts', 'osbuild')())
  File \"/usr/lib/python3.9/site-packages/osbuild/main_cli.py\", line 181, in osbuild_cli
    r = manifest.build(
  File \"/usr/lib/python3.9/site-packages/osbuild/pipeline.py\", line 477, in build
    res = pl.run(store, monitor, libdir, debug_break, stage_timeout)
  File \"/usr/lib/python3.9/site-packages/osbuild/pipeline.py\", line 376, in run
    results = self.build_stages(store,
  File \"/usr/lib/python3.9/site-packages/osbuild/pipeline.py\", line 348, in build_stages
    r = stage.run(tree,
  File \"/usr/lib/python3.9/site-packages/osbuild/pipeline.py\", line 213, in run
    data = ipmgr.map(ip, store)
  File \"/usr/lib/python3.9/site-packages/osbuild/inputs.py\", line 94, in map
    reply, _ = client.call_with_fds(\"map\", {}, fds)
  File \"/usr/lib/python3.9/site-packages/osbuild/host.py\", line 373, in call_with_fds
    kind, data = self.protocol.decode_message(ret)
  File \"/usr/lib/python3.9/site-packages/osbuild/host.py\", line 83, in decode_message
    raise ProtocolError(\"message empty\")
osbuild.host.ProtocolError: message empty
cannot run osbuild: exit status 1" into osbuild result: invalid character 'T' looking for beginning of value
...
input/packages (org.osbuild.files): Traceback (most recent call last):
input/packages (org.osbuild.files):   File "/usr/lib/osbuild/inputs/org.osbuild.files", line 226, in <module>
input/packages (org.osbuild.files):     main()
input/packages (org.osbuild.files):   File "/usr/lib/osbuild/inputs/org.osbuild.files", line 222, in main
input/packages (org.osbuild.files):     service.main()
input/packages (org.osbuild.files):   File "/usr/lib/python3.11/site-packages/osbuild/host.py", line 250, in main
input/packages (org.osbuild.files):     self.serve()
input/packages (org.osbuild.files):   File "/usr/lib/python3.11/site-packages/osbuild/host.py", line 284, in serve
input/packages (org.osbuild.files):     self.sock.send(reply, fds=reply_fds)
input/packages (org.osbuild.files):   File "/usr/lib/python3.11/site-packages/osbuild/util/jsoncomm.py", line 407, in send
input/packages (org.osbuild.files):     n = self._socket.sendmsg([serialized], cmsg, 0)
input/packages (org.osbuild.files):         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
input/packages (org.osbuild.files): OSError: [Errno 90] Message too long
```

The underlying issue is that the reply of the `map()` call is too
big for the buffer that `jsoncomm` uses. This problem existed before
for the args of map and was fixed by introducing a temporary file
in https://github.com/osbuild/osbuild/pull/1331 (and similarly
before in https://github.com/osbuild/osbuild/pull/824).

This commit writes the return values also into a file. This should
fix the crash above and make the function more symetrical as well.

Alternative/complementary version of
https://github.com/osbuild/osbuild/pull/1833

Closes: HMS-4537
This commit is contained in:
Michael Vogt 2024-08-07 10:18:57 +02:00 • committed by Simon de Vlieger
parent 29f926f305
commit 88c35ea306
3 changed files with 67 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

30
osbuild/inputs.py
View file

@ -88,10 +88,12 @@ class InputManager:
}
}
with make_args_file(store.tmp, args) as fd:
fds = [fd]
with make_args_and_reply_files(store.tmp, args) as (fd_args, fd_reply):
fds = [fd_args, fd_reply]
client = self.service_manager.start(f"input/{ip.name}", ip.info.path)
reply, _ = client.call_with_fds("map", {}, fds)
_, _ = client.call_with_fds("map", {}, fds)
with os.fdopen(os.dup(fd_reply)) as f:
reply = json.loads(f.read())
path = reply["path"]
@ -106,11 +108,12 @@ class InputManager:
@contextlib.contextmanager
def make_args_file(tmp, args):
with tempfile.TemporaryFile("w+", dir=tmp, encoding="utf-8") as f:
json.dump(args, f)
f.seek(0)
yield f.fileno()
def make_args_and_reply_files(tmp, args):
with tempfile.TemporaryFile("w+", dir=tmp, encoding="utf-8") as f_args, \
tempfile.TemporaryFile("w+", dir=tmp, encoding="utf-8") as f_reply:
json.dump(args, f_args)
f_args.seek(0)
yield f_args.fileno(), f_reply.fileno()
class InputService(host.Service):
@ -126,9 +129,11 @@ class InputService(host.Service):
def stop(self):
self.unmap()
def dispatch(self, method: str, _, _fds):
def dispatch(self, method: str, _, fds):
if method == "map":
with os.fdopen(_fds.steal(0)) as f:
# map() sends fd[0] to read the arguments from and fd[1] to
# write the reply back. This avoids running into EMSGSIZE
with os.fdopen(fds.steal(0)) as f:
args = json.load(f)
store = StoreClient(connect_to=args["api"]["store"])
r = self.map(store,
@ -136,6 +141,9 @@ class InputService(host.Service):
args["refs"],
args["target"],
args["options"])
return r, None
with os.fdopen(fds.steal(1), "w") as f:
f.write(json.dumps(r))
f.seek(0)
return "{}", None
raise host.ProtocolError("Unknown method")