jsoncomm: transparently handle huge messages via fds

The existing jsoncomm is a work of beautiy. For very big arguments
however the used `SOCK_SEQPACKET` hits the limitations of the
kernel network buffer size (see also [0]). This lead to various
workarounds in #824,#1331,#1836 where parts of the request are
encoded as part of the json method call and parts are done via
a side-channel via fd-passing.

This commit changes the code so that the fd channel is automatically
and transparently created and the workarounds are removed. A test
is added that ensures that very big messages can be passed.

[0] https://github.com/osbuild/osbuild/pull/1833

test/mod/test_inputs.py

@@ -1,9 +1,7 @@
-import json
 import os
-from unittest.mock import patch
+from unittest.mock import call, patch
 
 from osbuild import inputs
-from osbuild.util.jsoncomm import FdSet
 
 
 class FakeInputService(inputs.InputService):
@@ -11,7 +9,7 @@ class FakeInputService(inputs.InputService):
         # do not call "super().__init__()" here to make it testable
         self.map_calls = []
 
-    def map(self, _store, origin, refs, target, options):
+    def map(self, store, origin, refs, target, options):
         self.map_calls.append([origin, refs, target, options])
         return "complex", 2, "reply"
 
@@ -20,8 +18,6 @@ def test_inputs_dispatches_map(tmp_path):
     store_api_path = tmp_path / "api-store"
     store_api_path.write_text("")
 
-    args_path = tmp_path / "args"
-    reply_path = tmp_path / "reply"
     args = {
         "api": {
             "store": os.fspath(store_api_path),
@@ -31,17 +27,14 @@ def test_inputs_dispatches_map(tmp_path):
         "target": "some-target",
         "options": "some-options",
     }
-    args_path.write_text(json.dumps(args))
-    reply_path.write_text("")
 
-    with args_path.open() as f_args, reply_path.open("w") as f_reply:
-        fd_args, fd_reply = os.dup(f_args.fileno()), os.dup(f_reply.fileno())
-        fds = FdSet.from_list([fd_args, fd_reply])
-        fake_service = FakeInputService(args="some")
-        with patch.object(inputs, "StoreClient"):
-            r = fake_service.dispatch("map", None, fds)
-            assert r == ('{}', None)
-        assert fake_service.map_calls == [
-            ["some-origin", "some-refs", "some-target", "some-options"],
-        ]
-    assert reply_path.read_text() == '["complex", 2, "reply"]'
+    fake_service = FakeInputService(args="some")
+    with patch.object(inputs, "StoreClient") as mocked_store_client_klass:
+        r = fake_service.dispatch("map", args, None)
+    assert mocked_store_client_klass.call_args_list == [
+        call(connect_to=os.fspath(store_api_path)),
+    ]
+    assert fake_service.map_calls == [
+        ["some-origin", "some-refs", "some-target", "some-options"],
+    ]
+    assert r == (("complex", 2, "reply"), None)

test/mod/test_util_jsoncomm.py

@@ -2,13 +2,17 @@
 # Tests for the 'osbuild.util.jsoncomm' module.
 #
 
+# pylint: disable=protected-access
+
 import asyncio
 import errno
+import json
 import os
 import pathlib
 import tempfile
 import unittest
 from concurrent import futures
+from unittest.mock import patch
 
 import pytest
 
@@ -220,11 +224,39 @@ class TestUtilJsonComm(unittest.TestCase):
         pong, _, _ = a.recv()
         self.assertEqual(ping, pong)
 
-    def test_send_and_recv_tons_of_data_still_errors(self):
+    def test_sendmsg_errors_with_size_on_EMSGSIZE(self):
         a, _ = jsoncomm.Socket.new_pair()
 
-        ping = {"data": "1" * 1_000_000}
+        serialized = json.dumps({"data": "1" * 1_000_000}).encode()
         with pytest.raises(BufferError) as exc:
-            a.send(ping)
+            a._send_via_sendmsg(serialized, [])
         assert str(exc.value) == "jsoncomm message size 1000012 is too big"
         assert exc.value.__cause__.errno == errno.EMSGSIZE
+
+    def test_send_and_recv_tons_of_data_is_fine(self):
+        a, b = jsoncomm.Socket.new_pair()
+
+        ping = {"data": "tons" * 1_000_000}
+        a.send(ping)
+        pong, _, _ = b.send_and_recv(ping)
+        self.assertEqual(ping, pong)
+        pong, _, _ = a.recv()
+        self.assertEqual(ping, pong)
+
+    def test_send_small_data_via_sendmsg(self):
+        a, _ = jsoncomm.Socket.new_pair()
+        with patch.object(a, "_send_via_fd") as mock_send_via_fd, \
+                patch.object(a, "_send_via_sendmsg") as mock_send_via_sendmsg:
+            ping = {"data": "little"}
+            a.send(ping)
+        assert mock_send_via_fd.call_count == 0
+        assert mock_send_via_sendmsg.call_count == 1
+
+    def test_send_huge_data_via_fd(self):
+        a, _ = jsoncomm.Socket.new_pair()
+        with patch.object(a, "_send_via_fd") as mock_send_via_fd, \
+                patch.object(a, "_send_via_sendmsg") as mock_send_via_sendmsg:
+            ping = {"data": "tons" * 1_000_000}
+            a.send(ping)
+        assert mock_send_via_fd.call_count == 1
+        assert mock_send_via_sendmsg.call_count == 0