jsoncomm: transparently handle huge messages via fds

The existing jsoncomm is a work of beautiy. For very big arguments
however the used `SOCK_SEQPACKET` hits the limitations of the
kernel network buffer size (see also [0]). This lead to various
workarounds in #824,#1331,#1836 where parts of the request are
encoded as part of the json method call and parts are done via
a side-channel via fd-passing.

This commit changes the code so that the fd channel is automatically
and transparently created and the workarounds are removed. A test
is added that ensures that very big messages can be passed.

[0] https://github.com/osbuild/osbuild/pull/1833
This commit is contained in:
Michael Vogt 2024-08-07 16:49:30 +02:00 committed by Achilleas Koutsou
parent d67fa48c17
commit 0abdfb9041
6 changed files with 130 additions and 87 deletions

View file

@ -2,13 +2,17 @@
# Tests for the 'osbuild.util.jsoncomm' module.
#
# pylint: disable=protected-access
import asyncio
import errno
import json
import os
import pathlib
import tempfile
import unittest
from concurrent import futures
from unittest.mock import patch
import pytest
@ -220,11 +224,39 @@ class TestUtilJsonComm(unittest.TestCase):
pong, _, _ = a.recv()
self.assertEqual(ping, pong)
def test_send_and_recv_tons_of_data_still_errors(self):
def test_sendmsg_errors_with_size_on_EMSGSIZE(self):
a, _ = jsoncomm.Socket.new_pair()
ping = {"data": "1" * 1_000_000}
serialized = json.dumps({"data": "1" * 1_000_000}).encode()
with pytest.raises(BufferError) as exc:
a.send(ping)
a._send_via_sendmsg(serialized, [])
assert str(exc.value) == "jsoncomm message size 1000012 is too big"
assert exc.value.__cause__.errno == errno.EMSGSIZE
def test_send_and_recv_tons_of_data_is_fine(self):
a, b = jsoncomm.Socket.new_pair()
ping = {"data": "tons" * 1_000_000}
a.send(ping)
pong, _, _ = b.send_and_recv(ping)
self.assertEqual(ping, pong)
pong, _, _ = a.recv()
self.assertEqual(ping, pong)
def test_send_small_data_via_sendmsg(self):
a, _ = jsoncomm.Socket.new_pair()
with patch.object(a, "_send_via_fd") as mock_send_via_fd, \
patch.object(a, "_send_via_sendmsg") as mock_send_via_sendmsg:
ping = {"data": "little"}
a.send(ping)
assert mock_send_via_fd.call_count == 0
assert mock_send_via_sendmsg.call_count == 1
def test_send_huge_data_via_fd(self):
a, _ = jsoncomm.Socket.new_pair()
with patch.object(a, "_send_via_fd") as mock_send_via_fd, \
patch.object(a, "_send_via_sendmsg") as mock_send_via_sendmsg:
ping = {"data": "tons" * 1_000_000}
a.send(ping)
assert mock_send_via_fd.call_count == 1
assert mock_send_via_sendmsg.call_count == 0