sources: pass items via temporary file
Since source were converted to host services it now uses a unix socket instead of stdin to pass the arguments, which includes the list of items to download. The latter can become quite big, in fact too big to fit into a single package (NB: SOCK_SEQPACKET is used for the underlying transport). Therefore write the actual items to a temporary file and pass the fd of it along the message.
This commit is contained in:
parent
fe3bb30f4c
commit
879c56a3b5
1 changed files with 18 additions and 4 deletions
|
|
@ -1,5 +1,8 @@
|
|||
import abc
|
||||
import contextlib
|
||||
import os
|
||||
import json
|
||||
import tempfile
|
||||
|
||||
from . import host
|
||||
from .objectstore import ObjectStore
|
||||
|
|
@ -21,7 +24,6 @@ class Source:
|
|||
cache = os.path.join(store.store, "sources")
|
||||
|
||||
args = {
|
||||
"items": self.items,
|
||||
"options": self.options,
|
||||
"cache": cache,
|
||||
"output": None,
|
||||
|
|
@ -31,10 +33,19 @@ class Source:
|
|||
|
||||
client = mgr.start(f"source/{source}", self.info.path)
|
||||
|
||||
reply = client.call("download", args)
|
||||
with self.make_items_file(store.tmp) as fd:
|
||||
fds = [fd]
|
||||
reply = client.call_with_fds("download", args, fds)
|
||||
|
||||
return reply
|
||||
|
||||
@contextlib.contextmanager
|
||||
def make_items_file(self, tmp):
|
||||
with tempfile.TemporaryFile("w+", dir=tmp, encoding="utf-8") as f:
|
||||
json.dump(self.items, f)
|
||||
f.seek(0)
|
||||
yield f.fileno()
|
||||
|
||||
|
||||
class SourceService(host.Service):
|
||||
"""Source host service"""
|
||||
|
|
@ -43,9 +54,12 @@ class SourceService(host.Service):
|
|||
def download(self, items, cache, options):
|
||||
pass
|
||||
|
||||
def dispatch(self, method: str, args, _fds):
|
||||
def dispatch(self, method: str, args, fds):
|
||||
if method == "download":
|
||||
r = self.download(args["items"],
|
||||
with os.fdopen(fds.steal(0)) as f:
|
||||
items = json.load(f)
|
||||
|
||||
r = self.download(items,
|
||||
args["cache"],
|
||||
args["options"])
|
||||
return r, None
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue