sources: move parallelisation into source

This moves the parallelisation decisions into the sources themselves,
making the `download` method abstract inside `osbuild` itself.
This commit is contained in:
Simon de Vlieger 2024-01-23 11:45:13 +01:00 committed by Michael Vogt
parent 18e5481ae8
commit 2c42c46c48
6 changed files with 54 additions and 10 deletions

View file

@ -1,5 +1,4 @@
import abc
import concurrent.futures
import contextlib
import json
import os
@ -67,6 +66,10 @@ class SourceService(host.Service):
def fetch_one(self, checksum, desc) -> None:
"""Performs the actual fetch of an element described by its checksum and its descriptor"""
@abc.abstractmethod
def download(self, items: Dict) -> None:
"""Download all sources."""
def exists(self, checksum, _desc) -> bool:
"""Returns True if the item to download is in cache. """
return os.path.isfile(f"{self.cache}/{checksum}")
@ -76,14 +79,6 @@ class SourceService(host.Service):
"""Modify the input data before downloading. By default only transforms an item object to a Tupple."""
return checksum, desc
def download(self, items: Dict) -> None:
filtered = filter(lambda i: not self.exists(i[0], i[1]), items.items()) # discards items already in cache
transformed = map(lambda i: self.transform(i[0], i[1]), filtered) # prepare each item to be downloaded
with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as executor:
for _ in executor.map(self.fetch_one, *zip(*transformed)):
pass
@staticmethod
def load_items(fds):
with os.fdopen(fds.steal(0)) as f: