particle-os/debian-forge
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

sources: move parallelisation into source

Browse source 
This moves the parallelisation decisions into the sources themselves,
making the `download` method abstract inside `osbuild` itself.
This commit is contained in:
Simon de Vlieger 2024-01-23 11:45:13 +01:00 • committed by Michael Vogt
parent 18e5481ae8
commit 2c42c46c48
6 changed files with 54 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

10
sources/org.osbuild.inline
View file

@ -12,9 +12,11 @@ resource is decoded and written to the store.
import base64
import concurrent.futures
import contextlib
import os
import sys
from typing import Dict
from osbuild import sources
from osbuild.util.checksum import verify_file
@ -56,6 +58,14 @@ class InlineSource(sources.SourceService):
content_type = "org.osbuild.files"
def download(self, items: Dict) -> None:
filtered = filter(lambda i: not self.exists(i[0], i[1]), items.items()) # discards items already in cache
transformed = map(lambda i: self.transform(i[0], i[1]), filtered) # prepare each item to be downloaded
with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as executor:
for _ in executor.map(self.fetch_one, *zip(*transformed)):
pass
def fetch_one(self, checksum, desc):
target = os.path.join(self.cache, checksum)
floating = os.path.join(self.tmpdir, checksum)