particle-os/debian-forge
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

sources: add org.osbuild.files source

Browse source 
This source adds support for downloaded files. The files are
indexed by their content hash, and the only option is their URL.

The main usecase for this will be downloading rpms. Allowing depsolving
to be done outside of osbuild, network access to be restricted and
downloaded rpms to be reused between runs.

Each source is now passed two additional arguments, a cache directory
and an output directory. Both are in the source's namespace, and
the source is responsible for managing them. Each directory may
contain contents from previous runs, but neither is ever guaranteed
to do so.

Downloaded contents may be saved to the cache and resued between
runs, and the requested content should be written to the output dir.
If secrets are used, the source must only ever write contents to
the output that corresponds to the available secrets (rather than
contents from the cache from previous runs).

Each stage is passed an additional argument, a sources directory.
The directory is read-only, and contains a subdirectory named after
each used source, which will contain the requseted contents when
the `Get()` call returns (if the source uses this functionality).

Based on a patch by Lars Karlitski.

Signed-off-by: Tom Gundersen <teg@jklm.no>
This commit is contained in:
Tom Gundersen 2020-02-01 23:54:53 +01:00
parent 794ec97bf3
commit 7817ae5e8b
38 changed files with 348 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

3
.travis.yml
View file

@ -25,6 +25,9 @@ jobs:
script:
- sudo env "PATH=$PATH" python3 -m osbuild --libdir . --build-env samples/ubuntu1804.json samples/noop.json
- sudo env "PATH=$PATH" python3 -m osbuild --libdir . --build-env samples/ubuntu1804.json samples/noop.json
- name: sources-tests
before_install: sudo apt-get install -y rpm
script: sudo env "PATH=$PATH" python3 -m unittest -v test.test_sources
- name: f30-boot
before_install: sudo apt-get install -y systemd-container yum qemu-kvm
script: sudo env "PATH=$PATH" "OSBUILD_TEST_BUILD_ENV=samples/f27-build-from-ubuntu1804.json" python3 -m unittest -v test.test_boot

1
osbuild.spec
View file

@ -16,6 +16,7 @@ BuildRequires: python3-devel
Requires: bash
Requires: coreutils
Requires: curl
Requires: dnf
Requires: e2fsprogs
Requires: glibc

15
osbuild/pipeline.py
View file

@ -73,23 +73,26 @@ class Stage:
tree,
runner,
build_tree,
cache,
interactive=False,
libdir=None,
var="/var/tmp",
source_options=None,
secrets=None):
with buildroot.BuildRoot(build_tree, runner, libdir=libdir, var=var) as build_root:
with buildroot.BuildRoot(build_tree, runner, libdir=libdir, var=var) as build_root, \
tempfile.TemporaryDirectory(prefix="osbuild-sources-output-", dir=var) as sources_output:
if interactive:
print_header(f"{self.name}: {self.id}", self.options)
args = {
"tree": "/run/osbuild/tree",
"sources": "/run/osbuild/sources",
"options": self.options,
}
sources_dir = f"{libdir}/sources" if libdir else "/usr/lib/osbuild/sources"
ro_binds = []
ro_binds = [f"{sources_output}:/run/osbuild/sources"]
if not libdir:
osbuild_module_path = os.path.dirname(importlib.util.find_spec('osbuild').origin)
# This is a temporary workaround, once we have a common way to include osbuild in the
@ -98,7 +101,12 @@ class Stage:
ro_binds.append(f"{osbuild_module_path}:/run/osbuild/lib/stages/osbuild")
with API(f"{build_root.api}/osbuild", args, interactive) as api, \
sources.SourcesServer(f"{build_root.api}/sources", sources_dir, source_options, secrets):
sources.SourcesServer(f"{build_root.api}/sources",
sources_dir,
source_options,
f"{cache}/sources",
sources_output,
secrets):
r = build_root.run(
[f"/run/osbuild/lib/stages/{self.name}"],
binds=[f"{tree}:/run/osbuild/tree"],
@ -267,6 +275,7 @@ class Pipeline:
r = stage.run(tree,
self.runner,
build_tree,
store,
interactive=interactive,
libdir=libdir,
var=store,

22
osbuild/sources.py
View file

@ -6,23 +6,29 @@ import threading
class SourcesServer:
def __init__(self, socket_address, sources_dir, source_options, secrets=None):
# pylint: disable=too-many-instance-attributes
def __init__(self, socket_address, sources_libdir, options, cache, output, secrets=None):
self.socket_address = socket_address
self.sources_dir = sources_dir
self.source_options = source_options or {}
self.sources_libdir = sources_libdir
self.cache = cache
self.output = output
self.options = options or {}
self.secrets = secrets or {}
self.event_loop = asyncio.new_event_loop()
self.thread = threading.Thread(target=self._run_event_loop)
self.barrier = threading.Barrier(2)
def _run_source(self, source, checksums):
msg = {
"options": self.source_options.get(source, {}),
"options": self.options.get(source, {}),
"secrets": self.secrets.get(source, {}),
"cache": f"{self.cache}/{source}",
"output": f"{self.output}/{source}",
"checksums": checksums
}
r = subprocess.run(
[f"{self.sources_dir}/{source}"],
[f"{self.sources_libdir}/{source}"],
input=json.dumps(msg),
stdout=subprocess.PIPE,
encoding="utf-8",
@ -43,6 +49,7 @@ class SourcesServer:
def _run_event_loop(self):
sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
sock.bind(self.socket_address)
self.barrier.wait()
self.event_loop.add_reader(sock, self._dispatch, sock)
asyncio.set_event_loop(self.event_loop)
self.event_loop.run_forever()
@ -51,6 +58,7 @@ class SourcesServer:
def __enter__(self):
self.thread.start()
self.barrier.wait()
return self
def __exit__(self, *args):
@ -58,10 +66,10 @@ class SourcesServer:
self.thread.join()
def get(source, checksums):
def get(source, checksums, api_path="/run/osbuild/api/sources"):
with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1)
sock.connect("/run/osbuild/api/sources")
sock.connect(api_path)
msg = {
"source": source,
"checksums": checksums

108
sources/org.osbuild.files Executable file
View file

@ -0,0 +1,108 @@
#!/usr/bin/python3
import concurrent.futures
import itertools
import json
import os
import subprocess
import sys
import tempfile
def verify_checksum(filename, checksum):
algorithm, checksum = checksum.split(":", 1)
if algorithm not in ("md5", "sha1", "sha256", "sha384", "sha512"):
raise RuntimeError(f"unsupported checksum algorithm: {algorithm}")
ret = subprocess.run(
[f"{algorithm}sum", "-c"],
input=f"{checksum} {filename}",
stdout=subprocess.DEVNULL,
encoding="utf-8",
check=False
)
return ret.returncode == 0
def fetch(url, checksum, directory):
# Invariant: all files in @directory must be named after their (verified) checksum.
if os.path.isfile(f"{directory}/{checksum}"):
return
# Download to a temporary directory until we have verified the checksum. Use a
# subdirectory, so we avoid copying accross block devices.
with tempfile.TemporaryDirectory(prefix="osbuild-unverified-file-", dir=directory) as tmpdir:
# some mirrors are broken sometimes. retry manually, because curl doesn't on 404
for _ in range(3):
curl = subprocess.run([
"curl",
"--silent",
"--show-error",
"--fail",
"--location",
"--output", checksum,
url
], encoding="utf-8", cwd=tmpdir, check=False)
if curl.returncode == 0:
break
else:
raise RuntimeError(f"error downloading {url}")
if not verify_checksum(f"{tmpdir}/{checksum}", checksum):
raise RuntimeError(f"checksum mismatch: {checksum} {url}")
# The checksum has been verified, move the file into place. in case we race
# another download of the same file, we simply ignore the error as their
# contents are guaranteed to be the same.
try:
os.rename(f"{tmpdir}/{checksum}", f"{directory}/{checksum}")
except FileExistsError:
pass
def main(options, checksums, cache, output):
urls = options.get("urls", {})
os.makedirs(cache, exist_ok=True)
os.makedirs(output, exist_ok=True)
with concurrent.futures.ProcessPoolExecutor(max_workers=10) as executor:
requested_urls = []
for checksum in checksums:
try:
requested_urls.append(urls[checksum])
except KeyError:
json.dump({"error": f"unknown file: {checksum}"}, sys.stdout)
return 1
results = executor.map(fetch, requested_urls, checksums, itertools.repeat(cache))
try:
for _ in results:
pass
except RuntimeError as e:
json.dump({"error": e.args[0]}, sys.stdout)
return 1
for checksum in checksums:
try:
subprocess.run([
"cp",
"--reflink=auto",
f"{cache}/{checksum}",
f"{output}/{checksum}"],
check=True)
except FileExistsError:
continue
except Exception as e:
json.dump({"error": e.message}, sys.stdout)
return 1
json.dump({}, sys.stdout)
return 0
if __name__ == '__main__':
args = json.load(sys.stdin)
r = main(args["options"], args["checksums"], args["cache"], args["output"])
sys.exit(r)

4
test/sources_tests/org.osbuild.files/cases/empty.json Normal file
View file

@ -0,0 +1,4 @@
{
"expects": "success",
"checksums": []
}

6
test/sources_tests/org.osbuild.files/cases/invalid_checksum.json Normal file
View file

@ -0,0 +1,6 @@
{
"expects": "error",
"checksums": [
"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
]
}

6
test/sources_tests/org.osbuild.files/cases/missing_file.json Normal file
View file

@ -0,0 +1,6 @@
{
"expects": "error",
"checksums": [
"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
]
}

31
test/sources_tests/org.osbuild.files/cases/success.json Normal file
View file

@ -0,0 +1,31 @@
{
"expects": "success",
"checksums": [
"sha256:87428fc522803d31065e7bce3cf03fe475096631e5e07bbd7a0fde60c4cf25c7",
"sha256:0263829989b6fd954f72baaf2fc64bc2e2f01d692d4de72986ea808f6e99813f",
"sha256:a3a5e715f0cc574a73c3f9bebb6bc24f32ffd5b67b387244c2c909da779a1478",
"sha256:8d74beec1be996322ad76813bafb92d40839895d6dd7ee808b17ca201eac98be",
"sha256:a2bbdb2de53523b8099b37013f251546f3d65dbe7a0774fa41af0a4176992fd4",
"sha256:092fcfbbcfca3b5be7ae1b5e58538e92c35ab273ae13664fed0d67484c8e78a6",
"sha256:768c71d785bf6bbbf8c4d6af6582041f2659027140a962cd0c55b11eddfd5e3d",
"sha256:91ee5e9f42ba3d34e414443b36a27b797a56a47aad6bb1e4c1769e69c77ce0ca",
"sha256:50c393f158c3de2db92fa9661bfb00eda5b67c3a777c88524ed3417509631625",
"sha256:cee00b08a818db87e17e703273818e5194f83280e1ef3eae9214ff14675d9e6d",
"sha256:19732980d68fbd00358a0a4d98246c960400b87e4fa2a2e155db98be2b42ed6c",
"sha256:6d7ebc44c5bc26207e62f4f628f912e1a0f41ed11764891aa7dd99eab83228e7",
"sha256:01a60e35df88d8b49546cb3f8f4ba4f406870f9b8e1f394c9d48ab73548d748d",
"sha256:a4fb621495a0122493b2203591c448903c472e306a1ede54fabad829e01075c0",
"sha256:7427d152005f9ed0fa31c76ef9963cf4bb47dce6e2768111d9eb0edbfe59c704",
"sha256:fd6641673e7f3bf6e80e4bc5401fcb2821a1e117206c8e1c65cef23a58dc37ff",
"sha256:4adc33bd9fe74303c344be46e5916d65182fb218e248fe80452ab3f025b06c64",
"sha256:8e54b0ca18020275e4aef1ca0eb5e197e066c065c1864817652a8a39c55402cd",
"sha256:cbc80bb5c0c0f8944bf73b3a429505ac5cde16644978bc9a1e74c5755f8ca556",
"sha256:fe8edeeb98cc6d3b93cf2d57000254b84bd9eba34b4df7ce4b87db8b937b7703",
"sha256:ea46748e171abd2dd4dba5b86bb6589334d86bba2df8d50cbb16b36c83b0856a",
"sha256:73324e1ab1db72ee9eb4fdf1c90a586d67e00ab58330d1cbfea26ecd0a77fa4d",
"sha256:cf945b5236e101dbe0471d5200f28b1ae64f21c1f35bf55fcf40cd0fe42cd8e7",
"sha256:73cb3858a687a8494ca3323053016282f3dad39d42cf62ca4e79dda2aac7d9ac",
"sha256:3bb2abb69ebb27fbfe63c7639624c6ec5e331b841a5bc8c3ebc10b9285e90877",
"sha256:c865f6c5ab8d1b0bcd383a5e1e3879d22681c96bf462c269b7581d523fbe70ab"
]
}

6
test/sources_tests/org.osbuild.files/cases/unknown_checksum.json Normal file
View file

@ -0,0 +1,6 @@
{
"expects": "error",
"checksums": [
"sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
]
}

1
test/sources_tests/org.osbuild.files/data/a Normal file
View file

@ -0,0 +1 @@
a

1
test/sources_tests/org.osbuild.files/data/b Normal file
View file

@ -0,0 +1 @@
b

1
test/sources_tests/org.osbuild.files/data/c Normal file
View file

@ -0,0 +1 @@
c

1
test/sources_tests/org.osbuild.files/data/d Normal file
View file

@ -0,0 +1 @@
d

1
test/sources_tests/org.osbuild.files/data/e Normal file
View file

@ -0,0 +1 @@
e

1
test/sources_tests/org.osbuild.files/data/f Normal file
View file

@ -0,0 +1 @@
f

1
test/sources_tests/org.osbuild.files/data/g Normal file
View file

@ -0,0 +1 @@
g

1
test/sources_tests/org.osbuild.files/data/h Normal file
View file

@ -0,0 +1 @@
h

1
test/sources_tests/org.osbuild.files/data/i Normal file
View file

@ -0,0 +1 @@
i

1
test/sources_tests/org.osbuild.files/data/j Normal file
View file

@ -0,0 +1 @@
j

1
test/sources_tests/org.osbuild.files/data/k Normal file
View file

@ -0,0 +1 @@
k

1
test/sources_tests/org.osbuild.files/data/l Normal file
View file

@ -0,0 +1 @@
l

1
test/sources_tests/org.osbuild.files/data/m Normal file
View file

@ -0,0 +1 @@
m

1
test/sources_tests/org.osbuild.files/data/n Normal file
View file

@ -0,0 +1 @@
n

1
test/sources_tests/org.osbuild.files/data/o Normal file
View file

@ -0,0 +1 @@
o

1
test/sources_tests/org.osbuild.files/data/p Normal file
View file

@ -0,0 +1 @@
p

1
test/sources_tests/org.osbuild.files/data/q Normal file
View file

@ -0,0 +1 @@
q

1
test/sources_tests/org.osbuild.files/data/r Normal file
View file

@ -0,0 +1 @@
r

1
test/sources_tests/org.osbuild.files/data/s Normal file
View file

@ -0,0 +1 @@
s

1
test/sources_tests/org.osbuild.files/data/t Normal file
View file

@ -0,0 +1 @@
t

1
test/sources_tests/org.osbuild.files/data/u Normal file
View file

@ -0,0 +1 @@
u

1
test/sources_tests/org.osbuild.files/data/v Normal file
View file

@ -0,0 +1 @@
v

1
test/sources_tests/org.osbuild.files/data/w Normal file
View file

@ -0,0 +1 @@
w

1
test/sources_tests/org.osbuild.files/data/x Normal file
View file

@ -0,0 +1 @@
x

1
test/sources_tests/org.osbuild.files/data/y Normal file
View file

@ -0,0 +1 @@
y

1
test/sources_tests/org.osbuild.files/data/z Normal file
View file

@ -0,0 +1 @@
z

34
test/sources_tests/org.osbuild.files/sources.json Normal file
View file

@ -0,0 +1,34 @@
{
"org.osbuild.files": {
"urls": {
"sha256:87428fc522803d31065e7bce3cf03fe475096631e5e07bbd7a0fde60c4cf25c7": "http://localhost/test/sources_tests/org.osbuild.files/data/a",
"sha256:0263829989b6fd954f72baaf2fc64bc2e2f01d692d4de72986ea808f6e99813f": "http://localhost/test/sources_tests/org.osbuild.files/data/b",
"sha256:a3a5e715f0cc574a73c3f9bebb6bc24f32ffd5b67b387244c2c909da779a1478": "http://localhost/test/sources_tests/org.osbuild.files/data/c",
"sha256:8d74beec1be996322ad76813bafb92d40839895d6dd7ee808b17ca201eac98be": "http://localhost/test/sources_tests/org.osbuild.files/data/d",
"sha256:a2bbdb2de53523b8099b37013f251546f3d65dbe7a0774fa41af0a4176992fd4": "http://localhost/test/sources_tests/org.osbuild.files/data/e",
"sha256:092fcfbbcfca3b5be7ae1b5e58538e92c35ab273ae13664fed0d67484c8e78a6": "http://localhost/test/sources_tests/org.osbuild.files/data/f",
"sha256:768c71d785bf6bbbf8c4d6af6582041f2659027140a962cd0c55b11eddfd5e3d": "http://localhost/test/sources_tests/org.osbuild.files/data/g",
"sha256:91ee5e9f42ba3d34e414443b36a27b797a56a47aad6bb1e4c1769e69c77ce0ca": "http://localhost/test/sources_tests/org.osbuild.files/data/h",
"sha256:50c393f158c3de2db92fa9661bfb00eda5b67c3a777c88524ed3417509631625": "http://localhost/test/sources_tests/org.osbuild.files/data/i",
"sha256:cee00b08a818db87e17e703273818e5194f83280e1ef3eae9214ff14675d9e6d": "http://localhost/test/sources_tests/org.osbuild.files/data/j",
"sha256:19732980d68fbd00358a0a4d98246c960400b87e4fa2a2e155db98be2b42ed6c": "http://localhost/test/sources_tests/org.osbuild.files/data/k",
"sha256:6d7ebc44c5bc26207e62f4f628f912e1a0f41ed11764891aa7dd99eab83228e7": "http://localhost/test/sources_tests/org.osbuild.files/data/l",
"sha256:01a60e35df88d8b49546cb3f8f4ba4f406870f9b8e1f394c9d48ab73548d748d": "http://localhost/test/sources_tests/org.osbuild.files/data/m",
"sha256:a4fb621495a0122493b2203591c448903c472e306a1ede54fabad829e01075c0": "http://localhost/test/sources_tests/org.osbuild.files/data/n",
"sha256:7427d152005f9ed0fa31c76ef9963cf4bb47dce6e2768111d9eb0edbfe59c704": "http://localhost/test/sources_tests/org.osbuild.files/data/o",
"sha256:fd6641673e7f3bf6e80e4bc5401fcb2821a1e117206c8e1c65cef23a58dc37ff": "http://localhost/test/sources_tests/org.osbuild.files/data/p",
"sha256:4adc33bd9fe74303c344be46e5916d65182fb218e248fe80452ab3f025b06c64": "http://localhost/test/sources_tests/org.osbuild.files/data/q",
"sha256:8e54b0ca18020275e4aef1ca0eb5e197e066c065c1864817652a8a39c55402cd": "http://localhost/test/sources_tests/org.osbuild.files/data/r",
"sha256:cbc80bb5c0c0f8944bf73b3a429505ac5cde16644978bc9a1e74c5755f8ca556": "http://localhost/test/sources_tests/org.osbuild.files/data/s",
"sha256:fe8edeeb98cc6d3b93cf2d57000254b84bd9eba34b4df7ce4b87db8b937b7703": "http://localhost/test/sources_tests/org.osbuild.files/data/t",
"sha256:ea46748e171abd2dd4dba5b86bb6589334d86bba2df8d50cbb16b36c83b0856a": "http://localhost/test/sources_tests/org.osbuild.files/data/u",
"sha256:73324e1ab1db72ee9eb4fdf1c90a586d67e00ab58330d1cbfea26ecd0a77fa4d": "http://localhost/test/sources_tests/org.osbuild.files/data/v",
"sha256:cf945b5236e101dbe0471d5200f28b1ae64f21c1f35bf55fcf40cd0fe42cd8e7": "http://localhost/test/sources_tests/org.osbuild.files/data/w",
"sha256:73cb3858a687a8494ca3323053016282f3dad39d42cf62ca4e79dda2aac7d9ac": "http://localhost/test/sources_tests/org.osbuild.files/data/x",
"sha256:3bb2abb69ebb27fbfe63c7639624c6ec5e331b841a5bc8c3ebc10b9285e90877": "http://localhost/test/sources_tests/org.osbuild.files/data/y",
"sha256:c865f6c5ab8d1b0bcd383a5e1e3879d22681c96bf462c269b7581d523fbe70ab": "http://localhost/test/sources_tests/org.osbuild.files/data/z",
"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": "http://localhost/test/sources_tests/org.osbuild.files/data/a",
"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb": "http://localhost/test/sources_tests/org.osbuild.files/data/missing"
}
}
}

96
test/test_sources.py Normal file
View file

@ -0,0 +1,96 @@
import contextlib
import ctypes
import json
import os
import osbuild.sources
import socketserver
import subprocess
import tempfile
import threading
import unittest
from http import server
def errcheck(ret, func, args):
if ret == -1:
e = ctypes.get_errno()
raise OSError(e, os.strerror(e))
CLONE_NEWNET = 0x40000000
libc = ctypes.CDLL('libc.so.6', use_errno=True)
libc.setns.errcheck = errcheck
@contextlib.contextmanager
def netns():
# Grab a reference to the current namespace.
with open("/proc/self/ns/net") as oldnet:
# Create a new namespace and enter it.
libc.unshare(CLONE_NEWNET)
# Up the loopback device in the new namespace.
subprocess.run(["ip", "link", "set", "up", "dev", "lo"], check=True)
try:
yield
finally:
# Revert to the old namespace, dropping our
# reference to the new one.
libc.setns(oldnet.fileno(), CLONE_NEWNET)
@contextlib.contextmanager
def fileServer(path):
with netns():
# This is leaked until the program exits, but inaccessible after the with
# due to the network namespace.
barrier = threading.Barrier(2)
thread = threading.Thread(target=runFileServer, args=(path, barrier))
thread.daemon = True
thread.start()
barrier.wait()
yield
def runFileServer(path, barrier):
httpd = socketserver.TCPServer(('', 80), server.SimpleHTTPRequestHandler)
barrier.wait()
httpd.serve_forever()
class TestSources(unittest.TestCase):
def setUp(self):
self.sources = 'test/sources_tests'
def check_case(self, source, case, destdir, api_path):
expects = case["expects"]
if expects == "error":
with self.assertRaises(RuntimeError):
osbuild.sources.get(source, case["checksums"], api_path=api_path)
elif expects == "success":
r = osbuild.sources.get(source, case["checksums"], api_path=api_path)
self.assertEqual(r, {})
else:
raise ValueError(f"invalid expectation: {expects}")
def check_source(self, source):
source_options = {}
with open(f"{self.sources}/{source}/sources.json") as f:
source_options = json.load(f)
for case in os.listdir(f"{self.sources}/{source}/cases"):
with self.subTest(case=case):
case_options = {}
with open(f"{self.sources}/{source}/cases/{case}") as f:
case_options = json.load(f)
with tempfile.TemporaryDirectory() as tmpdir, \
fileServer(f"{self.sources}/{source}/data"), \
osbuild.sources.SourcesServer(f"{tmpdir}/sources-api", "./sources", source_options, f"{tmpdir}/cache", f"{tmpdir}/dst"):
self.check_case(source, case_options, f"{tmpdir}/dst", f"{tmpdir}/sources-api")
self.check_case(source, case_options, f"{tmpdir}/dst", f"{tmpdir}/sources-api")
def test_sources(self):
for source in os.listdir(self.sources):
with self.subTest(source=source):
self.check_source(source)