#!/usr/bin/python3 """Fetch OSTree commits from an repository Uses ostree to pull specific commits from (remote) repositories at the provided `url`. Can verify the commit, if one or more gpg keys are provided via `gpgkeys`. The secret providers currently supported are: - `org.osbuild.mtls` for downloading content that requires client certificate. The paths to the key and cert should be set in the environment in OSBUILD_SOURCES_OSTREE_SSL_CLIENT_KEY, OSBUILD_SOURCES_OSTREE_SSL_CLIENT_CERT, and optionally OSBUILD_SOURCES_OSTREE_SSL_CA_CERT. - `org.osbuild.rhsm.consumer` for downloading content using RHSM entitlement certificate. Secret fields `consumer_cert` and `consumer_key` must be set. To skip TLS verification, set OSBUILD_SOURCES_OSTREE_INSECURE environment variable to "true". To set a HTTP(S) proxy, set OSBUILD_SOURCES_OSTREE_PROXY environment variable to the proxy URL. """ import concurrent.futures import os import sys import uuid from typing import Dict from osbuild import sources from osbuild.util import ostree SCHEMA = """ "additionalProperties": false, "definitions": { "item": { "description": "The commits to fetch indexed their checksum", "type": "object", "additionalProperties": false, "patternProperties": { "[0-9a-f]{5,64}": { "type": "object", "additionalProperties": false, "required": ["remote"], "properties": { "remote": { "type": "object", "additionalProperties": false, "required": ["url"], "properties": { "url": { "type": "string", "description": "URL of the repository." }, "contenturl": { "type": "string", "description": "content URL of the repository." }, "gpgkeys": { "type": "array", "items": { "type": "string", "description": "GPG keys to verify the commits" } }, "secrets": { "type": "object", "additionalProperties": false, "required": [ "name" ], "properties": { "name": { "type": "string", "description": "Name of the secrets provider." } } } } } } } } } }, "properties": { "items": {"$ref": "#/definitions/item"}, "commits": {"$ref": "#/definitions/item"} }, "oneOf": [{ "required": ["items"] }, { "required": ["commits"] }] """ class OSTreeSource(sources.SourceService): content_type = "org.osbuild.ostree" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.repo = None def fetch_all(self, items: Dict) -> None: filtered = filter(lambda i: not self.exists(i[0], i[1]), items.items()) # discards items already in cache with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as executor: for _ in executor.map(self.fetch_one, *zip(*filtered)): pass def fetch_one(self, checksum, desc): commit = checksum remote = desc["remote"] # This is a temporary remote so we'll just use a random name name = str(uuid.uuid4()) ostree.setup_remote(self.repo, name, remote) # Transfer the commit: remote → cache print(f"pulling {commit}", file=sys.stderr) ostree.cli("pull", name, commit, repo=self.repo) # Remove the temporary remote again ostree.cli("remote", "delete", name, repo=self.repo) def setup(self, args): super().setup(args) # Prepare the cache and the output repo self.repo = os.path.join(self.cache, "repo") ostree.cli("init", mode="archive", repo=self.repo) # Make sure the cache repository uses locks to protect the metadata during # shared access. This is the default since `2018.5`, but lets document this # explicitly here. ostree.cli("config", "set", "repo.locking", "true", repo=self.repo) # pylint: disable=[no-self-use] def exists(self, checksum, _desc): try: ostree.show(self.repo, checksum) except RuntimeError: return False return True def main(): service = OSTreeSource.from_args(sys.argv[1:]) service.main() if __name__ == '__main__': main()