155 lines
4.5 KiB
Python
Executable file
155 lines
4.5 KiB
Python
Executable file
#!/usr/bin/python3
|
|
"""Fetch OSTree commits from an repository
|
|
|
|
Uses ostree to pull specific commits from (remote) repositories
|
|
at the provided `url`. Can verify the commit, if one or more
|
|
gpg keys are provided via `gpgkeys`. The secret providers currently
|
|
supported are:
|
|
|
|
- `org.osbuild.mtls` for downloading content that requires client
|
|
certificate. The paths to the key and cert should be set in the
|
|
environment in OSBUILD_SOURCES_OSTREE_SSL_CLIENT_KEY,
|
|
OSBUILD_SOURCES_OSTREE_SSL_CLIENT_CERT, and optionally
|
|
OSBUILD_SOURCES_OSTREE_SSL_CA_CERT.
|
|
|
|
- `org.osbuild.rhsm.consumer` for downloading content using RHSM
|
|
entitlement certificate. Secret fields `consumer_cert` and
|
|
`consumer_key` must be set.
|
|
|
|
To skip TLS verification, set OSBUILD_SOURCES_OSTREE_INSECURE environment
|
|
variable to "true". To set a HTTP(S) proxy, set OSBUILD_SOURCES_OSTREE_PROXY
|
|
environment variable to the proxy URL.
|
|
"""
|
|
|
|
|
|
import concurrent.futures
|
|
import os
|
|
import sys
|
|
import uuid
|
|
from typing import Dict
|
|
|
|
from osbuild import sources
|
|
from osbuild.util import ostree
|
|
|
|
SCHEMA = """
|
|
"additionalProperties": false,
|
|
"definitions": {
|
|
"item": {
|
|
"description": "The commits to fetch indexed their checksum",
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"patternProperties": {
|
|
"[0-9a-f]{5,64}": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["remote"],
|
|
"properties": {
|
|
"remote": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["url"],
|
|
"properties": {
|
|
"url": {
|
|
"type": "string",
|
|
"description": "URL of the repository."
|
|
},
|
|
"contenturl": {
|
|
"type": "string",
|
|
"description": "content URL of the repository."
|
|
},
|
|
"gpgkeys": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string",
|
|
"description": "GPG keys to verify the commits"
|
|
}
|
|
},
|
|
"secrets": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": [
|
|
"name"
|
|
],
|
|
"properties": {
|
|
"name": {
|
|
"type": "string",
|
|
"description": "Name of the secrets provider."
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"properties": {
|
|
"items": {"$ref": "#/definitions/item"},
|
|
"commits": {"$ref": "#/definitions/item"}
|
|
},
|
|
"oneOf": [{
|
|
"required": ["items"]
|
|
}, {
|
|
"required": ["commits"]
|
|
}]
|
|
"""
|
|
|
|
|
|
class OSTreeSource(sources.SourceService):
|
|
|
|
content_type = "org.osbuild.ostree"
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
self.repo = None
|
|
|
|
def fetch_all(self, items: Dict) -> None:
|
|
filtered = filter(lambda i: not self.exists(i[0], i[1]), items.items()) # discards items already in cache
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as executor:
|
|
for _ in executor.map(self.fetch_one, *zip(*filtered)):
|
|
pass
|
|
|
|
def fetch_one(self, checksum, desc):
|
|
commit = checksum
|
|
remote = desc["remote"]
|
|
# This is a temporary remote so we'll just use a random name
|
|
name = str(uuid.uuid4())
|
|
|
|
ostree.setup_remote(self.repo, name, remote)
|
|
|
|
# Transfer the commit: remote → cache
|
|
print(f"pulling {commit}", file=sys.stderr)
|
|
ostree.cli("pull", name, commit, repo=self.repo)
|
|
|
|
# Remove the temporary remote again
|
|
ostree.cli("remote", "delete", name, repo=self.repo)
|
|
|
|
def setup(self, args):
|
|
super().setup(args)
|
|
# Prepare the cache and the output repo
|
|
self.repo = os.path.join(self.cache, "repo")
|
|
ostree.cli("init", mode="archive", repo=self.repo)
|
|
|
|
# Make sure the cache repository uses locks to protect the metadata during
|
|
# shared access. This is the default since `2018.5`, but lets document this
|
|
# explicitly here.
|
|
ostree.cli("config", "set", "repo.locking", "true", repo=self.repo)
|
|
|
|
# pylint: disable=[no-self-use]
|
|
def exists(self, checksum, _desc):
|
|
try:
|
|
ostree.show(self.repo, checksum)
|
|
except RuntimeError:
|
|
return False
|
|
return True
|
|
|
|
|
|
def main():
|
|
service = OSTreeSource.from_args(sys.argv[1:])
|
|
service.main()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|