debian-forge/sources/org.osbuild.ostree

#!/usr/bin/python3
"""Fetch OSTree commits from an repository

Uses ostree to pull specific commits from (remote) repositories
at the provided `url`. Can verify the commit, if one or more
gpg keys are provided via `gpgkeys`. The secret providers currently
supported are:

- `org.osbuild.mtls` for downloading content that requires client
  certificate. The paths to the key and cert should be set in the
  environment in OSBUILD_SOURCES_OSTREE_SSL_CLIENT_KEY,
  OSBUILD_SOURCES_OSTREE_SSL_CLIENT_CERT, and optionally
  OSBUILD_SOURCES_OSTREE_SSL_CA_CERT.

- `org.osbuild.rhsm.consumer` for downloading content using RHSM
  entitlement certificate. Secret fields `consumer_cert` and
  `consumer_key` must be set.

To skip TLS verification, set OSBUILD_SOURCES_OSTREE_INSECURE environment
variable to "true". To set a HTTP(S) proxy, set OSBUILD_SOURCES_OSTREE_PROXY
environment variable to the proxy URL.
"""


import concurrent.futures
import os
import sys
import uuid
from typing import Dict

from osbuild import sources
from osbuild.util import ostree

SCHEMA = """
"additionalProperties": false,
"definitions": {
  "item": {
    "description": "The commits to fetch indexed their checksum",
    "type": "object",
    "additionalProperties": false,
    "patternProperties": {
      "[0-9a-f]{5,64}": {
        "type": "object",
        "additionalProperties": false,
        "required": ["remote"],
        "properties": {
          "remote": {
            "type": "object",
            "additionalProperties": false,
            "required": ["url"],
            "properties": {
              "url": {
                "type": "string",
                "description": "URL of the repository."
              },
              "contenturl": {
                "type": "string",
                "description": "content URL of the repository."
              },
              "gpgkeys": {
                "type": "array",
                "items": {
                  "type": "string",
                  "description": "GPG keys to verify the commits"
                }
              },
              "secrets": {
                "type": "object",
                "additionalProperties": false,
                "required": [
                  "name"
                ],
                "properties": {
                  "name": {
                    "type": "string",
                    "description": "Name of the secrets provider."
                  }
                }
              }
            }
          }
        }
      }
    }
  }
},
"properties": {
  "items": {"$ref": "#/definitions/item"},
  "commits": {"$ref": "#/definitions/item"}
},
"oneOf": [{
  "required": ["items"]
}, {
  "required": ["commits"]
}]
"""


class OSTreeSource(sources.SourceService):

    content_type = "org.osbuild.ostree"

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.repo = None

    def fetch_all(self, items: Dict) -> None:
        filtered = filter(lambda i: not self.exists(i[0], i[1]), items.items())  # discards items already in cache

        with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as executor:
            for _ in executor.map(self.fetch_one, *zip(*filtered)):
                pass

    def fetch_one(self, checksum, desc):
        commit = checksum
        remote = desc["remote"]
        # This is a temporary remote so we'll just use a random name
        name = str(uuid.uuid4())

        ostree.setup_remote(self.repo, name, remote)

        # Transfer the commit: remote → cache
        print(f"pulling {commit}", file=sys.stderr)
        ostree.cli("pull", name, commit, repo=self.repo)

        # Remove the temporary remote again
        ostree.cli("remote", "delete", name, repo=self.repo)

    def setup(self, args):
        super().setup(args)
        # Prepare the cache and the output repo
        self.repo = os.path.join(self.cache, "repo")
        ostree.cli("init", mode="archive", repo=self.repo)

        # Make sure the cache repository uses locks to protect the metadata during
        # shared access. This is the default since `2018.5`, but lets document this
        # explicitly here.
        ostree.cli("config", "set", "repo.locking", "true", repo=self.repo)

    # pylint: disable=[no-self-use]
    def exists(self, checksum, _desc):
        try:
            ostree.show(self.repo, checksum)
        except RuntimeError:
            return False
        return True


def main():
    service = OSTreeSource.from_args(sys.argv[1:])
    service.main()


if __name__ == '__main__':
    main()