c902a7a754
Port sources to also use the host services infrastructure that is used by inputs, devices and mounts. Sources are a bit different from the other services that they don't run for the duration of the stage but are run before anything is built. By using the same infrastructure we re-use the process management and inter process communcation. Additionally, this will forward all messages from sources to the existing monitoring framework. Adapt all existing sources and tests.
98 lines
2.5 KiB
Python
Executable file
98 lines
2.5 KiB
Python
Executable file
#!/usr/bin/python3
|
|
"""Source for binary data encoded inline in the manifest
|
|
|
|
This source can be used to transport data in the source
|
|
section of the manifest. Each resource is ascii-encoded
|
|
in the `data` property, where the encoding is specified
|
|
in the `encoding` property. The resources is content
|
|
addressed via the hash value of the raw data before the
|
|
ascii encoding. This hash value is verified after the
|
|
resource is decoded and written to the store.
|
|
"""
|
|
|
|
|
|
import base64
|
|
import contextlib
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
|
|
from typing import Dict
|
|
|
|
from osbuild import sources
|
|
from osbuild.util.checksum import verify_file
|
|
|
|
|
|
SCHEMA = """
|
|
"definitions": {
|
|
"item": {
|
|
"description": "Inline data indexed by their checksum",
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"patternProperties": {
|
|
"(md5|sha1|sha256|sha384|sha512):[0-9a-f]{32,128}": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["encoding", "data"],
|
|
"properties": {
|
|
"encoding": {
|
|
"description": "The specific encoding of `data`",
|
|
"enum": ["base64"]
|
|
},
|
|
"data": {
|
|
"description": "The ascii encoded raw data",
|
|
"type": "string"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"additionalProperties": false,
|
|
"required": ["items"],
|
|
"properties": {
|
|
"items": {"$ref": "#/definitions/item"}
|
|
}
|
|
"""
|
|
|
|
|
|
def process(items: Dict, cache: str, tmpdir):
|
|
for checksum, item in items.items():
|
|
target = os.path.join(cache, checksum)
|
|
floating = os.path.join(tmpdir, checksum)
|
|
|
|
if os.path.isfile(target):
|
|
return
|
|
|
|
data = base64.b64decode(item["data"])
|
|
|
|
# Write the bits to disk and then verify the checksum
|
|
# This ensures that 1) the data is ok and that 2) we
|
|
# wrote them correctly as well
|
|
with open(floating, "wb") as f:
|
|
f.write(data)
|
|
|
|
if not verify_file(floating, checksum):
|
|
raise RuntimeError("Checksum mismatch for {}".format(checksum))
|
|
|
|
with contextlib.suppress(FileExistsError):
|
|
os.rename(floating, target)
|
|
|
|
|
|
class InlineSource(sources.SourceService):
|
|
|
|
def download(self, items, cache, _options):
|
|
cache = os.path.join(cache, "org.osbuild.files")
|
|
os.makedirs(cache, exist_ok=True)
|
|
|
|
with tempfile.TemporaryDirectory(prefix=".unverified-", dir=cache) as tmpdir:
|
|
process(items, cache, tmpdir)
|
|
|
|
|
|
def main():
|
|
service = InlineSource.from_args(sys.argv[1:])
|
|
service.main()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|