1de74ce2c9
Before, the download method was defined in the inherited class of each program. With the same kind of workflow redefined every time. This contribution aims at making the workflow more clear and to generalize what can be in the SourceService class. The download worklow is as follow: Setup -> Filter -> Prepare -> Download The setup mainly step sets up caches. Where the download data will be stored in the end. The filter step is used to discard some of the items to download based on some criterion. By default, it is used to verify if an item is already in the cache using the item's checksum. The Prepare step goes from each element and let the overloading step the ability to alter each item before downloading it. This is used mainly for the curl command which for rhel must generate the subscriptions. Then the download step will call fetch_one for each item. Here the download can be performed sequentially or in parallel depending on the number of workers selected.
88 lines
2.2 KiB
Python
Executable file
88 lines
2.2 KiB
Python
Executable file
#!/usr/bin/python3
|
|
"""Source for binary data encoded inline in the manifest
|
|
|
|
This source can be used to transport data in the source
|
|
section of the manifest. Each resource is ascii-encoded
|
|
in the `data` property, where the encoding is specified
|
|
in the `encoding` property. The resources is content
|
|
addressed via the hash value of the raw data before the
|
|
ascii encoding. This hash value is verified after the
|
|
resource is decoded and written to the store.
|
|
"""
|
|
|
|
|
|
import base64
|
|
import contextlib
|
|
import os
|
|
import sys
|
|
|
|
from osbuild import sources
|
|
from osbuild.util.checksum import verify_file
|
|
|
|
|
|
SCHEMA = """
|
|
"definitions": {
|
|
"item": {
|
|
"description": "Inline data indexed by their checksum",
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"patternProperties": {
|
|
"(md5|sha1|sha256|sha384|sha512):[0-9a-f]{32,128}": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["encoding", "data"],
|
|
"properties": {
|
|
"encoding": {
|
|
"description": "The specific encoding of `data`",
|
|
"enum": ["base64"]
|
|
},
|
|
"data": {
|
|
"description": "The ascii encoded raw data",
|
|
"type": "string"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"additionalProperties": false,
|
|
"required": ["items"],
|
|
"properties": {
|
|
"items": {"$ref": "#/definitions/item"}
|
|
}
|
|
"""
|
|
|
|
|
|
class InlineSource(sources.SourceService):
|
|
|
|
content_type = "org.osbuild.files"
|
|
|
|
def fetch_one(self, checksum, desc):
|
|
target = os.path.join(self.cache, checksum)
|
|
floating = os.path.join(self.tmpdir, checksum)
|
|
|
|
if os.path.isfile(target):
|
|
return
|
|
|
|
data = base64.b64decode(desc["data"])
|
|
|
|
# Write the bits to disk and then verify the checksum
|
|
# This ensures that 1) the data is ok and that 2) we
|
|
# wrote them correctly as well
|
|
with open(floating, "wb") as f:
|
|
f.write(data)
|
|
|
|
if not verify_file(floating, checksum):
|
|
raise RuntimeError(f"Checksum mismatch for {format(checksum)}")
|
|
|
|
with contextlib.suppress(FileExistsError):
|
|
os.rename(floating, target)
|
|
|
|
|
|
def main():
|
|
service = InlineSource.from_args(sys.argv[1:])
|
|
service.main()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|