From fe00e1efd36e377ff86b68b0d995cbfa763e25a4 Mon Sep 17 00:00:00 2001 From: sanne Date: Fri, 14 Jan 2022 16:03:21 +0100 Subject: [PATCH] containers/osbuild-composer: Allow dnf-json to accept http connections Revert 83e16afda4109b1902be021cf87c5fafdc993e98: With dnf-json running in a container it's easy to run it standalone. --- containers/osbuild-composer/entrypoint.py | 39 +++++++++++++++--- dnf-json | 41 ++++++++++++------- .../regression-old-worker-new-composer.sh | 29 +++++++++++++ 3 files changed, 89 insertions(+), 20 deletions(-) diff --git a/containers/osbuild-composer/entrypoint.py b/containers/osbuild-composer/entrypoint.py index 685378e90..521f596e7 100644 --- a/containers/osbuild-composer/entrypoint.py +++ b/containers/osbuild-composer/entrypoint.py @@ -108,6 +108,13 @@ class Cli(contextlib.AbstractContextManager): dest="dnf_json", help="Disable dnf-json", ) + self._parser.add_argument( + "--dnf-json-port", + type=int, + default=0, + dest="dnf_json_port", + help="Specify the port dnf-json should listen on", + ) self._parser.set_defaults( builtin_worker=False, @@ -258,9 +265,22 @@ class Cli(contextlib.AbstractContextManager): "/usr/libexec/osbuild-composer/dnf-json", ] + if self.args.dnf_json_port: + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + sock.bind(("::", self.args.dnf_json_port)) + sock.listen() + else: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.bind("/run/osbuild-dnf-json/api.sock") + sock.listen() + dnfenv = os.environ.copy() - dnfenv["LISTEN_FDS"] = "0" - dnfenv["LISTEN_FDNAMES"] = "" + dnfenv["LISTEN_FDS"] = "1" + dnfenv["LISTEN_FD"] = str(sock.fileno()) return subprocess.Popen( cmd, @@ -268,6 +288,7 @@ class Cli(contextlib.AbstractContextManager): stdin=subprocess.DEVNULL, stderr=subprocess.STDOUT, env=dnfenv, + pass_fds=[sock.fileno()] ) def run(self): @@ -286,14 +307,20 @@ class Cli(contextlib.AbstractContextManager): if self.args.dnf_json: proc_dnf_json = self._spawn_dnf_json() - proc_composer = self._spawn_composer(sockets) + if any([self.args.weldr_api, self.args.composer_api, self.args.local_worker_api, self.args.remote_worker_api]): + proc_composer = self._spawn_composer(sockets) + + if proc_composer: + res = proc_composer.wait() - res = proc_composer.wait() if proc_worker: - proc_worker.terminate() + if proc_composer: + proc_worker.terminate() proc_worker.wait() + if proc_dnf_json: - proc_dnf_json.terminate() + if proc_composer: + proc_dnf_json.terminate() proc_dnf_json.wait() return res diff --git a/dnf-json b/dnf-json index ed209b4d5..45ea28743 100755 --- a/dnf-json +++ b/dnf-json @@ -275,6 +275,13 @@ class DnfJsonRequestHandler(BaseHTTPRequestHandler): self.wfile.write(json.dumps({"kind": kind, "reason": reason}).encode("utf-8")) + def response_failure(self, json_object): + self._send() + self.send_response(500) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps(json_object).encode("utf-8")) + def response_success(self, json_object): self._send() self.send_response(200) @@ -305,10 +312,19 @@ class DnfJsonRequestHandler(BaseHTTPRequestHandler): arguments = call["arguments"] repos = arguments.get("repos", {}) arch = arguments["arch"] - self.cache_dir = arguments["cachedir"] - cache_state = CacheState.load_cache_state_from_disk(self.cache_dir) module_platform_id = arguments["module_platform_id"] + # If dnf-json is run as a service, we don't want users to be able to set the cache + self.cache_dir = os.environ.get("OVERWRITE_CACHE_DIR", "") + if self.cache_dir: + self.cache_dir = os.path.join(self.cache_dir, arch) + else: + self.cache_dir = arguments.get("cachedir", "") + + if not self.cache_dir: + self.response_failure({ "kind": "Error", "reason": "No cache dir set" }) + cache_state = CacheState.load_cache_state_from_disk(self.cache_dir) + with tempfile.TemporaryDirectory() as persistdir: try: solver = Solver( @@ -358,24 +374,21 @@ class DnfJsonRequestHandler(BaseHTTPRequestHandler): log.info("Starting the dnf-json server") LISTEN_FDS = int(os.environ.get("LISTEN_FDS", 0)) -SOCK_PATH = "/run/osbuild-dnf-json/" -SOCK_NAME = "api.sock" +# set from entrypoint if differs from 3 +LISTEN_FD = int(os.environ.get("LISTEN_FD", 3)) # The dnf-json web server has to use forks to serve the requests. Because the # dnf library is leaking memory in its Cpp side. class SystemDActivationSocketServer(socketserver.ForkingMixIn, socketserver.UnixStreamServer): def server_bind(self): log.debug("service bind") - if LISTEN_FDS == 0: - log.debug("create new socket") - socketserver.UnixStreamServer.server_bind(self) - else: - log.debug("rebind socket") - log.debug("address_family: %d ", self.address_family) - log.debug("socket_type: %d ", self.socket_type) - self.socket = socket.fromfd(3, self.address_family, self.socket_type) + log.debug("rebind socket") + log.debug("address_family: %d ", self.address_family) + log.debug("socket_type: %d ", self.socket_type) + if LISTEN_FDS > 1: + log.warning("More than one LISTEN_FDS") + self.socket = socket.fromfd(LISTEN_FD, self.address_family, self.socket_type) # start the web server -pathlib.Path(SOCK_PATH).mkdir(parents=True, exist_ok=True) -server = SystemDActivationSocketServer(f"{SOCK_PATH}{SOCK_NAME}", DnfJsonRequestHandler) +server = SystemDActivationSocketServer('', DnfJsonRequestHandler) server.serve_forever() diff --git a/test/cases/regression-old-worker-new-composer.sh b/test/cases/regression-old-worker-new-composer.sh index 486534257..9f4bd5fab 100644 --- a/test/cases/regression-old-worker-new-composer.sh +++ b/test/cases/regression-old-worker-new-composer.sh @@ -52,6 +52,8 @@ rpm -q "$WORKER_RPM" # run container WELDR_DIR="$(mktemp -d)" WELDR_SOCK="$WELDR_DIR/api.socket" +DNF_DIR="$(mktemp -d)" +DNF_SOCK="$DNF_DIR/api.sock" sudo podman pull --creds "${V2_QUAY_USERNAME}":"${V2_QUAY_PASSWORD}" \ "quay.io/osbuild/osbuild-composer-ubi-pr:${CI_COMMIT_SHA}" @@ -66,6 +68,8 @@ sudo podman run \ -v /etc/pki/entitlement:/etc/pki/entitlement:Z \ -v "$REPOS/repositories":/usr/share/osbuild-composer/repositories:Z \ -v "$WELDR_DIR:/run/weldr/":Z \ + -v "$DNF_DIR:/run/osbuild-dnf-json/":Z \ + -e OVERWRITE_CACHE_DIR="/var/cache/dnf-json" \ -p 8700:8700 \ "quay.io/osbuild/osbuild-composer-ubi-pr:${CI_COMMIT_SHA}" \ --weldr-api --dnf-json --remote-worker-api \ @@ -130,3 +134,28 @@ if [[ $COMPOSE_STATUS != FINISHED ]]; then echo "Something went wrong with the compose. 😢" exit 1 fi + +tee "dnf-json-request.json" <