From f8b73346628dc0aed6203643a3a727fcf319cb83 Mon Sep 17 00:00:00 2001 From: Gerald Pinder Date: Sat, 13 Apr 2024 15:08:31 -0400 Subject: [PATCH] feat: Look for recipes in `./recipes/`, build files in `./files/`, and Containerfiles in `./containerfiles/` (#157) Recipe files can now be put into their own directory `./recipes/`. This directory is NEVER copied into the build so changes to a recipe will no longer cause cache misses for builds. Here is an example of my build changing the second to last module and only requiring the last 2 `RUN` layers to be run again. ``` => CACHED [stage-config 1/1] COPY ./config /config 0.0s => CACHED [stage-modules 1/2] COPY --from=ghcr.io/blue-build/modules:latest /modules /modules 0.0s => CACHED [stage-modules 2/2] COPY ./modules /modules 0.0s => CACHED [stage-keys 1/1] COPY cosign.pub /keys/jp-desktop-gaming.pub 0.0s => CACHED [stage-4 2/16] RUN --mount=type=bind,from=stage-keys,src=/keys,dst=/tmp/keys mkdir -p /usr/etc/pki/containers/ && cp /tmp/keys/* /usr/et 0.0s => CACHED [stage-bins 1/3] COPY --from=gcr.io/projectsigstore/cosign /ko-app/cosign /bins/cosign 0.0s => CACHED [stage-bins 2/3] COPY --from=docker.io/mikefarah/yq /usr/bin/yq /bins/yq 0.0s => CACHED [stage-bins 3/3] COPY --from=ghcr.io/blue-build/cli:main-installer /out/bluebuild /bins/bluebuild 0.0s => CACHED [stage-4 3/16] RUN --mount=type=bind,from=stage-bins,src=/bins,dst=/tmp/bins mkdir -p /usr/bin/ && cp /tmp/bins/* /usr/bin/ && ostree 0.0s => CACHED [stage-4 4/16] RUN --mount=type=tmpfs,target=/var --mount=type=bind,from=stage-config,src=/config,dst=/tmp/config,rw --mount=type=bind 0.0s => CACHED [stage-4 5/16] RUN --mount=type=tmpfs,target=/var --mount=type=bind,from=stage-config,src=/config,dst=/tmp/config,rw --mount=type=bind 0.0s => CACHED [stage-4 6/16] RUN --mount=type=tmpfs,target=/var --mount=type=bind,from=stage-config,src=/config,dst=/tmp/config,rw --mount=type=bind 0.0s => CACHED [stage-4 7/16] RUN --mount=type=tmpfs,target=/var --mount=type=bind,from=stage-config,src=/config,dst=/tmp/config,rw --mount=type=bind 0.0s => CACHED [stage-4 8/16] RUN --mount=type=tmpfs,target=/var --mount=type=bind,from=stage-config,src=/config,dst=/tmp/config,rw --mount=type=bind 0.0s => CACHED [stage-4 9/16] RUN --mount=type=tmpfs,target=/var --mount=type=bind,from=stage-config,src=/config,dst=/tmp/config,rw --mount=type=bind 0.0s => CACHED [stage-4 10/16] RUN --mount=type=tmpfs,target=/var --mount=type=bind,from=stage-config,src=/config,dst=/tmp/config,rw --mount=type=bind 0.0s => CACHED [stage-4 11/16] RUN --mount=type=tmpfs,target=/var --mount=type=bind,from=stage-config,src=/config,dst=/tmp/config,rw --mount=type=bind 0.0s => CACHED [stage-4 12/16] RUN --mount=type=tmpfs,target=/var --mount=type=bind,from=stage-config,src=/config,dst=/tmp/config,rw --mount=type=bind 0.0s => CACHED [stage-4 13/16] RUN --mount=type=tmpfs,target=/var --mount=type=bind,from=stage-config,src=/config,dst=/tmp/config,rw --mount=type=bind 0.0s => CACHED [stage-4 14/16] RUN --mount=type=tmpfs,target=/var --mount=type=bind,from=stage-config,src=/config,dst=/tmp/config,rw --mount=type=bind 0.0s => [stage-4 15/16] RUN --mount=type=tmpfs,target=/var --mount=type=bind,from=stage-config,src=/config,dst=/tmp/config,rw --mount=type=bind,from= 33.4s => [stage-4 16/16] RUN --mount=type=tmpfs,target=/var --mount=type=bind,from=stage-config,src=/config,dst=/tmp/config,rw --mount=type=bind,from=s 0.7s ``` Support was also added to put all build files into `./files/` instead of `./config/`. This is an all or nothing operation, meaning if there exists a directory of `files` then the `config` directory will be completely ignored. Work will have to be done in https://github.com/blue-build/modules to allow users to put their files directly in `./files/` and not `./files/files` for the `files` module or `./files/scripts` for the scripts module. Support was also added to move the `./config/containerfiles/` directory to the root of the project. Now the directories you can find in the root of projects are: ``` files/ containerfiles/ recipes/ ``` --- README.md | 8 ++-- integration-tests/Earthfile | 29 ++++++++++-- integration-tests/legacy-test-repo/.gitignore | 1 + .../config/akmods.yml | 0 .../containerfiles/labels/Containerfile | 1 + .../config/files/usr/.gitkeep | 0 .../config/files/usr/test-file | 0 .../legacy-test-repo/config/recipe.yml | 44 +++++++++++++++++++ .../config/scripts/example.sh | 0 integration-tests/legacy-test-repo/cosign.pub | 4 ++ .../legacy-test-repo/modules/.gitkeep | 0 .../modules/test-module/test-module.sh | 5 +++ .../containerfiles/labels/Containerfile | 1 + .../test-repo/files/files/usr/.gitkeep | 0 .../test-repo/files/files/usr/test-file | 1 + .../test-repo/files/scripts/example.sh | 10 +++++ .../test-repo/recipes/akmods.yml | 22 ++++++++++ .../test-repo/{config => recipes}/recipe.yml | 5 +++ recipe/src/module.rs | 19 -------- recipe/src/module_ext.rs | 16 ++++--- src/commands/build.rs | 22 ++++++---- src/commands/template.rs | 34 +++++++------- template/src/lib.rs | 32 +++++++++++--- template/templates/Containerfile.j2 | 5 +++ .../templates/modules/containerfile/README.md | 8 ++-- template/templates/modules/modules.j2 | 4 ++ template/templates/stages.j2 | 5 +++ utils/src/constants.rs | 11 ++--- 28 files changed, 217 insertions(+), 70 deletions(-) create mode 100644 integration-tests/legacy-test-repo/.gitignore rename integration-tests/{test-repo => legacy-test-repo}/config/akmods.yml (100%) create mode 100644 integration-tests/legacy-test-repo/config/containerfiles/labels/Containerfile rename integration-tests/{test-repo => legacy-test-repo}/config/files/usr/.gitkeep (100%) rename integration-tests/{test-repo => legacy-test-repo}/config/files/usr/test-file (100%) create mode 100644 integration-tests/legacy-test-repo/config/recipe.yml rename integration-tests/{test-repo => legacy-test-repo}/config/scripts/example.sh (100%) create mode 100644 integration-tests/legacy-test-repo/cosign.pub create mode 100644 integration-tests/legacy-test-repo/modules/.gitkeep create mode 100644 integration-tests/legacy-test-repo/modules/test-module/test-module.sh create mode 100644 integration-tests/test-repo/containerfiles/labels/Containerfile create mode 100644 integration-tests/test-repo/files/files/usr/.gitkeep create mode 100644 integration-tests/test-repo/files/files/usr/test-file create mode 100644 integration-tests/test-repo/files/scripts/example.sh create mode 100644 integration-tests/test-repo/recipes/akmods.yml rename integration-tests/test-repo/{config => recipes}/recipe.yml (84%) diff --git a/README.md b/README.md index 3817a98..4651679 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ You can then use this with `podman` or `buildah` to build and publish your image If you don't care about the details of the template, you can run the `build` command. ```bash -bluebuild build ./config/recipe.yaml +bluebuild build ./recipes/recipe.yaml ``` This will template out the file and build with `buildah` or `podman`. @@ -172,7 +172,7 @@ Currently, bluebuild completions are available for `bash`, `zsh`, `fish`, `power If you want to test your changes, you can do so by using the `rebase` command. This will create an image as a `.tar.gz` file, store it in `/etc/bluebuild`, an run `rpm-ostree rebase` on that newly built file. ```bash -sudo bluebuild rebase config/recipe.yml +sudo bluebuild rebase recipes/recipe.yml ``` You can initiate an immediate restart by adding the `--reboot/-r` option. @@ -182,7 +182,7 @@ You can initiate an immediate restart by adding the `--reboot/-r` option. When you've rebased onto a local image archive, you can update your image for your recipe by running: ```bash -sudo bluebuild upgrade config/recipe.yml +sudo bluebuild upgrade recipes/recipe.yml ``` The `--reboot` argument can be used with this command as well. @@ -272,7 +272,7 @@ build-image: - export COSIGN_PRIVATE_KEY=$(cat .secure_files/cosign.key) script: - sleep 5 # Wait a bit for the docker-in-docker service to start - - bluebuild build --push ./config/$RECIPE + - bluebuild build --push ./recipes/$RECIPE ``` ## Future Features diff --git a/integration-tests/Earthfile b/integration-tests/Earthfile index 1525415..603db5e 100644 --- a/integration-tests/Earthfile +++ b/integration-tests/Earthfile @@ -4,6 +4,7 @@ PROJECT blue-build/cli all: BUILD +test-image BUILD +test-secureblue + BUILD +test-legacy-image BUILD +build BUILD +rebase BUILD +upgrade @@ -15,6 +16,13 @@ test-image: DO +RUN_TESTS +test-legacy-image: + FROM +build-template --src=template-legacy-containerfile + WORKDIR /tmp/test + COPY ./test-scripts/*.sh ./ + + DO +RUN_TESTS + test-secureblue: FROM +build-template --src=template-secureblue WORKDIR /tmp/test @@ -30,6 +38,12 @@ build-template: template-containerfile: FROM +test-base + RUN bluebuild -vv template recipes/recipe.yml | tee Containerfile + + SAVE ARTIFACT /test + +template-legacy-containerfile: + FROM +legacy-base RUN bluebuild -vv template config/recipe.yml | tee Containerfile SAVE ARTIFACT /test @@ -43,18 +57,18 @@ template-secureblue: build: FROM +test-base - RUN --privileged bluebuild -vv build config/recipe.yml + RUN bluebuild -vv build recipes/recipe.yml rebase: FROM +test-base - RUN --privileged bluebuild -vv rebase config/recipe.yml + RUN bluebuild -vv rebase recipes/recipe.yml upgrade: FROM +test-base RUN mkdir -p /etc/bluebuild && touch /etc/bluebuild/cli_test.tar.gz - RUN --privileged bluebuild -vv upgrade config/recipe.yml + RUN bluebuild -vv upgrade recipes/recipe.yml secureblue-base: FROM +test-base @@ -64,8 +78,17 @@ secureblue-base: DO +GEN_KEYPAIR +legacy-base: + FROM +test-base + + RUN rm -fr /test + COPY ./legacy-test-repo /test + + DO +GEN_KEYPAIR + test-base: FROM ../+blue-build-cli-alpine + ENV CLICOLOR_FORCE=1 COPY ./mock-scripts/ /usr/bin/ diff --git a/integration-tests/legacy-test-repo/.gitignore b/integration-tests/legacy-test-repo/.gitignore new file mode 100644 index 0000000..37ba090 --- /dev/null +++ b/integration-tests/legacy-test-repo/.gitignore @@ -0,0 +1 @@ +/Containerfile diff --git a/integration-tests/test-repo/config/akmods.yml b/integration-tests/legacy-test-repo/config/akmods.yml similarity index 100% rename from integration-tests/test-repo/config/akmods.yml rename to integration-tests/legacy-test-repo/config/akmods.yml diff --git a/integration-tests/legacy-test-repo/config/containerfiles/labels/Containerfile b/integration-tests/legacy-test-repo/config/containerfiles/labels/Containerfile new file mode 100644 index 0000000..935350e --- /dev/null +++ b/integration-tests/legacy-test-repo/config/containerfiles/labels/Containerfile @@ -0,0 +1 @@ +LABEL org.test.label="this is a test" diff --git a/integration-tests/test-repo/config/files/usr/.gitkeep b/integration-tests/legacy-test-repo/config/files/usr/.gitkeep similarity index 100% rename from integration-tests/test-repo/config/files/usr/.gitkeep rename to integration-tests/legacy-test-repo/config/files/usr/.gitkeep diff --git a/integration-tests/test-repo/config/files/usr/test-file b/integration-tests/legacy-test-repo/config/files/usr/test-file similarity index 100% rename from integration-tests/test-repo/config/files/usr/test-file rename to integration-tests/legacy-test-repo/config/files/usr/test-file diff --git a/integration-tests/legacy-test-repo/config/recipe.yml b/integration-tests/legacy-test-repo/config/recipe.yml new file mode 100644 index 0000000..d34ad9c --- /dev/null +++ b/integration-tests/legacy-test-repo/config/recipe.yml @@ -0,0 +1,44 @@ +name: cli/test-legacy +description: This is my personal OS image. +base-image: ghcr.io/ublue-os/silverblue-surface +image-version: 39 +modules: + - from-file: akmods.yml + + - type: files + files: + - usr: /usr + + - type: script + scripts: + - example.sh + + - type: rpm-ostree + repos: + - https://copr.fedorainfracloud.org/coprs/atim/starship/repo/fedora-%OS_VERSION%/atim-starship-fedora-%OS_VERSION%.repo + install: + - micro + - starship + remove: + - firefox + - firefox-langpacks + + - type: default-flatpaks + notify: true + system: + install: + - org.mozilla.firefox + - org.gnome.Loupe + - one.ablaze.floorp//lightning + remove: + - org.gnome.eog + + - type: signing + + - type: test-module + + - type: containerfile + containerfiles: + - labels + snippets: + - RUN echo "This is a snippet" && ostree container commit diff --git a/integration-tests/test-repo/config/scripts/example.sh b/integration-tests/legacy-test-repo/config/scripts/example.sh similarity index 100% rename from integration-tests/test-repo/config/scripts/example.sh rename to integration-tests/legacy-test-repo/config/scripts/example.sh diff --git a/integration-tests/legacy-test-repo/cosign.pub b/integration-tests/legacy-test-repo/cosign.pub new file mode 100644 index 0000000..64d33e6 --- /dev/null +++ b/integration-tests/legacy-test-repo/cosign.pub @@ -0,0 +1,4 @@ +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEgJYNEq43hrKPwWgWah14yBOUjMCd +1eG8hOwIbOTSRq+siTLep8G2m5FSYit/ea+H+0IXZS0ruLdgzoPUI7Babw== +-----END PUBLIC KEY----- diff --git a/integration-tests/legacy-test-repo/modules/.gitkeep b/integration-tests/legacy-test-repo/modules/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/integration-tests/legacy-test-repo/modules/test-module/test-module.sh b/integration-tests/legacy-test-repo/modules/test-module/test-module.sh new file mode 100644 index 0000000..ab26db5 --- /dev/null +++ b/integration-tests/legacy-test-repo/modules/test-module/test-module.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -euo pipefail + +echo "This is a test module" diff --git a/integration-tests/test-repo/containerfiles/labels/Containerfile b/integration-tests/test-repo/containerfiles/labels/Containerfile new file mode 100644 index 0000000..935350e --- /dev/null +++ b/integration-tests/test-repo/containerfiles/labels/Containerfile @@ -0,0 +1 @@ +LABEL org.test.label="this is a test" diff --git a/integration-tests/test-repo/files/files/usr/.gitkeep b/integration-tests/test-repo/files/files/usr/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/integration-tests/test-repo/files/files/usr/test-file b/integration-tests/test-repo/files/files/usr/test-file new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/integration-tests/test-repo/files/files/usr/test-file @@ -0,0 +1 @@ + diff --git a/integration-tests/test-repo/files/scripts/example.sh b/integration-tests/test-repo/files/scripts/example.sh new file mode 100644 index 0000000..fdb2e04 --- /dev/null +++ b/integration-tests/test-repo/files/scripts/example.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +# Tell this script to exit if there are any errors. +# You should have this in every custom script, to ensure that your completed +# builds actually ran successfully without any errors! +set -oue pipefail + +# Your code goes here. +echo 'This is an example shell script' +echo 'Scripts here will run during build if specified in recipe.yml' diff --git a/integration-tests/test-repo/recipes/akmods.yml b/integration-tests/test-repo/recipes/akmods.yml new file mode 100644 index 0000000..68125de --- /dev/null +++ b/integration-tests/test-repo/recipes/akmods.yml @@ -0,0 +1,22 @@ +# TODO: Add back installs after upstream issues are fixed + +modules: + # Tests installing rpms from a combo image stage + - type: akmods + base: surface + nvidia-version: 545 + # install: + # - nvidia + # - openrazer + # - openrgb + + # Tests pulling main image + - type: akmods + + # Tests pulling image for main nvidia + - type: akmods + nvidia-version: 545 + + # Test pulling image for base asus + - type: akmods + base: asus diff --git a/integration-tests/test-repo/config/recipe.yml b/integration-tests/test-repo/recipes/recipe.yml similarity index 84% rename from integration-tests/test-repo/config/recipe.yml rename to integration-tests/test-repo/recipes/recipe.yml index 51bdf6b..1771dab 100644 --- a/integration-tests/test-repo/config/recipe.yml +++ b/integration-tests/test-repo/recipes/recipe.yml @@ -37,3 +37,8 @@ modules: - type: test-module + - type: containerfile + containerfiles: + - labels + snippets: + - RUN echo "This is a snippet" && ostree container commit diff --git a/recipe/src/module.rs b/recipe/src/module.rs index 17d23c4..6f2bb08 100644 --- a/recipe/src/module.rs +++ b/recipe/src/module.rs @@ -81,25 +81,6 @@ impl<'a> Module<'a> { }) } - #[must_use] - pub fn get_files_list(&'a self) -> Option> { - Some( - self.config - .get("files")? - .as_sequence()? - .iter() - .filter_map(|entry| entry.as_mapping()) - .flatten() - .filter_map(|(src, dest)| { - Some(( - format!("./config/files/{}", src.as_str()?), - dest.as_str()?.to_string(), - )) - }) - .collect(), - ) - } - pub fn generate_akmods_info(&'a self, os_version: &str) -> AkmodsInfo { trace!("generate_akmods_base({self:#?}, {os_version})"); diff --git a/recipe/src/module_ext.rs b/recipe/src/module_ext.rs index 1dbdbe8..761a3cc 100644 --- a/recipe/src/module_ext.rs +++ b/recipe/src/module_ext.rs @@ -1,7 +1,8 @@ -use std::{borrow::Cow, collections::HashSet, fs, path::PathBuf}; +use std::{borrow::Cow, collections::HashSet, fs, path::Path}; use anyhow::Result; -use log::trace; +use blue_build_utils::constants::{CONFIG_PATH, RECIPE_PATH}; +use log::{trace, warn}; use serde::{Deserialize, Serialize}; use typed_builder::TypedBuilder; @@ -20,11 +21,14 @@ impl ModuleExt<'_> { /// Can return an `anyhow` Error if the file cannot be read or deserialized /// into a [`ModuleExt`] pub fn parse_module_from_file(file_name: &str) -> Result { - let file_path = PathBuf::from("config").join(file_name); - let file_path = if file_path.is_absolute() { - file_path + let legacy_path = Path::new(CONFIG_PATH); + let recipe_path = Path::new(RECIPE_PATH); + + let file_path = if recipe_path.exists() && recipe_path.is_dir() { + recipe_path.join(file_name) } else { - std::env::current_dir()?.join(file_path) + warn!("Use of {CONFIG_PATH} for recipes is deprecated, please move your recipe files into {RECIPE_PATH}"); + legacy_path.join(file_name) }; let file = fs::read_to_string(file_path)?; diff --git a/src/commands/build.rs b/src/commands/build.rs index 5c03c8a..c86eea3 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -8,10 +8,10 @@ use anyhow::{bail, Result}; use blue_build_recipe::Recipe; use blue_build_utils::constants::{ ARCHIVE_SUFFIX, BUILD_ID_LABEL, CI_DEFAULT_BRANCH, CI_PROJECT_NAME, CI_PROJECT_NAMESPACE, - CI_PROJECT_URL, CI_REGISTRY, CI_SERVER_HOST, CI_SERVER_PROTOCOL, CONTAINER_FILE, COSIGN_PATH, - COSIGN_PRIVATE_KEY, GITHUB_REPOSITORY_OWNER, GITHUB_TOKEN, GITHUB_TOKEN_ISSUER_URL, - GITHUB_WORKFLOW_REF, GITIGNORE_PATH, LABELED_ERROR_MESSAGE, NO_LABEL_ERROR_MESSAGE, - RECIPE_PATH, SIGSTORE_ID_TOKEN, + CI_PROJECT_URL, CI_REGISTRY, CI_SERVER_HOST, CI_SERVER_PROTOCOL, CONFIG_PATH, CONTAINER_FILE, + COSIGN_PATH, COSIGN_PRIVATE_KEY, GITHUB_REPOSITORY_OWNER, GITHUB_TOKEN, + GITHUB_TOKEN_ISSUER_URL, GITHUB_WORKFLOW_REF, GITIGNORE_PATH, LABELED_ERROR_MESSAGE, + NO_LABEL_ERROR_MESSAGE, RECIPE_FILE, RECIPE_PATH, SIGSTORE_ID_TOKEN, }; use clap::Args; use colored::Colorize; @@ -174,10 +174,16 @@ impl BlueBuildCommand for BuildCommand { bail!("You cannot use '--archive' and '--push' at the same time"); } - let recipe_path = self - .recipe - .clone() - .unwrap_or_else(|| PathBuf::from(RECIPE_PATH)); + let recipe_path = self.recipe.clone().unwrap_or_else(|| { + let legacy_path = Path::new(CONFIG_PATH); + let recipe_path = Path::new(RECIPE_PATH); + if recipe_path.exists() && recipe_path.is_dir() { + recipe_path.join(RECIPE_FILE) + } else { + warn!("Use of {CONFIG_PATH} for recipes is deprecated, please move your recipe files into {RECIPE_PATH}"); + legacy_path.join(RECIPE_FILE) + } + }); TemplateCommand::builder() .recipe(&recipe_path) diff --git a/src/commands/template.rs b/src/commands/template.rs index 1d7f8ea..bf8a85f 100644 --- a/src/commands/template.rs +++ b/src/commands/template.rs @@ -1,13 +1,17 @@ -use std::{env, path::PathBuf}; +use std::{ + env, + path::{Path, PathBuf}, +}; use anyhow::Result; use blue_build_recipe::Recipe; use blue_build_template::{ContainerFileTemplate, Template}; use blue_build_utils::constants::{ - CI_PROJECT_NAME, CI_PROJECT_NAMESPACE, CI_REGISTRY, GITHUB_REPOSITORY_OWNER, RECIPE_PATH, + CI_PROJECT_NAME, CI_PROJECT_NAMESPACE, CI_REGISTRY, CONFIG_PATH, GITHUB_REPOSITORY_OWNER, + RECIPE_FILE, RECIPE_PATH, }; use clap::Args; -use log::{debug, info, trace}; +use log::{debug, info, trace, warn}; use typed_builder::TypedBuilder; use crate::{drivers::Driver, shadow}; @@ -49,14 +53,6 @@ pub struct TemplateCommand { impl BlueBuildCommand for TemplateCommand { fn try_run(&mut self) -> Result<()> { - info!( - "Templating for recipe at {}", - self.recipe - .clone() - .unwrap_or_else(|| PathBuf::from(RECIPE_PATH)) - .display() - ); - Driver::builder() .build_driver(self.drivers.build_driver) .inspect_driver(self.drivers.inspect_driver) @@ -71,10 +67,18 @@ impl TemplateCommand { fn template_file(&self) -> Result<()> { trace!("TemplateCommand::template_file()"); - let recipe_path = self - .recipe - .clone() - .unwrap_or_else(|| PathBuf::from(RECIPE_PATH)); + let recipe_path = self.recipe.clone().unwrap_or_else(|| { + let legacy_path = Path::new(CONFIG_PATH); + let recipe_path = Path::new(RECIPE_PATH); + if recipe_path.exists() && recipe_path.is_dir() { + recipe_path.join(RECIPE_FILE) + } else { + warn!("Use of {CONFIG_PATH} for recipes is deprecated, please move your recipe files into {RECIPE_PATH}"); + legacy_path.join(RECIPE_FILE) + } + }); + + info!("Templating for recipe at {}", recipe_path.display()); debug!("Deserializing recipe"); let recipe_de = Recipe::parse(&recipe_path)?; diff --git a/template/src/lib.rs b/template/src/lib.rs index 412c10e..47ff805 100644 --- a/template/src/lib.rs +++ b/template/src/lib.rs @@ -2,10 +2,11 @@ use std::{borrow::Cow, env, fs, path::Path, process}; use blue_build_recipe::Recipe; use blue_build_utils::constants::{ - CI_PROJECT_NAME, CI_PROJECT_NAMESPACE, CI_SERVER_HOST, CI_SERVER_PROTOCOL, COSIGN_PATH, - GITHUB_RESPOSITORY, GITHUB_SERVER_URL, + CI_PROJECT_NAME, CI_PROJECT_NAMESPACE, CI_SERVER_HOST, CI_SERVER_PROTOCOL, CONFIG_PATH, + CONTAINERFILES_PATH, CONTAINER_FILE, COSIGN_PATH, FILES_PATH, GITHUB_RESPOSITORY, + GITHUB_SERVER_URL, }; -use log::{debug, error, trace}; +use log::{debug, error, trace, warn}; use typed_builder::TypedBuilder; use uuid::Uuid; @@ -90,14 +91,22 @@ fn print_containerfile(containerfile: &str) -> String { trace!("print_containerfile({containerfile})"); debug!("Loading containerfile contents for {containerfile}"); - let path = format!("config/containerfiles/{containerfile}/Containerfile"); + let legacy_path = Path::new(CONFIG_PATH); + let containerfiles_path = Path::new(CONTAINERFILES_PATH); + + let path = if containerfiles_path.exists() && containerfiles_path.is_dir() { + containerfiles_path.join(format!("{containerfile}/{CONTAINER_FILE}")) + } else { + warn!("Use of {CONFIG_PATH} is deprecated for the containerfile module, please move your containerfile directories into {CONTAINERFILES_PATH}"); + legacy_path.join(format!("containerfiles/{containerfile}/{CONTAINER_FILE}")) + }; let file = fs::read_to_string(&path).unwrap_or_else(|e| { - error!("Failed to read file {path}: {e}"); + error!("Failed to read file {}: {e}", path.display()); process::exit(1); }); - debug!("Containerfile contents {path}:\n{file}"); + debug!("Containerfile contents {}:\n{file}", path.display()); file } @@ -139,6 +148,17 @@ fn modules_exists() -> bool { mod_path.exists() && mod_path.is_dir() } +fn files_dir_exists() -> bool { + let path = Path::new(FILES_PATH); + let exists = path.exists() && path.is_dir(); + + if !exists { + warn!("Use of the {CONFIG_PATH} directory is deprecated. Please move your non-recipe files into {FILES_PATH}"); + } + + exists +} + mod filters { #[allow(clippy::unnecessary_wraps)] pub fn replace(input: T, from: char, to: &str) -> askama::Result { diff --git a/template/templates/Containerfile.j2 b/template/templates/Containerfile.j2 index 390ca73..b02fceb 100644 --- a/template/templates/Containerfile.j2 +++ b/template/templates/Containerfile.j2 @@ -1,3 +1,4 @@ +{%- set files_dir_exists = self::files_dir_exists() %} {%- include "stages.j2" %} FROM {{ recipe.base_image }}:{{ recipe.image_version }} @@ -5,7 +6,11 @@ FROM {{ recipe.base_image }}:{{ recipe.image_version }} ARG RECIPE={{ recipe_path.display() }} ARG IMAGE_REGISTRY={{ registry }} +{%- if files_dir_exists %} +ARG CONFIG_DIRECTORY="/tmp/files" +{%- else %} ARG CONFIG_DIRECTORY="/tmp/config" +{%- endif %} ARG MODULE_DIRECTORY="/tmp/modules" ARG IMAGE_NAME="{{ recipe.name }}" ARG BASE_IMAGE="{{ recipe.base_image }}" diff --git a/template/templates/modules/containerfile/README.md b/template/templates/modules/containerfile/README.md index afee5ae..bcf51e3 100644 --- a/template/templates/modules/containerfile/README.md +++ b/template/templates/modules/containerfile/README.md @@ -29,7 +29,7 @@ This makes it really easy to copy a file or program from another image. ### `containerfiles:` -The `containerfiles` property allows you to tell the compiler which directory contains a `Containerfile` in `./config/containerfiles/`. +The `containerfiles` property allows you to tell the compiler which directory contains a `Containerfile` in `./containerfiles/`. Below is an example of how a `containerfile` module would be used with the `containerfiles` property: @@ -43,10 +43,10 @@ modules: In the example above, the compiler would look for these files: -- `./config/containerfiles/example/Containerfile` -- `./config/containerfiles/subroutine/Containerfile` +- `./containerfiles/example/Containerfile` +- `./containerfiles/subroutine/Containerfile` -You could then store files related to say the `subroutine` `Containerfile` in `./config/containerfiles/subroutine/` to keep it organized and portable for other recipes to use. +You could then store files related to say the `subroutine` `Containerfile` in `./containerfiles/subroutine/` to keep it organized and portable for other recipes to use. :::note **NOTE:** The instructions you add in your `Containerfile`'s each become a layer unlike other modules which are typically run as a single `RUN` command, thus creating only one layer. diff --git a/template/templates/modules/modules.j2 b/template/templates/modules/modules.j2 index 84929fa..a3f1310 100644 --- a/template/templates/modules/modules.j2 +++ b/template/templates/modules/modules.j2 @@ -18,7 +18,11 @@ RUN --mount=type=bind,from=stage-bins,src=/bins,dst=/tmp/bins \ {%- else %} RUN \ --mount=type=tmpfs,target=/var \ + {%- if files_dir_exists %} + --mount=type=bind,from=stage-files,src=/files,dst=/tmp/files,rw \ + {%- else %} --mount=type=bind,from=stage-config,src=/config,dst=/tmp/config,rw \ + {%- endif %} {%- if let Some(source) = module.source %} --mount=type=bind,from={{ source }},src=/modules,dst=/tmp/modules,rw \ {%- else %} diff --git a/template/templates/stages.j2 b/template/templates/stages.j2 index 9c56fa3..6c07f1d 100644 --- a/template/templates/stages.j2 +++ b/template/templates/stages.j2 @@ -1,8 +1,13 @@ # This stage is responsible for holding onto # your config without copying it directly into # the final image +{%- if files_dir_exists %} +FROM scratch as stage-files +COPY ./files /files +{%- else %} FROM scratch as stage-config COPY ./config /config +{%- endif %} # Copy modules # The default modules are inside blue-build/modules diff --git a/utils/src/constants.rs b/utils/src/constants.rs index 85329d3..4112c5c 100644 --- a/utils/src/constants.rs +++ b/utils/src/constants.rs @@ -1,14 +1,15 @@ // Paths pub const ARCHIVE_SUFFIX: &str = "tar.gz"; +pub const CONFIG_PATH: &str = "./config"; +pub const CONTAINERFILES_PATH: &str = "./containerfiles"; +pub const CONTAINER_FILE: &str = "Containerfile"; pub const COSIGN_PATH: &str = "./cosign.pub"; +pub const FILES_PATH: &str = "./files"; pub const GITIGNORE_PATH: &str = ".gitignore"; pub const LOCAL_BUILD: &str = "/etc/bluebuild"; -pub const CONTAINER_FILE: &str = "Containerfile"; pub const MODULES_PATH: &str = "./config/modules"; -pub const RECIPE_PATH: &str = "./config/recipe.yml"; -pub const RUN_PODMAN_SOCK: &str = "/run/podman/podman.sock"; -pub const VAR_RUN_PODMAN_PODMAN_SOCK: &str = "/var/run/podman/podman.sock"; -pub const VAR_RUN_PODMAN_SOCK: &str = "/var/run/podman.sock"; +pub const RECIPE_FILE: &str = "recipe.yml"; +pub const RECIPE_PATH: &str = "./recipes"; // Labels pub const BUILD_ID_LABEL: &str = "org.blue-build.build-id";