diff --git a/.rusty-hook.toml b/.rusty-hook.toml index ac4c348..89f9c9f 100644 --- a/.rusty-hook.toml +++ b/.rusty-hook.toml @@ -1,5 +1,5 @@ [hooks] -pre-commit = "cargo fmt --check && cargo test && cargo clippy -- -D warnings" +pre-commit = "cargo fmt --check && cargo test --all-features && cargo clippy --all-features -- -D warnings" [logging] verbose = true diff --git a/Cargo.lock b/Cargo.lock index 2f5143e..b2abbc6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -228,6 +228,7 @@ dependencies = [ "anyhow", "blue-build-utils", "chrono", + "colored", "indexmap 2.2.3", "log", "serde", diff --git a/Cargo.toml b/Cargo.toml index 87ef602..c8b4eb1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,6 +84,8 @@ uuid.workspace = true [features] default = [] +stages = ["blue-build-recipe/stages"] +copy = ["blue-build-recipe/copy"] [dev-dependencies] rusty-hook = "0.11.2" diff --git a/Earthfile b/Earthfile index b83a3e3..285d7d1 100644 --- a/Earthfile +++ b/Earthfile @@ -1,7 +1,7 @@ VERSION 0.8 PROJECT blue-build/cli -IMPORT github.com/blue-build/earthly-lib/cargo AS cargo +IMPORT github.com/earthly/lib/rust AS rust ARG --global IMAGE=ghcr.io/blue-build/cli @@ -21,17 +21,29 @@ build: lint: FROM +common - DO cargo+LINT + DO rust+CARGO --args="clippy -- -D warnings" + DO rust+CARGO --args="clippy --all-features -- -D warnings" + DO rust+CARGO --args="clippy --no-default-features -- -D warnings" test: FROM +common - DO cargo+TEST + DO rust+CARGO --args="test -- --show-output" + DO rust+CARGO --args="test --all-features -- --show-output" + DO rust+CARGO --args="test --no-default-features -- --show-output" install: FROM +common ARG --required BUILD_TARGET - DO cargo+BUILD_RELEASE --BUILD_TARGET=$BUILD_TARGET + DO rust+CARGO --args="build --release --target $BUILD_TARGET" --output="$BUILD_TARGET/release/[^\./]+" + + SAVE ARTIFACT target/$BUILD_TARGET/release/bluebuild + +install-all-features: + FROM +common + ARG --required BUILD_TARGET + + DO rust+CARGO --args="build --all-features --release --target $BUILD_TARGET" --output="$BUILD_TARGET/release/[^\./]+" SAVE ARTIFACT target/$BUILD_TARGET/release/bluebuild @@ -47,7 +59,7 @@ common: COPY --keep-ts --dir .git/ /app RUN touch build.rs - DO cargo+INIT + DO rust+INIT --keep_fingerprints=true build-scripts: FROM alpine @@ -65,8 +77,6 @@ blue-build-cli: FROM $BASE_IMAGE LABEL org.opencontainers.image.base.name="$BASE_IMAGE" - BUILD +install --BUILD_TARGET="x86_64-unknown-linux-gnu" - RUN dnf -y install dnf-plugins-core \ && dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo \ && dnf install --refresh -y \ @@ -84,7 +94,7 @@ blue-build-cli: COPY +cosign/cosign /usr/bin/cosign - COPY (+install/bluebuild --BUILD_TARGET="x86_64-unknown-linux-gnu") /usr/bin/bluebuild + DO --pass-args +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="x86_64-unknown-linux-gnu" RUN mkdir -p /bluebuild WORKDIR /bluebuild @@ -97,14 +107,12 @@ blue-build-cli-alpine: FROM $BASE_IMAGE LABEL org.opencontainers.image.base.name="$BASE_IMAGE" - BUILD +install --BUILD_TARGET="x86_64-unknown-linux-musl" - RUN apk update && apk add buildah podman skopeo fuse-overlayfs jq LABEL org.opencontainers.image.base.digest="$(skopeo inspect "docker://$BASE_IMAGE" | jq -r '.Digest')" COPY +cosign/cosign /usr/bin/cosign - COPY (+install/bluebuild --BUILD_TARGET="x86_64-unknown-linux-musl") /usr/bin/bluebuild + DO --pass-args +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="x86_64-unknown-linux-musl" RUN mkdir -p /bluebuild WORKDIR /bluebuild @@ -121,7 +129,7 @@ installer: LABEL org.opencontainers.image.base.digest="$(skopeo inspect "docker://$BASE_IMAGE" | jq -r '.Digest')" - COPY (+install/bluebuild --BUILD_TARGET="x86_64-unknown-linux-musl") /out/bluebuild + DO --pass-args +INSTALL --OUT_DIR="/out/" --BUILD_TARGET="x86_64-unknown-linux-musl" COPY install.sh /install.sh CMD ["cat", "/install.sh"] @@ -146,6 +154,18 @@ version: SAVE ARTIFACT /version +INSTALL: + FUNCTION + ARG TAGGED="false" + ARG --required BUILD_TARGET + ARG --required OUT_DIR + + IF [ "$TAGGED" = "true" ] + COPY (+install/bluebuild --BUILD_TARGET="$BUILD_TARGET") $OUT_DIR + ELSE + COPY (+install-all-features/bluebuild --BUILD_TARGET="$BUILD_TARGET") $OUT_DIR + END + SAVE_IMAGE: FUNCTION ARG SUFFIX="" diff --git a/integration-tests/Earthfile b/integration-tests/Earthfile index 5f268ec..bf6c7fe 100644 --- a/integration-tests/Earthfile +++ b/integration-tests/Earthfile @@ -92,11 +92,17 @@ secureblue-base: DO +GEN_KEYPAIR legacy-base: - FROM +test-base + FROM ../+blue-build-cli-alpine + ENV BB_TEST_LOCAL_IMAGE=/etc/bluebuild/cli_test.tar.gz + ENV CLICOLOR_FORCE=1 - RUN rm -fr /test + COPY ./mock-scripts/ /usr/bin/ + + WORKDIR /test COPY ./legacy-test-repo /test + DO ../+INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="x86_64-unknown-linux-musl" --TAGGED="true" + DO +GEN_KEYPAIR test-base: diff --git a/integration-tests/test-repo/files/scripts/bluebuild.sh b/integration-tests/test-repo/files/scripts/bluebuild.sh new file mode 100644 index 0000000..b808c5a --- /dev/null +++ b/integration-tests/test-repo/files/scripts/bluebuild.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -euo pipefail + +cargo install blue-build --debug --all-features --target x86_64-unknown-linux-gnu +mkdir -p /out/ +mv $CARGO_HOME/bin/bluebuild /out/bluebuild diff --git a/integration-tests/test-repo/recipes/bluebuild.yml b/integration-tests/test-repo/recipes/bluebuild.yml new file mode 100644 index 0000000..2959560 --- /dev/null +++ b/integration-tests/test-repo/recipes/bluebuild.yml @@ -0,0 +1,12 @@ +stages: + - name: blue-build + image: rust + modules: + - type: script + scripts: + - bluebuild.sh +modules: + - type: copy + from: blue-build + src: /out/bluebuild + dest: /usr/bin/bluebuild diff --git a/integration-tests/test-repo/recipes/recipe.yml b/integration-tests/test-repo/recipes/recipe.yml index ba09243..ea91aec 100644 --- a/integration-tests/test-repo/recipes/recipe.yml +++ b/integration-tests/test-repo/recipes/recipe.yml @@ -1,10 +1,12 @@ name: cli/test -description: This is my personal OS image. -base-image: ghcr.io/ublue-os/silverblue-main alt-tags: - gts - stable +description: This is my personal OS image. +base-image: ghcr.io/ublue-os/silverblue-main image-version: 40 +stages: + - from-file: stages.yml modules: - from-file: akmods.yml - from-file: flatpaks.yml @@ -36,3 +38,20 @@ modules: - labels snippets: - RUN echo "This is a snippet" && ostree container commit + + - type: copy + from: alpine-test + src: /test.txt + dest: / + - type: copy + from: ubuntu-test + src: /test.txt + dest: / + - type: copy + from: debian-test + src: /test.txt + dest: / + - type: copy + from: fedora-test + src: /test.txt + dest: / diff --git a/integration-tests/test-repo/recipes/stages.yml b/integration-tests/test-repo/recipes/stages.yml new file mode 100644 index 0000000..e2851c5 --- /dev/null +++ b/integration-tests/test-repo/recipes/stages.yml @@ -0,0 +1,32 @@ +stages: + - name: ubuntu-test + from: ubuntu + modules: + - from-file: stages.yml + - name: debian-test + from: debian + modules: + - from-file: stages.yml + - name: fedora-test + from: fedora + modules: + - from-file: stages.yml + - name: alpine-test + from: alpine + modules: + - from-file: stages.yml +modules: + - type: files + files: + - usr: /usr + - type: script + scripts: + - example.sh + snippets: + - echo "test" > /test.txt + - type: test-module + - type: containerfile + containerfiles: + - labels + snippets: + - RUN echo "This is a snippet" diff --git a/modules.json b/modules.json index b229517..3c5d30e 100644 --- a/modules.json +++ b/modules.json @@ -1,3 +1,4 @@ [ - "https://raw.githubusercontent.com/blue-build/cli/main/template/templates/modules/containerfile/module.yml" + "https://raw.githubusercontent.com/blue-build/cli/main/template/templates/modules/containerfile/module.yml", + "https://raw.githubusercontent.com/blue-build/cli/main/template/templates/modules/copy/module.yml" ] \ No newline at end of file diff --git a/recipe/Cargo.toml b/recipe/Cargo.toml index 616ad5d..520b935 100644 --- a/recipe/Cargo.toml +++ b/recipe/Cargo.toml @@ -14,11 +14,17 @@ chrono = "0.4" indexmap = { version = "2", features = ["serde"] } anyhow.workspace = true +colored.workspace = true log.workspace = true serde.workspace = true serde_yaml.workspace = true serde_json.workspace = true typed-builder.workspace = true +[features] +default = [] +stages = [] +copy = [] + [lints] workspace = true diff --git a/recipe/src/lib.rs b/recipe/src/lib.rs index c98d976..4d04119 100644 --- a/recipe/src/lib.rs +++ b/recipe/src/lib.rs @@ -2,8 +2,12 @@ pub mod akmods_info; pub mod module; pub mod module_ext; pub mod recipe; +pub mod stage; +pub mod stages_ext; pub use akmods_info::*; pub use module::*; pub use module_ext::*; pub use recipe::*; +pub use stage::*; +pub use stages_ext::*; diff --git a/recipe/src/module.rs b/recipe/src/module.rs index 003cae1..5da25f5 100644 --- a/recipe/src/module.rs +++ b/recipe/src/module.rs @@ -1,6 +1,8 @@ -use std::{borrow::Cow, process}; +use std::{borrow::Cow, path::PathBuf, process}; use anyhow::{bail, Result}; +use blue_build_utils::syntax_highlighting::highlight_ser; +use colored::Colorize; use indexmap::IndexMap; use log::{error, trace, warn}; use serde::{Deserialize, Serialize}; @@ -9,54 +11,34 @@ use typed_builder::TypedBuilder; use crate::{AkmodsInfo, ModuleExt}; -#[derive(Serialize, Deserialize, Debug, Clone, TypedBuilder)] -pub struct Module<'a> { - #[builder(default, setter(into, strip_option))] - #[serde(rename = "type", skip_serializing_if = "Option::is_none")] - pub module_type: Option>, - - #[builder(default, setter(into, strip_option))] - #[serde(rename = "from-file", skip_serializing_if = "Option::is_none")] - pub from_file: Option>, +#[derive(Serialize, Deserialize, Debug, Clone, TypedBuilder, Default)] +pub struct ModuleRequiredFields<'a> { + #[builder(default, setter(into))] + #[serde(rename = "type")] + pub module_type: Cow<'a, str>, #[builder(default, setter(into, strip_option))] #[serde(skip_serializing_if = "Option::is_none")] pub source: Option>, + #[builder(default)] + #[serde(rename = "no-cache", default, skip_serializing_if = "is_false")] + pub no_cache: bool, + #[serde(flatten)] #[builder(default, setter(into))] pub config: IndexMap, } -impl<'a> Module<'a> { - /// Get's any child modules. - /// - /// # Errors - /// Will error if the module cannot be - /// deserialized or the user uses another - /// property alongside `from-file:`. - pub fn get_modules(modules: &[Self]) -> Result> { - let mut found_modules = vec![]; - for module in modules { - found_modules.extend( - match module.from_file.as_ref() { - None => vec![module.clone()], - Some(file_name) => { - if module.module_type.is_some() || module.source.is_some() { - bail!("You cannot use the `type:` or `source:` property with `from-file:`"); - } - Self::get_modules(&ModuleExt::parse_module_from_file(file_name)?.modules)? - } - } - .into_iter(), - ); - } - Ok(found_modules) - } +#[allow(clippy::trivially_copy_pass_by_ref)] +const fn is_false(b: &bool) -> bool { + !*b +} +impl<'a> ModuleRequiredFields<'a> { #[must_use] pub fn get_module_type_list(&'a self, typ: &str, list_key: &str) -> Option> { - if self.module_type.as_ref()? == typ { + if self.module_type == typ { Some( self.config .get(list_key)? @@ -88,6 +70,24 @@ impl<'a> Module<'a> { }) } + #[must_use] + #[allow(clippy::missing_const_for_fn)] + pub fn get_copy_args(&'a self) -> Option<(Option<&'a str>, &'a str, &'a str)> { + #[cfg(feature = "copy")] + { + Some(( + self.config.get("from").and_then(|from| from.as_str()), + self.config.get("src")?.as_str()?, + self.config.get("dest")?.as_str()?, + )) + } + + #[cfg(not(feature = "copy"))] + { + None + } + } + #[must_use] pub fn generate_akmods_info(&'a self, os_version: &u64) -> AkmodsInfo { #[derive(Debug, Copy, Clone)] @@ -161,3 +161,90 @@ impl<'a> Module<'a> { .build() } } + +#[derive(Serialize, Deserialize, Debug, Clone, TypedBuilder, Default)] +pub struct Module<'a> { + #[builder(default, setter(strip_option))] + #[serde(flatten, skip_serializing_if = "Option::is_none")] + pub required_fields: Option>, + + #[builder(default, setter(into, strip_option))] + #[serde(rename = "from-file", skip_serializing_if = "Option::is_none")] + pub from_file: Option>, +} + +impl<'a> Module<'a> { + /// Get's any child modules. + /// + /// # Errors + /// Will error if the module cannot be + /// deserialized or the user uses another + /// property alongside `from-file:`. + pub fn get_modules( + modules: &[Self], + traversed_files: Option>, + ) -> Result> { + let mut found_modules = vec![]; + let traversed_files = traversed_files.unwrap_or_default(); + + for module in modules { + found_modules.extend( + match &module { + Module { + required_fields: Some(_), + from_file: None, + } => vec![module.clone()], + Module { + required_fields: None, + from_file: Some(file_name), + } => { + let file_name = PathBuf::from(file_name.as_ref()); + if traversed_files.contains(&file_name) { + bail!( + "{} File {} has already been parsed:\n{traversed_files:?}", + "Circular dependency detected!".bright_red(), + file_name.display().to_string().bold(), + ); + } + + let mut traversed_files = traversed_files.clone(); + traversed_files.push(file_name.clone()); + + Self::get_modules( + &ModuleExt::parse(&file_name)?.modules, + Some(traversed_files), + )? + } + _ => { + let from_example = Self::builder().from_file("test.yml").build(); + let module_example = Self::example(); + + bail!( + "Improper format for module. Must be in the format like:\n{}\n{}\n\n{}", + highlight_ser(&module_example, "yaml", None)?, + "or".bold(), + highlight_ser(&from_example, "yaml", None)? + ); + } + } + .into_iter(), + ); + } + Ok(found_modules) + } + + #[must_use] + pub fn example() -> Self { + Self::builder() + .required_fields( + ModuleRequiredFields::builder() + .module_type("module-name") + .config(IndexMap::from_iter([ + ("module".to_string(), Value::String("config".to_string())), + ("goes".to_string(), Value::String("here".to_string())), + ])) + .build(), + ) + .build() + } +} diff --git a/recipe/src/module_ext.rs b/recipe/src/module_ext.rs index a6e1885..f21abab 100644 --- a/recipe/src/module_ext.rs +++ b/recipe/src/module_ext.rs @@ -15,12 +15,12 @@ pub struct ModuleExt<'a> { } impl ModuleExt<'_> { - /// # Parse a module file returning a [`ModuleExt`] + /// Parse a module file returning a [`ModuleExt`] /// /// # Errors /// Can return an `anyhow` Error if the file cannot be read or deserialized /// into a [`ModuleExt`] - pub fn parse_module_from_file(file_name: &str) -> Result { + pub fn parse(file_name: &Path) -> Result { let legacy_path = Path::new(CONFIG_PATH); let recipe_path = Path::new(RECIPE_PATH); @@ -52,8 +52,20 @@ impl ModuleExt<'_> { self.modules .iter() - .filter(|module| module.module_type.as_ref().is_some_and(|t| t == "akmods")) - .map(|module| module.generate_akmods_info(os_version)) + .filter(|module| { + module + .required_fields + .as_ref() + .is_some_and(|rf| rf.module_type == "akmods") + }) + .filter_map(|module| { + Some( + module + .required_fields + .as_ref()? + .generate_akmods_info(os_version), + ) + }) .filter(|image| seen.insert(image.clone())) .collect() } diff --git a/recipe/src/recipe.rs b/recipe/src/recipe.rs index 107dfe0..e1c53f7 100644 --- a/recipe/src/recipe.rs +++ b/recipe/src/recipe.rs @@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize}; use serde_yaml::Value; use typed_builder::TypedBuilder; -use crate::{Module, ModuleExt}; +use crate::{Module, ModuleExt, StagesExt}; /// The build recipe. /// @@ -60,6 +60,14 @@ pub struct Recipe<'a> { #[builder(default, setter(into, strip_option))] pub alt_tags: Option>>, + /// The stages extension of the recipe. + /// + /// This hold the list of stages that can + /// be used to build software outside of + /// the final build image. + #[serde(flatten, skip_serializing_if = "Option::is_none")] + pub stages_ext: Option>, + /// The modules extension of the recipe. /// /// This holds the list of modules to be run on the image. @@ -207,7 +215,17 @@ impl<'a> Recipe<'a> { let mut recipe = serde_yaml::from_str::(&file) .map_err(blue_build_utils::serde_yaml_err(&file))?; - recipe.modules_ext.modules = Module::get_modules(&recipe.modules_ext.modules)?.into(); + recipe.modules_ext.modules = Module::get_modules(&recipe.modules_ext.modules, None)?.into(); + + #[cfg(feature = "stages")] + if let Some(ref mut stages_ext) = recipe.stages_ext { + stages_ext.stages = crate::Stage::get_stages(&stages_ext.stages, None)?.into(); + } + + #[cfg(not(feature = "stages"))] + { + recipe.stages_ext = None; + } Ok(recipe) } diff --git a/recipe/src/stage.rs b/recipe/src/stage.rs new file mode 100644 index 0000000..f9af1dd --- /dev/null +++ b/recipe/src/stage.rs @@ -0,0 +1,159 @@ +use std::{borrow::Cow, path::PathBuf}; + +use anyhow::{bail, Result}; +use blue_build_utils::syntax_highlighting::highlight_ser; +use colored::Colorize; +use serde::{Deserialize, Serialize}; +use typed_builder::TypedBuilder; + +use crate::{Module, ModuleExt, StagesExt}; + +/// Contains the required fields for a stage. +#[derive(Serialize, Deserialize, Debug, Clone, TypedBuilder)] +pub struct StageRequiredFields<'a> { + /// The name of the stage. + /// + /// This can then be referenced in the `copy` + /// module using the `from:` property. + #[builder(setter(into))] + pub name: Cow<'a, str>, + + /// The base image of the stage. + /// + /// This is set directly in a `FROM` instruction. + #[builder(setter(into))] + pub from: Cow<'a, str>, + + /// The shell to use in the stage. + #[builder(default, setter(into, strip_option))] + #[serde(skip_serializing_if = "Option::is_none")] + pub shell: Option>>, + + /// The modules extension for the stage + #[serde(flatten)] + pub modules_ext: ModuleExt<'a>, +} + +/// Corresponds to a stage in a Containerfile +/// +/// A stage has its own list of modules to run which +/// allows the user to reuse the modules thats provided to the main build. +#[derive(Serialize, Deserialize, Debug, Clone, TypedBuilder)] +pub struct Stage<'a> { + /// The requied fields for a stage. + #[builder(default, setter(strip_option))] + #[serde(flatten, skip_serializing_if = "Option::is_none")] + pub required_fields: Option>, + + /// A reference to another recipe file containing + /// one or more stages. + /// + /// An imported recipe file can contain just a single stage like: + /// + /// ```yaml + /// name: blue-build + /// image: rust + /// modules: + /// - type: containerfile + /// snippets: + /// - | + /// RUN cargo install blue-build --debug --all-features --target x86_64-unknown-linux-gnu \ + /// && mkdir -p /out/ \ + /// && mv $CARGO_HOME/bin/bluebuild /out/bluebuild + /// ``` + /// + /// Or it can contain multiple stages: + /// + /// ```yaml + /// stages: + /// - name: blue-build + /// image: rust + /// modules: + /// - type: containerfile + /// snippets: + /// - | + /// RUN cargo install blue-build --debug --all-features --target x86_64-unknown-linux-gnu \ + /// && mkdir -p /out/ \ + /// && mv $CARGO_HOME/bin/bluebuild /out/bluebuild + /// - name: hello-world + /// image: alpine + /// modules: + /// - type: script + /// snippets: + /// - echo "Hello World!" + /// ``` + #[builder(default, setter(into, strip_option))] + #[serde(rename = "from-file", skip_serializing_if = "Option::is_none")] + pub from_file: Option>, +} + +impl<'a> Stage<'a> { + /// Get's any child stages. + /// + /// # Errors + /// Will error if the stage cannot be + /// deserialized or the user uses another + /// property alongside `from-file:`. + pub fn get_stages(stages: &[Self], traversed_files: Option>) -> Result> { + let mut found_stages = vec![]; + let traversed_files = traversed_files.unwrap_or_default(); + + for stage in stages { + found_stages.extend( + match stage { + Stage { + required_fields: Some(_), + from_file: None, + } => vec![stage.clone()], + Stage { + required_fields: None, + from_file: Some(file_name), + } => { + let file_name = PathBuf::from(file_name.as_ref()); + if traversed_files.contains(&file_name) { + bail!( + "{} File {} has already been parsed:\n{traversed_files:?}", + "Circular dependency detected!".bright_red(), + file_name.display().to_string().bold(), + ); + } + let mut tf = traversed_files.clone(); + tf.push(file_name.clone()); + + Self::get_stages(&StagesExt::parse(&file_name)?.stages, Some(tf))? + } + _ => { + let from_example = Stage::builder().from_file("path/to/stage.yml").build(); + let stage_example = Self::example(); + + bail!( + "Improper format for stage. Must be in the format like:\n{}\n{}\n\n{}", + highlight_ser(&stage_example, "yaml", None)?, + "or".bold(), + highlight_ser(&from_example, "yaml", None)? + ); + } + } + .into_iter(), + ); + } + Ok(found_stages) + } + + #[must_use] + pub fn example() -> Self { + Stage::builder() + .required_fields( + StageRequiredFields::builder() + .name("stage-name") + .from("build/image:here") + .modules_ext( + ModuleExt::builder() + .modules(vec![Module::example()]) + .build(), + ) + .build(), + ) + .build() + } +} diff --git a/recipe/src/stages_ext.rs b/recipe/src/stages_ext.rs new file mode 100644 index 0000000..c02d4c9 --- /dev/null +++ b/recipe/src/stages_ext.rs @@ -0,0 +1,61 @@ +use std::{borrow::Cow, fs, path::Path}; + +use anyhow::{Context, Result}; +use blue_build_utils::constants::{CONFIG_PATH, RECIPE_PATH}; +use log::warn; +use serde::{Deserialize, Serialize}; +use typed_builder::TypedBuilder; + +use crate::{Module, Stage}; + +#[derive(Default, Serialize, Clone, Deserialize, Debug, TypedBuilder)] +pub struct StagesExt<'a> { + #[builder(default, setter(into))] + pub stages: Cow<'a, [Stage<'a>]>, +} + +impl<'a> StagesExt<'a> { + /// Parse a module file returning a [`StagesExt`] + /// + /// # Errors + /// Can return an `anyhow` Error if the file cannot be read or deserialized + /// into a [`StagesExt`] + pub fn parse(file_name: &Path) -> Result { + let legacy_path = Path::new(CONFIG_PATH); + let recipe_path = Path::new(RECIPE_PATH); + + let file_path = if recipe_path.exists() && recipe_path.is_dir() { + recipe_path.join(file_name) + } else { + warn!("Use of {CONFIG_PATH} for recipes is deprecated, please move your recipe files into {RECIPE_PATH}"); + legacy_path.join(file_name) + }; + + let file = fs::read_to_string(&file_path) + .context(format!("Failed to open {}", file_path.display()))?; + + serde_yaml::from_str::(&file).map_or_else( + |_| -> Result { + let mut stage = serde_yaml::from_str::(&file) + .map_err(blue_build_utils::serde_yaml_err(&file))?; + if let Some(ref mut rf) = stage.required_fields { + rf.modules_ext.modules = + Module::get_modules(&rf.modules_ext.modules, None)?.into(); + } + Ok(Self::builder().stages(vec![stage]).build()) + }, + |mut stages_ext| -> Result { + let mut stages: Vec = + stages_ext.stages.iter().map(ToOwned::to_owned).collect(); + for stage in &mut stages { + if let Some(ref mut rf) = stage.required_fields { + rf.modules_ext.modules = + Module::get_modules(&rf.modules_ext.modules, None)?.into(); + } + } + stages_ext.stages = stages.into(); + Ok(stages_ext) + }, + ) + } +} diff --git a/scripts/run_module.sh b/scripts/run_module.sh index 2330723..83f58b9 100644 --- a/scripts/run_module.sh +++ b/scripts/run_module.sh @@ -2,6 +2,8 @@ set -euo pipefail +source /tmp/scripts/exports.sh + # Function to print a centered text banner within a specified width print_banner() { local term_width=120 @@ -25,4 +27,3 @@ print_banner "Start '${module}' Module" chmod +x ${script_path} ${script_path} "${params}" print_banner "End '${module}' Module" -ostree container commit diff --git a/scripts/setup.sh b/scripts/setup.sh new file mode 100644 index 0000000..fcb7509 --- /dev/null +++ b/scripts/setup.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +if [ -f /etc/os-release ]; then + export ID="$(cat /etc/os-release | grep -E '^ID=' | awk -F '=' '{print $2}')" + + if [ "$ID" = "alpine" ]; then + echo "Setting up Alpine based image to run BlueBuild modules" + apk update + apk add --no-cache bash curl coreutils wget grep + elif [ "$ID" = "ubuntu" ] || [ "$ID" = "debian" ]; then + echo "Setting up Ubuntu based image to run BlueBuild modules" + apt-get update + apt-get install -y bash curl coreutils wget + elif [ "$ID" = "fedora" ]; then + echo "Settig up Fedora based image to run BlueBuild modules" + dnf install -y --refresh bash curl wget coreutils + else + echo "OS not detected, proceeding without setup" + fi +fi +cp /tmp/bins/yq /usr/bin/ diff --git a/src/drivers/opts/build.rs b/src/drivers/opts/build.rs index a8e3772..94de13c 100644 --- a/src/drivers/opts/build.rs +++ b/src/drivers/opts/build.rs @@ -37,7 +37,7 @@ pub struct PushOpts<'a> { pub struct BuildTagPushOpts<'a> { /// The base image name. /// - /// NOTE: You cannot have this set with `archive-path` set. + /// NOTE: You cannot have this set with `archive_path` set. #[builder(default, setter(into, strip_option))] pub image: Option>, diff --git a/template/src/lib.rs b/template/src/lib.rs index 9c3badf..29f4c0e 100644 --- a/template/src/lib.rs +++ b/template/src/lib.rs @@ -13,7 +13,7 @@ use uuid::Uuid; pub use askama::Template; #[derive(Debug, Clone, Template, TypedBuilder)] -#[template(path = "Containerfile.j2", escape = "none")] +#[template(path = "Containerfile.j2", escape = "none", whitespace = "minimize")] pub struct ContainerFileTemplate<'a> { recipe: &'a Recipe<'a>, diff --git a/template/templates/Containerfile.j2 b/template/templates/Containerfile.j2 index 23934b5..10b7210 100644 --- a/template/templates/Containerfile.j2 +++ b/template/templates/Containerfile.j2 @@ -1,7 +1,9 @@ +{%- import "modules/modules.j2" as modules -%} {%- set files_dir_exists = self::files_dir_exists() %} {%- include "stages.j2" %} -FROM {{ recipe.base_image }}:{{ recipe.image_version }} +# Main image +FROM {{ recipe.base_image }}:{{ recipe.image_version }} as {{ recipe.name|replace('/', "-") }} ARG RECIPE={{ recipe_path.display() }} ARG IMAGE_REGISTRY={{ registry }} @@ -15,7 +17,19 @@ ARG MODULE_DIRECTORY="/tmp/modules" ARG IMAGE_NAME="{{ recipe.name }}" ARG BASE_IMAGE="{{ recipe.base_image }}" -{% include "modules/modules.j2" %} +# Key RUN +RUN --mount=type=bind,from=stage-keys,src=/keys,dst=/tmp/keys \ + mkdir -p /usr/etc/pki/containers/ \ + && cp /tmp/keys/* /usr/etc/pki/containers/ \ + && ostree container commit + +# Bin RUN +RUN --mount=type=bind,from=stage-bins,src=/bins,dst=/tmp/bins \ + mkdir -p /usr/bin/ \ + && cp /tmp/bins/* /usr/bin/ \ + && ostree container commit + +{% call modules::main_modules_run(recipe.modules_ext, os_version) %} RUN rm -fr /tmp/* /var/* && ostree container commit diff --git a/template/templates/modules/akmods/akmods.j2 b/template/templates/modules/akmods/akmods.j2 index 3066d7b..4e19b61 100644 --- a/template/templates/modules/akmods/akmods.j2 +++ b/template/templates/modules/akmods/akmods.j2 @@ -1,4 +1,5 @@ {%- for info in recipe.modules_ext.get_akmods_info_list(os_version) %} +# Stage for AKmod {{ info.stage_name }} FROM scratch as stage-akmods-{{ info.stage_name }} COPY --from=ghcr.io/ublue-os/{{ info.images.0 }} /rpms /rpms COPY --from=ghcr.io/ublue-os/{{ info.images.1 }} /rpms /rpms diff --git a/template/templates/modules/containerfile/README.md b/template/templates/modules/containerfile/README.md index bcf51e3..cb96751 100644 --- a/template/templates/modules/containerfile/README.md +++ b/template/templates/modules/containerfile/README.md @@ -4,7 +4,7 @@ Only compiler-based builds can use this module as it is built-in to the BlueBuild CLI tool. ::: -The `containerfile` module is a tool for adding custom [`Containerfile`](https://github.com/containers/common/blob/main/docs/Containerfile.5.md) instructions for custom image builds. This is useful when you wish to use some feature directly available in a `Containerfile`, but not in a bash module, such as copying from other OCI images with `COPY --from`. +The `containerfile` module is a tool for adding custom [`Containerfile`](https://github.com/containers/common/blob/main/docs/Containerfile.5.md) instructions for custom image builds. This is useful when you wish to use some feature directly available in a `Containerfile`, but not in a bash module, such as using a `RUN` instruction with custom mounts. Since standard compiler-based BlueBuild image builds generate a `Containerfile` from your recipe, there is no need to manage it yourself. However, we know that we also have technical users that would like to have the ability to customize their `Containerfile`. This is where the `containerfile` module comes into play. @@ -18,10 +18,10 @@ The `snippets` property is the easiest to use when you just need to insert a few modules: - type: containerfile snippets: - - COPY --from=docker.io/mikefarah/yq /usr/bin/yq /usr/bin/yq + - RUN --mount=type=tmpfs,target=/tmp /some/script.sh ``` -This makes it really easy to copy a file or program from another image. +This makes it really easy to add individual, custom instructions. :::note **NOTE:** Each entry of a snippet will be its own layer in the final `Containerfile`. @@ -66,7 +66,7 @@ If you wanted to have some `snippets` run before any `containerfiles` have, you modules: - type: containerfile snippets: - - COPY --from=docker.io/mikefarah/yq /usr/bin/yq /usr/bin/yq + - RUN --mount=type=tmpfs,target=/tmp /some/script.sh - type: containerfile containerfiles: - example diff --git a/template/templates/modules/containerfile/module.yml b/template/templates/modules/containerfile/module.yml index 32c267d..67e3faa 100644 --- a/template/templates/modules/containerfile/module.yml +++ b/template/templates/modules/containerfile/module.yml @@ -4,7 +4,7 @@ readme: https://raw.githubusercontent.com/blue-build/cli/main/template/templates example: | type: containerfile snippets: - - COPY ./config/example-dir/example-file.txt /usr/etc/example/ + - RUN --mount=type=tmpfs,target=/tmp /some/script.sh containerfiles: - example - subroutine diff --git a/template/templates/modules/copy/README.md b/template/templates/modules/copy/README.md new file mode 100644 index 0000000..6c525dc --- /dev/null +++ b/template/templates/modules/copy/README.md @@ -0,0 +1,44 @@ +# `copy` + +:::caution +Only compiler-based builds can use this module as it is built-in to the BlueBuild CLI tool. +::: + +:::note +**NOTE:** This module is currently only available with the `use_unstable_cli` option on the GHA or using the `main` image. +::: + +The `copy` module is a short-hand method of adding a [`COPY`](https://docs.docker.com/reference/dockerfile/#copy) instruction into the image. This can be used to copy files from images, other stages, or even from the build context. + +## Usage + +The `copy` module's properties are a 1-1 match with the `COPY` instruction containing `src`, `dest`, and `from` (optional). The example below will `COPY` the file `/usr/bin/yq` from `docker.io/mikefarah/yq` into `/usr/bin/`. + +```yaml +mdoules: +- type: copy + from: docker.io/mikefarah/yq + src: /usr/bin/yq + dest: /usr/bin/ +``` + +Creating an instruction like: + +```dockerfile +COPY --linked --from=docker.io/mikefarah/yq /usr/bin/yq /usr/bin/ +``` + +Omitting `from:` will allow copying from the build context: + +```yaml +mdoules: +- type: copy + src: file/to/copy.conf + dest: /usr/etc/app/ +``` + +Creating an instruction like: + +```dockerfile +COPY --linked file/to/copy.conf /usr/etc/app/ +``` diff --git a/template/templates/modules/copy/copy.j2 b/template/templates/modules/copy/copy.j2 new file mode 100644 index 0000000..e82130d --- /dev/null +++ b/template/templates/modules/copy/copy.j2 @@ -0,0 +1,4 @@ +{%- if let Some((from_img, src, dest)) = module.get_copy_args() %} +COPY{% if let Some(from_img) = from_img %} --from={{ from_img }}{% endif %} {{ src }} {{ dest }} +{%- endif %} + diff --git a/template/templates/modules/copy/module.yml b/template/templates/modules/copy/module.yml new file mode 100644 index 0000000..2179b6f --- /dev/null +++ b/template/templates/modules/copy/module.yml @@ -0,0 +1,8 @@ +name: copy +shortdesc: The copy module is a direct translation of the `COPY` instruction in a Containerfile. +readme: https://raw.githubusercontent.com/blue-build/cli/main/template/templates/modules/copy/README.md +example: | + type: copy + from: docker.io/mikefarah/yq + src: /usr/bin/yq + dest: /usr/bin/ diff --git a/template/templates/modules/modules.j2 b/template/templates/modules/modules.j2 index 2699b4a..5663a3d 100644 --- a/template/templates/modules/modules.j2 +++ b/template/templates/modules/modules.j2 @@ -1,41 +1,66 @@ -# Key RUN -RUN --mount=type=bind,from=stage-keys,src=/keys,dst=/tmp/keys \ - mkdir -p /usr/etc/pki/containers/ \ - && cp /tmp/keys/* /usr/etc/pki/containers/ \ - && ostree container commit - -# Bin RUN -RUN --mount=type=bind,from=stage-bins,src=/bins,dst=/tmp/bins \ - mkdir -p /usr/bin/ \ - && cp /tmp/bins/* /usr/bin/ \ - && ostree container commit - +{% macro main_modules_run(modules_ext, os_version) %} # Module RUNs -{%- for module in recipe.modules_ext.modules %} - {%- if let Some(type) = module.module_type %} - {%- if type == "containerfile" %} - {%- include "modules/containerfile/containerfile.j2" %} - {%- else %} + {%- for module in modules_ext.modules %} + {%- if let Some(module) = module.required_fields %} + {%- if module.no_cache %} +ARG CACHEBUST="{{ build_id }}" + {%- endif %} + + {%- if module.module_type == "containerfile" %} + {%- include "modules/containerfile/containerfile.j2" %} + {%- else if module.module_type == "copy" %} + {%- include "modules/copy/copy.j2" %} + {%- else %} RUN \ - {%- if files_dir_exists %} + {%- if files_dir_exists %} --mount=type=bind,from=stage-files,src=/files,dst=/tmp/files,rw \ - {%- else %} + {%- else %} --mount=type=bind,from=stage-config,src=/config,dst=/tmp/config,rw \ - {%- endif %} - {%- if let Some(source) = module.source %} + {%- endif %} + {%- if let Some(source) = module.source %} --mount=type=bind,from={{ source }},src=/modules,dst=/tmp/modules,rw \ - {%- else %} + {%- else %} --mount=type=bind,from=stage-modules,src=/modules,dst=/tmp/modules,rw \ - {%- endif %} - {%- if type == "akmods" %} + {%- endif %} + {%- if module.module_type == "akmods" %} --mount=type=bind,from=stage-akmods-{{ module.generate_akmods_info(os_version).stage_name }},src=/rpms,dst=/tmp/rpms,rw \ - {%- endif %} + {%- endif %} --mount=type=bind,from=ghcr.io/blue-build/cli:{{ exports_tag }}-build-scripts,src=/scripts/,dst=/tmp/scripts/ \ --mount=type=cache,dst=/var/cache/rpm-ostree,id=rpm-ostree-cache-{{ recipe.name }}-{{ recipe.image_version }},sharing=locked \ - source /tmp/scripts/exports.sh \ - && /tmp/scripts/run_module.sh {{ type }} '{{ module.print_module_context() }}' + /tmp/scripts/run_module.sh '{{ module.module_type }}' '{{ module.print_module_context() }}' \ + && ostree container commit + {%- endif %} {%- endif %} - {%- endif %} -{%- endfor %} + {%- endfor %} +{% endmacro %} +{% macro stage_modules_run(modules_ext, os_version) %} +# Module RUNs + {%- for module in modules_ext.modules %} + {%- if let Some(module) = module.required_fields %} + {%- if module.no_cache %} +ARG CACHEBUST="{{ build_id }}" + {%- endif %} + {%- if module.module_type == "containerfile" %} + {%- include "modules/containerfile/containerfile.j2" %} + {%- else if module.module_type == "copy" %} + {%- include "modules/copy/copy.j2" %} + {%- else %} +RUN \ + {%- if files_dir_exists %} + --mount=type=bind,from=stage-files,src=/files,dst=/tmp/files,rw \ + {%- else %} + --mount=type=bind,from=stage-config,src=/config,dst=/tmp/config,rw \ + {%- endif %} + {%- if let Some(source) = module.source %} + --mount=type=bind,from={{ source }},src=/modules,dst=/tmp/modules,rw \ + {%- else %} + --mount=type=bind,from=stage-modules,src=/modules,dst=/tmp/modules,rw \ + {%- endif %} + --mount=type=bind,from=ghcr.io/blue-build/cli:{{ exports_tag }}-build-scripts,src=/scripts/,dst=/tmp/scripts/ \ + /tmp/scripts/run_module.sh '{{ module.module_type }}' '{{ module.print_module_context() }}' + {%- endif %} + {%- endif %} + {%- endfor %} +{% endmacro %} diff --git a/template/templates/stages.j2 b/template/templates/stages.j2 index 6c07f1d..24e87f3 100644 --- a/template/templates/stages.j2 +++ b/template/templates/stages.j2 @@ -2,29 +2,28 @@ # your config without copying it directly into # the final image {%- if files_dir_exists %} -FROM scratch as stage-files +FROM scratch AS stage-files COPY ./files /files -{%- else %} -FROM scratch as stage-config +{% else %} +FROM scratch AS stage-config COPY ./config /config -{%- endif %} +{% endif %} # Copy modules # The default modules are inside blue-build/modules # Custom modules overwrite defaults -FROM scratch as stage-modules +FROM scratch AS stage-modules COPY --from=ghcr.io/blue-build/modules:latest /modules /modules {%- if self::modules_exists() %} COPY ./modules /modules -{%- endif %} +{% endif %} # Bins to install # These are basic tools that are added to all images. # Generally used for the build process. We use a multi # stage process so that adding the bins into the image # can be added to the ostree commits. -FROM scratch as stage-bins - +FROM scratch AS stage-bins COPY --from=gcr.io/projectsigstore/cosign /ko-app/cosign /bins/cosign COPY --from=docker.io/mikefarah/yq /usr/bin/yq /bins/yq COPY --from=ghcr.io/blue-build/cli: @@ -40,10 +39,41 @@ latest-installer # # Currently only holds the current image's # public key. -FROM scratch as stage-keys - +FROM scratch AS stage-keys {%- if self::has_cosign_file() %} COPY cosign.pub /keys/{{ recipe.name|replace('/', "_") }}.pub -{%- endif %} +{% endif %} {%- include "modules/akmods/akmods.j2" %} + +{%- set files_dir_exists = self::files_dir_exists() %} +{%~ if let Some(stages_ext) = recipe.stages_ext %} + {%- for stage in stages_ext.stages %} + {%- if let Some(stage) = stage.required_fields %} +# {{ stage.name|capitalize }} stage +FROM {{ stage.from }} AS {{ stage.name }} + + {%- if stage.from != "scratch" %} +# Add compatibility for modules +RUN --mount=type=bind,from=stage-bins,src=/bins/,dst=/tmp/bins/ \ + --mount=type=bind,from=ghcr.io/blue-build/cli:{{ exports_tag }}-build-scripts,src=/scripts/,dst=/tmp/scripts/ \ + /tmp/scripts/setup.sh + + {%- if files_dir_exists %} +ARG CONFIG_DIRECTORY="/tmp/files" + {%- else %} +ARG CONFIG_DIRECTORY="/tmp/config" + {%- endif %} +ARG MODULE_DIRECTORY="/tmp/modules" + + {%- if let Some(shell_args) = stage.shell %} +SHELL [{% for arg in shell_args %}"{{ arg }}"{% if !loop.last %}, {% endif %}{% endfor %}] + {%- else %} +SHELL ["bash", "-c"] + {%- endif %} + {%- endif %} + + {% call modules::stage_modules_run(stage.modules_ext, os_version) %} + {%- endif %} + {%- endfor %} +{%- endif %} diff --git a/utils/src/syntax_highlighting.rs b/utils/src/syntax_highlighting.rs index f3f3c0e..0bacc43 100644 --- a/utils/src/syntax_highlighting.rs +++ b/utils/src/syntax_highlighting.rs @@ -35,9 +35,8 @@ impl std::fmt::Display for DefaultThemes { /// # Errors /// Will error if the theme doesn't exist, the syntax doesn't exist, or the file /// failed to serialize. -pub fn print(file: &str, file_type: &str, theme: Option) -> Result<()> { - trace!("syntax_highlighting::print({file}, {file_type}, {theme:?})"); - +pub fn highlight(file: &str, file_type: &str, theme: Option) -> Result { + trace!("syntax_highlighting::highlight(file, {file_type}, {theme:?})"); if atty::is(atty::Stream::Stdout) { let ss: SyntaxSet = if file_type == "dockerfile" || file_type == "Dockerfile" { dumps::from_uncompressed_data(include_bytes!(concat!( @@ -58,15 +57,43 @@ pub fn print(file: &str, file_type: &str, theme: Option) -> Resul .get(theme.unwrap_or_default().to_string().as_str()) .ok_or_else(|| anyhow!("Failed to get highlight theme"))?, ); + + let mut highlighted_lines: Vec = vec![]; for line in file.lines() { - let ranges = h.highlight_line(line, &ss)?; - let escaped = syntect::util::as_24_bit_terminal_escaped(&ranges, false); - println!("{escaped}"); + highlighted_lines.push(syntect::util::as_24_bit_terminal_escaped( + &h.highlight_line(line, &ss)?, + false, + )); } - println!("\x1b[0m"); + highlighted_lines.push("\x1b[0m".to_string()); + Ok(highlighted_lines.join("\n")) } else { - println!("{file}"); + Ok(file.to_string()) } +} + +/// Takes a serializable struct and serializes it with syntax highlighting. +/// +/// # Errors +/// Will error if the theme doesn't exist, the syntax doesn't exist, or the file +/// failed to serialize. +pub fn highlight_ser( + file: &T, + file_type: &str, + theme: Option, +) -> Result { + trace!("syntax_highlighting::highlight_ser(file, {file_type}, {theme:?})"); + highlight(serde_yaml::to_string(file)?.as_str(), file_type, theme) +} + +/// Prints the file with syntax highlighting. +/// +/// # Errors +/// Will error if the theme doesn't exist, the syntax doesn't exist, or the file +/// failed to serialize. +pub fn print(file: &str, file_type: &str, theme: Option) -> Result<()> { + trace!("syntax_highlighting::print(file, {file_type}, {theme:?})"); + println!("{}", highlight(file, file_type, theme)?); Ok(()) } @@ -75,11 +102,12 @@ pub fn print(file: &str, file_type: &str, theme: Option) -> Resul /// # Errors /// Will error if the theme doesn't exist, the syntax doesn't exist, or the file /// failed to serialize. -pub fn print_ser( +pub fn print_ser( file: &T, file_type: &str, theme: Option, ) -> Result<()> { + trace!("syntax_highlighting::print_ser(file, {file_type}, {theme:?})"); print(serde_yaml::to_string(file)?.as_str(), file_type, theme)?; Ok(()) }