From 8069006c03c12ae9401fa1d097eded2a2929469b Mon Sep 17 00:00:00 2001 From: Gerald Pinder Date: Sat, 18 May 2024 09:23:50 -0400 Subject: [PATCH] feat: Stages (#173) ## Stages A new property (`stages`) is being added to the recipe file schema. This property will allow users to define a list of Containerfile stages each with their own modules. Stages can be used to compile programs, perform parallel operations, and copy the results into the final image without contaminating the final image. ### Module Support Currently the only modules that work out-of-the-box are `copy`, `script`, `files`, and `containerfile`. Other modules are dependent on the programs installed on the image. In order to better support some of our essential modules, a setup script is ran at the start of each stage that is not `scratch`. This script will install `curl`, `wget`, `bash`, and `grep` and use the package manager for the detected distributions. At this time, the following distributions are supported: - Debian - Ubuntu - Fedora - Alpine Contributions to increase the size of this list is [welcome](https://github.com/blue-build/cli)! ### Syntax - **Required** - `from` - The full image ref (image name + tag). This will be set in the `FROM` statement of the stage. - `name` - The name of the stage. This is used when referencing the stage when using the `from:` property in the `copy` module. - `modules` - The list of modules to execute. The exact same syntax used by the main recipe `modules:` property. - **Optional** - `shell` - Allows a user to pass in an array of strings that are passed directly into the [`SHELL` instruction](https://docs.docker.com/reference/dockerfile/#shell). #### Example ```yaml stages: - name: ubuntu-test from: ubuntu modules: - type: files files: - usr: /usr - type: script scripts: - example.sh snippets: - echo "test" > /test.txt - type: test-module - type: containerfile containerfiles: - labels snippets: - RUN echo "This is a snippet" ``` ### Tasks - [x] `from-file:` - Allows the user to store their stages in a separate file so it can be included in multiple recipes - [x] `no-cache:` - This will be useful for stages that want to pull the latest changes from a git repo and not have to rely on the base image getting an update for the build to be triggered again. - [x] Add setup script to be able to install necessary programs to run `bluebuild` modules in stages - [x] Check for circular dependencies and error out ## `copy` module This is a 1-1 for the [`COPY` instruction](https://docs.docker.com/reference/dockerfile/#copy). It has the ability to copy files between stages, making this a very important addition to complete functionality for the stages feature. Each use of this "module" will become its own layer. ### Decision to use `--link` We use the `--link` [option](https://docs.docker.com/reference/dockerfile/#benefits-of-using---link) which allows that layer to have the same hash if the files haven't changed regardless of if the previous instructions have changed. This allows these layers to not have to be re-downloaded on the user's computer if the copied files haven't changed. ### Syntax - **Required** - `src` - The source directory/file from the repo OR when `from:` is set the image/stage that is specified. - `dest` - The destination directory/file inside the working image. - **Optional** - `from` - The stage/image to copy from. #### Example ```yaml modules: - type: copy from: ubuntu-test src: /test.txt dest: / ``` ### Tasks - [x] make `from:` optional - [x] Add README.md and module.yml ## Feature gating Gating this feature until we release for `v0.9.0`. The plan will be to build all features (including this one) for main branch builds. This means that these features will be available when using the `main` image and consequently the `use_unstable_cli:` option on the GitHub Action. All future `v0.9.0` features will be gated as well to allow for patches to `v0.8`. ### Tasks - [x] Build `--all-features` on non-tagged builds - [x] Add stages and copy features --- .rusty-hook.toml | 2 +- Cargo.lock | 1 + Cargo.toml | 2 + Earthfile | 44 +++-- integration-tests/Earthfile | 10 +- .../test-repo/files/scripts/bluebuild.sh | 7 + .../test-repo/recipes/bluebuild.yml | 12 ++ .../test-repo/recipes/recipe.yml | 23 ++- .../test-repo/recipes/stages.yml | 32 ++++ modules.json | 3 +- recipe/Cargo.toml | 6 + recipe/src/lib.rs | 4 + recipe/src/module.rs | 159 ++++++++++++++---- recipe/src/module_ext.rs | 20 ++- recipe/src/recipe.rs | 22 ++- recipe/src/stage.rs | 159 ++++++++++++++++++ recipe/src/stages_ext.rs | 61 +++++++ scripts/run_module.sh | 3 +- scripts/setup.sh | 21 +++ src/drivers/opts/build.rs | 2 +- template/src/lib.rs | 2 +- template/templates/Containerfile.j2 | 18 +- template/templates/modules/akmods/akmods.j2 | 1 + .../templates/modules/containerfile/README.md | 8 +- .../modules/containerfile/module.yml | 2 +- template/templates/modules/copy/README.md | 44 +++++ template/templates/modules/copy/copy.j2 | 4 + template/templates/modules/copy/module.yml | 8 + template/templates/modules/modules.j2 | 83 +++++---- template/templates/stages.j2 | 52 ++++-- utils/src/syntax_highlighting.rs | 46 ++++- 31 files changed, 742 insertions(+), 119 deletions(-) create mode 100644 integration-tests/test-repo/files/scripts/bluebuild.sh create mode 100644 integration-tests/test-repo/recipes/bluebuild.yml create mode 100644 integration-tests/test-repo/recipes/stages.yml create mode 100644 recipe/src/stage.rs create mode 100644 recipe/src/stages_ext.rs create mode 100644 scripts/setup.sh create mode 100644 template/templates/modules/copy/README.md create mode 100644 template/templates/modules/copy/copy.j2 create mode 100644 template/templates/modules/copy/module.yml diff --git a/.rusty-hook.toml b/.rusty-hook.toml index ac4c348..89f9c9f 100644 --- a/.rusty-hook.toml +++ b/.rusty-hook.toml @@ -1,5 +1,5 @@ [hooks] -pre-commit = "cargo fmt --check && cargo test && cargo clippy -- -D warnings" +pre-commit = "cargo fmt --check && cargo test --all-features && cargo clippy --all-features -- -D warnings" [logging] verbose = true diff --git a/Cargo.lock b/Cargo.lock index 2f5143e..b2abbc6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -228,6 +228,7 @@ dependencies = [ "anyhow", "blue-build-utils", "chrono", + "colored", "indexmap 2.2.3", "log", "serde", diff --git a/Cargo.toml b/Cargo.toml index 87ef602..c8b4eb1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,6 +84,8 @@ uuid.workspace = true [features] default = [] +stages = ["blue-build-recipe/stages"] +copy = ["blue-build-recipe/copy"] [dev-dependencies] rusty-hook = "0.11.2" diff --git a/Earthfile b/Earthfile index b83a3e3..285d7d1 100644 --- a/Earthfile +++ b/Earthfile @@ -1,7 +1,7 @@ VERSION 0.8 PROJECT blue-build/cli -IMPORT github.com/blue-build/earthly-lib/cargo AS cargo +IMPORT github.com/earthly/lib/rust AS rust ARG --global IMAGE=ghcr.io/blue-build/cli @@ -21,17 +21,29 @@ build: lint: FROM +common - DO cargo+LINT + DO rust+CARGO --args="clippy -- -D warnings" + DO rust+CARGO --args="clippy --all-features -- -D warnings" + DO rust+CARGO --args="clippy --no-default-features -- -D warnings" test: FROM +common - DO cargo+TEST + DO rust+CARGO --args="test -- --show-output" + DO rust+CARGO --args="test --all-features -- --show-output" + DO rust+CARGO --args="test --no-default-features -- --show-output" install: FROM +common ARG --required BUILD_TARGET - DO cargo+BUILD_RELEASE --BUILD_TARGET=$BUILD_TARGET + DO rust+CARGO --args="build --release --target $BUILD_TARGET" --output="$BUILD_TARGET/release/[^\./]+" + + SAVE ARTIFACT target/$BUILD_TARGET/release/bluebuild + +install-all-features: + FROM +common + ARG --required BUILD_TARGET + + DO rust+CARGO --args="build --all-features --release --target $BUILD_TARGET" --output="$BUILD_TARGET/release/[^\./]+" SAVE ARTIFACT target/$BUILD_TARGET/release/bluebuild @@ -47,7 +59,7 @@ common: COPY --keep-ts --dir .git/ /app RUN touch build.rs - DO cargo+INIT + DO rust+INIT --keep_fingerprints=true build-scripts: FROM alpine @@ -65,8 +77,6 @@ blue-build-cli: FROM $BASE_IMAGE LABEL org.opencontainers.image.base.name="$BASE_IMAGE" - BUILD +install --BUILD_TARGET="x86_64-unknown-linux-gnu" - RUN dnf -y install dnf-plugins-core \ && dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo \ && dnf install --refresh -y \ @@ -84,7 +94,7 @@ blue-build-cli: COPY +cosign/cosign /usr/bin/cosign - COPY (+install/bluebuild --BUILD_TARGET="x86_64-unknown-linux-gnu") /usr/bin/bluebuild + DO --pass-args +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="x86_64-unknown-linux-gnu" RUN mkdir -p /bluebuild WORKDIR /bluebuild @@ -97,14 +107,12 @@ blue-build-cli-alpine: FROM $BASE_IMAGE LABEL org.opencontainers.image.base.name="$BASE_IMAGE" - BUILD +install --BUILD_TARGET="x86_64-unknown-linux-musl" - RUN apk update && apk add buildah podman skopeo fuse-overlayfs jq LABEL org.opencontainers.image.base.digest="$(skopeo inspect "docker://$BASE_IMAGE" | jq -r '.Digest')" COPY +cosign/cosign /usr/bin/cosign - COPY (+install/bluebuild --BUILD_TARGET="x86_64-unknown-linux-musl") /usr/bin/bluebuild + DO --pass-args +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="x86_64-unknown-linux-musl" RUN mkdir -p /bluebuild WORKDIR /bluebuild @@ -121,7 +129,7 @@ installer: LABEL org.opencontainers.image.base.digest="$(skopeo inspect "docker://$BASE_IMAGE" | jq -r '.Digest')" - COPY (+install/bluebuild --BUILD_TARGET="x86_64-unknown-linux-musl") /out/bluebuild + DO --pass-args +INSTALL --OUT_DIR="/out/" --BUILD_TARGET="x86_64-unknown-linux-musl" COPY install.sh /install.sh CMD ["cat", "/install.sh"] @@ -146,6 +154,18 @@ version: SAVE ARTIFACT /version +INSTALL: + FUNCTION + ARG TAGGED="false" + ARG --required BUILD_TARGET + ARG --required OUT_DIR + + IF [ "$TAGGED" = "true" ] + COPY (+install/bluebuild --BUILD_TARGET="$BUILD_TARGET") $OUT_DIR + ELSE + COPY (+install-all-features/bluebuild --BUILD_TARGET="$BUILD_TARGET") $OUT_DIR + END + SAVE_IMAGE: FUNCTION ARG SUFFIX="" diff --git a/integration-tests/Earthfile b/integration-tests/Earthfile index 5f268ec..bf6c7fe 100644 --- a/integration-tests/Earthfile +++ b/integration-tests/Earthfile @@ -92,11 +92,17 @@ secureblue-base: DO +GEN_KEYPAIR legacy-base: - FROM +test-base + FROM ../+blue-build-cli-alpine + ENV BB_TEST_LOCAL_IMAGE=/etc/bluebuild/cli_test.tar.gz + ENV CLICOLOR_FORCE=1 - RUN rm -fr /test + COPY ./mock-scripts/ /usr/bin/ + + WORKDIR /test COPY ./legacy-test-repo /test + DO ../+INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="x86_64-unknown-linux-musl" --TAGGED="true" + DO +GEN_KEYPAIR test-base: diff --git a/integration-tests/test-repo/files/scripts/bluebuild.sh b/integration-tests/test-repo/files/scripts/bluebuild.sh new file mode 100644 index 0000000..b808c5a --- /dev/null +++ b/integration-tests/test-repo/files/scripts/bluebuild.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -euo pipefail + +cargo install blue-build --debug --all-features --target x86_64-unknown-linux-gnu +mkdir -p /out/ +mv $CARGO_HOME/bin/bluebuild /out/bluebuild diff --git a/integration-tests/test-repo/recipes/bluebuild.yml b/integration-tests/test-repo/recipes/bluebuild.yml new file mode 100644 index 0000000..2959560 --- /dev/null +++ b/integration-tests/test-repo/recipes/bluebuild.yml @@ -0,0 +1,12 @@ +stages: + - name: blue-build + image: rust + modules: + - type: script + scripts: + - bluebuild.sh +modules: + - type: copy + from: blue-build + src: /out/bluebuild + dest: /usr/bin/bluebuild diff --git a/integration-tests/test-repo/recipes/recipe.yml b/integration-tests/test-repo/recipes/recipe.yml index ba09243..ea91aec 100644 --- a/integration-tests/test-repo/recipes/recipe.yml +++ b/integration-tests/test-repo/recipes/recipe.yml @@ -1,10 +1,12 @@ name: cli/test -description: This is my personal OS image. -base-image: ghcr.io/ublue-os/silverblue-main alt-tags: - gts - stable +description: This is my personal OS image. +base-image: ghcr.io/ublue-os/silverblue-main image-version: 40 +stages: + - from-file: stages.yml modules: - from-file: akmods.yml - from-file: flatpaks.yml @@ -36,3 +38,20 @@ modules: - labels snippets: - RUN echo "This is a snippet" && ostree container commit + + - type: copy + from: alpine-test + src: /test.txt + dest: / + - type: copy + from: ubuntu-test + src: /test.txt + dest: / + - type: copy + from: debian-test + src: /test.txt + dest: / + - type: copy + from: fedora-test + src: /test.txt + dest: / diff --git a/integration-tests/test-repo/recipes/stages.yml b/integration-tests/test-repo/recipes/stages.yml new file mode 100644 index 0000000..e2851c5 --- /dev/null +++ b/integration-tests/test-repo/recipes/stages.yml @@ -0,0 +1,32 @@ +stages: + - name: ubuntu-test + from: ubuntu + modules: + - from-file: stages.yml + - name: debian-test + from: debian + modules: + - from-file: stages.yml + - name: fedora-test + from: fedora + modules: + - from-file: stages.yml + - name: alpine-test + from: alpine + modules: + - from-file: stages.yml +modules: + - type: files + files: + - usr: /usr + - type: script + scripts: + - example.sh + snippets: + - echo "test" > /test.txt + - type: test-module + - type: containerfile + containerfiles: + - labels + snippets: + - RUN echo "This is a snippet" diff --git a/modules.json b/modules.json index b229517..3c5d30e 100644 --- a/modules.json +++ b/modules.json @@ -1,3 +1,4 @@ [ - "https://raw.githubusercontent.com/blue-build/cli/main/template/templates/modules/containerfile/module.yml" + "https://raw.githubusercontent.com/blue-build/cli/main/template/templates/modules/containerfile/module.yml", + "https://raw.githubusercontent.com/blue-build/cli/main/template/templates/modules/copy/module.yml" ] \ No newline at end of file diff --git a/recipe/Cargo.toml b/recipe/Cargo.toml index 616ad5d..520b935 100644 --- a/recipe/Cargo.toml +++ b/recipe/Cargo.toml @@ -14,11 +14,17 @@ chrono = "0.4" indexmap = { version = "2", features = ["serde"] } anyhow.workspace = true +colored.workspace = true log.workspace = true serde.workspace = true serde_yaml.workspace = true serde_json.workspace = true typed-builder.workspace = true +[features] +default = [] +stages = [] +copy = [] + [lints] workspace = true diff --git a/recipe/src/lib.rs b/recipe/src/lib.rs index c98d976..4d04119 100644 --- a/recipe/src/lib.rs +++ b/recipe/src/lib.rs @@ -2,8 +2,12 @@ pub mod akmods_info; pub mod module; pub mod module_ext; pub mod recipe; +pub mod stage; +pub mod stages_ext; pub use akmods_info::*; pub use module::*; pub use module_ext::*; pub use recipe::*; +pub use stage::*; +pub use stages_ext::*; diff --git a/recipe/src/module.rs b/recipe/src/module.rs index 003cae1..5da25f5 100644 --- a/recipe/src/module.rs +++ b/recipe/src/module.rs @@ -1,6 +1,8 @@ -use std::{borrow::Cow, process}; +use std::{borrow::Cow, path::PathBuf, process}; use anyhow::{bail, Result}; +use blue_build_utils::syntax_highlighting::highlight_ser; +use colored::Colorize; use indexmap::IndexMap; use log::{error, trace, warn}; use serde::{Deserialize, Serialize}; @@ -9,54 +11,34 @@ use typed_builder::TypedBuilder; use crate::{AkmodsInfo, ModuleExt}; -#[derive(Serialize, Deserialize, Debug, Clone, TypedBuilder)] -pub struct Module<'a> { - #[builder(default, setter(into, strip_option))] - #[serde(rename = "type", skip_serializing_if = "Option::is_none")] - pub module_type: Option>, - - #[builder(default, setter(into, strip_option))] - #[serde(rename = "from-file", skip_serializing_if = "Option::is_none")] - pub from_file: Option>, +#[derive(Serialize, Deserialize, Debug, Clone, TypedBuilder, Default)] +pub struct ModuleRequiredFields<'a> { + #[builder(default, setter(into))] + #[serde(rename = "type")] + pub module_type: Cow<'a, str>, #[builder(default, setter(into, strip_option))] #[serde(skip_serializing_if = "Option::is_none")] pub source: Option>, + #[builder(default)] + #[serde(rename = "no-cache", default, skip_serializing_if = "is_false")] + pub no_cache: bool, + #[serde(flatten)] #[builder(default, setter(into))] pub config: IndexMap, } -impl<'a> Module<'a> { - /// Get's any child modules. - /// - /// # Errors - /// Will error if the module cannot be - /// deserialized or the user uses another - /// property alongside `from-file:`. - pub fn get_modules(modules: &[Self]) -> Result> { - let mut found_modules = vec![]; - for module in modules { - found_modules.extend( - match module.from_file.as_ref() { - None => vec![module.clone()], - Some(file_name) => { - if module.module_type.is_some() || module.source.is_some() { - bail!("You cannot use the `type:` or `source:` property with `from-file:`"); - } - Self::get_modules(&ModuleExt::parse_module_from_file(file_name)?.modules)? - } - } - .into_iter(), - ); - } - Ok(found_modules) - } +#[allow(clippy::trivially_copy_pass_by_ref)] +const fn is_false(b: &bool) -> bool { + !*b +} +impl<'a> ModuleRequiredFields<'a> { #[must_use] pub fn get_module_type_list(&'a self, typ: &str, list_key: &str) -> Option> { - if self.module_type.as_ref()? == typ { + if self.module_type == typ { Some( self.config .get(list_key)? @@ -88,6 +70,24 @@ impl<'a> Module<'a> { }) } + #[must_use] + #[allow(clippy::missing_const_for_fn)] + pub fn get_copy_args(&'a self) -> Option<(Option<&'a str>, &'a str, &'a str)> { + #[cfg(feature = "copy")] + { + Some(( + self.config.get("from").and_then(|from| from.as_str()), + self.config.get("src")?.as_str()?, + self.config.get("dest")?.as_str()?, + )) + } + + #[cfg(not(feature = "copy"))] + { + None + } + } + #[must_use] pub fn generate_akmods_info(&'a self, os_version: &u64) -> AkmodsInfo { #[derive(Debug, Copy, Clone)] @@ -161,3 +161,90 @@ impl<'a> Module<'a> { .build() } } + +#[derive(Serialize, Deserialize, Debug, Clone, TypedBuilder, Default)] +pub struct Module<'a> { + #[builder(default, setter(strip_option))] + #[serde(flatten, skip_serializing_if = "Option::is_none")] + pub required_fields: Option>, + + #[builder(default, setter(into, strip_option))] + #[serde(rename = "from-file", skip_serializing_if = "Option::is_none")] + pub from_file: Option>, +} + +impl<'a> Module<'a> { + /// Get's any child modules. + /// + /// # Errors + /// Will error if the module cannot be + /// deserialized or the user uses another + /// property alongside `from-file:`. + pub fn get_modules( + modules: &[Self], + traversed_files: Option>, + ) -> Result> { + let mut found_modules = vec![]; + let traversed_files = traversed_files.unwrap_or_default(); + + for module in modules { + found_modules.extend( + match &module { + Module { + required_fields: Some(_), + from_file: None, + } => vec![module.clone()], + Module { + required_fields: None, + from_file: Some(file_name), + } => { + let file_name = PathBuf::from(file_name.as_ref()); + if traversed_files.contains(&file_name) { + bail!( + "{} File {} has already been parsed:\n{traversed_files:?}", + "Circular dependency detected!".bright_red(), + file_name.display().to_string().bold(), + ); + } + + let mut traversed_files = traversed_files.clone(); + traversed_files.push(file_name.clone()); + + Self::get_modules( + &ModuleExt::parse(&file_name)?.modules, + Some(traversed_files), + )? + } + _ => { + let from_example = Self::builder().from_file("test.yml").build(); + let module_example = Self::example(); + + bail!( + "Improper format for module. Must be in the format like:\n{}\n{}\n\n{}", + highlight_ser(&module_example, "yaml", None)?, + "or".bold(), + highlight_ser(&from_example, "yaml", None)? + ); + } + } + .into_iter(), + ); + } + Ok(found_modules) + } + + #[must_use] + pub fn example() -> Self { + Self::builder() + .required_fields( + ModuleRequiredFields::builder() + .module_type("module-name") + .config(IndexMap::from_iter([ + ("module".to_string(), Value::String("config".to_string())), + ("goes".to_string(), Value::String("here".to_string())), + ])) + .build(), + ) + .build() + } +} diff --git a/recipe/src/module_ext.rs b/recipe/src/module_ext.rs index a6e1885..f21abab 100644 --- a/recipe/src/module_ext.rs +++ b/recipe/src/module_ext.rs @@ -15,12 +15,12 @@ pub struct ModuleExt<'a> { } impl ModuleExt<'_> { - /// # Parse a module file returning a [`ModuleExt`] + /// Parse a module file returning a [`ModuleExt`] /// /// # Errors /// Can return an `anyhow` Error if the file cannot be read or deserialized /// into a [`ModuleExt`] - pub fn parse_module_from_file(file_name: &str) -> Result { + pub fn parse(file_name: &Path) -> Result { let legacy_path = Path::new(CONFIG_PATH); let recipe_path = Path::new(RECIPE_PATH); @@ -52,8 +52,20 @@ impl ModuleExt<'_> { self.modules .iter() - .filter(|module| module.module_type.as_ref().is_some_and(|t| t == "akmods")) - .map(|module| module.generate_akmods_info(os_version)) + .filter(|module| { + module + .required_fields + .as_ref() + .is_some_and(|rf| rf.module_type == "akmods") + }) + .filter_map(|module| { + Some( + module + .required_fields + .as_ref()? + .generate_akmods_info(os_version), + ) + }) .filter(|image| seen.insert(image.clone())) .collect() } diff --git a/recipe/src/recipe.rs b/recipe/src/recipe.rs index 107dfe0..e1c53f7 100644 --- a/recipe/src/recipe.rs +++ b/recipe/src/recipe.rs @@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize}; use serde_yaml::Value; use typed_builder::TypedBuilder; -use crate::{Module, ModuleExt}; +use crate::{Module, ModuleExt, StagesExt}; /// The build recipe. /// @@ -60,6 +60,14 @@ pub struct Recipe<'a> { #[builder(default, setter(into, strip_option))] pub alt_tags: Option>>, + /// The stages extension of the recipe. + /// + /// This hold the list of stages that can + /// be used to build software outside of + /// the final build image. + #[serde(flatten, skip_serializing_if = "Option::is_none")] + pub stages_ext: Option>, + /// The modules extension of the recipe. /// /// This holds the list of modules to be run on the image. @@ -207,7 +215,17 @@ impl<'a> Recipe<'a> { let mut recipe = serde_yaml::from_str::(&file) .map_err(blue_build_utils::serde_yaml_err(&file))?; - recipe.modules_ext.modules = Module::get_modules(&recipe.modules_ext.modules)?.into(); + recipe.modules_ext.modules = Module::get_modules(&recipe.modules_ext.modules, None)?.into(); + + #[cfg(feature = "stages")] + if let Some(ref mut stages_ext) = recipe.stages_ext { + stages_ext.stages = crate::Stage::get_stages(&stages_ext.stages, None)?.into(); + } + + #[cfg(not(feature = "stages"))] + { + recipe.stages_ext = None; + } Ok(recipe) } diff --git a/recipe/src/stage.rs b/recipe/src/stage.rs new file mode 100644 index 0000000..f9af1dd --- /dev/null +++ b/recipe/src/stage.rs @@ -0,0 +1,159 @@ +use std::{borrow::Cow, path::PathBuf}; + +use anyhow::{bail, Result}; +use blue_build_utils::syntax_highlighting::highlight_ser; +use colored::Colorize; +use serde::{Deserialize, Serialize}; +use typed_builder::TypedBuilder; + +use crate::{Module, ModuleExt, StagesExt}; + +/// Contains the required fields for a stage. +#[derive(Serialize, Deserialize, Debug, Clone, TypedBuilder)] +pub struct StageRequiredFields<'a> { + /// The name of the stage. + /// + /// This can then be referenced in the `copy` + /// module using the `from:` property. + #[builder(setter(into))] + pub name: Cow<'a, str>, + + /// The base image of the stage. + /// + /// This is set directly in a `FROM` instruction. + #[builder(setter(into))] + pub from: Cow<'a, str>, + + /// The shell to use in the stage. + #[builder(default, setter(into, strip_option))] + #[serde(skip_serializing_if = "Option::is_none")] + pub shell: Option>>, + + /// The modules extension for the stage + #[serde(flatten)] + pub modules_ext: ModuleExt<'a>, +} + +/// Corresponds to a stage in a Containerfile +/// +/// A stage has its own list of modules to run which +/// allows the user to reuse the modules thats provided to the main build. +#[derive(Serialize, Deserialize, Debug, Clone, TypedBuilder)] +pub struct Stage<'a> { + /// The requied fields for a stage. + #[builder(default, setter(strip_option))] + #[serde(flatten, skip_serializing_if = "Option::is_none")] + pub required_fields: Option>, + + /// A reference to another recipe file containing + /// one or more stages. + /// + /// An imported recipe file can contain just a single stage like: + /// + /// ```yaml + /// name: blue-build + /// image: rust + /// modules: + /// - type: containerfile + /// snippets: + /// - | + /// RUN cargo install blue-build --debug --all-features --target x86_64-unknown-linux-gnu \ + /// && mkdir -p /out/ \ + /// && mv $CARGO_HOME/bin/bluebuild /out/bluebuild + /// ``` + /// + /// Or it can contain multiple stages: + /// + /// ```yaml + /// stages: + /// - name: blue-build + /// image: rust + /// modules: + /// - type: containerfile + /// snippets: + /// - | + /// RUN cargo install blue-build --debug --all-features --target x86_64-unknown-linux-gnu \ + /// && mkdir -p /out/ \ + /// && mv $CARGO_HOME/bin/bluebuild /out/bluebuild + /// - name: hello-world + /// image: alpine + /// modules: + /// - type: script + /// snippets: + /// - echo "Hello World!" + /// ``` + #[builder(default, setter(into, strip_option))] + #[serde(rename = "from-file", skip_serializing_if = "Option::is_none")] + pub from_file: Option>, +} + +impl<'a> Stage<'a> { + /// Get's any child stages. + /// + /// # Errors + /// Will error if the stage cannot be + /// deserialized or the user uses another + /// property alongside `from-file:`. + pub fn get_stages(stages: &[Self], traversed_files: Option>) -> Result> { + let mut found_stages = vec![]; + let traversed_files = traversed_files.unwrap_or_default(); + + for stage in stages { + found_stages.extend( + match stage { + Stage { + required_fields: Some(_), + from_file: None, + } => vec![stage.clone()], + Stage { + required_fields: None, + from_file: Some(file_name), + } => { + let file_name = PathBuf::from(file_name.as_ref()); + if traversed_files.contains(&file_name) { + bail!( + "{} File {} has already been parsed:\n{traversed_files:?}", + "Circular dependency detected!".bright_red(), + file_name.display().to_string().bold(), + ); + } + let mut tf = traversed_files.clone(); + tf.push(file_name.clone()); + + Self::get_stages(&StagesExt::parse(&file_name)?.stages, Some(tf))? + } + _ => { + let from_example = Stage::builder().from_file("path/to/stage.yml").build(); + let stage_example = Self::example(); + + bail!( + "Improper format for stage. Must be in the format like:\n{}\n{}\n\n{}", + highlight_ser(&stage_example, "yaml", None)?, + "or".bold(), + highlight_ser(&from_example, "yaml", None)? + ); + } + } + .into_iter(), + ); + } + Ok(found_stages) + } + + #[must_use] + pub fn example() -> Self { + Stage::builder() + .required_fields( + StageRequiredFields::builder() + .name("stage-name") + .from("build/image:here") + .modules_ext( + ModuleExt::builder() + .modules(vec![Module::example()]) + .build(), + ) + .build(), + ) + .build() + } +} diff --git a/recipe/src/stages_ext.rs b/recipe/src/stages_ext.rs new file mode 100644 index 0000000..c02d4c9 --- /dev/null +++ b/recipe/src/stages_ext.rs @@ -0,0 +1,61 @@ +use std::{borrow::Cow, fs, path::Path}; + +use anyhow::{Context, Result}; +use blue_build_utils::constants::{CONFIG_PATH, RECIPE_PATH}; +use log::warn; +use serde::{Deserialize, Serialize}; +use typed_builder::TypedBuilder; + +use crate::{Module, Stage}; + +#[derive(Default, Serialize, Clone, Deserialize, Debug, TypedBuilder)] +pub struct StagesExt<'a> { + #[builder(default, setter(into))] + pub stages: Cow<'a, [Stage<'a>]>, +} + +impl<'a> StagesExt<'a> { + /// Parse a module file returning a [`StagesExt`] + /// + /// # Errors + /// Can return an `anyhow` Error if the file cannot be read or deserialized + /// into a [`StagesExt`] + pub fn parse(file_name: &Path) -> Result { + let legacy_path = Path::new(CONFIG_PATH); + let recipe_path = Path::new(RECIPE_PATH); + + let file_path = if recipe_path.exists() && recipe_path.is_dir() { + recipe_path.join(file_name) + } else { + warn!("Use of {CONFIG_PATH} for recipes is deprecated, please move your recipe files into {RECIPE_PATH}"); + legacy_path.join(file_name) + }; + + let file = fs::read_to_string(&file_path) + .context(format!("Failed to open {}", file_path.display()))?; + + serde_yaml::from_str::(&file).map_or_else( + |_| -> Result { + let mut stage = serde_yaml::from_str::(&file) + .map_err(blue_build_utils::serde_yaml_err(&file))?; + if let Some(ref mut rf) = stage.required_fields { + rf.modules_ext.modules = + Module::get_modules(&rf.modules_ext.modules, None)?.into(); + } + Ok(Self::builder().stages(vec![stage]).build()) + }, + |mut stages_ext| -> Result { + let mut stages: Vec = + stages_ext.stages.iter().map(ToOwned::to_owned).collect(); + for stage in &mut stages { + if let Some(ref mut rf) = stage.required_fields { + rf.modules_ext.modules = + Module::get_modules(&rf.modules_ext.modules, None)?.into(); + } + } + stages_ext.stages = stages.into(); + Ok(stages_ext) + }, + ) + } +} diff --git a/scripts/run_module.sh b/scripts/run_module.sh index 2330723..83f58b9 100644 --- a/scripts/run_module.sh +++ b/scripts/run_module.sh @@ -2,6 +2,8 @@ set -euo pipefail +source /tmp/scripts/exports.sh + # Function to print a centered text banner within a specified width print_banner() { local term_width=120 @@ -25,4 +27,3 @@ print_banner "Start '${module}' Module" chmod +x ${script_path} ${script_path} "${params}" print_banner "End '${module}' Module" -ostree container commit diff --git a/scripts/setup.sh b/scripts/setup.sh new file mode 100644 index 0000000..fcb7509 --- /dev/null +++ b/scripts/setup.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +if [ -f /etc/os-release ]; then + export ID="$(cat /etc/os-release | grep -E '^ID=' | awk -F '=' '{print $2}')" + + if [ "$ID" = "alpine" ]; then + echo "Setting up Alpine based image to run BlueBuild modules" + apk update + apk add --no-cache bash curl coreutils wget grep + elif [ "$ID" = "ubuntu" ] || [ "$ID" = "debian" ]; then + echo "Setting up Ubuntu based image to run BlueBuild modules" + apt-get update + apt-get install -y bash curl coreutils wget + elif [ "$ID" = "fedora" ]; then + echo "Settig up Fedora based image to run BlueBuild modules" + dnf install -y --refresh bash curl wget coreutils + else + echo "OS not detected, proceeding without setup" + fi +fi +cp /tmp/bins/yq /usr/bin/ diff --git a/src/drivers/opts/build.rs b/src/drivers/opts/build.rs index a8e3772..94de13c 100644 --- a/src/drivers/opts/build.rs +++ b/src/drivers/opts/build.rs @@ -37,7 +37,7 @@ pub struct PushOpts<'a> { pub struct BuildTagPushOpts<'a> { /// The base image name. /// - /// NOTE: You cannot have this set with `archive-path` set. + /// NOTE: You cannot have this set with `archive_path` set. #[builder(default, setter(into, strip_option))] pub image: Option>, diff --git a/template/src/lib.rs b/template/src/lib.rs index 9c3badf..29f4c0e 100644 --- a/template/src/lib.rs +++ b/template/src/lib.rs @@ -13,7 +13,7 @@ use uuid::Uuid; pub use askama::Template; #[derive(Debug, Clone, Template, TypedBuilder)] -#[template(path = "Containerfile.j2", escape = "none")] +#[template(path = "Containerfile.j2", escape = "none", whitespace = "minimize")] pub struct ContainerFileTemplate<'a> { recipe: &'a Recipe<'a>, diff --git a/template/templates/Containerfile.j2 b/template/templates/Containerfile.j2 index 23934b5..10b7210 100644 --- a/template/templates/Containerfile.j2 +++ b/template/templates/Containerfile.j2 @@ -1,7 +1,9 @@ +{%- import "modules/modules.j2" as modules -%} {%- set files_dir_exists = self::files_dir_exists() %} {%- include "stages.j2" %} -FROM {{ recipe.base_image }}:{{ recipe.image_version }} +# Main image +FROM {{ recipe.base_image }}:{{ recipe.image_version }} as {{ recipe.name|replace('/', "-") }} ARG RECIPE={{ recipe_path.display() }} ARG IMAGE_REGISTRY={{ registry }} @@ -15,7 +17,19 @@ ARG MODULE_DIRECTORY="/tmp/modules" ARG IMAGE_NAME="{{ recipe.name }}" ARG BASE_IMAGE="{{ recipe.base_image }}" -{% include "modules/modules.j2" %} +# Key RUN +RUN --mount=type=bind,from=stage-keys,src=/keys,dst=/tmp/keys \ + mkdir -p /usr/etc/pki/containers/ \ + && cp /tmp/keys/* /usr/etc/pki/containers/ \ + && ostree container commit + +# Bin RUN +RUN --mount=type=bind,from=stage-bins,src=/bins,dst=/tmp/bins \ + mkdir -p /usr/bin/ \ + && cp /tmp/bins/* /usr/bin/ \ + && ostree container commit + +{% call modules::main_modules_run(recipe.modules_ext, os_version) %} RUN rm -fr /tmp/* /var/* && ostree container commit diff --git a/template/templates/modules/akmods/akmods.j2 b/template/templates/modules/akmods/akmods.j2 index 3066d7b..4e19b61 100644 --- a/template/templates/modules/akmods/akmods.j2 +++ b/template/templates/modules/akmods/akmods.j2 @@ -1,4 +1,5 @@ {%- for info in recipe.modules_ext.get_akmods_info_list(os_version) %} +# Stage for AKmod {{ info.stage_name }} FROM scratch as stage-akmods-{{ info.stage_name }} COPY --from=ghcr.io/ublue-os/{{ info.images.0 }} /rpms /rpms COPY --from=ghcr.io/ublue-os/{{ info.images.1 }} /rpms /rpms diff --git a/template/templates/modules/containerfile/README.md b/template/templates/modules/containerfile/README.md index bcf51e3..cb96751 100644 --- a/template/templates/modules/containerfile/README.md +++ b/template/templates/modules/containerfile/README.md @@ -4,7 +4,7 @@ Only compiler-based builds can use this module as it is built-in to the BlueBuild CLI tool. ::: -The `containerfile` module is a tool for adding custom [`Containerfile`](https://github.com/containers/common/blob/main/docs/Containerfile.5.md) instructions for custom image builds. This is useful when you wish to use some feature directly available in a `Containerfile`, but not in a bash module, such as copying from other OCI images with `COPY --from`. +The `containerfile` module is a tool for adding custom [`Containerfile`](https://github.com/containers/common/blob/main/docs/Containerfile.5.md) instructions for custom image builds. This is useful when you wish to use some feature directly available in a `Containerfile`, but not in a bash module, such as using a `RUN` instruction with custom mounts. Since standard compiler-based BlueBuild image builds generate a `Containerfile` from your recipe, there is no need to manage it yourself. However, we know that we also have technical users that would like to have the ability to customize their `Containerfile`. This is where the `containerfile` module comes into play. @@ -18,10 +18,10 @@ The `snippets` property is the easiest to use when you just need to insert a few modules: - type: containerfile snippets: - - COPY --from=docker.io/mikefarah/yq /usr/bin/yq /usr/bin/yq + - RUN --mount=type=tmpfs,target=/tmp /some/script.sh ``` -This makes it really easy to copy a file or program from another image. +This makes it really easy to add individual, custom instructions. :::note **NOTE:** Each entry of a snippet will be its own layer in the final `Containerfile`. @@ -66,7 +66,7 @@ If you wanted to have some `snippets` run before any `containerfiles` have, you modules: - type: containerfile snippets: - - COPY --from=docker.io/mikefarah/yq /usr/bin/yq /usr/bin/yq + - RUN --mount=type=tmpfs,target=/tmp /some/script.sh - type: containerfile containerfiles: - example diff --git a/template/templates/modules/containerfile/module.yml b/template/templates/modules/containerfile/module.yml index 32c267d..67e3faa 100644 --- a/template/templates/modules/containerfile/module.yml +++ b/template/templates/modules/containerfile/module.yml @@ -4,7 +4,7 @@ readme: https://raw.githubusercontent.com/blue-build/cli/main/template/templates example: | type: containerfile snippets: - - COPY ./config/example-dir/example-file.txt /usr/etc/example/ + - RUN --mount=type=tmpfs,target=/tmp /some/script.sh containerfiles: - example - subroutine diff --git a/template/templates/modules/copy/README.md b/template/templates/modules/copy/README.md new file mode 100644 index 0000000..6c525dc --- /dev/null +++ b/template/templates/modules/copy/README.md @@ -0,0 +1,44 @@ +# `copy` + +:::caution +Only compiler-based builds can use this module as it is built-in to the BlueBuild CLI tool. +::: + +:::note +**NOTE:** This module is currently only available with the `use_unstable_cli` option on the GHA or using the `main` image. +::: + +The `copy` module is a short-hand method of adding a [`COPY`](https://docs.docker.com/reference/dockerfile/#copy) instruction into the image. This can be used to copy files from images, other stages, or even from the build context. + +## Usage + +The `copy` module's properties are a 1-1 match with the `COPY` instruction containing `src`, `dest`, and `from` (optional). The example below will `COPY` the file `/usr/bin/yq` from `docker.io/mikefarah/yq` into `/usr/bin/`. + +```yaml +mdoules: +- type: copy + from: docker.io/mikefarah/yq + src: /usr/bin/yq + dest: /usr/bin/ +``` + +Creating an instruction like: + +```dockerfile +COPY --linked --from=docker.io/mikefarah/yq /usr/bin/yq /usr/bin/ +``` + +Omitting `from:` will allow copying from the build context: + +```yaml +mdoules: +- type: copy + src: file/to/copy.conf + dest: /usr/etc/app/ +``` + +Creating an instruction like: + +```dockerfile +COPY --linked file/to/copy.conf /usr/etc/app/ +``` diff --git a/template/templates/modules/copy/copy.j2 b/template/templates/modules/copy/copy.j2 new file mode 100644 index 0000000..e82130d --- /dev/null +++ b/template/templates/modules/copy/copy.j2 @@ -0,0 +1,4 @@ +{%- if let Some((from_img, src, dest)) = module.get_copy_args() %} +COPY{% if let Some(from_img) = from_img %} --from={{ from_img }}{% endif %} {{ src }} {{ dest }} +{%- endif %} + diff --git a/template/templates/modules/copy/module.yml b/template/templates/modules/copy/module.yml new file mode 100644 index 0000000..2179b6f --- /dev/null +++ b/template/templates/modules/copy/module.yml @@ -0,0 +1,8 @@ +name: copy +shortdesc: The copy module is a direct translation of the `COPY` instruction in a Containerfile. +readme: https://raw.githubusercontent.com/blue-build/cli/main/template/templates/modules/copy/README.md +example: | + type: copy + from: docker.io/mikefarah/yq + src: /usr/bin/yq + dest: /usr/bin/ diff --git a/template/templates/modules/modules.j2 b/template/templates/modules/modules.j2 index 2699b4a..5663a3d 100644 --- a/template/templates/modules/modules.j2 +++ b/template/templates/modules/modules.j2 @@ -1,41 +1,66 @@ -# Key RUN -RUN --mount=type=bind,from=stage-keys,src=/keys,dst=/tmp/keys \ - mkdir -p /usr/etc/pki/containers/ \ - && cp /tmp/keys/* /usr/etc/pki/containers/ \ - && ostree container commit - -# Bin RUN -RUN --mount=type=bind,from=stage-bins,src=/bins,dst=/tmp/bins \ - mkdir -p /usr/bin/ \ - && cp /tmp/bins/* /usr/bin/ \ - && ostree container commit - +{% macro main_modules_run(modules_ext, os_version) %} # Module RUNs -{%- for module in recipe.modules_ext.modules %} - {%- if let Some(type) = module.module_type %} - {%- if type == "containerfile" %} - {%- include "modules/containerfile/containerfile.j2" %} - {%- else %} + {%- for module in modules_ext.modules %} + {%- if let Some(module) = module.required_fields %} + {%- if module.no_cache %} +ARG CACHEBUST="{{ build_id }}" + {%- endif %} + + {%- if module.module_type == "containerfile" %} + {%- include "modules/containerfile/containerfile.j2" %} + {%- else if module.module_type == "copy" %} + {%- include "modules/copy/copy.j2" %} + {%- else %} RUN \ - {%- if files_dir_exists %} + {%- if files_dir_exists %} --mount=type=bind,from=stage-files,src=/files,dst=/tmp/files,rw \ - {%- else %} + {%- else %} --mount=type=bind,from=stage-config,src=/config,dst=/tmp/config,rw \ - {%- endif %} - {%- if let Some(source) = module.source %} + {%- endif %} + {%- if let Some(source) = module.source %} --mount=type=bind,from={{ source }},src=/modules,dst=/tmp/modules,rw \ - {%- else %} + {%- else %} --mount=type=bind,from=stage-modules,src=/modules,dst=/tmp/modules,rw \ - {%- endif %} - {%- if type == "akmods" %} + {%- endif %} + {%- if module.module_type == "akmods" %} --mount=type=bind,from=stage-akmods-{{ module.generate_akmods_info(os_version).stage_name }},src=/rpms,dst=/tmp/rpms,rw \ - {%- endif %} + {%- endif %} --mount=type=bind,from=ghcr.io/blue-build/cli:{{ exports_tag }}-build-scripts,src=/scripts/,dst=/tmp/scripts/ \ --mount=type=cache,dst=/var/cache/rpm-ostree,id=rpm-ostree-cache-{{ recipe.name }}-{{ recipe.image_version }},sharing=locked \ - source /tmp/scripts/exports.sh \ - && /tmp/scripts/run_module.sh {{ type }} '{{ module.print_module_context() }}' + /tmp/scripts/run_module.sh '{{ module.module_type }}' '{{ module.print_module_context() }}' \ + && ostree container commit + {%- endif %} {%- endif %} - {%- endif %} -{%- endfor %} + {%- endfor %} +{% endmacro %} +{% macro stage_modules_run(modules_ext, os_version) %} +# Module RUNs + {%- for module in modules_ext.modules %} + {%- if let Some(module) = module.required_fields %} + {%- if module.no_cache %} +ARG CACHEBUST="{{ build_id }}" + {%- endif %} + {%- if module.module_type == "containerfile" %} + {%- include "modules/containerfile/containerfile.j2" %} + {%- else if module.module_type == "copy" %} + {%- include "modules/copy/copy.j2" %} + {%- else %} +RUN \ + {%- if files_dir_exists %} + --mount=type=bind,from=stage-files,src=/files,dst=/tmp/files,rw \ + {%- else %} + --mount=type=bind,from=stage-config,src=/config,dst=/tmp/config,rw \ + {%- endif %} + {%- if let Some(source) = module.source %} + --mount=type=bind,from={{ source }},src=/modules,dst=/tmp/modules,rw \ + {%- else %} + --mount=type=bind,from=stage-modules,src=/modules,dst=/tmp/modules,rw \ + {%- endif %} + --mount=type=bind,from=ghcr.io/blue-build/cli:{{ exports_tag }}-build-scripts,src=/scripts/,dst=/tmp/scripts/ \ + /tmp/scripts/run_module.sh '{{ module.module_type }}' '{{ module.print_module_context() }}' + {%- endif %} + {%- endif %} + {%- endfor %} +{% endmacro %} diff --git a/template/templates/stages.j2 b/template/templates/stages.j2 index 6c07f1d..24e87f3 100644 --- a/template/templates/stages.j2 +++ b/template/templates/stages.j2 @@ -2,29 +2,28 @@ # your config without copying it directly into # the final image {%- if files_dir_exists %} -FROM scratch as stage-files +FROM scratch AS stage-files COPY ./files /files -{%- else %} -FROM scratch as stage-config +{% else %} +FROM scratch AS stage-config COPY ./config /config -{%- endif %} +{% endif %} # Copy modules # The default modules are inside blue-build/modules # Custom modules overwrite defaults -FROM scratch as stage-modules +FROM scratch AS stage-modules COPY --from=ghcr.io/blue-build/modules:latest /modules /modules {%- if self::modules_exists() %} COPY ./modules /modules -{%- endif %} +{% endif %} # Bins to install # These are basic tools that are added to all images. # Generally used for the build process. We use a multi # stage process so that adding the bins into the image # can be added to the ostree commits. -FROM scratch as stage-bins - +FROM scratch AS stage-bins COPY --from=gcr.io/projectsigstore/cosign /ko-app/cosign /bins/cosign COPY --from=docker.io/mikefarah/yq /usr/bin/yq /bins/yq COPY --from=ghcr.io/blue-build/cli: @@ -40,10 +39,41 @@ latest-installer # # Currently only holds the current image's # public key. -FROM scratch as stage-keys - +FROM scratch AS stage-keys {%- if self::has_cosign_file() %} COPY cosign.pub /keys/{{ recipe.name|replace('/', "_") }}.pub -{%- endif %} +{% endif %} {%- include "modules/akmods/akmods.j2" %} + +{%- set files_dir_exists = self::files_dir_exists() %} +{%~ if let Some(stages_ext) = recipe.stages_ext %} + {%- for stage in stages_ext.stages %} + {%- if let Some(stage) = stage.required_fields %} +# {{ stage.name|capitalize }} stage +FROM {{ stage.from }} AS {{ stage.name }} + + {%- if stage.from != "scratch" %} +# Add compatibility for modules +RUN --mount=type=bind,from=stage-bins,src=/bins/,dst=/tmp/bins/ \ + --mount=type=bind,from=ghcr.io/blue-build/cli:{{ exports_tag }}-build-scripts,src=/scripts/,dst=/tmp/scripts/ \ + /tmp/scripts/setup.sh + + {%- if files_dir_exists %} +ARG CONFIG_DIRECTORY="/tmp/files" + {%- else %} +ARG CONFIG_DIRECTORY="/tmp/config" + {%- endif %} +ARG MODULE_DIRECTORY="/tmp/modules" + + {%- if let Some(shell_args) = stage.shell %} +SHELL [{% for arg in shell_args %}"{{ arg }}"{% if !loop.last %}, {% endif %}{% endfor %}] + {%- else %} +SHELL ["bash", "-c"] + {%- endif %} + {%- endif %} + + {% call modules::stage_modules_run(stage.modules_ext, os_version) %} + {%- endif %} + {%- endfor %} +{%- endif %} diff --git a/utils/src/syntax_highlighting.rs b/utils/src/syntax_highlighting.rs index f3f3c0e..0bacc43 100644 --- a/utils/src/syntax_highlighting.rs +++ b/utils/src/syntax_highlighting.rs @@ -35,9 +35,8 @@ impl std::fmt::Display for DefaultThemes { /// # Errors /// Will error if the theme doesn't exist, the syntax doesn't exist, or the file /// failed to serialize. -pub fn print(file: &str, file_type: &str, theme: Option) -> Result<()> { - trace!("syntax_highlighting::print({file}, {file_type}, {theme:?})"); - +pub fn highlight(file: &str, file_type: &str, theme: Option) -> Result { + trace!("syntax_highlighting::highlight(file, {file_type}, {theme:?})"); if atty::is(atty::Stream::Stdout) { let ss: SyntaxSet = if file_type == "dockerfile" || file_type == "Dockerfile" { dumps::from_uncompressed_data(include_bytes!(concat!( @@ -58,15 +57,43 @@ pub fn print(file: &str, file_type: &str, theme: Option) -> Resul .get(theme.unwrap_or_default().to_string().as_str()) .ok_or_else(|| anyhow!("Failed to get highlight theme"))?, ); + + let mut highlighted_lines: Vec = vec![]; for line in file.lines() { - let ranges = h.highlight_line(line, &ss)?; - let escaped = syntect::util::as_24_bit_terminal_escaped(&ranges, false); - println!("{escaped}"); + highlighted_lines.push(syntect::util::as_24_bit_terminal_escaped( + &h.highlight_line(line, &ss)?, + false, + )); } - println!("\x1b[0m"); + highlighted_lines.push("\x1b[0m".to_string()); + Ok(highlighted_lines.join("\n")) } else { - println!("{file}"); + Ok(file.to_string()) } +} + +/// Takes a serializable struct and serializes it with syntax highlighting. +/// +/// # Errors +/// Will error if the theme doesn't exist, the syntax doesn't exist, or the file +/// failed to serialize. +pub fn highlight_ser( + file: &T, + file_type: &str, + theme: Option, +) -> Result { + trace!("syntax_highlighting::highlight_ser(file, {file_type}, {theme:?})"); + highlight(serde_yaml::to_string(file)?.as_str(), file_type, theme) +} + +/// Prints the file with syntax highlighting. +/// +/// # Errors +/// Will error if the theme doesn't exist, the syntax doesn't exist, or the file +/// failed to serialize. +pub fn print(file: &str, file_type: &str, theme: Option) -> Result<()> { + trace!("syntax_highlighting::print(file, {file_type}, {theme:?})"); + println!("{}", highlight(file, file_type, theme)?); Ok(()) } @@ -75,11 +102,12 @@ pub fn print(file: &str, file_type: &str, theme: Option) -> Resul /// # Errors /// Will error if the theme doesn't exist, the syntax doesn't exist, or the file /// failed to serialize. -pub fn print_ser( +pub fn print_ser( file: &T, file_type: &str, theme: Option, ) -> Result<()> { + trace!("syntax_highlighting::print_ser(file, {file_type}, {theme:?})"); print(serde_yaml::to_string(file)?.as_str(), file_type, theme)?; Ok(()) }