diff --git a/Cargo.lock b/Cargo.lock index fcc20ab..b737d51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -570,7 +570,10 @@ dependencies = [ "serde_json", "serde_yml", "syntect", + "tempfile", + "uuid", "which 8.0.0", + "zeroize", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index e7d7002..490022e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ tempfile = "3" tokio = { version = "1", features = ["rt", "rt-multi-thread"] } users = "0.11" uuid = { version = "1", features = ["v4"] } +zeroize = { version = "1", features = ["aarch64", "derive", "std", "serde"] } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/integration-tests/Earthfile b/integration-tests/Earthfile index 96d3c3d..c0b33eb 100644 --- a/integration-tests/Earthfile +++ b/integration-tests/Earthfile @@ -4,31 +4,11 @@ PROJECT blue-build/cli IMPORT github.com/earthly/lib/utils/dind AS dind all: - BUILD +test-image - # BUILD +test-legacy-image BUILD +build BUILD +switch BUILD +validate - -test-image: - FROM +build-template --src=template-containerfile - WORKDIR /tmp/test - COPY ./test-scripts/*.sh ./ - - DO +RUN_TESTS - -test-legacy-image: - FROM +build-template --src=template-legacy-containerfile - WORKDIR /tmp/test - COPY ./test-scripts/*.sh ./ - - DO +RUN_TESTS - -build-template: - ARG --required src - FROM DOCKERFILE \ - -f +$src/test/Containerfile \ - +$src/test/* + BUILD +template-containerfile + BUILD +template-legacy-containerfile template-containerfile: FROM +test-base diff --git a/integration-tests/test-repo/.gitignore b/integration-tests/test-repo/.gitignore index 4c749b6..90aaf15 100644 --- a/integration-tests/test-repo/.gitignore +++ b/integration-tests/test-repo/.gitignore @@ -1,2 +1,3 @@ /Containerfile /Containerfile.* +/secrets diff --git a/integration-tests/test-repo/recipes/common.yml b/integration-tests/test-repo/recipes/common.yml index 7929992..d851bcf 100644 --- a/integration-tests/test-repo/recipes/common.yml +++ b/integration-tests/test-repo/recipes/common.yml @@ -77,3 +77,41 @@ modules: from: fedora-test src: /test.txt dest: / + + # Testing secrets + - type: script + secrets: + - type: env + name: TEST_SECRET + snippets: + - '[ "$TEST_SECRET" == "test123" ]' + - type: script + secrets: + - type: file + source: ./secrets/test-secret + destination: /tmp/test-secret + snippets: + - '[ "$(cat /tmp/test-secret)" == "321tset" ]' + - type: script + secrets: + - type: exec + command: cat + args: + - ./test_secret_file.txt + output: + type: env + name: TEST_SECRET + snippets: + - '[ "$TEST_SECRET" == "TEST_PASS" ]' + - type: script + secrets: + - type: exec + command: cat + args: + - ./test_secret_file.txt + output: + type: file + destination: /tmp/test-secret + snippets: + - '[ "$(cat /tmp/test-secret)" == "TEST_PASS" ]' + diff --git a/integration-tests/test-repo/test_secret_file.txt b/integration-tests/test-repo/test_secret_file.txt new file mode 100644 index 0000000..15985b0 --- /dev/null +++ b/integration-tests/test-repo/test_secret_file.txt @@ -0,0 +1 @@ +TEST_PASS diff --git a/justfile b/justfile index 9e729e1..3b78766 100644 --- a/justfile +++ b/justfile @@ -1,5 +1,7 @@ export RUST_BACKTRACE := "1" export BB_CACHE_LAYERS := "true" +export TEST_SECRET := "test123" +# export BB_SKIP_VALIDATION := "true" set dotenv-load := true set positional-arguments := true @@ -133,11 +135,15 @@ cargo_bin := if env('CARGO_HOME', '') != '' { x"$HOME/.cargo/bin" } +generate-test-secret: + mkdir -p integration-tests/test-repo/secrets + echo "321tset" > integration-tests/test-repo/secrets/test-secret + # Run all integration tests -integration-tests: test-docker-build test-empty-files-build test-arm64-build test-podman-build test-buildah-build test-generate-iso-image test-generate-iso-recipe +integration-tests: generate-test-secret test-docker-build test-empty-files-build test-arm64-build test-podman-build test-buildah-build test-generate-iso-image test-generate-iso-recipe # Run docker driver integration test -test-docker-build: install-debug-all-features +test-docker-build: generate-test-secret install-debug-all-features cd integration-tests/test-repo \ && bluebuild build \ --retry-push \ @@ -148,7 +154,7 @@ test-docker-build: install-debug-all-features -vv \ recipes/recipe.yml recipes/recipe-gts.yml -test-empty-files-build: install-debug-all-features +test-empty-files-build: generate-test-secret install-debug-all-features cd integration-tests/empty-files-repo \ && bluebuild build \ --retry-push \ @@ -158,7 +164,7 @@ test-empty-files-build: install-debug-all-features {{ should_push }} \ -vv -test-rechunk-build: install-debug-all-features +test-rechunk-build: generate-test-secret install-debug-all-features cd integration-tests/test-repo \ && bluebuild build \ {{ should_push }} \ @@ -166,7 +172,7 @@ test-rechunk-build: install-debug-all-features --rechunk \ recipes/recipe-rechunk.yml -test-fresh-rechunk-build: install-debug-all-features +test-fresh-rechunk-build: generate-test-secret install-debug-all-features cd integration-tests/test-repo \ && bluebuild build \ {{ should_push }} \ @@ -176,7 +182,7 @@ test-fresh-rechunk-build: install-debug-all-features recipes/recipe-rechunk.yml # Run arm integration test -test-arm64-build: install-debug-all-features +test-arm64-build: generate-test-secret install-debug-all-features cd integration-tests/test-repo \ && bluebuild build \ --retry-push \ @@ -186,7 +192,7 @@ test-arm64-build: install-debug-all-features recipes/recipe-arm64.yml # Run docker driver external login integration test -test-docker-build-external-login: install-debug-all-features +test-docker-build-external-login: generate-test-secret install-debug-all-features cd integration-tests/test-repo \ && bluebuild build \ --retry-push \ @@ -196,7 +202,7 @@ test-docker-build-external-login: install-debug-all-features recipes/recipe-docker-external.yml # Run podman driver integration test -test-podman-build: install-debug-all-features +test-podman-build: generate-test-secret install-debug-all-features cd integration-tests/test-repo \ && bluebuild build \ --retry-push \ @@ -208,7 +214,7 @@ test-podman-build: install-debug-all-features recipes/recipe-podman.yml # Run buildah driver integration test -test-buildah-build: install-debug-all-features +test-buildah-build: generate-test-secret install-debug-all-features cd integration-tests/test-repo \ && bluebuild build \ --retry-push \ @@ -220,14 +226,14 @@ test-buildah-build: install-debug-all-features recipes/recipe-buildah.yml # Run ISO generator for images -test-generate-iso-image: install-debug-all-features +test-generate-iso-image: generate-test-secret install-debug-all-features #!/usr/bin/env bash set -eu ISO_OUT=$(mktemp -d) bluebuild generate-iso -vv --output-dir "$ISO_OUT" image ghcr.io/blue-build/cli/test:40 # Run ISO generator for images -test-generate-iso-recipe: install-debug-all-features +test-generate-iso-recipe: generate-test-secret install-debug-all-features #!/usr/bin/env bash set -eu ISO_OUT=$(mktemp -d) diff --git a/process/Cargo.toml b/process/Cargo.toml index 7368fb5..d768bf4 100644 --- a/process/Cargo.toml +++ b/process/Cargo.toml @@ -20,7 +20,6 @@ os_pipe = { version = "1", features = ["io_safety"] } rand = "0.9" signal-hook = { version = "0.3", features = ["extended-siginfo"] } sigstore = { version = "0.11", features = ["full-rustls-tls", "cached-client", "sigstore-trust-root", "sign"], default-features = false } -zeroize = { version = "1", features = ["aarch64", "derive", "serde"] } cached.workspace = true chrono.workspace = true @@ -42,6 +41,7 @@ tokio.workspace = true bon.workspace = true users.workspace = true uuid.workspace = true +zeroize.workspace = true [dev-dependencies] rstest.workspace = true diff --git a/process/drivers.rs b/process/drivers.rs index 389baf5..b825320 100644 --- a/process/drivers.rs +++ b/process/drivers.rs @@ -14,7 +14,7 @@ use std::{ time::Duration, }; -use blue_build_utils::semver::Version; +use blue_build_utils::{BUILD_ID, semver::Version}; use bon::{Builder, bon}; use cached::proc_macro::cached; use clap::Args; @@ -69,9 +69,6 @@ static SELECTED_SIGNING_DRIVER: std::sync::LazyLock>> = std::sync::LazyLock::new(|| RwLock::new(None)); -/// UUID used to mark the current builds -static BUILD_ID: std::sync::LazyLock = std::sync::LazyLock::new(Uuid::new_v4); - /// Args for selecting the various drivers to use for runtime. /// /// If the args are left uninitialized, the program will determine diff --git a/process/drivers/buildah_driver.rs b/process/drivers/buildah_driver.rs index e90049a..c5b38bb 100644 --- a/process/drivers/buildah_driver.rs +++ b/process/drivers/buildah_driver.rs @@ -1,11 +1,12 @@ use std::{io::Write, process::Stdio}; -use blue_build_utils::{credentials::Credentials, semver::Version}; +use blue_build_utils::{credentials::Credentials, secret::SecretArgs, semver::Version}; use colored::Colorize; use comlexr::cmd; use log::{debug, error, info, trace}; -use miette::{IntoDiagnostic, Result, bail, miette}; +use miette::{Context, IntoDiagnostic, Result, bail, miette}; use serde::Deserialize; +use tempfile::TempDir; use crate::{drivers::types::Platform, logging::CommandLogging}; @@ -50,9 +51,14 @@ impl BuildDriver for BuildahDriver { fn build(opts: &BuildOpts) -> Result<()> { trace!("BuildahDriver::build({opts:#?})"); + let temp_dir = TempDir::new() + .into_diagnostic() + .wrap_err("Failed to create temporary directory for secrets")?; + let command = cmd!( "buildah", "build", + for opts.secrets.args(&temp_dir)?, if !matches!(opts.platform, Platform::Native) => [ "--platform", opts.platform.to_string(), diff --git a/process/drivers/docker_driver.rs b/process/drivers/docker_driver.rs index 6c10d4c..afa9b50 100644 --- a/process/drivers/docker_driver.rs +++ b/process/drivers/docker_driver.rs @@ -8,6 +8,7 @@ use blue_build_utils::{ constants::{BLUE_BUILD, DOCKER_HOST, GITHUB_ACTIONS}, credentials::Credentials, get_env_var, + secret::SecretArgs, semver::Version, string_vec, }; @@ -194,6 +195,10 @@ impl BuildDriver for DockerDriver { fn build(opts: &BuildOpts) -> Result<()> { trace!("DockerDriver::build({opts:#?})"); + let temp_dir = TempDir::new() + .into_diagnostic() + .wrap_err("Failed to create temporary directory for secrets")?; + if opts.squash { warn!("Squash is deprecated for docker so this build will not squash"); } @@ -209,6 +214,7 @@ impl BuildDriver for DockerDriver { "-t", opts.image.to_string(), "-f", + for opts.secrets.args(&temp_dir)?, if let Some(cache_from) = opts.cache_from.as_ref() => [ "--cache-from", format!( @@ -381,6 +387,10 @@ impl BuildDriver for DockerDriver { fn build_tag_push(opts: &BuildTagPushOpts) -> Result> { trace!("DockerDriver::build_tag_push({opts:#?})"); + let temp_dir = TempDir::new() + .into_diagnostic() + .wrap_err("Failed to create temporary directory for secrets")?; + if opts.squash { warn!("Squash is deprecated for docker so this build will not squash"); } @@ -391,7 +401,7 @@ impl BuildDriver for DockerDriver { let first_image = final_images.first().unwrap(); - let status = build_tag_push_cmd(opts, first_image) + let status = build_tag_push_cmd(opts, first_image, &temp_dir)? .build_status(first_image, "Building Image") .into_diagnostic()?; @@ -408,13 +418,18 @@ impl BuildDriver for DockerDriver { } } -fn build_tag_push_cmd(opts: &BuildTagPushOpts<'_>, first_image: &str) -> Command { +fn build_tag_push_cmd( + opts: &BuildTagPushOpts<'_>, + first_image: &str, + temp_dir: &TempDir, +) -> Result { let c = cmd!( "docker", "buildx", format!("--builder={BLUE_BUILD}"), "build", ".", + for opts.secrets.args(temp_dir)?, match &opts.image { ImageRef::Remote(_remote) if opts.push => [ "--output", @@ -459,7 +474,7 @@ fn build_tag_push_cmd(opts: &BuildTagPushOpts<'_>, first_image: &str) -> Command ], ); trace!("{c:?}"); - c + Ok(c) } fn get_final_images(opts: &BuildTagPushOpts<'_>) -> Vec { diff --git a/process/drivers/opts/build.rs b/process/drivers/opts/build.rs index c301409..4db19a5 100644 --- a/process/drivers/opts/build.rs +++ b/process/drivers/opts/build.rs @@ -1,5 +1,6 @@ -use std::{borrow::Cow, path::Path}; +use std::{borrow::Cow, collections::HashSet, path::Path}; +use blue_build_utils::secret::Secret; use bon::Builder; use oci_distribution::Reference; @@ -33,6 +34,9 @@ pub struct BuildOpts<'scope> { #[builder(into)] pub cache_to: Option<&'scope Reference>, + + #[builder(default)] + pub secrets: HashSet<&'scope Secret>, } #[derive(Debug, Clone, Builder)] @@ -110,4 +114,8 @@ pub struct BuildTagPushOpts<'scope> { /// Cache layers to the registry. pub cache_to: Option<&'scope Reference>, + + /// Secrets to mount + #[builder(default)] + pub secrets: HashSet<&'scope Secret>, } diff --git a/process/drivers/opts/rechunk.rs b/process/drivers/opts/rechunk.rs index 23095ad..dccd355 100644 --- a/process/drivers/opts/rechunk.rs +++ b/process/drivers/opts/rechunk.rs @@ -1,5 +1,6 @@ -use std::{borrow::Cow, path::Path}; +use std::{borrow::Cow, collections::HashSet, path::Path}; +use blue_build_utils::secret::Secret; use bon::Builder; use oci_distribution::Reference; @@ -55,6 +56,9 @@ pub struct RechunkOpts<'scope> { /// Cache layers to the registry. pub cache_to: Option<&'scope Reference>, + + #[builder(default)] + pub secrets: HashSet<&'scope Secret>, } #[derive(Debug, Clone, Builder)] diff --git a/process/drivers/podman_driver.rs b/process/drivers/podman_driver.rs index 8a5c172..a0a6915 100644 --- a/process/drivers/podman_driver.rs +++ b/process/drivers/podman_driver.rs @@ -7,14 +7,14 @@ use std::{ use blue_build_utils::{ constants::SUDO_ASKPASS, credentials::Credentials, has_env_var, running_as_root, - semver::Version, + secret::SecretArgs, semver::Version, }; use cached::proc_macro::cached; use colored::Colorize; use comlexr::{cmd, pipe}; use indicatif::{ProgressBar, ProgressStyle}; use log::{debug, error, info, trace}; -use miette::{IntoDiagnostic, Report, Result, bail, miette}; +use miette::{Context, IntoDiagnostic, Report, Result, bail, miette}; use oci_distribution::Reference; use serde::Deserialize; use tempfile::TempDir; @@ -137,6 +137,10 @@ impl BuildDriver for PodmanDriver { fn build(opts: &BuildOpts) -> Result<()> { trace!("PodmanDriver::build({opts:#?})"); + let temp_dir = TempDir::new() + .into_diagnostic() + .wrap_err("Failed to create temporary directory for secrets")?; + let use_sudo = opts.privileged && !running_as_root(); let command = cmd!( if use_sudo { @@ -149,7 +153,10 @@ impl BuildDriver for PodmanDriver { "-p", SUDO_PROMPT, ], - if use_sudo => "podman", + if use_sudo => [ + "--preserve-env", + "podman", + ], "build", if !matches!(opts.platform, Platform::Native) => [ "--platform", @@ -178,6 +185,7 @@ impl BuildDriver for PodmanDriver { &*opts.containerfile, "-t", opts.image.to_string(), + for opts.secrets.args(&temp_dir)?, ".", ); @@ -409,11 +417,11 @@ impl ContainerMountDriver for PodmanDriver { } else { "podman" }, - if use_sudo && has_env_var(SUDO_ASKPASS) => [ - "-A", - "-p", - SUDO_PROMPT, - ], + if use_sudo && has_env_var(SUDO_ASKPASS) => [ + "-A", + "-p", + SUDO_PROMPT, + ], if use_sudo => "podman", "mount", opts.container_id, diff --git a/process/drivers/traits.rs b/process/drivers/traits.rs index ccc816e..babd607 100644 --- a/process/drivers/traits.rs +++ b/process/drivers/traits.rs @@ -128,6 +128,7 @@ pub trait BuildDriver: PrivateDriver { .squash(opts.squash) .maybe_cache_from(opts.cache_from) .maybe_cache_to(opts.cache_to) + .secrets(opts.secrets.clone()) .build(); info!("Building image {}", opts.image); @@ -289,6 +290,7 @@ pub trait RechunkDriver: RunDriver + BuildDriver + ContainerMountDriver { .privileged(true) .squash(true) .host_network(true) + .secrets(opts.secrets.clone()) .build(), )?; diff --git a/recipe/src/module.rs b/recipe/src/module.rs index 1b9b89d..529b8f2 100644 --- a/recipe/src/module.rs +++ b/recipe/src/module.rs @@ -1,7 +1,7 @@ use std::{borrow::Cow, path::PathBuf}; use blue_build_utils::{ - constants::BLUE_BUILD_MODULE_IMAGE_REF, syntax_highlighting::highlight_ser, + constants::BLUE_BUILD_MODULE_IMAGE_REF, secret::Secret, syntax_highlighting::highlight_ser, }; use bon::Builder; use colored::Colorize; @@ -35,6 +35,10 @@ pub struct ModuleRequiredFields<'a> { #[serde(skip_serializing_if = "Option::is_none")] pub env: Option>, + #[builder(into, default)] + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub secrets: Vec, + #[serde(flatten)] #[builder(default, into)] pub config: IndexMap, diff --git a/recipe/src/recipe.rs b/recipe/src/recipe.rs index 66fad52..50c66d6 100644 --- a/recipe/src/recipe.rs +++ b/recipe/src/recipe.rs @@ -1,5 +1,6 @@ -use std::{borrow::Cow, fs, path::Path}; +use std::{borrow::Cow, collections::HashSet, fs, path::Path}; +use blue_build_utils::secret::Secret; use bon::Builder; use log::{debug, trace}; use miette::{Context, IntoDiagnostic, Result}; @@ -135,4 +136,24 @@ impl Recipe<'_> { Some(MaybeVersion::Version(version)) => version.to_string(), } } + + #[must_use] + pub fn get_secrets(&self) -> HashSet<&Secret> { + self.modules_ext + .modules + .iter() + .filter_map(|module| Some(&module.required_fields.as_ref()?.secrets)) + .flatten() + .chain( + self.stages_ext + .as_ref() + .map_or_else(Vec::new, |stage| stage.stages.iter().collect()) + .iter() + .filter_map(|stage| Some(&stage.required_fields.as_ref()?.modules_ext.modules)) + .flatten() + .filter_map(|module| Some(&module.required_fields.as_ref()?.secrets)) + .flatten(), + ) + .collect() + } } diff --git a/src/commands/build.rs b/src/commands/build.rs index 60df8b6..2df0ca4 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -14,8 +14,8 @@ use blue_build_process_management::{ use blue_build_recipe::Recipe; use blue_build_utils::{ constants::{ - ARCHIVE_SUFFIX, BB_REGISTRY_NAMESPACE, CONFIG_PATH, CONTAINER_FILE, RECIPE_FILE, - RECIPE_PATH, + ARCHIVE_SUFFIX, BB_REGISTRY_NAMESPACE, BB_SKIP_VALIDATION, CONFIG_PATH, CONTAINER_FILE, + RECIPE_FILE, RECIPE_PATH, }, cowstr, credentials::{Credentials, CredentialsArgs}, @@ -135,6 +135,11 @@ pub struct BuildCommand { #[arg(long, env = blue_build_utils::constants::BB_CACHE_LAYERS)] cache_layers: bool, + /// Skips validation of the recipe file. + #[arg(long, env = BB_SKIP_VALIDATION)] + #[builder(default)] + skip_validation: bool, + #[clap(flatten)] #[builder(default)] credentials: CredentialsArgs, @@ -192,6 +197,7 @@ impl BlueBuildCommand for BuildCommand { } else { PathBuf::from(CONTAINER_FILE) })) + .skip_validation(self.skip_validation) .platform(self.platform) .recipe(recipe) .drivers(self.drivers) @@ -286,6 +292,7 @@ impl BuildCommand { .squash(self.squash) .maybe_cache_from(cache_image.as_ref()) .maybe_cache_to(cache_image.as_ref()) + .secrets(recipe.get_secrets()) .build() }, |archive_dir| { @@ -300,6 +307,7 @@ impl BuildCommand { .squash(self.squash) .maybe_cache_from(cache_image.as_ref()) .maybe_cache_to(cache_image.as_ref()) + .secrets(recipe.get_secrets()) .build() }, ))? @@ -368,6 +376,7 @@ impl BuildCommand { .clear_plan(self.rechunk_clear_plan) .maybe_cache_from(cache_image) .maybe_cache_to(cache_image) + .secrets(recipe.get_secrets()) .build(), ) } diff --git a/src/commands/generate.rs b/src/commands/generate.rs index 1491751..565ab19 100644 --- a/src/commands/generate.rs +++ b/src/commands/generate.rs @@ -1,5 +1,6 @@ use std::{ env, + ops::Not, path::{Path, PathBuf}, }; @@ -9,7 +10,9 @@ use blue_build_process_management::drivers::{ use blue_build_recipe::Recipe; use blue_build_template::{ContainerFileTemplate, Template}; use blue_build_utils::{ - constants::{BUILD_SCRIPTS_IMAGE_REF, CONFIG_PATH, RECIPE_FILE, RECIPE_PATH}, + constants::{ + BB_SKIP_VALIDATION, BUILD_SCRIPTS_IMAGE_REF, CONFIG_PATH, RECIPE_FILE, RECIPE_PATH, + }, syntax_highlighting::{self, DefaultThemes}, }; use bon::Builder; @@ -73,6 +76,11 @@ pub struct GenerateCommand { #[builder(default)] platform: Platform, + /// Skips validation of the recipe file. + #[arg(long, env = BB_SKIP_VALIDATION)] + #[builder(default)] + skip_validation: bool, + #[clap(flatten)] #[builder(default)] drivers: DriverArgs, @@ -101,10 +109,12 @@ impl GenerateCommand { } }); - ValidateCommand::builder() - .recipe(recipe_path.clone()) - .build() - .try_run()?; + if self.skip_validation.not() { + ValidateCommand::builder() + .recipe(recipe_path.clone()) + .build() + .try_run()?; + } let registry = if let (Some(registry), Some(registry_namespace)) = (&self.registry, &self.registry_namespace) diff --git a/src/commands/generate_iso.rs b/src/commands/generate_iso.rs index 03e889a..1cb0734 100644 --- a/src/commands/generate_iso.rs +++ b/src/commands/generate_iso.rs @@ -4,7 +4,11 @@ use std::{ }; use blue_build_recipe::Recipe; -use blue_build_utils::{constants::ARCHIVE_SUFFIX, string_vec, traits::CowCollecter}; +use blue_build_utils::{ + constants::{ARCHIVE_SUFFIX, BB_SKIP_VALIDATION}, + string_vec, + traits::CowCollecter, +}; use bon::Builder; use clap::{Args, Subcommand, ValueEnum}; use miette::{Context, IntoDiagnostic, Result, bail}; @@ -98,6 +102,10 @@ pub enum GenIsoSubcommand { /// The path to the recipe file for your image. #[arg()] recipe: PathBuf, + + /// Skips validation of the recipe file. + #[arg(long, env = BB_SKIP_VALIDATION)] + skip_validation: bool, }, } @@ -147,11 +155,16 @@ impl BlueBuildCommand for GenerateIsoCommand { env::current_dir().into_diagnostic()? }; - if let GenIsoSubcommand::Recipe { recipe } = &self.command { + if let GenIsoSubcommand::Recipe { + recipe, + skip_validation, + } = &self.command + { BuildCommand::builder() .recipe(vec![recipe.clone()]) .archive(image_out_dir.path()) .maybe_tempdir(self.tempdir.clone()) + .skip_validation(*skip_validation) .build() .try_run()?; } @@ -208,7 +221,10 @@ impl GenerateIsoCommand { ), ]); } - GenIsoSubcommand::Recipe { recipe } => { + GenIsoSubcommand::Recipe { + recipe, + skip_validation: _, + } => { let recipe = Recipe::parse(recipe)?; args.extend([ diff --git a/src/commands/switch.rs b/src/commands/switch.rs index 47f000c..c3b13dd 100644 --- a/src/commands/switch.rs +++ b/src/commands/switch.rs @@ -9,7 +9,10 @@ use blue_build_process_management::{ }; use blue_build_recipe::Recipe; use blue_build_utils::{ - constants::{ARCHIVE_SUFFIX, LOCAL_BUILD, OCI_ARCHIVE, OSTREE_UNVERIFIED_IMAGE, SUDO_ASKPASS}, + constants::{ + ARCHIVE_SUFFIX, BB_SKIP_VALIDATION, LOCAL_BUILD, OCI_ARCHIVE, OSTREE_UNVERIFIED_IMAGE, + SUDO_ASKPASS, + }, has_env_var, running_as_root, }; use bon::Builder; @@ -41,6 +44,11 @@ pub struct SwitchCommand { #[arg(long)] tempdir: Option, + /// Skips validation of the recipe file. + #[arg(long, env = BB_SKIP_VALIDATION)] + #[builder(default)] + skip_validation: bool, + #[clap(flatten)] #[builder(default)] drivers: DriverArgs, @@ -70,6 +78,7 @@ impl BlueBuildCommand for SwitchCommand { .recipe([self.recipe.clone()]) .archive(tempdir.path()) .maybe_tempdir(self.tempdir.clone()) + .skip_validation(self.skip_validation) .build() .try_run()?; diff --git a/template/src/lib.rs b/template/src/lib.rs index 1c339eb..473cc76 100644 --- a/template/src/lib.rs +++ b/template/src/lib.rs @@ -1,8 +1,9 @@ use std::{borrow::Cow, fs, path::Path, process}; use blue_build_recipe::{MaybeVersion, Recipe}; -use blue_build_utils::constants::{ - CONFIG_PATH, CONTAINER_FILE, CONTAINERFILES_PATH, COSIGN_PUB_PATH, FILES_PATH, +use blue_build_utils::{ + constants::{CONFIG_PATH, CONTAINER_FILE, CONTAINERFILES_PATH, COSIGN_PUB_PATH, FILES_PATH}, + secret::SecretMounts, }; use bon::Builder; use chrono::Utc; diff --git a/template/templates/modules/modules.j2 b/template/templates/modules/modules.j2 index 3cfb2dd..63fc7d1 100644 --- a/template/templates/modules/modules.j2 +++ b/template/templates/modules/modules.j2 @@ -12,6 +12,9 @@ ARG CACHEBUST="{{ build_id }}" {%- include "modules/copy/copy.j2" %} {%- else %} RUN \ + {%- for secret_mount in module.secrets.mounts() %} + {{ secret_mount }} \ + {%- endfor %} {%- if self::files_dir_exists() %} --mount=type=bind,from=stage-files,src=/files,dst=/tmp/files,rw \ {%- else if self::config_dir_exists() %} @@ -33,6 +36,9 @@ RUN \ --mount=type=bind,from={{ build_scripts_image }},src=/scripts/,dst=/tmp/scripts/ \ --mount=type=cache,dst=/var/cache/rpm-ostree,id=rpm-ostree-cache-{{ recipe.name }}-{{ recipe.image_version }},sharing=locked \ --mount=type=cache,dst=/var/cache/libdnf5,id=dnf-cache-{{ recipe.name }}-{{ recipe.image_version }},sharing=locked \ + {%- for secret_var in module.secrets.envs() %} + {{ secret_var }} \ + {%- endfor %} {%- for (key, value) in module.get_env() %} {{ key }}="{{ value | replace('"', "\\\"") }}" \ {%- endfor %} @@ -57,6 +63,9 @@ ARG CACHEBUST="{{ build_id }}" {%- include "modules/copy/copy.j2" %} {%- else %} RUN \ + {%- for secret_mount in module.secrets.mounts() %} + {{ secret_mount }} \ + {%- endfor %} {%- if self::files_dir_exists() %} --mount=type=bind,from=stage-files,src=/files,dst=/tmp/files,rw \ {%- else if self::config_dir_exists() %} @@ -70,6 +79,9 @@ RUN \ --mount=type=bind,from={{ module.get_module_image() }},src=/modules,dst=/tmp/modules,rw \ {%- endif %} --mount=type=bind,from={{ build_scripts_image }},src=/scripts/,dst=/tmp/scripts/ \ + {%- for secret_var in module.secrets.envs() %} + {{ secret_var }} \ + {%- endfor %} {%- for (key, value) in module.get_env() %} {{ key }}="{{ value | replace('"', "\\\"") }}" \ {%- endfor %} diff --git a/utils/Cargo.toml b/utils/Cargo.toml index 5f731bf..996e458 100644 --- a/utils/Cargo.toml +++ b/utils/Cargo.toml @@ -20,6 +20,7 @@ lenient_semver = "0.4" process_control = { version = "4", features = ["crossbeam-channel"] } which = "8" +bon.workspace = true cached.workspace = true chrono.workspace = true clap = { workspace = true, features = ["derive", "env"] } @@ -32,7 +33,9 @@ serde.workspace = true serde_json.workspace = true serde_yaml.workspace = true syntect.workspace = true -bon.workspace = true +tempfile.workspace = true +uuid.workspace = true +zeroize.workspace = true [build-dependencies] syntect.workspace = true diff --git a/utils/src/constants.rs b/utils/src/constants.rs index 88f3162..528c571 100644 --- a/utils/src/constants.rs +++ b/utils/src/constants.rs @@ -23,6 +23,7 @@ pub const BB_PASSWORD: &str = "BB_PASSWORD"; pub const BB_PRIVATE_KEY: &str = "BB_PRIVATE_KEY"; pub const BB_REGISTRY: &str = "BB_REGISTRY"; pub const BB_REGISTRY_NAMESPACE: &str = "BB_REGISTRY_NAMESPACE"; +pub const BB_SKIP_VALIDATION: &str = "BB_SKIP_VALIDATION"; pub const BB_USERNAME: &str = "BB_USERNAME"; pub const BB_BUILD_RECHUNK: &str = "BB_BUILD_RECHUNK"; pub const BB_BUILD_RECHUNK_CLEAR_PLAN: &str = "BB_BUILD_RECHUNK_CLEAR_PLAN"; diff --git a/utils/src/lib.rs b/utils/src/lib.rs index ddfd489..c0a2d6a 100644 --- a/utils/src/lib.rs +++ b/utils/src/lib.rs @@ -2,6 +2,7 @@ pub mod command_output; pub mod constants; pub mod credentials; mod macros; +pub mod secret; pub mod semver; pub mod syntax_highlighting; #[cfg(feature = "test")] @@ -27,11 +28,15 @@ use comlexr::cmd; use format_serde_error::SerdeError; use log::{trace, warn}; use miette::{Context, IntoDiagnostic, Result, miette}; +use uuid::Uuid; use crate::constants::CONTAINER_FILE; pub use command_output::*; +/// UUID used to mark the current builds +pub static BUILD_ID: std::sync::LazyLock = std::sync::LazyLock::new(Uuid::new_v4); + /// Checks for the existance of a given command. /// /// # Errors diff --git a/utils/src/secret.rs b/utils/src/secret.rs new file mode 100644 index 0000000..9946250 --- /dev/null +++ b/utils/src/secret.rs @@ -0,0 +1,233 @@ +use std::{ + collections::HashSet, + fs, + hash::{DefaultHasher, Hash, Hasher}, + ops::Not, + path::PathBuf, +}; + +use cached::proc_macro::cached; +use comlexr::cmd; +use miette::{Context, IntoDiagnostic, Result, bail}; +use serde::{Deserialize, Serialize}; +use tempfile::TempDir; +use zeroize::Zeroizing; + +use crate::{BUILD_ID, string}; + +mod private { + pub trait Private {} +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize, Serialize)] +#[serde(tag = "type")] +pub enum Secret { + #[serde(rename = "env")] + Env { name: String }, + #[serde(rename = "file")] + File { + source: PathBuf, + destination: PathBuf, + }, + #[serde(rename = "exec")] + Exec(SecretExec), + #[serde(rename = "ssh")] + Ssh, +} + +impl Secret { + #[must_use] + pub fn get_hash(&self) -> String { + get_hash(self) + } + + #[must_use] + pub fn mount(&self) -> String { + let hash = self.get_hash(); + let prefix = format!("--mount=type=secret,id={hash}"); + match self { + Self::Env { name: _ } + | Self::Exec(SecretExec { + command: _, + args: _, + output: SecretExecOutput::Env { name: _ }, + }) => format!("{prefix},dst=/tmp/secrets/{hash}"), + Self::File { + source: _, + destination, + } + | Self::Exec(SecretExec { + command: _, + args: _, + output: SecretExecOutput::File { destination }, + }) => format!("{prefix},dst={}", destination.display()), + Self::Ssh => string!("--ssh"), + } + } + + #[must_use] + pub fn env(&self) -> Option { + let hash = self.get_hash(); + match self { + Self::Env { name } + | Self::Exec(SecretExec { + command: _, + args: _, + output: SecretExecOutput::Env { name }, + }) => Some(format!(r#"{name}="$(cat /tmp/secrets/{hash})""#)), + _ => None, + } + } +} + +#[cached(key = "Secret", convert = "{secret.clone()}", sync_writes = "by_key")] +fn get_hash(secret: &Secret) -> String { + let mut hasher = DefaultHasher::new(); + secret.hash(&mut hasher); + BUILD_ID.hash(&mut hasher); + format!("{:x}", hasher.finish()) +} + +impl private::Private for Vec {} + +pub trait SecretMounts: private::Private { + fn mounts(&self) -> Vec; + fn envs(&self) -> Vec; +} + +impl SecretMounts for Vec { + fn mounts(&self) -> Vec { + self.iter().map(Secret::mount).collect() + } + + fn envs(&self) -> Vec { + self.iter().filter_map(Secret::env).collect() + } +} + +impl private::Private for HashSet<&Secret, H> {} + +#[allow(private_bounds)] +pub trait SecretArgs: private::Private { + /// Retrieves the args for the image builder. + /// + /// If exec based secrets are included, will run the commands + /// to put the results into files for mounting. + /// + /// # Errors + /// Will error if an exec based secret fails to run. + fn args(&self, temp_dir: &TempDir) -> Result>; +} + +impl SecretArgs for HashSet<&Secret, H> { + fn args(&self, temp_dir: &TempDir) -> Result> { + self.iter() + .map(|secret| { + Ok(match secret { + Secret::Env { name } => { + format!( + "--secret=id={},type=env,src={}", + secret.get_hash(), + name.trim() + ) + } + Secret::File { + source, + destination: _, + } => { + format!( + "--secret=id={},type=file,src={}", + secret.get_hash(), + source.display() + ) + } + Secret::Exec(exec) => { + let result = exec.exec()?; + let hash = secret.get_hash(); + let secret_path = temp_dir.path().join(&hash); + fs::write(&secret_path, result.value()) + .into_diagnostic() + .wrap_err("Failed to write secret to temp file")?; + format!("--secret=id={hash},src={}", secret_path.display()) + } + Secret::Ssh => string!("--ssh"), + }) + }) + .collect() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize, Serialize)] +pub struct SecretExec { + pub command: String, + pub args: Vec, + pub output: SecretExecOutput, +} + +impl SecretExec { + /// Executes the command to retrieve the secret value. + /// + /// # Errors + /// Will error if the command fails to execute. + pub fn exec(&self) -> Result { + let output = cmd!(&self.command, for &self.args) + .output() + .into_diagnostic() + .wrap_err_with(|| format!("Unable to execute `{}`", self.command))?; + + if output.status.success().not() { + bail!("Failed to execute `{}` to retrieve secret", self.command); + } + + String::from_utf8(output.stdout) + .map(SecretValue::from) + .into_diagnostic() + .wrap_err_with(|| "Failed to read output") + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize, Serialize)] +#[serde(tag = "type")] +pub enum SecretExecOutput { + #[serde(rename = "env")] + Env { name: String }, + #[serde(rename = "file")] + File { destination: PathBuf }, +} + +#[derive(Deserialize)] +pub struct SecretValue(Zeroizing); + +macro_rules! impl_secret_value { + ($($type:ty),*) => { + $( + impl From<$type> for SecretValue { + fn from(value: $type) -> Self { + Self(String::from(value.trim()).into()) + } + } + )* + }; +} + +impl_secret_value!(String, &String, &str); + +impl SecretValue { + /// Get the value of the secret. + #[must_use] + pub fn value(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for SecretValue { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "[REDACTED]") + } +} + +impl std::fmt::Debug for SecretValue { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "[REDACTED]") + } +}