From b2253d598ae23bc52afa1ff00457280b4d90028d Mon Sep 17 00:00:00 2001 From: Gerald Pinder Date: Mon, 28 Apr 2025 20:14:41 -0400 Subject: [PATCH] feat: Add cache layer support --- justfile | 1 + process/drivers/buildah_driver.rs | 16 ++ process/drivers/docker_driver.rs | 354 ++++++++++++++++++------------ process/drivers/opts/build.rs | 12 + process/drivers/opts/rechunk.rs | 6 + process/drivers/podman_driver.rs | 16 ++ process/drivers/traits.rs | 2 + src/commands/build.rs | 127 +++++++---- utils/src/constants.rs | 2 + 9 files changed, 351 insertions(+), 185 deletions(-) diff --git a/justfile b/justfile index 9def7ce..5663db1 100644 --- a/justfile +++ b/justfile @@ -1,4 +1,5 @@ export RUST_BACKTRACE := "1" +export BB_CACHE_LAYERS := "true" set dotenv-load := true set positional-arguments := true diff --git a/process/drivers/buildah_driver.rs b/process/drivers/buildah_driver.rs index 7a35484..319eb4b 100644 --- a/process/drivers/buildah_driver.rs +++ b/process/drivers/buildah_driver.rs @@ -65,6 +65,22 @@ impl BuildDriver for BuildahDriver { ], "--pull=true", format!("--layers={}", !opts.squash), + if let Some(cache_from) = opts.cache_from.as_ref() => [ + "--cache-from", + format!( + "{}/{}", + cache_from.registry(), + cache_from.repository() + ), + ], + if let Some(cache_to) = opts.cache_to.as_ref() => [ + "--cache-to", + format!( + "{}/{}", + cache_to.registry(), + cache_to.repository() + ), + ], "-f", &*opts.containerfile, "-t", diff --git a/process/drivers/docker_driver.rs b/process/drivers/docker_driver.rs index 731cc40..3811efe 100644 --- a/process/drivers/docker_driver.rs +++ b/process/drivers/docker_driver.rs @@ -1,17 +1,17 @@ use std::{ env, + ops::Not, path::Path, process::{Command, ExitStatus}, - sync::Mutex, }; use blue_build_utils::{ - constants::{BB_BUILDKIT_CACHE_GHA, DOCKER_HOST, GITHUB_ACTIONS}, + constants::{BB_BUILDKIT_CACHE_GHA, BLUE_BUILD, DOCKER_HOST, GITHUB_ACTIONS}, credentials::Credentials, semver::Version, string_vec, }; -use cached::proc_macro::cached; +use cached::proc_macro::{cached, once}; use colored::Colorize; use comlexr::{cmd, pipe}; use log::{debug, info, trace, warn}; @@ -38,78 +38,107 @@ use crate::{ use super::opts::{CreateContainerOpts, RemoveContainerOpts, RemoveImageOpts}; #[derive(Debug, Deserialize)] -struct DockerVerisonJsonClient { +struct VerisonJsonClient { #[serde(alias = "Version")] - pub version: Version, + version: Version, } #[derive(Debug, Deserialize)] -struct DockerVersionJson { +struct VersionJson { #[serde(alias = "Client")] - pub client: DockerVerisonJsonClient, + client: VerisonJsonClient, } -static DOCKER_SETUP: std::sync::LazyLock> = - std::sync::LazyLock::new(|| Mutex::new(false)); +#[derive(Debug, Deserialize)] +#[serde(rename_all = "PascalCase")] +struct ContextListItem { + name: String, +} #[derive(Debug)] pub struct DockerDriver; impl DockerDriver { fn setup() -> Result<()> { - trace!("DockerDriver::setup()"); + #[once(result = true, sync_writes = true)] + fn exec() -> Result<()> { + trace!("DockerDriver::setup()"); - if !Self::has_buildx() { - bail!("Docker Buildx is required to use the Docker driver"); - } + if !DockerDriver::has_buildx() { + bail!("Docker Buildx is required to use the Docker driver"); + } - let mut lock = DOCKER_SETUP.lock().expect("Should lock"); - - if *lock { - drop(lock); - return Ok(()); - } - - let ls_out = { - let c = cmd!("docker", "buildx", "ls", "--format={{.Name}}"); - trace!("{c:?}"); - c - } - .output() - .into_diagnostic()?; - - if !ls_out.status.success() { - bail!("{}", String::from_utf8_lossy(&ls_out.stderr)); - } - - let ls_out = String::from_utf8(ls_out.stdout).into_diagnostic()?; - - trace!("{ls_out}"); - - if !ls_out.lines().any(|line| line == "bluebuild") { - let create_out = { - let c = cmd!( - "docker", - "buildx", - "create", - "--bootstrap", - "--driver=docker-container", - "--name=bluebuild", - ); + let ls_out = { + let c = cmd!("docker", "buildx", "ls", "--format={{.Name}}"); trace!("{c:?}"); c } .output() .into_diagnostic()?; - if !create_out.status.success() { - bail!("{}", String::from_utf8_lossy(&create_out.stderr)); + if !ls_out.status.success() { + bail!("{}", String::from_utf8_lossy(&ls_out.stderr)); } + + let ls_out = String::from_utf8(ls_out.stdout).into_diagnostic()?; + + trace!("{ls_out}"); + + if !ls_out.lines().any(|line| line == BLUE_BUILD) { + let remote = env::var(DOCKER_HOST).is_ok(); + if remote { + let context_list = get_context_list()?; + trace!("{context_list:#?}"); + + if context_list.iter().any(|ctx| ctx.name == BLUE_BUILD).not() { + let context_out = { + let c = cmd!( + "docker", + "context", + "create", + "--from=default", + format!("{BLUE_BUILD}0") + ); + trace!("{c:?}"); + c + } + .output() + .into_diagnostic()?; + + if context_out.status.success().not() { + bail!("{}", String::from_utf8_lossy(&context_out.stderr)); + } + + let context_list = get_context_list()?; + trace!("{context_list:#?}"); + } + } + + let create_out = { + let c = cmd!( + "docker", + "buildx", + "create", + "--bootstrap", + "--driver=docker-container", + format!("--name={BLUE_BUILD}"), + if remote => format!("{BLUE_BUILD}0"), + ); + trace!("{c:?}"); + c + } + .output() + .into_diagnostic()?; + + if !create_out.status.success() { + bail!("{}", String::from_utf8_lossy(&create_out.stderr)); + } + } + + Ok(()) } - *lock = true; - drop(lock); - Ok(()) + exec() } #[must_use] @@ -120,6 +149,25 @@ impl DockerDriver { } } +fn get_context_list() -> Result> { + { + let c = cmd!("docker", "context", "ls", "--format=json"); + trace!("{c:?}"); + c + } + .output() + .into_diagnostic() + .and_then(|out| { + if out.status.success().not() { + bail!("{}", String::from_utf8_lossy(&out.stderr)); + } + String::from_utf8(out.stdout).into_diagnostic() + })? + .lines() + .map(|line| serde_json::from_str(line).into_diagnostic()) + .collect() +} + impl DriverVersion for DockerDriver { // First docker verison to use buildkit // https://docs.docker.com/build/buildkit/ @@ -136,8 +184,7 @@ impl DriverVersion for DockerDriver { .output() .into_diagnostic()?; - let version_json: DockerVersionJson = - serde_json::from_slice(&output.stdout).into_diagnostic()?; + let version_json: VersionJson = serde_json::from_slice(&output.stdout).into_diagnostic()?; Ok(version_json.client.version) } @@ -162,6 +209,22 @@ impl BuildDriver for DockerDriver { "-t", &*opts.image, "-f", + if let Some(cache_from) = opts.cache_from.as_ref() => [ + "--cache-from", + format!( + "type=registry,ref={registry}/{repository}", + registry = cache_from.registry(), + repository = cache_from.repository(), + ), + ], + if let Some(cache_to) = opts.cache_to.as_ref() => [ + "--cache-to", + format!( + "type=registry,ref={registry}/{repository},mode=max", + registry = cache_to.registry(), + repository = cache_to.repository(), + ), + ], &*opts.containerfile, ".", ); @@ -282,11 +345,7 @@ impl BuildDriver for DockerDriver { }); let buildx = scope.spawn(|| { - let run_setup = !env::var(DOCKER_HOST).is_ok_and(|dh| !dh.is_empty()); - - if run_setup { - Self::setup()?; - } + Self::setup()?; { let c = cmd!( @@ -294,7 +353,7 @@ impl BuildDriver for DockerDriver { "buildx", "prune", "--force", - if run_setup => "--builder=bluebuild", + format!("--builder={BLUE_BUILD}"), if opts.all => "--all", ); trace!("{c:?}"); @@ -327,87 +386,15 @@ impl BuildDriver for DockerDriver { warn!("Squash is deprecated for docker so this build will not squash"); } - let run_setup = !env::var(DOCKER_HOST).is_ok_and(|dh| !dh.is_empty()); + Self::setup()?; - if run_setup { - Self::setup()?; - } - - let final_images = match (opts.image, opts.archive_path.as_deref()) { - (Some(image), None) => { - let images = if opts.tags.is_empty() { - let image = image.to_string(); - string_vec![image] - } else { - opts.tags - .iter() - .map(|tag| { - format!("{}/{}:{tag}", image.resolve_registry(), image.repository()) - }) - .collect() - }; - - images - } - (None, Some(archive_path)) => { - string_vec![archive_path.display().to_string()] - } - (Some(_), Some(_)) => bail!("Cannot use both image and archive path"), - (None, None) => bail!("Need either the image or archive path set"), - }; + let final_images = get_final_images(opts)?; let first_image = final_images.first().unwrap(); - let status = { - let c = cmd!( - "docker", - "buildx", - if run_setup => "--builder=bluebuild", - "build", - ".", - match (opts.image, opts.archive_path.as_deref()) { - (Some(_), None) if opts.push => [ - "--output", - format!( - "type=image,name={first_image},push=true,compression={},oci-mediatypes=true", - opts.compression - ), - ], - (Some(_), None) if env::var(GITHUB_ACTIONS).is_err() => "--load", - (None, Some(archive_path)) => [ - "--output", - format!("type=oci,dest={}", archive_path.display()), - ], - _ => [], - }, - for opts.image.as_ref().map_or_else(Vec::new, |image| { - opts.tags.iter().flat_map(|tag| { - vec![ - "-t".to_string(), - format!("{}/{}:{tag}", image.resolve_registry(), image.repository()) - ] - }).collect() - }), - "--pull", - if !matches!(opts.platform, Platform::Native) => [ - "--platform", - opts.platform.to_string(), - ], - "-f", - &*opts.containerfile, - // https://github.com/moby/buildkit?tab=readme-ov-file#github-actions-cache-experimental - if env::var(BB_BUILDKIT_CACHE_GHA) - .map_or_else(|_| false, |e| e == "true") => [ - "--cache-from", - "type=gha", - "--cache-to", - "type=gha", - ], - ); - trace!("{c:?}"); - c - } - .build_status(first_image, "Building Image").into_diagnostic()?; + let status = build_tag_push_cmd(opts, first_image) + .build_status(first_image, "Building Image") + .into_diagnostic()?; if status.success() { if opts.push { @@ -422,6 +409,93 @@ impl BuildDriver for DockerDriver { } } +fn build_tag_push_cmd(opts: &BuildTagPushOpts<'_>, first_image: &str) -> Command { + // let remote = env::var(DOCKER_HOST).is_ok(); + let c = cmd!( + "docker", + // if remote => format!("--context={BLUE_BUILD}0"), + "buildx", + format!("--builder={BLUE_BUILD}"), + "build", + ".", + match (opts.image, opts.archive_path.as_deref()) { + (Some(_), None) if opts.push => [ + "--output", + format!( + "type=image,name={first_image},push=true,compression={},oci-mediatypes=true", + opts.compression + ), + ], + (Some(_), None) if env::var(GITHUB_ACTIONS).is_err() => "--load", + (None, Some(archive_path)) => [ + "--output", + format!("type=oci,dest={}", archive_path.display()), + ], + _ => [], + }, + for opts.image.as_ref().map_or_else(Vec::new, |image| { + opts.tags.iter().flat_map(|tag| { + vec![ + "-t".to_string(), + format!("{}/{}:{tag}", image.resolve_registry(), image.repository()) + ] + }).collect() + }), + "--pull", + if !matches!(opts.platform, Platform::Native) => [ + "--platform", + opts.platform.to_string(), + ], + "-f", + &*opts.containerfile, + // https://github.com/moby/buildkit?tab=readme-ov-file#github-actions-cache-experimental + if env::var(BB_BUILDKIT_CACHE_GHA) + .map_or_else(|_| false, |e| e == "true") => [ + "--cache-from", + "type=gha", + "--cache-to", + "type=gha", + ], + if let Some(cache_from) = opts.cache_from.as_ref() => [ + "--cache-from", + format!( + "type=registry,ref={cache_from}", + ), + ], + if let Some(cache_to) = opts.cache_to.as_ref() => [ + "--cache-to", + format!( + "type=registry,ref={cache_to},mode=max", + ), + ], + ); + trace!("{c:?}"); + c +} + +fn get_final_images(opts: &BuildTagPushOpts<'_>) -> Result> { + Ok(match (opts.image, opts.archive_path.as_deref()) { + (Some(image), None) => { + let images = if opts.tags.is_empty() { + let image = image.to_string(); + string_vec![image] + } else { + opts.tags + .iter() + .map(|tag| format!("{}/{}:{tag}", image.resolve_registry(), image.repository())) + .collect() + }; + + images + } + (None, Some(archive_path)) => { + string_vec![archive_path.display().to_string()] + } + (Some(_), Some(_)) => bail!("Cannot use both image and archive path"), + (None, None) => bail!("Need either the image or archive path set"), + }) +} + impl InspectDriver for DockerDriver { fn get_metadata(opts: &GetMetadataOpts) -> Result { get_metadata_cache(opts) @@ -438,17 +512,13 @@ fn get_metadata_cache(opts: &GetMetadataOpts) -> Result { trace!("DockerDriver::get_metadata({opts:#?})"); let image_str = opts.image.to_string(); - let run_setup = !env::var(DOCKER_HOST).is_ok_and(|dh| !dh.is_empty()); - - if run_setup { - DockerDriver::setup()?; - } + DockerDriver::setup()?; let output = { let c = cmd!( "docker", "buildx", - if run_setup => "--builder=bluebuild", + format!("--builder={BLUE_BUILD}"), "imagetools", "inspect", "--format", diff --git a/process/drivers/opts/build.rs b/process/drivers/opts/build.rs index f5c946b..b789655 100644 --- a/process/drivers/opts/build.rs +++ b/process/drivers/opts/build.rs @@ -27,6 +27,12 @@ pub struct BuildOpts<'scope> { #[builder(default)] pub privileged: bool, + + #[builder(into)] + pub cache_from: Option<&'scope Reference>, + + #[builder(into)] + pub cache_to: Option<&'scope Reference>, } #[derive(Debug, Clone, Builder)] @@ -108,4 +114,10 @@ pub struct BuildTagPushOpts<'scope> { /// Runs the build with elevated privileges #[builder(default)] pub privileged: bool, + + /// Cache layers from the registry. + pub cache_from: Option<&'scope Reference>, + + /// Cache layers to the registry. + pub cache_to: Option<&'scope Reference>, } diff --git a/process/drivers/opts/rechunk.rs b/process/drivers/opts/rechunk.rs index d9f59e5..23095ad 100644 --- a/process/drivers/opts/rechunk.rs +++ b/process/drivers/opts/rechunk.rs @@ -49,6 +49,12 @@ pub struct RechunkOpts<'scope> { #[builder(default)] pub clear_plan: bool, + + /// Cache layers from the registry. + pub cache_from: Option<&'scope Reference>, + + /// Cache layers to the registry. + pub cache_to: Option<&'scope Reference>, } #[derive(Debug, Clone, Builder)] diff --git a/process/drivers/podman_driver.rs b/process/drivers/podman_driver.rs index 95e4f8e..b7fec79 100644 --- a/process/drivers/podman_driver.rs +++ b/process/drivers/podman_driver.rs @@ -157,6 +157,22 @@ impl BuildDriver for PodmanDriver { "--platform", opts.platform.to_string(), ], + if let Some(cache_from) = opts.cache_from.as_ref() => [ + "--cache-from", + format!( + "{}/{}", + cache_from.registry(), + cache_from.repository() + ), + ], + if let Some(cache_to) = opts.cache_to.as_ref() => [ + "--cache-to", + format!( + "{}/{}", + cache_to.registry(), + cache_to.repository() + ), + ], "--pull=true", if opts.host_network => "--net=host", format!("--layers={}", !opts.squash), diff --git a/process/drivers/traits.rs b/process/drivers/traits.rs index f23d4a5..d142a99 100644 --- a/process/drivers/traits.rs +++ b/process/drivers/traits.rs @@ -136,6 +136,8 @@ pub trait BuildDriver: PrivateDriver { .containerfile(opts.containerfile.as_ref()) .platform(opts.platform) .squash(opts.squash) + .maybe_cache_from(opts.cache_from) + .maybe_cache_to(opts.cache_to) .build(); info!("Building image {full_image}"); diff --git a/src/commands/build.rs b/src/commands/build.rs index c157893..90a98d1 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -24,7 +24,7 @@ use blue_build_utils::{ }; use bon::Builder; use clap::Args; -use log::{info, trace, warn}; +use log::{debug, info, trace, warn}; use miette::{IntoDiagnostic, Result, bail}; use oci_distribution::Reference; use tempfile::TempDir; @@ -136,6 +136,13 @@ pub struct BuildCommand { #[arg(long)] tempdir: Option, + /// Automatically cache build layers to the registry. + /// + /// NOTE: Only works when using --push + #[builder(default)] + #[arg(long, env = blue_build_utils::constants::BB_CACHE_LAYERS)] + cache_layers: bool, + #[clap(flatten)] #[builder(default)] credentials: CredentialsArgs, @@ -298,6 +305,18 @@ impl BuildCommand { let image: Reference = format!("{image_name}:{}", tags.first().map_or("latest", |tag| tag)) .parse() .into_diagnostic()?; + let cache_image = (self.cache_layers && self.push).then(|| { + let cache_image = Reference::with_tag( + image.registry().to_string(), + image.repository().to_string(), + format!( + "{}-cache", + image.tag().expect("Reference should be built with tag") + ), + ); + debug!("Using {cache_image} for caching layers"); + cache_image + }); let build_fn = || -> Result> { Driver::build_tag_push(&self.archive.as_ref().map_or_else( @@ -312,6 +331,8 @@ impl BuildCommand { .retry_count(self.retry_count) .compression(self.compression_format) .squash(self.squash) + .maybe_cache_from(cache_image.as_ref()) + .maybe_cache_to(cache_image.as_ref()) .build() }, |archive_dir| { @@ -324,6 +345,8 @@ impl BuildCommand { recipe.name.to_lowercase().replace('/', "_"), ))) .squash(self.squash) + .maybe_cache_from(cache_image.as_ref()) + .maybe_cache_to(cache_image.as_ref()) .build() }, )) @@ -331,48 +354,12 @@ impl BuildCommand { #[cfg(feature = "rechunk")] let images = if self.rechunk { - use blue_build_process_management::drivers::{ - InspectDriver, RechunkDriver, - opts::{GetMetadataOpts, RechunkOpts}, - }; - - let base_image: Reference = format!("{}:{}", &recipe.base_image, &recipe.image_version) - .parse() - .into_diagnostic()?; - - Driver::rechunk( - &RechunkOpts::builder() - .image(&image_name) - .containerfile(containerfile) - .platform(self.platform) - .tags(tags.collect_cow_vec()) - .push(self.push) - .version(format!( - "{version}.", - version = Driver::get_os_version() - .oci_ref(&recipe.base_image_ref()?) - .platform(self.platform) - .call()?, - )) - .retry_push(self.retry_push) - .retry_count(self.retry_count) - .compression(self.compression_format) - .base_digest( - Driver::get_metadata( - &GetMetadataOpts::builder() - .image(&base_image) - .platform(self.platform) - .build(), - )? - .digest, - ) - .repo(Driver::get_repo_url()?) - .name(&*recipe.name) - .description(&*recipe.description) - .base_image(format!("{}:{}", &recipe.base_image, &recipe.image_version)) - .maybe_tempdir(self.tempdir.as_deref()) - .clear_plan(self.rechunk_clear_plan) - .build(), + self.rechunk( + containerfile, + &recipe, + &tags, + &image_name, + cache_image.as_ref(), )? } else { build_fn()? @@ -395,6 +382,60 @@ impl BuildCommand { Ok(images) } + #[cfg(feature = "rechunk")] + fn rechunk( + &self, + containerfile: &Path, + recipe: &Recipe<'_>, + tags: &[String], + image_name: &str, + cache_image: Option<&Reference>, + ) -> Result, miette::Error> { + use blue_build_process_management::drivers::{ + InspectDriver, RechunkDriver, + opts::{GetMetadataOpts, RechunkOpts}, + }; + let base_image: Reference = format!("{}:{}", &recipe.base_image, &recipe.image_version) + .parse() + .into_diagnostic()?; + Driver::rechunk( + &RechunkOpts::builder() + .image(image_name) + .containerfile(containerfile) + .platform(self.platform) + .tags(tags.collect_cow_vec()) + .push(self.push) + .version(format!( + "{version}.", + version = Driver::get_os_version() + .oci_ref(&recipe.base_image_ref()?) + .platform(self.platform) + .call()?, + )) + .retry_push(self.retry_push) + .retry_count(self.retry_count) + .compression(self.compression_format) + .base_digest( + Driver::get_metadata( + &GetMetadataOpts::builder() + .image(&base_image) + .platform(self.platform) + .build(), + )? + .digest, + ) + .repo(Driver::get_repo_url()?) + .name(&*recipe.name) + .description(&*recipe.description) + .base_image(format!("{}:{}", &recipe.base_image, &recipe.image_version)) + .maybe_tempdir(self.tempdir.as_deref()) + .clear_plan(self.rechunk_clear_plan) + .maybe_cache_from(cache_image) + .maybe_cache_to(cache_image) + .build(), + ) + } + fn image_name(&self, recipe: &Recipe) -> Result { let image_name = Driver::generate_image_name( GenerateImageNameOpts::builder() diff --git a/utils/src/constants.rs b/utils/src/constants.rs index c8563b1..06e7a38 100644 --- a/utils/src/constants.rs +++ b/utils/src/constants.rs @@ -19,6 +19,7 @@ pub const IMAGE_VERSION_LABEL: &str = "org.opencontainers.image.version"; // BlueBuild vars pub const BB_BUILDKIT_CACHE_GHA: &str = "BB_BUILDKIT_CACHE_GHA"; +pub const BB_CACHE_LAYERS: &str = "BB_CACHE_LAYERS"; pub const BB_PASSWORD: &str = "BB_PASSWORD"; pub const BB_PRIVATE_KEY: &str = "BB_PRIVATE_KEY"; pub const BB_REGISTRY: &str = "BB_REGISTRY"; @@ -76,6 +77,7 @@ pub const XDG_RUNTIME_DIR: &str = "XDG_RUNTIME_DIR"; pub const SUDO_ASKPASS: &str = "SUDO_ASKPASS"; // Misc +pub const BLUE_BUILD: &str = "bluebuild"; pub const BUILD_SCRIPTS_IMAGE_REF: &str = "ghcr.io/blue-build/cli/build-scripts"; pub const BLUE_BULID_IMAGE_REF: &str = "ghcr.io/blue-build/cli"; pub const BLUE_BUILD_MODULE_IMAGE_REF: &str = "ghcr.io/blue-build/modules";