From 0ba78fab321a5f4e589277e3de7f8ad4e5c309a7 Mon Sep 17 00:00:00 2001 From: Gerald Pinder Date: Fri, 1 Mar 2024 12:18:02 -0500 Subject: [PATCH] refactor: Update build command to use BuildStrategy (#88) --- .github/workflows/build-pr.yml | 55 ++ .github/workflows/build.yml | 56 +++ Earthfile | 13 +- integration-tests/Earthfile | 3 +- integration-tests/test-repo/config/recipe.yml | 5 +- src/commands/build.rs | 473 ++---------------- src/commands/build/build_strategy.rs | 56 --- src/lib.rs | 1 + src/strategies.rs | 121 +++++ src/strategies/buildah_strategy.rs | 91 ++++ src/strategies/docker_strategy.rs | 122 +++++ src/strategies/podman_api_strategy.rs | 208 ++++++++ src/strategies/podman_strategy.rs | 93 ++++ utils/src/constants.rs | 3 + 14 files changed, 818 insertions(+), 482 deletions(-) delete mode 100644 src/commands/build/build_strategy.rs create mode 100644 src/strategies.rs create mode 100644 src/strategies/buildah_strategy.rs create mode 100644 src/strategies/docker_strategy.rs create mode 100644 src/strategies/podman_api_strategy.rs create mode 100644 src/strategies/podman_strategy.rs diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml index da4ec1a..5868ee1 100644 --- a/.github/workflows/build-pr.yml +++ b/.github/workflows/build-pr.yml @@ -45,3 +45,58 @@ jobs: if: github.repository == 'blue-build/cli' run: earthly --ci -P ./integration-tests+all + docker-build: + timeout-minutes: 60 + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Maximize build space + uses: ublue-os/remove-unwanted-software@v6 + + - uses: sigstore/cosign-installer@v3.3.0 + - uses: earthly/actions-setup@v1 + with: + version: v0.8.3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + install: true + + - name: Earthly login + env: + EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} + if: env.EARTHLY_SAT_TOKEN != null + run: | + earthly account login --token ${{ secrets.EARTHLY_SAT_TOKEN }} >> /dev/null + earthly org s blue-build + earthly sat s blue-build-integration-tests + + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.ref }} + + - name: Install bluebuild + env: + BB_BUILDKIT_CACHE_GHA: true + run: | + earthly -a +install/bluebuild --BUILD_TARGET=x86_64-unknown-linux-musl /usr/local/bin/bluebuild + + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v3 + + - name: Run Build + env: + BB_BUILDKIT_CACHE_GHA: "true" + GH_TOKEN: ${{ github.token }} + GH_PR_EVENT_NUMBER: ${{ github.event.number }} + run: | + cd integration-tests/test-repo + echo -n "\n\n" | cosign generate-key-pair + export COSIGN_PRIVATE_KEY=$(cat cosign.key) + rm cosign.key + bluebuild build --push -vv diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index efb804b..2ccfd68 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -58,3 +58,59 @@ jobs: - name: Run integration tests if: github.repository == 'blue-build/cli' run: earthly --ci -P ./integration-tests+all + + docker-build: + timeout-minutes: 60 + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Maximize build space + uses: ublue-os/remove-unwanted-software@v6 + + - uses: sigstore/cosign-installer@v3.3.0 + - uses: earthly/actions-setup@v1 + with: + version: v0.8.3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + install: true + + - name: Earthly login + env: + EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} + if: env.EARTHLY_SAT_TOKEN != null + run: | + earthly account login --token ${{ secrets.EARTHLY_SAT_TOKEN }} >> /dev/null + earthly org s blue-build + earthly sat s blue-build-integration-tests + + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.ref }} + + - name: Install bluebuild + env: + BB_BUILDKIT_CACHE_GHA: true + run: | + earthly -a +install/bluebuild --BUILD_TARGET=x86_64-unknown-linux-musl /usr/local/bin/bluebuild + + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v3 + + - name: Run Build + env: + BB_BUILDKIT_CACHE_GHA: "true" + GH_TOKEN: ${{ github.token }} + GH_PR_EVENT_NUMBER: ${{ github.event.number }} + run: | + cd integration-tests/test-repo + echo -n "\n\n" | cosign generate-key-pair + export COSIGN_PRIVATE_KEY=$(cat cosign.key) + rm cosign.key + bluebuild build --push -vv diff --git a/Earthfile b/Earthfile index 0cbb800..2ad3ec2 100644 --- a/Earthfile +++ b/Earthfile @@ -68,7 +68,18 @@ blue-build-cli: BUILD +install --BUILD_TARGET="x86_64-unknown-linux-gnu" --NIGHTLY=$NIGHTLY - RUN dnf install --refresh -y buildah podman skopeo + RUN dnf -y install dnf-plugins-core \ + && dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo \ + && dnf install --refresh -y \ + jq \ + docker-ce \ + docker-ce-cli \ + containerd.io \ + docker-buildx-plugin \ + docker-compose-plugin \ + buildah \ + podman \ + skopeo COPY +cosign/cosign /usr/bin/cosign diff --git a/integration-tests/Earthfile b/integration-tests/Earthfile index 344ac5c..305bac3 100644 --- a/integration-tests/Earthfile +++ b/integration-tests/Earthfile @@ -37,7 +37,7 @@ rebase: upgrade: ARG NIGHTLY=false FROM +test-base --NIGHTLY=$NIGHTLY - RUN mkdir -p /etc/bluebuild && touch /etc/bluebuild/template.tar.gz + RUN mkdir -p /etc/bluebuild && touch /etc/bluebuild/cli_test.tar.gz RUN --privileged bluebuild -vv upgrade config/recipe.yml @@ -55,3 +55,4 @@ test-base: RUN echo -n "\n\n" | cosign generate-key-pair ENV COSIGN_PRIVATE_KEY=$(cat cosign.key) RUN rm cosign.key + diff --git a/integration-tests/test-repo/config/recipe.yml b/integration-tests/test-repo/config/recipe.yml index 5e1730d..7422214 100644 --- a/integration-tests/test-repo/config/recipe.yml +++ b/integration-tests/test-repo/config/recipe.yml @@ -1,4 +1,4 @@ -name: template +name: cli/test description: This is my personal OS image. base-image: ghcr.io/ublue-os/silverblue-surface image-version: 39 @@ -33,7 +33,8 @@ modules: remove: - org.gnome.eog - - type: signing + # Needs a bug to be fixed to allow / in image name + # - type: signing - type: test-module diff --git a/src/commands/build.rs b/src/commands/build.rs index 4c0cd0f..0e05feb 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -1,10 +1,8 @@ -#[cfg(feature = "podman-api")] -mod build_strategy; - use std::{ env, fs, path::{Path, PathBuf}, process::Command, + rc::Rc, }; use anyhow::{anyhow, bail, Result}; @@ -16,29 +14,11 @@ use log::{debug, info, trace, warn}; use typed_builder::TypedBuilder; use uuid::Uuid; -#[cfg(feature = "builtin-podman")] -use std::sync::Arc; - -#[cfg(feature = "podman-api")] -use podman_api::{ - opts::{ImageBuildOpts, ImagePushOpts, RegistryAuth}, - Podman, +use crate::{ + commands::template::TemplateCommand, + strategies::{determine_build_strategy, BuildStrategy}, }; -#[cfg(feature = "podman-api")] -use build_strategy::BuildStrategy; - -#[cfg(feature = "signal-hook-tokio")] -use signal_hook_tokio::Signals; - -#[cfg(feature = "tokio")] -use tokio::{ - runtime::Runtime, - sync::oneshot::{self, Sender}, -}; - -use crate::commands::template::TemplateCommand; - use super::BlueBuildCommand; #[derive(Debug, Default, Clone, TypedBuilder)] @@ -115,7 +95,7 @@ pub struct BuildCommand { /// The connection string used to connect /// to a remote podman socket. - #[cfg(feature = "podman-api")] + #[cfg(feature = "tls")] #[arg(short, long)] #[builder(default, setter(into, strip_option))] connection: Option, @@ -227,13 +207,6 @@ impl BlueBuildCommand for BuildCommand { .clone() .unwrap_or_else(|| PathBuf::from(RECIPE_PATH)); - #[cfg(not(feature = "podman-api"))] - if let Err(e1) = blue_build_utils::check_command_exists("buildah") { - blue_build_utils::check_command_exists("podman").map_err(|e2| { - anyhow!("Need either 'buildah' or 'podman' commands to proceed: {e1}, {e2}") - })?; - } - if self.push { blue_build_utils::check_command_exists("cosign")?; blue_build_utils::check_command_exists("skopeo")?; @@ -249,140 +222,17 @@ impl BlueBuildCommand for BuildCommand { info!("Building image for recipe at {}", recipe_path.display()); - #[cfg(feature = "builtin-podman")] - match BuildStrategy::determine_strategy()? { - BuildStrategy::Socket(socket) => Runtime::new()?.block_on(self.build_image_podman_api( - Podman::unix(socket), - build_id, - &recipe_path, - )), - _ => self.build_image(&recipe_path), - } + let credentials = self.get_login_creds(); - #[cfg(not(feature = "builtin-podman"))] - self.build_image(&recipe_path) + self.start( + &recipe_path, + determine_build_strategy(build_id, credentials, self.archive.is_some())?, + ) } } impl BuildCommand { - #[cfg(feature = "builtin-podman")] - async fn build_image_podman_api( - &self, - client: Podman, - build_id: Uuid, - recipe_path: &Path, - ) -> Result<()> { - use futures_util::StreamExt; - use signal_hook::consts::{SIGINT, SIGQUIT, SIGTERM}; - - trace!("BuildCommand::build_image({client:#?})"); - - let credentials = self.get_login_creds(); - - if self.push && credentials.is_none() { - bail!("Failed to get credentials"); - } - - let recipe = Recipe::parse(&recipe_path)?; - trace!("recipe: {recipe:#?}"); - - // Get values for image - let tags = recipe.generate_tags(); - let image_name = self.generate_full_image_name(&recipe)?; - let first_image_name = if self.archive.is_some() { - image_name.to_string() - } else { - tags.first() - .map_or_else(|| image_name.to_string(), |t| format!("{image_name}:{t}")) - }; - debug!("Full tag is {first_image_name}"); - - // Prepare for the signal trap - let client = Arc::new(client); - - let signals = Signals::new([SIGTERM, SIGINT, SIGQUIT])?; - let handle = signals.handle(); - - let (kill_tx, mut kill_rx) = oneshot::channel::<()>(); - - let signals_task = tokio::spawn(handle_signals(signals, kill_tx, build_id, client.clone())); - - // Get podman ready to build - let opts = ImageBuildOpts::builder(".") - .tag(&first_image_name) - .dockerfile("Containerfile") - .remove(true) - .layers(true) - .labels([(BUILD_ID_LABEL, build_id.to_string())]) - .pull(true) - .build(); - trace!("Build options: {opts:#?}"); - - info!("Building image {first_image_name}"); - match client.images().build(&opts) { - Ok(mut build_stream) => loop { - tokio::select! { - Some(chunk) = build_stream.next() => { - match chunk { - Ok(chunk) => chunk - .stream - .trim() - .lines() - .map(str::trim) - .filter(|line| !line.is_empty()) - .for_each(|line| info!("{line}")), - Err(e) => bail!("{e}"), - } - }, - _ = &mut kill_rx => { - break; - } - } - }, - Err(e) => bail!("{e}"), - }; - handle.close(); - signals_task.await??; - - if self.push { - debug!("Pushing is enabled"); - - let credentials = - credentials.ok_or_else(|| anyhow!("Should have checked for creds earlier"))?; - - push_images_podman_api(&tags, &image_name, &first_image_name, &client, &credentials) - .await?; - - let (registry, username, password) = ( - credentials.registry, - credentials.username, - credentials.password, - ); - - info!("Logging into registry using cosign"); - trace!("cosign login -u {username} -p [MASKED] {registry}"); - if !Command::new("cosign") - .arg("login") - .arg("-u") - .arg(&username) - .arg("-p") - .arg(&password) - .arg(®istry) - .output()? - .status - .success() - { - bail!("Failed to login for cosign!"); - } - info!("Cosign login success at {registry}"); - - sign_images(&image_name, tags.first().map(String::as_str))?; - } - - Ok(()) - } - - fn build_image(&self, recipe_path: &Path) -> Result<()> { + fn start(&self, recipe_path: &Path, build_strat: Rc) -> Result<()> { trace!("BuildCommand::build_image()"); let recipe = Recipe::parse(&recipe_path)?; @@ -392,16 +242,16 @@ impl BuildCommand { let image_name = self.generate_full_image_name(&recipe)?; if self.push { - self.login()?; + self.login(build_strat.clone())?; } - self.run_build(&image_name, &tags)?; + self.run_build(&image_name, &tags, build_strat)?; info!("Build complete!"); Ok(()) } - fn login(&self) -> Result<()> { + fn login(&self, build_strat: Rc) -> Result<()> { trace!("BuildCommand::login()"); info!("Attempting to login to the registry"); @@ -410,47 +260,22 @@ impl BuildCommand { .ok_or_else(|| anyhow!("Unable to get credentials"))?; let (registry, username, password) = ( - credentials.registry, - credentials.username, - credentials.password, + &credentials.registry, + &credentials.username, + &credentials.password, ); info!("Logging into the registry, {registry}"); - let login_output = match ( - blue_build_utils::check_command_exists("buildah"), - blue_build_utils::check_command_exists("podman"), - ) { - (Ok(()), _) => { - trace!("buildah login -u {username} -p [MASKED] {registry}"); - Command::new("buildah") - } - (Err(_), Ok(())) => { - trace!("podman login -u {username} -p [MASKED] {registry}"); - Command::new("podman") - } - _ => bail!("Need 'buildah' or 'podman' to login"), - } - .arg("login") - .arg("-u") - .arg(&username) - .arg("-p") - .arg(&password) - .arg(®istry) - .output()?; - - if !login_output.status.success() { - let err_out = String::from_utf8_lossy(&login_output.stderr); - bail!("Failed to login for buildah: {err_out}"); - } + build_strat.login()?; trace!("cosign login -u {username} -p [MASKED] {registry}"); let login_output = Command::new("cosign") .arg("login") .arg("-u") - .arg(&username) + .arg(username) .arg("-p") - .arg(&password) - .arg(®istry) + .arg(password) + .arg(registry) .output()?; if !login_output.status.success() { @@ -473,7 +298,7 @@ impl BuildCommand { format!( "oci-archive:{}/{}.{ARCHIVE_SUFFIX}", archive_dir.to_string_lossy().trim_end_matches('/'), - recipe.name.to_lowercase(), + recipe.name.to_lowercase().replace('/', "_"), ) } else { match ( @@ -535,7 +360,12 @@ impl BuildCommand { /// # Errors /// /// Will return `Err` if the build fails. - fn run_build(&self, image_name: &str, tags: &[String]) -> Result<()> { + fn run_build( + &self, + image_name: &str, + tags: &[String], + build_strat: Rc, + ) -> Result<()> { trace!("BuildCommand::run_build({image_name}, {tags:#?})"); let full_image = if self.archive.is_some() { @@ -546,46 +376,34 @@ impl BuildCommand { }; info!("Building image {full_image}"); - let status = match ( - blue_build_utils::check_command_exists("buildah"), - blue_build_utils::check_command_exists("podman"), - ) { - (Ok(()), _) => { - trace!("buildah build -t {full_image}"); - Command::new("buildah") - .arg("build") - .arg("-t") - .arg(&full_image) - .status()? - } - (Err(_), Ok(())) => { - trace!("podman build . -t {full_image}"); - Command::new("podman") - .arg("build") - .arg(".") - .arg("-t") - .arg(&full_image) - .status()? - } - (Err(e1), Err(e2)) => bail!("Need either 'buildah' or 'podman' to build: {e1}, {e2}"), - }; - - if status.success() { - info!("Successfully built {image_name}"); - } else { - bail!("Failed to build {image_name}"); - } + build_strat.build(&full_image)?; if tags.len() > 1 && self.archive.is_none() { - tag_images(tags, image_name, &full_image)?; + debug!("Tagging all images"); + + for tag in tags { + debug!("Tagging {image_name} with {tag}"); + + build_strat.tag(&full_image, image_name, tag)?; + + if self.push { + let retry = self.retry_push; + let retry_count = if retry { self.retry_count } else { 0 }; + + debug!("Pushing all images"); + // Push images with retries (1s delay between retries) + blue_build_utils::retry(retry_count, 1000, || { + debug!("Pushing image {image_name}:{tag}"); + + let tag_image = format!("{image_name}:{tag}"); + + build_strat.push(&tag_image) + })?; + } + } } if self.push { - let retry = self.retry_push; - let retry_count = if retry { self.retry_count } else { 0 }; - - // Push images with retries (1s delay between retries) - blue_build_utils::retry(retry_count, 1000, || push_images(tags, image_name))?; sign_images(image_name, tags.first().map(String::as_str))?; } @@ -887,192 +705,3 @@ fn check_cosign_files() -> Result<()> { } } } - -#[cfg(feature = "builtin-podman")] -async fn handle_signals( - mut signals: Signals, - kill: Sender<()>, - build_id: Uuid, - client: Arc, -) -> Result<()> { - use std::process; - - use futures_util::StreamExt; - use podman_api::opts::{ - ContainerListOpts, ContainerPruneFilter, ContainerPruneOpts, ImagePruneFilter, - ImagePruneOpts, - }; - use signal_hook::consts::{SIGHUP, SIGINT}; - use tokio::time::{self, Duration}; - - trace!("handle_signals(signals, {build_id}, {client:#?})"); - - while let Some(signal) = signals.next().await { - match signal { - SIGHUP => (), - SIGINT => { - kill.send(()).unwrap(); - info!("Recieved SIGINT, cleaning up build..."); - - time::sleep(Duration::from_secs(1)).await; - - let containers = client - .containers() - .list(&ContainerListOpts::builder().sync(true).all(true).build()) - .await?; - - trace!("{containers:#?}"); - - // Prune containers from this build - let container_prune_opts = ContainerPruneOpts::builder() - .filter([ContainerPruneFilter::LabelKeyVal( - BUILD_ID_LABEL.to_string(), - build_id.to_string(), - )]) - .build(); - client.containers().prune(&container_prune_opts).await?; - debug!("Pruned containers"); - - // Prune images from this build - let image_prune_opts = ImagePruneOpts::builder() - .filter([ImagePruneFilter::LabelKeyVal( - BUILD_ID_LABEL.to_string(), - build_id.to_string(), - )]) - .build(); - client.images().prune(&image_prune_opts).await?; - debug!("Pruned images"); - process::exit(2); - } - _ => unreachable!(), - } - } - - Ok(()) -} - -fn tag_images(tags: &[String], image_name: &str, full_image: &str) -> Result<()> { - debug!("Tagging all images"); - - for tag in tags { - debug!("Tagging {image_name} with {tag}"); - - let tag_image = format!("{image_name}:{tag}"); - - let status = match ( - blue_build_utils::check_command_exists("buildah"), - blue_build_utils::check_command_exists("podman"), - ) { - (Ok(()), _) => { - trace!("buildah tag {full_image} {tag_image}"); - Command::new("buildah") - } - (Err(_), Ok(())) => { - trace!("podman tag {full_image} {tag_image}"); - Command::new("podman") - } - (Err(e1), Err(e2)) => { - bail!("Need either 'buildah' or 'podman' to build: {e1}, {e2}") - } - } - .arg("tag") - .arg(full_image) - .arg(&tag_image) - .status()?; - - if status.success() { - info!("Successfully tagged {image_name}:{tag}!"); - } else { - bail!("Failed to tag image {image_name}:{tag}"); - } - } - - Ok(()) -} - -fn push_images(tags: &[String], image_name: &str) -> Result<()> { - debug!("Pushing all images"); - for tag in tags { - debug!("Pushing image {image_name}:{tag}"); - - let tag_image = format!("{image_name}:{tag}"); - - let status = match ( - blue_build_utils::check_command_exists("buildah"), - blue_build_utils::check_command_exists("podman"), - ) { - (Ok(()), _) => { - trace!("buildah push {tag_image}"); - Command::new("buildah") - } - (Err(_), Ok(())) => { - trace!("podman push {tag_image}"); - Command::new("podman") - } - (Err(e1), Err(e2)) => { - bail!("Need either 'buildah' or 'podman' to build: {e1}, {e2}") - } - } - .arg("push") - .arg(&tag_image) - .status()?; - - if status.success() { - info!("Successfully pushed {image_name}:{tag}!"); - } else { - bail!("Failed to push image {image_name}:{tag}"); - } - } - - Ok(()) -} - -#[cfg(feature = "podman-api")] -async fn push_images_podman_api( - tags: &[String], - image_name: &str, - first_image_name: &str, - client: &Podman, - credentials: &Credentials, -) -> Result<()> { - use podman_api::opts::ImageTagOpts; - - let first_image = client.images().get(first_image_name); - let (registry, username, password) = ( - &credentials.registry, - &credentials.username, - &credentials.password, - ); - - for tag in tags { - let full_image_name = format!("{image_name}:{tag}"); - - first_image - .tag(&ImageTagOpts::builder().repo(image_name).tag(tag).build()) - .await?; - debug!("Tagged image {full_image_name}"); - - let new_image = client.images().get(&full_image_name); - - info!("Pushing {full_image_name}"); - match new_image - .push( - &ImagePushOpts::builder() - .tls_verify(true) - .auth( - RegistryAuth::builder() - .username(username) - .password(password) - .server_address(registry) - .build(), - ) - .build(), - ) - .await - { - Ok(_) => info!("Pushed {full_image_name} successfully!"), - Err(e) => bail!("Failed to push image: {e}"), - } - } - Ok(()) -} diff --git a/src/commands/build/build_strategy.rs b/src/commands/build/build_strategy.rs deleted file mode 100644 index 5facf8f..0000000 --- a/src/commands/build/build_strategy.rs +++ /dev/null @@ -1,56 +0,0 @@ -use std::{ - env, - path::{Path, PathBuf}, -}; - -use anyhow::{bail, Result}; -use blue_build_utils::constants::*; -use log::trace; - -#[cfg(feature = "podman-api")] -#[derive(Debug, Clone, Default)] -pub enum BuildStrategy { - #[default] - Uninitialized, - Socket(PathBuf), - Buildah, - Podman, -} - -#[cfg(feature = "podman-api")] -impl BuildStrategy { - pub fn determine_strategy() -> Result { - trace!("BuildStrategy::determin_strategy()"); - - Ok( - match ( - env::var(XDG_RUNTIME_DIR), - PathBuf::from(RUN_PODMAN_SOCK), - PathBuf::from(VAR_RUN_PODMAN_PODMAN_SOCK), - PathBuf::from(VAR_RUN_PODMAN_SOCK), - blue_build_utils::check_command_exists("buildah"), - blue_build_utils::check_command_exists("podman"), - ) { - (Ok(xdg_runtime), _, _, _, _, _) - if Path::new(&format!("{xdg_runtime}/podman/podman.sock")).exists() => - { - Self::Socket(PathBuf::from(format!("{xdg_runtime}/podman/podman.sock"))) - } - (_, run_podman_podman_sock, _, _, _, _) if run_podman_podman_sock.exists() => { - Self::Socket(run_podman_podman_sock) - } - (_, _, var_run_podman_podman_sock, _, _, _) - if var_run_podman_podman_sock.exists() => - { - Self::Socket(var_run_podman_podman_sock) - } - (_, _, _, var_run_podman_sock, _, _) if var_run_podman_sock.exists() => { - Self::Socket(var_run_podman_sock) - } - (_, _, _, _, Ok(()), _) => Self::Buildah, - (_, _, _, _, _, Ok(())) => Self::Podman, - _ => bail!("Could not determine strategy"), - }, - ) - } -} diff --git a/src/lib.rs b/src/lib.rs index f12e4e3..68dd733 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,3 +4,4 @@ shadow_rs::shadow!(shadow); pub mod commands; +pub mod strategies; diff --git a/src/strategies.rs b/src/strategies.rs new file mode 100644 index 0000000..eb7cca2 --- /dev/null +++ b/src/strategies.rs @@ -0,0 +1,121 @@ +use std::{env, path::PathBuf, rc::Rc}; + +use anyhow::{bail, Result}; +use blue_build_utils::constants::*; +use log::trace; +use uuid::Uuid; + +#[cfg(feature = "podman-api")] +use podman_api::Podman; + +#[cfg(feature = "tokio")] +use tokio::runtime::Runtime; + +use crate::{ + commands::build::Credentials, + strategies::{ + buildah_strategy::BuildahStrategy, docker_strategy::DockerStrategy, + podman_strategy::PodmanStrategy, + }, +}; + +#[cfg(feature = "builtin-podman")] +use crate::strategies::podman_api_strategy::PodmanApiStrategy; + +mod buildah_strategy; +mod docker_strategy; +#[cfg(feature = "builtin-podman")] +mod podman_api_strategy; +mod podman_strategy; + +pub trait BuildStrategy { + fn build(&self, image: &str) -> Result<()>; + + fn tag(&self, src_image: &str, image_name: &str, tag: &str) -> Result<()>; + + fn push(&self, image: &str) -> Result<()>; + + fn login(&self) -> Result<()>; +} + +pub fn determine_build_strategy( + uuid: Uuid, + creds: Option, + oci_required: bool, +) -> Result> { + trace!("BuildStrategy::determine_strategy({uuid})"); + + Ok( + match ( + env::var(XDG_RUNTIME_DIR), + PathBuf::from(RUN_PODMAN_SOCK), + PathBuf::from(VAR_RUN_PODMAN_PODMAN_SOCK), + PathBuf::from(VAR_RUN_PODMAN_SOCK), + blue_build_utils::check_command_exists("docker"), + blue_build_utils::check_command_exists("podman"), + blue_build_utils::check_command_exists("buildah"), + ) { + #[cfg(feature = "builtin-podman")] + (Ok(xdg_runtime), _, _, _, _, _, _) + if PathBuf::from(format!("{xdg_runtime}/podman/podman.sock")).exists() => + { + Rc::new( + PodmanApiStrategy::builder() + .client( + Podman::unix(PathBuf::from(format!( + "{xdg_runtime}/podman/podman.sock" + ))) + .into(), + ) + .rt(Runtime::new()?) + .uuid(uuid) + .creds(creds) + .build(), + ) + } + #[cfg(feature = "builtin-podman")] + (_, run_podman_podman_sock, _, _, _, _, _) if run_podman_podman_sock.exists() => { + Rc::new( + PodmanApiStrategy::builder() + .client(Podman::unix(run_podman_podman_sock).into()) + .rt(Runtime::new()?) + .uuid(uuid) + .creds(creds) + .build(), + ) + } + #[cfg(feature = "builtin-podman")] + (_, _, var_run_podman_podman_sock, _, _, _, _) + if var_run_podman_podman_sock.exists() => + { + Rc::new( + PodmanApiStrategy::builder() + .client(Podman::unix(var_run_podman_podman_sock).into()) + .rt(Runtime::new()?) + .uuid(uuid) + .creds(creds) + .build(), + ) + } + #[cfg(feature = "builtin-podman")] + (_, _, _, var_run_podman_sock, _, _, _) if var_run_podman_sock.exists() => Rc::new( + PodmanApiStrategy::builder() + .client(Podman::unix(var_run_podman_sock).into()) + .rt(Runtime::new()?) + .uuid(uuid) + .creds(creds) + .build(), + ), + (_, _, _, _, Ok(_docker), _, _) if !oci_required => { + Rc::new(DockerStrategy::builder().creds(creds).build()) + } + (_, _, _, _, _, Ok(_podman), _) => { + Rc::new(PodmanStrategy::builder().creds(creds).build()) + } + (_, _, _, _, _, _, Ok(_buildah)) => { + Rc::new(BuildahStrategy::builder().creds(creds).build()) + } + _ => bail!("Could not determine strategy"), + }, + ) +} diff --git a/src/strategies/buildah_strategy.rs b/src/strategies/buildah_strategy.rs new file mode 100644 index 0000000..016d83b --- /dev/null +++ b/src/strategies/buildah_strategy.rs @@ -0,0 +1,91 @@ +use std::process::Command; + +use anyhow::{anyhow, bail, Result}; +use log::{info, trace}; +use typed_builder::TypedBuilder; + +use crate::commands::build::Credentials; + +use super::BuildStrategy; + +#[derive(Debug, TypedBuilder)] +pub struct BuildahStrategy { + creds: Option, +} + +impl BuildStrategy for BuildahStrategy { + fn build(&self, image: &str) -> Result<()> { + trace!("buildah build -t {image}"); + let status = Command::new("buildah") + .arg("build") + .arg("-t") + .arg(image) + .status()?; + + if status.success() { + info!("Successfully built {image}"); + } else { + bail!("Failed to build {image}"); + } + Ok(()) + } + + fn tag(&self, src_image: &str, image_name: &str, tag: &str) -> Result<()> { + let dest_image = format!("{image_name}:{tag}"); + trace!("buildah tag {src_image} {dest_image}"); + let status = Command::new("buildah") + .arg("tag") + .arg(src_image) + .arg(&dest_image) + .status()?; + + if status.success() { + info!("Successfully tagged {dest_image}!"); + } else { + bail!("Failed to tag image {dest_image}"); + } + Ok(()) + } + + fn push(&self, image: &str) -> Result<()> { + trace!("buildah push {image}"); + let status = Command::new("buildah").arg("push").arg(image).status()?; + + if status.success() { + info!("Successfully pushed {image}!"); + } else { + bail!("Failed to push image {image}") + } + Ok(()) + } + + fn login(&self) -> Result<()> { + let (registry, username, password) = self + .creds + .as_ref() + .map(|credentials| { + ( + &credentials.registry, + &credentials.username, + &credentials.password, + ) + }) + .ok_or_else(|| anyhow!("Unable to login, missing credentials!"))?; + + trace!("buildah login -u {username} -p [MASKED] {registry}"); + let output = Command::new("buildah") + .arg("login") + .arg("-u") + .arg(username) + .arg("-p") + .arg(password) + .arg(registry) + .output()?; + + if !output.status.success() { + let err_out = String::from_utf8_lossy(&output.stderr); + bail!("Failed to login for buildah: {err_out}"); + } + Ok(()) + } +} diff --git a/src/strategies/docker_strategy.rs b/src/strategies/docker_strategy.rs new file mode 100644 index 0000000..307d977 --- /dev/null +++ b/src/strategies/docker_strategy.rs @@ -0,0 +1,122 @@ +use std::{env, process::Command}; + +use anyhow::{anyhow, bail, Result}; +use blue_build_utils::constants::*; +use log::{info, trace}; +use typed_builder::TypedBuilder; + +use crate::commands::build::Credentials; + +use super::BuildStrategy; + +#[derive(Debug, TypedBuilder)] +pub struct DockerStrategy { + creds: Option, +} + +impl BuildStrategy for DockerStrategy { + fn build(&self, image: &str) -> Result<()> { + let docker_help = Command::new("docker") + .arg("build") + .arg("--help") + .output()? + .stdout; + let docker_help = String::from_utf8_lossy(&docker_help); + + trace!("docker"); + let mut command = Command::new("docker"); + + if docker_help.lines().filter(|l| l.contains("buildx")).count() > 0 { + trace!("buildx build --load"); + command.arg("buildx").arg("build").arg("--load"); + } else { + trace!("build"); + command.arg("build"); + } + + // https://github.com/moby/buildkit?tab=readme-ov-file#github-actions-cache-experimental + if env::var(BB_BUILDKIT_CACHE_GHA).map_or_else(|_| false, |e| e == "true") { + trace!("--cache-from type=gha --cache-to type=gha"); + command + .arg("--cache-from") + .arg("type=gha") + .arg("--cache-to") + .arg("type=gha"); + } + + trace!("-t {image} -f Containerfile ."); + command + .arg("-t") + .arg(image) + .arg("-f") + .arg("Containerfile") + .arg("."); + + if command.status()?.success() { + info!("Successfully built {image}"); + } else { + bail!("Failed to build {image}"); + } + Ok(()) + } + + fn tag(&self, src_image: &str, image_name: &str, tag: &str) -> Result<()> { + let dest_image = format!("{image_name}:{tag}"); + + trace!("docker tag {src_image} {dest_image}"); + let status = Command::new("docker") + .arg("tag") + .arg(src_image) + .arg(&dest_image) + .status()?; + + if status.success() { + info!("Successfully tagged {dest_image}!"); + } else { + bail!("Failed to tag image {dest_image}"); + } + Ok(()) + } + + fn push(&self, image: &str) -> Result<()> { + trace!("docker push {image}"); + let status = Command::new("docker").arg("push").arg(image).status()?; + + if status.success() { + info!("Successfully pushed {image}!"); + } else { + bail!("Failed to push image {image}") + } + Ok(()) + } + + fn login(&self) -> Result<()> { + let (registry, username, password) = self + .creds + .as_ref() + .map(|credentials| { + ( + &credentials.registry, + &credentials.username, + &credentials.password, + ) + }) + .ok_or_else(|| anyhow!("Unable to login, missing credentials!"))?; + + trace!("docker login -u {username} -p [MASKED] {registry}"); + let output = Command::new("docker") + .arg("login") + .arg("-u") + .arg(username) + .arg("-p") + .arg(password) + .arg(registry) + .output()?; + + if !output.status.success() { + let err_out = String::from_utf8_lossy(&output.stderr); + bail!("Failed to login for buildah: {err_out}"); + } + Ok(()) + } +} diff --git a/src/strategies/podman_api_strategy.rs b/src/strategies/podman_api_strategy.rs new file mode 100644 index 0000000..5857379 --- /dev/null +++ b/src/strategies/podman_api_strategy.rs @@ -0,0 +1,208 @@ +use anyhow::Context; +use anyhow::{anyhow, bail, Result}; +use blue_build_utils::constants::*; +use futures_util::StreamExt; +use log::{debug, error}; +use log::{info, trace}; +use podman_api::{ + opts::{ + ContainerListOpts, ContainerPruneFilter, ContainerPruneOpts, ImageBuildOpts, + ImagePruneFilter, ImagePruneOpts, ImagePushOpts, ImageTagOpts, RegistryAuth, + }, + Podman, +}; +use signal_hook::consts::{SIGHUP, SIGINT, SIGQUIT, SIGTERM}; +use signal_hook_tokio::Signals; +use std::sync::Arc; +use tokio::{ + runtime::Runtime, + sync::oneshot::{self, Sender}, + time::{self, Duration}, +}; +use typed_builder::TypedBuilder; +use uuid::Uuid; + +use crate::commands::build::Credentials; + +use super::BuildStrategy; + +#[derive(Debug, TypedBuilder)] +pub struct PodmanApiStrategy { + client: Arc, + rt: Runtime, + uuid: Uuid, + creds: Option, +} + +impl BuildStrategy for PodmanApiStrategy { + fn build(&self, image: &str) -> Result<()> { + self.rt.block_on(async { + let signals = Signals::new([SIGTERM, SIGINT, SIGQUIT])?; + let handle = signals.handle(); + + let (kill_tx, mut kill_rx) = oneshot::channel::<()>(); + + let signals_task = tokio::spawn(handle_signals( + signals, + kill_tx, + self.uuid, + self.client.clone(), + )); + + // Get podman ready to build + let opts = ImageBuildOpts::builder(".") + .tag(image) + .dockerfile("Containerfile") + .remove(true) + .layers(true) + .labels([(BUILD_ID_LABEL, self.uuid.to_string())]) + .pull(true) + .build(); + trace!("Build options: {opts:#?}"); + + info!("Building image {image}"); + match self.client.images().build(&opts) { + Ok(mut build_stream) => loop { + tokio::select! { + Some(chunk) = build_stream.next() => { + match chunk { + Ok(chunk) => chunk + .stream + .trim() + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .for_each(|line| info!("{line}")), + Err(e) => bail!("{e}"), + } + }, + _ = &mut kill_rx => { + break; + }, + else => { + break; + } + } + }, + Err(e) => bail!("{e}"), + }; + handle.close(); + signals_task.await?; + Ok(()) + }) + } + + fn tag(&self, src_image: &str, image_name: &str, tag: &str) -> Result<()> { + let first_image = self.client.images().get(src_image); + self.rt.block_on(async { + first_image + .tag(&ImageTagOpts::builder().repo(image_name).tag(tag).build()) + .await + .context("Failed to tag image")?; + debug!("Tagged image {image_name}:{tag}"); + Ok(()) + }) + } + + fn push(&self, image: &str) -> Result<()> { + let (username, password, registry) = self + .creds + .as_ref() + .map(|c| (&c.username, &c.password, &c.registry)) + .ok_or_else(|| anyhow!("No credentials provided, unable to push"))?; + + self.rt.block_on(async { + let new_image = self.client.images().get(image); + info!("Pushing {image}"); + match new_image + .push( + &ImagePushOpts::builder() + .tls_verify(true) + .auth( + RegistryAuth::builder() + .username(username) + .password(password) + .server_address(registry) + .build(), + ) + .build(), + ) + .await + { + Ok(_) => info!("Pushed {image} successfully!"), + Err(e) => bail!("Failed to push image: {e}"), + }; + Ok(()) + }) + } + + fn login(&self) -> Result<()> { + debug!("No login step for Socket based building, skipping..."); + Ok(()) + } +} + +async fn handle_signals( + mut signals: Signals, + kill: Sender<()>, + build_id: Uuid, + client: Arc, +) { + use std::process; + + trace!("handle_signals(signals, {build_id}, {client:#?})"); + + while let Some(signal) = signals.next().await { + match signal { + SIGHUP => (), + SIGINT => { + kill.send(()).unwrap(); + info!("Recieved SIGINT, cleaning up build..."); + + time::sleep(Duration::from_secs(1)).await; + + let containers = match client + .containers() + .list(&ContainerListOpts::builder().sync(true).all(true).build()) + .await + { + Ok(list) => list, + Err(e) => { + error!("{e}"); + process::exit(1); + } + }; + + trace!("{containers:#?}"); + + // Prune containers from this build + let container_prune_opts = ContainerPruneOpts::builder() + .filter([ContainerPruneFilter::LabelKeyVal( + BUILD_ID_LABEL.to_string(), + build_id.to_string(), + )]) + .build(); + if let Err(e) = client.containers().prune(&container_prune_opts).await { + error!("{e}"); + process::exit(1); + } + debug!("Pruned containers"); + + // Prune images from this build + let image_prune_opts = ImagePruneOpts::builder() + .filter([ImagePruneFilter::LabelKeyVal( + BUILD_ID_LABEL.to_string(), + build_id.to_string(), + )]) + .build(); + if let Err(e) = client.images().prune(&image_prune_opts).await { + error!("{e}"); + process::exit(1); + } + debug!("Pruned images"); + process::exit(2); + } + _ => unreachable!(), + } + } +} diff --git a/src/strategies/podman_strategy.rs b/src/strategies/podman_strategy.rs new file mode 100644 index 0000000..1aac6a4 --- /dev/null +++ b/src/strategies/podman_strategy.rs @@ -0,0 +1,93 @@ +use std::process::Command; + +use anyhow::{anyhow, bail, Result}; +use log::{info, trace}; +use typed_builder::TypedBuilder; + +use crate::commands::build::Credentials; + +use super::BuildStrategy; + +#[derive(Debug, TypedBuilder)] +pub struct PodmanStrategy { + creds: Option, +} + +impl BuildStrategy for PodmanStrategy { + fn build(&self, image: &str) -> Result<()> { + trace!("podman build . -t {image}"); + let status = Command::new("podman") + .arg("build") + .arg(".") + .arg("-t") + .arg(image) + .status()?; + + if status.success() { + info!("Successfully built {image}"); + } else { + bail!("Failed to build {image}"); + } + Ok(()) + } + + fn tag(&self, src_image: &str, image_name: &str, tag: &str) -> Result<()> { + let dest_image = format!("{image_name}:{tag}"); + + trace!("podman tag {src_image} {dest_image}"); + let status = Command::new("podman") + .arg("tag") + .arg(src_image) + .arg(&dest_image) + .status()?; + + if status.success() { + info!("Successfully tagged {dest_image}!"); + } else { + bail!("Failed to tag image {dest_image}"); + } + Ok(()) + } + + fn push(&self, image: &str) -> Result<()> { + trace!("podman push {image}"); + let status = Command::new("podman").arg("push").arg(image).status()?; + + if status.success() { + info!("Successfully pushed {image}!"); + } else { + bail!("Failed to push image {image}") + } + Ok(()) + } + + fn login(&self) -> Result<()> { + let (registry, username, password) = self + .creds + .as_ref() + .map(|credentials| { + ( + &credentials.registry, + &credentials.username, + &credentials.password, + ) + }) + .ok_or_else(|| anyhow!("Unable to login, missing credentials!"))?; + + trace!("podman login -u {username} -p [MASKED] {registry}"); + let output = Command::new("podman") + .arg("login") + .arg("-u") + .arg(username) + .arg("-p") + .arg(password) + .arg(registry) + .output()?; + + if !output.status.success() { + let err_out = String::from_utf8_lossy(&output.stderr); + bail!("Failed to login for buildah: {err_out}"); + } + Ok(()) + } +} diff --git a/utils/src/constants.rs b/utils/src/constants.rs index bebf4ec..576fb5d 100644 --- a/utils/src/constants.rs +++ b/utils/src/constants.rs @@ -13,6 +13,9 @@ pub const VAR_RUN_PODMAN_SOCK: &str = "/var/run/podman.sock"; // Labels pub const BUILD_ID_LABEL: &str = "org.blue-build.build-id"; +// BlueBuild vars +pub const BB_BUILDKIT_CACHE_GHA: &str = "BB_BUILDKIT_CACHE_GHA"; + // Cosign vars pub const COSIGN_PRIVATE_KEY: &str = "COSIGN_PRIVATE_KEY"; pub const GITHUB_TOKEN_ISSUER_URL: &str = "https://token.actions.githubusercontent.com";