diff --git a/Cargo.lock b/Cargo.lock index 887dc3a..88323b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -288,6 +288,7 @@ dependencies = [ "derive_builder", "env_logger", "futures-util", + "indexmap 2.1.0", "log", "podman-api", "rusty-hook", diff --git a/Cargo.toml b/Cargo.toml index 7ba0d34..0141d0c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ clap-verbosity-flag = "2.1.1" derive_builder = "0.12.0" env_logger = "0.10.1" futures-util = { version = "0.3.30", optional = true } +indexmap = { version = "2.1.0", features = ["serde"] } log = "0.4.20" podman-api = { version = "0.10.0", optional = true } serde = { version = "1.0.188", features = ["derive"] } diff --git a/Earthfile b/Earthfile index f1c68f7..9d97f32 100644 --- a/Earthfile +++ b/Earthfile @@ -23,6 +23,8 @@ default: BUILD +installer --NIGHTLY=$NIGHTLY BUILD +integration-test-template --NIGHTLY=$NIGHTLY BUILD +integration-test-build --NIGHTLY=$NIGHTLY + BUILD +integration-test-rebase --NIGHTLY=$NIGHTLY + BUILD +integration-test-upgrade --NIGHTLY=$NIGHTLY nightly: BUILD +default --NIGHTLY=true @@ -124,7 +126,20 @@ integration-test-build: ARG NIGHTLY=false FROM +integration-test-base --NIGHTLY=$NIGHTLY - RUN --entrypoint --privileged podman info && bb -vv build config/recipe-jp-desktop.yml + RUN --privileged bb -vv build config/recipe-jp-desktop.yml + +integration-test-rebase: + ARG NIGHTLY=false + FROM +integration-test-base --NIGHTLY=$NIGHTLY + + RUN --privileged bb -vv rebase config/recipe-jp-desktop.yml + +integration-test-upgrade: + ARG NIGHTLY=false + FROM +integration-test-base --NIGHTLY=$NIGHTLY + RUN mkdir -p /etc/blue-build && touch /etc/blue-build/jp-desktop.tar.gz + + RUN --privileged bb -vv upgrade config/recipe-jp-desktop.yml integration-test-base: ARG NIGHTLY=false @@ -132,10 +147,16 @@ integration-test-base: FROM +blue-build-cli-alpine --NIGHTLY=$NIGHTLY RUN echo "#!/bin/sh - echo 'Running podman'" > /usr/bin/podman + echo 'Running podman'" > /usr/bin/podman \ + && chmod +x /usr/bin/podman RUN echo "#!/bin/sh - echo 'Running buildah'" > /usr/bin/buildah + echo 'Running buildah'" > /usr/bin/buildah \ + && chmod +x /usr/bin/buildah + + RUN echo "#!/bin/sh + echo 'Running rpm-ostree'" > /usr/bin/rpm-ostree \ + && chmod +x /usr/bin/rpm-ostree GIT CLONE https://gitlab.com/wunker-bunker/wunker-os.git /test WORKDIR /test diff --git a/src/bin/bb.rs b/src/bin/bb.rs index 29f3f57..3ab3ba8 100644 --- a/src/bin/bb.rs +++ b/src/bin/bb.rs @@ -1,4 +1,4 @@ -use blue_build::{self, build, template}; +use blue_build::{self, build, local, template}; use clap::{Parser, Subcommand}; use clap_verbosity_flag::{InfoLevel, Verbosity}; use env_logger::WriteStyle; @@ -25,6 +25,29 @@ enum CommandArgs { /// Generate a Containerfile from a recipe Template(template::TemplateCommand), + /// Upgrade your current OS with the + /// local image saved at `/etc/blue-build/`. + /// + /// This requires having rebased already onto + /// a local archive already by using the `rebase` + /// subcommand. + /// + /// NOTE: This can only be used if you have `rpm-ostree` + /// installed and if the `--push` and `--rebase` option isn't + /// used. This image will not be signed. + Upgrade(local::UpgradeCommand), + + /// Rebase your current OS onto the image + /// being built. + /// + /// This will create a tarball of your image at + /// `/etc/blue-build/` and invoke `rpm-ostree` to + /// rebase onto the image using `oci-archive`. + /// + /// NOTE: This can only be used if you have `rpm-ostree` + /// installed. + Rebase(local::RebaseCommand), + /// Initialize a new Ublue Starting Point repo #[cfg(feature = "init")] Init(init::InitCommand), @@ -46,8 +69,9 @@ fn main() { match args.command { CommandArgs::Build(mut command) => command.run(), - CommandArgs::Template(command) => command.run(), + CommandArgs::Upgrade(command) => command.run(), + CommandArgs::Rebase(command) => command.run(), #[cfg(feature = "init")] CommandArgs::Init(command) => command.run(), diff --git a/src/build.rs b/src/build.rs index c27dcc4..9b2ee11 100644 --- a/src/build.rs +++ b/src/build.rs @@ -11,7 +11,6 @@ use anyhow::{anyhow, bail, Result}; use clap::Args; use log::{debug, error, info, trace, warn}; use typed_builder::TypedBuilder; -use users::{Users, UsersCache}; #[cfg(feature = "podman-api")] use podman_api::{ @@ -30,40 +29,31 @@ use futures_util::StreamExt; use tokio::runtime::Runtime; use crate::{ - ops, + ops::{self, ARCHIVE_SUFFIX}, template::{Recipe, TemplateCommand}, }; -const LOCAL_BUILD: &str = "/etc/blue-build"; - #[derive(Debug, Clone, Args, TypedBuilder)] pub struct BuildCommand { /// The recipe file to build an image #[arg()] recipe: PathBuf, - /// Rebase your current OS onto the image - /// being built. - /// - /// This will create a tarball of your image at - /// `/etc/blue-build/` and invoke `rpm-ostree` to - /// rebase onto the image using `oci-archive`. - /// - /// NOTE: This can only be used if you have `rpm-ostree` - /// installed and if the `--push` option isn't - /// used. This image will not be signed. - #[arg(short, long)] - rebase: bool, - /// Push the image with all the tags. /// - /// Requires `--registry`, `--registry-path`, + /// Requires `--registry`, /// `--username`, and `--password` if not /// building in CI. #[arg(short, long)] #[builder(default)] push: bool, + /// Archives the built image into a tarfile + /// in the specified directory. + #[arg(short, long)] + #[builder(default, setter(into, strip_option))] + archive: Option, + /// The registry's domain name. #[arg(long)] #[builder(default, setter(into, strip_option))] @@ -142,8 +132,8 @@ impl BuildCommand { pub fn try_run(&mut self) -> Result<()> { trace!("BuildCommand::try_run()"); - if self.push && self.rebase { - bail!("You cannot use '--rebase' and '--push' at the same time"); + if self.push && self.archive.is_some() { + bail!("You cannot use '--archive' and '--push' at the same time"); } #[cfg(not(feature = "podman-api"))] @@ -153,18 +143,6 @@ impl BuildCommand { })?; } - if self.rebase { - ops::check_command_exists("rpm-ostree")?; - - let cache = UsersCache::new(); - - if cache.get_current_uid() != 0 { - bail!("You need to be root to rebase a local image! Try using 'sudo'."); - } - - clean_local_build_dir()?; - } - if self.push { ops::check_command_exists("cosign")?; ops::check_command_exists("skopeo")?; @@ -182,8 +160,7 @@ impl BuildCommand { #[cfg(feature = "podman-api")] match BuildStrategy::determine_strategy()? { BuildStrategy::Socket(socket) => { - let rt = Runtime::new()?; - rt.block_on(self.build_image_podman_api(Podman::unix(socket))) + Runtime::new()?.block_on(self.build_image_podman_api(Podman::unix(socket))) } _ => self.build_image(), } @@ -200,6 +177,7 @@ impl BuildCommand { error!("Failed to build image: {e}"); process::exit(1); } + info!("Finished building!"); } #[cfg(feature = "podman-api")] @@ -253,10 +231,17 @@ impl BuildCommand { // Get values for image let tags = recipe.generate_tags(); let image_name = self.generate_full_image_name(&recipe)?; - let first_image_name = if tags.is_empty() || self.rebase { - image_name.clone() - } else { - format!("{}:{}", &image_name, &tags[0]) + let first_image_name = match &self.archive { + Some(archive_dir) => format!( + "oci-archive:{}", + archive_dir + .join(format!("{image_name}{ARCHIVE_SUFFIX}")) + .display() + ), + None => tags + .first() + .map(|t| format!("{image_name}:{t}")) + .unwrap_or(image_name.to_string()), }; debug!("Full tag is {first_image_name}"); @@ -270,21 +255,29 @@ impl BuildCommand { .build(); trace!("Build options: {opts:#?}"); + info!("Building image {first_image_name}"); match client.images().build(&opts) { Ok(mut build_stream) => { while let Some(chunk) = build_stream.next().await { match chunk { - Ok(chunk) => debug!("{}", chunk.stream.trim()), - Err(e) => error!("{}", e), + Ok(chunk) => chunk + .stream + .trim() + .lines() + .map(|line| line.trim()) + .filter(|line| !line.is_empty()) + .for_each(|line| info!("{line}")), + Err(e) => bail!("{e}"), } } } - Err(e) => error!("{}", e), + Err(e) => bail!("{e}"), }; if self.push { debug!("Pushing is enabled"); + info!("Logging into registry using cosign"); trace!("cosign login -u {username} -p [MASKED] {registry}"); if !Command::new("cosign") .arg("login") @@ -334,20 +327,7 @@ impl BuildCommand { } } - self.sign_images(&image_name, &tags[0])?; - } else if self.rebase { - debug!("Rebasing onto locally built image {image_name}"); - - if Command::new("rpm-ostree") - .arg("rebase") - .arg(format!("ostree-unverified-image:{first_image_name}")) - .status()? - .success() - { - info!("Successfully rebased to {first_image_name}"); - } else { - bail!("Failed to rebase to {first_image_name}"); - } + self.sign_images(&image_name, tags.first().map(|x| x.as_str()))?; } Ok(()) } @@ -367,10 +347,6 @@ impl BuildCommand { info!("Build complete!"); - if self.rebase { - info!("Be sure to restart your computer to use your new changes!"); - } - Ok(()) } @@ -411,6 +387,7 @@ impl BuildCommand { _ => bail!("Need '--password' set in order to login"), }; + info!("Logging into the registry, {registry}"); if !match ( ops::check_command_exists("buildah"), ops::check_command_exists("podman"), @@ -457,15 +434,12 @@ impl BuildCommand { Ok(()) } - fn generate_full_image_name(&self, recipe: &Recipe) -> Result { - info!("Generating full image name"); + pub fn generate_full_image_name(&self, recipe: &Recipe) -> Result { trace!("BuildCommand::generate_full_image_name({recipe:#?})"); + info!("Generating full image name"); - let image_name = if self.rebase { - let local_build_path = PathBuf::from(LOCAL_BUILD); - - let image_path = local_build_path.join(format!("{}.tar.gz", &recipe.name)); - format!("oci-archive:{}", image_path.display()) + let image_name = if self.archive.is_some() { + recipe.name.to_string() } else { match ( env::var("CI_REGISTRY").ok(), @@ -514,7 +488,7 @@ impl BuildCommand { } }; - info!("Using image name '{image_name}'"); + debug!("Using image name '{image_name}'"); Ok(image_name) } @@ -522,18 +496,20 @@ impl BuildCommand { fn run_build(&self, image_name: &str, tags: &[String]) -> Result<()> { trace!("BuildCommand::run_build({image_name}, {tags:#?})"); - let mut tags_iter = tags.iter(); - - let first_tag = tags_iter - .next() - .ok_or(anyhow!("We got here with no tags!?"))?; - - let full_image = if self.rebase { - image_name.to_owned() - } else { - format!("{image_name}:{first_tag}") + let full_image = match &self.archive { + Some(archive_dir) => format!( + "oci-archive:{}", + archive_dir + .join(format!("{image_name}{ARCHIVE_SUFFIX}")) + .display() + ), + None => tags + .first() + .map(|t| format!("{image_name}:{t}")) + .unwrap_or(image_name.to_string()), }; + info!("Building image {full_image}"); let status = match ( ops::check_command_exists("buildah"), ops::check_command_exists("podman"), @@ -564,10 +540,10 @@ impl BuildCommand { bail!("Failed to build {image_name}"); } - if tags.len() > 1 && !self.rebase { + if tags.len() > 1 && self.archive.is_none() { debug!("Tagging all images"); - for tag in tags_iter { + for tag in tags { debug!("Tagging {image_name} with {tag}"); let tag_image = format!("{image_name}:{tag}"); @@ -635,32 +611,22 @@ impl BuildCommand { } } - self.sign_images(image_name, first_tag)?; - } else if self.rebase { - debug!("Rebasing onto locally built image {image_name}"); - - if Command::new("rpm-ostree") - .arg("rebase") - .arg(format!("ostree-unverified-image:{full_image}")) - .status()? - .success() - { - info!("Successfully rebased to {full_image}"); - } else { - bail!("Failed to rebase to {full_image}"); - } + self.sign_images(image_name, tags.first().map(|x| x.as_str()))?; } Ok(()) } - fn sign_images(&self, image_name: &str, tag: &str) -> Result<()> { - trace!("BuildCommand::sign_images({image_name}, {tag})"); + fn sign_images(&self, image_name: &str, tag: Option<&str>) -> Result<()> { + trace!("BuildCommand::sign_images({image_name}, {tag:?})"); env::set_var("COSIGN_PASSWORD", ""); env::set_var("COSIGN_YES", "true"); let image_digest = get_image_digest(image_name, tag)?; + let image_name_tag = tag + .map(|t| format!("{image_name}:{t}")) + .unwrap_or(image_name.to_owned()); match ( env::var("CI_DEFAULT_BRANCH"), @@ -708,7 +674,7 @@ impl BuildCommand { let cert_oidc = format!("{ci_server_protocol}://{ci_server_host}"); - trace!("cosign verify --certificate-identity {cert_ident} --certificate-oidc-issuer {cert_oidc} {image_name}:{tag}"); + trace!("cosign verify --certificate-identity {cert_ident} --certificate-oidc-issuer {cert_oidc} {image_name_tag}"); if !Command::new("cosign") .arg("verify") @@ -716,7 +682,7 @@ impl BuildCommand { .arg(&cert_ident) .arg("--certificate-oidc-issuer") .arg(&cert_oidc) - .arg(&format!("{image_name}:{tag}")) + .arg(&image_name_tag) .status()? .success() { @@ -746,12 +712,12 @@ impl BuildCommand { bail!("Failed to sign image: {image_digest}"); } - trace!("cosign verify --key ./cosign.pub {image_name}:{tag}"); + trace!("cosign verify --key ./cosign.pub {image_name_tag}"); if !Command::new("cosign") .arg("verify") .arg("--key=./cosign.pub") - .arg(&format!("{image_name}:{tag}")) + .arg(&image_name_tag) .status()? .success() { @@ -765,10 +731,13 @@ impl BuildCommand { } } -fn get_image_digest(image_name: &str, tag: &str) -> Result { - trace!("get_image_digest({image_name}, {tag})"); +fn get_image_digest(image_name: &str, tag: Option<&str>) -> Result { + trace!("get_image_digest({image_name}, {tag:?})"); - let image_url = format!("docker://{image_name}:{tag}"); + let image_url = match tag { + Some(tag) => format!("docker://{image_name}:{tag}"), + None => format!("docker://{image_name}"), + }; trace!("skopeo inspect --format='{{.Digest}}' {image_url}"); let image_digest = String::from_utf8( @@ -831,33 +800,3 @@ fn check_cosign_files() -> Result<()> { } } } - -fn clean_local_build_dir() -> Result<()> { - trace!("clean_local_build_dir()"); - let local_build_path = Path::new(LOCAL_BUILD); - - if !local_build_path.exists() { - trace!( - "Creating build output dir at {}", - local_build_path.display() - ); - fs::create_dir_all(local_build_path)?; - } else { - debug!("Cleaning out build dir {LOCAL_BUILD}"); - - let entries = fs::read_dir(LOCAL_BUILD)?; - - for entry in entries { - let entry = entry?; - let path = entry.path(); - trace!("Found {}", path.display()); - - if path.is_file() && path.ends_with(".tar.gz") { - trace!("Removing {}", path.display()); - fs::remove_file(path)?; - } - } - } - - Ok(()) -} diff --git a/src/lib.rs b/src/lib.rs index d1c7f90..356c4e7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,5 +11,6 @@ pub mod init; pub mod build; +pub mod local; mod ops; pub mod template; diff --git a/src/local.rs b/src/local.rs new file mode 100644 index 0000000..4762e2e --- /dev/null +++ b/src/local.rs @@ -0,0 +1,211 @@ +use std::{ + fs, + path::{Path, PathBuf}, + process::{self, Command}, +}; + +use anyhow::{bail, Result}; +use clap::{Args, Subcommand}; +use log::{debug, error, info, trace}; +use typed_builder::TypedBuilder; +use users::{Users, UsersCache}; + +use crate::{ + build::BuildCommand, + ops::{self, ARCHIVE_SUFFIX, LOCAL_BUILD}, + template::Recipe, +}; + +#[derive(Default, Clone, Debug, TypedBuilder, Args)] +pub struct LocalCommonArgs { + /// The recipe file to build an image. + #[arg()] + recipe: PathBuf, + + /// Reboot your system after + /// the update is complete. + #[arg(short, long)] + #[builder(default)] + reboot: bool, +} + +#[derive(Default, Clone, Debug, TypedBuilder, Args)] +pub struct UpgradeCommand { + #[clap(flatten)] + common: LocalCommonArgs, +} + +impl UpgradeCommand { + pub fn try_run(&self) -> Result<()> { + trace!("UpgradeCommand::try_run()"); + + check_can_run()?; + + let recipe: Recipe = + serde_yaml::from_str(fs::read_to_string(&self.common.recipe)?.as_str())?; + let mut build = BuildCommand::builder() + .recipe(self.common.recipe.clone()) + .archive(LOCAL_BUILD) + .build(); + + let image_name = build.generate_full_image_name(&recipe)?; + clean_local_build_dir(&image_name, false)?; + debug!("Image name is {image_name}"); + + build.try_run()?; + + info!("Upgrading from locally built image {image_name}"); + + let image_name = format!("ostree-unverified-image:{image_name}"); + + let status = if self.common.reboot { + debug!("Upgrading image {image_name} and rebooting"); + + Command::new("rpm-ostree") + .arg("upgrade") + .arg("--reboot") + .status()? + } else { + debug!("Upgrading image {image_name}"); + + Command::new("rpm-ostree").arg("upgrade").status()? + }; + + if status.success() { + info!("Successfully upgraded image {image_name}"); + } else { + bail!("Failed to upgrade image {image_name}"); + } + Ok(()) + } + + pub fn run(&self) { + trace!("UpgradeCommand::run()"); + + if let Err(e) = self.try_run() { + error!("Failed to upgrade image: {e}"); + process::exit(1); + } + } +} + +#[derive(Default, Clone, Debug, TypedBuilder, Args)] +pub struct RebaseCommand { + #[clap(flatten)] + common: LocalCommonArgs, +} + +impl RebaseCommand { + pub fn try_run(&self) -> Result<()> { + trace!("RebaseCommand::try_run()"); + + check_can_run()?; + + let recipe: Recipe = + serde_yaml::from_str(fs::read_to_string(&self.common.recipe)?.as_str())?; + let mut build = BuildCommand::builder() + .recipe(self.common.recipe.clone()) + .archive(LOCAL_BUILD) + .build(); + + let image_name = build.generate_full_image_name(&recipe)?; + clean_local_build_dir(&image_name, true)?; + debug!("Image name is {image_name}"); + + build.try_run()?; + + info!("Rebasing onto locally built image {image_name}"); + + let image_name = format!("ostree-unverified-image:{image_name}"); + + let status = if self.common.reboot { + debug!("Rebasing image {image_name} and rebooting"); + + Command::new("rpm-ostree") + .arg("rebase") + .arg("--reboot") + .arg(&image_name) + .status()? + } else { + debug!("Rebasing image {image_name}"); + + Command::new("rpm-ostree") + .arg("rebase") + .arg(&image_name) + .status()? + }; + + if status.success() { + info!("Successfully rebased to {image_name}"); + } else { + bail!("Failed to rebase to {image_name}"); + } + Ok(()) + } + + pub fn run(&self) { + trace!("RebaseCommand::run()"); + + if let Err(e) = self.try_run() { + error!("Failed to rebase onto new image: {e}"); + process::exit(1); + } + } +} + +fn check_can_run() -> Result<()> { + trace!("check_can_run()"); + + ops::check_command_exists("rpm-ostree")?; + + let cache = UsersCache::new(); + + if cache.get_current_uid() != 0 { + bail!("You need to be root to rebase a local image! Try using 'sudo'."); + } + Ok(()) +} + +fn clean_local_build_dir(image_name: &str, rebase: bool) -> Result<()> { + trace!("clean_local_build_dir()"); + + let local_build_path = Path::new(LOCAL_BUILD); + let image_file_name = format!("{image_name}.tar.gz"); + let image_file_path = local_build_path.join(image_file_name); + + if !image_file_path.exists() && !rebase { + bail!( + "Cannot upgrade {} as the image doesn't exist", + image_file_path.display() + ); + } + + if !local_build_path.exists() { + debug!( + "Creating build output dir at {}", + local_build_path.display() + ); + fs::create_dir_all(local_build_path)?; + } else { + debug!("Cleaning out build dir {LOCAL_BUILD}"); + + let entries = fs::read_dir(LOCAL_BUILD)?; + + for entry in entries { + let entry = entry?; + let path = entry.path(); + trace!("Found {}", path.display()); + + if path.is_file() && path.ends_with(ARCHIVE_SUFFIX) { + if !rebase && path == image_file_path { + debug!("Not rebasing, keeping {}", image_file_path.display()); + continue; + } + trace!("Removing {}", path.display()); + fs::remove_file(path)?; + } + } + } + + Ok(()) +} diff --git a/src/ops.rs b/src/ops.rs index b6a3f7d..e5828da 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -8,6 +8,9 @@ use anyhow::{anyhow, bail, Result}; use clap::ValueEnum; use log::{debug, trace}; +pub const LOCAL_BUILD: &str = "/etc/blue-build"; +pub const ARCHIVE_SUFFIX: &str = ".tar.gz"; + pub fn check_command_exists(command: &str) -> Result<()> { trace!("check_command_exists({command})"); debug!("Checking if {command} exists"); diff --git a/src/template.rs b/src/template.rs index d389cb5..8123445 100644 --- a/src/template.rs +++ b/src/template.rs @@ -9,6 +9,7 @@ use anyhow::Result; use askama::Template; use chrono::Local; use clap::Args; +use indexmap::IndexMap; use log::{debug, error, info, trace, warn}; use serde::{Deserialize, Serialize}; use serde_yaml::Value; @@ -40,11 +41,14 @@ pub struct Recipe { #[serde(alias = "image-version")] pub image_version: String, + #[serde(alias = "blue-build-tag")] + pub blue_build_tag: Option, + #[serde(flatten)] pub modules_ext: ModuleExt, #[serde(flatten)] - pub extra: HashMap, + pub extra: IndexMap, } impl Recipe { @@ -137,7 +141,7 @@ pub struct Module { pub from_file: Option, #[serde(flatten)] - pub config: HashMap, + pub config: IndexMap, } #[derive(Debug, Clone, Args, TypedBuilder)] diff --git a/templates/Containerfile b/templates/Containerfile index d7bcbe8..74b9a67 100644 --- a/templates/Containerfile +++ b/templates/Containerfile @@ -21,7 +21,12 @@ COPY --from=docker.io/mikefarah/yq /usr/bin/yq /usr/bin/yq COPY --from=gcr.io/projectsigstore/cosign /ko-app/cosign /usr/bin/cosign -COPY --from=registry.gitlab.com/wunker-bunker/blue-build:latest-installer /out/bb /usr/bin/bb +COPY --from=registry.gitlab.com/wunker-bunker/blue-build: +{%- if let Some(tag) = recipe.blue_build_tag -%} +{{ tag }} +{%- else -%} +latest-installer +{%- endif %} /out/bb /usr/bin/bb COPY config /tmp/config/