particle-os/particle-os-cli
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(rechunk): Add the ability to rechunk an image

Browse source
This commit is contained in:
Gerald Pinder 2024-11-29 00:16:10 -05:00
parent ffa1789422
commit b4fbac2a66
22 changed files with 1002 additions and 190 deletions
Download patch file Download diff file Expand all files Collapse all files

28
process/drivers/docker_driver.rs
View file

@ -33,7 +33,7 @@ use crate::{
types::Platform,
},
logging::CommandLogging,
signal_handler::{add_cid, remove_cid, ContainerId, ContainerRuntime},
signal_handler::{add_cid, remove_cid, ContainerRuntime, ContainerSignalId},
};
#[derive(Debug, Deserialize)]
@ -427,32 +427,34 @@ fn get_metadata_cache(opts: &GetMetadataOpts) -> Result<ImageMetadata> {
}
impl RunDriver for DockerDriver {
fn run(opts: &RunOpts) -> std::io::Result<ExitStatus> {
fn run(opts: &RunOpts) -> Result<ExitStatus> {
trace!("DockerDriver::run({opts:#?})");
let cid_path = TempDir::new()?;
let cid_path = TempDir::new().into_diagnostic()?;
let cid_file = cid_path.path().join("cid");
let cid = ContainerId::new(&cid_file, ContainerRuntime::Docker, false);
let cid = ContainerSignalId::new(&cid_file, ContainerRuntime::Docker, false);
add_cid(&cid);
let status = docker_run(opts, &cid_file).build_status(&*opts.image, "Running container")?;
let status = docker_run(opts, &cid_file)
.build_status(&*opts.image, "Running container")
.into_diagnostic()?;
remove_cid(&cid);
Ok(status)
}
fn run_output(opts: &RunOpts) -> std::io::Result<std::process::Output> {
fn run_output(opts: &RunOpts) -> Result<std::process::Output> {
trace!("DockerDriver::run({opts:#?})");
let cid_path = TempDir::new()?;
let cid_path = TempDir::new().into_diagnostic()?;
let cid_file = cid_path.path().join("cid");
let cid = ContainerId::new(&cid_file, ContainerRuntime::Docker, false);
let cid = ContainerSignalId::new(&cid_file, ContainerRuntime::Docker, false);
add_cid(&cid);
let output = docker_run(opts, &cid_file).output()?;
let output = docker_run(opts, &cid_file).output().into_diagnostic()?;
remove_cid(&cid);
@ -469,6 +471,7 @@ fn docker_run(opts: &RunOpts, cid_file: &Path) -> Command {
if opts.privileged => "--privileged",
if opts.remove => "--rm",
if opts.pull => "--pull=always",
if let Some(user) = opts.user.as_ref() => format!("--user={user}"),
for RunOptsVolume { path_or_vol_name, container_path } in opts.volumes.iter() => [
"--volume",
format!("{path_or_vol_name}:{container_path}"),
@ -477,13 +480,6 @@ fn docker_run(opts: &RunOpts, cid_file: &Path) -> Command {
"--env",
format!("{key}={value}"),
],
|command| {
match (opts.uid, opts.gid) {
(Some(uid), None) => cmd!(command, "-u", format!("{uid}")),
(Some(uid), Some(gid)) => cmd!(command, "-u", format!("{}:{}", uid, gid)),
_ => {}
}
},
&*opts.image,
for arg in opts.args.iter() => &**arg,
);

4
process/drivers/opts.rs
View file

@ -3,12 +3,16 @@ use clap::ValueEnum;
pub use build::*;
pub use ci::*;
pub use inspect::*;
#[cfg(feature = "rechunk")]
pub use rechunk::*;
pub use run::*;
pub use signing::*;
mod build;
mod ci;
mod inspect;
#[cfg(feature = "rechunk")]
mod rechunk;
mod run;
mod signing;

3
process/drivers/opts/build.rs
View file

@ -20,6 +20,9 @@ pub struct BuildOpts<'scope> {
#[builder(default)]
pub platform: Platform,
#[builder(default)]
pub host_network: bool,
}
#[derive(Debug, Clone, Builder)]

47
process/drivers/opts/rechunk.rs Normal file
View file

@ -0,0 +1,47 @@
use std::{borrow::Cow, path::Path};
use bon::Builder;
use crate::drivers::types::Platform;
use super::CompressionType;
#[derive(Debug, Clone, Builder)]
#[builder(on(Cow<'_, str>, into))]
pub struct RechunkOpts<'scope> {
pub image: Cow<'scope, str>,
#[builder(into)]
pub containerfile: Cow<'scope, Path>,
#[builder(default)]
pub platform: Platform,
pub version: Cow<'scope, str>,
pub name: Cow<'scope, str>,
pub description: Cow<'scope, str>,
pub base_digest: Cow<'scope, str>,
pub base_image: Cow<'scope, str>,
pub repo: Cow<'scope, str>,
/// The list of tags for the image being built.
#[builder(default, into)]
pub tags: Vec<Cow<'scope, str>>,
/// Enable pushing the image.
#[builder(default)]
pub push: bool,
/// Enable retry logic for pushing.
#[builder(default)]
pub retry_push: bool,
/// Number of times to retry pushing.
///
/// Defaults to 1.
#[builder(default = 1)]
pub retry_count: u8,
/// The compression type to use when pushing.
#[builder(default)]
pub compression: CompressionType,
}

5
process/drivers/opts/run.rs
View file

@ -15,8 +15,9 @@ pub struct RunOpts<'scope> {
#[builder(default, into)]
pub volumes: Vec<RunOptsVolume<'scope>>,
pub uid: Option<u32>,
pub gid: Option<u32>,
#[builder(into)]
pub user: Option<Cow<'scope, str>>,
#[builder(default)]
pub privileged: bool,

159
process/drivers/podman_driver.rs
View file

@ -10,7 +10,7 @@ use blue_build_utils::{cmd, credentials::Credentials};
use cached::proc_macro::cached;
use colored::Colorize;
use indicatif::{ProgressBar, ProgressStyle};
use log::{debug, error, info, trace, warn};
use log::{debug, error, info, trace};
use miette::{bail, miette, IntoDiagnostic, Report, Result};
use oci_distribution::Reference;
use semver::Version;
@ -24,7 +24,13 @@ use crate::{
BuildDriver, DriverVersion, InspectDriver, RunDriver,
},
logging::{CommandLogging, Logger},
signal_handler::{add_cid, remove_cid, ContainerId, ContainerRuntime},
signal_handler::{add_cid, remove_cid, ContainerRuntime, ContainerSignalId},
};
#[cfg(feature = "rechunk")]
use super::{
types::{ContainerId, MountId},
ContainerMountDriver, RechunkDriver,
};
#[derive(Deserialize, Debug, Clone)]
@ -136,6 +142,7 @@ impl BuildDriver for PodmanDriver {
opts.platform.to_string(),
],
"--pull=true",
if opts.host_network => "--net=host",
format!("--layers={}", !opts.squash),
"-f",
&*opts.containerfile,
@ -334,39 +341,151 @@ fn get_metadata_cache(opts: &GetMetadataOpts) -> Result<ImageMetadata> {
.inspect(|metadata| trace!("{metadata:#?}"))
}
#[cfg(feature = "rechunk")]
impl ContainerMountDriver for PodmanDriver {
fn create_container(image: &Reference) -> Result<ContainerId> {
let output = {
let c = cmd!("podman", "create", image.to_string(), "bash");
trace!("{c:?}");
c
}
.output()
.into_diagnostic()?;
if !output.status.success() {
bail!("Failed to create a container from image {image}");
}
Ok(ContainerId(
String::from_utf8(output.stdout.trim_ascii().to_vec()).into_diagnostic()?,
))
}
fn remove_container(container_id: &super::types::ContainerId) -> Result<()> {
let output = {
let c = cmd!("podman", "rm", container_id);
trace!("{c:?}");
c
}
.output()
.into_diagnostic()?;
if !output.status.success() {
bail!("Failed to remove container {container_id}");
}
Ok(())
}
fn remove_image(image: &Reference) -> Result<()> {
let output = {
let c = cmd!("podman", "rmi", image.to_string());
trace!("{c:?}");
c
}
.output()
.into_diagnostic()?;
if !output.status.success() {
bail!("Failed to remove the image {image}");
}
Ok(())
}
fn mount_container(container_id: &super::types::ContainerId) -> Result<MountId> {
let output = {
let c = cmd!("podman", "mount", container_id);
trace!("{c:?}");
c
}
.output()
.into_diagnostic()?;
if !output.status.success() {
bail!("Failed to mount container {container_id}");
}
Ok(MountId(
String::from_utf8(output.stdout.trim_ascii().to_vec()).into_diagnostic()?,
))
}
fn unmount_container(container_id: &super::types::ContainerId) -> Result<()> {
let output = {
let c = cmd!("podman", "unmount", container_id);
trace!("{c:?}");
c
}
.output()
.into_diagnostic()?;
if !output.status.success() {
bail!("Failed to unmount container {container_id}");
}
Ok(())
}
fn remove_volume(volume_id: &str) -> Result<()> {
let output = {
let c = cmd!("podman", "volume", "rm", volume_id);
trace!("{c:?}");
c
}
.output()
.into_diagnostic()?;
if !output.status.success() {
bail!("Failed to remove volume {volume_id}");
}
Ok(())
}
}
#[cfg(feature = "rechunk")]
impl RechunkDriver for PodmanDriver {}
impl RunDriver for PodmanDriver {
fn run(opts: &RunOpts) -> std::io::Result<ExitStatus> {
fn run(opts: &RunOpts) -> Result<ExitStatus> {
trace!("PodmanDriver::run({opts:#?})");
let cid_path = TempDir::new()?;
if !nix::unistd::Uid::effective().is_root() {
bail!("You must be root to run privileged podman!");
}
let cid_path = TempDir::new().into_diagnostic()?;
let cid_file = cid_path.path().join("cid");
let cid = ContainerId::new(&cid_file, ContainerRuntime::Podman, opts.privileged);
let cid = ContainerSignalId::new(&cid_file, ContainerRuntime::Podman, opts.privileged);
add_cid(&cid);
let status = if opts.privileged {
podman_run(opts, &cid_file).status()?
} else {
podman_run(opts, &cid_file).build_status(&*opts.image, "Running container")?
};
let status = podman_run(opts, &cid_file)
.build_status(&*opts.image, "Running container")
.into_diagnostic()?;
remove_cid(&cid);
Ok(status)
}
fn run_output(opts: &RunOpts) -> std::io::Result<std::process::Output> {
fn run_output(opts: &RunOpts) -> Result<std::process::Output> {
trace!("PodmanDriver::run_output({opts:#?})");
let cid_path = TempDir::new()?;
if !nix::unistd::Uid::effective().is_root() {
bail!("You must be root to run privileged podman!");
}
let cid_path = TempDir::new().into_diagnostic()?;
let cid_file = cid_path.path().join("cid");
let cid = ContainerId::new(&cid_file, ContainerRuntime::Podman, opts.privileged);
let cid = ContainerSignalId::new(&cid_file, ContainerRuntime::Podman, opts.privileged);
add_cid(&cid);
let output = podman_run(opts, &cid_file).output()?;
let output = podman_run(opts, &cid_file).output().into_diagnostic()?;
remove_cid(&cid);
@ -376,16 +495,7 @@ impl RunDriver for PodmanDriver {
fn podman_run(opts: &RunOpts, cid_file: &Path) -> Command {
let command = cmd!(
if opts.privileged {
warn!(
"Running 'podman' in privileged mode requires '{}'",
"sudo".bold().red()
);
"sudo"
} else {
"podman"
},
if opts.privileged => "podman",
"podman",
"run",
format!("--cidfile={}", cid_file.display()),
if opts.privileged => [
@ -394,6 +504,7 @@ fn podman_run(opts: &RunOpts, cid_file: &Path) -> Command {
],
if opts.remove => "--rm",
if opts.pull => "--pull=always",
if let Some(user) = opts.user.as_ref() => format!("--user={user}"),
for RunOptsVolume { path_or_vol_name, container_path } in opts.volumes.iter() => [
"--volume",
format!("{path_or_vol_name}:{container_path}"),

24
process/drivers/skopeo_driver.rs
View file

@ -65,3 +65,27 @@ fn get_metadata_cache(opts: &GetMetadataOpts) -> Result<ImageMetadata> {
}
serde_json::from_slice(&output.stdout).into_diagnostic()
}
#[cfg(feature = "rechunk")]
impl super::OciCopy for SkopeoDriver {
fn copy_oci_dir(
oci_dir: &super::types::OciDir,
registry: &oci_distribution::Reference,
) -> Result<()> {
use crate::logging::CommandLogging;
let status = {
let c = cmd!("skopeo", "copy", oci_dir, format!("docker://{registry}"),);
trace!("{c:?}");
c
}
.build_status(registry.to_string(), format!("Copying {oci_dir} to"))
.into_diagnostic()?;
if !status.success() {
bail!("Failed to copy {oci_dir} to {registry}");
}
Ok(())
}
}

254
process/drivers/traits.rs
View file

@ -30,6 +30,11 @@ use super::{
skopeo_driver::SkopeoDriver,
types::ImageMetadata,
};
#[cfg(feature = "rechunk")]
use super::{
opts::RechunkOpts,
types::{ContainerId, MountId},
};
trait PrivateDriver {}
@ -209,13 +214,258 @@ pub trait RunDriver: PrivateDriver {
///
/// # Errors
/// Will error if there is an issue running the container.
fn run(opts: &RunOpts) -> std::io::Result<ExitStatus>;
fn run(opts: &RunOpts) -> Result<ExitStatus>;
/// Run a container to perform an action and capturing output.
///
/// # Errors
/// Will error if there is an issue running the container.
fn run_output(opts: &RunOpts) -> std::io::Result<Output>;
fn run_output(opts: &RunOpts) -> Result<Output>;
}
#[allow(private_bounds)]
#[cfg(feature = "rechunk")]
pub(super) trait ContainerMountDriver: PrivateDriver {
/// Creates container
///
/// # Errors
/// Will error if the container create command fails.
fn create_container(image: &Reference) -> Result<ContainerId>;
/// Removes a container
///
/// # Errors
/// Will error if the container remove command fails.
fn remove_container(container_id: &ContainerId) -> Result<()>;
/// Removes an image
///
/// # Errors
/// Will error if the image remove command fails.
fn remove_image(image: &Reference) -> Result<()>;
/// Mounts the container
///
/// # Errors
/// Will error if the container mount command fails.
fn mount_container(container_id: &ContainerId) -> Result<MountId>;
/// Unmount the container
///
/// # Errors
/// Will error if the container unmount command fails.
fn unmount_container(container_id: &ContainerId) -> Result<()>;
/// Remove a volume
///
/// # Errors
/// Will error if the volume remove command fails.
fn remove_volume(volume_id: &str) -> Result<()>;
}
#[cfg(feature = "rechunk")]
pub(super) trait OciCopy {
fn copy_oci_dir(
oci_dir: &super::types::OciDir,
registry: &oci_distribution::Reference,
) -> Result<()>;
}
#[allow(private_bounds)]
#[cfg(feature = "rechunk")]
pub trait RechunkDriver: RunDriver + BuildDriver + ContainerMountDriver {
const RECHUNK_IMAGE: &str = "ghcr.io/hhd-dev/rechunk:v1.0.1";
/// Perform a rechunk build of a recipe.
///
/// # Errors
/// Will error if the rechunk process fails.
fn rechunk(opts: &RechunkOpts) -> Result<Vec<String>> {
let ostree_cache_id = &uuid::Uuid::new_v4().to_string();
let raw_image =
&Reference::try_from(format!("localhost/{ostree_cache_id}/raw-rechunk")).unwrap();
let current_dir = &std::env::current_dir().into_diagnostic()?;
let current_dir = &*current_dir.to_string_lossy();
let full_image = Reference::try_from(opts.tags.first().map_or_else(
|| opts.image.to_string(),
|tag| format!("{}:{tag}", opts.image),
))
.into_diagnostic()?;
Self::build(
&BuildOpts::builder()
.image(raw_image.to_string())
.containerfile(&*opts.containerfile)
.platform(opts.platform)
.squash(true)
.host_network(true)
.build(),
)?;
let container = &Self::create_container(raw_image)?;
let mount = &Self::mount_container(container)?;
Self::prune_image(mount, container, raw_image, opts)?;
Self::create_ostree_commit(mount, ostree_cache_id, container, raw_image, opts)?;
let temp_dir = tempfile::TempDir::new().into_diagnostic()?;
let temp_dir_str = &*temp_dir.path().to_string_lossy();
Self::rechunk_image(ostree_cache_id, temp_dir_str, current_dir, opts)?;
let mut image_list = Vec::with_capacity(opts.tags.len());
if opts.push {
let oci_dir = &super::types::OciDir::try_from(temp_dir.path().join(ostree_cache_id))?;
for tag in &opts.tags {
let tagged_image = Reference::with_tag(
full_image.registry().to_string(),
full_image.repository().to_string(),
tag.to_string(),
);
blue_build_utils::retry(opts.retry_count, 5, || {
debug!("Pushing image {tagged_image}");
Driver::copy_oci_dir(oci_dir, &tagged_image)
})?;
image_list.push(tagged_image.into());
}
}
Ok(image_list)
}
/// Step 1 of the rechunk process that prunes excess files.
///
/// # Errors
/// Will error if the prune process fails.
fn prune_image(
mount: &MountId,
container: &ContainerId,
raw_image: &Reference,
opts: &RechunkOpts<'_>,
) -> Result<(), miette::Error> {
let status = Self::run(
&RunOpts::builder()
.image(Self::RECHUNK_IMAGE)
.remove(true)
.user("0:0")
.privileged(true)
.volumes(crate::run_volumes! {
mount => "/var/tree",
})
.env_vars(crate::run_envs! {
"TREE" => "/var/tree",
})
.args(bon::vec!["/sources/rechunk/1_prune.sh"])
.build(),
)?;
if !status.success() {
Self::unmount_container(container)?;
Self::remove_container(container)?;
Self::remove_image(raw_image)?;
bail!("Failed to run prune step for {}", &opts.image);
}
Ok(())
}
/// Step 2 of the rechunk process that creates the ostree commit.
///
/// # Errors
/// Will error if the ostree commit process fails.
fn create_ostree_commit(
mount: &MountId,
ostree_cache_id: &str,
container: &ContainerId,
raw_image: &Reference,
opts: &RechunkOpts<'_>,
) -> Result<()> {
let status = Self::run(
&RunOpts::builder()
.image(Self::RECHUNK_IMAGE)
.remove(true)
.user("0:0")
.privileged(true)
.volumes(crate::run_volumes! {
mount => "/var/tree",
ostree_cache_id => "/var/ostree",
})
.env_vars(crate::run_envs! {
"TREE" => "/var/tree",
"REPO" => "/var/ostree/repo",
"RESET_TIMESTAMP" => "1",
})
.args(bon::vec!["/sources/rechunk/2_create.sh"])
.build(),
)?;
Self::unmount_container(container)?;
Self::remove_container(container)?;
Self::remove_image(raw_image)?;
if !status.success() {
bail!("Failed to run Ostree create step for {}", &opts.image);
}
Ok(())
}
/// Step 3 of the rechunk process that generates the final chunked image.
///
/// # Errors
/// Will error if the chunk process fails.
fn rechunk_image(
ostree_cache_id: &str,
temp_dir_str: &str,
current_dir: &str,
opts: &RechunkOpts<'_>,
) -> Result<()> {
let status = Self::run(
&RunOpts::builder()
.image(Self::RECHUNK_IMAGE)
.remove(true)
.user("0:0")
.privileged(true)
.volumes(crate::run_volumes! {
ostree_cache_id => "/var/ostree",
temp_dir_str => "/workspace",
current_dir => "/var/git"
})
.env_vars(crate::run_envs! {
"REPO" => "/var/ostree/repo",
"PREV_REF" => &*opts.image,
"OUT_NAME" => ostree_cache_id,
// "PREV_REF_FAIL" => "true",
"VERSION" => format!("{}", opts.version),
"OUT_REF" => format!("oci:{ostree_cache_id}"),
"GIT_DIR" => "/var/git",
"LABELS" => format!(
"{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}",
format_args!("{}={}", blue_build_utils::constants::BUILD_ID_LABEL, Driver::get_build_id()),
format_args!("org.opencontainers.image.title={}", &opts.name),
format_args!("org.opencontainers.image.description={}", &opts.description),
format_args!("org.opencontainers.image.source={}", &opts.repo),
format_args!("org.opencontainers.image.base.digest={}", &opts.base_digest),
format_args!("org.opencontainers.image.base.name={}", &opts.base_image),
"org.opencontainers.image.created=<timestamp>",
"io.artifacthub.package.readme-url=https://raw.githubusercontent.com/blue-build/cli/main/README.md",
)
})
.args(bon::vec!["/sources/rechunk/3_chunk.sh"])
.build(),
)?;
Self::remove_volume(ostree_cache_id)?;
if !status.success() {
bail!("Failed to run rechunking for {}", &opts.image);
}
Ok(())
}
}
/// Allows agnostic management of signature keys.

71
process/drivers/types.rs
View file

@ -235,3 +235,74 @@ impl ImageMetadata {
)
}
}
#[cfg(feature = "rechunk")]
pub struct ContainerId(pub(super) String);
#[cfg(feature = "rechunk")]
impl std::fmt::Display for ContainerId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", &self.0)
}
}
#[cfg(feature = "rechunk")]
impl AsRef<std::ffi::OsStr> for ContainerId {
fn as_ref(&self) -> &std::ffi::OsStr {
self.0.as_ref()
}
}
#[cfg(feature = "rechunk")]
pub struct MountId(pub(super) String);
#[cfg(feature = "rechunk")]
impl std::fmt::Display for MountId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", &self.0)
}
}
#[cfg(feature = "rechunk")]
impl AsRef<std::ffi::OsStr> for MountId {
fn as_ref(&self) -> &std::ffi::OsStr {
self.0.as_ref()
}
}
#[cfg(feature = "rechunk")]
impl<'a> From<&'a MountId> for std::borrow::Cow<'a, str> {
fn from(value: &'a MountId) -> Self {
Self::Borrowed(&value.0)
}
}
#[cfg(feature = "rechunk")]
pub struct OciDir(String);
#[cfg(feature = "rechunk")]
impl std::fmt::Display for OciDir {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", &self.0)
}
}
#[cfg(feature = "rechunk")]
impl AsRef<std::ffi::OsStr> for OciDir {
fn as_ref(&self) -> &std::ffi::OsStr {
self.0.as_ref()
}
}
#[cfg(feature = "rechunk")]
impl TryFrom<std::path::PathBuf> for OciDir {
type Error = miette::Report;
fn try_from(value: std::path::PathBuf) -> Result<Self, Self::Error> {
if !value.is_dir() {
miette::bail!("OCI directory doesn't exist at {}", value.display());
}
Ok(Self(format!("oci:{}", value.display())))
}
}