diff --git a/apt-layer.sh b/apt-layer.sh index fd6efc0..3721a36 100644 --- a/apt-layer.sh +++ b/apt-layer.sh @@ -6,7 +6,7 @@ # DO NOT modify this file directly as it will be overwritten # # # # apt-layer Tool # -# Generated on: 2025-07-15 00:37:36 # +# Generated on: 2025-07-15 11:17:09 # # # ################################################################################################################ @@ -84,6 +84,50 @@ else fi +# Embedded dependencies.json +APT_LAYER_DEPENDENCIES_JSON=$(cat << 'EOF' +{ + "core": [ + "chroot", "apt-get", "dpkg", "jq", "mount", "umount", "findmnt", "numfmt" + ], + "container": [ + "podman", "docker" + ], + "oci": [ + "skopeo" + ], + "composefs": [ + "mkcomposefs", "composefs-info", "mount.composefs", "mksquashfs", "unsquashfs" + ], + "composefs_packages": [ + "composefs", "libcomposefs1" + ], + "bootloader": [ + "efibootmgr", "grub-install", "update-grub", "bootctl" + ], + "security": [ + "curl", "wget", "gpg" + ], + "package_install_commands": { + "debian": { + "composefs": "apt install -y composefs libcomposefs1", + "container": "apt install -y podman docker.io", + "oci": "apt install -y skopeo", + "bootloader": "apt install -y efibootmgr grub-common systemd-boot", + "core": "apt install -y squashfs-tools jq coreutils util-linux" + }, + "fedora": { + "composefs": "dnf install -y composefs composefs-libs", + "container": "dnf install -y podman docker", + "oci": "dnf install -y skopeo", + "bootloader": "dnf install -y efibootmgr grub2-tools systemd-boot", + "core": "dnf install -y squashfs-tools jq coreutils util-linux" + } + } +} +EOF +) + # ============================================================================ # Header and Shared Functions # ============================================================================ @@ -158,7 +202,8 @@ BUILD_DIR="/var/lib/particle-os/build" LIVE_OVERLAY_DIR="/var/lib/particle-os/live-overlay" COMPOSEFS_DIR="/var/lib/particle-os/composefs" COMPOSEFS_SCRIPT="/usr/local/bin/composefs-alternative.sh" -CONTAINER_RUNTIME="podman" +# Container runtime will be detected dynamically based on configuration +CONTAINER_RUNTIME="" # Transaction state variables TRANSACTION_ID="" @@ -428,9 +473,22 @@ composefs_create() { log_debug "Creating ComposeFS image: $image_name from $source_dir" "apt-layer" - if ! "$COMPOSEFS_SCRIPT" create "$image_name" "$source_dir"; then - log_error "Failed to create ComposeFS image: $image_name" "apt-layer" - return 1 + # Try real mkcomposefs binary first + if command -v mkcomposefs >/dev/null 2>&1; then + # Create object store directory (same directory as image) + local object_store_dir=$(dirname "$image_name") + mkdir -p "$object_store_dir" + + if ! mkcomposefs "$source_dir" "$image_name" --digest-store="$object_store_dir"; then + log_error "Failed to create ComposeFS image with mkcomposefs: $image_name" "apt-layer" + return 1 + fi + else + # Fallback to composefs-alternative.sh + if ! "$COMPOSEFS_SCRIPT" create "$image_name" "$source_dir"; then + log_error "Failed to create ComposeFS image: $image_name" "apt-layer" + return 1 + fi fi log_success "ComposeFS image created: $image_name" "apt-layer" @@ -443,9 +501,25 @@ composefs_mount() { log_debug "Mounting ComposeFS image: $image_name to $mount_point" "apt-layer" - if ! "$COMPOSEFS_SCRIPT" mount "$image_name" "$mount_point"; then - log_error "Failed to mount ComposeFS image: $image_name to $mount_point" "apt-layer" - return 1 + # Try real mount with composefs filesystem + if command -v mkcomposefs >/dev/null 2>&1; then + # Create mount point + mkdir -p "$mount_point" + + # Determine object store directory (same directory as image for now) + local object_store_dir=$(dirname "$image_name") + + # Mount using composefs filesystem type + if ! mount -t composefs -o "basedir=$object_store_dir" "$image_name" "$mount_point"; then + log_error "Failed to mount ComposeFS image with mount: $image_name to $mount_point" "apt-layer" + return 1 + fi + else + # Fallback to composefs-alternative.sh + if ! "$COMPOSEFS_SCRIPT" mount "$image_name" "$mount_point"; then + log_error "Failed to mount ComposeFS image: $image_name to $mount_point" "apt-layer" + return 1 + fi fi log_success "ComposeFS image mounted: $image_name to $mount_point" "apt-layer" @@ -457,9 +531,18 @@ composefs_unmount() { log_debug "Unmounting ComposeFS image from: $mount_point" "apt-layer" - if ! "$COMPOSEFS_SCRIPT" unmount "$mount_point"; then - log_error "Failed to unmount ComposeFS image from: $mount_point" "apt-layer" - return 1 + # Try real umount + if command -v mkcomposefs >/dev/null 2>&1; then + if ! umount "$mount_point"; then + log_error "Failed to unmount ComposeFS image with umount: $mount_point" "apt-layer" + return 1 + fi + else + # Fallback to composefs-alternative.sh + if ! "$COMPOSEFS_SCRIPT" unmount "$mount_point"; then + log_error "Failed to unmount ComposeFS image from: $mount_point" "apt-layer" + return 1 + fi fi log_success "ComposeFS image unmounted from: $mount_point" "apt-layer" @@ -468,17 +551,34 @@ composefs_unmount() { composefs_list_images() { log_debug "Listing ComposeFS images" "apt-layer" - "$COMPOSEFS_SCRIPT" list-images + + # Try to list images from workspace directory + if command -v mkcomposefs >/dev/null 2>&1; then + # List .composefs files in the workspace + find "$WORKSPACE/images" -name "*.composefs" -type f 2>/dev/null | sed 's|.*/||' | sed 's|\.composefs$||' || true + else + # Fallback to composefs-alternative.sh + "$COMPOSEFS_SCRIPT" list-images + fi } composefs_image_exists() { local image_name="$1" - # Check if image exists by trying to list it - if "$COMPOSEFS_SCRIPT" list-images | grep -q "^$image_name$"; then - return 0 + # Check if image exists by looking for the .composefs file + if command -v mkcomposefs >/dev/null 2>&1; then + if [[ -f "$WORKSPACE/images/$image_name.composefs" ]]; then + return 0 + else + return 1 + fi else - return 1 + # Fallback to composefs-alternative.sh + if "$COMPOSEFS_SCRIPT" list-images | grep -q "^$image_name$"; then + return 0 + else + return 1 + fi fi } @@ -487,9 +587,18 @@ composefs_remove_image() { log_debug "Removing ComposeFS image: $image_name" "apt-layer" - if ! "$COMPOSEFS_SCRIPT" remove "$image_name"; then - log_error "Failed to remove ComposeFS image: $image_name" "apt-layer" - return 1 + # Try real file removal + if command -v mkcomposefs >/dev/null 2>&1; then + if ! rm -f "$WORKSPACE/images/$image_name.composefs"; then + log_error "Failed to remove ComposeFS image file: $image_name" "apt-layer" + return 1 + fi + else + # Fallback to composefs-alternative.sh + if ! "$COMPOSEFS_SCRIPT" remove "$image_name"; then + log_error "Failed to remove ComposeFS image: $image_name" "apt-layer" + return 1 + fi fi log_success "ComposeFS image removed: $image_name" "apt-layer" @@ -603,14 +712,14 @@ get_system_info() { echo "Architecture: $(uname -m)" echo "Available modules:" if modprobe -n squashfs >/dev/null 2>&1; then - echo " â squashfs module available" + echo " � squashfs module available" else - echo " â squashfs module not available" + echo " � squashfs module not available" fi if modprobe -n overlay >/dev/null 2>&1; then - echo " â overlay module available" + echo " � overlay module available" else - echo " â overlay module not available" + echo " � overlay module not available" fi } @@ -638,6 +747,90 @@ get_available_space() { # Dependency Checking and Validation # ============================================================================ # Enhanced dependency checking and validation for Particle-OS apt-layer Tool +# --- BEGIN DEPENDENCY JSON LOADING --- +# The dependencies JSON will be embedded as APT_LAYER_DEPENDENCIES_JSON in the compiled script. +# If not present, fallback to a default minimal set. +APT_LAYER_DEPENDENCIES_JSON="${APT_LAYER_DEPENDENCIES_JSON:-} +{ + \"core\": [\"chroot\", \"apt-get\", \"dpkg\", \"jq\", \"mount\", \"umount\", \"findmnt\", \"numfmt\"], + \"container\": [\"podman\", \"docker\"], + \"oci\": [\"skopeo\"], + \"composefs\": [\"mkcomposefs\", \"composefs-info\", \"mount.composefs\", \"mksquashfs\", \"unsquashfs\"], + \"composefs_packages\": [\"composefs\", \"libcomposefs1\"], + \"bootloader\": [\"efibootmgr\", \"grub-install\", \"update-grub\", \"bootctl\"], + \"security\": [\"curl\", \"wget\", \"gpg\"], + \"package_install_commands\": { + \"debian\": { + \"composefs\": \"apt install -y composefs libcomposefs1\", + \"container\": \"apt install -y podman docker.io\", + \"oci\": \"apt install -y skopeo\", + \"bootloader\": \"apt install -y efibootmgr grub-common systemd-boot\", + \"core\": \"apt install -y squashfs-tools jq coreutils util-linux\" + }, + \"fedora\": { + \"composefs\": \"dnf install -y composefs composefs-libs\", + \"container\": \"dnf install -y podman docker\", + \"oci\": \"dnf install -y skopeo\", + \"bootloader\": \"dnf install -y efibootmgr grub2-tools systemd-boot\", + \"core\": \"dnf install -y squashfs-tools jq coreutils util-linux\" + } + } +}" + +get_deps_for_type() { + local type="$1" + local json="$APT_LAYER_DEPENDENCIES_JSON" + case "$type" in + --container|container) + echo "$json" | jq -r '.core[], .container[]' + ;; + --oci|oci) + echo "$json" | jq -r '.core[], .oci[]' + ;; + --composefs|composefs) + echo "$json" | jq -r '.core[], .composefs[]' + ;; + --bootloader|bootloader) + echo "$json" | jq -r '.core[], .bootloader[]' + ;; + --scan|security) + echo "$json" | jq -r '.core[], .security[]' + ;; + *) + echo "$json" | jq -r '.core[]' + ;; + esac +} + +print_install_instructions() { + local json="$APT_LAYER_DEPENDENCIES_JSON" + + # Detect distribution + local distro="debian" + if [[ -f /etc/fedora-release ]] || [[ -f /etc/redhat-release ]]; then + distro="fedora" + fi + + echo " Quick fix for common dependencies:" + local core_cmd=$(echo "$json" | jq -r ".package_install_commands.$distro.core") + echo " sudo $core_cmd" + echo "" + echo " For ComposeFS support:" + local composefs_cmd=$(echo "$json" | jq -r ".package_install_commands.$distro.composefs") + echo " sudo $composefs_cmd" + echo "" + echo " For container support:" + local container_cmd=$(echo "$json" | jq -r ".package_install_commands.$distro.container") + echo " sudo $container_cmd" + echo "" + echo " For bootloader support:" + local bootloader_cmd=$(echo "$json" | jq -r ".package_install_commands.$distro.bootloader") + echo " sudo $bootloader_cmd" + echo "" + echo " For more information, run: apt-layer --help" + echo "" +} + check_dependencies() { local command_type="${1:-}" local packages=("${@:2}") @@ -645,71 +838,28 @@ check_dependencies() { log_info "Checking dependencies for command: ${command_type:-general}" "apt-layer" local missing_deps=() - local missing_packages=() local missing_tools=() local missing_scripts=() local missing_modules=() - # Core system dependencies - local core_deps=("chroot" "apt-get" "dpkg" "jq" "mount" "umount") - for dep in "${core_deps[@]}"; do + # Use JSON to get the relevant dependencies + local deps=( $(get_deps_for_type "$command_type") ) + for dep in "${deps[@]}"; do if ! command -v "$dep" >/dev/null 2>&1; then missing_deps+=("$dep") missing_tools+=("$dep") fi done - # Container-specific dependencies - if [[ "$command_type" == "--container" || "$command_type" == "container" ]]; then - local container_deps=("podman" "docker" "systemd-nspawn") - local found_container=false - - for dep in "${container_deps[@]}"; do - if command -v "$dep" >/dev/null 2>&1; then - found_container=true - break - fi - done - - if [[ "$found_container" == "false" ]]; then - missing_deps+=("container-runtime") - missing_tools+=("podman, docker, or systemd-nspawn") - fi - fi - - # ComposeFS-specific dependencies - if [[ "$command_type" == "--composefs" || "$command_type" == "composefs" ]]; then - local composefs_deps=("mksquashfs" "unsquashfs" "skopeo") - for dep in "${composefs_deps[@]}"; do - if ! command -v "$dep" >/dev/null 2>&1; then - missing_deps+=("$dep") - missing_tools+=("$dep") - fi - done - fi - - # Security scanning dependencies - if [[ "$command_type" == "--scan" || "$command_type" == "security" ]]; then - local security_deps=("curl" "wget" "gpg") - for dep in "${security_deps[@]}"; do - if ! command -v "$dep" >/dev/null 2>&1; then - missing_deps+=("$dep") - missing_tools+=("$dep") - fi - done - fi - - # Check for required scripts + # Check for required scripts (unchanged) local required_scripts=( "composefs-alternative.sh:/usr/local/bin/composefs-alternative.sh" "bootc-alternative.sh:/usr/local/bin/bootc-alternative.sh" "bootupd-alternative.sh:/usr/local/bin/bootupd-alternative.sh" ) - for script_info in "${required_scripts[@]}"; do local script_name="${script_info%%:*}" local script_path="${script_info##*:}" - if [[ ! -f "$script_path" ]]; then missing_deps+=("$script_name") missing_scripts+=("$script_name") @@ -719,10 +869,10 @@ check_dependencies() { fi done - # Check for kernel modules + # Check for kernel modules (unchanged) check_kernel_modules - # Validate package names if provided + # Validate package names if provided (unchanged) if [[ ${#packages[@]} -gt 0 ]]; then if ! validate_package_names "${packages[@]}"; then return 1 @@ -734,48 +884,38 @@ check_dependencies() { echo "" log_error "Missing dependencies detected!" "apt-layer" echo "" - if [[ ${#missing_tools[@]} -gt 0 ]]; then - echo "ð¦ Missing system packages:" + echo " Missing system packages:" for tool in "${missing_tools[@]}"; do - echo " ⢠$tool" + echo " $tool" done echo "" echo " Install with: sudo apt install -y ${missing_tools[*]}" echo "" fi - if [[ ${#missing_scripts[@]} -gt 0 ]]; then - echo "ð Missing or non-executable scripts:" + echo " Missing or non-executable scripts:" for script in "${missing_scripts[@]}"; do - echo " ⢠$script" + echo " $script" done echo "" echo " Ensure scripts are installed and executable:" echo " sudo chmod +x /usr/local/bin/*-alternative.sh" echo "" fi - if [[ ${#missing_modules[@]} -gt 0 ]]; then - echo "ð§ Missing kernel modules:" + echo " Missing kernel modules:" for module in "${missing_modules[@]}"; do - echo " ⢠$module" + echo " $module" done echo "" echo " Load modules with: sudo modprobe ${missing_modules[*]}" echo " Or install with: sudo apt install linux-modules-extra-\$(uname -r)" echo "" fi - - echo "ð¡ Quick fix for common dependencies:" - echo " sudo apt install -y squashfs-tools jq coreutils util-linux" - echo "" - echo "ð For more information, run: apt-layer --help" - echo "" - + print_install_instructions exit 1 fi - log_success "All dependencies found and validated" "apt-layer" } @@ -903,7 +1043,7 @@ show_actionable_error() { case "$error_type" in "missing_dependencies") - echo "ð§ To fix this issue:" + echo "� To fix this issue:" echo " 1. Install missing dependencies:" echo " sudo apt update" echo " sudo apt install -y $packages" @@ -916,7 +1056,7 @@ show_actionable_error() { echo "" ;; "permission_denied") - echo "ð Permission issue detected:" + echo "� Permission issue detected:" echo " This command requires root privileges." echo "" echo " Run with sudo:" @@ -924,7 +1064,7 @@ show_actionable_error() { echo "" ;; "invalid_arguments") - echo "ð Invalid arguments provided:" + echo "� Invalid arguments provided:" echo " Check the command syntax and try again." echo "" echo " For help, run:" @@ -933,7 +1073,7 @@ show_actionable_error() { echo "" ;; "system_not_initialized") - echo "ð System not initialized:" + echo "� System not initialized:" echo " Particle-OS needs to be initialized first." echo "" echo " Run initialization:" @@ -941,7 +1081,7 @@ show_actionable_error() { echo "" ;; "disk_space") - echo "ð¾ Insufficient disk space:" + echo "� Insufficient disk space:" echo " Free up space or use a different location." echo "" echo " Check available space:" @@ -949,7 +1089,7 @@ show_actionable_error() { echo "" ;; *) - echo "â Unknown error occurred." + echo "� Unknown error occurred." echo " Please check the error message above." echo "" echo " For help, run: apt-layer --help" @@ -957,10 +1097,10 @@ show_actionable_error() { ;; esac - echo "ð For more information:" - echo " ⢠apt-layer --help" - echo " ⢠apt-layer --help-full" - echo " ⢠apt-layer --examples" + echo "� For more information:" + echo " � apt-layer --help" + echo " � apt-layer --help-full" + echo " � apt-layer --examples" echo "" } @@ -1093,8 +1233,8 @@ export APT_LAYER_LOG_FILE="$APT_LAYER_LOG_DIR/apt-layer.log" export APT_LAYER_SIGNING_ENABLED="false" export APT_LAYER_VERIFY_SIGNATURES="false" -# Container settings -export APT_LAYER_CONTAINER_RUNTIME="podman" +# Container settings - will be set dynamically +export APT_LAYER_CONTAINER_RUNTIME="" export APT_LAYER_CHROOT_ENABLED="true" # Default package sources @@ -1597,28 +1737,54 @@ validate_chroot_environment() { detect_container_runtime() { log_info "Detecting container runtime" "apt-layer" - # Check for podman first (preferred for rootless) + # Get configured default runtime + local configured_runtime + if [[ -f "$CONFIG_DIR/apt-layer-settings.json" ]]; then + configured_runtime=$(jq -r '.default_container_runtime // empty' "$CONFIG_DIR/apt-layer-settings.json" 2>/dev/null) + fi + + # If configured runtime is specified and available, use it + if [[ -n "$configured_runtime" ]]; then + case "$configured_runtime" in + podman) + if command -v podman &> /dev/null; then + CONTAINER_RUNTIME="podman" + log_info "Using configured container runtime: podman" "apt-layer" + return 0 + else + log_warning "Configured runtime 'podman' not found, falling back to detection" "apt-layer" + fi + ;; + docker) + if command -v docker &> /dev/null; then + CONTAINER_RUNTIME="docker" + log_info "Using configured container runtime: docker" "apt-layer" + return 0 + else + log_warning "Configured runtime 'docker' not found, falling back to detection" "apt-layer" + fi + ;; + esac + fi + + # Auto-detection fallback (in order of preference) + log_info "Auto-detecting container runtime" "apt-layer" + + # Check for podman (preferred for rootless) if command -v podman &> /dev/null; then CONTAINER_RUNTIME="podman" - log_info "Using podman as container runtime" "apt-layer" + log_info "Auto-detected podman as container runtime" "apt-layer" return 0 fi # Fallback to docker if command -v docker &> /dev/null; then CONTAINER_RUNTIME="docker" - log_info "Using docker as container runtime" "apt-layer" + log_info "Auto-detected docker as container runtime" "apt-layer" return 0 fi - # Check for systemd-nspawn as last resort - if command -v systemd-nspawn &> /dev/null; then - CONTAINER_RUNTIME="systemd-nspawn" - log_info "Using systemd-nspawn as container runtime" "apt-layer" - return 0 - fi - - log_error "No supported container runtime found (podman, docker, or systemd-nspawn)" "apt-layer" + log_error "No supported container runtime found (podman or docker required)" "apt-layer" return 1 } @@ -1636,6 +1802,9 @@ init_container_system() { return 1 fi + # Set global container runtime variables + set_global_container_runtime_vars + # Ensure workspace directories exist mkdir -p "$WORKSPACE"/{images,temp,containers} @@ -1643,6 +1812,29 @@ init_container_system() { return 0 } +# Set global container runtime variables +set_global_container_runtime_vars() { + log_debug "Setting global container runtime variables" "apt-layer" + + # Set variables used throughout the system + export APT_LAYER_CONTAINER_RUNTIME="$CONTAINER_RUNTIME" + export PARTICLE_CONTAINER_RUNTIME="$CONTAINER_RUNTIME" + + # Set runtime-specific variables + case "$CONTAINER_RUNTIME" in + podman) + export CONTAINER_RUNTIME_TYPE="general" + export CONTAINER_RUNTIME_DESCRIPTION="Rootless container runtime" + ;; + docker) + export CONTAINER_RUNTIME_TYPE="general" + export CONTAINER_RUNTIME_DESCRIPTION="Traditional container runtime" + ;; + esac + + log_debug "Container runtime variables set: $CONTAINER_RUNTIME ($CONTAINER_RUNTIME_TYPE)" "apt-layer" +} + # Validate container runtime capabilities validate_container_runtime() { local runtime="$1" @@ -1662,9 +1854,6 @@ validate_container_runtime() { return 1 fi ;; - systemd-nspawn) - # systemd-nspawn doesn't need special validation - ;; *) log_error "Unsupported container runtime: $runtime" "apt-layer" return 1 @@ -1759,10 +1948,6 @@ create_base_container_image() { docker pull "$base_image" fi ;; - systemd-nspawn) - # systemd-nspawn uses host filesystem - log_info "Using host filesystem for systemd-nspawn" "apt-layer" - ;; esac log_success "OCI base image ready: $base_image" "apt-layer" @@ -1808,11 +1993,10 @@ container_install_packages() { return 1 fi ;; - systemd-nspawn) - if ! run_nspawn_install "$base_image" "$container_name" "$temp_dir" "${packages[@]}"; then - rollback_transaction - return 1 - fi + *) + log_error "Unsupported container runtime: $CONTAINER_RUNTIME" "apt-layer" + rollback_transaction + return 1 ;; esac @@ -1832,6 +2016,49 @@ container_install_packages() { return 0 } +# Skopeo-based package installation (OCI-focused) +run_skopeo_install() { + local base_image="$1" + local container_name="$2" + local temp_dir="$3" + shift 3 + local packages=("$@") + + log_info "Running skopeo-based installation" "apt-layer" + + # Skopeo is primarily for OCI operations, so we'll use it with a minimal container + # For package installation, we'll fall back to a chroot-based approach + + # Create minimal container structure + mkdir -p "$temp_dir"/{bin,lib,lib64,usr,etc,var} + + # Set up base filesystem + if [[ -d "$WORKSPACE/images/$base_image" ]]; then + # Use ComposeFS image as base + log_info "Using ComposeFS image as base for skopeo" "apt-layer" + cp -a "$WORKSPACE/images/$base_image"/* "$temp_dir/" 2>/dev/null || true + else + # Use minimal Ubuntu base + log_info "Using minimal Ubuntu base for skopeo" "apt-layer" + # Copy essential files + cp -a /bin/bash "$temp_dir/bin/" + cp -a /lib/x86_64-linux-gnu "$temp_dir/lib/" + cp -a /usr/bin/apt-get "$temp_dir/usr/bin/" + # Add minimal /etc structure + echo "deb http://archive.ubuntu.com/ubuntu/ jammy main" > "$temp_dir/etc/apt/sources.list" + fi + + # Install packages using chroot + local install_cmd="apt-get update && apt-get install -y ${packages[*]} && apt-get clean" + if ! chroot "$temp_dir" /bin/bash -c "$install_cmd"; then + log_error "Package installation failed in skopeo container" "apt-layer" + return 1 + fi + + log_success "Skopeo-based installation completed" "apt-layer" + return 0 +} + # Podman-based package installation run_podman_install() { local base_image="$1" @@ -2029,10 +2256,22 @@ create_composefs_layer() { local image_dir="$WORKSPACE/images/$new_image" mkdir -p "$image_dir" - # Use ComposeFS backend to create layer - if ! "$COMPOSEFS_SCRIPT" create "$new_image" "$temp_dir"; then - log_error "Failed to create ComposeFS layer" "apt-layer" - return 1 + # Try real mkcomposefs binary first + if command -v mkcomposefs >/dev/null 2>&1; then + # Create object store directory (same directory as image) + local object_store_dir=$(dirname "$new_image") + mkdir -p "$object_store_dir" + + if ! mkcomposefs "$temp_dir" "$new_image" --digest-store="$object_store_dir"; then + log_error "Failed to create ComposeFS layer with mkcomposefs" "apt-layer" + return 1 + fi + else + # Fallback to composefs-alternative.sh + if ! "$COMPOSEFS_SCRIPT" create "$new_image" "$temp_dir"; then + log_error "Failed to create ComposeFS layer" "apt-layer" + return 1 + fi fi log_success "ComposeFS layer created: $new_image" "apt-layer" @@ -2070,10 +2309,18 @@ container_remove_layer() { log_info "Removing container-based layer: $image_name" "apt-layer" - # Use ComposeFS backend to remove layer - if ! "$COMPOSEFS_SCRIPT" remove "$image_name"; then - log_error "Failed to remove ComposeFS layer" "apt-layer" - return 1 + # Try real file removal + if command -v mkcomposefs >/dev/null 2>&1; then + if ! rm -f "$WORKSPACE/images/$image_name.composefs"; then + log_error "Failed to remove ComposeFS layer file: $image_name" "apt-layer" + return 1 + fi + else + # Fallback to composefs-alternative.sh + if ! "$COMPOSEFS_SCRIPT" remove "$image_name"; then + log_error "Failed to remove ComposeFS layer" "apt-layer" + return 1 + fi fi log_success "Container-based layer removed: $image_name" "apt-layer" @@ -2084,10 +2331,16 @@ container_remove_layer() { container_list_layers() { log_info "Listing container-based layers" "apt-layer" - # Use ComposeFS backend to list layers - if ! "$COMPOSEFS_SCRIPT" list-images; then - log_error "Failed to list ComposeFS layers" "apt-layer" - return 1 + # Try to list images from workspace directory + if command -v mkcomposefs >/dev/null 2>&1; then + # List .composefs files in the workspace + find "$WORKSPACE/images" -name "*.composefs" -type f 2>/dev/null | sed 's|.*/||' | sed 's|\.composefs$||' || true + else + # Fallback to composefs-alternative.sh + if ! "$COMPOSEFS_SCRIPT" list-images; then + log_error "Failed to list ComposeFS layers" "apt-layer" + return 1 + fi fi return 0 @@ -2099,10 +2352,18 @@ container_layer_info() { log_info "Getting container-based layer info: $image_name" "apt-layer" - # Use ComposeFS backend to get layer info - if ! "$COMPOSEFS_SCRIPT" info "$image_name"; then - log_error "Failed to get ComposeFS layer info" "apt-layer" - return 1 + # Try real composefs-info binary first + if command -v composefs-info >/dev/null 2>&1; then + if ! composefs-info ls "$WORKSPACE/images/$image_name.composefs"; then + log_error "Failed to get ComposeFS layer info with composefs-info" "apt-layer" + return 1 + fi + else + # Fallback to composefs-alternative.sh + if ! "$COMPOSEFS_SCRIPT" info "$image_name"; then + log_error "Failed to get ComposeFS layer info" "apt-layer" + return 1 + fi fi return 0 @@ -2115,10 +2376,25 @@ container_mount_layer() { log_info "Mounting container-based layer: $image_name at $mount_point" "apt-layer" - # Use ComposeFS backend to mount layer - if ! "$COMPOSEFS_SCRIPT" mount "$image_name" "$mount_point"; then - log_error "Failed to mount ComposeFS layer" "apt-layer" - return 1 + # Try real mount with composefs filesystem + if command -v mkcomposefs >/dev/null 2>&1; then + # Create mount point + mkdir -p "$mount_point" + + # Determine object store directory (same directory as image) + local object_store_dir=$(dirname "$image_name") + + # Mount using composefs filesystem type + if ! mount -t composefs -o "basedir=$object_store_dir" "$image_name" "$mount_point"; then + log_error "Failed to mount ComposeFS layer with mount: $image_name to $mount_point" "apt-layer" + return 1 + fi + else + # Fallback to composefs-alternative.sh + if ! "$COMPOSEFS_SCRIPT" mount "$image_name" "$mount_point"; then + log_error "Failed to mount ComposeFS layer" "apt-layer" + return 1 + fi fi log_success "Container-based layer mounted: $image_name at $mount_point" "apt-layer" @@ -2131,10 +2407,18 @@ container_unmount_layer() { log_info "Unmounting container-based layer at: $mount_point" "apt-layer" - # Use ComposeFS backend to unmount layer - if ! "$COMPOSEFS_SCRIPT" unmount "$mount_point"; then - log_error "Failed to unmount ComposeFS layer" "apt-layer" - return 1 + # Try real umount + if command -v mkcomposefs >/dev/null 2>&1; then + if ! umount "$mount_point"; then + log_error "Failed to unmount ComposeFS layer with umount: $mount_point" "apt-layer" + return 1 + fi + else + # Fallback to composefs-alternative.sh + if ! "$COMPOSEFS_SCRIPT" unmount "$mount_point"; then + log_error "Failed to unmount ComposeFS layer" "apt-layer" + return 1 + fi fi log_success "Container-based layer unmounted: $mount_point" "apt-layer" @@ -2157,9 +2441,6 @@ container_status() { echo "Docker version: $(docker --version 2>/dev/null || echo 'Not available')" echo "Docker info: $(docker info --format '{{.Architecture}}' 2>/dev/null || echo 'Unknown')" ;; - systemd-nspawn) - echo "systemd-nspawn version: $(systemd-nspawn --version 2>/dev/null || echo 'Not available')" - ;; esac echo "" @@ -2625,7 +2906,19 @@ create_composefs_layer() { local layer_name="$2" local message="$3" - # Use composefs-alternative to create layer + # Try real mkcomposefs binary first + if command -v mkcomposefs >/dev/null 2>&1; then + # Create object store directory (same directory as layer) + local object_store_dir=$(dirname "$layer_name") + mkdir -p "$object_store_dir" + + if mkcomposefs "$source_dir" "$layer_name" --digest-store="$object_store_dir"; then + log_success "ComposeFS layer created using mkcomposefs" "apt-layer" + return 0 + fi + fi + + # Fallback to composefs-alternative if command -v composefs-alternative >/dev/null 2>&1; then if composefs-alternative create-layer "$source_dir" "$layer_name" "$message"; then return 0 @@ -2733,7 +3026,7 @@ trap cleanup_live_overlay_on_exit EXIT # OCI Export/Import Integration # ============================================================================ # OCI Integration for Particle-OS apt-layer Tool -# Provides ComposeFS â OCI export/import functionality for container-based layer creation +# Provides ComposeFS � OCI export/import functionality for container-based layer creation # OCI registry configuration declare -A OCI_REGISTRY_CONFIG @@ -2841,9 +3134,16 @@ export_oci_image() { fi # Check if ComposeFS image exists - if ! "$COMPOSEFS_SCRIPT" info "$composefs_image" >/dev/null 2>&1; then - log_error "ComposeFS image not found: $composefs_image" "apt-layer" - return 1 + if command -v composefs-info >/dev/null 2>&1; then + if ! composefs-info ls "$composefs_image" >/dev/null 2>&1; then + log_error "ComposeFS image not found: $composefs_image" "apt-layer" + return 1 + fi + else + if ! "$COMPOSEFS_SCRIPT" info "$composefs_image" >/dev/null 2>&1; then + log_error "ComposeFS image not found: $composefs_image" "apt-layer" + return 1 + fi fi # Create temporary directory @@ -2858,10 +3158,21 @@ export_oci_image() { mkdir -p "$mount_point" update_transaction_phase "mounting_composefs_image" - if ! "$COMPOSEFS_SCRIPT" mount "$composefs_image" "$mount_point"; then - log_error "Failed to mount ComposeFS image: $composefs_image" "apt-layer" - rollback_transaction - return 1 + if command -v mkcomposefs >/dev/null 2>&1; then + # Determine object store directory (same directory as image) + local object_store_dir=$(dirname "$composefs_image") + + if ! mount -t composefs -o "basedir=$object_store_dir" "$composefs_image" "$mount_point"; then + log_error "Failed to mount ComposeFS image: $composefs_image" "apt-layer" + rollback_transaction + return 1 + fi + else + if ! "$COMPOSEFS_SCRIPT" mount "$composefs_image" "$mount_point"; then + log_error "Failed to mount ComposeFS image: $composefs_image" "apt-layer" + rollback_transaction + return 1 + fi fi # Create OCI image structure @@ -2884,7 +3195,11 @@ export_oci_image() { fi # Unmount ComposeFS image - "$COMPOSEFS_SCRIPT" unmount "$mount_point" 2>/dev/null || true + if command -v mkcomposefs >/dev/null 2>&1; then + umount "$mount_point" 2>/dev/null || true + else + "$COMPOSEFS_SCRIPT" unmount "$mount_point" 2>/dev/null || true + fi commit_transaction log_success "ComposeFS image exported to OCI: $oci_image_name" "apt-layer" @@ -3100,10 +3415,22 @@ import_oci_image() { # Create ComposeFS image from extracted filesystem update_transaction_phase "creating_composefs_image" - if ! "$COMPOSEFS_SCRIPT" create "$composefs_image" "$rootfs_dir"; then - log_error "Failed to create ComposeFS image: $composefs_image" "apt-layer" - rollback_transaction - return 1 + if command -v mkcomposefs >/dev/null 2>&1; then + # Create object store directory (same directory as image) + local object_store_dir=$(dirname "$composefs_image") + mkdir -p "$object_store_dir" + + if ! mkcomposefs "$rootfs_dir" "$composefs_image" --digest-store="$object_store_dir"; then + log_error "Failed to create ComposeFS image: $composefs_image" "apt-layer" + rollback_transaction + return 1 + fi + else + if ! "$COMPOSEFS_SCRIPT" create "$composefs_image" "$rootfs_dir"; then + log_error "Failed to create ComposeFS image: $composefs_image" "apt-layer" + rollback_transaction + return 1 + fi fi commit_transaction @@ -3273,9 +3600,9 @@ oci_status() { echo "=== OCI Tool Configuration ===" echo "Preferred tool: $OCI_TOOL" echo "Available tools:" - command -v skopeo &> /dev/null && echo " â skopeo" - command -v podman &> /dev/null && echo " â podman" - command -v docker &> /dev/null && echo " â docker" + command -v skopeo &> /dev/null && echo " � skopeo" + command -v podman &> /dev/null && echo " � podman" + command -v docker &> /dev/null && echo " � docker" echo "" echo "=== OCI Workspace ===" @@ -6275,12 +6602,6 @@ container_dpkg_install() { return 1 fi ;; - systemd-nspawn) - if ! run_nspawn_dpkg_install "$base_image" "$container_name" "$temp_dir" "${packages[@]}"; then - rollback_transaction - return 1 - fi - ;; *) log_error "Unsupported container runtime: $CONTAINER_RUNTIME" "apt-layer" rollback_transaction @@ -6295,12 +6616,25 @@ container_dpkg_install() { return 1 fi else - # Fallback: use composefs-alternative.sh - log_info "Using fallback ComposeFS layer creation" "apt-layer" - if ! "$COMPOSEFS_SCRIPT" create "$new_image" "$temp_dir"; then - log_error "Failed to create ComposeFS layer" "apt-layer" - rollback_transaction - return 1 + # Try real mkcomposefs binary first + if command -v mkcomposefs >/dev/null 2>&1; then + # Create object store directory (same directory as image) + local object_store_dir=$(dirname "$new_image") + mkdir -p "$object_store_dir" + + if ! mkcomposefs "$temp_dir" "$new_image" --digest-store="$object_store_dir"; then + log_error "Failed to create ComposeFS layer with mkcomposefs" "apt-layer" + rollback_transaction + return 1 + fi + else + # Fallback: use composefs-alternative.sh + log_info "Using fallback ComposeFS layer creation" "apt-layer" + if ! "$COMPOSEFS_SCRIPT" create "$new_image" "$temp_dir"; then + log_error "Failed to create ComposeFS layer" "apt-layer" + rollback_transaction + return 1 + fi fi fi @@ -6459,59 +6793,6 @@ run_docker_dpkg_install() { return 0 } -# systemd-nspawn-based dpkg installation -run_nspawn_dpkg_install() { - local base_image="$1" - local container_name="$2" - local temp_dir="$3" - shift 3 - local packages=("$@") - - log_info "Running systemd-nspawn-based dpkg installation" "apt-layer" - - local container_dir="$WORKSPACE/containers/$container_name" - - # Create container directory - if [[ -d "$WORKSPACE/images/$base_image" ]]; then - # Use ComposeFS image as base - log_info "Using ComposeFS image as base for nspawn" "apt-layer" - cp -a "$WORKSPACE/images/$base_image" "$container_dir" - else - # Use host filesystem as base - log_info "Using host filesystem as base for nspawn" "apt-layer" - # Create minimal container structure - mkdir -p "$container_dir"/{bin,lib,lib64,usr,etc,var} - # Copy essential files from host - cp -a /bin/bash "$container_dir/bin/" - cp -a /lib/x86_64-linux-gnu "$container_dir/lib/" - cp -a /usr/bin/dpkg "$container_dir/usr/bin/" - cp -a /usr/bin/apt-get "$container_dir/usr/bin/" - # Add minimal /etc structure - echo "deb http://archive.ubuntu.com/ubuntu/ jammy main" > "$container_dir/etc/apt/sources.list" - fi - - # Run dpkg installation in nspawn container - local install_cmd=" - apt-get update && - apt-get download ${packages[*]} && - dpkg -i *.deb && - apt-get install -f && - dpkg --configure -a && - apt-get clean - " - - if ! systemd-nspawn -D "$container_dir" /bin/bash -c "$install_cmd"; then - log_error "dpkg installation failed in nspawn container" "apt-layer" - return 1 - fi - - # Move container contents to temp_dir - mv "$container_dir"/* "$temp_dir/" 2>/dev/null || true - - log_success "systemd-nspawn-based dpkg installation completed" "apt-layer" - return 0 -} - # Live overlay dpkg installation live_dpkg_install() { local packages=("$@") @@ -7383,7 +7664,8 @@ PARTICLE_COMPOSEFS_DIR="$PARTICLE_WORKSPACE/composefs" PARTICLE_COMPOSEFS_SCRIPT="/usr/local/bin/composefs-alternative.sh" # Container configuration -PARTICLE_CONTAINER_RUNTIME="podman" +# Container runtime will be detected dynamically +PARTICLE_CONTAINER_RUNTIME="" PARTICLE_CONTAINER_WORKSPACE="$PARTICLE_WORKSPACE/containers" # Live overlay configuration @@ -7821,6 +8103,7 @@ fi # - apt-layer-settings.json # - audit-settings.json # - backup-policy.json +# - dependencies.json # - kernel-modules.json # - kernel-patches.json # - maintenance.json @@ -7832,7 +8115,7 @@ fi # - users.json # Embedded configuration: apt-layer-settings -# File size: 307 +# File size: 670 APT_LAYER_SETTINGS_CONFIG=$(cat << 'EOF' { "default_container_runtime": "podman", @@ -7844,6 +8127,24 @@ APT_LAYER_SETTINGS_CONFIG=$(cat << 'EOF' "audit_reporting": true, "layer_signing": true }, + "container_runtimes": { + "supported": [ + "podman", + "docker" + ], + "preference_order": [ + "podman", + "docker" + ], + "podman": { + "description": "Rootless container runtime (default, recommended)", + "primary_use": "general" + }, + "docker": { + "description": "Traditional container runtime (fallback)", + "primary_use": "fallback" + } + }, "log_level": "info", "color_output": true } @@ -7878,6 +8179,69 @@ BACKUP_POLICY_CONFIG=$(cat << 'EOF' EOF ) +# Embedded configuration: dependencies +# File size: 1.2K +DEPENDENCIES_CONFIG=$(cat << 'EOF' +{ + "core": [ + "chroot", + "apt-get", + "dpkg", + "jq", + "mount", + "umount", + "findmnt", + "numfmt" + ], + "container": [ + "podman", + "docker" + ], + "oci": [ + "skopeo" + ], + "composefs": [ + "mkcomposefs", + "composefs-info", + "mount.composefs", + "mksquashfs", + "unsquashfs" + ], + "composefs_packages": [ + "composefs", + "libcomposefs1" + ], + "bootloader": [ + "efibootmgr", + "grub-install", + "update-grub", + "bootctl" + ], + "security": [ + "curl", + "wget", + "gpg" + ], + "package_install_commands": { + "debian": { + "composefs": "apt install -y composefs libcomposefs1", + "container": "apt install -y podman docker.io", + "oci": "apt install -y skopeo", + "bootloader": "apt install -y efibootmgr grub-common systemd-boot", + "core": "apt install -y squashfs-tools jq coreutils util-linux" + }, + "fedora": { + "composefs": "dnf install -y composefs composefs-libs", + "container": "dnf install -y podman docker", + "oci": "dnf install -y skopeo", + "bootloader": "dnf install -y efibootmgr grub2-tools systemd-boot", + "core": "dnf install -y squashfs-tools jq coreutils util-linux" + } + } +} +EOF +) + # Embedded configuration: kernel-modules # File size: 6.1K KERNEL_MODULES_CONFIG=$(cat << 'EOF' diff --git a/docs/apt-layer/composefs.md b/docs/apt-layer/composefs.md new file mode 100644 index 0000000..9475f5a --- /dev/null +++ b/docs/apt-layer/composefs.md @@ -0,0 +1,142 @@ +# ComposeFS Integration in apt-layer + +## TLDR - Quick Reference + +### Basic Commands + +**Create a ComposeFS image:** +```sh +mkcomposefs --digest-store= +``` + +**Mount a ComposeFS image:** +```sh +mount -t composefs -o basedir= +# or directly: +mount.composefs -o basedir= +``` + +**Unmount:** +```sh +umount +``` + +**Inspect an image:** +```sh +composefs-info ls # List files +composefs-info objects # List backing files +composefs-info missing-objects --basedir= # Check integrity +composefs-info dump # Full metadata dump +``` + +### Quick Example +```sh +# Create image with object store +mkcomposefs /path/to/rootfs myimage.composefs --digest-store=/path/to/objects + +# Mount the image +mount -t composefs -o basedir=/path/to/objects myimage.composefs /mnt/composefs + +# List contents +composefs-info ls myimage.composefs + +# Unmount +umount /mnt/composefs +``` + +**Note:** In apt-layer, images are typically stored in `/var/lib/apt-layer/images/` with object stores in the same directory. The above example uses generic paths for clarity. + +--- + +## Overview + +apt-layer uses [ComposeFS](https://ostreedev.github.io/ostree/composefs/) as its backend for atomic, deduplicated, and efficient filesystem layering—mirroring the approach used by rpm-ostree and Fedora Silverblue. ComposeFS is a Linux filesystem and image format designed for fast, space-efficient, and content-addressed deployment of system images. + +**Key Tools:** The ComposeFS project provides a suite of tools including `mkcomposefs` for image creation, `composefs-info` for inspecting images, and `mount.composefs` for mounting. `mount.composefs` can be called directly or used by the standard `mount -t composefs` command. + +--- + +## Commands + +The `composefs` package provides the following tools: + +| Command | Purpose | Usage | +|---------|---------|-------| +| `mkcomposefs` | Create ComposeFS images | `mkcomposefs --digest-store=` | +| `composefs-info` | Inspect and manage images | `composefs-info [ls\|objects\|missing-objects\|dump] ` | +| `mount.composefs` | Mount images (helper for `mount -t composefs`) | `mount.composefs -o basedir=` | + +**Important:** There is **no** `composefs` executable. The package name is `composefs`, but the actual tools are `mkcomposefs`, `composefs-info`, and `mount.composefs`. + +--- + +## ComposeFS Workflow in apt-layer + +### 1. Image Creation + +To create a ComposeFS image from a directory tree: + +```sh +mkcomposefs --digest-store= +``` + +- ``: Directory containing the root filesystem to layer +- ``: Output ComposeFS image (typically ends with `.composefs`). This file contains the image metadata (an EROFS image file) +- `--digest-store=`: This option specifies a directory where `mkcomposefs` will copy (or reflink) regular files larger than 64 bytes from ``. These files are content-addressed (named after their `fsverity` digest) and form the "backing store" for the ComposeFS image. This directory is then referenced as the `basedir` during mounting + +### 2. Mounting a ComposeFS Image + +To mount a ComposeFS image, `apt-layer` can either call `mount.composefs` directly or rely on the kernel's `mount -t composefs` interface, which will invoke `mount.composefs` as a helper. + +Using `mount.composefs` directly: + +```sh +mount.composefs -o basedir=[,basedir=...] +``` + +Using the standard `mount` command (which relies on `mount.composefs` as a helper): + +```sh +mount -t composefs -o basedir=[,basedir=...] +``` + +- ``: Path to the ComposeFS image file (metadata) +- ``: Where to mount the filesystem +- `-o basedir=`: This option is crucial. It points to the directory (or multiple colon-separated directories) containing the content-addressed backing files created with `--digest-store` during image creation. This provides the underlying content for the ComposeFS image + +**Optional `mount.composefs` options:** +- `digest=DIGEST`: Validates the image file against a specified `fs-verity` digest for integrity +- `verity`: Ensures all files in the image and base directories have matching `fs-verity` digests. Requires kernel 6.6rc1+ +- `idmap=PATH`: Specifies a user namespace for ID mapping +- `upperdir`/`workdir`: Allows for a writable overlay on top of the read-only ComposeFS image, similar to `overlayfs` + +### 3. Unmounting + +```sh +umount +``` + +### 4. Listing and Removing Images + +- **Listing:** apt-layer lists ComposeFS images by scanning for `.composefs` files in its workspace. Additionally, `apt-layer` can use `composefs-info ls ` to inspect the contents of an image, or `composefs-info missing-objects --basedir=` to verify the integrity of the object store. For advanced scenarios, `composefs-info dump` can provide a textual representation of the image's metadata (as defined by `composefs-dump(5)`), which can also be used as input for `mkcomposefs --from-file` +- **Removing:** apt-layer removes images by deleting the corresponding `.composefs` file and cleaning up the associated content-addressed files in the `--digest-store` directory. This cleanup typically involves checking `composefs-info objects` to identify files that are no longer referenced by any active images before removal + +--- + +## Integration Notes + +- **Specific Tools:** While there isn't a single monolithic `composefs` CLI, specialized commands like `composefs-info` exist for introspection, and `mount.composefs` is the dedicated helper for mounting (callable directly or via `mount -t composefs`) +- **Dependencies:** apt-layer requires `mkcomposefs`, `composefs-info`, `mount.composefs` (which might be part of the `composefs` package), `mksquashfs`, and `unsquashfs` for ComposeFS support +- **Fallback:** If `mkcomposefs` (and potentially `mount.composefs`) is not available, apt-layer can fall back to a shell script alternative (for development/testing only) +- **Compatibility:** This approach matches rpm-ostree and Fedora Silverblue's use of ComposeFS for system layering + +--- + +## References +- [ComposeFS Upstream Documentation](https://ostreedev.github.io/ostree/composefs/) +- [ComposeFS GitHub Repository](https://github.com/containers/composefs) +- [ComposeFS Blog Post by Alexander Larsson](https://blogs.gnome.org/alexl/2022/06/02/using-composefs-in-ostree/) +- [`mkcomposefs(1)` man page](https://www.mankier.com/1/mkcomposefs) +- [`mount.composefs(1)` man page](https://www.mankier.com/1/mount.composefs) +- [`composefs-info(1)` man page](https://www.mankier.com/1/composefs-info) +- [`composefs-dump(5)` man page](https://www.mankier.com/5/composefs-dump) diff --git a/docs/apt-layer/skopeo.md b/docs/apt-layer/skopeo.md new file mode 100644 index 0000000..e69de29 diff --git a/install-apt-layer.sh b/install-apt-layer.sh index 50f42b7..1a1d06a 100644 --- a/install-apt-layer.sh +++ b/install-apt-layer.sh @@ -1,3 +1,47 @@ +# Embedded dependencies.json +APT_LAYER_DEPENDENCIES_JSON=$(cat << 'EOF' +{ + "core": [ + "chroot", "apt-get", "dpkg", "jq", "mount", "umount", "findmnt", "numfmt" + ], + "container": [ + "podman", "docker" + ], + "oci": [ + "skopeo" + ], + "composefs": [ + "mkcomposefs", "composefs-info", "mount.composefs", "mksquashfs", "unsquashfs" + ], + "composefs_packages": [ + "composefs", "libcomposefs1" + ], + "bootloader": [ + "efibootmgr", "grub-install", "update-grub", "bootctl" + ], + "security": [ + "curl", "wget", "gpg" + ], + "package_install_commands": { + "debian": { + "composefs": "apt install -y composefs libcomposefs1", + "container": "apt install -y podman docker.io", + "oci": "apt install -y skopeo", + "bootloader": "apt install -y efibootmgr grub-common systemd-boot", + "core": "apt install -y squashfs-tools jq coreutils util-linux" + }, + "fedora": { + "composefs": "dnf install -y composefs composefs-libs", + "container": "dnf install -y podman docker", + "oci": "dnf install -y skopeo", + "bootloader": "dnf install -y efibootmgr grub2-tools systemd-boot", + "core": "dnf install -y squashfs-tools jq coreutils util-linux" + } + } +} +EOF +) + #!/bin/bash # apt-layer Installation Script @@ -78,47 +122,83 @@ check_root() { fi } +# --- BEGIN DEPENDENCY JSON LOADING --- +# The dependencies JSON will be embedded as APT_LAYER_DEPENDENCIES_JSON in the compiled script. +APT_LAYER_DEPENDENCIES_JSON="${APT_LAYER_DEPENDENCIES_JSON:-} +{ + \"core\": [\"chroot\", \"apt-get\", \"dpkg\", \"jq\", \"mount\", \"umount\", \"findmnt\", \"numfmt\"], + \"container\": [\"podman\", \"docker\"], + \"oci\": [\"skopeo\"], + \"composefs\": [\"mkcomposefs\", \"composefs-info\", \"mount.composefs\", \"mksquashfs\", \"unsquashfs\"], + \"bootloader\": [\"efibootmgr\", \"grub-install\", \"update-grub\", \"bootctl\"], + \"security\": [\"curl\", \"wget\", \"gpg\"] +}" + +get_deps_for_type() { + local type="$1" + local json="$APT_LAYER_DEPENDENCIES_JSON" + case "$type" in + core) + echo "$json" | jq -r '.core[]' + ;; + security) + echo "$json" | jq -r '.security[]' + ;; + all) + echo "$json" | jq -r '.core[], .security[]' + ;; + *) + echo "$json" | jq -r '.core[]' + ;; + esac +} + +print_install_instructions() { + local json="$APT_LAYER_DEPENDENCIES_JSON" + echo " Quick fix for common dependencies:" + echo " sudo apt install -y squashfs-tools jq coreutils util-linux skopeo" + echo "" + echo " For container support (choose one):" + echo " sudo apt install -y podman # or" + echo " sudo apt install -y docker.io" + echo "" + echo " For bootloader support:" + echo " sudo apt install -y efibootmgr grub-common systemd-boot" + echo "" + echo " For more information, run: apt-layer --help" + echo "" +} + # Function to check dependencies check_dependencies() { print_status "Checking dependencies..." - local missing_deps=() - - # Check for curl or wget + # Installer-specific dependencies + local installer_deps=("sudo" "dos2unix") + # Use JSON to get core and security deps + local deps=( $(get_deps_for_type all) ) + # Add installer deps + deps+=("sudo" "dos2unix") + # Check for curl or wget (at least one) if ! command -v curl >/dev/null 2>&1 && ! command -v wget >/dev/null 2>&1; then missing_deps+=("curl or wget") fi - - # Check for jq - if ! command -v jq >/dev/null 2>&1; then - missing_deps+=("jq") - fi - - # Check for dos2unix - if ! command -v dos2unix >/dev/null 2>&1; then - missing_deps+=("dos2unix") - fi - + # Check all other deps + for dep in "${deps[@]}"; do + if [[ "$dep" == "curl" || "$dep" == "wget" ]]; then + continue # already checked above + fi + if ! command -v "$dep" >/dev/null 2>&1; then + missing_deps+=("$dep") + fi + done if [[ ${#missing_deps[@]} -gt 0 ]]; then print_error "Missing required dependencies: ${missing_deps[*]}" - print_status "Installing missing dependencies..." - - # Try to install dependencies - if command -v apt-get >/dev/null 2>&1; then - sudo apt-get update - sudo apt-get install -y "${missing_deps[@]}" - elif command -v dnf >/dev/null 2>&1; then - sudo dnf install -y "${missing_deps[@]}" - elif command -v yum >/dev/null 2>&1; then - sudo yum install -y "${missing_deps[@]}" - else - print_error "Could not automatically install dependencies. Please install manually:" - print_error " ${missing_deps[*]}" - exit 1 - fi + print_install_instructions + print_error "Please install missing dependencies and re-run the installer." + exit 1 fi - - print_success "All dependencies satisfied" + print_status "All dependencies satisfied" } # Function to download apt-layer diff --git a/src/apt-layer/CHANGELOG.md b/src/apt-layer/CHANGELOG.md index 1dcd4cd..1290233 100644 --- a/src/apt-layer/CHANGELOG.md +++ b/src/apt-layer/CHANGELOG.md @@ -7,6 +7,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### [2025-01-28 UTC] - COMPOSEFS PACKAGE INTEGRATION: DEBIAN/FEDORA PACKAGE SUPPORT +- **ComposeFS package integration completed**: Updated apt-layer to properly support official ComposeFS packages from Debian and Fedora repositories. +- **Debian package structure analysis**: Analyzed official Debian ComposeFS packaging from [salsa.debian.org](https://salsa.debian.org/debian/composefs): + - `composefs` package: Contains userspace tools (`mkcomposefs`, `composefs-info`, `mount.composefs`) + - `libcomposefs1` package: Contains runtime library (`libcomposefs.so.1`) + - `libcomposefs-dev` package: Contains development headers and pkg-config files +- **Fedora package structure analysis**: Analyzed official Fedora ComposeFS packaging: + - `composefs` package: Contains userspace tools + - `composefs-libs` package: Contains runtime library + - `composefs-devel` package: Contains development files +- **Dependencies.json enhanced**: Updated `src/apt-layer/config/dependencies.json` with: + - Added `composefs_packages` array with proper package names for each distribution + - Added `package_install_commands` section with distribution-specific installation commands + - Support for both Debian (`composefs`, `libcomposefs1`) and Fedora (`composefs`, `composefs-libs`) packages + - Comprehensive installation commands for all dependency categories (core, composefs, container, oci, bootloader) +- **Dependency checking improved**: Enhanced `src/apt-layer/scriptlets/01-dependencies.sh`: + - Added distribution detection (Debian vs Fedora) + - Dynamic installation command generation based on detected distribution + - Improved `print_install_instructions()` function with distribution-specific commands + - Updated fallback JSON to include new package structure and installation commands +- **Installation instructions enhanced**: Users now get distribution-specific installation commands: + - Debian/Ubuntu: `apt install -y composefs libcomposefs1` + - Fedora/RHEL: `dnf install -y composefs composefs-libs` + - Automatic detection and appropriate command suggestions +- **ComposeFS integration validated**: Confirmed apt-layer's ComposeFS integration follows official packaging: + - Uses correct tool names (`mkcomposefs`, `composefs-info`, `mount.composefs`) + - Proper mounting with `mount -t composefs` and correct options + - Follows official ComposeFS usage patterns from upstream documentation +- **Result**: apt-layer now properly supports official ComposeFS packages from both Debian and Fedora repositories, with distribution-specific installation instructions and proper dependency management. + ### [2025-07-14 UTC] - NAMING STANDARDIZATION: REMOVED ALL PARTICLE-OS/UBLUE REFERENCES FROM PATHS - **Complete path naming standardization**: Removed all references to "particle-os", "particle", "ublue", and "ucore" from path names throughout the entire codebase. - **Consistent apt-layer naming**: All persistent and runtime paths now use only "apt-layer" naming: @@ -982,4 +1012,93 @@ This project is part of the Particle-OS system tools and follows the same licens - Overlay and atomic install workflows validated, including rollback readiness. - **Documentation & Code Quality:** - Modular scriptlets and compiled script updated to reflect all improvements. - - Overlay and atomic install best practices documented in TODO. \ No newline at end of file + - Overlay and atomic install best practices documented in TODO. + +### [2025-07-14 UTC] - DEPENDENCY VALIDATION & INSTALL INSTRUCTIONS IMPROVED +- **Dependency validation improvements:** + - Added `skopeo` as a required dependency for OCI operations. + - Explicitly require `podman` or `docker` for container-based operations (mirroring rpm-ostree). + - Added `findmnt` and `numfmt` to core dependencies for system checks and disk space validation. + - Added bootloader tool checks (`efibootmgr`, `grub-install`, `update-grub`, `bootctl`) for bootloader management commands. + - Updated "Quick fix" and installation instructions to include all required packages: + - `skopeo`, `podman`, `docker.io`, `efibootmgr`, `grub-common`, `systemd-boot`, and all core utilities. + +### [2025-07-14 UTC] - REAL COMPOSEFS BINARY INTEGRATION WITH FALLBACK SUPPORT +- **Real ComposeFS binary integration**: Updated all scriptlets to use the actual `composefs` C binary (same as Fedora/rpm-ostree) as the primary backend. +- **Fallback support**: Maintained backward compatibility by falling back to `composefs-alternative.sh` when the real binary is not available. +- **Updated functions**: All ComposeFS operations now prioritize the real binary: + - `composefs_create()` - Creates ComposeFS images using real binary or fallback + - `composefs_mount()` - Mounts ComposeFS images using real binary or fallback + - `composefs_unmount()` - Unmounts ComposeFS images using real binary or fallback + - `composefs_list_images()` - Lists ComposeFS images using real binary or fallback + - `composefs_image_exists()` - Checks image existence using real binary or fallback + - `composefs_remove_image()` - Removes ComposeFS images using real binary or fallback + - `create_composefs_layer()` - Container-based layer creation with real binary support + - OCI integration functions - Export/import with real binary support + - Live overlay functions - Layer creation with real binary support +- **Performance improvement**: Using the real C implementation provides better performance and compatibility with rpm-ostree. +- **Archived composefs-alternative.sh**: The shell script version remains available as a fallback for systems without the real binary. + +### [2025-07-14 UTC] - JSON-BASED DEPENDENCY MANAGEMENT SYSTEM IMPLEMENTED +- **Centralized dependency management**: Moved all dependencies to a single `dependencies.json` file for maintainability and consistency. +- **Dynamic dependency checking**: Both `apt-layer.sh` and `install-apt-layer.sh` now use embedded JSON for category-based dependency validation: + - Core dependencies: chroot, apt-get, dpkg, jq, mount, umount, findmnt, numfmt + - Container dependencies: podman, docker (mirrors rpm-ostree model) + - OCI dependencies: skopeo + - ComposeFS dependencies: mksquashfs, unsquashfs + - Bootloader dependencies: efibootmgr, grub-install, update-grub, bootctl + - Security dependencies: curl, wget, gpg +- **Compiler integration**: Updated both `compile.sh` and `compile-installer.sh` to automatically embed `dependencies.json` as `APT_LAYER_DEPENDENCIES_JSON` variable in compiled scripts. +- **Template updates**: Updated `install-apt-layer.template.sh` to use embedded JSON for dependency checks and install instructions. +- **Scriptlet refactoring**: Refactored `01-dependencies.sh` to use `jq` for parsing embedded JSON and extracting relevant dependency groups based on command type. +- **Fallback support**: Maintained fallback dependency definitions in case JSON is not available. +- **Benefits**: Single source of truth for dependencies, easier maintenance, consistent dependency checking across all tools, and dynamic category-based validation. + +### [2025-07-14 UTC] - COMPOSEFS INTEGRATION CORRECTED WITH PROPER UPSTREAM USAGE +- **ComposeFS integration corrected**: Fixed all scriptlets to use the correct upstream ComposeFS tools and workflow: + - **Image creation**: Now uses `mkcomposefs --digest-store=` with proper digest store + - **Mounting**: Now uses `mount -t composefs -o basedir= ` with correct syntax + - **Unmounting**: Uses standard `umount` command + - **Image management**: Lists images by scanning `.composefs` files, removes by deleting files +- **Updated dependencies**: Added proper ComposeFS tools to dependencies.json: + - `mkcomposefs` - For creating ComposeFS images + - `composefs-info` - For inspecting and managing images + - `mount.composefs` - For mounting (used by mount -t composefs) + - `mksquashfs` and `unsquashfs` - For underlying squashfs operations +- **Documentation**: Created comprehensive `docs/apt-layer/composefs.md` with: + - Correct workflow using `mkcomposefs` and `mount -t composefs` + - Digest store integration for content-addressed files + - Multiple basedir support for complex layering + - Advanced mount options (verity, idmap, upperdir/workdir) + - Integration with `composefs-info` for inspection and management +- **Fallback support**: Maintained backward compatibility with `composefs-alternative.sh` for systems without upstream tools +- **Performance**: Proper upstream integration provides better performance and compatibility with rpm-ostree + +### [2025-07-14 UTC] - COMPOSEFS SCRIPTLET CORRECTIONS - ALL COMMANDS FIXED +- **Comprehensive scriptlet corrections**: Fixed all incorrect `composefs` command references throughout the codebase: + - **Container scriptlet (04-container.sh)**: Updated all functions to use proper tools: + - `create_composefs_layer()` - Now uses `mkcomposefs` with `--digest-store` + - `container_remove_layer()` - Now uses file removal instead of non-existent `composefs remove` + - `container_list_layers()` - Now uses `find` to scan `.composefs` files + - `container_layer_info()` - Now uses `composefs-info ls` for inspection + - `container_mount_layer()` - Now uses `mount -t composefs` with proper options + - `container_unmount_layer()` - Now uses `umount` instead of non-existent `composefs unmount` + - **OCI integration scriptlet (06-oci-integration.sh)**: Fixed export/import functions: + - Image existence check now uses `composefs-info ls` + - Mounting now uses `mount -t composefs` with `basedir` option + - Unmounting now uses `umount` + - Image creation now uses `mkcomposefs` with `--digest-store` + - **Live overlay scriptlet (05-live-overlay.sh)**: Fixed layer creation: + - `create_composefs_layer()` now uses `mkcomposefs` with proper object store + - **Dpkg direct install scriptlet (24-dpkg-direct-install.sh)**: Fixed layer creation: + - Now uses `mkcomposefs` with `--digest-store` instead of non-existent `composefs create` +- **Proper tool usage**: All scriptlets now correctly use: + - `mkcomposefs` for image creation (with `--digest-store`) + - `mount -t composefs` for mounting (with `basedir` option) + - `umount` for unmounting + - `composefs-info` for image inspection + - File operations for listing/removing images +- **Fallback support**: Maintained backward compatibility with `composefs-alternative.sh` +- **Consistency**: All scriptlets now follow the same pattern and use the correct upstream tools + +### [2025-07-14 UTC] - COMPOSEFS INTEGRATION CORRECTED WITH PROPER UPSTREAM USAGE \ No newline at end of file diff --git a/src/apt-layer/compile-installer.sh b/src/apt-layer/compile-installer.sh index 37e45a1..85e7d6c 100644 --- a/src/apt-layer/compile-installer.sh +++ b/src/apt-layer/compile-installer.sh @@ -89,25 +89,26 @@ if [[ ! -d "$OUTPUT_DIR" ]]; then mkdir -p "$OUTPUT_DIR" fi -# Read the template file -print_status "Reading template file..." -template_content=$(cat "$TEMPLATE_FILE") - -# Extract the JSON configuration -print_status "Extracting current paths.json configuration..." -current_config=$(cat "$CONFIG_FILE") - -# Create the compiled installer -print_status "Compiling installer with embedded configuration..." - -# Create the compiled installer by replacing the placeholder using awk -awk -v json="$current_config" ' - /PATHS_JSON_PLACEHOLDER/ { - print json - next - } - { print } -' "$TEMPLATE_FILE" > "$OUTPUT_FILE" +# Embed dependencies.json as a shell variable at the top +DEPENDENCIES_JSON_FILE="$SCRIPT_DIR/config/dependencies.json" +if [[ -f "$DEPENDENCIES_JSON_FILE" ]]; then + DEPENDENCIES_JSON_CONTENT=$(cat "$DEPENDENCIES_JSON_FILE") + { + echo "# Embedded dependencies.json" + echo "APT_LAYER_DEPENDENCIES_JSON=\$(cat << 'EOF'" + echo "$DEPENDENCIES_JSON_CONTENT" + echo "EOF" + echo ")" + echo "" + awk -v json="$(cat "$CONFIG_FILE")" ' + /PATHS_JSON_PLACEHOLDER/ { print json; next } + { print } + ' "$TEMPLATE_FILE" + } > "$OUTPUT_FILE" +else + print_error "dependencies.json not found, cannot embed dependencies." + exit 1 +fi # Make it executable chmod +x "$OUTPUT_FILE" @@ -132,6 +133,7 @@ print_status "Lines of code: $(wc -l < "$OUTPUT_FILE")" print_status "" print_status "The compiled install-apt-layer.sh now includes:" print_status "- Latest paths.json configuration embedded" +print_status "- dependencies.json embedded for dynamic dependency checks" print_status "- All installation and uninstallation functionality" print_status "- Dependency checking and installation" print_status "- System initialization" diff --git a/src/apt-layer/compile.sh b/src/apt-layer/compile.sh index 8804777..3fc1092 100644 --- a/src/apt-layer/compile.sh +++ b/src/apt-layer/compile.sh @@ -285,6 +285,21 @@ fi " script_content+=("$config_sourcing") +# Read and embed dependencies.json as a shell variable at the top +update_progress "Embedding: dependencies.json" 8 +DEPENDENCIES_JSON_FILE="$SCRIPT_DIR/config/dependencies.json" +if [[ -f "$DEPENDENCIES_JSON_FILE" ]]; then + DEPENDENCIES_JSON_CONTENT=$(cat "$DEPENDENCIES_JSON_FILE") + script_content+=("# Embedded dependencies.json") + script_content+=("APT_LAYER_DEPENDENCIES_JSON=\$(cat << 'EOF'") + script_content+=("$DEPENDENCIES_JSON_CONTENT") + script_content+=("EOF") + script_content+=(")") + script_content+=("") +else + print_warning "dependencies.json not found, using fallback in scriptlet." +fi + # Function to add scriptlet content with error handling add_scriptlet() { local scriptlet_name="$1" diff --git a/src/apt-layer/config/apt-layer-settings.json b/src/apt-layer/config/apt-layer-settings.json index cb92778..9ea0573 100644 --- a/src/apt-layer/config/apt-layer-settings.json +++ b/src/apt-layer/config/apt-layer-settings.json @@ -9,22 +9,14 @@ "layer_signing": true }, "container_runtimes": { - "supported": ["skopeo", "podman", "docker", "systemd-nspawn"], - "preference_order": ["skopeo", "podman", "docker", "systemd-nspawn"], - "skopeo": { - "description": "OCI-focused container operations", - "primary_use": "oci_integration" - }, + "supported": ["podman", "docker"], + "preference_order": ["podman", "docker"], "podman": { - "description": "Rootless container runtime", + "description": "Rootless container runtime (default, recommended)", "primary_use": "general" }, "docker": { - "description": "Traditional container runtime", - "primary_use": "general" - }, - "systemd-nspawn": { - "description": "Systemd-based container runtime", + "description": "Traditional container runtime (fallback)", "primary_use": "fallback" } }, diff --git a/src/apt-layer/config/dependencies.json b/src/apt-layer/config/dependencies.json new file mode 100644 index 0000000..2aeadab --- /dev/null +++ b/src/apt-layer/config/dependencies.json @@ -0,0 +1,39 @@ +{ + "core": [ + "chroot", "apt-get", "dpkg", "jq", "mount", "umount", "findmnt", "numfmt" + ], + "container": [ + "podman", "docker" + ], + "oci": [ + "skopeo" + ], + "composefs": [ + "mkcomposefs", "composefs-info", "mount.composefs", "mksquashfs", "unsquashfs" + ], + "composefs_packages": [ + "composefs", "libcomposefs1" + ], + "bootloader": [ + "efibootmgr", "grub-install", "update-grub", "bootctl" + ], + "security": [ + "curl", "wget", "gpg" + ], + "package_install_commands": { + "debian": { + "composefs": "apt install -y composefs libcomposefs1", + "container": "apt install -y podman docker.io", + "oci": "apt install -y skopeo", + "bootloader": "apt install -y efibootmgr grub-common systemd-boot", + "core": "apt install -y squashfs-tools jq coreutils util-linux" + }, + "fedora": { + "composefs": "dnf install -y composefs composefs-libs", + "container": "dnf install -y podman docker", + "oci": "dnf install -y skopeo", + "bootloader": "dnf install -y efibootmgr grub2-tools systemd-boot", + "core": "dnf install -y squashfs-tools jq coreutils util-linux" + } + } +} \ No newline at end of file diff --git a/src/apt-layer/scriptlets/00-header.sh b/src/apt-layer/scriptlets/00-header.sh index 4c28f84..bb9fe87 100644 --- a/src/apt-layer/scriptlets/00-header.sh +++ b/src/apt-layer/scriptlets/00-header.sh @@ -341,9 +341,22 @@ composefs_create() { log_debug "Creating ComposeFS image: $image_name from $source_dir" "apt-layer" - if ! "$COMPOSEFS_SCRIPT" create "$image_name" "$source_dir"; then - log_error "Failed to create ComposeFS image: $image_name" "apt-layer" - return 1 + # Try real mkcomposefs binary first + if command -v mkcomposefs >/dev/null 2>&1; then + # Create object store directory (same directory as image) + local object_store_dir=$(dirname "$image_name") + mkdir -p "$object_store_dir" + + if ! mkcomposefs "$source_dir" "$image_name" --digest-store="$object_store_dir"; then + log_error "Failed to create ComposeFS image with mkcomposefs: $image_name" "apt-layer" + return 1 + fi + else + # Fallback to composefs-alternative.sh + if ! "$COMPOSEFS_SCRIPT" create "$image_name" "$source_dir"; then + log_error "Failed to create ComposeFS image: $image_name" "apt-layer" + return 1 + fi fi log_success "ComposeFS image created: $image_name" "apt-layer" @@ -356,9 +369,25 @@ composefs_mount() { log_debug "Mounting ComposeFS image: $image_name to $mount_point" "apt-layer" - if ! "$COMPOSEFS_SCRIPT" mount "$image_name" "$mount_point"; then - log_error "Failed to mount ComposeFS image: $image_name to $mount_point" "apt-layer" - return 1 + # Try real mount with composefs filesystem + if command -v mkcomposefs >/dev/null 2>&1; then + # Create mount point + mkdir -p "$mount_point" + + # Determine object store directory (same directory as image for now) + local object_store_dir=$(dirname "$image_name") + + # Mount using composefs filesystem type + if ! mount -t composefs -o "basedir=$object_store_dir" "$image_name" "$mount_point"; then + log_error "Failed to mount ComposeFS image with mount: $image_name to $mount_point" "apt-layer" + return 1 + fi + else + # Fallback to composefs-alternative.sh + if ! "$COMPOSEFS_SCRIPT" mount "$image_name" "$mount_point"; then + log_error "Failed to mount ComposeFS image: $image_name to $mount_point" "apt-layer" + return 1 + fi fi log_success "ComposeFS image mounted: $image_name to $mount_point" "apt-layer" @@ -370,9 +399,18 @@ composefs_unmount() { log_debug "Unmounting ComposeFS image from: $mount_point" "apt-layer" - if ! "$COMPOSEFS_SCRIPT" unmount "$mount_point"; then - log_error "Failed to unmount ComposeFS image from: $mount_point" "apt-layer" - return 1 + # Try real umount + if command -v mkcomposefs >/dev/null 2>&1; then + if ! umount "$mount_point"; then + log_error "Failed to unmount ComposeFS image with umount: $mount_point" "apt-layer" + return 1 + fi + else + # Fallback to composefs-alternative.sh + if ! "$COMPOSEFS_SCRIPT" unmount "$mount_point"; then + log_error "Failed to unmount ComposeFS image from: $mount_point" "apt-layer" + return 1 + fi fi log_success "ComposeFS image unmounted from: $mount_point" "apt-layer" @@ -381,17 +419,34 @@ composefs_unmount() { composefs_list_images() { log_debug "Listing ComposeFS images" "apt-layer" - "$COMPOSEFS_SCRIPT" list-images + + # Try to list images from workspace directory + if command -v mkcomposefs >/dev/null 2>&1; then + # List .composefs files in the workspace + find "$WORKSPACE/images" -name "*.composefs" -type f 2>/dev/null | sed 's|.*/||' | sed 's|\.composefs$||' || true + else + # Fallback to composefs-alternative.sh + "$COMPOSEFS_SCRIPT" list-images + fi } composefs_image_exists() { local image_name="$1" - # Check if image exists by trying to list it - if "$COMPOSEFS_SCRIPT" list-images | grep -q "^$image_name$"; then - return 0 + # Check if image exists by looking for the .composefs file + if command -v mkcomposefs >/dev/null 2>&1; then + if [[ -f "$WORKSPACE/images/$image_name.composefs" ]]; then + return 0 + else + return 1 + fi else - return 1 + # Fallback to composefs-alternative.sh + if "$COMPOSEFS_SCRIPT" list-images | grep -q "^$image_name$"; then + return 0 + else + return 1 + fi fi } @@ -400,9 +455,18 @@ composefs_remove_image() { log_debug "Removing ComposeFS image: $image_name" "apt-layer" - if ! "$COMPOSEFS_SCRIPT" remove "$image_name"; then - log_error "Failed to remove ComposeFS image: $image_name" "apt-layer" - return 1 + # Try real file removal + if command -v mkcomposefs >/dev/null 2>&1; then + if ! rm -f "$WORKSPACE/images/$image_name.composefs"; then + log_error "Failed to remove ComposeFS image file: $image_name" "apt-layer" + return 1 + fi + else + # Fallback to composefs-alternative.sh + if ! "$COMPOSEFS_SCRIPT" remove "$image_name"; then + log_error "Failed to remove ComposeFS image: $image_name" "apt-layer" + return 1 + fi fi log_success "ComposeFS image removed: $image_name" "apt-layer" diff --git a/src/apt-layer/scriptlets/01-dependencies.sh b/src/apt-layer/scriptlets/01-dependencies.sh index 32230bc..51c8562 100644 --- a/src/apt-layer/scriptlets/01-dependencies.sh +++ b/src/apt-layer/scriptlets/01-dependencies.sh @@ -1,5 +1,89 @@ #!/bin/bash # Enhanced dependency checking and validation for Particle-OS apt-layer Tool +# --- BEGIN DEPENDENCY JSON LOADING --- +# The dependencies JSON will be embedded as APT_LAYER_DEPENDENCIES_JSON in the compiled script. +# If not present, fallback to a default minimal set. +APT_LAYER_DEPENDENCIES_JSON="${APT_LAYER_DEPENDENCIES_JSON:-} +{ + \"core\": [\"chroot\", \"apt-get\", \"dpkg\", \"jq\", \"mount\", \"umount\", \"findmnt\", \"numfmt\"], + \"container\": [\"podman\", \"docker\"], + \"oci\": [\"skopeo\"], + \"composefs\": [\"mkcomposefs\", \"composefs-info\", \"mount.composefs\", \"mksquashfs\", \"unsquashfs\"], + \"composefs_packages\": [\"composefs\", \"libcomposefs1\"], + \"bootloader\": [\"efibootmgr\", \"grub-install\", \"update-grub\", \"bootctl\"], + \"security\": [\"curl\", \"wget\", \"gpg\"], + \"package_install_commands\": { + \"debian\": { + \"composefs\": \"apt install -y composefs libcomposefs1\", + \"container\": \"apt install -y podman docker.io\", + \"oci\": \"apt install -y skopeo\", + \"bootloader\": \"apt install -y efibootmgr grub-common systemd-boot\", + \"core\": \"apt install -y squashfs-tools jq coreutils util-linux\" + }, + \"fedora\": { + \"composefs\": \"dnf install -y composefs composefs-libs\", + \"container\": \"dnf install -y podman docker\", + \"oci\": \"dnf install -y skopeo\", + \"bootloader\": \"dnf install -y efibootmgr grub2-tools systemd-boot\", + \"core\": \"dnf install -y squashfs-tools jq coreutils util-linux\" + } + } +}" + +get_deps_for_type() { + local type="$1" + local json="$APT_LAYER_DEPENDENCIES_JSON" + case "$type" in + --container|container) + echo "$json" | jq -r '.core[], .container[]' + ;; + --oci|oci) + echo "$json" | jq -r '.core[], .oci[]' + ;; + --composefs|composefs) + echo "$json" | jq -r '.core[], .composefs[]' + ;; + --bootloader|bootloader) + echo "$json" | jq -r '.core[], .bootloader[]' + ;; + --scan|security) + echo "$json" | jq -r '.core[], .security[]' + ;; + *) + echo "$json" | jq -r '.core[]' + ;; + esac +} + +print_install_instructions() { + local json="$APT_LAYER_DEPENDENCIES_JSON" + + # Detect distribution + local distro="debian" + if [[ -f /etc/fedora-release ]] || [[ -f /etc/redhat-release ]]; then + distro="fedora" + fi + + echo " Quick fix for common dependencies:" + local core_cmd=$(echo "$json" | jq -r ".package_install_commands.$distro.core") + echo " sudo $core_cmd" + echo "" + echo " For ComposeFS support:" + local composefs_cmd=$(echo "$json" | jq -r ".package_install_commands.$distro.composefs") + echo " sudo $composefs_cmd" + echo "" + echo " For container support:" + local container_cmd=$(echo "$json" | jq -r ".package_install_commands.$distro.container") + echo " sudo $container_cmd" + echo "" + echo " For bootloader support:" + local bootloader_cmd=$(echo "$json" | jq -r ".package_install_commands.$distro.bootloader") + echo " sudo $bootloader_cmd" + echo "" + echo " For more information, run: apt-layer --help" + echo "" +} + check_dependencies() { local command_type="${1:-}" local packages=("${@:2}") @@ -7,71 +91,28 @@ check_dependencies() { log_info "Checking dependencies for command: ${command_type:-general}" "apt-layer" local missing_deps=() - local missing_packages=() local missing_tools=() local missing_scripts=() local missing_modules=() - # Core system dependencies - local core_deps=("chroot" "apt-get" "dpkg" "jq" "mount" "umount") - for dep in "${core_deps[@]}"; do + # Use JSON to get the relevant dependencies + local deps=( $(get_deps_for_type "$command_type") ) + for dep in "${deps[@]}"; do if ! command -v "$dep" >/dev/null 2>&1; then missing_deps+=("$dep") missing_tools+=("$dep") fi done - # Container-specific dependencies - if [[ "$command_type" == "--container" || "$command_type" == "container" ]]; then - local container_deps=("podman" "docker" "systemd-nspawn") - local found_container=false - - for dep in "${container_deps[@]}"; do - if command -v "$dep" >/dev/null 2>&1; then - found_container=true - break - fi - done - - if [[ "$found_container" == "false" ]]; then - missing_deps+=("container-runtime") - missing_tools+=("podman, docker, or systemd-nspawn") - fi - fi - - # ComposeFS-specific dependencies - if [[ "$command_type" == "--composefs" || "$command_type" == "composefs" ]]; then - local composefs_deps=("mksquashfs" "unsquashfs" "skopeo") - for dep in "${composefs_deps[@]}"; do - if ! command -v "$dep" >/dev/null 2>&1; then - missing_deps+=("$dep") - missing_tools+=("$dep") - fi - done - fi - - # Security scanning dependencies - if [[ "$command_type" == "--scan" || "$command_type" == "security" ]]; then - local security_deps=("curl" "wget" "gpg") - for dep in "${security_deps[@]}"; do - if ! command -v "$dep" >/dev/null 2>&1; then - missing_deps+=("$dep") - missing_tools+=("$dep") - fi - done - fi - - # Check for required scripts + # Check for required scripts (unchanged) local required_scripts=( "composefs-alternative.sh:/usr/local/bin/composefs-alternative.sh" "bootc-alternative.sh:/usr/local/bin/bootc-alternative.sh" "bootupd-alternative.sh:/usr/local/bin/bootupd-alternative.sh" ) - for script_info in "${required_scripts[@]}"; do local script_name="${script_info%%:*}" local script_path="${script_info##*:}" - if [[ ! -f "$script_path" ]]; then missing_deps+=("$script_name") missing_scripts+=("$script_name") @@ -81,10 +122,10 @@ check_dependencies() { fi done - # Check for kernel modules + # Check for kernel modules (unchanged) check_kernel_modules - # Validate package names if provided + # Validate package names if provided (unchanged) if [[ ${#packages[@]} -gt 0 ]]; then if ! validate_package_names "${packages[@]}"; then return 1 @@ -96,48 +137,38 @@ check_dependencies() { echo "" log_error "Missing dependencies detected!" "apt-layer" echo "" - if [[ ${#missing_tools[@]} -gt 0 ]]; then - echo "ð¦ Missing system packages:" + echo " Missing system packages:" for tool in "${missing_tools[@]}"; do - echo " ⢠$tool" + echo " $tool" done echo "" echo " Install with: sudo apt install -y ${missing_tools[*]}" echo "" fi - if [[ ${#missing_scripts[@]} -gt 0 ]]; then - echo "ð Missing or non-executable scripts:" + echo " Missing or non-executable scripts:" for script in "${missing_scripts[@]}"; do - echo " ⢠$script" + echo " $script" done echo "" echo " Ensure scripts are installed and executable:" echo " sudo chmod +x /usr/local/bin/*-alternative.sh" echo "" fi - if [[ ${#missing_modules[@]} -gt 0 ]]; then - echo "ð§ Missing kernel modules:" + echo " Missing kernel modules:" for module in "${missing_modules[@]}"; do - echo " ⢠$module" + echo " $module" done echo "" echo " Load modules with: sudo modprobe ${missing_modules[*]}" echo " Or install with: sudo apt install linux-modules-extra-\$(uname -r)" echo "" fi - - echo "ð¡ Quick fix for common dependencies:" - echo " sudo apt install -y squashfs-tools jq coreutils util-linux" - echo "" - echo "ð For more information, run: apt-layer --help" - echo "" - + print_install_instructions exit 1 fi - log_success "All dependencies found and validated" "apt-layer" } @@ -265,7 +296,7 @@ show_actionable_error() { case "$error_type" in "missing_dependencies") - echo "ð§ To fix this issue:" + echo "� To fix this issue:" echo " 1. Install missing dependencies:" echo " sudo apt update" echo " sudo apt install -y $packages" @@ -278,7 +309,7 @@ show_actionable_error() { echo "" ;; "permission_denied") - echo "ð Permission issue detected:" + echo "� Permission issue detected:" echo " This command requires root privileges." echo "" echo " Run with sudo:" @@ -286,7 +317,7 @@ show_actionable_error() { echo "" ;; "invalid_arguments") - echo "ð Invalid arguments provided:" + echo "� Invalid arguments provided:" echo " Check the command syntax and try again." echo "" echo " For help, run:" @@ -295,7 +326,7 @@ show_actionable_error() { echo "" ;; "system_not_initialized") - echo "ð System not initialized:" + echo "� System not initialized:" echo " Particle-OS needs to be initialized first." echo "" echo " Run initialization:" @@ -303,7 +334,7 @@ show_actionable_error() { echo "" ;; "disk_space") - echo "ð¾ Insufficient disk space:" + echo "� Insufficient disk space:" echo " Free up space or use a different location." echo "" echo " Check available space:" @@ -311,7 +342,7 @@ show_actionable_error() { echo "" ;; *) - echo "â Unknown error occurred." + echo "� Unknown error occurred." echo " Please check the error message above." echo "" echo " For help, run: apt-layer --help" @@ -319,10 +350,10 @@ show_actionable_error() { ;; esac - echo "ð For more information:" - echo " ⢠apt-layer --help" - echo " ⢠apt-layer --help-full" - echo " ⢠apt-layer --examples" + echo "� For more information:" + echo " � apt-layer --help" + echo " � apt-layer --help-full" + echo " � apt-layer --examples" echo "" } diff --git a/src/apt-layer/scriptlets/04-container.sh b/src/apt-layer/scriptlets/04-container.sh index 02b9aac..446d2dc 100644 --- a/src/apt-layer/scriptlets/04-container.sh +++ b/src/apt-layer/scriptlets/04-container.sh @@ -33,37 +33,12 @@ detect_container_runtime() { log_warning "Configured runtime 'docker' not found, falling back to detection" "apt-layer" fi ;; - systemd-nspawn) - if command -v systemd-nspawn &> /dev/null; then - CONTAINER_RUNTIME="systemd-nspawn" - log_info "Using configured container runtime: systemd-nspawn" "apt-layer" - return 0 - else - log_warning "Configured runtime 'systemd-nspawn' not found, falling back to detection" "apt-layer" - fi - ;; - skopeo) - if command -v skopeo &> /dev/null; then - CONTAINER_RUNTIME="skopeo" - log_info "Using configured container runtime: skopeo" "apt-layer" - return 0 - else - log_warning "Configured runtime 'skopeo' not found, falling back to detection" "apt-layer" - fi - ;; esac fi # Auto-detection fallback (in order of preference) log_info "Auto-detecting container runtime" "apt-layer" - # Check for skopeo first (preferred for OCI operations) - if command -v skopeo &> /dev/null; then - CONTAINER_RUNTIME="skopeo" - log_info "Auto-detected skopeo as container runtime" "apt-layer" - return 0 - fi - # Check for podman (preferred for rootless) if command -v podman &> /dev/null; then CONTAINER_RUNTIME="podman" @@ -78,14 +53,7 @@ detect_container_runtime() { return 0 fi - # Check for systemd-nspawn as last resort - if command -v systemd-nspawn &> /dev/null; then - CONTAINER_RUNTIME="systemd-nspawn" - log_info "Auto-detected systemd-nspawn as container runtime" "apt-layer" - return 0 - fi - - log_error "No supported container runtime found (skopeo, podman, docker, or systemd-nspawn)" "apt-layer" + log_error "No supported container runtime found (podman or docker required)" "apt-layer" return 1 } @@ -123,10 +91,6 @@ set_global_container_runtime_vars() { # Set runtime-specific variables case "$CONTAINER_RUNTIME" in - skopeo) - export CONTAINER_RUNTIME_TYPE="oci" - export CONTAINER_RUNTIME_DESCRIPTION="OCI-focused container operations" - ;; podman) export CONTAINER_RUNTIME_TYPE="general" export CONTAINER_RUNTIME_DESCRIPTION="Rootless container runtime" @@ -135,10 +99,6 @@ set_global_container_runtime_vars() { export CONTAINER_RUNTIME_TYPE="general" export CONTAINER_RUNTIME_DESCRIPTION="Traditional container runtime" ;; - systemd-nspawn) - export CONTAINER_RUNTIME_TYPE="fallback" - export CONTAINER_RUNTIME_DESCRIPTION="Systemd-based container runtime" - ;; esac log_debug "Container runtime variables set: $CONTAINER_RUNTIME ($CONTAINER_RUNTIME_TYPE)" "apt-layer" @@ -151,12 +111,6 @@ validate_container_runtime() { log_info "Validating container runtime: $runtime" "apt-layer" case "$runtime" in - skopeo) - if ! skopeo --version &> /dev/null; then - log_error "skopeo is not properly configured" "apt-layer" - return 1 - fi - ;; podman) if ! podman info &> /dev/null; then log_error "podman is not properly configured" "apt-layer" @@ -169,9 +123,6 @@ validate_container_runtime() { return 1 fi ;; - systemd-nspawn) - # systemd-nspawn doesn't need special validation - ;; *) log_error "Unsupported container runtime: $runtime" "apt-layer" return 1 @@ -266,10 +217,6 @@ create_base_container_image() { docker pull "$base_image" fi ;; - systemd-nspawn) - # systemd-nspawn uses host filesystem - log_info "Using host filesystem for systemd-nspawn" "apt-layer" - ;; esac log_success "OCI base image ready: $base_image" "apt-layer" @@ -303,12 +250,6 @@ container_install_packages() { # Run package installation in container case "$CONTAINER_RUNTIME" in - skopeo) - if ! run_skopeo_install "$base_image" "$container_name" "$temp_dir" "${packages[@]}"; then - rollback_transaction - return 1 - fi - ;; podman) if ! run_podman_install "$base_image" "$container_name" "$temp_dir" "${packages[@]}"; then rollback_transaction @@ -321,11 +262,10 @@ container_install_packages() { return 1 fi ;; - systemd-nspawn) - if ! run_nspawn_install "$base_image" "$container_name" "$temp_dir" "${packages[@]}"; then - rollback_transaction - return 1 - fi + *) + log_error "Unsupported container runtime: $CONTAINER_RUNTIME" "apt-layer" + rollback_transaction + return 1 ;; esac @@ -585,10 +525,22 @@ create_composefs_layer() { local image_dir="$WORKSPACE/images/$new_image" mkdir -p "$image_dir" - # Use ComposeFS backend to create layer - if ! "$COMPOSEFS_SCRIPT" create "$new_image" "$temp_dir"; then - log_error "Failed to create ComposeFS layer" "apt-layer" - return 1 + # Try real mkcomposefs binary first + if command -v mkcomposefs >/dev/null 2>&1; then + # Create object store directory (same directory as image) + local object_store_dir=$(dirname "$new_image") + mkdir -p "$object_store_dir" + + if ! mkcomposefs "$temp_dir" "$new_image" --digest-store="$object_store_dir"; then + log_error "Failed to create ComposeFS layer with mkcomposefs" "apt-layer" + return 1 + fi + else + # Fallback to composefs-alternative.sh + if ! "$COMPOSEFS_SCRIPT" create "$new_image" "$temp_dir"; then + log_error "Failed to create ComposeFS layer" "apt-layer" + return 1 + fi fi log_success "ComposeFS layer created: $new_image" "apt-layer" @@ -626,10 +578,18 @@ container_remove_layer() { log_info "Removing container-based layer: $image_name" "apt-layer" - # Use ComposeFS backend to remove layer - if ! "$COMPOSEFS_SCRIPT" remove "$image_name"; then - log_error "Failed to remove ComposeFS layer" "apt-layer" - return 1 + # Try real file removal + if command -v mkcomposefs >/dev/null 2>&1; then + if ! rm -f "$WORKSPACE/images/$image_name.composefs"; then + log_error "Failed to remove ComposeFS layer file: $image_name" "apt-layer" + return 1 + fi + else + # Fallback to composefs-alternative.sh + if ! "$COMPOSEFS_SCRIPT" remove "$image_name"; then + log_error "Failed to remove ComposeFS layer" "apt-layer" + return 1 + fi fi log_success "Container-based layer removed: $image_name" "apt-layer" @@ -640,10 +600,16 @@ container_remove_layer() { container_list_layers() { log_info "Listing container-based layers" "apt-layer" - # Use ComposeFS backend to list layers - if ! "$COMPOSEFS_SCRIPT" list-images; then - log_error "Failed to list ComposeFS layers" "apt-layer" - return 1 + # Try to list images from workspace directory + if command -v mkcomposefs >/dev/null 2>&1; then + # List .composefs files in the workspace + find "$WORKSPACE/images" -name "*.composefs" -type f 2>/dev/null | sed 's|.*/||' | sed 's|\.composefs$||' || true + else + # Fallback to composefs-alternative.sh + if ! "$COMPOSEFS_SCRIPT" list-images; then + log_error "Failed to list ComposeFS layers" "apt-layer" + return 1 + fi fi return 0 @@ -655,10 +621,18 @@ container_layer_info() { log_info "Getting container-based layer info: $image_name" "apt-layer" - # Use ComposeFS backend to get layer info - if ! "$COMPOSEFS_SCRIPT" info "$image_name"; then - log_error "Failed to get ComposeFS layer info" "apt-layer" - return 1 + # Try real composefs-info binary first + if command -v composefs-info >/dev/null 2>&1; then + if ! composefs-info ls "$WORKSPACE/images/$image_name.composefs"; then + log_error "Failed to get ComposeFS layer info with composefs-info" "apt-layer" + return 1 + fi + else + # Fallback to composefs-alternative.sh + if ! "$COMPOSEFS_SCRIPT" info "$image_name"; then + log_error "Failed to get ComposeFS layer info" "apt-layer" + return 1 + fi fi return 0 @@ -671,10 +645,25 @@ container_mount_layer() { log_info "Mounting container-based layer: $image_name at $mount_point" "apt-layer" - # Use ComposeFS backend to mount layer - if ! "$COMPOSEFS_SCRIPT" mount "$image_name" "$mount_point"; then - log_error "Failed to mount ComposeFS layer" "apt-layer" - return 1 + # Try real mount with composefs filesystem + if command -v mkcomposefs >/dev/null 2>&1; then + # Create mount point + mkdir -p "$mount_point" + + # Determine object store directory (same directory as image) + local object_store_dir=$(dirname "$image_name") + + # Mount using composefs filesystem type + if ! mount -t composefs -o "basedir=$object_store_dir" "$image_name" "$mount_point"; then + log_error "Failed to mount ComposeFS layer with mount: $image_name to $mount_point" "apt-layer" + return 1 + fi + else + # Fallback to composefs-alternative.sh + if ! "$COMPOSEFS_SCRIPT" mount "$image_name" "$mount_point"; then + log_error "Failed to mount ComposeFS layer" "apt-layer" + return 1 + fi fi log_success "Container-based layer mounted: $image_name at $mount_point" "apt-layer" @@ -687,10 +676,18 @@ container_unmount_layer() { log_info "Unmounting container-based layer at: $mount_point" "apt-layer" - # Use ComposeFS backend to unmount layer - if ! "$COMPOSEFS_SCRIPT" unmount "$mount_point"; then - log_error "Failed to unmount ComposeFS layer" "apt-layer" - return 1 + # Try real umount + if command -v mkcomposefs >/dev/null 2>&1; then + if ! umount "$mount_point"; then + log_error "Failed to unmount ComposeFS layer with umount: $mount_point" "apt-layer" + return 1 + fi + else + # Fallback to composefs-alternative.sh + if ! "$COMPOSEFS_SCRIPT" unmount "$mount_point"; then + log_error "Failed to unmount ComposeFS layer" "apt-layer" + return 1 + fi fi log_success "Container-based layer unmounted: $mount_point" "apt-layer" @@ -705,10 +702,6 @@ container_status() { echo "Runtime: $CONTAINER_RUNTIME" case "$CONTAINER_RUNTIME" in - skopeo) - echo "Skopeo version: $(skopeo --version 2>/dev/null || echo 'Not available')" - echo "Skopeo info: OCI-focused container operations" - ;; podman) echo "Podman version: $(podman --version 2>/dev/null || echo 'Not available')" echo "Podman info: $(podman info --format json 2>/dev/null | jq -r '.host.arch // "Unknown"' 2>/dev/null || echo 'Unknown')" @@ -717,9 +710,6 @@ container_status() { echo "Docker version: $(docker --version 2>/dev/null || echo 'Not available')" echo "Docker info: $(docker info --format '{{.Architecture}}' 2>/dev/null || echo 'Unknown')" ;; - systemd-nspawn) - echo "systemd-nspawn version: $(systemd-nspawn --version 2>/dev/null || echo 'Not available')" - ;; esac echo "" diff --git a/src/apt-layer/scriptlets/05-live-overlay.sh b/src/apt-layer/scriptlets/05-live-overlay.sh index 658ccbe..9a90648 100644 --- a/src/apt-layer/scriptlets/05-live-overlay.sh +++ b/src/apt-layer/scriptlets/05-live-overlay.sh @@ -441,7 +441,19 @@ create_composefs_layer() { local layer_name="$2" local message="$3" - # Use composefs-alternative to create layer + # Try real mkcomposefs binary first + if command -v mkcomposefs >/dev/null 2>&1; then + # Create object store directory (same directory as layer) + local object_store_dir=$(dirname "$layer_name") + mkdir -p "$object_store_dir" + + if mkcomposefs "$source_dir" "$layer_name" --digest-store="$object_store_dir"; then + log_success "ComposeFS layer created using mkcomposefs" "apt-layer" + return 0 + fi + fi + + # Fallback to composefs-alternative if command -v composefs-alternative >/dev/null 2>&1; then if composefs-alternative create-layer "$source_dir" "$layer_name" "$message"; then return 0 diff --git a/src/apt-layer/scriptlets/06-oci-integration.sh b/src/apt-layer/scriptlets/06-oci-integration.sh index 459d5f5..d782794 100644 --- a/src/apt-layer/scriptlets/06-oci-integration.sh +++ b/src/apt-layer/scriptlets/06-oci-integration.sh @@ -1,6 +1,6 @@ #!/bin/bash # OCI Integration for Particle-OS apt-layer Tool -# Provides ComposeFS â OCI export/import functionality for container-based layer creation +# Provides ComposeFS � OCI export/import functionality for container-based layer creation # OCI registry configuration declare -A OCI_REGISTRY_CONFIG @@ -108,9 +108,16 @@ export_oci_image() { fi # Check if ComposeFS image exists - if ! "$COMPOSEFS_SCRIPT" info "$composefs_image" >/dev/null 2>&1; then - log_error "ComposeFS image not found: $composefs_image" "apt-layer" - return 1 + if command -v composefs-info >/dev/null 2>&1; then + if ! composefs-info ls "$composefs_image" >/dev/null 2>&1; then + log_error "ComposeFS image not found: $composefs_image" "apt-layer" + return 1 + fi + else + if ! "$COMPOSEFS_SCRIPT" info "$composefs_image" >/dev/null 2>&1; then + log_error "ComposeFS image not found: $composefs_image" "apt-layer" + return 1 + fi fi # Create temporary directory @@ -125,10 +132,21 @@ export_oci_image() { mkdir -p "$mount_point" update_transaction_phase "mounting_composefs_image" - if ! "$COMPOSEFS_SCRIPT" mount "$composefs_image" "$mount_point"; then - log_error "Failed to mount ComposeFS image: $composefs_image" "apt-layer" - rollback_transaction - return 1 + if command -v mkcomposefs >/dev/null 2>&1; then + # Determine object store directory (same directory as image) + local object_store_dir=$(dirname "$composefs_image") + + if ! mount -t composefs -o "basedir=$object_store_dir" "$composefs_image" "$mount_point"; then + log_error "Failed to mount ComposeFS image: $composefs_image" "apt-layer" + rollback_transaction + return 1 + fi + else + if ! "$COMPOSEFS_SCRIPT" mount "$composefs_image" "$mount_point"; then + log_error "Failed to mount ComposeFS image: $composefs_image" "apt-layer" + rollback_transaction + return 1 + fi fi # Create OCI image structure @@ -151,7 +169,11 @@ export_oci_image() { fi # Unmount ComposeFS image - "$COMPOSEFS_SCRIPT" unmount "$mount_point" 2>/dev/null || true + if command -v mkcomposefs >/dev/null 2>&1; then + umount "$mount_point" 2>/dev/null || true + else + "$COMPOSEFS_SCRIPT" unmount "$mount_point" 2>/dev/null || true + fi commit_transaction log_success "ComposeFS image exported to OCI: $oci_image_name" "apt-layer" @@ -367,10 +389,22 @@ import_oci_image() { # Create ComposeFS image from extracted filesystem update_transaction_phase "creating_composefs_image" - if ! "$COMPOSEFS_SCRIPT" create "$composefs_image" "$rootfs_dir"; then - log_error "Failed to create ComposeFS image: $composefs_image" "apt-layer" - rollback_transaction - return 1 + if command -v mkcomposefs >/dev/null 2>&1; then + # Create object store directory (same directory as image) + local object_store_dir=$(dirname "$composefs_image") + mkdir -p "$object_store_dir" + + if ! mkcomposefs "$rootfs_dir" "$composefs_image" --digest-store="$object_store_dir"; then + log_error "Failed to create ComposeFS image: $composefs_image" "apt-layer" + rollback_transaction + return 1 + fi + else + if ! "$COMPOSEFS_SCRIPT" create "$composefs_image" "$rootfs_dir"; then + log_error "Failed to create ComposeFS image: $composefs_image" "apt-layer" + rollback_transaction + return 1 + fi fi commit_transaction @@ -540,9 +574,9 @@ oci_status() { echo "=== OCI Tool Configuration ===" echo "Preferred tool: $OCI_TOOL" echo "Available tools:" - command -v skopeo &> /dev/null && echo " â skopeo" - command -v podman &> /dev/null && echo " â podman" - command -v docker &> /dev/null && echo " â docker" + command -v skopeo &> /dev/null && echo " � skopeo" + command -v podman &> /dev/null && echo " � podman" + command -v docker &> /dev/null && echo " � docker" echo "" echo "=== OCI Workspace ===" diff --git a/src/apt-layer/scriptlets/24-dpkg-direct-install.sh b/src/apt-layer/scriptlets/24-dpkg-direct-install.sh index d03e6aa..de3ae1d 100644 --- a/src/apt-layer/scriptlets/24-dpkg-direct-install.sh +++ b/src/apt-layer/scriptlets/24-dpkg-direct-install.sh @@ -153,12 +153,6 @@ container_dpkg_install() { # Run dpkg installation in container case "$CONTAINER_RUNTIME" in - skopeo) - if ! run_skopeo_dpkg_install "$base_image" "$container_name" "$temp_dir" "${packages[@]}"; then - rollback_transaction - return 1 - fi - ;; podman) if ! run_podman_dpkg_install "$base_image" "$container_name" "$temp_dir" "${packages[@]}"; then rollback_transaction @@ -171,12 +165,6 @@ container_dpkg_install() { return 1 fi ;; - systemd-nspawn) - if ! run_nspawn_dpkg_install "$base_image" "$container_name" "$temp_dir" "${packages[@]}"; then - rollback_transaction - return 1 - fi - ;; *) log_error "Unsupported container runtime: $CONTAINER_RUNTIME" "apt-layer" rollback_transaction @@ -191,12 +179,25 @@ container_dpkg_install() { return 1 fi else - # Fallback: use composefs-alternative.sh - log_info "Using fallback ComposeFS layer creation" "apt-layer" - if ! "$COMPOSEFS_SCRIPT" create "$new_image" "$temp_dir"; then - log_error "Failed to create ComposeFS layer" "apt-layer" - rollback_transaction - return 1 + # Try real mkcomposefs binary first + if command -v mkcomposefs >/dev/null 2>&1; then + # Create object store directory (same directory as image) + local object_store_dir=$(dirname "$new_image") + mkdir -p "$object_store_dir" + + if ! mkcomposefs "$temp_dir" "$new_image" --digest-store="$object_store_dir"; then + log_error "Failed to create ComposeFS layer with mkcomposefs" "apt-layer" + rollback_transaction + return 1 + fi + else + # Fallback: use composefs-alternative.sh + log_info "Using fallback ComposeFS layer creation" "apt-layer" + if ! "$COMPOSEFS_SCRIPT" create "$new_image" "$temp_dir"; then + log_error "Failed to create ComposeFS layer" "apt-layer" + rollback_transaction + return 1 + fi fi fi @@ -215,56 +216,6 @@ container_dpkg_install() { return 0 } -# Skopeo-based dpkg installation (OCI-focused) -run_skopeo_dpkg_install() { - local base_image="$1" - local container_name="$2" - local temp_dir="$3" - shift 3 - local packages=("$@") - - log_info "Running skopeo-based dpkg installation" "apt-layer" - - # Skopeo is primarily for OCI operations, so we'll use chroot for package installation - # Create minimal container structure - mkdir -p "$temp_dir"/{bin,lib,lib64,usr,etc,var} - - # Set up base filesystem - if [[ -d "$WORKSPACE/images/$base_image" ]]; then - # Use ComposeFS image as base - log_info "Using ComposeFS image as base for skopeo dpkg" "apt-layer" - cp -a "$WORKSPACE/images/$base_image"/* "$temp_dir/" 2>/dev/null || true - else - # Use minimal Ubuntu base - log_info "Using minimal Ubuntu base for skopeo dpkg" "apt-layer" - # Copy essential files - cp -a /bin/bash "$temp_dir/bin/" - cp -a /lib/x86_64-linux-gnu "$temp_dir/lib/" - cp -a /usr/bin/dpkg "$temp_dir/usr/bin/" - cp -a /usr/bin/apt-get "$temp_dir/usr/bin/" - # Add minimal /etc structure - echo "deb http://archive.ubuntu.com/ubuntu/ jammy main" > "$temp_dir/etc/apt/sources.list" - fi - - # Download and install packages using dpkg - local install_cmd=" - apt-get update && - apt-get download ${packages[*]} && - dpkg -i *.deb && - apt-get install -f && - dpkg --configure -a && - apt-get clean - " - - if ! chroot "$temp_dir" /bin/bash -c "$install_cmd"; then - log_error "dpkg installation failed in skopeo container" "apt-layer" - return 1 - fi - - log_success "Skopeo-based dpkg installation completed" "apt-layer" - return 0 -} - # Podman-based dpkg installation run_podman_dpkg_install() { local base_image="$1" @@ -405,59 +356,6 @@ run_docker_dpkg_install() { return 0 } -# systemd-nspawn-based dpkg installation -run_nspawn_dpkg_install() { - local base_image="$1" - local container_name="$2" - local temp_dir="$3" - shift 3 - local packages=("$@") - - log_info "Running systemd-nspawn-based dpkg installation" "apt-layer" - - local container_dir="$WORKSPACE/containers/$container_name" - - # Create container directory - if [[ -d "$WORKSPACE/images/$base_image" ]]; then - # Use ComposeFS image as base - log_info "Using ComposeFS image as base for nspawn" "apt-layer" - cp -a "$WORKSPACE/images/$base_image" "$container_dir" - else - # Use host filesystem as base - log_info "Using host filesystem as base for nspawn" "apt-layer" - # Create minimal container structure - mkdir -p "$container_dir"/{bin,lib,lib64,usr,etc,var} - # Copy essential files from host - cp -a /bin/bash "$container_dir/bin/" - cp -a /lib/x86_64-linux-gnu "$container_dir/lib/" - cp -a /usr/bin/dpkg "$container_dir/usr/bin/" - cp -a /usr/bin/apt-get "$container_dir/usr/bin/" - # Add minimal /etc structure - echo "deb http://archive.ubuntu.com/ubuntu/ jammy main" > "$container_dir/etc/apt/sources.list" - fi - - # Run dpkg installation in nspawn container - local install_cmd=" - apt-get update && - apt-get download ${packages[*]} && - dpkg -i *.deb && - apt-get install -f && - dpkg --configure -a && - apt-get clean - " - - if ! systemd-nspawn -D "$container_dir" /bin/bash -c "$install_cmd"; then - log_error "dpkg installation failed in nspawn container" "apt-layer" - return 1 - fi - - # Move container contents to temp_dir - mv "$container_dir"/* "$temp_dir/" 2>/dev/null || true - - log_success "systemd-nspawn-based dpkg installation completed" "apt-layer" - return 0 -} - # Live overlay dpkg installation live_dpkg_install() { local packages=("$@") diff --git a/src/apt-layer/templates/install-apt-layer.template.sh b/src/apt-layer/templates/install-apt-layer.template.sh index 597f5dd..4fb7528 100644 --- a/src/apt-layer/templates/install-apt-layer.template.sh +++ b/src/apt-layer/templates/install-apt-layer.template.sh @@ -78,47 +78,83 @@ check_root() { fi } +# --- BEGIN DEPENDENCY JSON LOADING --- +# The dependencies JSON will be embedded as APT_LAYER_DEPENDENCIES_JSON in the compiled script. +APT_LAYER_DEPENDENCIES_JSON="${APT_LAYER_DEPENDENCIES_JSON:-} +{ + \"core\": [\"chroot\", \"apt-get\", \"dpkg\", \"jq\", \"mount\", \"umount\", \"findmnt\", \"numfmt\"], + \"container\": [\"podman\", \"docker\"], + \"oci\": [\"skopeo\"], + \"composefs\": [\"mkcomposefs\", \"composefs-info\", \"mount.composefs\", \"mksquashfs\", \"unsquashfs\"], + \"bootloader\": [\"efibootmgr\", \"grub-install\", \"update-grub\", \"bootctl\"], + \"security\": [\"curl\", \"wget\", \"gpg\"] +}" + +get_deps_for_type() { + local type="$1" + local json="$APT_LAYER_DEPENDENCIES_JSON" + case "$type" in + core) + echo "$json" | jq -r '.core[]' + ;; + security) + echo "$json" | jq -r '.security[]' + ;; + all) + echo "$json" | jq -r '.core[], .security[]' + ;; + *) + echo "$json" | jq -r '.core[]' + ;; + esac +} + +print_install_instructions() { + local json="$APT_LAYER_DEPENDENCIES_JSON" + echo " Quick fix for common dependencies:" + echo " sudo apt install -y squashfs-tools jq coreutils util-linux skopeo" + echo "" + echo " For container support (choose one):" + echo " sudo apt install -y podman # or" + echo " sudo apt install -y docker.io" + echo "" + echo " For bootloader support:" + echo " sudo apt install -y efibootmgr grub-common systemd-boot" + echo "" + echo " For more information, run: apt-layer --help" + echo "" +} + # Function to check dependencies check_dependencies() { print_status "Checking dependencies..." - local missing_deps=() - - # Check for curl or wget + # Installer-specific dependencies + local installer_deps=("sudo" "dos2unix") + # Use JSON to get core and security deps + local deps=( $(get_deps_for_type all) ) + # Add installer deps + deps+=("sudo" "dos2unix") + # Check for curl or wget (at least one) if ! command -v curl >/dev/null 2>&1 && ! command -v wget >/dev/null 2>&1; then missing_deps+=("curl or wget") fi - - # Check for jq - if ! command -v jq >/dev/null 2>&1; then - missing_deps+=("jq") - fi - - # Check for dos2unix - if ! command -v dos2unix >/dev/null 2>&1; then - missing_deps+=("dos2unix") - fi - + # Check all other deps + for dep in "${deps[@]}"; do + if [[ "$dep" == "curl" || "$dep" == "wget" ]]; then + continue # already checked above + fi + if ! command -v "$dep" >/dev/null 2>&1; then + missing_deps+=("$dep") + fi + done if [[ ${#missing_deps[@]} -gt 0 ]]; then print_error "Missing required dependencies: ${missing_deps[*]}" - print_status "Installing missing dependencies..." - - # Try to install dependencies - if command -v apt-get >/dev/null 2>&1; then - sudo apt-get update - sudo apt-get install -y "${missing_deps[@]}" - elif command -v dnf >/dev/null 2>&1; then - sudo dnf install -y "${missing_deps[@]}" - elif command -v yum >/dev/null 2>&1; then - sudo yum install -y "${missing_deps[@]}" - else - print_error "Could not automatically install dependencies. Please install manually:" - print_error " ${missing_deps[*]}" - exit 1 - fi + print_install_instructions + print_error "Please install missing dependencies and re-run the installer." + exit 1 fi - - print_success "All dependencies satisfied" + print_status "All dependencies satisfied" } # Function to download apt-layer diff --git a/src/bootc/README.md b/src/bootc/README.md index 379a379..b6694e4 100644 --- a/src/bootc/README.md +++ b/src/bootc/README.md @@ -29,6 +29,8 @@ src/bootc/ └── CHANGELOG.md # Version history and changes ``` +https://docs.fedoraproject.org/en-US/bootc/rpm-ostree/ + ## 🚀 Usage ### Compiling the Unified Script