diff --git a/.gitignore b/.gitignore index 79a3b9f..0642f6b 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ output-* *.qcow2 *.iso container.tar +test-integration-* # Image files (large binary artifacts) *.img diff --git a/bib/create-ostree-bootable-image.go b/bib/create-ostree-bootable-image.go index 0377dc6..580fc25 100644 --- a/bib/create-ostree-bootable-image.go +++ b/bib/create-ostree-bootable-image.go @@ -93,27 +93,97 @@ func main() { os.Exit(1) } - // Create a single partition - fmt.Println("Creating partition...") - cmd = exec.Command("sudo", "parted", loopDevice, "mkpart", "primary", "ext4", "1MiB", "100%") + // Create BIOS Boot Partition (1MB) for GRUB + fmt.Println("Creating BIOS Boot Partition...") + cmd = exec.Command("sudo", "parted", loopDevice, "mkpart", "primary", "1MiB", "2MiB") cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { - fmt.Printf("Error creating partition: %v\n", err) + fmt.Printf("Error creating BIOS Boot Partition: %v\n", err) os.Exit(1) } - // Get the partition device - partitionDevice := loopDevice + "p1" + // Set BIOS Boot Partition type flag + fmt.Println("Setting BIOS Boot Partition type...") + cmd = exec.Command("sudo", "parted", loopDevice, "set", "1", "bios_grub", "on") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + fmt.Printf("Warning: could not set BIOS Boot Partition type: %v\n", err) + } + + // Create EFI System Partition (501MB) for UEFI boot + fmt.Println("Creating EFI System Partition...") + cmd = exec.Command("sudo", "parted", loopDevice, "mkpart", "primary", "fat32", "2MiB", "503MiB") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + fmt.Printf("Error creating EFI System Partition: %v\n", err) + os.Exit(1) + } + + // Set EFI System Partition type flag + fmt.Println("Setting EFI System Partition type...") + cmd = exec.Command("sudo", "parted", loopDevice, "set", "2", "esp", "on") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + fmt.Printf("Warning: could not set EFI System Partition type: %v\n", err) + } + + // Create Boot Partition (1GB) for GRUB and kernel + fmt.Println("Creating Boot Partition...") + cmd = exec.Command("sudo", "parted", loopDevice, "mkpart", "primary", "ext4", "503MiB", "1527MiB") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + fmt.Printf("Error creating Boot Partition: %v\n", err) + os.Exit(1) + } + + // Create Root Partition (remaining space) + fmt.Println("Creating Root Partition...") + cmd = exec.Command("sudo", "parted", loopDevice, "mkpart", "primary", "ext4", "1527MiB", "100%") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + fmt.Printf("Error creating Root Partition: %v\n", err) + os.Exit(1) + } + + // Get the partition devices + bootPartitionDevice := loopDevice + "p3" + partitionDevice := loopDevice + "p4" fmt.Printf("Partition device: %s\n", partitionDevice) - // Format the partition with ext4 - fmt.Println("Formatting partition with ext4...") + // Format EFI partition with FAT32 + fmt.Printf("Formatting EFI partition with FAT32...\n") + efiPartitionDevice := loopDevice + "p2" + cmd = exec.Command("sudo", "mkfs.fat", "-F", "32", efiPartitionDevice) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + fmt.Printf("Error formatting EFI partition: %v\n", err) + os.Exit(1) + } + + // Format Boot partition with ext4 + fmt.Printf("Formatting Boot partition with ext4...\n") + cmd = exec.Command("sudo", "mkfs.ext4", bootPartitionDevice) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + fmt.Printf("Error formatting Boot partition: %v\n", err) + os.Exit(1) + } + + // Format Root partition with ext4 + fmt.Printf("Formatting Root partition with ext4...\n") cmd = exec.Command("sudo", "mkfs.ext4", partitionDevice) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { - fmt.Printf("Error formatting partition: %v\n", err) + fmt.Printf("Error formatting Root partition: %v\n", err) os.Exit(1) } @@ -124,19 +194,77 @@ func main() { os.Exit(1) } - // Mount the partition - fmt.Printf("Mounting partition to %s...\n", mountPoint) + // Mount Root partition + fmt.Printf("Mounting Root partition to %s...\n", mountPoint) cmd = exec.Command("sudo", "mount", partitionDevice, mountPoint) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { - fmt.Printf("Error mounting partition: %v\n", err) + fmt.Printf("Error mounting Root partition: %v\n", err) os.Exit(1) } + // Create boot directory first + fmt.Printf("Creating boot directory structure...\n") + bootMountPoint := filepath.Join(mountPoint, "boot") + + // Create boot directory and set ownership + cmd = exec.Command("sudo", "mkdir", "-p", bootMountPoint) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + fmt.Printf("Warning: could not create boot directory: %v\n", err) + } + + // Set ownership to root + cmd = exec.Command("sudo", "chown", "root:root", bootMountPoint) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + fmt.Printf("Warning: could not set ownership: %v\n", err) + } + + // Mount Boot partition + fmt.Printf("Mounting Boot partition to %s...\n", bootMountPoint) + cmd = exec.Command("sudo", "mount", bootPartitionDevice, bootMountPoint) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + fmt.Printf("Warning: could not mount Boot partition: %v\n", err) + } + + // Now create EFI directory inside the mounted boot partition + efiMountPoint := filepath.Join(mountPoint, "boot", "efi") + cmd = exec.Command("sudo", "mkdir", "-p", efiMountPoint) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + fmt.Printf("Warning: could not create EFI directory: %v\n", err) + } + + // Set ownership to root + cmd = exec.Command("sudo", "chown", "root:root", efiMountPoint) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + fmt.Printf("Warning: could not set ownership: %v\n", err) + } + + // Mount EFI partition + fmt.Printf("Mounting EFI partition to %s...\n", efiMountPoint) + cmd = exec.Command("sudo", "mount", efiPartitionDevice, efiMountPoint) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + fmt.Printf("Warning: could not mount EFI partition: %v\n", err) + } + // Clean up mount on exit defer func() { - fmt.Printf("Unmounting %s...\n", mountPoint) + fmt.Printf("Unmounting all partitions...\n") + // Unmount in reverse order: EFI, Boot, then Root + exec.Command("sudo", "umount", efiMountPoint).Run() + exec.Command("sudo", "umount", bootMountPoint).Run() exec.Command("sudo", "umount", mountPoint).Run() }() @@ -186,7 +314,13 @@ func main() { // Create OSTree-specific fstab fmt.Println("Creating OSTree fstab...") fstabContent := `# /etc/fstab for OSTree particle-os -/dev/sda1 / ext4 rw,errors=remount-ro 0 1 +# Root filesystem (4th partition) +/dev/sda4 / ext4 rw,errors=remount-ro 0 1 +# Boot partition (3rd partition) +/dev/sda3 /boot ext4 rw,errors=remount-ro 0 2 +# EFI System Partition (2nd partition) +/dev/sda2 /boot/efi vfat umask=0077,shortname=winnt 0 2 +# BIOS Boot Partition (1st partition) - not mounted proc /proc proc defaults 0 0 sysfs /sys sysfs defaults 0 0 devpts /dev/pts devpts gid=5,mode=620 0 0 @@ -203,17 +337,15 @@ tmpfs /run tmpfs defaults 0 0 // Try to install GRUB for OSTree fmt.Println("Installing GRUB for OSTree...") - // Bind mount necessary directories for GRUB - grubDirs := []string{"/dev", "/proc", "/sys"} - for _, dir := range grubDirs { - bindMount := filepath.Join(mountPoint, dir) - if err := exec.Command("sudo", "mount", "--bind", dir, bindMount).Run(); err != nil { - fmt.Printf("Warning: could not bind mount %s: %v\n", dir, err) - } - } - - // Install GRUB - cmd = exec.Command("sudo", "chroot", mountPoint, "grub-install", "--target=i386-pc", loopDevice) + // Install GRUB using podman run to access the Debian container with GRUB tools + // Mount the host's /dev directory to access loop devices + // Use the actual loop device name that was created + grubInstallCmd := fmt.Sprintf("grub-install --target=i386-pc --boot-directory=/mnt/boot --root-directory=/mnt %s", loopDevice) + cmd = exec.Command("sudo", "podman", "run", "--rm", "--privileged", "--tty", + "-v", "/dev:/dev:z", + "-v", mountPoint+":/mnt:z", + "238208cf481a", + "bash", "-c", grubInstallCmd) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { @@ -222,19 +354,21 @@ tmpfs /run tmpfs defaults 0 0 fmt.Println("GRUB installed successfully") } - // Generate GRUB config - cmd = exec.Command("sudo", "chroot", mountPoint, "update-grub") + // Generate GRUB config using podman run + grubMkconfigCmd := fmt.Sprintf("grub-mkconfig -o /mnt/boot/grub/grub.cfg") + cmd = exec.Command("sudo", "podman", "run", "--rm", "--privileged", "--tty", + "-v", mountPoint+":/mnt:z", + "238208cf481a", + "bash", "-c", grubMkconfigCmd) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { fmt.Printf("Warning: GRUB config generation failed: %v\n", err) + } else { + fmt.Println("GRUB config generated successfully") } - // Unbind mount directories - for _, dir := range grubDirs { - bindMount := filepath.Join(mountPoint, dir) - exec.Command("sudo", "umount", bindMount).Run() - } + // GRUB installation complete } else { // Handle standard bootable image setup diff --git a/bib/create-working-minimal-image b/bib/create-working-minimal-image deleted file mode 100755 index 7813f43..0000000 Binary files a/bib/create-working-minimal-image and /dev/null differ diff --git a/bib/particle-os-fixed b/bib/particle-os-fixed deleted file mode 100755 index 38c059a..0000000 Binary files a/bib/particle-os-fixed and /dev/null differ diff --git a/bin/bib-canary-arm64 b/bin/bib-canary-arm64 deleted file mode 100755 index b3a467b..0000000 Binary files a/bin/bib-canary-arm64 and /dev/null differ diff --git a/build-apt-ostree-in-container.sh b/build-apt-ostree-in-container.sh new file mode 100755 index 0000000..3583805 --- /dev/null +++ b/build-apt-ostree-in-container.sh @@ -0,0 +1,129 @@ +#!/bin/bash +# Build apt-ostree in Debian container environment +# This script uses the existing deb-bootc-image-builder container infrastructure + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Configuration +WORKSPACE_PATH="/home/rob/Documents/Projects/overseer" +APT_OSTREE_PATH="$WORKSPACE_PATH/apt-ostree" +CONTAINER_NAME="apt-ostree-builder" +IMAGE_NAME="apt-ostree-builder:latest" + +# Functions +log_info() { + echo -e "${BLUE}ℹ️ $1${NC}" +} + +log_success() { + echo -e "${GREEN}✅ $1${NC}" +} + +log_warning() { + echo -e "${YELLOW}⚠️ $1${NC}" +} + +log_error() { + echo -e "${RED}❌ $1${NC}" +} + +cleanup() { + log_info "Cleaning up container..." + podman rm -f "$CONTAINER_NAME" 2>/dev/null || true +} + +# Set up cleanup on exit +trap cleanup EXIT + +# Check if we're in the right directory +if [[ ! -d "$APT_OSTREE_PATH" ]]; then + log_error "apt-ostree directory not found at $APT_OSTREE_PATH" + exit 1 +fi + +log_info "Building apt-ostree in Debian container environment..." + +# Create a temporary Dockerfile for building apt-ostree +cat > /tmp/apt-ostree.Dockerfile << 'EOF' +FROM debian:trixie + +# Install build dependencies +RUN apt-get update && apt-get install -y \ + build-essential \ + cargo \ + rustc \ + pkg-config \ + libostree-dev \ + libapt-pkg-dev \ + libglib2.0-dev \ + libgirepository1.0-dev \ + libgio-2.0-dev \ + libpolkit-gobject-1-dev \ + debootstrap \ + ostree \ + git \ + curl \ + wget \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Set working directory +WORKDIR /workspace + +# Copy apt-ostree source +COPY . /workspace/ + +# Build command +CMD ["bash", "-c", "cargo build --release && echo 'Build completed successfully'"] +EOF + +# Build the container image +log_info "Building container image..." +cd "$APT_OSTREE_PATH" +if ! podman build -f /tmp/apt-ostree.Dockerfile -t "$IMAGE_NAME" .; then + log_error "Failed to build container image" + exit 1 +fi +log_success "Container image built successfully" + +# Run the build in the container +log_info "Building apt-ostree in container..." +if ! podman run --rm \ + --name "$CONTAINER_NAME" \ + -v "$APT_OSTREE_PATH:/workspace:z" \ + "$IMAGE_NAME"; then + log_error "Failed to build apt-ostree in container" + exit 1 +fi + +log_success "apt-ostree built successfully in Debian container!" + +# Check if the binary was created +if [[ -f "$APT_OSTREE_PATH/target/release/apt-ostree" ]]; then + log_success "Binary found at: $APT_OSTREE_PATH/target/release/apt-ostree" + + # Test the binary + log_info "Testing apt-ostree binary..." + if "$APT_OSTREE_PATH/target/release/apt-ostree" --help >/dev/null 2>&1; then + log_success "apt-ostree binary works correctly!" + "$APT_OSTREE_PATH/target/release/apt-ostree" --version + else + log_error "apt-ostree binary failed to run" + exit 1 + fi +else + log_error "Binary not found after build" + exit 1 +fi + +# Clean up temporary files +rm -f /tmp/apt-ostree.Dockerfile + +log_success "apt-ostree build and test completed successfully!" diff --git a/demo-alternative-solution.sh b/demo-alternative-solution.sh new file mode 100755 index 0000000..38b975b --- /dev/null +++ b/demo-alternative-solution.sh @@ -0,0 +1,156 @@ +#!/bin/bash +# Demonstration of working alternative solution +# Shows complete Debian Atomic pipeline: debootstrap + ostree → bootable image + +set -euo pipefail + +echo "🚀 Debian Atomic Pipeline Demonstration" +echo "Using working alternative solution: debootstrap + ostree" +echo "=" * 60 + +# Configuration +WORKSPACE_PATH="/home/rob/Documents/Projects/overseer" +DEB_BOOTC_PATH="$WORKSPACE_PATH/deb-bootc-image-builder" +ATOMIC_CONFIGS_PATH="$WORKSPACE_PATH/debian-atomic-configs" + +# Step 1: Create OSTree commit using alternative solution +echo +echo "Step 1: Creating OSTree commit using debootstrap + ostree" +echo "Running: create-debian-atomic.py minimal" + +cd "$ATOMIC_CONFIGS_PATH" + +# Run the alternative solution with a persistent work directory +WORK_DIR="/tmp/debian-atomic-demo" +if [[ -d "$WORK_DIR" ]]; then + rm -rf "$WORK_DIR" +fi + +echo "Creating persistent build in: $WORK_DIR" +if distrobox-enter debian-trixie -- bash -c "cd '$ATOMIC_CONFIGS_PATH' && sudo python3 create-debian-atomic.py minimal '$WORK_DIR'"; then + echo "✅ Step 1 completed: OSTree commit created successfully" +else + echo "❌ Step 1 failed" + exit 1 +fi + +# Step 2: Verify the repository and commit +echo +echo "Step 2: Verifying OSTree repository and commit" + +REPO_PATH="$WORK_DIR/repo" +if [[ ! -d "$REPO_PATH" ]]; then + echo "❌ Repository not found at $REPO_PATH" + exit 1 +fi + +echo "✅ Repository found at: $REPO_PATH" + +# List references +echo "Repository references:" +ostree refs --repo="$REPO_PATH" || echo "No references found" + +# List commits +echo "Repository commits:" +find "$REPO_PATH/objects" -type d -name "????????????????????????????????????????????????????????????????" 2>/dev/null | head -5 || echo "No commits found" + +# Step 3: Extract rootfs from OSTree commit +echo +echo "Step 3: Extracting rootfs from OSTree commit" + +ROOTFS_PATH="/tmp/debian-atomic-rootfs" +if [[ -d "$ROOTFS_PATH" ]]; then + rm -rf "$ROOTFS_PATH" +fi + +# Get the reference +REF_FILE=$(find "$REPO_PATH/refs" -type f | head -1) +if [[ -z "$REF_FILE" ]]; then + echo "❌ No reference found" + exit 1 +fi + +COMMIT_ID=$(cat "$REF_FILE") +echo "Using commit: $COMMIT_ID" + +# Extract rootfs +echo "Extracting rootfs to: $ROOTFS_PATH" +if ostree checkout --repo="$REPO_PATH" --subpath=/ "$COMMIT_ID" "$ROOTFS_PATH"; then + echo "✅ Rootfs extracted successfully" +else + echo "❌ Failed to extract rootfs" + exit 1 +fi + +# Verify rootfs structure +if [[ -d "$ROOTFS_PATH/usr" ]] && [[ -d "$ROOTFS_PATH/etc" ]]; then + echo "✅ Rootfs structure verified" +else + echo "❌ Invalid rootfs structure" + exit 1 +fi + +# Step 4: Create bootable image +echo +echo "Step 4: Creating bootable image using deb-bootc-image-builder" + +# Build deb-bootc-image-builder if needed +BIB_BIN="$DEB_BOOTC_PATH/bib/create-ostree-bootable-image" +if [[ ! -f "$BIB_BIN" ]]; then + echo "Building deb-bootc-image-builder..." + cd "$DEB_BOOTC_PATH" + if ! go build -o bib/create-ostree-bootable-image ./bib/create-ostree-bootable-image.go; then + echo "❌ Failed to build deb-bootc-image-builder" + exit 1 + fi + echo "✅ deb-bootc-image-builder built successfully" +fi + +# Create bootable image +OUTPUT_IMAGE="/tmp/debian-atomic-minimal.img" +echo "Creating bootable image: $OUTPUT_IMAGE" + +if "$BIB_BIN" "$ROOTFS_PATH" "$OUTPUT_IMAGE"; then + echo "✅ Bootable image created successfully" + + # Show image information + if [[ -f "$OUTPUT_IMAGE" ]]; then + IMAGE_SIZE=$(du -h "$OUTPUT_IMAGE" | cut -f1) + echo "Image size: $IMAGE_SIZE" + + # Copy to workspace + FINAL_IMAGE="$WORKSPACE_PATH/debian-atomic-minimal-demo.img" + cp "$OUTPUT_IMAGE" "$FINAL_IMAGE" + echo "Image copied to: $FINAL_IMAGE" + + echo + echo "🎉 Pipeline completed successfully!" + echo + echo "📋 Summary:" + echo "✅ OSTree commit created using debootstrap + ostree" + echo "✅ Rootfs extracted from OSTree commit" + echo "✅ Bootable image created: $FINAL_IMAGE" + echo + echo "🧪 To test the image:" + echo "qemu-system-x86_64 -m 2G -drive file=$FINAL_IMAGE,format=raw -nographic" + + else + echo "❌ Output image not found" + exit 1 + fi +else + echo "❌ Failed to create bootable image" + exit 1 +fi + +echo +echo "🔍 Repository details:" +echo "Repository: $REPO_PATH" +echo "Commit: $COMMIT_ID" +echo "Rootfs: $ROOTFS_PATH" +echo "Image: $OUTPUT_IMAGE" + +echo +echo "💡 This demonstrates that the alternative solution (debootstrap + ostree)" +echo " is fully functional and can generate bootable images, bypassing the" +echo " apt-ostree integration issues." diff --git a/go.mod b/go.mod deleted file mode 100644 index 7000078..0000000 --- a/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module particle-os - -go 1.25.0 diff --git a/integrate-alternative-solution.sh b/integrate-alternative-solution.sh new file mode 100755 index 0000000..1c0f050 --- /dev/null +++ b/integrate-alternative-solution.sh @@ -0,0 +1,276 @@ +#!/bin/bash +# Integration script using the working alternative solution +# Creates complete Debian Atomic pipeline: treefile → debootstrap + ostree → bootable image + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Configuration +WORKSPACE_PATH="/home/rob/Documents/Projects/overseer" +DEB_BOOTC_PATH="$WORKSPACE_PATH/deb-bootc-image-builder" +ATOMIC_CONFIGS_PATH="$WORKSPACE_PATH/debian-atomic-configs" +TEMP_DIR=$(mktemp -d /tmp/debian-atomic-alt-XXXXXX) + +# Functions +log_info() { + echo -e "${BLUE}ℹ️ $1${NC}" +} + +log_success() { + echo -e "${GREEN}✅ $1${NC}" +} + +log_warning() { + echo -e "${YELLOW}⚠️ $1${NC}" +} + +log_error() { + echo -e "${RED}❌ $1${NC}" +} + +cleanup() { + log_info "Cleaning up temporary files..." + if [[ -d "$TEMP_DIR" ]]; then + rm -rf "$TEMP_DIR" + fi +} + +# Set up cleanup on exit +trap cleanup EXIT + +# Validate paths +validate_paths() { + log_info "Validating required components..." + + local missing_paths=() + + [[ ! -d "$DEB_BOOTC_PATH" ]] && missing_paths+=("deb-bootc-image-builder") + [[ ! -d "$ATOMIC_CONFIGS_PATH" ]] && missing_paths+=("debian-atomic-configs") + + if [[ ${#missing_paths[@]} -gt 0 ]]; then + log_error "Missing required components: ${missing_paths[*]}" + exit 1 + fi + + log_success "All required components found" +} + +# Build deb-bootc-image-builder if needed +build_deb_bootc() { + local bib_bin="$DEB_BOOTC_PATH/bib/create-ostree-bootable-image" + + if [[ ! -f "$bib_bin" ]]; then + log_info "Building deb-bootc-image-builder..." + cd "$DEB_BOOTC_PATH" + + if ! go build -o bib/create-ostree-bootable-image ./bib/create-ostree-bootable-image.go; then + log_error "Failed to build deb-bootc-image-builder" + exit 1 + fi + + log_success "deb-bootc-image-builder built successfully" + else + log_info "deb-bootc-image-builder binary found, skipping build" + fi +} + +# Step 1: Create OSTree commit using alternative solution (debootstrap + ostree) +create_ostree_commit() { + local variant="$1" + + log_info "Step 1: Creating OSTree commit using alternative solution (debootstrap + ostree)" + + cd "$TEMP_DIR" + + # Run the alternative solution script + log_info "Running create-debian-atomic.py for variant: $variant" + + if ! distrobox-enter debian-trixie -- bash -c "cd '$TEMP_DIR' && sudo python3 '$ATOMIC_CONFIGS_PATH/create-debian-atomic.py' '$variant'"; then + log_error "Failed to create OSTree commit using alternative solution" + return 1 + fi + + log_success "OSTree commit created successfully using alternative solution" + + # The script cleans up automatically, so we need to run it again to keep the repository + # Let's run it with a persistent work directory + local persistent_dir="$TEMP_DIR/persistent-build" + log_info "Creating persistent build in: $persistent_dir" + + if ! distrobox-enter debian-trixie -- bash -c "cd '$TEMP_DIR' && sudo python3 '$ATOMIC_CONFIGS_PATH/create-debian-atomic.py' '$variant' '$persistent_dir'"; then + log_error "Failed to create persistent OSTree commit" + return 1 + fi + + # Verify the repository was created + local repo_path="$persistent_dir/repo" + if [[ ! -d "$repo_path" ]]; then + log_error "OSTree repository not found at $repo_path" + return 1 + fi + + # Get commit information + local refs_dir="$repo_path/refs" + if [[ ! -d "$refs_dir" ]]; then + log_error "No refs directory found" + return 1 + fi + + # Find the reference file + local ref_file + ref_file=$(find "$refs_dir" -type f -name "*$variant*" | head -1) + if [[ -z "$ref_file" ]]; then + log_error "No reference found for variant $variant" + return 1 + fi + + local commit_id + commit_id=$(cat "$ref_file") + log_info "Found commit: $commit_id" + + # Verify the commit exists + local objects_dir="$repo_path/objects" + local commit_path="$objects_dir/${commit_id:0:2}/${commit_id:2}" + if [[ ! -d "$commit_path" ]]; then + log_error "Commit object not found at $commit_path" + return 1 + fi + + log_success "Successfully created commit: $commit_id" + echo "$commit_id" +} + +# Step 2: Extract rootfs from OSTree commit and create bootable image +create_bootable_image() { + local commit_id="$1" + local repo_path="$TEMP_DIR/persistent-build/repo" + + log_info "Step 2: Creating bootable image from OSTree commit" + + cd "$TEMP_DIR" + + # Extract the rootfs from the OSTree commit + local rootfs_path="$TEMP_DIR/rootfs" + log_info "Extracting rootfs from OSTree commit..." + + # Use ostree to checkout the commit to a directory + if ! ostree checkout --repo="$repo_path" --subpath=/ "$commit_id" "$rootfs_path"; then + log_error "Failed to extract rootfs from OSTree commit" + return 1 + fi + + log_success "Rootfs extracted to: $rootfs_path" + + # Verify rootfs structure + if [[ ! -d "$rootfs_path/usr" ]] || [[ ! -d "$rootfs_path/etc" ]]; then + log_error "Invalid rootfs structure" + return 1 + fi + + # Create bootable image using deb-bootc-image-builder + local bib_bin="$DEB_BOOTC_PATH/bib/create-ostree-bootable-image" + local output_image="$TEMP_DIR/debian-atomic.img" + + log_info "Creating bootable image from rootfs..." + if ! "$bib_bin" "$rootfs_path" "$output_image"; then + log_error "Failed to create bootable image" + return 1 + fi + + log_success "Bootable image created successfully" + + # Check if image was created + if [[ ! -f "$output_image" ]]; then + log_error "Output image not found: $output_image" + return 1 + fi + + # Show image information + local image_size=$(du -h "$output_image" | cut -f1) + log_info "Image created: $output_image ($image_size)" + + echo "$output_image" +} + +# Main pipeline function +run_pipeline() { + local variant="$1" + + log_info "🚀 Starting Debian Atomic pipeline (Alternative Solution)" + log_info "Variant: $variant" + log_info "Working directory: $TEMP_DIR" + + # Step 1: Create OSTree commit using alternative solution + log_info "Executing Step 1: Create OSTree commit using alternative solution..." + local commit_id + if ! commit_id=$(create_ostree_commit "$variant"); then + log_error "Step 1 failed: Could not create OSTree commit" + return 1 + fi + + log_success "Step 1 completed: OSTree commit $commit_id created" + + # Step 2: Create bootable image + log_info "Executing Step 2: Create bootable image..." + local output_image + if ! output_image=$(create_bootable_image "$commit_id"); then + log_error "Step 2 failed: Could not create bootable image" + return 1 + fi + + log_success "Step 2 completed: Bootable image created" + + log_success "🎉 Pipeline completed successfully!" + log_info "Output image: $output_image" + + # Copy image to workspace for easy access + local final_image="$WORKSPACE_PATH/debian-atomic-$variant-alternative.img" + cp "$output_image" "$final_image" + log_info "Image copied to: $final_image" + + # Show testing instructions + echo + log_info "🧪 To test the image:" + echo "qemu-system-x86_64 -m 2G -drive file=$final_image,format=raw -nographic" + echo + log_info "📋 Summary:" + log_success "✅ OSTree commit created using alternative solution (debootstrap + ostree)" + log_success "✅ Bootable image created: $final_image" + log_success "✅ Debian Atomic pipeline completed successfully using alternative solution!" + + return 0 +} + +# Main function +main() { + if [[ $# -lt 1 ]]; then + echo "Usage: $0 " + echo "Example: $0 minimal" + echo + echo "Available variants: minimal, gnome, plasma, cosmic, sway, budgie" + exit 1 + fi + + local variant="$1" + + # Validate and build components + validate_paths + build_deb_bootc + + # Run the pipeline + if ! run_pipeline "$variant"; then + log_error "Pipeline failed" + exit 1 + fi + + log_success "Pipeline completed successfully" +} + +# Run main function +main "$@" diff --git a/integrate-apt-ostree-bootable.sh b/integrate-apt-ostree-bootable.sh new file mode 100755 index 0000000..13a73df --- /dev/null +++ b/integrate-apt-ostree-bootable.sh @@ -0,0 +1,278 @@ +#!/bin/bash +# Integration script between apt-ostree and deb-bootc-image-builder +# Creates complete Debian Atomic pipeline: treefile → apt-ostree → bootable image + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Configuration +WORKSPACE_PATH="/home/rob/Documents/Projects/overseer" +APT_OSTREE_PATH="$WORKSPACE_PATH/apt-ostree" +DEB_BOOTC_PATH="$WORKSPACE_PATH/deb-bootc-image-builder" +ATOMIC_CONFIGS_PATH="$WORKSPACE_PATH/debian-atomic-configs" +TEMP_DIR=$(mktemp -d ./debian-atomic-bootable-XXXXXX) + +# Functions +log_info() { + echo -e "${BLUE}ℹ️ $1${NC}" +} + +log_success() { + echo -e "${GREEN}✅ $1${NC}" +} + +log_warning() { + echo -e "${YELLOW}⚠️ $1${NC}" +} + +log_error() { + echo -e "${RED}❌ $1${NC}" +} + +cleanup() { + log_info "Cleaning up temporary files..." + if [[ -d "$TEMP_DIR" ]]; then + rm -rf "$TEMP_DIR" + fi +} + +# Set up cleanup on exit (disabled for debugging) +# trap cleanup EXIT + +# Validate paths +validate_paths() { + log_info "Validating required components..." + + local missing_paths=() + + [[ ! -d "$APT_OSTREE_PATH" ]] && missing_paths+=("apt-ostree") + [[ ! -d "$DEB_BOOTC_PATH" ]] && missing_paths+=("deb-bootc-image-builder") + [[ ! -d "$ATOMIC_CONFIGS_PATH" ]] && missing_paths+=("debian-atomic-configs") + + if [[ ${#missing_paths[@]} -gt 0 ]]; then + log_error "Missing required components: ${missing_paths[*]}" + exit 1 + fi + + log_success "All required components found" +} + +# Step 1: Create OSTree commit using apt-ostree in container +create_ostree_commit() { + local treefile="$1" + local treefile_name=$(basename "$treefile") + + log_info "Step 1: Creating OSTree commit from $treefile_name" + + # Run apt-ostree in the container to create the OSTree commit + log_info "Running apt-ostree compose tree in Debian container..." + + local output + local exit_code + output=$(podman run --rm --privileged \ + -v "$APT_OSTREE_PATH:/workspace:z" \ + -v "$ATOMIC_CONFIGS_PATH:/configs:z" \ + -v "$TEMP_DIR:/shared:z" \ + -v "$DEB_BOOTC_PATH:/shared/deb-bootc:z" \ + apt-ostree-builder:latest \ + bash -c "cd /workspace && ./target/release/apt-ostree compose tree /configs/treefiles/$treefile_name --workdir /shared/apt-ostree-build" 2>&1) + exit_code=$? + + log_info "Command completed with exit code: $exit_code" + if [[ $exit_code -ne 0 ]]; then + log_error "apt-ostree failed with exit code $exit_code" + log_error "Output: $output" + return 1 + fi + + log_success "OSTree commit created successfully" + + # Extract the commit hash from the output + local commit_hash=$(echo "$output" | grep "Commit hash:" | tail -1 | awk '{print $3}') + if [[ -z "$commit_hash" ]]; then + log_error "Could not extract commit hash from output" + return 1 + fi + + log_info "Commit hash: $commit_hash" + echo "$commit_hash" +} + +# Step 2: Create bootable image from OSTree commit +create_bootable_image() { + local commit_hash="$1" + + log_info "Step 2: Creating bootable image from OSTree commit" + + # Run the entire step 2 process inside the container where all tools are available + log_info "Running bootable image creation in Debian container..." + + local output + local exit_code + output=$(podman run --rm --privileged \ + -v "$APT_OSTREE_PATH:/workspace:z" \ + -v "$ATOMIC_CONFIGS_PATH:/configs:z" \ + -v "$TEMP_DIR:/shared:z" \ + -v "$DEB_BOOTC_PATH:/shared/deb-bootc:z" \ + apt-ostree-builder:latest \ + bash -c " + cd /workspace + echo 'Checking repository at /shared/apt-ostree-build/repo' + ls -la /shared/apt-ostree-build/ + if [[ -d /shared/apt-ostree-build/repo ]]; then + echo 'Repository found, extracting rootfs...' + ostree checkout --repo=/shared/apt-ostree-build/repo --subpath=/ '$commit_hash' /shared/rootfs + if [[ \$? -eq 0 ]]; then + echo 'Rootfs extracted successfully' + ls -la /shared/rootfs/ + if [[ -d /shared/rootfs/usr ]] && [[ -d /shared/rootfs/etc ]]; then + echo 'Rootfs structure verified' + echo 'Creating bootable image using deb-bootc-image-builder...' + echo 'Checking if binary exists...' + ls -la /shared/deb-bootc/bib/ + if [[ -f /shared/deb-bootc/bib/create-ostree-bootable-image ]]; then + echo 'Binary found, checking permissions...' + ls -la /shared/deb-bootc/bib/create-ostree-bootable-image + echo 'Running create-ostree-bootable-image...' + # Use the existing create-ostree-bootable-image binary + /shared/deb-bootc/bib/create-ostree-bootable-image /shared/rootfs /shared/debian-atomic-bootable.img + echo 'Command completed with exit code: $?' + else + echo 'Binary not found!' + exit 1 + fi + if [[ \$? -eq 0 ]]; then + echo 'Bootable image created successfully' + ls -la /shared/ + exit 0 + else + echo 'Failed to create bootable image' + exit 1 + fi + else + echo 'Invalid rootfs structure' + exit 1 + fi + else + echo 'Failed to extract rootfs' + exit 1 + fi + else + echo 'Repository not found' + exit 1 + fi + " 2>&1) + exit_code=$? + + log_info "Container execution completed with exit code: $exit_code" + log_info "Container output:" + echo "$output" + + if [[ $exit_code -ne 0 ]]; then + log_error "Failed to create bootable image in container" + return 1 + fi + + # Check if the image was created + local output_image="$TEMP_DIR/debian-atomic-bootable.img" + if [[ ! -f "$output_image" ]]; then + log_error "Output image not found: $output_image" + return 1 + fi + + log_success "Bootable image created successfully" + + # Show image information + local image_size=$(du -h "$output_image" | cut -f1) + log_info "Image created: $output_image ($image_size)" + + echo "$output_image" +} + +# Main pipeline function +run_pipeline() { + local treefile="$1" + local variant="${2:-minimal}" + + log_info "🚀 Starting Debian Atomic Bootable Image Pipeline" + log_info "Treefile: $treefile" + log_info "Variant: $variant" + log_info "Working directory: $TEMP_DIR" + + # Step 1: Create OSTree commit + log_info "Executing Step 1: Create OSTree commit..." + local commit_hash + if ! commit_hash=$(create_ostree_commit "$treefile"); then + log_error "Step 1 failed: Could not create OSTree commit" + return 1 + fi + + log_success "Step 1 completed: OSTree commit $commit_hash created" + + # Step 2: Create bootable image + log_info "Executing Step 2: Create bootable image..." + local output_image + if ! output_image=$(create_bootable_image "$commit_hash"); then + log_error "Step 2 failed: Could not create bootable image" + return 1 + fi + + log_success "Step 2 completed: Bootable image created" + + log_success "🎉 Pipeline completed successfully!" + log_info "Output image: $output_image" + + # Copy image to workspace for easy access + local final_image="$WORKSPACE_PATH/debian-atomic-$variant-bootable.img" + cp "$output_image" "$final_image" + log_info "Image copied to: $final_image" + + # Show testing instructions + echo + log_info "🧪 To test the bootable image:" + echo "qemu-system-x86_64 -m 2G -drive file=$final_image,format=raw -nographic -serial stdio" + echo + log_info "📋 Summary:" + log_success "✅ OSTree commit created from $treefile" + log_success "✅ Bootable image created: $final_image" + log_success "✅ Debian Atomic bootable image pipeline completed successfully!" + + return 0 +} + +# Main function +main() { + if [[ $# -lt 1 ]]; then + echo "Usage: $0 [variant]" + echo "Example: $0 debian-minimal-apt-ostree.yaml minimal" + echo + echo "Available treefiles:" + if [[ -d "$ATOMIC_CONFIGS_PATH/treefiles" ]]; then + ls "$ATOMIC_CONFIGS_PATH/treefiles/"*.yaml 2>/dev/null | sed 's/.*\///' || echo "No treefiles found" + fi + exit 1 + fi + + local treefile="$1" + local variant="${2:-minimal}" + + # Validate and run the pipeline + validate_paths + + # Run the pipeline + if ! run_pipeline "$treefile" "$variant"; then + log_error "Pipeline failed" + exit 1 + fi + + log_success "Pipeline completed successfully" +} + +# Run main function +main "$@" diff --git a/integrate-apt-ostree.sh b/integrate-apt-ostree.sh new file mode 100755 index 0000000..7401d4a --- /dev/null +++ b/integrate-apt-ostree.sh @@ -0,0 +1,320 @@ +#!/bin/bash +# Integration script between apt-ostree and deb-bootc-image-builder +# Creates complete Debian Atomic pipeline: treefile → apt-ostree → bootable image + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Configuration +WORKSPACE_PATH="/home/rob/Documents/Projects/overseer" +APT_OSTREE_PATH="$WORKSPACE_PATH/apt-ostree" +DEB_BOOTC_PATH="$WORKSPACE_PATH/deb-bootc-image-builder" +ATOMIC_CONFIGS_PATH="$WORKSPACE_PATH/debian-atomic-configs" +TEMP_DIR=$(mktemp -d /tmp/debian-atomic-XXXXXX) + +# Functions +log_info() { + echo -e "${BLUE}ℹ️ $1${NC}" +} + +log_success() { + echo -e "${GREEN}✅ $1${NC}" +} + +log_warning() { + echo -e "${YELLOW}⚠️ $1${NC}" +} + +log_error() { + echo -e "${RED}❌ $1${NC}" +} + +cleanup() { + log_info "Cleaning up temporary files..." + if [[ -d "$TEMP_DIR" ]]; then + rm -rf "$TEMP_DIR" + fi +} + +# Set up cleanup on exit +trap cleanup EXIT + +# Validate paths +validate_paths() { + log_info "Validating required components..." + + local missing_paths=() + + [[ ! -d "$APT_OSTREE_PATH" ]] && missing_paths+=("apt-ostree") + [[ ! -d "$DEB_BOOTC_PATH" ]] && missing_paths+=("deb-bootc-image-builder") + [[ ! -d "$ATOMIC_CONFIGS_PATH" ]] && missing_paths+=("debian-atomic-configs") + + if [[ ${#missing_paths[@]} -gt 0 ]]; then + log_error "Missing required components: ${missing_paths[*]}" + exit 1 + fi + + log_success "All required components found" +} + +# Build apt-ostree if needed +build_apt_ostree() { + local apt_ostree_bin="$APT_OSTREE_PATH/target/debug/apt-ostree" + + if [[ ! -f "$apt_ostree_bin" ]]; then + log_info "Building apt-ostree..." + cd "$APT_OSTREE_PATH" + + if ! cargo build --bin apt-ostree; then + log_error "Failed to build apt-ostree" + exit 1 + fi + + log_success "apt-ostree built successfully" + else + log_info "apt-ostree binary found, skipping build" + fi +} + +# Build deb-bootc-image-builder if needed +build_deb_bootc() { + local bib_bin="$DEB_BOOTC_PATH/bib/create-ostree-bootable-image" + + if [[ ! -f "$bib_bin" ]]; then + log_info "Building deb-bootc-image-builder..." + cd "$DEB_BOOTC_PATH" + + if ! go build -o bib/create-ostree-bootable-image ./bib/create-ostree-bootable-image.go; then + log_error "Failed to build deb-bootc-image-builder" + exit 1 + fi + + log_success "deb-bootc-image-builder built successfully" + else + log_info "deb-bootc-image-builder binary found, skipping build" + fi +} + +# Step 1: Create OSTree commit using apt-ostree +create_ostree_commit() { + echo "DEBUG: create_ostree_commit function called with: $1" + + local treefile="$1" + local treefile_path="$ATOMIC_CONFIGS_PATH/treefiles/$treefile" + + log_info "Step 1: Creating OSTree commit from $treefile" + log_info "Treefile path: $treefile_path" + + if [[ ! -f "$treefile_path" ]]; then + log_error "Treefile not found: $treefile_path" + return 1 + fi + + # Run apt-ostree inside the Debian container where libraries are available + local apt_ostree_bin="$APT_OSTREE_PATH/target/release/apt-ostree" + log_info "Running apt-ostree inside Debian container..." + log_info "apt-ostree binary: $apt_ostree_bin" + + # Create a temporary working directory that's shared between host and container + local container_workdir="$TEMP_DIR/apt-ostree-build" + + # Run the command in the container and capture output + log_info "Executing: podman run --rm --privileged -v $APT_OSTREE_PATH:/workspace:z -v $ATOMIC_CONFIGS_PATH:/configs:z -v $TEMP_DIR:/shared:z apt-ostree-builder:latest bash -c 'cd /workspace && ./target/release/apt-ostree compose tree /configs/treefiles/$(basename $treefile_path) --workdir /shared/apt-ostree-build'" + + local output + local exit_code + output=$(podman run --rm --privileged -v "$APT_OSTREE_PATH:/workspace:z" -v "$ATOMIC_CONFIGS_PATH:/configs:z" -v "$TEMP_DIR:/shared:z" apt-ostree-builder:latest bash -c "cd /workspace && ./target/release/apt-ostree compose tree /configs/treefiles/$(basename $treefile_path) --workdir /shared/apt-ostree-build" 2>&1) + exit_code=$? + + log_info "Command completed with exit code: $exit_code" + log_info "Command output:" + echo "$output" + + if [[ $exit_code -ne 0 ]]; then + log_error "apt-ostree failed with exit code $exit_code" + return 1 + fi + + log_success "OSTree commit created successfully" + + # apt-ostree creates the repository in the shared workdir + local repo_path="$TEMP_DIR/apt-ostree-build/repo" + log_info "Checking for repository at: $repo_path" + + if [[ ! -d "$repo_path" ]]; then + log_error "OSTree repository not found at $repo_path" + return 1 + fi + + log_info "Repository found, checking commits..." + + # Get commit information + local commits_dir="$repo_path/commits" + if [[ ! -d "$commits_dir" ]]; then + log_error "No commits directory found" + return 1 + fi + + local commits=($(ls "$commits_dir" | grep -E "^commit_")) + log_info "Found commits: ${commits[*]}" + + if [[ ${#commits[@]} -eq 0 ]]; then + log_error "No commits found in repository" + return 1 + fi + + local commit_id="${commits[0]}" + log_info "Selected commit: $commit_id" + + # Read metadata if available + local metadata_file="$commits_dir/$commit_id/metadata.json" + if [[ -f "$metadata_file" ]]; then + log_info "Commit metadata:" + cat "$metadata_file" | jq '.' 2>/dev/null || cat "$metadata_file" + fi + + log_success "Successfully created commit: $commit_id" + echo "$commit_id" +} + +# Step 2: Extract rootfs from OSTree commit and create bootable image +create_bootable_image() { + local commit_id="$1" + local repo_path="/tmp/apt-ostree-build/repo" + + log_info "Step 2: Creating bootable image from OSTree commit" + + cd "$TEMP_DIR" + + # Extract the rootfs from the OSTree commit + local rootfs_path="$TEMP_DIR/rootfs" + log_info "Extracting rootfs from OSTree commit..." + + # Use ostree to checkout the commit to a directory + if ! ostree checkout --repo="$repo_path" --subpath=/ "$commit_id" "$rootfs_path"; then + log_error "Failed to extract rootfs from OSTree commit" + exit 1 + fi + + log_success "Rootfs extracted to: $rootfs_path" + + # Verify rootfs structure + if [[ ! -d "$rootfs_path/usr" ]] || [[ ! -d "$rootfs_path/etc" ]]; then + log_error "Invalid rootfs structure" + exit 1 + fi + + # Create bootable image using deb-bootc-image-builder + local bib_bin="$DEB_BOOTC_PATH/bib/create-ostree-bootable-image" + local output_image="$TEMP_DIR/debian-atomic.img" + + log_info "Creating bootable image from rootfs..." + if ! "$bib_bin" "$rootfs_path" "$output_image"; then + log_error "Failed to create bootable image" + exit 1 + fi + + log_success "Bootable image created successfully" + + # Check if image was created + if [[ ! -f "$output_image" ]]; then + log_error "Output image not found: $output_image" + exit 1 + fi + + # Show image information + local image_size=$(du -h "$output_image" | cut -f1) + log_info "Image created: $output_image ($image_size)" + + echo "$output_image" +} + +# Main pipeline function +run_pipeline() { + local treefile="$1" + local variant="${2:-minimal}" + + log_info "🚀 Starting Debian Atomic pipeline" + log_info "Treefile: $treefile" + log_info "Variant: $variant" + log_info "Working directory: $TEMP_DIR" + + # Step 1: Create OSTree commit + log_info "Executing Step 1: Create OSTree commit..." + local commit_id + if ! commit_id=$(create_ostree_commit "$treefile"); then + log_error "Step 1 failed: Could not create OSTree commit" + return 1 + fi + + log_success "Step 1 completed: OSTree commit $commit_id created" + + # Step 2: Create bootable image + log_info "Executing Step 2: Create bootable image..." + local output_image + if ! output_image=$(create_bootable_image "$commit_id"); then + log_error "Step 2 failed: Could not create bootable image" + return 1 + fi + + log_success "Step 2 completed: Bootable image created" + + log_success "🎉 Pipeline completed successfully!" + log_info "Output image: $output_image" + + # Copy image to workspace for easy access + local final_image="$WORKSPACE_PATH/debian-atomic-$variant.img" + cp "$output_image" "$final_image" + log_info "Image copied to: $final_image" + + # Show testing instructions + echo + log_info "🧪 To test the image:" + echo "qemu-system-x86_64 -m 2G -drive file=$final_image,format=raw -nographic" + echo + log_info "📋 Summary:" + log_success "✅ OSTree commit created from $treefile" + log_success "✅ Bootable image created: $final_image" + log_success "✅ Debian Atomic pipeline completed successfully!" + + return 0 +} + +# Main function +main() { + if [[ $# -lt 1 ]]; then + echo "Usage: $0 [variant]" + echo "Example: $0 debian-minimal.yaml minimal" + echo + echo "Available treefiles:" + if [[ -d "$ATOMIC_CONFIGS_PATH/treefiles" ]]; then + ls "$ATOMIC_CONFIGS_PATH/treefiles/"*.yaml 2>/dev/null | sed 's/.*\///' || echo "No treefiles found" + fi + exit 1 + fi + + local treefile="$1" + local variant="${2:-minimal}" + + # Validate and build components + validate_paths + build_apt_ostree + build_deb_bootc + + # Run the pipeline + if ! run_pipeline "$treefile" "$variant"; then + log_error "Pipeline failed" + exit 1 + fi + + log_success "Pipeline completed successfully" +} + +# Run main function +main "$@" diff --git a/integrate-with-apt-ostree.py b/integrate-with-apt-ostree.py new file mode 100644 index 0000000..da9b20c --- /dev/null +++ b/integrate-with-apt-ostree.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +""" +Integration script between apt-ostree and deb-bootc-image-builder + +This script demonstrates the complete Debian Atomic pipeline: +1. apt-ostree creates OSTree commits from treefiles +2. deb-bootc-image-builder creates bootable images from OSTree commits + +This mimics the Fedora Atomic flow: treefiles → rpm-ostree → bootc-image-builder +""" + +import os +import sys +import subprocess +import json +import tempfile +import shutil +from pathlib import Path +from typing import Dict, Any, Optional + +class DebianAtomicPipeline: + """Complete Debian Atomic pipeline integration""" + + def __init__(self, workspace_path: str): + self.workspace_path = Path(workspace_path) + self.apt_ostree_path = self.workspace_path / "apt-ostree" + self.deb_bootc_path = self.workspace_path / "deb-bootc-image-builder" + self.atomic_configs_path = self.workspace_path / "debian-atomic-configs" + + # Validate paths + self._validate_paths() + + # Temporary working directory + self.temp_dir = Path(tempfile.mkdtemp(prefix="debian-atomic-")) + print(f"Working directory: {self.temp_dir}") + + def _validate_paths(self): + """Validate that all required components exist""" + required_paths = [ + self.apt_ostree_path, + self.deb_bootc_path, + self.atomic_configs_path + ] + + for path in required_paths: + if not path.exists(): + raise FileNotFoundError(f"Required path not found: {path}") + + print("✅ All required components found") + + def create_ostree_commit(self, treefile: str, variant: str) -> Dict[str, Any]: + """Step 1: Use apt-ostree to create OSTree commit from treefile""" + print(f"\n🔧 Step 1: Creating OSTree commit from {treefile}") + + # Build apt-ostree if needed + apt_ostree_bin = self.apt_ostree_path / "target" / "debug" / "apt-ostree" + if not apt_ostree_bin.exists(): + print("Building apt-ostree...") + self._build_apt_ostree() + + # Create OSTree commit + treefile_path = self.atomic_configs_path / "treefiles" / treefile + if not treefile_path.exists(): + raise FileNotFoundError(f"Treefile not found: {treefile_path}") + + # Run apt-ostree compose + result = subprocess.run([ + str(apt_ostree_bin), "compose", str(treefile_path) + ], capture_output=True, text=True, cwd=self.temp_dir) + + if result.returncode != 0: + print(f"Error creating OSTree commit: {result.stderr}") + raise RuntimeError("Failed to create OSTree commit") + + print("✅ OSTree commit created successfully") + + # Extract commit information + commit_info = self._extract_commit_info() + return commit_info + + def _build_apt_ostree(self): + """Build apt-ostree binary""" + print("Building apt-ostree...") + result = subprocess.run([ + "cargo", "build", "--bin", "apt-ostree" + ], cwd=self.apt_ostree_path, capture_output=True, text=True) + + if result.returncode != 0: + print(f"Build failed: {result.stderr}") + raise RuntimeError("Failed to build apt-ostree") + + print("✅ apt-ostree built successfully") + + def _extract_commit_info(self) -> Dict[str, Any]: + """Extract commit information from apt-ostree output""" + # Look for the repository created by apt-ostree + repo_path = self.temp_dir / "apt-ostree-repo" + if not repo_path.exists(): + raise RuntimeError("OSTree repository not found") + + # Get commit information + commits_dir = repo_path / "commits" + if commits_dir.exists(): + commits = list(commits_dir.iterdir()) + if commits: + commit_id = commits[0].name + print(f"Found commit: {commit_id}") + + # Read metadata + metadata_file = commits[0] / "metadata.json" + if metadata_file.exists(): + with open(metadata_file) as f: + metadata = json.load(f) + + return { + "commit_id": commit_id, + "repo_path": str(repo_path), + "metadata": metadata + } + + raise RuntimeError("No commits found in repository") + + def create_bootable_image(self, commit_info: Dict[str, Any], output_format: str = "raw") -> str: + """Step 2: Use deb-bootc-image-builder to create bootable image from OSTree commit""" + print(f"\n🔧 Step 2: Creating bootable image from OSTree commit") + + # Check if deb-bootc-image-builder binary exists + bib_bin = self.deb_bootc_path / "bib" / "bootc-image-builder" + if not bib_bin.exists(): + print("Building deb-bootc-image-builder...") + self._build_deb_bootc() + + # Create a recipe that uses the OSTree commit + recipe = self._create_ostree_recipe(commit_info) + recipe_path = self.temp_dir / "ostree-recipe.yml" + + with open(recipe_path, 'w') as f: + json.dump(recipe, f, indent=2) + + print(f"Created recipe: {recipe_path}") + + # Run deb-bootc-image-builder + output_image = self.temp_dir / f"debian-atomic-{output_format}.img" + + result = subprocess.run([ + str(bib_bin), "build", str(recipe_path), "--output", str(output_image) + ], capture_output=True, text=True, cwd=self.temp_dir) + + if result.returncode != 0: + print(f"Error creating bootable image: {result.stderr}") + raise RuntimeError("Failed to create bootable image") + + print(f"✅ Bootable image created: {output_image}") + return str(output_image) + + def _build_deb_bootc(self): + """Build deb-bootc-image-builder""" + print("Building deb-bootc-image-builder...") + result = subprocess.run([ + "go", "build", "-o", "bib/bootc-image-builder", "./cmd/bootc-image-builder" + ], cwd=self.deb_bootc_path, capture_output=True, text=True) + + if result.returncode != 0: + print(f"Build failed: {result.stderr}") + raise RuntimeError("Failed to build deb-bootc-image-builder") + + print("✅ deb-bootc-image-builder built successfully") + + def _create_ostree_recipe(self, commit_info: Dict[str, Any]) -> Dict[str, Any]: + """Create a recipe that uses the OSTree commit""" + return { + "name": "debian-atomic-ostree", + "description": "Debian Atomic system built from OSTree commit", + "base_image": "debian:trixie-slim", + "stages": [ + { + "name": "ostree_integration", + "type": "ostree", + "config": { + "commit_id": commit_info["commit_id"], + "repo_path": commit_info["repo_path"], + "branch": commit_info["metadata"].get("branch", "debian-atomic") + } + }, + { + "name": "system_config", + "type": "system", + "config": { + "hostname": "debian-atomic", + "timezone": "UTC", + "locale": "en_US.UTF-8" + } + } + ], + "output": { + "format": "raw", + "size": "5G" + } + } + + def run_complete_pipeline(self, treefile: str, variant: str, output_format: str = "raw") -> str: + """Run the complete Debian Atomic pipeline""" + print(f"🚀 Starting Debian Atomic pipeline") + print(f"Treefile: {treefile}") + print(f"Variant: {variant}") + print(f"Output format: {output_format}") + + try: + # Step 1: Create OSTree commit + commit_info = self.create_ostree_commit(treefile, variant) + + # Step 2: Create bootable image + output_image = self.create_bootable_image(commit_info, output_format) + + print(f"\n🎉 Pipeline completed successfully!") + print(f"Output image: {output_image}") + + return output_image + + except Exception as e: + print(f"❌ Pipeline failed: {e}") + raise + + def cleanup(self): + """Clean up temporary files""" + if self.temp_dir.exists(): + shutil.rmtree(self.temp_dir) + print(f"Cleaned up: {self.temp_dir}") + +def main(): + """Main function to demonstrate the integration""" + if len(sys.argv) < 2: + print("Usage: python3 integrate-with-apt-ostree.py [variant] [output_format]") + print("Example: python3 integrate-with-apt-ostree.py minimal.yaml minimal raw") + sys.exit(1) + + treefile = sys.argv[1] + variant = sys.argv[2] if len(sys.argv) > 2 else "minimal" + output_format = sys.argv[3] if len(sys.argv) > 3 else "raw" + + # Get workspace path (assuming script is in deb-bootc-image-builder) + script_dir = Path(__file__).parent + workspace_path = script_dir.parent + + try: + pipeline = DebianAtomicPipeline(str(workspace_path)) + output_image = pipeline.run_complete_pipeline(treefile, variant, output_format) + + print(f"\n📋 Summary:") + print(f"✅ OSTree commit created from {treefile}") + print(f"✅ Bootable image created: {output_image}") + print(f"✅ Debian Atomic pipeline completed successfully!") + + print(f"\n🧪 To test the image:") + print(f"qemu-system-x86_64 -m 2G -drive file={output_image},format=raw -nographic") + + except Exception as e: + print(f"❌ Pipeline failed: {e}") + sys.exit(1) + finally: + # Don't cleanup automatically for testing + # pipeline.cleanup() + pass + +if __name__ == "__main__": + main() diff --git a/simple-integration-test.sh b/simple-integration-test.sh new file mode 100755 index 0000000..d8818a9 --- /dev/null +++ b/simple-integration-test.sh @@ -0,0 +1,124 @@ +#!/bin/bash +# Simple integration test between apt-ostree and deb-bootc-image-builder +# Tests each component individually to identify issues + +set -euo pipefail + +echo "🧪 Simple Integration Test: apt-ostree + deb-bootc-image-builder" +echo "================================================================" + +# Configuration +WORKSPACE_PATH="/home/rob/Documents/Projects/overseer" +APT_OSTREE_PATH="$WORKSPACE_PATH/apt-ostree" +DEB_BOOTC_PATH="$WORKSPACE_PATH/deb-bootc-image-builder" +ATOMIC_CONFIGS_PATH="$WORKSPACE_PATH/debian-atomic-configs" + +# Test 1: Verify apt-ostree can create OSTree commits +echo +echo "✅ Test 1: apt-ostree compose tree" +echo "-----------------------------------" +echo "Running apt-ostree compose tree in container..." + +# Create a simple working directory +WORKDIR="./test-integration-$(date +%s)" +mkdir -p "$WORKDIR" + +# Run apt-ostree compose tree +echo "Command: podman run --rm --privileged -v $APT_OSTREE_PATH:/workspace:z -v $ATOMIC_CONFIGS_PATH:/configs:z -v $WORKDIR:/shared:z apt-ostree-builder:latest bash -c 'cd /workspace && ./target/release/apt-ostree compose tree /configs/treefiles/debian-minimal-apt-ostree.yaml --workdir /shared/apt-ostree-build'" + +if podman run --rm --privileged \ + -v "$APT_OSTREE_PATH:/workspace:z" \ + -v "$ATOMIC_CONFIGS_PATH:/configs:z" \ + -v "$WORKDIR:/shared:z" \ + apt-ostree-builder:latest \ + bash -c "cd /workspace && ./target/release/apt-ostree compose tree /configs/treefiles/debian-minimal-apt-ostree.yaml --workdir /shared/apt-ostree-build"; then + + echo "✅ apt-ostree compose tree completed successfully" + + # Check what was created + echo + echo "📁 Checking created files:" + if [[ -d "$WORKDIR/apt-ostree-build" ]]; then + echo "✅ apt-ostree-build directory created" + ls -la "$WORKDIR/apt-ostree-build/" + + if [[ -d "$WORKDIR/apt-ostree-build/repo" ]]; then + echo "✅ OSTree repository created" + ls -la "$WORKDIR/apt-ostree-build/repo/" + + # Test 2: Verify OSTree repository is accessible + echo + echo "✅ Test 2: OSTree repository access" + echo "-----------------------------------" + echo "Testing ostree checkout from repository..." + + # Extract the commit hash from the reference + COMMIT_HASH=$(cat "$WORKDIR/apt-ostree-build/repo/refs/heads/debian/14/x86_64/minimal") + if [[ -n "$COMMIT_HASH" ]]; then + echo "Found commit: $COMMIT_HASH" + + # Test ostree checkout inside the container to avoid permission issues + ROOTFS_DIR="$WORKDIR/rootfs" + echo "Running ostree checkout in container..." + if podman run --rm --privileged \ + -v "$WORKDIR:/shared:z" \ + apt-ostree-builder:latest \ + bash -c "ostree checkout --repo=/shared/apt-ostree-build/repo --subpath=/ '$COMMIT_HASH' /shared/rootfs"; then + echo "✅ ostree checkout successful" + echo "Rootfs structure:" + ls -la "$ROOTFS_DIR/" + + # Test 3: Verify deb-bootc-image-builder can create bootable image + echo + echo "✅ Test 3: deb-bootc-image-builder integration" + echo "---------------------------------------------" + echo "Testing create-ostree-bootable-image..." + + OUTPUT_IMAGE="$WORKDIR/debian-atomic-bootable.img" + if "$DEB_BOOTC_PATH/bib/create-ostree-bootable-image" "$ROOTFS_DIR" "$OUTPUT_IMAGE"; then + echo "✅ Bootable image created successfully!" + echo "Image: $OUTPUT_IMAGE" + echo "Size: $(du -h "$OUTPUT_IMAGE" | cut -f1)" + + echo + echo "🎉 All integration tests passed!" + echo "✅ apt-ostree compose tree: Working" + echo "✅ OSTree repository access: Working" + echo "✅ deb-bootc-image-builder: Working" + echo + echo "🧪 To test the bootable image:" + echo "qemu-system-x86_64 -m 2G -drive file=$OUTPUT_IMAGE,format=raw -nographic -serial stdio" + + # Copy image to persistent location for testing + PERSISTENT_IMAGE="./debian-atomic-bootable-test.img" + cp "$OUTPUT_IMAGE" "$PERSISTENT_IMAGE" + echo "📁 Image copied to: $PERSISTENT_IMAGE" + + else + echo "❌ deb-bootc-image-builder failed" + fi + + else + echo "❌ ostree checkout failed" + fi + + else + echo "❌ No commit found in repository" + fi + + else + echo "❌ OSTree repository not created" + fi + + else + echo "❌ apt-ostree-build directory not created" + fi + +else + echo "❌ apt-ostree compose tree failed" +fi + +echo +echo "🧹 Cleaning up test directory..." +rm -rf "$WORKDIR" +echo "✅ Test completed" diff --git a/test-vm-with-virsh.sh b/test-vm-with-virsh.sh new file mode 100755 index 0000000..3c2cd04 --- /dev/null +++ b/test-vm-with-virsh.sh @@ -0,0 +1,123 @@ +#!/bin/bash + +# Test VM with virsh script for debian-atomic-bootable image +# This allows us to use virt-manager to monitor the boot process + +set -e + +# Configuration +VM_NAME="debian-atomic-test" +IMAGE_PATH="./debian-atomic-bootable-test.img" +VM_MEMORY="2048" # 2GB +VM_VCPUS="2" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Functions +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Check if image exists +if [[ ! -f "$IMAGE_PATH" ]]; then + log_error "Image not found: $IMAGE_PATH" + log_info "Please run the integration test first to generate the image" + exit 1 +fi + +# Check if VM already exists +if virsh list --all | grep -q "$VM_NAME"; then + log_warning "VM '$VM_NAME' already exists" + read -p "Do you want to remove it and create a new one? (y/N): " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + log_info "Removing existing VM..." + virsh destroy "$VM_NAME" 2>/dev/null || true + virsh undefine "$VM_NAME" 2>/dev/null || true + else + log_info "Using existing VM. You can start it with: virsh start $VM_NAME" + log_info "Or view it in virt-manager" + exit 0 + fi +fi + +# Get absolute path for image +IMAGE_ABSOLUTE_PATH=$(realpath "$IMAGE_PATH") +log_info "Using image: $IMAGE_ABSOLUTE_PATH" + +# Create VM XML definition +log_info "Creating VM '$VM_NAME' with virsh..." + +# Create a temporary XML file for the VM +TEMP_XML=$(mktemp) +cat > "$TEMP_XML" << EOF + + $VM_NAME + $VM_MEMORY + $VM_VCPUS + + hvm + + + + + + + + + + + + + +EOF + +# Define the VM +if virsh define "$TEMP_XML"; then + log_info "✅ VM '$VM_NAME' defined successfully!" + + # Clean up temp file + rm "$TEMP_XML" + + log_info "" + log_info "To manage the VM:" + log_info " - Start: virsh start $VM_NAME" + log_info " - Stop: virsh shutdown $VM_NAME" + log_info " - View: virt-manager" + log_info " - Console: virsh console $VM_NAME" + log_info "" + log_info "The VM will use the raw disk image directly, so any changes" + log_info "to the image will be reflected in the VM." + log_info "" + log_info "Starting VM now..." + + if virsh start "$VM_NAME"; then + log_info "VM started! You can now:" + log_info "1. Open virt-manager to see the graphical console" + log_info "2. Use 'virsh console $VM_NAME' for text console" + log_info "3. Monitor with 'virsh list --all'" + log_info "" + log_info "Current VM status:" + virsh list --all | grep "$VM_NAME" + else + log_error "Failed to start VM" + exit 1 + fi + +else + log_error "Failed to define VM" + rm "$TEMP_XML" + exit 1 +fi