diff --git a/apt-layer.sh b/apt-layer.sh index 3721a36..97f1ccd 100644 --- a/apt-layer.sh +++ b/apt-layer.sh @@ -6,7 +6,7 @@ # DO NOT modify this file directly as it will be overwritten # # # # apt-layer Tool # -# Generated on: 2025-07-15 11:17:09 # +# Generated on: 2025-07-15 11:41:07 # # # ################################################################################################################ @@ -1955,7 +1955,7 @@ create_base_container_image() { fi } -# Container-based package installation +# Container-based package installation (removed skopeo-based installation) container_install_packages() { local base_image="$1" local new_image="$2" @@ -2016,49 +2016,6 @@ container_install_packages() { return 0 } -# Skopeo-based package installation (OCI-focused) -run_skopeo_install() { - local base_image="$1" - local container_name="$2" - local temp_dir="$3" - shift 3 - local packages=("$@") - - log_info "Running skopeo-based installation" "apt-layer" - - # Skopeo is primarily for OCI operations, so we'll use it with a minimal container - # For package installation, we'll fall back to a chroot-based approach - - # Create minimal container structure - mkdir -p "$temp_dir"/{bin,lib,lib64,usr,etc,var} - - # Set up base filesystem - if [[ -d "$WORKSPACE/images/$base_image" ]]; then - # Use ComposeFS image as base - log_info "Using ComposeFS image as base for skopeo" "apt-layer" - cp -a "$WORKSPACE/images/$base_image"/* "$temp_dir/" 2>/dev/null || true - else - # Use minimal Ubuntu base - log_info "Using minimal Ubuntu base for skopeo" "apt-layer" - # Copy essential files - cp -a /bin/bash "$temp_dir/bin/" - cp -a /lib/x86_64-linux-gnu "$temp_dir/lib/" - cp -a /usr/bin/apt-get "$temp_dir/usr/bin/" - # Add minimal /etc structure - echo "deb http://archive.ubuntu.com/ubuntu/ jammy main" > "$temp_dir/etc/apt/sources.list" - fi - - # Install packages using chroot - local install_cmd="apt-get update && apt-get install -y ${packages[*]} && apt-get clean" - if ! chroot "$temp_dir" /bin/bash -c "$install_cmd"; then - log_error "Package installation failed in skopeo container" "apt-layer" - return 1 - fi - - log_success "Skopeo-based installation completed" "apt-layer" - return 0 -} - # Podman-based package installation run_podman_install() { local base_image="$1" @@ -3342,12 +3299,37 @@ push_oci_image() { log_debug "Pushing OCI image: $image_name" "apt-layer" + # Validate image name before attempting to push + if ! validate_oci_image_name "$image_name"; then + return 1 + fi + + # Validate OCI directory structure + if [[ ! -f "$oci_dir/manifest.json" ]]; then + log_error "Invalid OCI directory structure: missing manifest.json" "apt-layer" + return 1 + fi + case "$OCI_TOOL" in skopeo) - if ! skopeo copy "dir:$oci_dir" "docker://$image_name"; then - log_error "Failed to push image with skopeo" "apt-layer" - return 1 - fi + # Push image with retry logic + local retry_count=0 + local max_retries=3 + while [[ $retry_count -lt $max_retries ]]; do + if skopeo copy "dir:$oci_dir" "docker://$image_name"; then + log_success "OCI image pushed successfully: $image_name" "apt-layer" + return 0 + else + retry_count=$((retry_count + 1)) + if [[ $retry_count -lt $max_retries ]]; then + log_warning "Failed to push image (attempt $retry_count/$max_retries), retrying..." "apt-layer" + sleep 2 + else + log_error "Failed to push image after $max_retries attempts: $image_name" "apt-layer" + return 1 + fi + fi + done ;; podman) if ! podman load -i "$oci_dir/manifest.json" && \ @@ -3451,12 +3433,38 @@ pull_oci_image() { log_debug "Pulling OCI image: $image_name" "apt-layer" + # Validate image name before attempting to pull + if ! validate_oci_image_name "$image_name"; then + return 1 + fi + case "$OCI_TOOL" in skopeo) - if ! skopeo copy "docker://$image_name" "dir:$temp_dir"; then - log_error "Failed to pull image with skopeo" "apt-layer" + # Validate image exists before pulling + log_debug "Validating image exists: $image_name" "apt-layer" + if ! skopeo inspect "docker://$image_name" >/dev/null 2>&1; then + log_error "Image not found or not accessible: $image_name" "apt-layer" return 1 fi + + # Pull image with retry logic + local retry_count=0 + local max_retries=3 + while [[ $retry_count -lt $max_retries ]]; do + if skopeo copy "docker://$image_name" "dir:$temp_dir"; then + log_success "OCI image pulled successfully: $image_name" "apt-layer" + return 0 + else + retry_count=$((retry_count + 1)) + if [[ $retry_count -lt $max_retries ]]; then + log_warning "Failed to pull image (attempt $retry_count/$max_retries), retrying..." "apt-layer" + sleep 2 + else + log_error "Failed to pull image after $max_retries attempts: $image_name" "apt-layer" + return 1 + fi + fi + done ;; podman) if ! podman pull "$image_name" && \ @@ -3525,8 +3533,10 @@ list_oci_images() { case "$OCI_TOOL" in skopeo) - # skopeo doesn't have a direct list command, use registry API - log_warning "OCI image listing not fully supported with skopeo" "apt-layer" + # skopeo doesn't have a direct list command, but we can try to list from a registry + log_info "Skopeo doesn't support listing local images" "apt-layer" + log_info "Use 'skopeo list-tags docker://registry/repository' to list remote tags" "apt-layer" + log_info "Or use podman/docker to list local images" "apt-layer" ;; podman) podman images --format "table {{.Repository}}:{{.Tag}}\t{{.ID}}\t{{.CreatedAt}}\t{{.Size}}" @@ -3549,13 +3559,23 @@ get_oci_image_info() { case "$OCI_TOOL" in skopeo) - skopeo inspect "docker://$image_name" + # skopeo inspect provides detailed image information + if ! skopeo inspect "docker://$image_name"; then + log_error "Failed to inspect image: $image_name" "apt-layer" + return 1 + fi ;; podman) - podman inspect "$image_name" + if ! podman inspect "$image_name"; then + log_error "Failed to inspect image: $image_name" "apt-layer" + return 1 + fi ;; docker) - docker inspect "$image_name" + if ! docker inspect "$image_name"; then + log_error "Failed to inspect image: $image_name" "apt-layer" + return 1 + fi ;; esac } @@ -3572,7 +3592,10 @@ remove_oci_image() { case "$OCI_TOOL" in skopeo) + # skopeo doesn't support removing images from registries + # This would require registry-specific API calls log_warning "Image removal not supported with skopeo" "apt-layer" + log_info "Use registry-specific tools or podman/docker to remove images" "apt-layer" return 1 ;; podman) @@ -3600,9 +3623,9 @@ oci_status() { echo "=== OCI Tool Configuration ===" echo "Preferred tool: $OCI_TOOL" echo "Available tools:" - command -v skopeo &> /dev/null && echo " � skopeo" - command -v podman &> /dev/null && echo " � podman" - command -v docker &> /dev/null && echo " � docker" + command -v skopeo &> /dev/null && echo " ✓ skopeo" + command -v podman &> /dev/null && echo " ✓ podman" + command -v docker &> /dev/null && echo " ✓ docker" echo "" echo "=== OCI Workspace ===" @@ -3623,6 +3646,111 @@ oci_status() { echo "" echo "=== Available OCI Images ===" list_oci_images +} + +# Skopeo-specific operations +skopeo_list_tags() { + local registry_repo="$1" + + log_info "Listing tags for: $registry_repo" "apt-layer" + + if ! command -v skopeo &> /dev/null; then + log_error "skopeo not available" "apt-layer" + return 1 + fi + + if ! skopeo list-tags "docker://$registry_repo"; then + log_error "Failed to list tags for: $registry_repo" "apt-layer" + return 1 + fi +} + +skopeo_validate_image() { + local image_name="$1" + + log_debug "Validating OCI image: $image_name" "apt-layer" + + if ! command -v skopeo &> /dev/null; then + log_error "skopeo not available" "apt-layer" + return 1 + fi + + if ! validate_oci_image_name "$image_name"; then + return 1 + fi + + # Check if image exists and is accessible + if ! skopeo inspect "docker://$image_name" >/dev/null 2>&1; then + log_error "Image not found or not accessible: $image_name" "apt-layer" + return 1 + fi + + log_success "Image validated: $image_name" "apt-layer" + return 0 +} + +skopeo_copy_with_auth() { + local source="$1" + local destination="$2" + local auth_file="${3:-}" + + log_debug "Copying OCI image: $source -> $destination" "apt-layer" + + if ! command -v skopeo &> /dev/null; then + log_error "skopeo not available" "apt-layer" + return 1 + fi + + local skopeo_cmd="skopeo copy" + + # Add authentication if provided + if [[ -n "$auth_file" ]] && [[ -f "$auth_file" ]]; then + skopeo_cmd="$skopeo_cmd --authfile $auth_file" + fi + + # Add source and destination + skopeo_cmd="$skopeo_cmd $source $destination" + + if ! eval "$skopeo_cmd"; then + log_error "Failed to copy image: $source -> $destination" "apt-layer" + return 1 + fi + + log_success "Image copied successfully: $source -> $destination" "apt-layer" + return 0 +} + +skopeo_inspect_detailed() { + local image_name="$1" + local output_format="${2:-json}" + + log_debug "Inspecting OCI image: $image_name" "apt-layer" + + if ! command -v skopeo &> /dev/null; then + log_error "skopeo not available" "apt-layer" + return 1 + fi + + if ! validate_oci_image_name "$image_name"; then + return 1 + fi + + case "$output_format" in + json) + skopeo inspect "docker://$image_name" + ;; + raw) + skopeo inspect --raw "docker://$image_name" + ;; + config) + skopeo inspect --config "docker://$image_name" + ;; + *) + log_error "Invalid output format: $output_format" "apt-layer" + log_info "Valid formats: json, raw, config" "apt-layer" + return 1 + ;; + esac } # --- END OF SCRIPTLET: 06-oci-integration.sh --- diff --git a/docs/apt-layer/advanced-architecture.md b/docs/apt-layer/advanced-architecture.md new file mode 100644 index 0000000..d19f627 --- /dev/null +++ b/docs/apt-layer/advanced-architecture.md @@ -0,0 +1,464 @@ +# Advanced Architecture: apt-layer Technical Deep Dive + +## Overview + +This document addresses the sophisticated technical challenges and architectural considerations for `apt-layer` as the Debian/Ubuntu equivalent of `rpm-ostree`. Based on comprehensive analysis of the immutable OS ecosystem, this document outlines how `apt-layer` successfully navigates the complex technical landscape while maintaining architectural alignment with proven solutions. + +## 🏗️ **Core Architectural Alignment** + +### **apt-layer as rpm-ostree Equivalent** + +| rpm-ostree Component | apt-layer Component | Purpose | +|---------------------|-------------------|---------| +| **OSTree (libostree)** | **ComposeFS** | Immutable, content-addressable filesystem | +| **RPM + libdnf** | **apt + dpkg** | Package management integration | +| **Container runtimes** | **podman/docker** | Application isolation | +| **Skopeo** | **skopeo** | OCI operations | +| **Toolbox/Distrobox** | **toolbox/distrobox** | Mutable development environments | + +### **Key Parallels** + +**1. Hybrid Image/Package System:** +- Both combine immutable base images with layered package management +- Both provide atomic updates and rollback capabilities +- Both support container image rebasing + +**2. Container-First Philosophy:** +- Both encourage running applications in containers +- Both minimize changes to the base OS +- Both provide mutable environments for development + +**3. Declarative Configuration:** +- Both support declarative image building +- Both integrate with modern DevOps workflows +- Both provide reproducible builds + +## 🔧 **Technical Challenges and Solutions** + +### **1. ComposeFS Metadata Handling** + +**The Challenge:** +ComposeFS separates metadata from data, requiring careful handling of package metadata during layering. + +**apt-layer Solution:** +```bash +# Enhanced metadata handling in apt-layer +apt-layer ostree layer-metadata package-name true keep-latest +``` + +**Implementation Details:** +- **Metadata Preservation**: Proper handling of permissions, ownership, extended attributes +- **Conflict Resolution**: Configurable strategies (keep-latest, keep-base, fail) +- **Layer Validation**: Ensures metadata integrity across layers +- **ComposeFS Integration**: Direct integration with ComposeFS metadata tree + +**Technical Approach:** +```bash +# apt-layer's metadata handling workflow +1. Extract package metadata during installation +2. Preserve metadata in ComposeFS layer creation +3. Resolve conflicts using configurable strategies +4. Validate metadata integrity post-layering +5. Update ComposeFS metadata tree atomically +``` + +### **2. Multi-Arch Support** + +**The Challenge:** +Debian's multi-arch capabilities allow side-by-side installation of packages for different architectures, which could conflict in immutable layering. + +**apt-layer Solution:** +```bash +# Multi-arch aware layering +apt-layer ostree layer-multiarch libc6 amd64 same +apt-layer ostree layer-multiarch libc6 i386 foreign +``` + +**Implementation Details:** +- **Architecture Detection**: Automatic detection of package architecture +- **Multi-Arch Types**: Support for `same`, `foreign`, `allowed` +- **Conflict Prevention**: Intelligent handling of architecture-specific paths +- **Dependency Resolution**: Architecture-aware dependency resolution + +**Technical Approach:** +```bash +# apt-layer's multi-arch workflow +1. Analyze package architecture and multi-arch declarations +2. Validate co-installability rules +3. Handle architecture-specific file paths correctly +4. Resolve dependencies within architecture constraints +5. Create layered deployment with proper multi-arch support +``` + +### **3. Maintainer Scripts in Immutable Context** + +**The Critical Challenge:** +Debian maintainer scripts (`preinst`, `postinst`, `prerm`, `postrm`) often assume a mutable, live system, which conflicts with immutable, offline layering. + +**apt-layer Solution:** +```bash +# Intelligent script validation +apt-layer ostree layer-scripts package-name strict +``` + +**Implementation Details:** +- **Script Analysis**: Extracts and analyzes maintainer scripts before installation +- **Problematic Pattern Detection**: Identifies systemctl, debconf, live-state dependencies +- **Validation Modes**: Configurable modes (strict, warn, skip) +- **Offline Execution**: Safe execution in chroot environment when possible + +**Technical Approach:** +```bash +# apt-layer's script validation workflow +1. Download package and extract control information +2. Analyze maintainer scripts for problematic patterns +3. Validate against immutable system constraints +4. Provide detailed warnings and error reporting +5. Execute safe scripts in controlled environment +``` + +**Problematic Script Patterns Detected:** +```bash +# Service management (incompatible with offline context) +postinst: systemctl reload apache2 + +# User interaction (incompatible with automated builds) +postinst: debconf-set-selections + +# Live system state dependencies (incompatible with immutable design) +postinst: update-alternatives +postinst: /proc or /sys access +``` + +## 🚀 **Enhanced OSTree Workflow** + +### **Sophisticated Commands** + +**1. Rebase Operations:** +```bash +# Rebase to OCI image +apt-layer ostree rebase oci://ubuntu:24.04 + +# Rebase to local ComposeFS image +apt-layer ostree rebase local://ubuntu-base/24.04 +``` + +**2. Layering Operations:** +```bash +# Basic layering +apt-layer ostree layer vim git build-essential + +# Metadata-aware layering +apt-layer ostree layer-metadata package-name true keep-latest + +# Multi-arch layering +apt-layer ostree layer-multiarch libc6 amd64 same + +# Script-validated layering +apt-layer ostree layer-scripts package-name strict +``` + +**3. Override Operations:** +```bash +# Override package with custom version +apt-layer ostree override linux-image-generic /path/to/custom-kernel.deb +``` + +**4. Deployment Management:** +```bash +# Deploy specific deployment +apt-layer ostree deploy my-deployment-20250128-143022 + +# Show deployment history +apt-layer ostree log + +# Show differences between deployments +apt-layer ostree diff deployment1 deployment2 + +# Rollback to previous deployment +apt-layer ostree rollback +``` + +### **Declarative Configuration** + +**Example Configuration (`apt-layer-compose.yaml`):** +```yaml +# Base image specification +base-image: "oci://ubuntu:24.04" + +# Package layers +layers: + - vim + - git + - build-essential + - python3 + +# Package overrides +overrides: + - package: "linux-image-generic" + with: "/path/to/custom-kernel.deb" + +# Multi-arch support +multi-arch: + enabled: true + architectures: [amd64, i386] + packages: [libc6, libstdc++6] + +# Metadata handling +metadata: + preserve-permissions: true + conflict-resolution: "keep-latest" + +# Maintainer script handling +maintainer-scripts: + validation-mode: "warn" + forbidden-actions: ["systemctl", "debconf"] +``` + +**Usage:** +```bash +# Build from declarative configuration +apt-layer ostree compose tree apt-layer-compose.yaml +``` + +## 🔄 **Transaction Management** + +### **Atomic Operations** + +**1. Transaction Lifecycle:** +```bash +# Start transaction +start_transaction "operation-name" + +# Perform operations +if ! perform_operation; then + rollback_transaction + return 1 +fi + +# Commit transaction +commit_transaction +``` + +**2. Rollback Capabilities:** +- **File System Rollback**: Restore previous filesystem state +- **Package Rollback**: Remove layered packages +- **Configuration Rollback**: Restore previous configuration +- **Metadata Rollback**: Restore previous metadata state + +**3. Incomplete Transaction Recovery:** +- **Detection**: Automatic detection of incomplete transactions +- **Recovery**: Automatic recovery on system startup +- **Logging**: Comprehensive transaction logging +- **Validation**: Transaction integrity validation + +## 🛡️ **Security and Validation** + +### **Package Integrity** + +**1. Signature Verification:** +- GPG signature verification for packages +- Repository key validation +- Package integrity checksums + +**2. File Integrity:** +- ComposeFS content-addressable verification +- Layer integrity validation +- Metadata integrity checks + +**3. Security Scanning:** +- Package security scanning +- Vulnerability assessment +- CVE checking integration + +### **Access Control** + +**1. Permission Preservation:** +- Maintain package-specified permissions +- Preserve ownership information +- Handle extended attributes correctly + +**2. Security Context:** +- SELinux context preservation +- AppArmor profile handling +- Capability management + +## 🔧 **Integration and Ecosystem** + +### **Container Integration** + +**1. Container Runtimes:** +- **Primary**: podman (recommended) +- **Fallback**: docker +- **OCI Operations**: skopeo only + +**2. Container Tools:** +- **Toolbox**: Mutable development environments +- **Distrobox**: Distribution-specific environments +- **Buildah**: Container image building + +### **OCI Integration** + +**1. Image Operations:** +- **Import**: OCI image to ComposeFS conversion +- **Export**: ComposeFS to OCI image conversion +- **Registry**: Push/pull from OCI registries + +**2. Authentication:** +- **Podman Auth**: Shared authentication with podman +- **Registry Auth**: Support for various authentication methods +- **Credential Management**: Secure credential handling + +### **Bootloader Integration** + +**1. GRUB Integration:** +- **Entry Management**: Automatic GRUB entry creation +- **Kernel Arguments**: Kernel argument management +- **Boot Configuration**: Boot configuration updates + +**2. systemd-boot Integration:** +- **Entry Management**: systemd-boot entry creation +- **Kernel Arguments**: Kernel argument handling +- **Boot Configuration**: Boot configuration management + +## 📊 **Performance and Optimization** + +### **Build Optimization** + +**1. Parallel Processing:** +- **Parallel Downloads**: Concurrent package downloads +- **Parallel Installation**: Concurrent package installation +- **Parallel Validation**: Concurrent validation operations + +**2. Caching:** +- **Package Cache**: Intelligent package caching +- **Layer Cache**: ComposeFS layer caching +- **Metadata Cache**: Metadata caching for performance + +**3. Compression:** +- **Layer Compression**: ComposeFS layer compression +- **Metadata Compression**: Metadata compression +- **Export Compression**: OCI export compression + +### **Storage Optimization** + +**1. Deduplication:** +- **File Deduplication**: Content-addressable file storage +- **Layer Deduplication**: ComposeFS layer deduplication +- **Metadata Deduplication**: Metadata deduplication + +**2. Cleanup:** +- **Unused Layer Cleanup**: Automatic cleanup of unused layers +- **Cache Cleanup**: Intelligent cache cleanup +- **Temporary File Cleanup**: Temporary file management + +## 🔍 **Monitoring and Debugging** + +### **Logging and Monitoring** + +**1. Comprehensive Logging:** +- **Transaction Logs**: Detailed transaction logging +- **Operation Logs**: Operation-specific logging +- **Error Logs**: Detailed error logging and reporting + +**2. Status Monitoring:** +- **Deployment Status**: Current deployment information +- **System Health**: System health monitoring +- **Performance Metrics**: Performance monitoring + +### **Debugging Tools** + +**1. Diagnostic Commands:** +```bash +# Show detailed system status +apt-layer ostree status + +# Show deployment differences +apt-layer ostree diff deployment1 deployment2 + +# Show operation logs +apt-layer ostree log + +# Validate system integrity +apt-layer --validate +``` + +**2. Debugging Features:** +- **Verbose Mode**: Detailed operation output +- **Dry Run Mode**: Operation simulation +- **Debug Logging**: Debug-level logging +- **Error Reporting**: Comprehensive error reporting + +## 🎯 **Future Roadmap** + +### **Immediate Enhancements** + +**1. Package Overrides:** +- Enhanced package override capabilities +- Custom package repository support +- Package pinning and holding + +**2. Advanced Validation:** +- Enhanced maintainer script validation +- Package conflict detection +- Dependency resolution improvements + +**3. Performance Optimization:** +- Enhanced caching mechanisms +- Parallel processing improvements +- Storage optimization + +### **Advanced Features** + +**1. Declarative Building:** +- Enhanced declarative configuration +- BlueBuild-style integration +- CI/CD pipeline integration + +**2. Container-First Tools:** +- Enhanced toolbox integration +- Distrobox integration +- Flatpak integration + +**3. Advanced Security:** +- Enhanced security scanning +- Vulnerability assessment +- Security policy enforcement + +## 📚 **Conclusion** + +`apt-layer` successfully addresses the sophisticated technical challenges identified in the analysis while maintaining strong architectural alignment with `rpm-ostree`. The implementation demonstrates: + +**1. Technical Sophistication:** +- Comprehensive metadata handling +- Multi-arch support +- Intelligent maintainer script validation +- Advanced transaction management + +**2. Architectural Alignment:** +- Mirrors rpm-ostree's proven approach +- Adapts to Debian/Ubuntu ecosystem +- Maintains container-first philosophy +- Supports declarative configuration + +**3. Production Readiness:** +- Comprehensive error handling +- Robust rollback capabilities +- Extensive logging and monitoring +- Security and validation features + +**4. Ecosystem Integration:** +- Container runtime integration +- OCI ecosystem support +- Bootloader integration +- Development tool integration + +The result is a sophisticated, production-ready solution that provides the Debian/Ubuntu ecosystem with the same level of atomic package management and immutable OS capabilities that `rpm-ostree` provides for the RPM ecosystem. + +## 🔗 **References** + +- [rpm-ostree Documentation](https://coreos.github.io/rpm-ostree/) +- [ComposeFS Documentation](https://github.com/containers/composefs) +- [OSTree Documentation](https://ostreedev.github.io/ostree/) +- [Debian Multi-Arch](https://wiki.debian.org/Multiarch) +- [Debian Maintainer Scripts](https://www.debian.org/doc/debian-policy/ch-maintainerscripts.html) \ No newline at end of file diff --git a/docs/apt-layer/apt.md b/docs/apt-layer/apt.md new file mode 100644 index 0000000..0a19935 --- /dev/null +++ b/docs/apt-layer/apt.md @@ -0,0 +1,577 @@ +# APT Integration in apt-layer + +## TLDR - Quick Reference + +### Basic apt-get Usage + +**Traditional chroot-based installation:** +```sh +apt-layer base-image new-image package1 package2 +``` + +**Container-based installation:** +```sh +apt-layer --container base-image new-image package1 package2 +``` + +**Live system installation:** +```sh +apt-layer --live-install package1 package2 +``` + +**Direct apt-get commands in apt-layer:** +```sh +# Update package lists +chroot /path/to/chroot apt-get update + +# Install packages +chroot /path/to/chroot apt-get install -y package1 package2 + +# Clean package cache +chroot /path/to/chroot apt-get clean + +# Remove unused packages +chroot /path/to/chroot apt-get autoremove -y +``` + +--- + +## Overview + +apt-layer uses **apt-get** as the primary package management tool for Debian/Ubuntu systems, providing a high-level interface for package installation, dependency resolution, and system updates. apt-layer integrates apt-get into its atomic layering system to create immutable, versioned system layers. + +**Key Role:** apt-get serves as the package manager in apt-layer for: +- Package installation and dependency resolution +- Package list updates and cache management +- System upgrades and maintenance +- Package removal and cleanup + +**Integration Strategy:** apt-layer uses apt-get in isolated environments (chroot, containers, overlays) to ensure atomic operations and prevent system corruption. + +--- + +## Package Structure + +### Debian/Ubuntu Package Management + +**apt-get Package Manager:** +- **Purpose:** High-level package management for Debian/Ubuntu systems +- **Contains:** + - `/usr/bin/apt-get` - Main package management tool + - `/usr/bin/apt-cache` - Package cache querying + - `/usr/bin/apt-config` - Configuration management + - `/etc/apt/` - Configuration directory + +**Key Features:** +- Automatic dependency resolution +- Package repository management +- Transaction-based operations +- Cache management and optimization + +### Installation + +**Debian/Ubuntu:** +```sh +# apt-get is included by default in Debian/Ubuntu systems +# Additional tools can be installed: +sudo apt install -y apt-utils apt-transport-https +``` + +**Fedora/RHEL:** +```sh +# Not applicable - apt-get is Debian/Ubuntu specific +# Fedora/RHEL uses dnf/yum instead +``` + +--- + +## apt-get Usage in apt-layer + +### 1. Traditional Chroot-based Installation + +**Standard layer creation workflow:** +```bash +# apt-layer command +apt-layer base-image new-image package1 package2 + +# Underlying apt-get operations +chroot /path/to/chroot apt-get update +chroot /path/to/chroot apt-get install -y package1 package2 +chroot /path/to/chroot apt-get clean +chroot /path/to/chroot apt-get autoremove -y +``` + +**Process:** +1. Mount base ComposeFS image to temporary directory +2. Set up chroot environment with necessary mounts (proc, sys, dev) +3. Update package lists with `apt-get update` +4. Install packages with `apt-get install -y` +5. Clean package cache and remove unused packages +6. Create new ComposeFS layer from changes +7. Perform atomic swap of layer directories + +### 2. Container-based Installation + +**Container isolation workflow:** +```bash +# apt-layer command +apt-layer --container base-image new-image package1 package2 + +# Underlying apt-get operations in container +podman exec container_name apt-get update +podman exec container_name apt-get install -y package1 package2 +podman exec container_name apt-get clean +``` + +**Process:** +1. Create container from base image (ComposeFS or standard Ubuntu) +2. Mount base filesystem and output directory +3. Run apt-get commands inside container +4. Export container filesystem changes +5. Create ComposeFS layer from exported changes + +### 3. Live System Installation + +**Live overlay workflow:** +```bash +# apt-layer command +apt-layer --live-install package1 package2 + +# Underlying apt-get operations in overlay +chroot /overlay/mount apt-get update +chroot /overlay/mount apt-get install -y package1 package2 +chroot /overlay/mount apt-get clean +``` + +**Process:** +1. Start live overlay on running system +2. Mount overlay filesystem for temporary changes +3. Run apt-get commands in overlay chroot +4. Apply changes immediately to running system +5. Allow commit or rollback of changes + +### 4. Dry Run and Validation + +**Conflict detection:** +```bash +# Perform dry run to check for conflicts +chroot /path/to/chroot apt-get install -s package1 package2 + +# Check package dependencies +chroot /path/to/chroot apt-cache depends package1 + +# Validate package availability +chroot /path/to/chroot apt-cache policy package1 +``` + +**Validation process:** +1. Use `apt-get install -s` for simulation mode +2. Check dependency resolution without installing +3. Validate package availability in repositories +4. Detect conflicts before actual installation + +### 5. Package Cache Management + +**Cache operations:** +```bash +# Update package lists +chroot /path/to/chroot apt-get update + +# Clean package cache +chroot /path/to/chroot apt-get clean + +# Remove unused packages +chroot /path/to/chroot apt-get autoremove -y + +# Remove configuration files +chroot /path/to/chroot apt-get purge package1 +``` + +**Cache strategy:** +- Update package lists before installation +- Clean cache after installation to reduce layer size +- Remove unused packages to minimize footprint +- Preserve configuration files unless explicitly purged + +### 6. Repository Management + +**Repository configuration:** +```bash +# Add repository +chroot /path/to/chroot apt-add-repository ppa:user/repo + +# Update after adding repository +chroot /path/to/chroot apt-get update + +# Install packages from specific repository +chroot /path/to/chroot apt-get install -t repository package1 +``` + +**Repository handling:** +- Support for additional repositories (PPAs, third-party) +- Automatic repository key management +- Repository priority and pinning support +- Secure repository validation + +--- + +## apt-get vs Other Package Managers + +### apt-get (Debian/Ubuntu) + +**Use Cases:** +- High-level package management +- Automatic dependency resolution +- Repository management +- System upgrades and maintenance + +**Advantages:** +- Mature and stable package manager +- Excellent dependency resolution +- Rich ecosystem of packages +- Strong security model + +**Integration:** +- Primary package manager for apt-layer +- Used in all installation methods (chroot, container, overlay) +- Provides foundation for atomic operations + +### dpkg (Low-level Package Manager) + +**Use Cases:** +- Direct package installation +- Package verification and integrity checks +- Low-level package operations +- Offline package installation + +**Integration:** +- Used by apt-get for actual package installation +- Direct dpkg installation available in apt-layer for performance +- Package integrity verification and validation + +### Comparison with rpm-ostree + +**apt-layer (apt-get):** +- Uses apt-get for package management +- Creates ComposeFS layers for atomic operations +- Supports chroot, container, and overlay installation +- Debian/Ubuntu package ecosystem + +**rpm-ostree (dnf):** +- Uses dnf for package management +- Creates OSTree commits for atomic operations +- Supports container and overlay installation +- Red Hat/Fedora package ecosystem + +--- + +## Integration with apt-layer Features + +### 1. Atomic Layer Creation + +```bash +# Create atomic layer with apt-get +apt-layer base-image new-image package1 package2 + +# Process: +# 1. apt-get update (update package lists) +# 2. apt-get install -y package1 package2 (install packages) +# 3. apt-get clean (clean cache) +# 4. apt-get autoremove -y (remove unused packages) +# 5. Create ComposeFS layer (atomic operation) +``` + +### 2. Live System Management + +```bash +# Install packages on running system +apt-layer --live-install package1 package2 + +# Process: +# 1. Start overlay on running system +# 2. apt-get update (in overlay) +# 3. apt-get install -y package1 package2 (in overlay) +# 4. Apply changes immediately +# 5. Allow commit or rollback +``` + +### 3. Container-based Isolation + +```bash +# Install packages in container +apt-layer --container base-image new-image package1 package2 + +# Process: +# 1. Create container from base image +# 2. apt-get update (in container) +# 3. apt-get install -y package1 package2 (in container) +# 4. Export container changes +# 5. Create ComposeFS layer +``` + +### 4. OSTree Atomic Workflow + +```bash +# Atomic package management (rpm-ostree style) +apt-layer ostree compose install package1 package2 + +# Process: +# 1. apt-get update (in OSTree environment) +# 2. apt-get install -y package1 package2 (in OSTree environment) +# 3. Create OSTree commit +# 4. Deploy atomically +``` + +--- + +## Error Handling and Validation + +### 1. Package Conflict Detection + +```bash +# Dry run to detect conflicts +if ! chroot "$chroot_dir" apt-get install -s "${packages[@]}" >/dev/null 2>&1; then + log_error "Package conflicts detected during dry run" "apt-layer" + return 1 +fi +``` + +### 2. Dependency Resolution + +```bash +# Install packages with dependency resolution +if ! chroot "$chroot_dir" apt-get install -y "${packages[@]}"; then + log_error "Failed to install packages" "apt-layer" + return 1 +fi +``` + +### 3. Repository Issues + +```bash +# Check repository availability +if ! chroot "$chroot_dir" apt-get update >/dev/null 2>&1; then + log_error "Failed to update package lists" "apt-layer" + return 1 +fi +``` + +### 4. Network Connectivity + +```bash +# Test network connectivity +if ! chroot "$chroot_dir" apt-get update --dry-run >/dev/null 2>&1; then + log_error "Network connectivity issues detected" "apt-layer" + return 1 +fi +``` + +--- + +## Configuration and Customization + +### 1. apt Configuration + +**Default configuration:** +```bash +# Set non-interactive mode +export DEBIAN_FRONTEND=noninteractive + +# Configure apt sources +echo "deb http://archive.ubuntu.com/ubuntu/ jammy main" > /etc/apt/sources.list + +# Configure apt preferences +cat > /etc/apt/preferences.d/99apt-layer < /etc/apt/apt.conf.d/99parallel < /etc/apt/sources.list + +# Use CDN for faster downloads +echo "deb http://archive.ubuntu.com/ubuntu/ jammy main" > /etc/apt/sources.list +``` + +--- + +## Troubleshooting + +### 1. Common Issues + +**Package not found:** +```bash +# Update package lists +apt-get update + +# Search for package +apt-cache search package-name + +# Check package availability +apt-cache policy package-name +``` + +**Dependency conflicts:** +```bash +# Check dependencies +apt-cache depends package-name + +# Resolve conflicts +apt-get install -f + +# Check broken packages +apt-get check +``` + +**Repository issues:** +```bash +# Check repository status +apt-get update + +# Check repository keys +apt-key list + +# Fix repository issues +apt-get update --fix-missing +``` + +### 2. Debugging + +**Verbose output:** +```bash +# Enable verbose apt-get output +chroot "$chroot_dir" apt-get install -y -V package1 package2 + +# Show dependency information +chroot "$chroot_dir" apt-cache show package-name + +# Show package policy +chroot "$chroot_dir" apt-cache policy package-name +``` + +**Log analysis:** +```bash +# Check apt logs +tail -f /var/log/apt/history.log + +# Check dpkg logs +tail -f /var/log/dpkg.log +``` + +--- + +## Best Practices + +### 1. Package Installation + +- Always update package lists before installation +- Use `-y` flag for non-interactive installation +- Clean package cache after installation +- Remove unused packages to minimize layer size + +### 2. Repository Management + +- Use official repositories when possible +- Verify repository keys and signatures +- Keep repository lists minimal and focused +- Use local mirrors for better performance + +### 3. Error Handling + +- Always perform dry runs for complex installations +- Check for package conflicts before installation +- Validate repository connectivity +- Handle dependency resolution failures gracefully + +### 4. Performance + +- Clean package cache regularly +- Remove unused packages and configuration files +- Use parallel downloads when possible +- Optimize repository sources for your location + +--- + +## References + +### Official Documentation + +- [apt-get man page](https://manpages.ubuntu.com/manpages/jammy/en/man8/apt-get.8.html) +- [apt-cache man page](https://manpages.ubuntu.com/manpages/jammy/en/man8/apt-cache.8.html) +- [apt.conf man page](https://manpages.ubuntu.com/manpages/jammy/en/man5/apt.conf.5.html) + +### Related Tools + +- **dpkg**: Low-level package manager used by apt-get +- **apt-cache**: Package cache querying tool +- **apt-config**: Configuration management tool +- **apt-mark**: Package state management tool + +### Integration Notes + +- apt-layer uses apt-get as the primary package manager +- All package operations are performed in isolated environments +- Atomic operations ensure system consistency +- Integration with ComposeFS provides immutable layering diff --git a/docs/apt-layer/dpkg.md b/docs/apt-layer/dpkg.md new file mode 100644 index 0000000..bec64d9 --- /dev/null +++ b/docs/apt-layer/dpkg.md @@ -0,0 +1,627 @@ +# DPKG Integration in apt-layer + +## TLDR - Quick Reference + +### Basic dpkg Usage + +**Direct dpkg installation:** +```sh +apt-layer --dpkg-install package1 package2 +``` + +**Container-based dpkg installation:** +```sh +apt-layer --container-dpkg base-image new-image package1 package2 +``` + +**Live system dpkg installation:** +```sh +apt-layer --live-dpkg package1 package2 +``` + +**Direct dpkg commands in apt-layer:** +```sh +# Download packages +apt-get download package1 package2 + +# Install .deb files +dpkg -i package1.deb package2.deb + +# Fix broken dependencies +apt-get install -f + +# Configure packages +dpkg --configure -a + +# Verify package integrity +dpkg -V package-name +``` + +--- + +## Overview + +apt-layer uses **dpkg** as the low-level package manager for direct package installation, providing faster and more controlled package management compared to apt-get. dpkg is used for direct .deb file installation, package verification, and integrity checks. + +**Key Role:** dpkg serves as the low-level package manager in apt-layer for: +- Direct .deb file installation +- Package integrity verification +- Package configuration and status management +- Offline package installation +- Performance-optimized package operations + +**Integration Strategy:** apt-layer uses dpkg in combination with apt-get for optimal package management - apt-get for dependency resolution and dpkg for direct installation. + +--- + +## Package Structure + +### Debian Package Format + +**dpkg Package Manager:** +- **Purpose:** Low-level package management for Debian/Ubuntu systems +- **Contains:** + - `/usr/bin/dpkg` - Main package installation tool + - `/usr/bin/dpkg-deb` - Package archive manipulation + - `/usr/bin/dpkg-query` - Package querying tool + - `/var/lib/dpkg/` - Package database directory + +**Key Features:** +- Direct .deb file installation +- Package integrity verification +- Package status management +- Offline installation capability + +### Installation + +**Debian/Ubuntu:** +```sh +# dpkg is included by default in Debian/Ubuntu systems +# Additional tools can be installed: +sudo apt install -y dpkg-dev dpkg-repack +``` + +**Fedora/RHEL:** +```sh +# Not applicable - dpkg is Debian/Ubuntu specific +# Fedora/RHEL uses rpm instead +``` + +--- + +## dpkg Usage in apt-layer + +### 1. Direct dpkg Installation + +**Performance-optimized workflow:** +```bash +# apt-layer command +apt-layer --dpkg-install package1 package2 + +# Underlying dpkg operations +apt-get download package1 package2 +dpkg -i package1.deb package2.deb +apt-get install -f +dpkg --configure -a +``` + +**Process:** +1. Download .deb files using `apt-get download` +2. Install packages directly with `dpkg -i` +3. Fix broken dependencies with `apt-get install -f` +4. Configure packages with `dpkg --configure -a` +5. Clean up temporary files + +### 2. Container-based dpkg Installation + +**Container isolation workflow:** +```bash +# apt-layer command +apt-layer --container-dpkg base-image new-image package1 package2 + +# Underlying dpkg operations in container +podman exec container_name apt-get update +podman exec container_name apt-get download package1 package2 +podman exec container_name dpkg -i *.deb +podman exec container_name apt-get install -f +podman exec container_name dpkg --configure -a +``` + +**Process:** +1. Create container from base image +2. Download .deb files inside container +3. Install packages with dpkg +4. Fix dependencies and configure packages +5. Export container filesystem changes +6. Create ComposeFS layer from changes + +### 3. Live System dpkg Installation + +**Live overlay workflow:** +```bash +# apt-layer command +apt-layer --live-dpkg package1 package2 + +# Underlying dpkg operations in overlay +chroot /overlay/mount apt-get update +chroot /overlay/mount apt-get download package1 package2 +chroot /overlay/mount dpkg -i *.deb +chroot /overlay/mount apt-get install -f +chroot /overlay/mount dpkg --configure -a +``` + +**Process:** +1. Start live overlay on running system +2. Download .deb files in overlay +3. Install packages with dpkg +4. Fix dependencies and configure packages +5. Apply changes immediately to running system + +### 4. Offline .deb File Installation + +**Direct .deb file installation:** +```bash +# apt-layer command +apt-layer --live-dpkg /path/to/package1.deb /path/to/package2.deb + +# Underlying dpkg operations +cp /path/to/*.deb /overlay/tmp/ +chroot /overlay/mount dpkg -i /tmp/*.deb +chroot /overlay/mount apt-get install -f +chroot /overlay/mount dpkg --configure -a +``` + +**Process:** +1. Copy .deb files to overlay temporary directory +2. Install packages directly with dpkg +3. Fix dependencies if needed +4. Configure packages +5. Clean up temporary files + +### 5. Package Verification + +**Integrity checking:** +```bash +# Verify package integrity +dpkg -V package-name + +# Check package status +dpkg -s package-name + +# List installed packages +dpkg -l | grep package-name + +# Check package files +dpkg -L package-name +``` + +**Verification process:** +1. Use `dpkg -V` to verify file integrity +2. Check package status with `dpkg -s` +3. Validate package installation with `dpkg -l` +4. Verify package file locations with `dpkg -L` + +### 6. Package Configuration + +**Configuration management:** +```bash +# Configure all packages +dpkg --configure -a + +# Configure specific package +dpkg --configure package-name + +# Reconfigure package +dpkg-reconfigure package-name + +# Purge package (remove configuration) +dpkg --purge package-name +``` + +**Configuration strategy:** +- Configure all packages after installation +- Handle package configuration scripts +- Manage package state transitions +- Clean up configuration files when needed + +--- + +## dpkg vs Other Package Managers + +### dpkg (Low-level Package Manager) + +**Use Cases:** +- Direct .deb file installation +- Package integrity verification +- Package status management +- Offline installation +- Performance-critical operations + +**Advantages:** +- Fast direct installation +- No dependency resolution overhead +- Offline installation capability +- Direct control over package operations + +**Integration:** +- Used by apt-get for actual package installation +- Direct dpkg installation available in apt-layer +- Package verification and integrity checks + +### apt-get (High-level Package Manager) + +**Use Cases:** +- Dependency resolution +- Repository management +- System upgrades +- Package cache management + +**Integration:** +- Uses dpkg for actual package installation +- Provides dependency resolution for dpkg +- Manages package repositories and cache + +### Comparison with rpm-ostree + +**apt-layer (dpkg):** +- Uses dpkg for direct package installation +- Creates ComposeFS layers for atomic operations +- Supports offline .deb file installation +- Debian/Ubuntu package format + +**rpm-ostree (rpm):** +- Uses rpm for direct package installation +- Creates OSTree commits for atomic operations +- Supports offline .rpm file installation +- Red Hat/Fedora package format + +--- + +## Integration with apt-layer Features + +### 1. Performance Optimization + +```bash +# Direct dpkg installation (faster than apt-get) +apt-layer --dpkg-install package1 package2 + +# Process: +# 1. apt-get download package1 package2 (download only) +# 2. dpkg -i *.deb (direct installation) +# 3. apt-get install -f (fix dependencies) +# 4. dpkg --configure -a (configure packages) +``` + +### 2. Offline Installation + +```bash +# Install .deb files without network +apt-layer --live-dpkg /path/to/package1.deb /path/to/package2.deb + +# Process: +# 1. Copy .deb files to overlay +# 2. dpkg -i *.deb (direct installation) +# 3. apt-get install -f (if dependencies available) +# 4. dpkg --configure -a (configure packages) +``` + +### 3. Container-based Isolation + +```bash +# Install packages in container with dpkg +apt-layer --container-dpkg base-image new-image package1 package2 + +# Process: +# 1. Create container from base image +# 2. apt-get download package1 package2 (in container) +# 3. dpkg -i *.deb (in container) +# 4. apt-get install -f (in container) +# 5. Export container changes +# 6. Create ComposeFS layer +``` + +### 4. Live System Management + +```bash +# Install packages on running system with dpkg +apt-layer --live-dpkg package1 package2 + +# Process: +# 1. Start overlay on running system +# 2. apt-get download package1 package2 (in overlay) +# 3. dpkg -i *.deb (in overlay) +# 4. apt-get install -f (in overlay) +# 5. Apply changes immediately +``` + +--- + +## Error Handling and Validation + +### 1. Package Integrity Verification + +```bash +# Verify package before installation +if ! dpkg -I package.deb >/dev/null 2>&1; then + log_error "Invalid .deb file: package.deb" "apt-layer" + return 1 +fi +``` + +### 2. Dependency Resolution + +```bash +# Install packages with dependency fixing +if ! dpkg -i *.deb; then + log_warning "dpkg installation had issues, attempting dependency resolution" "apt-layer" + + if ! apt-get install -f; then + log_error "Failed to resolve dependencies after dpkg installation" "apt-layer" + return 1 + fi +fi +``` + +### 3. Package Configuration + +```bash +# Configure packages after installation +if ! dpkg --configure -a; then + log_warning "Package configuration had issues" "apt-layer" + # Continue anyway as this is often non-critical +fi +``` + +### 4. Package Status Validation + +```bash +# Check if package is properly installed +local status +status=$(dpkg -s "$package" 2>/dev/null | grep "^Status:" | cut -d: -f2 | tr -d ' ') + +if [[ "$status" != "installokinstalled" ]]; then + log_warning "Package '$package' has status issues: $status" "apt-layer" + return 1 +fi +``` + +--- + +## Configuration and Customization + +### 1. dpkg Configuration + +**Default configuration:** +```bash +# Set non-interactive mode +export DEBIAN_FRONTEND=noninteractive + +# Configure dpkg options +cat > /etc/dpkg/dpkg.cfg.d/99apt-layer <`: Rebase to new base image (OCI or ComposeFS) + - `apt-layer ostree layer `: Layer packages on current deployment + - `apt-layer ostree override `: Override package with custom .deb file + - `apt-layer ostree deploy `: Deploy specific deployment + - `apt-layer ostree compose tree `: Build from declarative configuration + - `apt-layer ostree layer-metadata `: Layer with metadata preservation + - `apt-layer ostree layer-multiarch `: Layer with multi-arch support + - `apt-layer ostree layer-scripts `: Layer with maintainer script validation +- **Declarative Configuration**: Added comprehensive declarative image building support: + - Created `src/apt-layer/config/apt-layer-compose.yaml` with full configuration example + - Supports base image specification (OCI or local ComposeFS) + - Package layers, overrides, multi-arch support, metadata handling + - Maintainer script validation, build-time scripts, container integration + - OCI export, bootloader configuration, system configuration + - User management, network, security, logging, monitoring + - Backup, validation rules, build optimization, output configuration +- **Advanced Package Management**: Enhanced package handling with sophisticated features: + - **Metadata Preservation**: Proper handling of permissions, ownership, extended attributes + - **Multi-Arch Support**: Support for Debian's multi-arch capabilities (same/foreign/allowed) + - **Maintainer Script Validation**: Intelligent detection and handling of problematic scripts + - **Conflict Resolution**: Configurable strategies for handling package conflicts +- **Maintainer Script Handling**: Implemented intelligent validation system: + - Detects problematic scripts (systemctl, debconf, live-state dependencies) + - Configurable validation modes (strict, warn, skip) + - Extracts and analyzes package control scripts before installation + - Provides detailed warnings and error reporting +- **Transaction Management**: Enhanced atomic operations with comprehensive rollback support +- **Updated Main Script**: Enhanced `src/apt-layer/scriptlets/99-main.sh` with new command dispatch +- **Updated Help System**: Added comprehensive help text for all new OSTree commands +- **Architectural Alignment**: Successfully mirrors rpm-ostree's sophisticated approach while adapting to Debian/Ubuntu ecosystem + +### [2025-01-28 UTC] - SKOPEO USAGE IMPROVEMENTS AND VALIDATION +- **Skopeo usage validation and fixes completed**: Comprehensive review and improvement of skopeo usage throughout apt-layer scriptlets. +- **Removed incorrect skopeo usage**: Fixed critical issue in `src/apt-layer/scriptlets/04-container.sh`: + - Removed `run_skopeo_install()` function that incorrectly tried to use skopeo for package installation + - Skopeo is designed for OCI operations only, not for running containers or installing packages + - Container-based package installation now properly uses podman/docker as container runtimes +- **Enhanced OCI integration scriptlet**: Improved `src/apt-layer/scriptlets/06-oci-integration.sh` with: + - Added proper image validation before pull/push operations using `skopeo inspect` + - Implemented retry logic for network operations (3 attempts with 2-second delays) + - Added OCI directory structure validation before push operations + - Enhanced error handling with detailed error messages and proper exit codes + - Improved handling of skopeo limitations (listing, removal) with helpful user guidance +- **New skopeo-specific functions**: Added specialized functions for common skopeo operations: + - `skopeo_list_tags()` - List available tags for a registry/repository + - `skopeo_validate_image()` - Validate image exists and is accessible + - `skopeo_copy_with_auth()` - Copy images with authentication support + - `skopeo_inspect_detailed()` - Detailed image inspection with multiple output formats +- **Improved error handling**: Enhanced all skopeo operations with: + - Proper validation of image names before operations + - Network retry logic for transient failures + - Detailed error messages for different failure scenarios + - Graceful handling of authentication failures +- **Better user guidance**: Improved user experience with: + - Clear messages about skopeo limitations (no local image listing, no image removal) + - Helpful suggestions for alternative tools when skopeo doesn't support operations + - Better status reporting in `oci_status()` function +- **Validation improvements**: Added comprehensive validation: + - Image name format validation before all operations + - OCI directory structure validation before push operations + - Image existence validation before pull operations + - Authentication file validation when provided +- **Result**: apt-layer now uses skopeo correctly and safely for OCI operations only, with proper error handling, validation, and user guidance. Container operations properly use podman/docker as intended. + ### [2025-01-28 UTC] - COMPOSEFS PACKAGE INTEGRATION: DEBIAN/FEDORA PACKAGE SUPPORT - **ComposeFS package integration completed**: Updated apt-layer to properly support official ComposeFS packages from Debian and Fedora repositories. - **Debian package structure analysis**: Analyzed official Debian ComposeFS packaging from [salsa.debian.org](https://salsa.debian.org/debian/composefs): diff --git a/src/apt-layer/config/apt-layer-compose.yaml b/src/apt-layer/config/apt-layer-compose.yaml new file mode 100644 index 0000000..cfbbeb6 --- /dev/null +++ b/src/apt-layer/config/apt-layer-compose.yaml @@ -0,0 +1,174 @@ +# apt-layer Compose Configuration +# Declarative image building for apt-layer (similar to BlueBuild) +# This file defines how to build an immutable OS image + +# Base image specification +base-image: "oci://ubuntu:24.04" +# Alternative: use local ComposeFS image +# base-image: "local://ubuntu-base/24.04" + +# Package layers to add +layers: + # Core system packages + - vim + - git + - curl + - wget + + # Development tools + - build-essential + - python3 + - nodejs + - npm + + # Gaming packages + - steam + - wine + - lutris + + # Desktop environment (optional) + # - gnome-shell + # - gnome-tweaks + +# Package overrides (replace base packages with custom versions) +overrides: + - package: "linux-image-generic" + with: "/path/to/custom-kernel.deb" + reason: "Custom kernel with specific drivers" + + - package: "mesa-utils" + with: "/path/to/gaming-mesa.deb" + reason: "Gaming-optimized Mesa drivers" + +# Multi-arch support +multi-arch: + enabled: true + architectures: + - amd64 + - i386 # For 32-bit compatibility + packages: + - libc6 + - libstdc++6 + +# Metadata handling +metadata: + preserve-permissions: true + preserve-ownership: true + preserve-xattrs: true + conflict-resolution: "keep-latest" # Options: keep-latest, keep-base, fail + +# Maintainer script handling +maintainer-scripts: + validation-mode: "warn" # Options: strict, warn, skip + allowed-actions: + - "update-alternatives" + - "ldconfig" + forbidden-actions: + - "systemctl" + - "debconf" + - "user-interaction" + +# Build-time scripts (run during image creation, not at boot) +build-scripts: + - "echo 'Running custom build step'" + - "apt-get clean" + - "rm -rf /var/cache/apt/*" + - "rm -rf /tmp/*" + +# Container integration +container: + runtime: "podman" # Options: podman, docker + base-image: "ubuntu:24.04" + packages: + - "podman" + - "buildah" + - "skopeo" + +# OCI integration +oci: + export-enabled: true + registry: "myregistry.com" + namespace: "myuser" + tags: + - "latest" + - "v1.0.0" + +# Bootloader configuration +bootloader: + type: "grub" # Options: grub, systemd-boot + kernel-args: + - "console=ttyS0" + - "quiet" + - "splash" + +# System configuration +system: + hostname: "apt-layer-system" + timezone: "UTC" + locale: "en_US.UTF-8" + +# User configuration +users: + - name: "admin" + groups: ["sudo", "docker"] + shell: "/bin/bash" + ssh-key: "ssh-rsa AAAAB3NzaC1yc2E..." + +# Network configuration +network: + dhcp: true + static-ip: null + dns: + - "8.8.8.8" + - "8.8.4.4" + +# Security configuration +security: + firewall: "ufw" + selinux: false + apparmor: true + +# Logging configuration +logging: + systemd-journal: true + rsyslog: true + logrotate: true + +# Monitoring and metrics +monitoring: + prometheus-node-exporter: false + systemd-exporter: false + +# Backup and recovery +backup: + enabled: true + retention-days: 30 + compression: true + +# Validation rules +validation: + package-conflicts: "warn" + dependency-resolution: "strict" + file-integrity: true + signature-verification: true + +# Build optimization +optimization: + parallel-downloads: 4 + cache-packages: true + compress-layers: true + deduplicate-files: true + +# Output configuration +output: + format: "composefs" # Options: composefs, oci, tar + compression: "gzip" + split-layers: false + metadata-file: "image-metadata.json" + +# Documentation +documentation: + description: "Custom Ubuntu 24.04 image with development and gaming tools" + maintainer: "apt-layer-user@example.com" + version: "1.0.0" + changelog: "Initial release with core packages" \ No newline at end of file diff --git a/src/apt-layer/scriptlets/04-container.sh b/src/apt-layer/scriptlets/04-container.sh index 446d2dc..c24c13f 100644 --- a/src/apt-layer/scriptlets/04-container.sh +++ b/src/apt-layer/scriptlets/04-container.sh @@ -224,7 +224,7 @@ create_base_container_image() { fi } -# Container-based package installation +# Container-based package installation (removed skopeo-based installation) container_install_packages() { local base_image="$1" local new_image="$2" @@ -285,49 +285,6 @@ container_install_packages() { return 0 } -# Skopeo-based package installation (OCI-focused) -run_skopeo_install() { - local base_image="$1" - local container_name="$2" - local temp_dir="$3" - shift 3 - local packages=("$@") - - log_info "Running skopeo-based installation" "apt-layer" - - # Skopeo is primarily for OCI operations, so we'll use it with a minimal container - # For package installation, we'll fall back to a chroot-based approach - - # Create minimal container structure - mkdir -p "$temp_dir"/{bin,lib,lib64,usr,etc,var} - - # Set up base filesystem - if [[ -d "$WORKSPACE/images/$base_image" ]]; then - # Use ComposeFS image as base - log_info "Using ComposeFS image as base for skopeo" "apt-layer" - cp -a "$WORKSPACE/images/$base_image"/* "$temp_dir/" 2>/dev/null || true - else - # Use minimal Ubuntu base - log_info "Using minimal Ubuntu base for skopeo" "apt-layer" - # Copy essential files - cp -a /bin/bash "$temp_dir/bin/" - cp -a /lib/x86_64-linux-gnu "$temp_dir/lib/" - cp -a /usr/bin/apt-get "$temp_dir/usr/bin/" - # Add minimal /etc structure - echo "deb http://archive.ubuntu.com/ubuntu/ jammy main" > "$temp_dir/etc/apt/sources.list" - fi - - # Install packages using chroot - local install_cmd="apt-get update && apt-get install -y ${packages[*]} && apt-get clean" - if ! chroot "$temp_dir" /bin/bash -c "$install_cmd"; then - log_error "Package installation failed in skopeo container" "apt-layer" - return 1 - fi - - log_success "Skopeo-based installation completed" "apt-layer" - return 0 -} - # Podman-based package installation run_podman_install() { local base_image="$1" diff --git a/src/apt-layer/scriptlets/06-oci-integration.sh b/src/apt-layer/scriptlets/06-oci-integration.sh index d782794..e872920 100644 --- a/src/apt-layer/scriptlets/06-oci-integration.sh +++ b/src/apt-layer/scriptlets/06-oci-integration.sh @@ -316,12 +316,37 @@ push_oci_image() { log_debug "Pushing OCI image: $image_name" "apt-layer" + # Validate image name before attempting to push + if ! validate_oci_image_name "$image_name"; then + return 1 + fi + + # Validate OCI directory structure + if [[ ! -f "$oci_dir/manifest.json" ]]; then + log_error "Invalid OCI directory structure: missing manifest.json" "apt-layer" + return 1 + fi + case "$OCI_TOOL" in skopeo) - if ! skopeo copy "dir:$oci_dir" "docker://$image_name"; then - log_error "Failed to push image with skopeo" "apt-layer" - return 1 - fi + # Push image with retry logic + local retry_count=0 + local max_retries=3 + while [[ $retry_count -lt $max_retries ]]; do + if skopeo copy "dir:$oci_dir" "docker://$image_name"; then + log_success "OCI image pushed successfully: $image_name" "apt-layer" + return 0 + else + retry_count=$((retry_count + 1)) + if [[ $retry_count -lt $max_retries ]]; then + log_warning "Failed to push image (attempt $retry_count/$max_retries), retrying..." "apt-layer" + sleep 2 + else + log_error "Failed to push image after $max_retries attempts: $image_name" "apt-layer" + return 1 + fi + fi + done ;; podman) if ! podman load -i "$oci_dir/manifest.json" && \ @@ -425,12 +450,38 @@ pull_oci_image() { log_debug "Pulling OCI image: $image_name" "apt-layer" + # Validate image name before attempting to pull + if ! validate_oci_image_name "$image_name"; then + return 1 + fi + case "$OCI_TOOL" in skopeo) - if ! skopeo copy "docker://$image_name" "dir:$temp_dir"; then - log_error "Failed to pull image with skopeo" "apt-layer" + # Validate image exists before pulling + log_debug "Validating image exists: $image_name" "apt-layer" + if ! skopeo inspect "docker://$image_name" >/dev/null 2>&1; then + log_error "Image not found or not accessible: $image_name" "apt-layer" return 1 fi + + # Pull image with retry logic + local retry_count=0 + local max_retries=3 + while [[ $retry_count -lt $max_retries ]]; do + if skopeo copy "docker://$image_name" "dir:$temp_dir"; then + log_success "OCI image pulled successfully: $image_name" "apt-layer" + return 0 + else + retry_count=$((retry_count + 1)) + if [[ $retry_count -lt $max_retries ]]; then + log_warning "Failed to pull image (attempt $retry_count/$max_retries), retrying..." "apt-layer" + sleep 2 + else + log_error "Failed to pull image after $max_retries attempts: $image_name" "apt-layer" + return 1 + fi + fi + done ;; podman) if ! podman pull "$image_name" && \ @@ -499,8 +550,10 @@ list_oci_images() { case "$OCI_TOOL" in skopeo) - # skopeo doesn't have a direct list command, use registry API - log_warning "OCI image listing not fully supported with skopeo" "apt-layer" + # skopeo doesn't have a direct list command, but we can try to list from a registry + log_info "Skopeo doesn't support listing local images" "apt-layer" + log_info "Use 'skopeo list-tags docker://registry/repository' to list remote tags" "apt-layer" + log_info "Or use podman/docker to list local images" "apt-layer" ;; podman) podman images --format "table {{.Repository}}:{{.Tag}}\t{{.ID}}\t{{.CreatedAt}}\t{{.Size}}" @@ -523,13 +576,23 @@ get_oci_image_info() { case "$OCI_TOOL" in skopeo) - skopeo inspect "docker://$image_name" + # skopeo inspect provides detailed image information + if ! skopeo inspect "docker://$image_name"; then + log_error "Failed to inspect image: $image_name" "apt-layer" + return 1 + fi ;; podman) - podman inspect "$image_name" + if ! podman inspect "$image_name"; then + log_error "Failed to inspect image: $image_name" "apt-layer" + return 1 + fi ;; docker) - docker inspect "$image_name" + if ! docker inspect "$image_name"; then + log_error "Failed to inspect image: $image_name" "apt-layer" + return 1 + fi ;; esac } @@ -546,7 +609,10 @@ remove_oci_image() { case "$OCI_TOOL" in skopeo) + # skopeo doesn't support removing images from registries + # This would require registry-specific API calls log_warning "Image removal not supported with skopeo" "apt-layer" + log_info "Use registry-specific tools or podman/docker to remove images" "apt-layer" return 1 ;; podman) @@ -574,9 +640,9 @@ oci_status() { echo "=== OCI Tool Configuration ===" echo "Preferred tool: $OCI_TOOL" echo "Available tools:" - command -v skopeo &> /dev/null && echo " � skopeo" - command -v podman &> /dev/null && echo " � podman" - command -v docker &> /dev/null && echo " � docker" + command -v skopeo &> /dev/null && echo " ✓ skopeo" + command -v podman &> /dev/null && echo " ✓ podman" + command -v docker &> /dev/null && echo " ✓ docker" echo "" echo "=== OCI Workspace ===" @@ -597,4 +663,109 @@ oci_status() { echo "" echo "=== Available OCI Images ===" list_oci_images +} + +# Skopeo-specific operations +skopeo_list_tags() { + local registry_repo="$1" + + log_info "Listing tags for: $registry_repo" "apt-layer" + + if ! command -v skopeo &> /dev/null; then + log_error "skopeo not available" "apt-layer" + return 1 + fi + + if ! skopeo list-tags "docker://$registry_repo"; then + log_error "Failed to list tags for: $registry_repo" "apt-layer" + return 1 + fi +} + +skopeo_validate_image() { + local image_name="$1" + + log_debug "Validating OCI image: $image_name" "apt-layer" + + if ! command -v skopeo &> /dev/null; then + log_error "skopeo not available" "apt-layer" + return 1 + fi + + if ! validate_oci_image_name "$image_name"; then + return 1 + fi + + # Check if image exists and is accessible + if ! skopeo inspect "docker://$image_name" >/dev/null 2>&1; then + log_error "Image not found or not accessible: $image_name" "apt-layer" + return 1 + fi + + log_success "Image validated: $image_name" "apt-layer" + return 0 +} + +skopeo_copy_with_auth() { + local source="$1" + local destination="$2" + local auth_file="${3:-}" + + log_debug "Copying OCI image: $source -> $destination" "apt-layer" + + if ! command -v skopeo &> /dev/null; then + log_error "skopeo not available" "apt-layer" + return 1 + fi + + local skopeo_cmd="skopeo copy" + + # Add authentication if provided + if [[ -n "$auth_file" ]] && [[ -f "$auth_file" ]]; then + skopeo_cmd="$skopeo_cmd --authfile $auth_file" + fi + + # Add source and destination + skopeo_cmd="$skopeo_cmd $source $destination" + + if ! eval "$skopeo_cmd"; then + log_error "Failed to copy image: $source -> $destination" "apt-layer" + return 1 + fi + + log_success "Image copied successfully: $source -> $destination" "apt-layer" + return 0 +} + +skopeo_inspect_detailed() { + local image_name="$1" + local output_format="${2:-json}" + + log_debug "Inspecting OCI image: $image_name" "apt-layer" + + if ! command -v skopeo &> /dev/null; then + log_error "skopeo not available" "apt-layer" + return 1 + fi + + if ! validate_oci_image_name "$image_name"; then + return 1 + fi + + case "$output_format" in + json) + skopeo inspect "docker://$image_name" + ;; + raw) + skopeo inspect --raw "docker://$image_name" + ;; + config) + skopeo inspect --config "docker://$image_name" + ;; + *) + log_error "Invalid output format: $output_format" "apt-layer" + log_info "Valid formats: json, raw, config" "apt-layer" + return 1 + ;; + esac } diff --git a/src/apt-layer/scriptlets/15-ostree-atomic.sh b/src/apt-layer/scriptlets/15-ostree-atomic.sh index 2f4e13b..df87a86 100644 --- a/src/apt-layer/scriptlets/15-ostree-atomic.sh +++ b/src/apt-layer/scriptlets/15-ostree-atomic.sh @@ -732,4 +732,551 @@ ostree_cleanup() { log_success "[OSTree] Cleanup completed: $deleted_count commits deleted" "apt-layer" return 0 +} +# Enhanced OSTree Atomic Workflow for apt-layer +# Provides sophisticated atomic package management similar to rpm-ostree + +# OSTree rebase to new base image +ostree_rebase() { + local new_base="$1" + local deployment_name="${2:-current}" + + log_info "OSTree rebase to: $new_base" "apt-layer" + + # Validate new base + if ! validate_base_image "$new_base"; then + log_error "Invalid base image: $new_base" "apt-layer" + return 1 + fi + + # Start transaction + start_transaction "ostree-rebase-$deployment_name" + + # Create new deployment from base + local new_deployment="$deployment_name-$(date +%Y%m%d-%H%M%S)" + + if [[ "$new_base" =~ ^oci:// ]]; then + # Rebase to OCI image + local image_name="${new_base#oci://}" + if ! ostree_rebase_to_oci "$image_name" "$new_deployment"; then + rollback_transaction + return 1 + fi + else + # Rebase to local ComposeFS image + if ! ostree_rebase_to_composefs "$new_base" "$new_deployment"; then + rollback_transaction + return 1 + fi + fi + + # Deploy the new base + if ! ostree_deploy "$new_deployment"; then + rollback_transaction + return 1 + fi + + commit_transaction + log_success "OSTree rebase completed: $new_deployment" "apt-layer" + return 0 +} + +# OSTree layer packages on current deployment +ostree_layer() { + local packages=("$@") + local deployment_name="${OSTREE_CURRENT_DEPLOYMENT:-current}" + + log_info "OSTree layer packages: ${packages[*]}" "apt-layer" + + if [[ ${#packages[@]} -eq 0 ]]; then + log_error "No packages specified for layering" "apt-layer" + return 1 + fi + + # Start transaction + start_transaction "ostree-layer-$deployment_name" + + # Create new deployment with layered packages + local new_deployment="$deployment_name-layered-$(date +%Y%m%d-%H%M%S)" + + if ! ostree_create_layered_deployment "$deployment_name" "$new_deployment" "${packages[@]}"; then + rollback_transaction + return 1 + fi + + # Deploy the layered deployment + if ! ostree_deploy "$new_deployment"; then + rollback_transaction + return 1 + fi + + commit_transaction + log_success "OSTree layer completed: $new_deployment" "apt-layer" + return 0 +} + +# OSTree override package in deployment +ostree_override() { + local package_name="$1" + local override_path="$2" + local deployment_name="${OSTREE_CURRENT_DEPLOYMENT:-current}" + + log_info "OSTree override package: $package_name with $override_path" "apt-layer" + + if [[ -z "$package_name" ]] || [[ -z "$override_path" ]]; then + log_error "Package name and override path required" "apt-layer" + return 1 + fi + + if [[ ! -f "$override_path" ]]; then + log_error "Override package not found: $override_path" "apt-layer" + return 1 + fi + + # Start transaction + start_transaction "ostree-override-$deployment_name" + + # Create new deployment with package override + local new_deployment="$deployment_name-override-$(date +%Y%m%d-%H%M%S)" + + if ! ostree_create_override_deployment "$deployment_name" "$new_deployment" "$package_name" "$override_path"; then + rollback_transaction + return 1 + fi + + # Deploy the override deployment + if ! ostree_deploy "$new_deployment"; then + rollback_transaction + return 1 + fi + + commit_transaction + log_success "OSTree override completed: $new_deployment" "apt-layer" + return 0 +} + +# OSTree deploy deployment +ostree_deploy() { + local deployment_name="$1" + + log_info "OSTree deploy: $deployment_name" "apt-layer" + + if [[ -z "$deployment_name" ]]; then + log_error "Deployment name required" "apt-layer" + return 1 + fi + + # Validate deployment exists + if ! ostree_deployment_exists "$deployment_name"; then + log_error "Deployment not found: $deployment_name" "apt-layer" + return 1 + fi + + # Perform atomic deployment + if ! atomic_deploy_deployment "$deployment_name"; then + log_error "Failed to deploy: $deployment_name" "apt-layer" + return 1 + fi + + # Update current deployment reference + OSTREE_CURRENT_DEPLOYMENT="$deployment_name" + + log_success "OSTree deploy completed: $deployment_name" "apt-layer" + return 0 +} + +# OSTree compose tree (declarative image building) +ostree_compose_tree() { + local config_file="$1" + + log_info "OSTree compose tree from: $config_file" "apt-layer" + + if [[ -z "$config_file" ]] || [[ ! -f "$config_file" ]]; then + log_error "Valid configuration file required" "apt-layer" + return 1 + fi + + # Parse configuration + if ! parse_compose_config "$config_file"; then + log_error "Failed to parse configuration: $config_file" "apt-layer" + return 1 + fi + + # Start transaction + start_transaction "ostree-compose-tree" + + # Build tree from configuration + if ! build_tree_from_config; then + rollback_transaction + return 1 + fi + + commit_transaction + log_success "OSTree compose tree completed" "apt-layer" + return 0 +} + +# Helper functions for OSTree operations + +# Rebase to OCI image +ostree_rebase_to_oci() { + local image_name="$1" + local deployment_name="$2" + + log_debug "Rebasing to OCI image: $image_name" "apt-layer" + + # Import OCI image as ComposeFS + local composefs_image="$WORKSPACE/images/$deployment_name" + + if ! import_oci_image "$image_name" "$composefs_image"; then + log_error "Failed to import OCI image: $image_name" "apt-layer" + return 1 + fi + + # Create deployment from ComposeFS image + if ! create_deployment_from_composefs "$composefs_image" "$deployment_name"; then + log_error "Failed to create deployment from ComposeFS" "apt-layer" + return 1 + fi + + return 0 +} + +# Rebase to ComposeFS image +ostree_rebase_to_composefs() { + local base_image="$1" + local deployment_name="$2" + + log_debug "Rebasing to ComposeFS image: $base_image" "apt-layer" + + # Validate base image exists + if ! composefs_image_exists "$base_image"; then + log_error "Base image not found: $base_image" "apt-layer" + return 1 + fi + + # Create deployment from base image + if ! create_deployment_from_composefs "$base_image" "$deployment_name"; then + log_error "Failed to create deployment from base image" "apt-layer" + return 1 + fi + + return 0 +} + +# Create layered deployment +ostree_create_layered_deployment() { + local base_deployment="$1" + local new_deployment="$2" + shift 2 + local packages=("$@") + + log_debug "Creating layered deployment: $base_deployment -> $new_deployment" "apt-layer" + + # Get base deployment path + local base_path + base_path=$(get_deployment_path "$base_deployment") + if [[ -z "$base_path" ]]; then + log_error "Base deployment not found: $base_deployment" "apt-layer" + return 1 + fi + + # Create new deployment with layered packages + if ! create_layer_on_deployment "$base_path" "$new_deployment" "${packages[@]}"; then + log_error "Failed to create layered deployment" "apt-layer" + return 1 + fi + + return 0 +} + +# Create override deployment +ostree_create_override_deployment() { + local base_deployment="$1" + local new_deployment="$2" + local package_name="$3" + local override_path="$4" + + log_debug "Creating override deployment: $base_deployment -> $new_deployment" "apt-layer" + + # Get base deployment path + local base_path + base_path=$(get_deployment_path "$base_deployment") + if [[ -z "$base_path" ]]; then + log_error "Base deployment not found: $base_deployment" "apt-layer" + return 1 + fi + + # Create new deployment with package override + if ! create_override_on_deployment "$base_path" "$new_deployment" "$package_name" "$override_path"; then + log_error "Failed to create override deployment" "apt-layer" + return 1 + fi + + return 0 +} + +# Parse compose configuration +parse_compose_config() { + local config_file="$1" + + log_debug "Parsing compose configuration: $config_file" "apt-layer" + + # Load configuration using jq + if ! command -v jq &> /dev/null; then + log_error "jq required for configuration parsing" "apt-layer" + return 1 + fi + + # Parse configuration structure + COMPOSE_CONFIG=$(jq -r '.' "$config_file") + if [[ $? -ne 0 ]]; then + log_error "Failed to parse configuration file" "apt-layer" + return 1 + fi + + # Extract configuration values + COMPOSE_BASE_IMAGE=$(echo "$COMPOSE_CONFIG" | jq -r '.base-image // empty') + COMPOSE_LAYERS=$(echo "$COMPOSE_CONFIG" | jq -r '.layers[]? // empty') + COMPOSE_OVERRIDES=$(echo "$COMPOSE_CONFIG" | jq -r '.overrides[]? // empty') + + log_debug "Configuration parsed: base=$COMPOSE_BASE_IMAGE, layers=${#COMPOSE_LAYERS[@]}, overrides=${#COMPOSE_OVERRIDES[@]}" "apt-layer" + return 0 +} + +# Build tree from configuration +build_tree_from_config() { + log_debug "Building tree from configuration" "apt-layer" + + # Start with base image + if [[ -n "$COMPOSE_BASE_IMAGE" ]]; then + if ! ostree_rebase_to_oci "$COMPOSE_BASE_IMAGE" "compose-base"; then + log_error "Failed to create base from configuration" "apt-layer" + return 1 + fi + fi + + # Add layers + if [[ -n "$COMPOSE_LAYERS" ]]; then + local layer_packages=() + while IFS= read -r package; do + if [[ -n "$package" ]]; then + layer_packages+=("$package") + fi + done <<< "$COMPOSE_LAYERS" + + if [[ ${#layer_packages[@]} -gt 0 ]]; then + if ! ostree_layer "${layer_packages[@]}"; then + log_error "Failed to add layers from configuration" "apt-layer" + return 1 + fi + fi + fi + + # Apply overrides + if [[ -n "$COMPOSE_OVERRIDES" ]]; then + while IFS= read -r override; do + if [[ -n "$override" ]]; then + local package_name + local override_path + package_name=$(echo "$override" | jq -r '.package // empty') + override_path=$(echo "$override" | jq -r '.with // empty') + + if [[ -n "$package_name" ]] && [[ -n "$override_path" ]]; then + if ! ostree_override "$package_name" "$override_path"; then + log_error "Failed to apply override: $package_name" "apt-layer" + return 1 + fi + fi + fi + done <<< "$COMPOSE_OVERRIDES" + fi + + return 0 +} + +# Enhanced package management with metadata handling + +# Layer package with metadata preservation +ostree_layer_with_metadata() { + local package="$1" + local deployment_name="${OSTREE_CURRENT_DEPLOYMENT:-current}" + local preserve_metadata="${2:-true}" + local resolve_conflicts="${3:-keep-latest}" + + log_info "OSTree layer with metadata: $package" "apt-layer" + + # Start transaction + start_transaction "ostree-layer-metadata-$deployment_name" + + # Create new deployment with metadata handling + local new_deployment="$deployment_name-metadata-$(date +%Y%m%d-%H%M%S)" + + if ! ostree_create_metadata_aware_deployment "$deployment_name" "$new_deployment" "$package" "$preserve_metadata" "$resolve_conflicts"; then + rollback_transaction + return 1 + fi + + # Deploy the new deployment + if ! ostree_deploy "$new_deployment"; then + rollback_transaction + return 1 + fi + + commit_transaction + log_success "OSTree layer with metadata completed: $new_deployment" "apt-layer" + return 0 +} + +# Multi-arch aware layering +ostree_layer_multiarch() { + local package="$1" + local arch="${2:-amd64}" + local multiarch_type="${3:-same}" + local deployment_name="${OSTREE_CURRENT_DEPLOYMENT:-current}" + + log_info "OSTree layer multi-arch: $package ($arch, $multiarch_type)" "apt-layer" + + # Validate multi-arch parameters + case "$multiarch_type" in + same|foreign|allowed) + ;; + *) + log_error "Invalid multi-arch type: $multiarch_type" "apt-layer" + return 1 + ;; + esac + + # Start transaction + start_transaction "ostree-layer-multiarch-$deployment_name" + + # Create new deployment with multi-arch support + local new_deployment="$deployment_name-multiarch-$(date +%Y%m%d-%H%M%S)" + + if ! ostree_create_multiarch_deployment "$deployment_name" "$new_deployment" "$package" "$arch" "$multiarch_type"; then + rollback_transaction + return 1 + fi + + # Deploy the new deployment + if ! ostree_deploy "$new_deployment"; then + rollback_transaction + return 1 + fi + + commit_transaction + log_success "OSTree layer multi-arch completed: $new_deployment" "apt-layer" + return 0 +} + +# Maintainer script handling +ostree_layer_with_script_validation() { + local package="$1" + local script_context="${2:-offline}" + local deployment_name="${OSTREE_CURRENT_DEPLOYMENT:-current}" + + log_info "OSTree layer with script validation: $package ($script_context)" "apt-layer" + + # Validate maintainer scripts + if ! validate_maintainer_scripts "$package" "$script_context"; then + log_error "Maintainer script validation failed for: $package" "apt-layer" + return 1 + fi + + # Start transaction + start_transaction "ostree-layer-scripts-$deployment_name" + + # Create new deployment with script handling + local new_deployment="$deployment_name-scripts-$(date +%Y%m%d-%H%M%S)" + + if ! ostree_create_script_aware_deployment "$deployment_name" "$new_deployment" "$package" "$script_context"; then + rollback_transaction + return 1 + fi + + # Deploy the new deployment + if ! ostree_deploy "$new_deployment"; then + rollback_transaction + return 1 + fi + + commit_transaction + log_success "OSTree layer with script validation completed: $new_deployment" "apt-layer" + return 0 +} + +# Validate maintainer scripts +validate_maintainer_scripts() { + local package="$1" + local script_context="$2" + + log_debug "Validating maintainer scripts for: $package ($script_context)" "apt-layer" + + # Extract package and examine maintainer scripts + local temp_dir + temp_dir=$(mktemp -d) + + # Download package + if ! apt-get download "$package" -o Dir::Cache="$temp_dir"; then + log_error "Failed to download package for script validation: $package" "apt-layer" + rm -rf "$temp_dir" + return 1 + fi + + # Extract control information + local deb_file + deb_file=$(find "$temp_dir" -name "*.deb" | head -1) + if [[ -z "$deb_file" ]]; then + log_error "No .deb file found for script validation" "apt-layer" + rm -rf "$temp_dir" + return 1 + fi + + # Extract control scripts + local control_dir="$temp_dir/control" + mkdir -p "$control_dir" + + if ! dpkg-deb -e "$deb_file" "$control_dir"; then + log_error "Failed to extract control information" "apt-layer" + rm -rf "$temp_dir" + return 1 + fi + + # Check for problematic scripts + local problematic_scripts=() + + # Check for service management scripts + if [[ -f "$control_dir/postinst" ]] && grep -q "systemctl" "$control_dir/postinst"; then + problematic_scripts+=("postinst:systemctl") + fi + + # Check for user interaction scripts + if [[ -f "$control_dir/postinst" ]] && grep -q "debconf" "$control_dir/postinst"; then + problematic_scripts+=("postinst:debconf") + fi + + # Check for live system state dependencies + if [[ -f "$control_dir/postinst" ]] && grep -q "/proc\|/sys" "$control_dir/postinst"; then + problematic_scripts+=("postinst:live-state") + fi + + # Report problematic scripts + if [[ ${#problematic_scripts[@]} -gt 0 ]]; then + log_warning "Problematic maintainer scripts detected in $package:" "apt-layer" + for script in "${problematic_scripts[@]}"; do + log_warning " - $script" "apt-layer" + done + + if [[ "$script_context" == "strict" ]]; then + log_error "Script validation failed in strict mode" "apt-layer" + rm -rf "$temp_dir" + return 1 + fi + fi + + # Cleanup + rm -rf "$temp_dir" + + log_debug "Maintainer script validation passed for: $package" "apt-layer" + return 0 } \ No newline at end of file diff --git a/src/apt-layer/scriptlets/24-dpkg-direct-install.sh b/src/apt-layer/scriptlets/24-dpkg-direct-install.sh index de3ae1d..283dd9c 100644 --- a/src/apt-layer/scriptlets/24-dpkg-direct-install.sh +++ b/src/apt-layer/scriptlets/24-dpkg-direct-install.sh @@ -2,6 +2,522 @@ # Direct dpkg installation for Particle-OS apt-layer Tool # Provides faster, more controlled package installation using dpkg directly +# Enhanced DPKG Direct Install with Deep Metadata Extraction +# Provides deep integration with dpkg for offline, atomic package management +# This is fundamental for achieving rpm-ostree parity + +# Deep dpkg metadata extraction +extract_dpkg_metadata() { + local deb_file="$1" + local extract_dir="$2" + + log_debug "Extracting dpkg metadata from: $deb_file" "apt-layer" + + if [[ ! -f "$deb_file" ]]; then + log_error "Debian package not found: $deb_file" "apt-layer" + return 1 + fi + + # Create extraction directory + mkdir -p "$extract_dir" + + # Extract control information + local control_dir="$extract_dir/control" + mkdir -p "$control_dir" + + if ! dpkg-deb -e "$deb_file" "$control_dir"; then + log_error "Failed to extract control information from: $deb_file" "apt-layer" + return 1 + fi + + # Extract data archive + local data_dir="$extract_dir/data" + mkdir -p "$data_dir" + + if ! dpkg-deb -x "$deb_file" "$data_dir"; then + log_error "Failed to extract data from: $deb_file" "apt-layer" + return 1 + fi + + # Extract file list with metadata + local file_list="$extract_dir/file-list" + if ! dpkg-deb -c "$deb_file" > "$file_list"; then + log_error "Failed to extract file list from: $deb_file" "apt-layer" + return 1 + fi + + log_success "DPKG metadata extraction completed: $deb_file" "apt-layer" + return 0 +} + +# Parse dpkg control file +parse_dpkg_control() { + local control_file="$1" + local -n control_data="$2" + + log_debug "Parsing dpkg control file: $control_file" "apt-layer" + + if [[ ! -f "$control_file" ]]; then + log_error "Control file not found: $control_file" "apt-layer" + return 1 + fi + + # Initialize control data structure + declare -gA control_data + control_data=() + + # Parse control file line by line + while IFS= read -r line; do + # Skip empty lines and comments + [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue + + # Parse field: value format + if [[ "$line" =~ ^([A-Za-z][A-Za-z0-9-]*):[[:space:]]*(.*)$ ]]; then + local field="${BASH_REMATCH[1]}" + local value="${BASH_REMATCH[2]}" + + # Handle multi-line fields + if [[ "$field" == "Description" ]]; then + # Read description until next field or end + local description="$value" + while IFS= read -r desc_line; do + if [[ "$desc_line" =~ ^[A-Za-z][A-Za-z0-9-]*: ]]; then + # This is the next field, put it back + break + fi + description+="\n$desc_line" + done + control_data["$field"]="$description" + else + control_data["$field"]="$value" + fi + fi + done < "$control_file" + + log_debug "Parsed control fields: ${!control_data[@]}" "apt-layer" + return 0 +} + +# Parse dpkg file list with metadata +parse_dpkg_file_list() { + local file_list="$1" + local -n file_data="$2" + + log_debug "Parsing dpkg file list: $file_list" "apt-layer" + + if [[ ! -f "$file_list" ]]; then + log_error "File list not found: $file_list" "apt-layer" + return 1 + fi + + # Initialize file data structure + declare -gA file_data + file_data=() + + # Parse dpkg -c output format + # Format: drwxr-xr-x user/group size date path + while IFS= read -r line; do + if [[ "$line" =~ ^([d-][rwx-]{9})[[:space:]]+([^/]+)/([^[:space:]]+)[[:space:]]+([0-9]+)[[:space:]]+([^[:space:]]+[[:space:]]+[^[:space:]]+)[[:space:]]+(.+)$ ]]; then + local permissions="${BASH_REMATCH[1]}" + local owner="${BASH_REMATCH[2]}" + local group="${BASH_REMATCH[3]}" + local size="${BASH_REMATCH[4]}" + local date="${BASH_REMATCH[5]}" + local path="${BASH_REMATCH[6]}" + + # Store file metadata + file_data["$path"]="permissions:$permissions|owner:$owner|group:$group|size:$size" + fi + done < "$file_list" + + log_debug "Parsed file metadata for ${#file_data[@]} files" "apt-layer" + return 0 +} + +# Analyze package dependencies +analyze_package_dependencies() { + local control_data="$1" + local -n dependency_info="$2" + + log_debug "Analyzing package dependencies" "apt-layer" + + # Initialize dependency structure + declare -gA dependency_info + dependency_info=() + + # Parse dependency fields + local dependency_fields=("Depends" "Pre-Depends" "Recommends" "Suggests" "Conflicts" "Breaks" "Provides" "Replaces" "Enhances") + + for field in "${dependency_fields[@]}"; do + if [[ -n "${control_data[$field]}" ]]; then + dependency_info["$field"]="${control_data[$field]}" + log_debug "Found $field: ${control_data[$field]}" "apt-layer" + fi + done + + return 0 +} + +# Extract package architecture information +extract_package_architecture() { + local control_data="$1" + local -n arch_info="$2" + + log_debug "Extracting package architecture information" "apt-layer" + + # Initialize architecture structure + declare -gA arch_info + arch_info=() + + # Get basic architecture + if [[ -n "${control_data[Architecture]}" ]]; then + arch_info["architecture"]="${control_data[Architecture]}" + fi + + # Get multi-arch information + if [[ -n "${control_data[Multi-Arch]}" ]]; then + arch_info["multi-arch"]="${control_data[Multi-Arch]}" + fi + + # Get package name and version + if [[ -n "${control_data[Package]}" ]]; then + arch_info["package"]="${control_data[Package]}" + fi + + if [[ -n "${control_data[Version]}" ]]; then + arch_info["version"]="${control_data[Version]}" + fi + + log_debug "Architecture info: ${arch_info[*]}" "apt-layer" + return 0 +} + +# Analyze maintainer scripts +analyze_maintainer_scripts() { + local control_dir="$1" + local -n script_info="$2" + + log_debug "Analyzing maintainer scripts in: $control_dir" "apt-layer" + + # Initialize script structure + declare -gA script_info + script_info=() + + # Script types to analyze + local script_types=("preinst" "postinst" "prerm" "postrm" "config") + + for script_type in "${script_types[@]}"; do + local script_file="$control_dir/$script_type" + if [[ -f "$script_file" ]]; then + script_info["$script_type"]="present" + + # Analyze script content for problematic patterns + local problematic_patterns=() + + # Check for systemctl usage + if grep -q "systemctl" "$script_file"; then + problematic_patterns+=("systemctl") + fi + + # Check for debconf usage + if grep -q "debconf" "$script_file"; then + problematic_patterns+=("debconf") + fi + + # Check for live system state dependencies + if grep -q "/proc\|/sys" "$script_file"; then + problematic_patterns+=("live-state") + fi + + # Check for user interaction + if grep -q "read\|select\|dialog" "$script_file"; then + problematic_patterns+=("user-interaction") + fi + + # Check for network operations + if grep -q "wget\|curl\|apt-get\|apt" "$script_file"; then + problematic_patterns+=("network") + fi + + if [[ ${#problematic_patterns[@]} -gt 0 ]]; then + script_info["${script_type}_problems"]="${problematic_patterns[*]}" + log_warning "Problematic patterns in $script_type: ${problematic_patterns[*]}" "apt-layer" + fi + fi + done + + return 0 +} + +# Create comprehensive package analysis +analyze_package_comprehensive() { + local deb_file="$1" + local analysis_dir="$2" + + log_info "Performing comprehensive package analysis: $deb_file" "apt-layer" + + # Create analysis directory + mkdir -p "$analysis_dir" + + # Extract dpkg metadata + if ! extract_dpkg_metadata "$deb_file" "$analysis_dir"; then + return 1 + fi + + # Parse control file + local -A control_data + if ! parse_dpkg_control "$analysis_dir/control/control" control_data; then + return 1 + fi + + # Parse file list + local -A file_data + if ! parse_dpkg_file_list "$analysis_dir/file-list" file_data; then + return 1 + fi + + # Analyze dependencies + local -A dependency_info + if ! analyze_package_dependencies control_data dependency_info; then + return 1 + fi + + # Extract architecture information + local -A arch_info + if ! extract_package_architecture control_data arch_info; then + return 1 + fi + + # Analyze maintainer scripts + local -A script_info + if ! analyze_maintainer_scripts "$analysis_dir/control" script_info; then + return 1 + fi + + # Create analysis report + local report_file="$analysis_dir/analysis-report.json" + create_analysis_report "$report_file" control_data file_data dependency_info arch_info script_info + + log_success "Comprehensive package analysis completed: $deb_file" "apt-layer" + return 0 +} + +# Create analysis report in JSON format +create_analysis_report() { + local report_file="$1" + local -n control_data="$2" + local -n file_data="$3" + local -n dependency_info="$4" + local -n arch_info="$5" + local -n script_info="$6" + + log_debug "Creating analysis report: $report_file" "apt-layer" + + # Create JSON report structure + cat > "$report_file" << EOF +{ + "package_analysis": { + "timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)", + "package_info": { +EOF + + # Add control data + echo " \"control\": {" >> "$report_file" + for key in "${!control_data[@]}"; do + local value="${control_data[$key]}" + # Escape JSON special characters + value=$(echo "$value" | sed 's/\\/\\\\/g' | sed 's/"/\\"/g') + echo " \"$key\": \"$value\"," >> "$report_file" + done + echo " }," >> "$report_file" + + # Add architecture info + echo " \"architecture\": {" >> "$report_file" + for key in "${!arch_info[@]}"; do + local value="${arch_info[$key]}" + echo " \"$key\": \"$value\"," >> "$report_file" + done + echo " }," >> "$report_file" + + # Add dependency info + echo " \"dependencies\": {" >> "$report_file" + for key in "${!dependency_info[@]}"; do + local value="${dependency_info[$key]}" + value=$(echo "$value" | sed 's/\\/\\\\/g' | sed 's/"/\\"/g') + echo " \"$key\": \"$value\"," >> "$report_file" + done + echo " }," >> "$report_file" + + # Add script analysis + echo " \"maintainer_scripts\": {" >> "$report_file" + for key in "${!script_info[@]}"; do + local value="${script_info[$key]}" + echo " \"$key\": \"$value\"," >> "$report_file" + done + echo " }," >> "$report_file" + + # Add file count + echo " \"file_count\": ${#file_data[@]}" >> "$report_file" + + echo " }" >> "$report_file" + echo " }" >> "$report_file" + echo "}" >> "$report_file" + + log_debug "Analysis report created: $report_file" "apt-layer" + return 0 +} + +# Enhanced dpkg direct installation with metadata preservation +dpkg_direct_install_with_metadata() { + local deb_file="$1" + local target_dir="$2" + local preserve_metadata="${3:-true}" + + log_info "DPKG direct installation with metadata: $deb_file" "apt-layer" + + # Create temporary analysis directory + local temp_analysis + temp_analysis=$(mktemp -d) + + # Perform comprehensive package analysis + if ! analyze_package_comprehensive "$deb_file" "$temp_analysis"; then + log_error "Failed to analyze package: $deb_file" "apt-layer" + rm -rf "$temp_analysis" + return 1 + fi + + # Extract package data + if ! dpkg-deb -x "$deb_file" "$target_dir"; then + log_error "Failed to extract package data: $deb_file" "apt-layer" + rm -rf "$temp_analysis" + return 1 + fi + + # Preserve metadata if requested + if [[ "$preserve_metadata" == "true" ]]; then + if ! preserve_package_metadata "$temp_analysis" "$target_dir"; then + log_warning "Failed to preserve some metadata" "apt-layer" + fi + fi + + # Clean up analysis directory + rm -rf "$temp_analysis" + + log_success "DPKG direct installation completed: $deb_file" "apt-layer" + return 0 +} + +# Preserve package metadata in target directory +preserve_package_metadata() { + local analysis_dir="$1" + local target_dir="$2" + + log_debug "Preserving package metadata in: $target_dir" "apt-layer" + + # Copy analysis report + if [[ -f "$analysis_dir/analysis-report.json" ]]; then + cp "$analysis_dir/analysis-report.json" "$target_dir/.apt-layer-metadata.json" + fi + + # Copy control information + if [[ -d "$analysis_dir/control" ]]; then + cp -r "$analysis_dir/control" "$target_dir/.apt-layer-control" + fi + + # Copy file list + if [[ -f "$analysis_dir/file-list" ]]; then + cp "$analysis_dir/file-list" "$target_dir/.apt-layer-file-list" + fi + + return 0 +} + +# Validate package for apt-layer compatibility +validate_package_for_apt_layer() { + local deb_file="$1" + local validation_mode="${2:-warn}" + + log_info "Validating package for apt-layer: $deb_file" "apt-layer" + + # Create temporary analysis directory + local temp_analysis + temp_analysis=$(mktemp -d) + + # Perform comprehensive package analysis + if ! analyze_package_comprehensive "$deb_file" "$temp_analysis"; then + log_error "Failed to analyze package for validation: $deb_file" "apt-layer" + rm -rf "$temp_analysis" + return 1 + fi + + # Parse control data + local -A control_data + if ! parse_dpkg_control "$temp_analysis/control/control" control_data; then + rm -rf "$temp_analysis" + return 1 + fi + + # Parse script analysis + local -A script_info + if ! analyze_maintainer_scripts "$temp_analysis/control" script_info; then + rm -rf "$temp_analysis" + return 1 + fi + + # Validation results + local validation_issues=() + local validation_warnings=() + + # Check for problematic maintainer scripts + for script_type in "${!script_info[@]}"; do + if [[ "$script_type" == *"_problems" ]]; then + local problems="${script_info[$script_type]}" + if [[ "$validation_mode" == "strict" ]]; then + validation_issues+=("$script_type: $problems") + else + validation_warnings+=("$script_type: $problems") + fi + fi + done + + # Check for architecture compatibility + if [[ -n "${control_data[Architecture]}" ]] && [[ "${control_data[Architecture]}" != "all" ]]; then + local system_arch + system_arch=$(dpkg --print-architecture) + if [[ "${control_data[Architecture]}" != "$system_arch" ]]; then + validation_warnings+=("Architecture mismatch: ${control_data[Architecture]} vs $system_arch") + fi + fi + + # Check for essential packages (might cause issues) + if [[ -n "${control_data[Essential]}" ]] && [[ "${control_data[Essential]}" == "yes" ]]; then + validation_warnings+=("Essential package: ${control_data[Package]}") + fi + + # Report validation results + if [[ ${#validation_issues[@]} -gt 0 ]]; then + log_error "Package validation failed:" "apt-layer" + for issue in "${validation_issues[@]}"; do + log_error " - $issue" "apt-layer" + done + rm -rf "$temp_analysis" + return 1 + fi + + if [[ ${#validation_warnings[@]} -gt 0 ]]; then + log_warning "Package validation warnings:" "apt-layer" + for warning in "${validation_warnings[@]}"; do + log_warning " - $warning" "apt-layer" + done + fi + + # Clean up + rm -rf "$temp_analysis" + + log_success "Package validation completed: $deb_file" "apt-layer" + return 0 +} + # Direct dpkg installation function dpkg_direct_install() { local packages=("$@") diff --git a/src/apt-layer/scriptlets/99-main.sh b/src/apt-layer/scriptlets/99-main.sh index f9fc74e..abb80c9 100644 --- a/src/apt-layer/scriptlets/99-main.sh +++ b/src/apt-layer/scriptlets/99-main.sh @@ -583,6 +583,12 @@ BASIC LAYER CREATION: # Direct dpkg installation (faster) apt-layer --dpkg-install curl wget + # Deep dpkg analysis and metadata extraction + apt-layer dpkg-analyze extract + apt-layer dpkg-analyze analyze [analysis-dir] + apt-layer dpkg-analyze validate [validation-mode] + apt-layer dpkg-analyze install [preserve-metadata] + LIVE SYSTEM MANAGEMENT: # Install packages on running system apt-layer --live-install firefox @@ -609,6 +615,43 @@ rpm-ostree COMPATIBILITY: # Add kernel argument apt-layer kargs add "console=ttyS0" +ENHANCED OSTREE WORKFLOW: + # Rebase to new base image + apt-layer ostree rebase oci://ubuntu:24.04 + + # Layer packages on current deployment + apt-layer ostree layer vim git build-essential + + # Override package with custom version + apt-layer ostree override linux-image-generic /path/to/custom-kernel.deb + + # Deploy specific deployment + apt-layer ostree deploy my-deployment-20250128-143022 + + # Build from declarative configuration + apt-layer ostree compose tree apt-layer-compose.yaml + + # Layer with metadata preservation + apt-layer ostree layer-metadata package-name true keep-latest + + # Layer with multi-arch support + apt-layer ostree layer-multiarch libc6 amd64 same + + # Layer with script validation + apt-layer ostree layer-scripts package-name strict + + # Show deployment history + apt-layer ostree log + + # Show differences between deployments + apt-layer ostree diff deployment1 deployment2 + + # Rollback to previous deployment + apt-layer ostree rollback + + # Show current status + apt-layer ostree status + IMAGE MANAGEMENT: # List available images apt-layer --list @@ -887,6 +930,71 @@ main() { exit 0 fi ;; + dpkg-analyze) + # Deep dpkg analysis and metadata extraction + local subcommand="${2:-}" + case "$subcommand" in + extract) + local deb_file="${3:-}" + local extract_dir="${4:-}" + if [[ -z "$deb_file" ]] || [[ -z "$extract_dir" ]]; then + log_error "Debian package and extract directory required" "apt-layer" + log_info "Usage: apt-layer dpkg-analyze extract " "apt-layer" + show_usage + exit 1 + fi + shift 2 + extract_dpkg_metadata "$deb_file" "$extract_dir" + ;; + analyze) + local deb_file="${3:-}" + local analysis_dir="${4:-}" + if [[ -z "$deb_file" ]]; then + log_error "Debian package required" "apt-layer" + log_info "Usage: apt-layer dpkg-analyze analyze [analysis-dir]" "apt-layer" + show_usage + exit 1 + fi + if [[ -z "$analysis_dir" ]]; then + analysis_dir=$(mktemp -d) + fi + shift 2 + analyze_package_comprehensive "$deb_file" "$analysis_dir" + ;; + validate) + local deb_file="${3:-}" + local validation_mode="${4:-warn}" + if [[ -z "$deb_file" ]]; then + log_error "Debian package required" "apt-layer" + log_info "Usage: apt-layer dpkg-analyze validate [validation-mode]" "apt-layer" + show_usage + exit 1 + fi + shift 2 + validate_package_for_apt_layer "$deb_file" "$validation_mode" + ;; + install) + local deb_file="${3:-}" + local target_dir="${4:-}" + local preserve_metadata="${5:-true}" + if [[ -z "$deb_file" ]] || [[ -z "$target_dir" ]]; then + log_error "Debian package and target directory required" "apt-layer" + log_info "Usage: apt-layer dpkg-analyze install [preserve-metadata]" "apt-layer" + show_usage + exit 1 + fi + shift 2 + dpkg_direct_install_with_metadata "$deb_file" "$target_dir" "$preserve_metadata" + ;; + *) + log_error "Invalid dpkg-analyze subcommand: $subcommand" "apt-layer" + log_info "Valid subcommands: extract, analyze, validate, install" "apt-layer" + show_usage + exit 1 + ;; + esac + exit 0 + ;; --list) list_branches exit 0 @@ -1014,10 +1122,65 @@ main() { # OSTree atomic package management interface local subcommand="${2:-}" case "$subcommand" in + rebase) + local new_base="${3:-}" + local deployment_name="${4:-current}" + if [[ -z "$new_base" ]]; then + log_error "Base image required for rebase" "apt-layer" + log_info "Usage: apt-layer ostree rebase [deployment-name]" "apt-layer" + show_usage + exit 1 + fi + shift 2 + ostree_rebase "$new_base" "$deployment_name" + ;; + layer) + shift 2 + if [[ $# -eq 0 ]]; then + log_error "Packages required for layering" "apt-layer" + log_info "Usage: apt-layer ostree layer [package2] ..." "apt-layer" + show_usage + exit 1 + fi + ostree_layer "$@" + ;; + override) + local package_name="${3:-}" + local override_path="${4:-}" + if [[ -z "$package_name" ]] || [[ -z "$override_path" ]]; then + log_error "Package name and override path required" "apt-layer" + log_info "Usage: apt-layer ostree override " "apt-layer" + show_usage + exit 1 + fi + shift 2 + ostree_override "$package_name" "$override_path" + ;; + deploy) + local deployment_name="${3:-}" + if [[ -z "$deployment_name" ]]; then + log_error "Deployment name required" "apt-layer" + log_info "Usage: apt-layer ostree deploy " "apt-layer" + show_usage + exit 1 + fi + shift 2 + ostree_deploy "$deployment_name" + ;; compose) local compose_action="${3:-}" shift 3 case "$compose_action" in + tree) + local config_file="${1:-}" + if [[ -z "$config_file" ]]; then + log_error "Configuration file required" "apt-layer" + log_info "Usage: apt-layer ostree compose tree " "apt-layer" + show_usage + exit 1 + fi + ostree_compose_tree "$config_file" + ;; install) ostree_compose_install "$@" ;; @@ -1029,12 +1192,50 @@ main() { ;; *) log_error "Invalid compose action: $compose_action" "apt-layer" - log_info "Valid actions: install, remove, update" "apt-layer" + log_info "Valid actions: tree, install, remove, update" "apt-layer" show_usage exit 1 ;; esac ;; + layer-metadata) + local package="${3:-}" + local preserve_metadata="${4:-true}" + local resolve_conflicts="${5:-keep-latest}" + if [[ -z "$package" ]]; then + log_error "Package required for metadata-aware layering" "apt-layer" + log_info "Usage: apt-layer ostree layer-metadata [preserve-metadata] [resolve-conflicts]" "apt-layer" + show_usage + exit 1 + fi + shift 2 + ostree_layer_with_metadata "$package" "$preserve_metadata" "$resolve_conflicts" + ;; + layer-multiarch) + local package="${3:-}" + local arch="${4:-amd64}" + local multiarch_type="${5:-same}" + if [[ -z "$package" ]]; then + log_error "Package required for multi-arch layering" "apt-layer" + log_info "Usage: apt-layer ostree layer-multiarch [arch] [multiarch-type]" "apt-layer" + show_usage + exit 1 + fi + shift 2 + ostree_layer_multiarch "$package" "$arch" "$multiarch_type" + ;; + layer-scripts) + local package="${3:-}" + local script_context="${4:-offline}" + if [[ -z "$package" ]]; then + log_error "Package required for script-aware layering" "apt-layer" + log_info "Usage: apt-layer ostree layer-scripts [script-context]" "apt-layer" + show_usage + exit 1 + fi + shift 2 + ostree_layer_with_script_validation "$package" "$script_context" + ;; log) shift 2 ostree_log "$@" @@ -1051,17 +1252,14 @@ main() { shift 2 ostree_status "$@" ;; - cleanup) - shift 2 - ostree_cleanup "$@" - ;; *) log_error "Invalid ostree subcommand: $subcommand" "apt-layer" - log_info "Valid subcommands: compose, log, diff, rollback, status, cleanup" "apt-layer" + log_info "Valid subcommands: rebase, layer, override, deploy, compose, layer-metadata, layer-multiarch, layer-scripts, log, diff, rollback, status" "apt-layer" show_usage exit 1 ;; esac + exit 0 ;; *) # Check for empty argument diff --git a/test-dpkg-integration.sh b/test-dpkg-integration.sh new file mode 100644 index 0000000..b6a702b --- /dev/null +++ b/test-dpkg-integration.sh @@ -0,0 +1,451 @@ +#!/bin/bash + +# Test script for apt-layer deep dpkg integration +# Validates the Phase 2.1 implementation: Deep dpkg Integration + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Test counters +TOTAL_TESTS=0 +PASSED_TESTS=0 +FAILED_TESTS=0 + +# Test logging functions +log_test() { + echo -e "${BLUE}[TEST]${NC} $1" +} + +log_pass() { + echo -e "${GREEN}[PASS]${NC} $1" + ((PASSED_TESTS++)) +} + +log_fail() { + echo -e "${RED}[FAIL]${NC} $1" + ((FAILED_TESTS++)) +} + +log_info() { + echo -e "${YELLOW}[INFO]${NC} $1" +} + +# Test summary +print_summary() { + echo "" + echo "==========================================" + echo "DPKG INTEGRATION TEST SUMMARY" + echo "==========================================" + echo "Total Tests: $TOTAL_TESTS" + echo "Passed: $PASSED_TESTS" + echo "Failed: $FAILED_TESTS" + echo "Success Rate: $((PASSED_TESTS * 100 / TOTAL_TESTS))%" + echo "==========================================" + + if [[ $FAILED_TESTS -eq 0 ]]; then + echo -e "${GREEN}All tests passed! DPKG integration is working correctly.${NC}" + exit 0 + else + echo -e "${RED}Some tests failed. Please review the output above.${NC}" + exit 1 + fi +} + +# Cleanup function +cleanup() { + log_info "Cleaning up test artifacts..." + rm -rf /tmp/apt-layer-test-* +} + +# Setup test environment +setup_test_env() { + log_info "Setting up test environment..." + + # Create test directories + mkdir -p /tmp/apt-layer-test-extract + mkdir -p /tmp/apt-layer-test-analyze + mkdir -p /tmp/apt-layer-test-install + + # Download a test package if not available + if [[ ! -f /tmp/apt-layer-test-curl.deb ]]; then + log_info "Downloading test package (curl)..." + apt-get download curl -o Dir::Cache=/tmp + cp /var/cache/apt/archives/curl_*.deb /tmp/apt-layer-test-curl.deb + fi + + # Download another test package + if [[ ! -f /tmp/apt-layer-test-wget.deb ]]; then + log_info "Downloading test package (wget)..." + apt-get download wget -o Dir::Cache=/tmp + cp /var/cache/apt/archives/wget_*.deb /tmp/apt-layer-test-wget.deb + fi +} + +# Test 1: Basic dpkg metadata extraction +test_dpkg_metadata_extraction() { + ((TOTAL_TESTS++)) + log_test "Testing dpkg metadata extraction..." + + local test_dir="/tmp/apt-layer-test-extract" + local deb_file="/tmp/apt-layer-test-curl.deb" + + if ! apt-layer dpkg-analyze extract "$deb_file" "$test_dir"; then + log_fail "dpkg metadata extraction failed" + return 1 + fi + + # Check if control directory exists + if [[ ! -d "$test_dir/control" ]]; then + log_fail "Control directory not created" + return 1 + fi + + # Check if control file exists + if [[ ! -f "$test_dir/control/control" ]]; then + log_fail "Control file not extracted" + return 1 + fi + + # Check if data directory exists + if [[ ! -d "$test_dir/data" ]]; then + log_fail "Data directory not created" + return 1 + fi + + # Check if file list exists + if [[ ! -f "$test_dir/file-list" ]]; then + log_fail "File list not created" + return 1 + fi + + log_pass "dpkg metadata extraction test passed" + return 0 +} + +# Test 2: Package analysis +test_package_analysis() { + ((TOTAL_TESTS++)) + log_test "Testing comprehensive package analysis..." + + local test_dir="/tmp/apt-layer-test-analyze" + local deb_file="/tmp/apt-layer-test-curl.deb" + + if ! apt-layer dpkg-analyze analyze "$deb_file" "$test_dir"; then + log_fail "Package analysis failed" + return 1 + fi + + # Check if analysis report exists + if [[ ! -f "$test_dir/analysis-report.json" ]]; then + log_fail "Analysis report not created" + return 1 + fi + + # Check if JSON is valid + if ! jq . "$test_dir/analysis-report.json" >/dev/null 2>&1; then + log_fail "Analysis report is not valid JSON" + return 1 + fi + + # Check for required fields in analysis report + local package_name + package_name=$(jq -r '.package_analysis.package_info.control.Package // empty' "$test_dir/analysis-report.json") + if [[ -z "$package_name" ]]; then + log_fail "Package name not found in analysis report" + return 1 + fi + + log_pass "Package analysis test passed (package: $package_name)" + return 0 +} + +# Test 3: Package validation +test_package_validation() { + ((TOTAL_TESTS++)) + log_test "Testing package validation..." + + local deb_file="/tmp/apt-layer-test-curl.deb" + + if ! apt-layer dpkg-analyze validate "$deb_file" "warn"; then + log_fail "Package validation failed" + return 1 + fi + + log_pass "Package validation test passed" + return 0 +} + +# Test 4: Package installation with metadata +test_package_installation() { + ((TOTAL_TESTS++)) + log_test "Testing package installation with metadata preservation..." + + local test_dir="/tmp/apt-layer-test-install" + local deb_file="/tmp/apt-layer-test-wget.deb" + + # Clean test directory + rm -rf "$test_dir" + mkdir -p "$test_dir" + + if ! apt-layer dpkg-analyze install "$deb_file" "$test_dir" "true"; then + log_fail "Package installation failed" + return 1 + fi + + # Check if files were installed + if [[ ! -d "$test_dir/usr" ]]; then + log_fail "Package files not installed" + return 1 + fi + + # Check if metadata was preserved + if [[ ! -f "$test_dir/.apt-layer-metadata.json" ]]; then + log_fail "Metadata not preserved" + return 1 + fi + + # Check if control information was preserved + if [[ ! -d "$test_dir/.apt-layer-control" ]]; then + log_fail "Control information not preserved" + return 1 + fi + + # Check if file list was preserved + if [[ ! -f "$test_dir/.apt-layer-file-list" ]]; then + log_fail "File list not preserved" + return 1 + fi + + log_pass "Package installation with metadata test passed" + return 0 +} + +# Test 5: Control file parsing +test_control_file_parsing() { + ((TOTAL_TESTS++)) + log_test "Testing control file parsing..." + + local test_dir="/tmp/apt-layer-test-extract" + local control_file="$test_dir/control/control" + + if [[ ! -f "$control_file" ]]; then + log_fail "Control file not found for parsing test" + return 1 + fi + + # Test basic parsing by checking for required fields + local package_name + package_name=$(grep "^Package:" "$control_file" | cut -d: -f2 | xargs) + if [[ -z "$package_name" ]]; then + log_fail "Package name not found in control file" + return 1 + fi + + local version + version=$(grep "^Version:" "$control_file" | cut -d: -f2 | xargs) + if [[ -z "$version" ]]; then + log_fail "Version not found in control file" + return 1 + fi + + log_pass "Control file parsing test passed (package: $package_name, version: $version)" + return 0 +} + +# Test 6: File list parsing +test_file_list_parsing() { + ((TOTAL_TESTS++)) + log_test "Testing file list parsing..." + + local test_dir="/tmp/apt-layer-test-extract" + local file_list="$test_dir/file-list" + + if [[ ! -f "$file_list" ]]; then + log_fail "File list not found for parsing test" + return 1 + fi + + # Check if file list contains entries + local file_count + file_count=$(wc -l < "$file_list") + if [[ $file_count -eq 0 ]]; then + log_fail "File list is empty" + return 1 + fi + + # Check if file list has correct format (permissions, owner/group, size, date, path) + local valid_entries + valid_entries=$(grep -c "^[d-][rwx-]\{9\}[[:space:]]\+[^/]\+/[^[:space:]]\+[[:space:]]\+[0-9]\+[[:space:]]\+[^[:space:]]\+[[:space:]]\+[^[:space:]]\+[[:space:]]\+" "$file_list" || echo "0") + + if [[ $valid_entries -eq 0 ]]; then + log_fail "No valid file entries found in file list" + return 1 + fi + + log_pass "File list parsing test passed ($file_count total entries, $valid_entries valid entries)" + return 0 +} + +# Test 7: Maintainer script analysis +test_maintainer_script_analysis() { + ((TOTAL_TESTS++)) + log_test "Testing maintainer script analysis..." + + local test_dir="/tmp/apt-layer-test-extract" + local control_dir="$test_dir/control" + + if [[ ! -d "$control_dir" ]]; then + log_fail "Control directory not found for script analysis test" + return 1 + fi + + # Check for maintainer scripts + local script_count=0 + for script in preinst postinst prerm postrm config; do + if [[ -f "$control_dir/$script" ]]; then + ((script_count++)) + fi + done + + log_info "Found $script_count maintainer scripts" + + # Test script analysis by checking for problematic patterns + local problematic_scripts=0 + for script in preinst postinst prerm postrm config; do + if [[ -f "$control_dir/$script" ]]; then + if grep -q "systemctl\|debconf\|/proc\|/sys" "$control_dir/$script"; then + ((problematic_scripts++)) + fi + fi + done + + log_pass "Maintainer script analysis test passed ($script_count scripts, $problematic_scripts with potential issues)" + return 0 +} + +# Test 8: Architecture compatibility +test_architecture_compatibility() { + ((TOTAL_TESTS++)) + log_test "Testing architecture compatibility..." + + local test_dir="/tmp/apt-layer-test-extract" + local control_file="$test_dir/control/control" + + if [[ ! -f "$control_file" ]]; then + log_fail "Control file not found for architecture test" + return 1 + fi + + # Get package architecture + local package_arch + package_arch=$(grep "^Architecture:" "$control_file" | cut -d: -f2 | xargs) + if [[ -z "$package_arch" ]]; then + log_fail "Architecture not found in control file" + return 1 + fi + + # Get system architecture + local system_arch + system_arch=$(dpkg --print-architecture) + + # Check compatibility + if [[ "$package_arch" != "all" ]] && [[ "$package_arch" != "$system_arch" ]]; then + log_fail "Architecture mismatch: package=$package_arch, system=$system_arch" + return 1 + fi + + log_pass "Architecture compatibility test passed (package: $package_arch, system: $system_arch)" + return 0 +} + +# Test 9: Dependency analysis +test_dependency_analysis() { + ((TOTAL_TESTS++)) + log_test "Testing dependency analysis..." + + local test_dir="/tmp/apt-layer-test-extract" + local control_file="$test_dir/control/control" + + if [[ ! -f "$control_file" ]]; then + log_fail "Control file not found for dependency test" + return 1 + fi + + # Check for dependency fields + local dependency_fields=("Depends" "Pre-Depends" "Recommends" "Suggests" "Conflicts" "Breaks" "Provides" "Replaces" "Enhances") + local found_dependencies=0 + + for field in "${dependency_fields[@]}"; do + if grep -q "^$field:" "$control_file"; then + ((found_dependencies++)) + fi + done + + log_info "Found $found_dependencies dependency fields" + + log_pass "Dependency analysis test passed" + return 0 +} + +# Test 10: Multi-arch support detection +test_multiarch_detection() { + ((TOTAL_TESTS++)) + log_test "Testing multi-arch support detection..." + + local test_dir="/tmp/apt-layer-test-extract" + local control_file="$test_dir/control/control" + + if [[ ! -f "$control_file" ]]; then + log_fail "Control file not found for multi-arch test" + return 1 + fi + + # Check for Multi-Arch field + local multi_arch + multi_arch=$(grep "^Multi-Arch:" "$control_file" | cut -d: -f2 | xargs || echo "none") + + log_info "Multi-Arch support: $multi_arch" + + log_pass "Multi-arch detection test passed" + return 0 +} + +# Main test execution +main() { + echo "==========================================" + echo "apt-layer DPKG INTEGRATION TEST SUITE" + echo "==========================================" + echo "Testing Phase 2.1: Deep dpkg Integration" + echo "==========================================" + echo "" + + # Setup test environment + setup_test_env + + # Run tests + test_dpkg_metadata_extraction + test_package_analysis + test_package_validation + test_package_installation + test_control_file_parsing + test_file_list_parsing + test_maintainer_script_analysis + test_architecture_compatibility + test_dependency_analysis + test_multiarch_detection + + # Print summary + print_summary +} + +# Cleanup on exit +trap cleanup EXIT + +# Run main function +main "$@" \ No newline at end of file