particle-os-tools/src/apt-layer/compile.sh
Joe Particle 4f00e140a3
Some checks failed
Compile apt-layer (v2) / compile (push) Failing after 3h13m50s
feat: add changelog, docs, and update various scripts for modular CLI milestone
2025-07-17 09:22:29 +00:00

635 lines
24 KiB
Bash
Raw Blame History

#!/bin/bash
# apt-layer Compiler
# Merges multiple scriptlets into a single self-contained apt-layer.sh
# Based on apt-layer installer compile.sh and ComposeFS compile.sh
set -euo pipefail
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m'
# Function to print colored output
print_status() {
echo -e "${GREEN}[INFO]${NC} $1"
}
print_warning() {
echo -e "${YELLOW}[WARNING]${NC} $1"
}
print_error() {
echo -e "${RED}[ERROR]${NC} $1"
}
print_header() {
echo -e "${BLUE}================================${NC}"
echo -e "${BLUE}$1${NC}"
echo -e "${BLUE}================================${NC}"
}
# Function to show progress
update_progress() {
local status_message="$1"
local percent="$2"
local activity="${3:-Compiling}"
echo -e "${CYAN}[$activity]${NC} $status_message (${percent}%)"
}
# Check dependencies
check_dependencies() {
local missing_deps=()
# Check for jq (required for JSON processing)
if ! command -v jq &> /dev/null; then
missing_deps+=("jq")
fi
# Check for bash (required for syntax validation)
if ! command -v bash &> /dev/null; then
missing_deps+=("bash")
fi
# Check for dos2unix (for Windows line ending conversion)
if ! command -v dos2unix &> /dev/null; then
# Check if our custom dos2unix.sh exists
if [[ ! -f "$(dirname "$SCRIPT_DIR")/../dos2unix.sh" ]]; then
missing_deps+=("dos2unix")
fi
fi
if [[ ${#missing_deps[@]} -gt 0 ]]; then
print_error "Missing required dependencies: ${missing_deps[*]}"
print_error "Please install missing packages and try again"
exit 1
fi
print_status "All dependencies found"
}
# Validate JSON files
validate_json_files() {
local config_dir="$1"
if [[ -d "$config_dir" ]]; then
print_status "Validating JSON files in $config_dir"
local json_files=($(find "$config_dir" -name "*.json" -type f))
for json_file in "${json_files[@]}"; do
# Convert line endings before validation
convert_line_endings "$json_file"
if ! jq empty "$json_file" 2>/dev/null; then
print_error "Invalid JSON in file: $json_file"
exit 1
fi
print_status "<EFBFBD> Validated: $json_file"
done
fi
}
# Convert Windows line endings to Unix line endings
convert_line_endings() {
local file="$1"
local dos2unix_cmd=""
# Try to use system dos2unix first
if command -v dos2unix &> /dev/null; then
dos2unix_cmd="dos2unix"
elif [[ -f "$(dirname "$SCRIPT_DIR")/../dos2unix.sh" ]]; then
dos2unix_cmd="$(dirname "$SCRIPT_DIR")/../dos2unix.sh"
# Make sure our dos2unix.sh is executable
chmod +x "$dos2unix_cmd" 2>/dev/null || true
else
print_warning "dos2unix not available, skipping line ending conversion for: $file"
return 0
fi
# Check if file has Windows line endings
if grep -q $'\r' "$file" 2>/dev/null; then
print_status "Converting Windows line endings to Unix: $file"
if "$dos2unix_cmd" -q "$file"; then
print_status "<EFBFBD> Converted: $file"
else
print_warning "Failed to convert line endings for: $file"
fi
fi
}
# --- Scriptlet/Function Dependency Validation ---
validate_scriptlet_dependencies() {
print_header "Validating Scriptlet and Function Dependencies"
local missing_scriptlets=()
local all_scriptlets=(
"00-header.sh" "01-dependencies.sh" "02-transactions.sh" "03-traditional.sh" "04-container.sh" \
"05-live-overlay.sh" "06-oci-integration.sh" "07-bootloader.sh" "08-system-init.sh" \
"09-atomic-deployment.sh" "10-rpm-ostree-compat.sh" "15-ostree-atomic.sh" \
"20-daemon-integration.sh" "24-dpkg-direct-install.sh" "99-main.sh"
)
# 1. Check all scriptlets exist
for s in "${all_scriptlets[@]}"; do
if [[ ! -f "$SCRIPTLETS_DIR/$s" ]]; then
print_error "Missing scriptlet: $s"
missing_scriptlets+=("$s")
fi
done
if [[ ${#missing_scriptlets[@]} -gt 0 ]]; then
print_error "Compilation aborted due to missing scriptlets."
exit 1
fi
# 2. Parse for function definitions and calls
local defined_funcs=()
local called_funcs=()
for s in "${all_scriptlets[@]}"; do
local file="$SCRIPTLETS_DIR/$s"
# Find function definitions
while read -r line; do
if [[ "$line" =~ ^[[:space:]]*([a-zA-Z0-9_]+)[[:space:]]*\(\)[[:space:]]*\{ ]]; then
defined_funcs+=("${BASH_REMATCH[1]}")
fi
done < "$file"
# Find function calls (simple heuristic: foo ... or foo() ...)
while read -r line; do
if [[ "$line" =~ ^[[:space:]]*([a-zA-Z0-9_]+)[[:space:]]*\(\)?[[:space:]] ]]; then
called_funcs+=("${BASH_REMATCH[1]}")
fi
done < "$file"
done
# Remove duplicates
defined_funcs=( $(printf "%s\n" "${defined_funcs[@]}" | sort -u) )
called_funcs=( $(printf "%s\n" "${called_funcs[@]}" | sort -u) )
# 3. Warn if a function is called but not defined
local missing_funcs=()
for f in "${called_funcs[@]}"; do
if ! [[ " ${defined_funcs[*]} " =~ " $f " ]]; then
# Ignore bash builtins and common commands
if ! command -v "$f" &>/dev/null; then
print_warning "Function called but not defined: $f"
missing_funcs+=("$f")
fi
fi
done
# 4. Print summary
print_status "Function definitions found: ${#defined_funcs[@]}"
print_status "Function calls found: ${#called_funcs[@]}"
if [[ ${#missing_funcs[@]} -gt 0 ]]; then
print_warning "Missing function definitions: ${missing_funcs[*]}"
else
print_status "All called functions are defined or available as commands."
fi
}
# Get script directory and project root
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SCRIPTLETS_DIR="$SCRIPT_DIR/scriptlets"
TEMP_DIR="$SCRIPT_DIR/temp"
# Parse command line arguments
OUTPUT_FILE="$(dirname "$SCRIPT_DIR")/../apt-layer.sh" # Default output path
while [[ $# -gt 0 ]]; do
case $1 in
-o|--output)
OUTPUT_FILE="$2"
shift 2
;;
-h|--help)
echo "Usage: $0 [-o|--output OUTPUT_PATH]"
echo " -o, --output Specify output file path (default: ../apt-layer.sh)"
echo " -h, --help Show this help message"
exit 0
;;
*)
print_error "Unknown option: $1"
echo "Use -h or --help for usage information"
exit 1
;;
esac
done
# Ensure output directory exists
OUTPUT_DIR="$(dirname "$OUTPUT_FILE")"
if [[ ! -d "$OUTPUT_DIR" ]]; then
print_status "Creating output directory: $OUTPUT_DIR"
mkdir -p "$OUTPUT_DIR"
fi
print_header "apt-layer Compiler"
# Check dependencies first
check_dependencies
# Check if scriptlets directory exists
if [[ ! -d "$SCRIPTLETS_DIR" ]]; then
print_error "Scriptlets directory not found: $SCRIPTLETS_DIR"
exit 1
fi
# Validate JSON files if config directory exists
if [[ -d "$SCRIPT_DIR/config" ]]; then
validate_json_files "$SCRIPT_DIR/config"
fi
# Call validation before merging scriptlets
validate_scriptlet_dependencies
# Create temporary directory
rm -rf "$TEMP_DIR"
mkdir -p "$TEMP_DIR"
# Variable to sync between sections
update_progress "Pre-req: Creating temporary directory" 0
# Create the script in memory
script_content=()
# Add header
update_progress "Adding: Header" 5
header="#!/bin/bash
################################################################################################################
# #
# WARNING: This file is automatically generated #
# DO NOT modify this file directly as it will be overwritten #
# #
# apt-layer Tool #
# Generated on: $(date '+%Y-%m-%d %H:%M:%S') #
# #
################################################################################################################
set -euo pipefail
# apt-layer Tool - Self-contained version
# This script contains all components merged into a single file
# Enhanced version with container support, multiple package managers, and LIVE SYSTEM LAYERING
# Inspired by Vanilla OS Apx approach, ParticleOS apt-layer, and rpm-ostree live layering
"
script_content+=("$header")
# Add version info
update_progress "Adding: Version" 10
version_info="# Version: $(date '+%y.%m.%d')
# apt-layer Tool
# Enhanced with Container Support and LIVE SYSTEM LAYERING
"
script_content+=("$version_info")
# Add fallback logging functions (always defined first)
update_progress "Adding: Fallback Logging" 11
fallback_logging="# Fallback logging functions (always defined first)
# Color definitions
RED='\\033[0;31m'
GREEN='\\033[0;32m'
YELLOW='\\033[1;33m'
BLUE='\\033[0;34m'
CYAN='\\033[0;36m'
PURPLE='\\033[0;35m'
NC='\\033[0m'
log_info() {
local message=\"\$1\"
local script_name=\"\${2:-apt-layer}\"
echo -e \"\${BLUE}[INFO]\${NC} [\$script_name] \$message\"
}
log_debug() {
local message=\"\$1\"
local script_name=\"\${2:-apt-layer}\"
echo -e \"\${YELLOW}[DEBUG]\${NC} [\$script_name] \$message\"
}
log_error() {
local message=\"\$1\"
local script_name=\"\${2:-apt-layer}\"
echo -e \"\${RED}[ERROR]\${NC} [\$script_name] \$message\" >&2
}
log_warning() {
local message=\"\$1\"
local script_name=\"\${2:-apt-layer}\"
echo -e \"\${YELLOW}[WARNING]\${NC} [\$script_name] \$message\" >&2
}
log_success() {
local message=\"\$1\"
local script_name=\"\${2:-apt-layer}\"
echo -e \"\${GREEN}[SUCCESS]\${NC} [\$script_name] \$message\"
}
log_layer() {
local message=\"\$1\"
local script_name=\"\${2:-apt-layer}\"
echo -e \"\${PURPLE}[LAYER]\${NC} [\$script_name] \$message\"
}
log_transaction() {
local message=\"\$1\"
local script_name=\"\${2:-apt-layer}\"
echo -e \"\${CYAN}[TRANSACTION]\${NC} [\$script_name] \$message\"
}
"
script_content+=("$fallback_logging")
# Add apt-layer configuration sourcing
update_progress "Adding: Configuration Sourcing" 12
config_sourcing="# Source apt-layer configuration (if available, skip for help commands)
# Skip configuration loading for help commands to avoid permission issues
if [[ \"\${1:-}\" != \"--help\" && \"\${1:-}\" != \"-h\" && \"\${1:-}\" != \"--help-full\" && \"\${1:-}\" != \"--examples\" ]]; then
if [[ -f \"/usr/local/etc/apt-layer-config.sh\" ]]; then
source \"/usr/local/etc/apt-layer-config.sh\"
log_info \"Loaded apt-layer configuration\" \"apt-layer\"
else
log_warning \"apt-layer configuration not found, using defaults\" \"apt-layer\"
fi
else
log_info \"Skipping configuration loading for help command\" \"apt-layer\"
fi
"
script_content+=("$config_sourcing")
# Read and embed dependencies.json as a shell variable at the top
update_progress "Embedding: dependencies.json" 8
DEPENDENCIES_JSON_FILE="$SCRIPT_DIR/config/dependencies.json"
if [[ -f "$DEPENDENCIES_JSON_FILE" ]]; then
DEPENDENCIES_JSON_CONTENT=$(cat "$DEPENDENCIES_JSON_FILE")
script_content+=("# Embedded dependencies.json")
script_content+=("APT_LAYER_DEPENDENCIES_JSON=\$(cat << 'EOF'")
script_content+=("$DEPENDENCIES_JSON_CONTENT")
script_content+=("EOF")
script_content+=(")")
script_content+=("")
else
print_warning "dependencies.json not found, using fallback in scriptlet."
fi
# Function to add scriptlet content with error handling
add_scriptlet() {
local scriptlet_name="$1"
local scriptlet_file="$SCRIPTLETS_DIR/$scriptlet_name"
local description="$2"
local is_critical="${3:-false}"
if [[ -f "$scriptlet_file" ]]; then
print_status "Including $scriptlet_name"
# Convert line endings before processing
convert_line_endings "$scriptlet_file"
script_content+=("# ============================================================================")
script_content+=("# $description")
script_content+=("# ============================================================================")
# Read and add scriptlet content, excluding the shebang if present
local content
if head -1 "$scriptlet_file" | grep -q "^#!/"; then
content=$(tail -n +2 "$scriptlet_file")
else
content=$(cat "$scriptlet_file")
fi
script_content+=("$content")
script_content+=("")
script_content+=("# --- END OF SCRIPTLET: $scriptlet_name ---")
script_content+=("")
else
if [[ "$is_critical" == "true" ]]; then
print_error "CRITICAL: $scriptlet_name not found - compilation cannot continue"
exit 1
else
print_warning "$scriptlet_name not found, skipping"
fi
fi
}
# Add scriptlets in logical dependency order
update_progress "Adding: Header and Shared Functions" 13
add_scriptlet "00-header.sh" "Header and Shared Functions" "true"
update_progress "Adding: Dependencies" 16
add_scriptlet "01-dependencies.sh" "Dependency Checking and Validation" "true"
update_progress "Adding: Transaction Management" 21
add_scriptlet "02-transactions.sh" "Transaction Management" "true"
update_progress "Adding: Traditional Layer Creation" 26
add_scriptlet "03-traditional.sh" "Traditional Layer Creation"
update_progress "Adding: Container-based Layer Creation" 31
add_scriptlet "04-container.sh" "Container-based Layer Creation (Apx-style)"
update_progress "Adding: Live Overlay System" 36
add_scriptlet "05-live-overlay.sh" "Live Overlay System (rpm-ostree style)"
update_progress "Adding: OCI Integration" 41
add_scriptlet "06-oci-integration.sh" "OCI Export/Import Integration"
update_progress "Adding: Bootloader Integration" 46
add_scriptlet "07-bootloader.sh" "Bootloader Integration (UEFI/GRUB/systemd-boot)"
update_progress "Adding: System Initialization" 48
add_scriptlet "08-system-init.sh" "System Initialization and Path Management"
update_progress "Adding: Atomic Deployment System" 52
add_scriptlet "09-atomic-deployment.sh" "Atomic Deployment System"
update_progress "Adding: rpm-ostree Compatibility" 57
add_scriptlet "10-rpm-ostree-compat.sh" "rpm-ostree Compatibility Layer"
update_progress "Adding: OSTree Atomic Package Management" 62
add_scriptlet "15-ostree-atomic.sh" "OSTree Atomic Package Management"
update_progress "Adding: Daemon Integration" 65
add_scriptlet "20-daemon-integration.sh" "Daemon Integration (apt-ostree.py)"
update_progress "Adding: Direct dpkg Installation" 67
add_scriptlet "24-dpkg-direct-install.sh" "Direct dpkg Installation (Performance Optimization)"
update_progress "Adding: Main Dispatch" 72
add_scriptlet "99-main.sh" "Main Dispatch and Help" "true"
# Add embedded configuration files if they exist
update_progress "Adding: Embedded Configuration" 99
if [[ -d "$SCRIPT_DIR/config" ]]; then
script_content+=("# ============================================================================")
script_content+=("# Embedded Configuration Files")
script_content+=("# ============================================================================")
script_content+=("")
script_content+=("# Enterprise-grade JSON configuration system")
script_content+=("# All configuration files are embedded for self-contained operation")
script_content+=("# Configuration can be overridden via command-line arguments")
script_content+=("")
# Find and embed JSON files
json_files=($(find "$SCRIPT_DIR/config" -name "*.json" -type f | sort))
# Add configuration summary
script_content+=("# Configuration files to be embedded:")
for json_file in "${json_files[@]}"; do
filename=$(basename "$json_file" .json)
script_content+=("# - $filename.json")
done
script_content+=("")
for json_file in "${json_files[@]}"; do
filename=$(basename "$json_file" .json)
# Convert filename to valid variable name (replace hyphens with underscores)
variable_name=$(echo "${filename^^}" | tr '-' '_')"_CONFIG"
print_status "Processing configuration: $filename"
# Check file size first
file_size=$(stat -c%s "$json_file" 2>/dev/null || echo "0")
# For very large files (>5MB), suggest external loading
if [[ $file_size -gt 5242880 ]]; then # 5MB
print_warning "Very large configuration file detected ($(numfmt --to=iec $file_size)): $json_file"
print_warning "Consider using external file loading for better performance"
print_warning "This file will be embedded but may impact script startup time"
# Add external loading option as comment
script_content+=("# Large configuration file: $filename")
script_content+=("# Consider using external loading for better performance")
script_content+=("# Example: load_config_from_file \"$filename\"")
elif [[ $file_size -gt 1048576 ]]; then # 1MB
print_warning "Large configuration file detected ($(numfmt --to=iec $file_size)): $json_file"
fi
# Convert line endings before processing
convert_line_endings "$json_file"
# Validate JSON before processing
if ! jq '.' "$json_file" >> /dev/null; then
print_error "Invalid JSON in configuration file: $json_file"
exit 1
fi
# Embed with safety comment
script_content+=("# Embedded configuration: $filename")
script_content+=("# File size: $(numfmt --to=iec $file_size)")
script_content+=("$variable_name=\$(cat << 'EOF'")
# Use jq to ensure safe JSON output (prevents shell injection)
script_content+=("$(jq -r '.' "$json_file")")
script_content+=("EOF")
script_content+=(")")
script_content+=("")
done
# Add external loading function for future use
script_content+=("# ============================================================================")
script_content+=("# External Configuration Loading (Future Enhancement)")
script_content+=("# ============================================================================")
script_content+=("")
script_content+=("# Function to load configuration from external files")
script_content+=("# Usage: load_config_from_file \"config-name\"")
script_content+=("load_config_from_file() {")
script_content+=(" local config_name=\"\$1\"")
script_content+=(" local config_file=\"/etc/apt-layer/config/\${config_name}.json\"")
script_content+=(" if [[ -f \"\$config_file\" ]]; then")
script_content+=(" jq -r '.' \"\$config_file\"")
script_content+=(" else")
script_content+=(" log_error \"Configuration file not found: \$config_file\" \"apt-layer\"")
script_content+=(" exit 1")
script_content+=(" fi")
script_content+=("}")
script_content+=("")
fi
# Add main function call
script_content+=("# ============================================================================")
script_content+=("# Main Execution")
script_content+=("# ============================================================================")
script_content+=("")
script_content+=("# Run main function if script is executed directly")
script_content+=("if [[ \"\${BASH_SOURCE[0]}\" == \"\${0}\" ]]; then")
script_content+=(" main \"\$@\"")
script_content+=("fi")
# Write the compiled script
update_progress "Writing: Compiled script" 99
printf '%s\n' "${script_content[@]}" > "$OUTPUT_FILE"
# Make it executable
chmod +x "$OUTPUT_FILE"
# Validate the script
update_progress "Validating: Script syntax" 100
if bash -n "$OUTPUT_FILE"; then
print_status "Syntax validation passed"
else
print_error "Syntax validation failed"
print_error "Removing invalid script: $OUTPUT_FILE"
rm -f "$OUTPUT_FILE"
exit 1
fi
# Clean up
rm -rf "$TEMP_DIR"
print_header "Compilation Complete!"
print_status "Output file: $OUTPUT_FILE"
print_status "File size: $(du -h "$OUTPUT_FILE" | cut -f1)"
print_status "Lines of code: $(wc -l < "$OUTPUT_FILE")"
print_status ""
print_status "The compiled apt-layer.sh is now self-contained and includes:"
print_status "- apt-layer configuration integration"
print_status "- Transactional operations with automatic rollback"
print_status "- Traditional chroot-based layer creation"
print_status "- Container-based layer creation (Apx-style)"
print_status "- OCI export/import integration"
print_status "- Live overlay system (rpm-ostree style)"
print_status "- Bootloader integration (UEFI/GRUB/systemd-boot)"
print_status "- Atomic deployment system with rollback"
print_status "- rpm-ostree compatibility layer (1:1 command mapping)"
print_status "- ComposeFS backend integration"
print_status "- Dependency validation and error handling"
print_status "- Comprehensive JSON configuration system"
print_status "- Direct dpkg installation (Performance optimization)"
print_status "- All dependencies merged into a single file"
print_status ""
print_status "apt-layer compilation complete with all core rpm-ostree-like features!"
print_status ""
print_status "Usage:"
print_status " sudo ./apt-layer.sh ubuntu-base/24.04 gaming/24.04 steam wine"
print_status " sudo ./apt-layer.sh --container ubuntu-base/24.04 dev/24.04 vscode git"
print_status " sudo ./apt-layer.sh --live-overlay start"
print_status " sudo ./apt-layer.sh --live-install steam wine"
print_status " sudo ./apt-layer.sh --live-overlay commit \"Add gaming packages\""
print_status " sudo ./apt-layer.sh kargs add \"console=ttyS0\""
print_status " sudo ./apt-layer.sh kargs list"
print_status " sudo ./apt-layer.sh --advanced-install steam wine"
print_status " sudo ./apt-layer.sh --advanced-remove package-name"
print_status " sudo ./apt-layer.sh --add-user username package_manager"
print_status " sudo ./apt-layer.sh --list-users"
print_status " sudo ./apt-layer.sh --generate-key my-signing-key sigstore"
print_status " sudo ./apt-layer.sh --sign-layer layer.squashfs my-signing-key"
print_status " sudo ./apt-layer.sh --verify-layer layer.squashfs"
print_status " sudo ./apt-layer.sh --query-audit json --user=admin --since=2024-01-01"
print_status " sudo ./apt-layer.sh --export-audit csv --output=audit-export.csv"
print_status " sudo ./apt-layer.sh --generate-compliance-report sox monthly html"
print_status " sudo ./apt-layer.sh --audit-status"
print_status " sudo ./apt-layer.sh --scan-package package-name"
print_status " sudo ./apt-layer.sh --scan-layer layer.squashfs"
print_status " sudo ./apt-layer.sh --generate-security-report package html"
print_status " sudo ./apt-layer.sh --security-status"
print_status " sudo ./apt-layer.sh --update-cve-database"
print_status " sudo ./apt-layer.sh --dpkg-install curl wget"
print_status " sudo ./apt-layer.sh --container-dpkg base-image new-image packages"
print_status " sudo ./apt-layer.sh --live-dpkg firefox"
print_status " sudo ./apt-layer.sh admin health"
print_status " sudo ./apt-layer.sh admin perf"
print_status " sudo ./apt-layer.sh admin cleanup --dry-run --days 30"
print_status " sudo ./apt-layer.sh admin backup"
print_status " sudo ./apt-layer.sh admin restore"
print_status " sudo ./apt-layer.sh --list"
print_status " sudo ./apt-layer.sh --help"
print_status ""
print_status "Ready for distribution! <20>"