particle-os-tools/test-production-integration.sh

#!/bin/bash

# Test script for apt-layer Production Integration
# Validates the Phase 2.4 implementation: Production Integration

set -e

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Test counters
TOTAL_TESTS=0
PASSED_TESTS=0
FAILED_TESTS=0

# Test logging functions
log_test() {
    echo -e "${BLUE}[TEST]${NC} $1"
}

log_pass() {
    echo -e "${GREEN}[PASS]${NC} $1"
    ((PASSED_TESTS++))
}

log_fail() {
    echo -e "${RED}[FAIL]${NC} $1"
    ((FAILED_TESTS++))
}

log_info() {
    echo -e "${YELLOW}[INFO]${NC} $1"
}

# Test summary
print_summary() {
    echo ""
    echo "=========================================="
    echo "PRODUCTION INTEGRATION TEST SUMMARY"
    echo "=========================================="
    echo "Total Tests: $TOTAL_TESTS"
    echo "Passed: $PASSED_TESTS"
    echo "Failed: $FAILED_TESTS"
    echo "Success Rate: $((PASSED_TESTS * 100 / TOTAL_TESTS))%"
    echo "=========================================="
    
    if [[ $FAILED_TESTS -eq 0 ]]; then
        echo -e "${GREEN}All tests passed! Production integration is working correctly.${NC}"
        exit 0
    else
        echo -e "${RED}Some tests failed. Please review the output above.${NC}"
        exit 1
    fi
}

# Cleanup function
cleanup() {
    log_info "Cleaning up test artifacts..."
    
    # Remove test systemd files
    rm -f /etc/systemd/system/apt-layer-test.service
    rm -f /etc/systemd/system/apt-layer-test-maintenance.service
    rm -f /etc/systemd/system/apt-layer-test-maintenance.timer
    
    # Remove test GRUB files
    rm -f /etc/grub.d/15_apt-layer-test
    
    # Remove test deployments
    rm -rf /tmp/apt-layer-production-test-*
    
    # Reload systemd if needed
    if command -v systemctl &> /dev/null; then
        systemctl daemon-reload 2>/dev/null || true
    fi
}

# Setup test environment
setup_test_env() {
    log_info "Setting up production test environment..."
    
    # Create test directories
    mkdir -p /tmp/apt-layer-production-test-deployments
    mkdir -p /tmp/apt-layer-production-test-backups
    mkdir -p /tmp/apt-layer-production-test-layers
    mkdir -p /tmp/apt-layer-production-test-mounts
    
    # Create test layers
    local source1="/tmp/apt-layer-production-test-source1"
    local source2="/tmp/apt-layer-production-test-source2"
    
    mkdir -p "$source1" "$source2"
    echo "Base layer content" > "$source1/base.txt"
    echo "Additional layer content" > "$source2/additional.txt"
    
    # Create test layers
    apt-layer composefs create "$source1" "/tmp/apt-layer-production-test-layers/base.composefs" "base-layer"
    apt-layer composefs create "$source2" "/tmp/apt-layer-production-test-layers/additional.composefs" "additional-layer"
    
    # Set environment variables for testing
    export DEPLOYMENT_DIR="/tmp/apt-layer-production-test-deployments"
    export BACKUP_DIR="/tmp/apt-layer-production-test-backups"
    export LOG_DIR="/tmp/apt-layer-production-test-logs"
    
    log_info "Production test environment setup completed"
}

# Test 1: Systemd integration setup
test_systemd_integration() {
    ((TOTAL_TESTS++))
    log_test "Testing systemd integration setup..."
    
    # Test systemd service creation
    if ! apt-layer production setup-systemd "apt-layer-test" "notify" "root"; then
        log_fail "Systemd integration setup failed"
        return 1
    fi
    
    # Check if service files were created
    if [[ ! -f "/etc/systemd/system/apt-layer-test.service" ]]; then
        log_fail "Systemd service file not created"
        return 1
    fi
    
    if [[ ! -f "/etc/systemd/system/apt-layer-test-maintenance.service" ]]; then
        log_fail "Systemd maintenance service file not created"
        return 1
    fi
    
    if [[ ! -f "/etc/systemd/system/apt-layer-test-maintenance.timer" ]]; then
        log_fail "Systemd maintenance timer file not created"
        return 1
    fi
    
    # Check service file content
    if ! grep -q "apt-layer daemon" "/etc/systemd/system/apt-layer-test.service"; then
        log_fail "Systemd service file missing ExecStart"
        return 1
    fi
    
    log_pass "Systemd integration setup test passed"
    return 0
}

# Test 2: GRUB integration setup
test_grub_integration() {
    ((TOTAL_TESTS++))
    log_test "Testing GRUB integration setup..."
    
    # Test GRUB integration setup
    if ! apt-layer production setup-grub "/tmp/apt-layer-production-test-grub" "/tmp/apt-layer-production-test-grub.cfg"; then
        log_fail "GRUB integration setup failed"
        return 1
    fi
    
    # Check if GRUB config was created
    if [[ ! -f "/tmp/apt-layer-production-test-grub" ]]; then
        log_fail "GRUB config file not created"
        return 1
    fi
    
    # Check GRUB config content
    if ! grep -q "GRUB_PARTICLE_OS_ENABLED" "/tmp/apt-layer-production-test-grub"; then
        log_fail "GRUB config missing apt-layer settings"
        return 1
    fi
    
    log_pass "GRUB integration setup test passed"
    return 0
}

# Test 3: systemd-boot integration setup
test_systemd_boot_integration() {
    ((TOTAL_TESTS++))
    log_test "Testing systemd-boot integration setup..."
    
    # Test systemd-boot integration setup
    if ! apt-layer production setup-systemd-boot "/tmp/apt-layer-production-test-efi"; then
        log_fail "systemd-boot integration setup failed"
        return 1
    fi
    
    # Check if loader config was created
    if [[ ! -f "/tmp/apt-layer-production-test-efi/loader/loader.conf" ]]; then
        log_fail "systemd-boot loader config not created"
        return 1
    fi
    
    # Check loader config content
    if ! grep -q "apt-layer" "/tmp/apt-layer-production-test-efi/loader/loader.conf"; then
        log_fail "systemd-boot loader config missing apt-layer settings"
        return 1
    fi
    
    log_pass "systemd-boot integration setup test passed"
    return 0
}

# Test 4: Deployment creation
test_deployment_creation() {
    ((TOTAL_TESTS++))
    log_test "Testing deployment creation..."
    
    local base_layer="/tmp/apt-layer-production-test-layers/base.composefs"
    local additional_layer="/tmp/apt-layer-production-test-layers/additional.composefs"
    local deployment_name="deployment-test-$(date +%Y%m%d-%H%M%S)"
    
    # Create deployment
    if ! apt-layer production create-deployment "$deployment_name" "$base_layer" "$additional_layer"; then
        log_fail "Deployment creation failed"
        return 1
    fi
    
    # Check if deployment directory was created
    local deployment_path="$DEPLOYMENT_DIR/$deployment_name"
    if [[ ! -d "$deployment_path" ]]; then
        log_fail "Deployment directory not created"
        return 1
    fi
    
    # Check deployment files
    if [[ ! -f "$deployment_path/metadata.json" ]]; then
        log_fail "Deployment metadata not created"
        return 1
    fi
    
    if [[ ! -f "$deployment_path/composed.composefs" ]]; then
        log_fail "Composed layer not created"
        return 1
    fi
    
    if [[ ! -f "$deployment_path/deploy.sh" ]]; then
        log_fail "Deployment script not created"
        return 1
    fi
    
    if [[ ! -x "$deployment_path/deploy.sh" ]]; then
        log_fail "Deployment script not executable"
        return 1
    fi
    
    # Check metadata content
    if ! command -v jq &> /dev/null; then
        log_info "jq not available, skipping metadata validation"
    else
        local status
        status=$(jq -r '.status' "$deployment_path/metadata.json" 2>/dev/null || echo "unknown")
        if [[ "$status" != "ready" ]]; then
            log_fail "Deployment status not ready: $status"
            return 1
        fi
    fi
    
    log_pass "Deployment creation test passed"
    return 0
}

# Test 5: Deployment listing
test_deployment_listing() {
    ((TOTAL_TESTS++))
    log_test "Testing deployment listing..."
    
    # List deployments
    if ! apt-layer production list-deployments; then
        log_fail "Deployment listing failed"
        return 1
    fi
    
    log_pass "Deployment listing test passed"
    return 0
}

# Test 6: Deployment health checking
test_deployment_health_check() {
    ((TOTAL_TESTS++))
    log_test "Testing deployment health checking..."
    
    # Find a deployment to check
    local deployments
    mapfile -t deployments < <(find "$DEPLOYMENT_DIR" -maxdepth 1 -type d -name "deployment-*" -printf "%f\n")
    
    if [[ ${#deployments[@]} -eq 0 ]]; then
        log_fail "No deployments found for health check"
        return 1
    fi
    
    local test_deployment="${deployments[0]}"
    
    # Check deployment health
    if ! apt-layer production health-check "$test_deployment"; then
        log_fail "Deployment health check failed"
        return 1
    fi
    
    # Check if health report was created
    local health_report="$DEPLOYMENT_DIR/$test_deployment/health-report.json"
    if [[ ! -f "$health_report" ]]; then
        log_fail "Health report not created"
        return 1
    fi
    
    log_pass "Deployment health check test passed"
    return 0
}

# Test 7: Deployment backup
test_deployment_backup() {
    ((TOTAL_TESTS++))
    log_test "Testing deployment backup..."
    
    # Find a deployment to backup
    local deployments
    mapfile -t deployments < <(find "$DEPLOYMENT_DIR" -maxdepth 1 -type d -name "deployment-*" -printf "%f\n")
    
    if [[ ${#deployments[@]} -eq 0 ]]; then
        log_fail "No deployments found for backup"
        return 1
    fi
    
    local test_deployment="${deployments[0]}"
    
    # Create deployment backup
    if ! apt-layer production backup-deployment "$test_deployment"; then
        log_fail "Deployment backup failed"
        return 1
    fi
    
    # Check if backup was created
    local backups
    mapfile -t backups < <(find "$BACKUP_DIR" -maxdepth 1 -type d -name "*-backup-*" -printf "%f\n")
    
    if [[ ${#backups[@]} -eq 0 ]]; then
        log_fail "No backup created"
        return 1
    fi
    
    # Check backup metadata
    local backup_path="$BACKUP_DIR/${backups[0]}"
    if [[ ! -f "$backup_path/backup-metadata.json" ]]; then
        log_fail "Backup metadata not created"
        return 1
    fi
    
    log_pass "Deployment backup test passed"
    return 0
}

# Test 8: Production status
test_production_status() {
    ((TOTAL_TESTS++))
    log_test "Testing production status..."
    
    # Check production status
    if ! apt-layer production status; then
        log_fail "Production status check failed"
        return 1
    fi
    
    log_pass "Production status test passed"
    return 0
}

# Test 9: Maintenance mode
test_maintenance_mode() {
    ((TOTAL_TESTS++))
    log_test "Testing maintenance mode..."
    
    # Run maintenance (should not fail even if no cleanup needed)
    if ! apt-layer maintenance; then
        log_fail "Maintenance mode failed"
        return 1
    fi
    
    log_pass "Maintenance mode test passed"
    return 0
}

# Test 10: Daemon mode (basic test)
test_daemon_mode() {
    ((TOTAL_TESTS++))
    log_test "Testing daemon mode..."
    
    # Test daemon startup (run for a short time)
    timeout 5s apt-layer daemon || true
    
    # Check if PID file was created (if daemon started successfully)
    if [[ -f "/var/run/apt-layer.pid" ]]; then
        # Clean up PID file
        rm -f "/var/run/apt-layer.pid"
        log_pass "Daemon mode test passed"
        return 0
    else
        log_info "Daemon mode test completed (PID file not created due to timeout)"
        log_pass "Daemon mode test passed"
        return 0
    fi
}

# Test 11: Rollback functionality
test_rollback_functionality() {
    ((TOTAL_TESTS++))
    log_test "Testing rollback functionality..."
    
    # Create a second deployment for rollback testing
    local base_layer="/tmp/apt-layer-production-test-layers/base.composefs"
    local deployment_name1="deployment-rollback1-$(date +%Y%m%d-%H%M%S)"
    local deployment_name2="deployment-rollback2-$(date +%Y%m%d-%H%M%S)"
    
    # Create two deployments
    apt-layer production create-deployment "$deployment_name1" "$base_layer"
    apt-layer production create-deployment "$deployment_name2" "$base_layer"
    
    # Deploy the first deployment
    if ! apt-layer production deploy "$deployment_name1"; then
        log_fail "Failed to deploy first deployment for rollback test"
        return 1
    fi
    
    # Test rollback (should rollback to previous deployment)
    if ! apt-layer production rollback; then
        log_fail "Rollback failed"
        return 1
    fi
    
    log_pass "Rollback functionality test passed"
    return 0
}

# Test 12: Integration testing
test_integration() {
    ((TOTAL_TESTS++))
    log_test "Testing production integration..."
    
    # Test that all components work together
    local base_layer="/tmp/apt-layer-production-test-layers/base.composefs"
    local deployment_name="deployment-integration-$(date +%Y%m%d-%H%M%S)"
    
    # Create and deploy a deployment
    if ! apt-layer production create-deployment "$deployment_name" "$base_layer"; then
        log_fail "Integration test: deployment creation failed"
        return 1
    fi
    
    if ! apt-layer production deploy "$deployment_name"; then
        log_fail "Integration test: deployment failed"
        return 1
    fi
    
    if ! apt-layer production health-check "$deployment_name"; then
        log_fail "Integration test: health check failed"
        return 1
    fi
    
    if ! apt-layer production backup-deployment "$deployment_name"; then
        log_fail "Integration test: backup failed"
        return 1
    fi
    
    log_pass "Production integration test passed"
    return 0
}

# Main test execution
main() {
    echo "=========================================="
    echo "apt-layer PRODUCTION INTEGRATION TEST SUITE"
    echo "=========================================="
    echo "Testing Phase 2.4: Production Integration"
    echo "=========================================="
    echo ""
    
    # Setup test environment
    setup_test_env
    
    # Run tests
    test_systemd_integration
    test_grub_integration
    test_systemd_boot_integration
    test_deployment_creation
    test_deployment_listing
    test_deployment_health_check
    test_deployment_backup
    test_production_status
    test_maintenance_mode
    test_daemon_mode
    test_rollback_functionality
    test_integration
    
    # Print summary
    print_summary
}

# Cleanup on exit
trap cleanup EXIT

# Run main function
main "$@"