debian-forge/scripts/error-handling-test.sh

#!/bin/bash
# Error Handling Test Script for debian-forge
# This script tests error handling and recovery mechanisms

set -e

echo "🔧 Debian Forge Error Handling Tests"
echo "===================================="

# Configuration
TEST_DIR="./error-tests"
RESULTS_DIR="./error-results"

# Create directories
mkdir -p "$TEST_DIR" "$RESULTS_DIR"

# Test cases for error handling
declare -A ERROR_TESTS=(
    ["invalid-manifest"]="invalid-manifest.json"
    ["missing-packages"]="missing-packages.json"
    ["invalid-repository"]="invalid-repository.json"
    ["network-failure"]="network-failure.json"
)

echo ""
echo "🧪 Running Error Handling Tests..."
echo "=================================="

# Create test manifests
mkdir -p "$TEST_DIR"

# 1. Invalid manifest (malformed JSON)
cat > "$TEST_DIR/invalid-manifest.json" << 'EOF'
{
  "version": "2",
  "pipelines": [
    {
      "runner": "org.osbuild.linux",
      "name": "build",
      "stages": [
        {
          "type": "org.osbuild.debootstrap",
          "options": {
            "suite": "trixie",
            "mirror": "http://deb.debian.org/debian",
            "arch": "amd64"
          }
        }
      ]
    }
  ]
  // Missing closing brace - invalid JSON
}
EOF

# 2. Missing packages manifest
cat > "$TEST_DIR/missing-packages.json" << 'EOF'
{
  "version": "2",
  "pipelines": [
    {
      "runner": "org.osbuild.linux",
      "name": "build",
      "stages": [
        {
          "type": "org.osbuild.debootstrap",
          "options": {
            "suite": "trixie",
            "mirror": "http://deb.debian.org/debian",
            "arch": "amd64"
          }
        },
        {
          "type": "org.osbuild.apt",
          "options": {
            "packages": [
              "nonexistent-package-12345",
              "another-missing-package-67890"
            ]
          }
        }
      ]
    }
  ]
}
EOF

# 3. Invalid repository manifest
cat > "$TEST_DIR/invalid-repository.json" << 'EOF'
{
  "version": "2",
  "pipelines": [
    {
      "runner": "org.osbuild.linux",
      "name": "build",
      "stages": [
        {
          "type": "org.osbuild.debootstrap",
          "options": {
            "suite": "trixie",
            "mirror": "http://invalid-mirror-that-does-not-exist.com/debian",
            "arch": "amd64"
          }
        }
      ]
    }
  ]
}
EOF

# 4. Network failure simulation manifest
cat > "$TEST_DIR/network-failure.json" << 'EOF'
{
  "version": "2",
  "pipelines": [
    {
      "runner": "org.osbuild.linux",
      "name": "build",
      "stages": [
        {
          "type": "org.osbuild.debootstrap",
          "options": {
            "suite": "trixie",
            "mirror": "http://192.168.1.999/debian",
            "arch": "amd64"
          }
        }
      ]
    }
  ]
}
EOF

# Test results
declare -A TEST_RESULTS
declare -A ERROR_MESSAGES

echo ""
echo "🔍 Testing Error Scenarios..."
echo "============================="

for test_name in "${!ERROR_TESTS[@]}"; do
    manifest="${ERROR_TESTS[$test_name]}"
    manifest_path="$TEST_DIR/$manifest"
    
    echo ""
    echo "🧪 Testing: $test_name"
    echo "----------------------"
    
    # Run test and capture output
    if python3 -m osbuild "$manifest_path" --output-dir "$TEST_DIR/${test_name}_output" --libdir . --json > "$RESULTS_DIR/${test_name}_result.json" 2>&1; then
        TEST_RESULTS[$test_name]="SUCCESS"
        ERROR_MESSAGES[$test_name]="No error detected"
        echo "✅ Test passed (unexpected)"
    else
        TEST_RESULTS[$test_name]="FAILED"
        ERROR_MESSAGES[$test_name]="Error detected as expected"
        echo "❌ Test failed (expected)"
        
        # Extract error message
        if [ -f "$RESULTS_DIR/${test_name}_result.json" ]; then
            error_msg=$(jq -r '.message // .error // "Unknown error"' "$RESULTS_DIR/${test_name}_result.json" 2>/dev/null || echo "JSON parse error")
            ERROR_MESSAGES[$test_name]="$error_msg"
            echo "   Error: $error_msg"
        fi
    fi
done

echo ""
echo "📊 Error Handling Summary"
echo "========================="

# Create error handling report
cat > "$RESULTS_DIR/error-handling-report.md" << EOF
# Debian Forge Error Handling Report

Generated: $(date)

## Test Results

| Test Case | Result | Error Message |
|-----------|--------|---------------|
EOF

for test_name in "${!ERROR_TESTS[@]}"; do
    result="${TEST_RESULTS[$test_name]}"
    error_msg="${ERROR_MESSAGES[$test_name]}"
    
    if [ "$result" = "SUCCESS" ]; then
        status="✅ PASS"
    else
        status="❌ FAIL"
    fi
    
    echo "| $test_name | $status | $error_msg |" >> "$RESULTS_DIR/error-handling-report.md"
done

cat >> "$RESULTS_DIR/error-handling-report.md" << EOF

## Error Analysis

### JSON Validation Errors
- **Invalid manifest**: Should fail with JSON schema validation error
- **Expected behavior**: Clear error message about malformed JSON

### Package Resolution Errors
- **Missing packages**: Should fail with package not found error
- **Expected behavior**: Clear error message about missing packages

### Network Errors
- **Invalid repository**: Should fail with network/connection error
- **Expected behavior**: Clear error message about repository access

### Recovery Recommendations

1. **JSON Validation**
   - Implement better JSON schema validation
   - Provide clear error messages for malformed manifests
   - Add manifest validation tools

2. **Package Resolution**
   - Improve package not found error messages
   - Add package availability checking
   - Implement package suggestion system

3. **Network Errors**
   - Add network connectivity checks
   - Implement retry mechanisms
   - Provide fallback repository options

4. **General Error Handling**
   - Add error recovery mechanisms
   - Implement graceful degradation
   - Provide detailed error logging

## Next Steps

1. Implement comprehensive error handling
2. Add error recovery mechanisms
3. Improve error messages
4. Add validation tools
5. Implement retry logic

EOF

echo ""
echo "📄 Error Handling Report Generated"
echo "=================================="
echo "📄 Report: $RESULTS_DIR/error-handling-report.md"
echo "📁 Results: $RESULTS_DIR/"

echo ""
echo "🎯 Error Handling Summary:"
echo "=========================="

for test_name in "${!ERROR_TESTS[@]}"; do
    result="${TEST_RESULTS[$test_name]}"
    error_msg="${ERROR_MESSAGES[$test_name]}"
    
    if [ "$result" = "SUCCESS" ]; then
        echo "✅ $test_name: PASSED (unexpected)"
    else
        echo "❌ $test_name: FAILED (expected) - $error_msg"
    fi
done

echo ""
echo "🔧 Error handling testing completed!"