bootc-docs/internals/bootc-internals-examples-troubleshooting.md

# bootc internals - Examples and Troubleshooting

## Overview

This document provides practical examples and troubleshooting guidance for the `bootc internals` system, covering common use cases, error scenarios, and debugging techniques.

## Common Use Cases

### 1. System Maintenance

#### Daily Cleanup Operations

```bash
#!/bin/bash
# Daily cleanup script for bootc systems

# Run filesystem consistency check
echo "Running filesystem consistency check..."
bootc internals fsck

# Perform cleanup operations
echo "Performing cleanup operations..."
bootc internals cleanup

# Check system status
echo "Checking system status..."
bootc status

# Log results
echo "Maintenance completed at $(date)" >> /var/log/bootc-maintenance.log
```

#### Weekly Deep Cleanup

```bash
#!/bin/bash
# Weekly deep cleanup script

# Run comprehensive fsck
echo "Running comprehensive filesystem check..."
bootc internals fsck --verbose

# Clean up old deployments
echo "Cleaning up old deployments..."
bootc internals cleanup --aggressive

# Check for unused images
echo "Checking for unused images..."
bootc image list --unused

# Remove unused images
echo "Removing unused images..."
bootc image prune

# Log results
echo "Deep cleanup completed at $(date)" >> /var/log/bootc-maintenance.log
```

### 2. Development and Testing

#### Test Environment Setup

```bash
#!/bin/bash
# Setup test environment for bootc development

# Create test directory
mkdir -p /tmp/bootc-test
cd /tmp/bootc-test

# Test composefs repository
echo "Testing composefs repository..."
bootc internals test-composefs

# Test loopback device allocation
echo "Testing loopback device allocation..."
bootc internals allocate-cleanup-loopback --file-path /tmp/test-image.img

# Test directory diff functionality
echo "Testing directory diff..."
bootc internals dir-diff \
    --pristine-etc /tmp/pristine-etc \
    --current-etc /tmp/current-etc \
    --new-etc /tmp/new-etc \
    --perform-merge

# Clean up test environment
echo "Cleaning up test environment..."
rm -rf /tmp/bootc-test
```

#### Container Image Testing

```bash
#!/bin/bash
# Test container image operations

# Test image pull
echo "Testing image pull..."
bootc internals ostree-ext container pull quay.io/myorg/test-image:latest

# Test image build
echo "Testing image build..."
bootc internals ostree-ext container build /path/to/containerfile

# Test image verification
echo "Testing image verification..."
bootc internals fsverity measure /path/to/image

# Test image deployment
echo "Testing image deployment..."
bootc internals ostree-ext container deploy quay.io/myorg/test-image:latest
```

### 3. System Integration

#### Systemd Service Integration

```bash
#!/bin/bash
# Setup systemd service for bootc internals

# Create service file
cat > /etc/systemd/system/bootc-maintenance.service << EOF
[Unit]
Description=Bootc Maintenance Service
After=multi-user.target

[Service]
Type=oneshot
ExecStart=/usr/local/bin/bootc-maintenance.sh
User=root
Group=root

[Install]
WantedBy=multi-user.target
EOF

# Create timer file
cat > /etc/systemd/system/bootc-maintenance.timer << EOF
[Unit]
Description=Bootc Maintenance Timer
Requires=bootc-maintenance.service

[Timer]
OnCalendar=daily
Persistent=true

[Install]
WantedBy=timers.target
EOF

# Enable and start timer
systemctl daemon-reload
systemctl enable bootc-maintenance.timer
systemctl start bootc-maintenance.timer

# Check timer status
systemctl status bootc-maintenance.timer
```

#### Monitoring Integration

```bash
#!/bin/bash
# Setup monitoring for bootc internals

# Create monitoring script
cat > /usr/local/bin/bootc-monitor.sh << 'EOF'
#!/bin/bash

# Check system health
HEALTH_STATUS=$(bootc internals fsck 2>&1 | grep -c "✗")
if [ $HEALTH_STATUS -gt 0 ]; then
    echo "WARNING: Filesystem consistency issues detected"
    # Send alert to monitoring system
    curl -X POST "https://monitoring.example.com/alerts" \
        -d '{"service": "bootc", "status": "warning", "message": "Filesystem consistency issues"}'
fi

# Check storage usage
STORAGE_USAGE=$(df /var/lib/bootc | tail -1 | awk '{print $5}' | sed 's/%//')
if [ $STORAGE_USAGE -gt 80 ]; then
    echo "WARNING: Storage usage is high: ${STORAGE_USAGE}%"
    # Run cleanup
    bootc internals cleanup
fi

# Check deployment status
DEPLOYMENT_STATUS=$(bootc status --format json | jq -r '.status.booted.state')
if [ "$DEPLOYMENT_STATUS" != "active" ]; then
    echo "WARNING: Deployment status is not active: $DEPLOYMENT_STATUS"
fi
EOF

chmod +x /usr/local/bin/bootc-monitor.sh

# Add to crontab
echo "*/5 * * * * /usr/local/bin/bootc-monitor.sh" | crontab -
```

### 4. Debugging and Diagnostics

#### System Diagnostics

```bash
#!/bin/bash
# Comprehensive system diagnostics

echo "=== Bootc System Diagnostics ==="
echo "Date: $(date)"
echo "Hostname: $(hostname)"
echo "Uptime: $(uptime)"
echo

echo "=== System Status ==="
bootc status --format json | jq '.'
echo

echo "=== Filesystem Check ==="
bootc internals fsck
echo

echo "=== Storage Usage ==="
df -h /var/lib/bootc
echo

echo "=== Memory Usage ==="
free -h
echo

echo "=== Process Information ==="
ps aux | grep bootc
echo

echo "=== Systemd Services ==="
systemctl status bootc-* --no-pager
echo

echo "=== Journal Logs ==="
journalctl -u bootc-* --since "1 hour ago" --no-pager
echo

echo "=== OSTree Status ==="
ostree admin status
echo

echo "=== Composefs Status ==="
bootc internals cfs status
echo

echo "=== Diagnostics Complete ==="
```

#### Performance Analysis

```bash
#!/bin/bash
# Performance analysis script

echo "=== Bootc Performance Analysis ==="
echo "Date: $(date)"
echo

echo "=== Command Execution Times ==="
time bootc status
time bootc internals fsck
time bootc internals cleanup
echo

echo "=== Resource Usage ==="
echo "Memory usage:"
ps aux | grep bootc | awk '{sum+=$6} END {print sum/1024 " MB"}'

echo "CPU usage:"
top -bn1 | grep bootc | awk '{print $9}'

echo "Disk I/O:"
iotop -bn1 | grep bootc
echo

echo "=== System Load ==="
uptime
echo

echo "=== Performance Analysis Complete ==="
```

## Troubleshooting Guide

### 1. Common Error Scenarios

#### Permission Denied Errors

**Error**: `Permission denied: This command must be executed as the root user`

**Cause**: Command executed without root privileges

**Solution**:
```bash
# Check current user
whoami

# Switch to root
sudo su -

# Or use sudo
sudo bootc internals <command>
```

**Prevention**:
```bash
# Check if running as root
if [ "$EUID" -ne 0 ]; then
    echo "This script must be run as root"
    exit 1
fi
```

#### Filesystem Consistency Errors

**Error**: `✗ ostree-repo: Repository integrity check failed`

**Cause**: OSTree repository corruption

**Solution**:
```bash
# Check repository status
ostree admin status

# Verify repository integrity
ostree fsck

# If corruption detected, restore from backup
ostree admin deploy --from-commit <backup-commit>

# Or reinstall if no backup available
bootc install to-disk /dev/sda
```

**Prevention**:
```bash
# Regular integrity checks
bootc internals fsck

# Backup important deployments
ostree admin pin <deployment-id>
```

#### Storage Space Errors

**Error**: `No space left on device`

**Cause**: Insufficient storage space

**Solution**:
```bash
# Check storage usage
df -h /var/lib/bootc

# Run cleanup
bootc internals cleanup

# Remove old deployments
ostree admin cleanup

# Check for large files
du -sh /var/lib/bootc/* | sort -hr

# Remove unnecessary files
rm -rf /var/lib/bootc/tmp/*
```

**Prevention**:
```bash
# Monitor storage usage
df -h /var/lib/bootc | awk 'NR==2 {print $5}' | sed 's/%//'

# Set up alerts
if [ $STORAGE_USAGE -gt 80 ]; then
    echo "WARNING: Storage usage is high"
    bootc internals cleanup
fi
```

#### Systemd Generator Errors

**Error**: `Failed to generate systemd unit`

**Cause**: Systemd generator failure

**Solution**:
```bash
# Check systemd status
systemctl status bootc-systemd-generator

# Check generator logs
journalctl -u bootc-systemd-generator

# Manually run generator
bootc internals systemd-generator /run/systemd/system

# Reload systemd
systemctl daemon-reload
```

**Prevention**:
```bash
# Regular generator testing
bootc internals systemd-generator /tmp/test-units

# Check systemd configuration
systemd-analyze verify /etc/systemd/system/bootc-*
```

### 2. Debugging Techniques

#### Enable Debug Logging

```bash
# Set debug log level
export RUST_LOG=debug

# Run command with debug output
bootc internals fsck

# Check debug logs
journalctl -u bootc-* --since "1 hour ago" | grep DEBUG
```

#### Verbose Output

```bash
# Enable verbose output
bootc internals fsck --verbose

# Check verbose logs
journalctl -u bootc-* --since "1 hour ago" | grep -v INFO
```

#### System Information

```bash
# Gather system information
uname -a
lsb_release -a
systemctl --version
ostree --version
bootc --version

# Check system configuration
cat /etc/os-release
cat /proc/version
cat /proc/cpuinfo | head -20
```

#### Network Diagnostics

```bash
# Check network connectivity
ping -c 3 registry.example.com

# Check DNS resolution
nslookup registry.example.com

# Check firewall rules
iptables -L
firewall-cmd --list-all

# Check proxy settings
echo $HTTP_PROXY
echo $HTTPS_PROXY
```

### 3. Recovery Procedures

#### System Recovery

```bash
#!/bin/bash
# System recovery script

echo "=== Bootc System Recovery ==="
echo "Date: $(date)"
echo

# Check system status
echo "Checking system status..."
bootc status

# Check filesystem integrity
echo "Checking filesystem integrity..."
bootc internals fsck

# If errors found, attempt repair
if [ $? -ne 0 ]; then
    echo "Filesystem errors detected, attempting repair..."
    
    # Try to repair OSTree repository
    ostree fsck --repair
    
    # If repair fails, restore from backup
    if [ $? -ne 0 ]; then
        echo "Repair failed, restoring from backup..."
        ostree admin deploy --from-commit <backup-commit>
    fi
fi

# Clean up system
echo "Cleaning up system..."
bootc internals cleanup

# Verify system is working
echo "Verifying system is working..."
bootc status
bootc internals fsck

echo "Recovery complete"
```

#### Data Recovery

```bash
#!/bin/bash
# Data recovery script

echo "=== Bootc Data Recovery ==="
echo "Date: $(date)"
echo

# Check for available backups
echo "Checking for available backups..."
ls -la /var/lib/bootc/backups/

# List available deployments
echo "Available deployments:"
ostree admin status

# Check deployment history
echo "Deployment history:"
ostree log <deployment-id>

# Attempt to recover specific deployment
echo "Attempting to recover deployment..."
ostree admin deploy <deployment-id>

# Verify recovery
echo "Verifying recovery..."
bootc status
```

### 4. Performance Optimization

#### System Optimization

```bash
#!/bin/bash
# System optimization script

echo "=== Bootc System Optimization ==="
echo "Date: $(date)"
echo

# Optimize OSTree repository
echo "Optimizing OSTree repository..."
ostree admin cleanup

# Optimize composefs repository
echo "Optimizing composefs repository..."
bootc internals cfs optimize

# Clean up temporary files
echo "Cleaning up temporary files..."
bootc internals cleanup

# Optimize systemd services
echo "Optimizing systemd services..."
systemctl daemon-reload
systemctl restart bootc-*

# Check system performance
echo "Checking system performance..."
bootc internals fsck
df -h /var/lib/bootc
free -h
```

#### Storage Optimization

```bash
#!/bin/bash
# Storage optimization script

echo "=== Bootc Storage Optimization ==="
echo "Date: $(date)"
echo

# Check storage usage
echo "Current storage usage:"
df -h /var/lib/bootc

# Remove old deployments
echo "Removing old deployments..."
ostree admin cleanup --keep=3

# Remove unused images
echo "Removing unused images..."
bootc image prune

# Compress repository
echo "Compressing repository..."
ostree admin cleanup --compress

# Check optimized storage usage
echo "Optimized storage usage:"
df -h /var/lib/bootc
```

### 5. Monitoring and Alerting

#### Health Check Script

```bash
#!/bin/bash
# Health check script

HEALTH_STATUS=0

echo "=== Bootc Health Check ==="
echo "Date: $(date)"
echo

# Check system status
echo "Checking system status..."
if ! bootc status > /dev/null 2>&1; then
    echo "ERROR: System status check failed"
    HEALTH_STATUS=1
fi

# Check filesystem integrity
echo "Checking filesystem integrity..."
if ! bootc internals fsck > /dev/null 2>&1; then
    echo "ERROR: Filesystem integrity check failed"
    HEALTH_STATUS=1
fi

# Check storage usage
echo "Checking storage usage..."
STORAGE_USAGE=$(df /var/lib/bootc | tail -1 | awk '{print $5}' | sed 's/%//')
if [ $STORAGE_USAGE -gt 90 ]; then
    echo "ERROR: Storage usage is critical: ${STORAGE_USAGE}%"
    HEALTH_STATUS=1
elif [ $STORAGE_USAGE -gt 80 ]; then
    echo "WARNING: Storage usage is high: ${STORAGE_USAGE}%"
fi

# Check memory usage
echo "Checking memory usage..."
MEMORY_USAGE=$(free | grep Mem | awk '{printf "%.0f", $3/$2 * 100.0}')
if [ $MEMORY_USAGE -gt 90 ]; then
    echo "ERROR: Memory usage is critical: ${MEMORY_USAGE}%"
    HEALTH_STATUS=1
elif [ $MEMORY_USAGE -gt 80 ]; then
    echo "WARNING: Memory usage is high: ${MEMORY_USAGE}%"
fi

# Check systemd services
echo "Checking systemd services..."
if ! systemctl is-active bootc-* > /dev/null 2>&1; then
    echo "ERROR: Some bootc services are not active"
    HEALTH_STATUS=1
fi

# Report health status
if [ $HEALTH_STATUS -eq 0 ]; then
    echo "Health check passed"
else
    echo "Health check failed"
fi

exit $HEALTH_STATUS
```

#### Alerting Script

```bash
#!/bin/bash
# Alerting script

# Send alert to monitoring system
send_alert() {
    local severity=$1
    local message=$2
    
    curl -X POST "https://monitoring.example.com/alerts" \
        -H "Content-Type: application/json" \
        -d "{
            \"service\": \"bootc\",
            \"severity\": \"$severity\",
            \"message\": \"$message\",
            \"timestamp\": \"$(date -Iseconds)\"
        }"
}

# Check system health
if ! /usr/local/bin/bootc-health-check.sh; then
    send_alert "critical" "Bootc system health check failed"
fi

# Check storage usage
STORAGE_USAGE=$(df /var/lib/bootc | tail -1 | awk '{print $5}' | sed 's/%//')
if [ $STORAGE_USAGE -gt 90 ]; then
    send_alert "critical" "Storage usage is critical: ${STORAGE_USAGE}%"
elif [ $STORAGE_USAGE -gt 80 ]; then
    send_alert "warning" "Storage usage is high: ${STORAGE_USAGE}%"
fi

# Check memory usage
MEMORY_USAGE=$(free | grep Mem | awk '{printf "%.0f", $3/$2 * 100.0}')
if [ $MEMORY_USAGE -gt 90 ]; then
    send_alert "critical" "Memory usage is critical: ${MEMORY_USAGE}%"
elif [ $MEMORY_USAGE -gt 80 ]; then
    send_alert "warning" "Memory usage is high: ${MEMORY_USAGE}%"
fi
```

This comprehensive examples and troubleshooting guide provides practical solutions for common issues and advanced debugging techniques for the bootc internals system.