bootc-docs/upgrade/examples-and-troubleshooting.md

# bootc upgrade - Examples and Troubleshooting

## Practical Examples

### 1. Basic Update Operations

#### Check for Updates

```bash
# Check if updates are available
bootc upgrade --check

# Expected output:
# Update available for: quay.io/myorg/debian-bootc:latest
#   Version: 1.2.3
#   Digest: sha256:abc123...
```

#### Download Updates

```bash
# Download and stage updates
bootc upgrade

# Expected output:
# Update available for: quay.io/myorg/debian-bootc:latest
#   Version: 1.2.3
#   Digest: sha256:abc123...
# Staging update...
# Update staged successfully
```

#### Apply Updates

```bash
# Apply updates and reboot
bootc upgrade --apply

# Expected output:
# Update available for: quay.io/myorg/debian-bootc:latest
#   Version: 1.2.3
#   Digest: sha256:abc123...
# Staging update...
# Update staged successfully
# Rebooting system...
```

### 2. Automated Update Scripts

#### Simple Update Script

```bash
#!/bin/bash
# update-system.sh

set -e

echo "Checking for updates..."
if bootc upgrade --check; then
    echo "Updates available, applying..."
    bootc upgrade --apply --soft-reboot=auto
else
    echo "System is up to date"
fi
```

#### Advanced Update Script

```bash
#!/bin/bash
# advanced-update.sh

set -e

# Configuration
IMAGE_SOURCE="quay.io/myorg/debian-bootc:latest"
LOG_FILE="/var/log/bootc-updates.log"
MAX_RETRIES=3

# Logging function
log() {
    echo "$(date): $1" | tee -a "$LOG_FILE"
}

# Update function
update_system() {
    local retry_count=0
    
    while [ $retry_count -lt $MAX_RETRIES ]; do
        log "Attempting update (attempt $((retry_count + 1)))"
        
        if bootc upgrade --check; then
            log "Updates available, applying..."
            if bootc upgrade --apply --soft-reboot=auto; then
                log "Update applied successfully"
                return 0
            else
                log "Update failed, retrying..."
                retry_count=$((retry_count + 1))
                sleep 30
            fi
        else
            log "No updates available"
            return 0
        fi
    done
    
    log "Update failed after $MAX_RETRIES attempts"
    return 1
}

# Main execution
log "Starting update process"
update_system
log "Update process completed"
```

### 3. CI/CD Integration

#### GitHub Actions

```yaml
name: Update System

on:
  schedule:
    - cron: '0 2 * * *'  # Daily at 2 AM
  workflow_dispatch:

jobs:
  update:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      
      - name: Setup SSH
        uses: webfactory/ssh-agent@v0.7.0
        with:
          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
      
      - name: Update System
        run: |
          ssh -o StrictHostKeyChecking=no user@${{ secrets.HOST_IP }} '
            bootc upgrade --check
            if [ $? -eq 0 ]; then
              bootc upgrade --apply --soft-reboot=auto
            fi
          '
```

#### GitLab CI

```yaml
stages:
  - update

update_system:
  stage: update
  script:
    - ssh user@$HOST_IP 'bootc upgrade --check'
    - ssh user@$HOST_IP 'bootc upgrade --apply --soft-reboot=auto'
  only:
    - schedules
    - web
```

#### Jenkins Pipeline

```groovy
pipeline {
    agent any
    
    stages {
        stage('Update System') {
            steps {
                script {
                    sh '''
                        ssh user@${HOST_IP} 'bootc upgrade --check'
                        if [ $? -eq 0 ]; then
                            ssh user@${HOST_IP} 'bootc upgrade --apply --soft-reboot=auto'
                        fi
                    '''
                }
            }
        }
    }
    
    triggers {
        cron('0 2 * * *')  // Daily at 2 AM
    }
}
```

### 4. Monitoring and Alerting

#### Update Monitoring Script

```bash
#!/bin/bash
# monitor-updates.sh

# Configuration
WEBHOOK_URL="https://hooks.slack.com/services/YOUR/WEBHOOK/URL"
HOSTNAME=$(hostname)

# Check for updates
check_updates() {
    if bootc upgrade --check > /dev/null 2>&1; then
        return 0  # Updates available
    else
        return 1  # No updates
    fi
}

# Send notification
send_notification() {
    local message="$1"
    curl -X POST -H 'Content-type: application/json' \
        --data "{\"text\":\"$HOSTNAME: $message\"}" \
        "$WEBHOOK_URL"
}

# Main monitoring loop
while true; do
    if check_updates; then
        send_notification "Updates available for $HOSTNAME"
        # Wait longer after finding updates
        sleep 3600  # 1 hour
    else
        # Check more frequently when no updates
        sleep 300   # 5 minutes
    fi
done
```

#### Prometheus Metrics

```bash
#!/bin/bash
# prometheus-metrics.sh

# Create metrics file
METRICS_FILE="/var/lib/prometheus/node-exporter/bootc-updates.prom"

# Check update status
if bootc upgrade --check > /dev/null 2>&1; then
    UPDATES_AVAILABLE=1
else
    UPDATES_AVAILABLE=0
fi

# Write metrics
cat > "$METRICS_FILE" << EOF
# HELP bootc_updates_available Whether updates are available
# TYPE bootc_updates_available gauge
bootc_updates_available $UPDATES_AVAILABLE
EOF
```

### 5. Systemd Integration

#### Update Service

```ini
# /etc/systemd/system/bootc-update.service
[Unit]
Description=Update bootc system
After=network-online.target
Wants=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/bin/update-system.sh
StandardOutput=journal
StandardError=journal

[Install]
WantedBy=multi-user.target
```

#### Update Timer

```ini
# /etc/systemd/system/bootc-update.timer
[Unit]
Description=Update bootc system daily
Requires=bootc-update.service

[Timer]
OnCalendar=daily
Persistent=true

[Install]
WantedBy=timers.target
```

#### Enable and Start

```bash
# Enable and start timer
sudo systemctl enable bootc-update.timer
sudo systemctl start bootc-update.timer

# Check status
sudo systemctl status bootc-update.timer
sudo systemctl list-timers bootc-update.timer
```

## Troubleshooting Guide

### 1. Common Issues

#### No Updates Available

**Problem**: `bootc upgrade --check` shows no updates
**Possible Causes**:
- System is already up to date
- Registry connectivity issues
- Image source not configured
- Authentication problems

**Solutions**:
```bash
# Check current status
bootc status

# Verify image source
bootc edit

# Test registry connectivity
podman pull quay.io/myorg/debian-bootc:latest

# Check authentication
podman login quay.io
```

#### Update Fails to Download

**Problem**: `bootc upgrade` fails during download
**Possible Causes**:
- Network connectivity issues
- Registry authentication problems
- Insufficient disk space
- Registry rate limiting

**Solutions**:
```bash
# Check network connectivity
ping quay.io

# Check disk space
df -h

# Check authentication
podman login quay.io

# Retry with verbose output
RUST_LOG=debug bootc upgrade
```

#### Update Fails to Apply

**Problem**: `bootc upgrade --apply` fails
**Possible Causes**:
- Staged update corrupted
- Insufficient disk space
- Bootloader configuration issues
- System compatibility problems

**Solutions**:
```bash
# Check staged status
bootc status

# Verify system compatibility
bootc upgrade --check

# Check system logs
journalctl -u bootc-fetch-apply-updates.service

# Try rollback
bootc rollback
```

#### Soft Reboot Fails

**Problem**: `--soft-reboot=required` fails
**Possible Causes**:
- Soft reboot not supported
- Insufficient memory
- Kernel compatibility issues
- Hardware limitations

**Solutions**:
```bash
# Check soft reboot support
bootc upgrade --apply --soft-reboot=required

# Use auto mode (fallback to regular reboot)
bootc upgrade --apply --soft-reboot=auto

# Use regular reboot
bootc upgrade --apply
```

### 2. Debug Commands

#### Enable Debug Logging

```bash
# Enable debug output
RUST_LOG=debug bootc upgrade --check

# Enable trace output
RUST_LOG=trace bootc upgrade --check

# Save debug output to file
RUST_LOG=debug bootc upgrade --check 2>&1 | tee debug.log
```

#### Check System Status

```bash
# Check current status
bootc status

# Check status in JSON format
bootc status --json

# Check system configuration
bootc edit

# Check system logs
journalctl -u bootc-fetch-apply-updates.service
```

#### Verify Image Integrity

```bash
# Check image source
bootc edit

# Verify image exists
podman pull quay.io/myorg/debian-bootc:latest

# Check image labels
podman inspect quay.io/myorg/debian-bootc:latest | jq '.[0].Config.Labels'
```

### 3. Recovery Procedures

#### Rollback from Failed Update

```bash
# Check current status
bootc status

# Rollback to previous version
bootc rollback

# Verify rollback
bootc status

# Check system logs
journalctl -u bootc-fetch-apply-updates.service
```

#### Reset System State

```bash
# Check for local modifications
bootc status

# Reset local modifications (if any)
rpm-ostree reset

# Check status again
bootc status

# Try update again
bootc upgrade --check
```

#### Manual Image Switch

```bash
# Switch to different image
bootc switch quay.io/myorg/debian-bootc:v1.0

# Check status
bootc status

# Apply update
bootc upgrade --apply
```

### 4. Performance Issues

#### Slow Update Downloads

**Problem**: Updates take too long to download
**Solutions**:
```bash
# Check network speed
speedtest-cli

# Use local registry mirror
bootc switch registry.local/myorg/debian-bootc:latest

# Check disk I/O
iostat -x 1

# Use quiet mode
bootc upgrade --quiet
```

#### High Memory Usage

**Problem**: Update process uses too much memory
**Solutions**:
```bash
# Check memory usage
free -h

# Check swap usage
swapon -s

# Monitor during update
htop

# Use check mode first
bootc upgrade --check
```

#### Disk Space Issues

**Problem**: Insufficient disk space for updates
**Solutions**:
```bash
# Check disk usage
df -h

# Clean up old deployments
ostree admin cleanup

# Check OSTree usage
ostree admin status

# Free up space
sudo apt autoremove
sudo apt autoclean
```

### 5. Network Issues

#### Registry Connectivity

**Problem**: Cannot connect to registry
**Solutions**:
```bash
# Test connectivity
ping quay.io

# Check DNS resolution
nslookup quay.io

# Test with curl
curl -I https://quay.io

# Check firewall
sudo ufw status
```

#### Authentication Issues

**Problem**: Registry authentication fails
**Solutions**:
```bash
# Login to registry
podman login quay.io

# Check credentials
cat ~/.docker/config.json

# Test authentication
podman pull quay.io/myorg/debian-bootc:latest

# Check token expiration
podman logout quay.io
podman login quay.io
```

#### Proxy Configuration

**Problem**: Updates fail behind proxy
**Solutions**:
```bash
# Configure proxy for podman
mkdir -p ~/.config/containers
cat > ~/.config/containers/containers.conf << EOF
[containers]
http_proxy = "http://proxy.example.com:8080"
https_proxy = "http://proxy.example.com:8080"
no_proxy = "localhost,127.0.0.1"
EOF

# Test proxy
podman pull quay.io/myorg/debian-bootc:latest
```

### 6. Advanced Troubleshooting

#### System Logs Analysis

```bash
# Check bootc logs
journalctl -u bootc-fetch-apply-updates.service -f

# Check system logs
journalctl -f

# Check kernel logs
dmesg | tail -50

# Check boot logs
journalctl -b
```

#### Network Diagnostics

```bash
# Check network interfaces
ip addr show

# Check routing table
ip route show

# Check DNS configuration
cat /etc/resolv.conf

# Test DNS resolution
dig quay.io
```

#### System Health Check

```bash
# Check system resources
htop

# Check disk health
sudo smartctl -a /dev/sda

# Check memory
free -h

# Check CPU
lscpu
```

This comprehensive examples and troubleshooting guide provides practical solutions for common bootc upgrade issues and integration patterns.