bootc-docs/internals/bootc-internals-examples-troubleshooting.md

bootc internals - Examples and Troubleshooting

Overview

This document provides practical examples and troubleshooting guidance for the bootc internals system, covering common use cases, error scenarios, and debugging techniques.

Common Use Cases

1. System Maintenance

Daily Cleanup Operations

#!/bin/bash
# Daily cleanup script for bootc systems

# Run filesystem consistency check
echo "Running filesystem consistency check..."
bootc internals fsck

# Perform cleanup operations
echo "Performing cleanup operations..."
bootc internals cleanup

# Check system status
echo "Checking system status..."
bootc status

# Log results
echo "Maintenance completed at $(date)" >> /var/log/bootc-maintenance.log

Weekly Deep Cleanup

#!/bin/bash
# Weekly deep cleanup script

# Run comprehensive fsck
echo "Running comprehensive filesystem check..."
bootc internals fsck --verbose

# Clean up old deployments
echo "Cleaning up old deployments..."
bootc internals cleanup --aggressive

# Check for unused images
echo "Checking for unused images..."
bootc image list --unused

# Remove unused images
echo "Removing unused images..."
bootc image prune

# Log results
echo "Deep cleanup completed at $(date)" >> /var/log/bootc-maintenance.log

2. Development and Testing

Test Environment Setup

#!/bin/bash
# Setup test environment for bootc development

# Create test directory
mkdir -p /tmp/bootc-test
cd /tmp/bootc-test

# Test composefs repository
echo "Testing composefs repository..."
bootc internals test-composefs

# Test loopback device allocation
echo "Testing loopback device allocation..."
bootc internals allocate-cleanup-loopback --file-path /tmp/test-image.img

# Test directory diff functionality
echo "Testing directory diff..."
bootc internals dir-diff \
    --pristine-etc /tmp/pristine-etc \
    --current-etc /tmp/current-etc \
    --new-etc /tmp/new-etc \
    --perform-merge

# Clean up test environment
echo "Cleaning up test environment..."
rm -rf /tmp/bootc-test

Container Image Testing

#!/bin/bash
# Test container image operations

# Test image pull
echo "Testing image pull..."
bootc internals ostree-ext container pull quay.io/myorg/test-image:latest

# Test image build
echo "Testing image build..."
bootc internals ostree-ext container build /path/to/containerfile

# Test image verification
echo "Testing image verification..."
bootc internals fsverity measure /path/to/image

# Test image deployment
echo "Testing image deployment..."
bootc internals ostree-ext container deploy quay.io/myorg/test-image:latest

3. System Integration

Systemd Service Integration

#!/bin/bash
# Setup systemd service for bootc internals

# Create service file
cat > /etc/systemd/system/bootc-maintenance.service << EOF
[Unit]
Description=Bootc Maintenance Service
After=multi-user.target

[Service]
Type=oneshot
ExecStart=/usr/local/bin/bootc-maintenance.sh
User=root
Group=root

[Install]
WantedBy=multi-user.target
EOF

# Create timer file
cat > /etc/systemd/system/bootc-maintenance.timer << EOF
[Unit]
Description=Bootc Maintenance Timer
Requires=bootc-maintenance.service

[Timer]
OnCalendar=daily
Persistent=true

[Install]
WantedBy=timers.target
EOF

# Enable and start timer
systemctl daemon-reload
systemctl enable bootc-maintenance.timer
systemctl start bootc-maintenance.timer

# Check timer status
systemctl status bootc-maintenance.timer

Monitoring Integration

#!/bin/bash
# Setup monitoring for bootc internals

# Create monitoring script
cat > /usr/local/bin/bootc-monitor.sh << 'EOF'
#!/bin/bash

# Check system health
HEALTH_STATUS=$(bootc internals fsck 2>&1 | grep -c "✗")
if [ $HEALTH_STATUS -gt 0 ]; then
    echo "WARNING: Filesystem consistency issues detected"
    # Send alert to monitoring system
    curl -X POST "https://monitoring.example.com/alerts" \
        -d '{"service": "bootc", "status": "warning", "message": "Filesystem consistency issues"}'
fi

# Check storage usage
STORAGE_USAGE=$(df /var/lib/bootc | tail -1 | awk '{print $5}' | sed 's/%//')
if [ $STORAGE_USAGE -gt 80 ]; then
    echo "WARNING: Storage usage is high: ${STORAGE_USAGE}%"
    # Run cleanup
    bootc internals cleanup
fi

# Check deployment status
DEPLOYMENT_STATUS=$(bootc status --format json | jq -r '.status.booted.state')
if [ "$DEPLOYMENT_STATUS" != "active" ]; then
    echo "WARNING: Deployment status is not active: $DEPLOYMENT_STATUS"
fi
EOF

chmod +x /usr/local/bin/bootc-monitor.sh

# Add to crontab
echo "*/5 * * * * /usr/local/bin/bootc-monitor.sh" | crontab -

4. Debugging and Diagnostics

System Diagnostics

#!/bin/bash
# Comprehensive system diagnostics

echo "=== Bootc System Diagnostics ==="
echo "Date: $(date)"
echo "Hostname: $(hostname)"
echo "Uptime: $(uptime)"
echo

echo "=== System Status ==="
bootc status --format json | jq '.'
echo

echo "=== Filesystem Check ==="
bootc internals fsck
echo

echo "=== Storage Usage ==="
df -h /var/lib/bootc
echo

echo "=== Memory Usage ==="
free -h
echo

echo "=== Process Information ==="
ps aux | grep bootc
echo

echo "=== Systemd Services ==="
systemctl status bootc-* --no-pager
echo

echo "=== Journal Logs ==="
journalctl -u bootc-* --since "1 hour ago" --no-pager
echo

echo "=== OSTree Status ==="
ostree admin status
echo

echo "=== Composefs Status ==="
bootc internals cfs status
echo

echo "=== Diagnostics Complete ==="

Performance Analysis

#!/bin/bash
# Performance analysis script

echo "=== Bootc Performance Analysis ==="
echo "Date: $(date)"
echo

echo "=== Command Execution Times ==="
time bootc status
time bootc internals fsck
time bootc internals cleanup
echo

echo "=== Resource Usage ==="
echo "Memory usage:"
ps aux | grep bootc | awk '{sum+=$6} END {print sum/1024 " MB"}'

echo "CPU usage:"
top -bn1 | grep bootc | awk '{print $9}'

echo "Disk I/O:"
iotop -bn1 | grep bootc
echo

echo "=== System Load ==="
uptime
echo

echo "=== Performance Analysis Complete ==="

Troubleshooting Guide

1. Common Error Scenarios

Permission Denied Errors

Error: Permission denied: This command must be executed as the root user

Cause: Command executed without root privileges

Solution:

# Check current user
whoami

# Switch to root
sudo su -

# Or use sudo
sudo bootc internals <command>

Prevention:

# Check if running as root
if [ "$EUID" -ne 0 ]; then
    echo "This script must be run as root"
    exit 1
fi

Filesystem Consistency Errors

Error: ✗ ostree-repo: Repository integrity check failed

Cause: OSTree repository corruption

Solution:

# Check repository status
ostree admin status

# Verify repository integrity
ostree fsck

# If corruption detected, restore from backup
ostree admin deploy --from-commit <backup-commit>

# Or reinstall if no backup available
bootc install to-disk /dev/sda

Prevention:

# Regular integrity checks
bootc internals fsck

# Backup important deployments
ostree admin pin <deployment-id>

Storage Space Errors

Error: No space left on device

Cause: Insufficient storage space

Solution:

# Check storage usage
df -h /var/lib/bootc

# Run cleanup
bootc internals cleanup

# Remove old deployments
ostree admin cleanup

# Check for large files
du -sh /var/lib/bootc/* | sort -hr

# Remove unnecessary files
rm -rf /var/lib/bootc/tmp/*

Prevention:

# Monitor storage usage
df -h /var/lib/bootc | awk 'NR==2 {print $5}' | sed 's/%//'

# Set up alerts
if [ $STORAGE_USAGE -gt 80 ]; then
    echo "WARNING: Storage usage is high"
    bootc internals cleanup
fi

Systemd Generator Errors

Error: Failed to generate systemd unit

Cause: Systemd generator failure

Solution:

# Check systemd status
systemctl status bootc-systemd-generator

# Check generator logs
journalctl -u bootc-systemd-generator

# Manually run generator
bootc internals systemd-generator /run/systemd/system

# Reload systemd
systemctl daemon-reload

Prevention:

# Regular generator testing
bootc internals systemd-generator /tmp/test-units

# Check systemd configuration
systemd-analyze verify /etc/systemd/system/bootc-*

2. Debugging Techniques

Enable Debug Logging

# Set debug log level
export RUST_LOG=debug

# Run command with debug output
bootc internals fsck

# Check debug logs
journalctl -u bootc-* --since "1 hour ago" | grep DEBUG

Verbose Output

# Enable verbose output
bootc internals fsck --verbose

# Check verbose logs
journalctl -u bootc-* --since "1 hour ago" | grep -v INFO

System Information

# Gather system information
uname -a
lsb_release -a
systemctl --version
ostree --version
bootc --version

# Check system configuration
cat /etc/os-release
cat /proc/version
cat /proc/cpuinfo | head -20

Network Diagnostics

# Check network connectivity
ping -c 3 registry.example.com

# Check DNS resolution
nslookup registry.example.com

# Check firewall rules
iptables -L
firewall-cmd --list-all

# Check proxy settings
echo $HTTP_PROXY
echo $HTTPS_PROXY

3. Recovery Procedures

System Recovery

#!/bin/bash
# System recovery script

echo "=== Bootc System Recovery ==="
echo "Date: $(date)"
echo

# Check system status
echo "Checking system status..."
bootc status

# Check filesystem integrity
echo "Checking filesystem integrity..."
bootc internals fsck

# If errors found, attempt repair
if [ $? -ne 0 ]; then
    echo "Filesystem errors detected, attempting repair..."
    
    # Try to repair OSTree repository
    ostree fsck --repair
    
    # If repair fails, restore from backup
    if [ $? -ne 0 ]; then
        echo "Repair failed, restoring from backup..."
        ostree admin deploy --from-commit <backup-commit>
    fi
fi

# Clean up system
echo "Cleaning up system..."
bootc internals cleanup

# Verify system is working
echo "Verifying system is working..."
bootc status
bootc internals fsck

echo "Recovery complete"

Data Recovery

#!/bin/bash
# Data recovery script

echo "=== Bootc Data Recovery ==="
echo "Date: $(date)"
echo

# Check for available backups
echo "Checking for available backups..."
ls -la /var/lib/bootc/backups/

# List available deployments
echo "Available deployments:"
ostree admin status

# Check deployment history
echo "Deployment history:"
ostree log <deployment-id>

# Attempt to recover specific deployment
echo "Attempting to recover deployment..."
ostree admin deploy <deployment-id>

# Verify recovery
echo "Verifying recovery..."
bootc status

4. Performance Optimization

System Optimization

#!/bin/bash
# System optimization script

echo "=== Bootc System Optimization ==="
echo "Date: $(date)"
echo

# Optimize OSTree repository
echo "Optimizing OSTree repository..."
ostree admin cleanup

# Optimize composefs repository
echo "Optimizing composefs repository..."
bootc internals cfs optimize

# Clean up temporary files
echo "Cleaning up temporary files..."
bootc internals cleanup

# Optimize systemd services
echo "Optimizing systemd services..."
systemctl daemon-reload
systemctl restart bootc-*

# Check system performance
echo "Checking system performance..."
bootc internals fsck
df -h /var/lib/bootc
free -h

Storage Optimization

#!/bin/bash
# Storage optimization script

echo "=== Bootc Storage Optimization ==="
echo "Date: $(date)"
echo

# Check storage usage
echo "Current storage usage:"
df -h /var/lib/bootc

# Remove old deployments
echo "Removing old deployments..."
ostree admin cleanup --keep=3

# Remove unused images
echo "Removing unused images..."
bootc image prune

# Compress repository
echo "Compressing repository..."
ostree admin cleanup --compress

# Check optimized storage usage
echo "Optimized storage usage:"
df -h /var/lib/bootc

5. Monitoring and Alerting

Health Check Script

#!/bin/bash
# Health check script

HEALTH_STATUS=0

echo "=== Bootc Health Check ==="
echo "Date: $(date)"
echo

# Check system status
echo "Checking system status..."
if ! bootc status > /dev/null 2>&1; then
    echo "ERROR: System status check failed"
    HEALTH_STATUS=1
fi

# Check filesystem integrity
echo "Checking filesystem integrity..."
if ! bootc internals fsck > /dev/null 2>&1; then
    echo "ERROR: Filesystem integrity check failed"
    HEALTH_STATUS=1
fi

# Check storage usage
echo "Checking storage usage..."
STORAGE_USAGE=$(df /var/lib/bootc | tail -1 | awk '{print $5}' | sed 's/%//')
if [ $STORAGE_USAGE -gt 90 ]; then
    echo "ERROR: Storage usage is critical: ${STORAGE_USAGE}%"
    HEALTH_STATUS=1
elif [ $STORAGE_USAGE -gt 80 ]; then
    echo "WARNING: Storage usage is high: ${STORAGE_USAGE}%"
fi

# Check memory usage
echo "Checking memory usage..."
MEMORY_USAGE=$(free | grep Mem | awk '{printf "%.0f", $3/$2 * 100.0}')
if [ $MEMORY_USAGE -gt 90 ]; then
    echo "ERROR: Memory usage is critical: ${MEMORY_USAGE}%"
    HEALTH_STATUS=1
elif [ $MEMORY_USAGE -gt 80 ]; then
    echo "WARNING: Memory usage is high: ${MEMORY_USAGE}%"
fi

# Check systemd services
echo "Checking systemd services..."
if ! systemctl is-active bootc-* > /dev/null 2>&1; then
    echo "ERROR: Some bootc services are not active"
    HEALTH_STATUS=1
fi

# Report health status
if [ $HEALTH_STATUS -eq 0 ]; then
    echo "Health check passed"
else
    echo "Health check failed"
fi

exit $HEALTH_STATUS

Alerting Script

#!/bin/bash
# Alerting script

# Send alert to monitoring system
send_alert() {
    local severity=$1
    local message=$2
    
    curl -X POST "https://monitoring.example.com/alerts" \
        -H "Content-Type: application/json" \
        -d "{
            \"service\": \"bootc\",
            \"severity\": \"$severity\",
            \"message\": \"$message\",
            \"timestamp\": \"$(date -Iseconds)\"
        }"
}

# Check system health
if ! /usr/local/bin/bootc-health-check.sh; then
    send_alert "critical" "Bootc system health check failed"
fi

# Check storage usage
STORAGE_USAGE=$(df /var/lib/bootc | tail -1 | awk '{print $5}' | sed 's/%//')
if [ $STORAGE_USAGE -gt 90 ]; then
    send_alert "critical" "Storage usage is critical: ${STORAGE_USAGE}%"
elif [ $STORAGE_USAGE -gt 80 ]; then
    send_alert "warning" "Storage usage is high: ${STORAGE_USAGE}%"
fi

# Check memory usage
MEMORY_USAGE=$(free | grep Mem | awk '{printf "%.0f", $3/$2 * 100.0}')
if [ $MEMORY_USAGE -gt 90 ]; then
    send_alert "critical" "Memory usage is critical: ${MEMORY_USAGE}%"
elif [ $MEMORY_USAGE -gt 80 ]; then
    send_alert "warning" "Memory usage is high: ${MEMORY_USAGE}%"
fi

This comprehensive examples and troubleshooting guide provides practical solutions for common issues and advanced debugging techniques for the bootc internals system.