811 lines
20 KiB
Markdown
811 lines
20 KiB
Markdown
# Debian Forge Deployment Documentation
|
|
|
|
## Overview
|
|
This document covers deploying Debian Forge in production environments, including system requirements, configuration, monitoring, and maintenance procedures.
|
|
|
|
## System Requirements
|
|
|
|
### Minimum Requirements
|
|
- **CPU**: 4 cores (8 recommended)
|
|
- **Memory**: 8GB RAM (16GB recommended)
|
|
- **Storage**: 100GB available space (500GB recommended)
|
|
- **Network**: Stable internet connection for package downloads
|
|
- **OS**: Debian 12+ or Ubuntu 22.04+
|
|
|
|
### Recommended Production Specs
|
|
- **CPU**: 8+ cores with good single-thread performance
|
|
- **Memory**: 32GB+ RAM for concurrent builds
|
|
- **Storage**: 1TB+ SSD with good I/O performance
|
|
- **Network**: Gigabit connection with low latency
|
|
- **OS**: Debian 12+ with LTS support
|
|
|
|
### Storage Requirements
|
|
```
|
|
/var/lib/debian-forge/ # Build artifacts and cache
|
|
├── builds/ # Build outputs (50-200GB)
|
|
├── cache/ # Package cache (20-100GB)
|
|
├── ostree/ # OSTree repositories (100-500GB)
|
|
└── logs/ # Build logs (10-50GB)
|
|
|
|
/tmp/ # Temporary build space (50-100GB)
|
|
.osbuild/ # OSBuild cache (20-100GB)
|
|
```
|
|
|
|
## Production Deployment
|
|
|
|
### 1. System Preparation
|
|
```bash
|
|
# Update system
|
|
sudo apt update && sudo apt upgrade -y
|
|
|
|
# Install essential packages
|
|
sudo apt install -y \
|
|
python3-pip \
|
|
python3-venv \
|
|
python3-dev \
|
|
build-essential \
|
|
ostree \
|
|
debootstrap \
|
|
mmdebstrap \
|
|
sbuild \
|
|
pbuilder \
|
|
bubblewrap \
|
|
qemu-utils \
|
|
curl \
|
|
skopeo \
|
|
git \
|
|
nginx \
|
|
postgresql \
|
|
redis-server \
|
|
supervisor \
|
|
logrotate \
|
|
fail2ban \
|
|
ufw
|
|
|
|
# Configure firewall
|
|
sudo ufw allow ssh
|
|
sudo ufw allow 80/tcp
|
|
sudo ufw allow 443/tcp
|
|
sudo ufw allow 8080/tcp # Debian Forge API
|
|
sudo ufw enable
|
|
```
|
|
|
|
### 2. User and Security Setup
|
|
```bash
|
|
# Create dedicated user
|
|
sudo useradd -m -s /bin/bash debian-forge
|
|
sudo usermod -aG sudo debian-forge
|
|
sudo usermod -aG sbuild debian-forge
|
|
|
|
# Configure sudo access
|
|
echo "debian-forge ALL=(ALL) NOPASSWD: /usr/bin/apt, /usr/bin/apt-get, /usr/bin/dpkg" | sudo tee /etc/sudoers.d/debian-forge
|
|
|
|
# Set up SSH keys
|
|
sudo mkdir -p /home/debian-forge/.ssh
|
|
sudo chown debian-forge:debian-forge /home/debian-forge/.ssh
|
|
sudo chmod 700 /home/debian-forge/.ssh
|
|
|
|
# Copy your SSH key
|
|
sudo -u debian-forge ssh-keygen -t ed25519 -C "debian-forge@$(hostname)"
|
|
```
|
|
|
|
### 3. Database Setup
|
|
```bash
|
|
# Configure PostgreSQL
|
|
sudo -u postgres createuser debian-forge
|
|
sudo -u postgres createdb debian_forge
|
|
sudo -u postgres psql -c "GRANT ALL PRIVILEGES ON DATABASE debian_forge TO debian_forge;"
|
|
|
|
# Configure Redis
|
|
sudo systemctl enable redis-server
|
|
sudo systemctl start redis-server
|
|
```
|
|
|
|
### 4. Application Deployment
|
|
```bash
|
|
# Switch to debian-forge user
|
|
sudo su - debian-forge
|
|
|
|
# Clone repository
|
|
git clone https://github.com/your-org/debian-forge.git
|
|
cd debian-forge
|
|
|
|
# Create virtual environment
|
|
python3 -m venv venv
|
|
source venv/bin/activate
|
|
|
|
# Install dependencies
|
|
pip install -r requirements.txt
|
|
|
|
# Install additional production packages
|
|
pip install gunicorn uwsgi psycopg2-binary redis supervisor
|
|
```
|
|
|
|
### 5. Configuration Files
|
|
|
|
#### Environment Configuration
|
|
```bash
|
|
# /home/debian-forge/debian-forge/.env
|
|
DEBIAN_FORGE_ENV=production
|
|
DEBIAN_FORGE_DEBUG=false
|
|
DEBIAN_FORGE_SECRET_KEY=your-secret-key-here
|
|
DEBIAN_FORGE_DATABASE_URL=postgresql://debian-forge@localhost/debian_forge
|
|
DEBIAN_FORGE_REDIS_URL=redis://localhost:6379/0
|
|
DEBIAN_FORGE_LOG_LEVEL=INFO
|
|
DEBIAN_FORGE_MAX_CONCURRENT_BUILDS=4
|
|
DEBIAN_FORGE_BUILD_TIMEOUT=3600
|
|
DEBIAN_FORGE_CACHE_SIZE=50GB
|
|
DEBIAN_FORGE_OSTREE_REPO_PATH=/var/lib/debian-forge/ostree
|
|
```
|
|
|
|
#### Build Environment Configuration
|
|
```bash
|
|
# /home/debian-forge/debian-forge/config/build-env.conf
|
|
[build_environment]
|
|
max_concurrent_builds = 4
|
|
build_timeout = 3600
|
|
resource_limits_cpu = 80
|
|
resource_limits_memory = 85
|
|
resource_limits_disk = 90
|
|
cleanup_after_build = true
|
|
cache_retention_days = 30
|
|
|
|
[ostree]
|
|
repo_path = /var/lib/debian-forge/ostree
|
|
max_repo_size = 100GB
|
|
cleanup_old_commits = true
|
|
commit_retention_days = 90
|
|
|
|
[apt]
|
|
proxy_url = http://192.168.1.101:3142
|
|
mirror_url = http://deb.debian.org/debian
|
|
security_url = http://security.debian.org/debian-security
|
|
updates_url = http://deb.debian.org/debian
|
|
```
|
|
|
|
### 6. Service Configuration
|
|
|
|
#### Supervisor Configuration
|
|
```ini
|
|
# /etc/supervisor/conf.d/debian-forge.conf
|
|
[program:debian-forge-api]
|
|
command=/home/debian-forge/debian-forge/venv/bin/gunicorn -w 4 -b 127.0.0.1:8080 --timeout 300 --max-requests 1000 --max-requests-jitter 100 app:app
|
|
directory=/home/debian-forge/debian-forge
|
|
user=debian-forge
|
|
autostart=true
|
|
autorestart=true
|
|
redirect_stderr=true
|
|
stdout_logfile=/var/log/debian-forge/api.log
|
|
stdout_logfile_maxbytes=50MB
|
|
stdout_logfile_backups=10
|
|
|
|
[program:debian-forge-worker]
|
|
command=/home/debian-forge/debian-forge/venv/bin/python -m build_orchestrator
|
|
directory=/home/debian-forge/debian-forge
|
|
user=debian-forge
|
|
autostart=true
|
|
autorestart=true
|
|
redirect_stderr=true
|
|
stdout_logfile=/var/log/debian-forge/worker.log
|
|
stdout_logfile_maxbytes=50MB
|
|
stdout_logfile_backups=10
|
|
|
|
[program:debian-forge-cleanup]
|
|
command=/home/debian-forge/debian-forge/venv/bin/python -m cleanup_manager
|
|
directory=/home/debian-forge/debian-forge
|
|
user=debian-forge
|
|
autostart=true
|
|
autorestart=true
|
|
redirect_stderr=true
|
|
stdout_logfile=/var/log/debian-forge/cleanup.log
|
|
stdout_logfile_maxbytes=50MB
|
|
stdout_logfile_backups=10
|
|
```
|
|
|
|
#### Nginx Configuration
|
|
```nginx
|
|
# /etc/nginx/sites-available/debian-forge
|
|
server {
|
|
listen 80;
|
|
server_name your-domain.com;
|
|
return 301 https://$server_name$request_uri;
|
|
}
|
|
|
|
server {
|
|
listen 443 ssl http2;
|
|
server_name your-domain.com;
|
|
|
|
ssl_certificate /etc/letsencrypt/live/your-domain.com/fullchain.pem;
|
|
ssl_certificate_key /etc/letsencrypt/live/your-domain.com/privkey.pem;
|
|
ssl_protocols TLSv1.2 TLSv1.3;
|
|
ssl_ciphers ECDHE-RSA-AES256-GCM-SHA512:DHE-RSA-AES256-GCM-SHA512:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384;
|
|
ssl_prefer_server_ciphers off;
|
|
|
|
client_max_body_size 100M;
|
|
proxy_read_timeout 300s;
|
|
proxy_connect_timeout 75s;
|
|
|
|
location / {
|
|
proxy_pass http://127.0.0.1:8080;
|
|
proxy_set_header Host $host;
|
|
proxy_set_header X-Real-IP $remote_addr;
|
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
|
proxy_set_header X-Forwarded-Proto $scheme;
|
|
}
|
|
|
|
location /static/ {
|
|
alias /home/debian-forge/debian-forge/static/;
|
|
expires 1y;
|
|
add_header Cache-Control "public, immutable";
|
|
}
|
|
|
|
location /logs/ {
|
|
alias /var/log/debian-forge/;
|
|
auth_basic "Restricted Access";
|
|
auth_basic_user_file /etc/nginx/.htpasswd;
|
|
}
|
|
}
|
|
```
|
|
|
|
### 7. SSL Certificate Setup
|
|
```bash
|
|
# Install Certbot
|
|
sudo apt install -y certbot python3-certbot-nginx
|
|
|
|
# Obtain SSL certificate
|
|
sudo certbot --nginx -d your-domain.com
|
|
|
|
# Test auto-renewal
|
|
sudo certbot renew --dry-run
|
|
```
|
|
|
|
### 8. Monitoring and Logging
|
|
|
|
#### Logrotate Configuration
|
|
```bash
|
|
# /etc/logrotate.d/debian-forge
|
|
/var/log/debian-forge/*.log {
|
|
daily
|
|
missingok
|
|
rotate 30
|
|
compress
|
|
delaycompress
|
|
notifempty
|
|
create 644 debian-forge debian-forge
|
|
postrotate
|
|
systemctl reload supervisor
|
|
endscript
|
|
}
|
|
```
|
|
|
|
#### Monitoring Scripts
|
|
```bash
|
|
#!/bin/bash
|
|
# /home/debian-forge/debian-forge/scripts/monitor.sh
|
|
|
|
# Check service status
|
|
check_service() {
|
|
local service=$1
|
|
if ! systemctl is-active --quiet $service; then
|
|
echo "ERROR: $service is not running"
|
|
systemctl restart $service
|
|
echo "$(date): Restarted $service" >> /var/log/debian-forge/monitor.log
|
|
fi
|
|
}
|
|
|
|
# Check disk space
|
|
check_disk() {
|
|
local usage=$(df /var/lib/debian-forge | tail -1 | awk '{print $5}' | sed 's/%//')
|
|
if [ $usage -gt 90 ]; then
|
|
echo "WARNING: Disk usage is ${usage}%"
|
|
# Trigger cleanup
|
|
/home/debian-forge/debian-forge/venv/bin/python -m cleanup_manager --force
|
|
fi
|
|
}
|
|
|
|
# Check memory usage
|
|
check_memory() {
|
|
local usage=$(free | grep Mem | awk '{printf "%.0f", $3/$2 * 100.0}')
|
|
if [ $usage -gt 90 ]; then
|
|
echo "WARNING: Memory usage is ${usage}%"
|
|
fi
|
|
}
|
|
|
|
# Main monitoring loop
|
|
while true; do
|
|
check_service debian-forge-api
|
|
check_service debian-forge-worker
|
|
check_service debian-forge-cleanup
|
|
check_disk
|
|
check_memory
|
|
sleep 300 # Check every 5 minutes
|
|
done
|
|
```
|
|
|
|
## Production Configuration
|
|
|
|
### 1. Performance Tuning
|
|
|
|
#### System Tuning
|
|
```bash
|
|
# /etc/sysctl.conf
|
|
# Increase file descriptor limits
|
|
fs.file-max = 65536
|
|
fs.inotify.max_user_watches = 524288
|
|
|
|
# Network tuning
|
|
net.core.somaxconn = 65535
|
|
net.core.netdev_max_backlog = 5000
|
|
net.ipv4.tcp_max_syn_backlog = 65535
|
|
|
|
# Memory tuning
|
|
vm.swappiness = 10
|
|
vm.dirty_ratio = 15
|
|
vm.dirty_background_ratio = 5
|
|
```
|
|
|
|
#### Application Tuning
|
|
```python
|
|
# /home/debian-forge/debian-forge/config/performance.conf
|
|
[performance]
|
|
max_workers = 4
|
|
worker_timeout = 300
|
|
max_requests = 1000
|
|
max_requests_jitter = 100
|
|
keepalive = 2
|
|
worker_connections = 1000
|
|
|
|
[caching]
|
|
cache_size = 50GB
|
|
cache_ttl = 86400
|
|
cache_cleanup_interval = 3600
|
|
|
|
[build_optimization]
|
|
parallel_stages = true
|
|
stage_cache_enabled = true
|
|
artifact_compression = true
|
|
```
|
|
|
|
### 2. Security Configuration
|
|
|
|
#### Fail2ban Configuration
|
|
```ini
|
|
# /etc/fail2ban/jail.local
|
|
[debian-forge-api]
|
|
enabled = true
|
|
port = 8080
|
|
filter = debian-forge-api
|
|
logpath = /var/log/debian-forge/api.log
|
|
maxretry = 5
|
|
bantime = 3600
|
|
findtime = 600
|
|
|
|
[debian-forge-ssh]
|
|
enabled = true
|
|
port = ssh
|
|
filter = sshd
|
|
logpath = /var/log/auth.log
|
|
maxretry = 3
|
|
bantime = 3600
|
|
findtime = 600
|
|
```
|
|
|
|
#### Access Control
|
|
```bash
|
|
# /etc/nginx/.htpasswd (for log access)
|
|
sudo htpasswd -c /etc/nginx/.htpasswd admin
|
|
|
|
# SSH key-based authentication only
|
|
sudo sed -i 's/PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config
|
|
sudo systemctl reload ssh
|
|
```
|
|
|
|
### 3. Backup Configuration
|
|
|
|
#### Backup Script
|
|
```bash
|
|
#!/bin/bash
|
|
# /home/debian-forge/debian-forge/scripts/backup.sh
|
|
|
|
BACKUP_DIR="/var/backups/debian-forge"
|
|
DATE=$(date +%Y%m%d_%H%M%S)
|
|
RETENTION_DAYS=30
|
|
|
|
# Create backup directory
|
|
mkdir -p $BACKUP_DIR
|
|
|
|
# Database backup
|
|
pg_dump -U debian-forge debian_forge > $BACKUP_DIR/db_$DATE.sql
|
|
|
|
# Configuration backup
|
|
tar -czf $BACKUP_DIR/config_$DATE.tar.gz \
|
|
/home/debian-forge/debian-forge/config \
|
|
/etc/supervisor/conf.d/debian-forge.conf \
|
|
/etc/nginx/sites-available/debian-forge
|
|
|
|
# OSTree repository backup
|
|
rsync -av --delete /var/lib/debian-forge/ostree/ $BACKUP_DIR/ostree_$DATE/
|
|
|
|
# Cleanup old backups
|
|
find $BACKUP_DIR -type f -mtime +$RETENTION_DAYS -delete
|
|
find $BACKUP_DIR -type d -mtime +$RETENTION_DAYS -exec rm -rf {} +
|
|
|
|
echo "Backup completed: $DATE" >> /var/log/debian-forge/backup.log
|
|
```
|
|
|
|
#### Automated Backup
|
|
```bash
|
|
# /etc/cron.daily/debian-forge-backup
|
|
#!/bin/bash
|
|
/home/debian-forge/debian-forge/scripts/backup.sh
|
|
```
|
|
|
|
## Maintenance Procedures
|
|
|
|
### 1. Regular Maintenance
|
|
|
|
#### Daily Tasks
|
|
```bash
|
|
# Check service status
|
|
sudo supervisorctl status
|
|
|
|
# Monitor logs
|
|
tail -f /var/log/debian-forge/*.log
|
|
|
|
# Check disk space
|
|
df -h /var/lib/debian-forge
|
|
|
|
# Check build queue
|
|
curl -s http://localhost:8080/api/v1/queue/status
|
|
```
|
|
|
|
#### Weekly Tasks
|
|
```bash
|
|
# Update system packages
|
|
sudo apt update && sudo apt upgrade -y
|
|
|
|
# Clean old build artifacts
|
|
/home/debian-forge/debian-forge/venv/bin/python -m cleanup_manager --force
|
|
|
|
# Rotate logs
|
|
sudo logrotate -f /etc/logrotate.d/debian-forge
|
|
|
|
# Check SSL certificate expiration
|
|
sudo certbot certificates
|
|
```
|
|
|
|
#### Monthly Tasks
|
|
```bash
|
|
# Review and clean old OSTree commits
|
|
/home/debian-forge/debian-forge/venv/bin/python -m ostree_cleanup --older-than 90
|
|
|
|
# Update application dependencies
|
|
cd /home/debian-forge/debian-forge
|
|
source venv/bin/activate
|
|
pip install --upgrade -r requirements.txt
|
|
|
|
# Review and update security configurations
|
|
sudo fail2ban-client status
|
|
sudo ufw status
|
|
```
|
|
|
|
### 2. Troubleshooting
|
|
|
|
#### Common Issues
|
|
|
|
**Service Not Starting**
|
|
```bash
|
|
# Check supervisor status
|
|
sudo supervisorctl status
|
|
|
|
# Check logs
|
|
sudo tail -f /var/log/supervisor/supervisord.log
|
|
sudo tail -f /var/log/debian-forge/*.log
|
|
|
|
# Restart services
|
|
sudo supervisorctl restart debian-forge-api
|
|
sudo supervisorctl restart debian-forge-worker
|
|
```
|
|
|
|
**Build Failures**
|
|
```bash
|
|
# Check build logs
|
|
tail -f /var/log/debian-forge/worker.log
|
|
|
|
# Check system resources
|
|
htop
|
|
df -h
|
|
free -h
|
|
|
|
# Restart worker
|
|
sudo supervisorctl restart debian-forge-worker
|
|
```
|
|
|
|
**Database Issues**
|
|
```bash
|
|
# Check PostgreSQL status
|
|
sudo systemctl status postgresql
|
|
|
|
# Check connection
|
|
sudo -u debian-forge psql -d debian_forge -c "SELECT version();"
|
|
|
|
# Restart database
|
|
sudo systemctl restart postgresql
|
|
```
|
|
|
|
### 3. Recovery Procedures
|
|
|
|
#### Service Recovery
|
|
```bash
|
|
#!/bin/bash
|
|
# /home/debian-forge/debian-forge/scripts/recovery.sh
|
|
|
|
echo "Starting Debian Forge recovery..."
|
|
|
|
# Stop all services
|
|
sudo supervisorctl stop all
|
|
|
|
# Clean up temporary files
|
|
sudo rm -rf /tmp/debian-forge-*
|
|
sudo rm -rf /var/tmp/debian-forge-*
|
|
|
|
# Restart database
|
|
sudo systemctl restart postgresql
|
|
sudo systemctl restart redis-server
|
|
|
|
# Wait for services to be ready
|
|
sleep 10
|
|
|
|
# Start services
|
|
sudo supervisorctl start all
|
|
|
|
# Check status
|
|
sudo supervisorctl status
|
|
|
|
echo "Recovery completed"
|
|
```
|
|
|
|
#### Data Recovery
|
|
```bash
|
|
#!/bin/bash
|
|
# /home/debian-forge/debian-forge/scripts/data-recovery.sh
|
|
|
|
BACKUP_DIR="/var/backups/debian-forge"
|
|
LATEST_BACKUP=$(ls -t $BACKUP_DIR/db_*.sql | head -1)
|
|
|
|
if [ -n "$LATEST_BACKUP" ]; then
|
|
echo "Restoring from backup: $LATEST_BACKUP"
|
|
|
|
# Stop services
|
|
sudo supervisorctl stop all
|
|
|
|
# Restore database
|
|
sudo -u postgres dropdb debian_forge
|
|
sudo -u postgres createdb debian_forge
|
|
sudo -u postgres psql debian_forge < $LATEST_BACKUP
|
|
|
|
# Restart services
|
|
sudo supervisorctl start all
|
|
|
|
echo "Data recovery completed"
|
|
else
|
|
echo "No backup found for recovery"
|
|
exit 1
|
|
fi
|
|
```
|
|
|
|
## Scaling Considerations
|
|
|
|
### 1. Horizontal Scaling
|
|
|
|
#### Load Balancer Configuration
|
|
```nginx
|
|
# /etc/nginx/sites-available/debian-forge-cluster
|
|
upstream debian_forge_backend {
|
|
server 192.168.1.10:8080;
|
|
server 192.168.1.11:8080;
|
|
server 192.168.1.12:8080;
|
|
}
|
|
|
|
server {
|
|
listen 443 ssl http2;
|
|
server_name your-domain.com;
|
|
|
|
location / {
|
|
proxy_pass http://debian_forge_backend;
|
|
proxy_set_header Host $host;
|
|
proxy_set_header X-Real-IP $remote_addr;
|
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
|
proxy_set_header X-Forwarded-Proto $scheme;
|
|
}
|
|
}
|
|
```
|
|
|
|
#### Shared Storage
|
|
```bash
|
|
# NFS configuration for shared storage
|
|
# /etc/exports
|
|
/var/lib/debian-forge 192.168.1.0/24(rw,sync,no_subtree_check)
|
|
|
|
# Mount on worker nodes
|
|
# /etc/fstab
|
|
192.168.1.10:/var/lib/debian-forge /var/lib/debian-forge nfs defaults 0 0
|
|
```
|
|
|
|
### 2. Vertical Scaling
|
|
|
|
#### Resource Optimization
|
|
```python
|
|
# /home/debian-forge/debian-forge/config/scaling.conf
|
|
[scaling]
|
|
max_concurrent_builds = 8
|
|
worker_processes = 8
|
|
memory_limit = 32GB
|
|
cpu_limit = 8
|
|
|
|
[cache]
|
|
cache_size = 100GB
|
|
cache_ttl = 172800 # 48 hours
|
|
```
|
|
|
|
## Monitoring and Alerting
|
|
|
|
### 1. Health Checks
|
|
|
|
#### Application Health
|
|
```python
|
|
# /home/debian-forge/debian-forge/health_check.py
|
|
import requests
|
|
import psutil
|
|
import os
|
|
|
|
def check_health():
|
|
health_status = {
|
|
"status": "healthy",
|
|
"checks": {}
|
|
}
|
|
|
|
# Check API endpoint
|
|
try:
|
|
response = requests.get("http://localhost:8080/health", timeout=5)
|
|
health_status["checks"]["api"] = "healthy" if response.status_code == 200 else "unhealthy"
|
|
except:
|
|
health_status["checks"]["api"] = "unhealthy"
|
|
|
|
# Check system resources
|
|
cpu_percent = psutil.cpu_percent()
|
|
memory_percent = psutil.virtual_memory().percent
|
|
disk_percent = psutil.disk_usage('/var/lib/debian-forge').percent
|
|
|
|
health_status["checks"]["cpu"] = "healthy" if cpu_percent < 90 else "warning"
|
|
health_status["checks"]["memory"] = "healthy" if memory_percent < 90 else "warning"
|
|
health_status["checks"]["disk"] = "healthy" if disk_percent < 90 else "warning"
|
|
|
|
# Overall status
|
|
if any(check == "unhealthy" for check in health_status["checks"].values()):
|
|
health_status["status"] = "unhealthy"
|
|
elif any(check == "warning" for check in health_status["checks"].values()):
|
|
health_status["status"] = "degraded"
|
|
|
|
return health_status
|
|
```
|
|
|
|
#### Monitoring Dashboard
|
|
```html
|
|
<!-- /home/debian-forge/debian-forge/templates/monitor.html -->
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>Debian Forge Monitor</title>
|
|
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
|
|
</head>
|
|
<body>
|
|
<h1>Debian Forge System Monitor</h1>
|
|
|
|
<div class="status-grid">
|
|
<div class="status-card">
|
|
<h3>API Status</h3>
|
|
<div id="api-status">Checking...</div>
|
|
</div>
|
|
|
|
<div class="status-card">
|
|
<h3>Build Queue</h3>
|
|
<div id="queue-status">Checking...</div>
|
|
</div>
|
|
|
|
<div class="status-card">
|
|
<h3>System Resources</h3>
|
|
<canvas id="resource-chart"></canvas>
|
|
</div>
|
|
</div>
|
|
|
|
<script>
|
|
// Update status every 30 seconds
|
|
setInterval(updateStatus, 30000);
|
|
|
|
function updateStatus() {
|
|
fetch('/api/v1/health')
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
document.getElementById('api-status').textContent = data.status;
|
|
// Update other status elements
|
|
});
|
|
}
|
|
</script>
|
|
</body>
|
|
</html>
|
|
```
|
|
|
|
### 2. Alerting Configuration
|
|
|
|
#### Email Alerts
|
|
```python
|
|
# /home/debian-forge/debian-forge/scripts/alert.py
|
|
import smtplib
|
|
from email.mime.text import MIMEText
|
|
import os
|
|
|
|
def send_alert(subject, message, severity="INFO"):
|
|
smtp_server = os.getenv("SMTP_SERVER", "localhost")
|
|
smtp_port = int(os.getenv("SMTP_PORT", "587"))
|
|
smtp_user = os.getenv("SMTP_USER")
|
|
smtp_password = os.getenv("SMTP_PASSWORD")
|
|
alert_email = os.getenv("ALERT_EMAIL")
|
|
|
|
msg = MIMEText(message)
|
|
msg['Subject'] = f"[{severity}] Debian Forge: {subject}"
|
|
msg['From'] = smtp_user
|
|
msg['To'] = alert_email
|
|
|
|
try:
|
|
with smtplib.SMTP(smtp_server, smtp_port) as server:
|
|
server.starttls()
|
|
server.login(smtp_user, smtp_password)
|
|
server.send_message(msg)
|
|
print(f"Alert sent: {subject}")
|
|
except Exception as e:
|
|
print(f"Failed to send alert: {e}")
|
|
```
|
|
|
|
#### Alert Rules
|
|
```yaml
|
|
# /home/debian-forge/debian-forge/config/alerts.yaml
|
|
alerts:
|
|
- name: "High CPU Usage"
|
|
condition: "cpu_percent > 90"
|
|
severity: "WARNING"
|
|
cooldown: 300
|
|
|
|
- name: "High Memory Usage"
|
|
condition: "memory_percent > 90"
|
|
severity: "WARNING"
|
|
cooldown: 300
|
|
|
|
- name: "High Disk Usage"
|
|
condition: "disk_percent > 90"
|
|
severity: "CRITICAL"
|
|
cooldown: 60
|
|
|
|
- name: "Service Down"
|
|
condition: "service_status != 'running'"
|
|
severity: "CRITICAL"
|
|
cooldown: 0
|
|
|
|
- name: "Build Queue Full"
|
|
condition: "queue_size > 100"
|
|
severity: "WARNING"
|
|
cooldown: 600
|
|
```
|
|
|
|
## Conclusion
|
|
|
|
This deployment guide provides a comprehensive approach to deploying Debian Forge in production. Key points to remember:
|
|
|
|
1. **Security First**: Always configure firewalls, fail2ban, and SSL certificates
|
|
2. **Monitoring**: Implement comprehensive monitoring and alerting
|
|
3. **Backups**: Regular automated backups with tested recovery procedures
|
|
4. **Maintenance**: Scheduled maintenance windows for updates and cleanup
|
|
5. **Scaling**: Plan for both horizontal and vertical scaling from the start
|
|
6. **Documentation**: Keep deployment and maintenance procedures up to date
|
|
|
|
For additional support, refer to the troubleshooting section or create an issue in the project repository.
|