Complete file structure reorganization for 1:1 osbuild compatibility

This commit is contained in:
robojerk 2025-08-26 15:38:59 -07:00
parent 61e7caaddb
commit 56f029cbc0
77 changed files with 5 additions and 956 deletions

4
.gitignore vendored
View file

@ -31,3 +31,7 @@ venv
debian-forge-docs/debos
debian-forge-docs/koji
# Embedded git repositories
docs/debian/debos/
docs/debian/koji/

View file

@ -1,43 +0,0 @@
{
"version": "2",
"pipelines": [
{
"name": "debian-base",
"build": "name:debian-base",
"stages": [
{
"name": "org.osbuild.debootstrap",
"options": {
"suite": "bookworm",
"mirror": "http://deb.debian.org/debian",
"arch": "amd64",
"variant": "minbase",
"apt_proxy": "http://192.168.1.101:3142"
}
},
{
"name": "org.osbuild.apt",
"options": {
"packages": ["systemd", "systemd-sysv", "dbus", "udev"],
"recommends": false,
"update": true,
"apt_proxy": "http://192.168.1.101:3142"
}
},
{
"name": "org.osbuild.ostree.commit",
"options": {
"repository": "debian-atomic",
"branch": "debian/bookworm",
"subject": "Debian Bookworm base system",
"metadata": {
"version": "12",
"variant": "minbase",
"arch": "amd64"
}
}
}
]
}
]
}

View file

@ -1,6 +0,0 @@
2025-08-22T18:43:44.007228 - Build build-000001: Build submitted - Priority: 5
2025-08-22T18:43:44.008134 - Build build-000001: Build submitted - Priority: 5
2025-08-22T18:43:45.009838 - Build build-000001: Build submitted - Priority: 5
2025-08-22T20:45:25.433439 - Build build-000001: Build submitted - Priority: 5
2025-08-22T20:45:45.179487 - Build build-000001: Build submitted - Priority: 5
2025-08-22T20:45:55.222544 - Build build-000001: Build submitted - Priority: 5

View file

@ -1,3 +0,0 @@
2025-08-22T18:43:44.007550 - Build build-000002: Build submitted - Priority: 3
2025-08-22T18:43:44.008287 - Build build-000002: Build submitted - Priority: 3
2025-08-22T18:43:45.010066 - Build build-000002: Build submitted - Priority: 4

View file

@ -1,2 +0,0 @@
2025-08-22T18:43:44.007774 - Build build-000003: Build submitted - Priority: 7
2025-08-22T18:43:45.010198 - Build build-000003: Build submitted - Priority: 3

View file

@ -1 +0,0 @@
2025-08-22T18:43:45.010403 - Build build-000004: Build submitted - Priority: 2

View file

@ -1 +0,0 @@
2025-08-22T18:43:45.010639 - Build build-000005: Build submitted - Priority: 1

Binary file not shown.

View file

@ -1,5 +0,0 @@
{
"repository": "http://deb.debian.org/debian",
"suite": "bookworm",
"last_sync": "2025-08-23T11:03:24.790058"
}

Binary file not shown.

View file

@ -1,40 +0,0 @@
{
"environments": [
{
"name": "bookworm-amd64",
"suite": "bookworm",
"architecture": "amd64",
"mirror": "http://deb.debian.org/debian",
"components": [
"main",
"contrib",
"non-free-firmware"
],
"extra_repositories": [],
"build_dependencies": [
"build-essential",
"devscripts",
"debhelper"
],
"enabled": true
},
{
"name": "sid-amd64",
"suite": "sid",
"architecture": "amd64",
"mirror": "http://deb.debian.org/debian",
"components": [
"main",
"contrib",
"non-free-firmware"
],
"extra_repositories": [],
"build_dependencies": [
"build-essential",
"devscripts",
"debhelper"
],
"enabled": true
}
]
}

View file

@ -1,99 +0,0 @@
{
"flavors": [
{
"name": "gnome",
"display_name": "GNOME",
"description": "Modern, intuitive desktop environment",
"packages": [
"task-gnome-desktop",
"gnome-core"
],
"dependencies": [
"gnome-session",
"gnome-shell",
"gdm3"
],
"variants": [
"bookworm",
"sid",
"testing"
],
"enabled": true,
"priority": 100
},
{
"name": "kde",
"display_name": "KDE Plasma",
"description": "Feature-rich, customizable desktop",
"packages": [
"task-kde-desktop",
"plasma-desktop"
],
"dependencies": [
"kde-plasma-desktop",
"sddm"
],
"variants": [
"bookworm",
"sid",
"testing"
],
"enabled": true,
"priority": 200
},
{
"name": "xfce",
"display_name": "Xfce",
"description": "Lightweight, fast desktop environment",
"packages": [
"task-xfce-desktop",
"xfce4"
],
"dependencies": [
"xfce4-session",
"lightdm"
],
"variants": [
"bookworm",
"sid",
"testing"
],
"enabled": true,
"priority": 300
},
{
"name": "mate",
"display_name": "MATE",
"description": "Traditional GNOME 2 desktop",
"packages": [
"task-mate-desktop",
"mate-desktop"
],
"dependencies": [
"mate-session-manager",
"lightdm"
],
"variants": [
"bookworm",
"sid",
"testing"
],
"enabled": true,
"priority": 400
},
{
"name": "minimal",
"display_name": "Minimal",
"description": "Minimal system without desktop",
"packages": [],
"dependencies": [],
"variants": [
"bookworm",
"sid",
"testing"
],
"enabled": true,
"priority": 500
}
]
}

View file

@ -1,80 +0,0 @@
{
"variants": [
{
"name": "bookworm",
"codename": "Bookworm",
"version": "12",
"status": "stable",
"release_date": "2023-06-10T00:00:00",
"end_of_life": "2026-06-10T00:00:00",
"architectures": [
"amd64",
"arm64",
"armel",
"armhf",
"i386",
"mips64el",
"mipsel",
"ppc64el",
"s390x"
],
"mirrors": [
"http://deb.debian.org/debian",
"http://security.debian.org/debian-security"
],
"security_support": true,
"updates_support": true,
"backports_support": true
},
{
"name": "sid",
"codename": "Sid",
"version": "unstable",
"status": "unstable",
"release_date": null,
"end_of_life": null,
"architectures": [
"amd64",
"arm64",
"armel",
"armhf",
"i386",
"mips64el",
"mipsel",
"ppc64el",
"s390x"
],
"mirrors": [
"http://deb.debian.org/debian"
],
"security_support": false,
"updates_support": false,
"backports_support": false
},
{
"name": "testing",
"codename": "Trixie",
"version": "13",
"status": "testing",
"release_date": null,
"end_of_life": null,
"architectures": [
"amd64",
"arm64",
"armel",
"armhf",
"i386",
"mips64el",
"mipsel",
"ppc64el",
"s390x"
],
"mirrors": [
"http://deb.debian.org/debian"
],
"security_support": false,
"updates_support": false,
"backports_support": false
}
]
}

View file

@ -1,147 +0,0 @@
# Debian Forge Deployment Guide
*Generated on: 2025-08-23 09:39:21*
## System Requirements
### Hardware Requirements
- **CPU**: 4 cores minimum, 8+ cores recommended
- **Memory**: 8GB minimum, 16GB+ recommended
- **Storage**: 50GB minimum, SSD recommended
- **Network**: 1Gbps minimum, 10Gbps recommended
### Software Requirements
- **Operating System**: Debian 12+ (Bookworm)
- **Kernel**: Linux 5.15+
- **Python**: 3.8+
- **Database**: SQLite (default) or PostgreSQL
## Installation
### Prerequisites
```bash
# Update system
sudo apt update && sudo apt upgrade -y
# Install required packages
sudo apt install -y python3 python3-pip python3-venv git
sudo apt install -y build-essential libssl-dev libffi-dev
# Install Go (for CLI and Composer)
sudo apt install -y golang-go
```
### Source Installation
```bash
# Clone repositories
git clone <debian-forge-repo>
git clone <debian-forge-cli-repo>
git clone <debian-forge-composer-repo>
# Set up Python environment
cd debian-forge
python3 -m venv venv
source venv/bin/activate
pip install -r requirements.txt
```
## Configuration
### Environment Configuration
```bash
# Create configuration file
cp config.example.yaml config.yaml
# Edit configuration
nano config.yaml
```
### Database Configuration
- **SQLite**: Default, no additional configuration needed
- **PostgreSQL**: Configure connection parameters
- **Database Initialization**: Run setup scripts
### Security Configuration
- **SSL/TLS**: Configure HTTPS certificates
- **Firewall**: Configure network security
- **User Authentication**: Set up initial admin user
## Service Configuration
### Systemd Service
```ini
[Unit]
Description=Debian Forge Service
After=network.target
[Service]
Type=simple
User=debian-forge
WorkingDirectory=/opt/debian-forge
ExecStart=/opt/debian-forge/venv/bin/python main.py
Restart=always
[Install]
WantedBy=multi-user.target
```
### Nginx Configuration
```nginx
server {
listen 80;
server_name debian-forge.example.com;
return 301 https://$server_name$request_uri;
}
server {
listen 443 ssl;
server_name debian-forge.example.com;
ssl_certificate /path/to/cert.pem;
ssl_certificate_key /path/to/key.pem;
location / {
proxy_pass http://127.0.0.1:8080;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
}
}
```
## Deployment Steps
### 1. System Preparation
- Verify system requirements
- Install prerequisites
- Configure system settings
### 2. Application Installation
- Clone source repositories
- Install dependencies
- Configure application
### 3. Service Setup
- Create system user
- Configure systemd service
- Set up reverse proxy
### 4. Initial Configuration
- Initialize database
- Create admin user
- Configure security settings
### 5. Testing and Validation
- Test service startup
- Verify web interface
- Test basic functionality
## Monitoring and Maintenance
### Health Checks
- **Service Status**: Check systemd service status
- **Web Interface**: Verify web interface accessibility
- **Database Health**: Check database connectivity
- **Performance Metrics**: Monitor system performance
### Backup Procedures
- **Configuration Files**: Backup configuration directory
- **Database**: Regular database backups
- **User Data**: Backup user uploads and generated images

View file

@ -1,125 +0,0 @@
# Debian Forge Maintenance Guide
*Generated on: 2025-08-23 09:39:21*
## Regular Maintenance Tasks
### Daily Tasks
- **System Health Check**: Verify all services are running
- **Performance Monitoring**: Review performance metrics
- **Error Log Review**: Check for new error messages
- **Backup Verification**: Ensure backups completed successfully
### Weekly Tasks
- **Performance Analysis**: Review weekly performance trends
- **Security Audit**: Run security vulnerability scans
- **Database Maintenance**: Clean up old data and optimize
- **Log Rotation**: Rotate and compress log files
### Monthly Tasks
- **System Updates**: Apply security and feature updates
- **Capacity Planning**: Review resource usage trends
- **Security Review**: Update security configurations
- **Documentation Review**: Update operational procedures
## Troubleshooting
### Common Issues and Solutions
#### Service Won't Start
1. Check systemd service status: `systemctl status debian-forge`
2. Review service logs: `journalctl -u debian-forge`
3. Verify configuration files
4. Check file permissions and ownership
#### Performance Issues
1. Monitor system resources: `htop`, `iotop`
2. Check database performance
3. Review build queue length
4. Analyze performance metrics
#### Authentication Problems
1. Verify user database integrity
2. Check password policies
3. Review authentication logs
4. Test user login process
## Backup and Recovery
### Backup Procedures
#### Configuration Backup
```bash
# Backup configuration directory
tar -czf config-backup-$(date +%Y%m%d).tar.gz config/
# Backup database files
cp *.db backup/
```
#### Database Backup
```bash
# SQLite backup
sqlite3 users.db .dump > backup/users-$(date +%Y%m%d).sql
# PostgreSQL backup
pg_dump debian_forge > backup/postgres-$(date +%Y%m%d).sql
```
### Recovery Procedures
#### Configuration Recovery
1. Stop the service: `systemctl stop debian-forge`
2. Restore configuration files
3. Verify file permissions
4. Start the service: `systemctl start debian-forge`
#### Database Recovery
1. Stop the service
2. Restore database from backup
3. Verify database integrity
4. Start the service
## Performance Optimization
### System Tuning
- **CPU Optimization**: Adjust process priorities
- **Memory Management**: Configure swap and memory limits
- **Disk I/O**: Optimize storage configuration
- **Network Tuning**: Optimize network parameters
### Application Tuning
- **Database Optimization**: Index optimization and query tuning
- **Build Optimization**: Parallel build processing
- **Cache Management**: Implement and tune caching
- **Resource Pooling**: Optimize resource allocation
## Security Maintenance
### Regular Security Tasks
- **Vulnerability Scanning**: Run security audits
- **Access Review**: Review user access and permissions
- **Security Updates**: Apply security patches
- **Configuration Review**: Review security settings
### Incident Response
1. **Detection**: Identify security incidents
2. **Assessment**: Evaluate incident severity
3. **Containment**: Limit incident impact
4. **Eradication**: Remove security threats
5. **Recovery**: Restore normal operations
6. **Lessons Learned**: Document and improve procedures
## Monitoring and Alerting
### Key Metrics to Monitor
- **System Resources**: CPU, memory, disk, network
- **Application Performance**: Response times, throughput
- **Build Queue**: Queue length, processing times
- **Security Events**: Authentication failures, access attempts
### Alerting Configuration
- **Threshold Alerts**: Resource usage alerts
- **Performance Alerts**: Response time and error rate alerts
- **Security Alerts**: Security incident notifications
- **Service Alerts**: Service availability notifications

View file

@ -1,36 +0,0 @@
# Debian Forge Documentation Index
*Generated on: 2025-08-23 09:39:21*
## Documentation Overview
This directory contains comprehensive documentation for the Debian Forge project.
## Available Documentation
### 📚 [Technical Documentation](TECHNICAL_DOCUMENTATION.md)
Comprehensive technical reference including architecture, API documentation, and system specifications.
### 📖 [User Guide](USER_GUIDE.md)
User-friendly guide for using Debian Forge, including getting started, blueprint creation, and troubleshooting.
### 🚀 [Deployment Guide](DEPLOYMENT_GUIDE.md)
Step-by-step deployment instructions, system requirements, and configuration details.
### 🔧 [Maintenance Guide](MAINTENANCE_GUIDE.md)
Operational procedures, troubleshooting guides, and maintenance best practices.
## Quick Start
1. **New Users**: Start with the [User Guide](USER_GUIDE.md)
2. **System Administrators**: Review the [Deployment Guide](DEPLOYMENT_GUIDE.md)
3. **Developers**: Reference the [Technical Documentation](TECHNICAL_DOCUMENTATION.md)
4. **Operations**: Use the [Maintenance Guide](MAINTENANCE_GUIDE.md)
## Documentation Maintenance
This documentation is automatically generated and should be updated when:
- New features are added to the system
- Configuration options change
- Security procedures are updated
- Deployment processes are modified

View file

@ -1,205 +0,0 @@
# Debian Forge Technical Documentation
*Generated on: 2025-08-23 09:39:21*
## Architecture Overview
Debian Forge is a fork of OSBuild, adapted for Debian with 1:1 compatibility goals.
### Core Components
- **debian-forge**: Core OSBuild fork with Debian-specific modifications
- **debian-forge-cli**: CLI tools for image building (fork of osbuild/image-builder-cli)
- **debian-forge-composer**: Web service and orchestration (fork of osbuild/osbuild-composer)
## Technical Specifications
### System Requirements
- **Operating System**: Debian 12+ or compatible
- **Python**: 3.8+
- **Database**: SQLite (default), PostgreSQL (optional)
- **Memory**: 4GB minimum, 8GB recommended
- **Storage**: 20GB minimum for base system
### Dependencies
- **Core**: Python standard library
- **Database**: sqlite3 (built-in)
- **Security**: OWASP Top 10 compliance
- **Monitoring**: Performance metrics collection
## API Documentation
### Core Modules
#### setup
- **File**: `setup.py`
- **Purpose**: Core functionality module
#### build_orchestrator
- **File**: `build_orchestrator.py`
- **Purpose**: Core functionality module
#### artifact_manager
- **File**: `artifact_manager.py`
- **Purpose**: Core functionality module
#### build_environment
- **File**: `build_environment.py`
- **Purpose**: Core functionality module
#### osbuild_integration
- **File**: `osbuild_integration.py`
- **Purpose**: Core functionality module
#### composer_client
- **File**: `composer_client.py`
- **Purpose**: Core functionality module
#### composer_status_monitor
- **File**: `composer_status_monitor.py`
- **Purpose**: Core functionality module
#### composer_build_history
- **File**: `composer_build_history.py`
- **Purpose**: Core functionality module
#### debian_repository_manager
- **File**: `debian_repository_manager.py`
- **Purpose**: Core functionality module
#### debian_package_resolver
- **File**: `debian_package_resolver.py`
- **Purpose**: Core functionality module
#### debian_atomic_blueprint_generator
- **File**: `debian_atomic_blueprint_generator.py`
- **Purpose**: Core functionality module
#### composer-build-history
- **File**: `composer-build-history.py`
- **Purpose**: Core functionality module
#### composer-status-monitor
- **File**: `composer-status-monitor.py`
- **Purpose**: Core functionality module
#### user_management
- **File**: `user_management.py`
- **Purpose**: Core functionality module
#### test_user_management
- **File**: `test_user_management.py`
- **Purpose**: Core functionality module
#### test_composer_auth
- **File**: `test_composer_auth.py`
- **Purpose**: Core functionality module
#### composer_client_simple
- **File**: `composer_client_simple.py`
- **Purpose**: Core functionality module
#### test_composer_simple
- **File**: `test_composer_simple.py`
- **Purpose**: Core functionality module
#### admin_interface
- **File**: `admin_interface.py`
- **Purpose**: Core functionality module
#### test_admin_interface
- **File**: `test_admin_interface.py`
- **Purpose**: Core functionality module
#### admin_interface_simple
- **File**: `admin_interface_simple.py`
- **Purpose**: System administration and configuration interface
#### test_admin_simple
- **File**: `test_admin_simple.py`
- **Purpose**: Core functionality module
#### cli_integration
- **File**: `cli_integration.py`
- **Purpose**: Integration with debian-forge-cli for command-line operations
#### composer_integration
- **File**: `composer_integration.py`
- **Purpose**: Core functionality module
#### test_unified_integration
- **File**: `test_unified_integration.py`
- **Purpose**: Core functionality module
#### composer_integration_simple
- **File**: `composer_integration_simple.py`
- **Purpose**: Integration with debian-forge-composer web service
#### unified_integration
- **File**: `unified_integration.py`
- **Purpose**: Unified interface for CLI and Composer integration
#### test_integration_simple
- **File**: `test_integration_simple.py`
- **Purpose**: Testing framework for integration modules
#### security_hardening
- **File**: `security_hardening.py`
- **Purpose**: Security testing, vulnerability assessment, and compliance
#### test_security_hardening
- **File**: `test_security_hardening.py`
- **Purpose**: Core functionality module
#### production_optimization
- **File**: `production_optimization.py`
- **Purpose**: Performance monitoring, load testing, and optimization
#### test_production_optimization
- **File**: `test_production_optimization.py`
- **Purpose**: Core functionality module
## Database Schema
### SQLite Databases
- **users.db**: User management and authentication
- **production_metrics.db**: Performance monitoring and load testing
- **security_vulnerabilities.db**: Security audit results
## Security Architecture
### Security Features
- **Authentication**: User management with role-based access control
- **Input Validation**: Comprehensive input sanitization
- **Data Protection**: Secure data handling and storage
- **File Permissions**: Secure file access controls
- **SQL Injection Protection**: Parameterized queries
- **XSS Protection**: Output sanitization
### Compliance
- **OWASP Top 10**: Web application security compliance
- **CIS Benchmarks**: Security configuration guidelines
- **Risk Assessment**: Automated vulnerability detection
## Performance Architecture
### Monitoring
- **Real-time Metrics**: CPU, memory, disk I/O, network I/O
- **Build Metrics**: Active builds, queue length, response times
- **Load Testing**: Multi-scenario performance testing
### Optimization
- **Bottleneck Detection**: Automated performance analysis
- **Recommendations**: Prioritized optimization suggestions
- **Historical Data**: Performance trend analysis
## Integration Architecture
### CLI Integration
- **debian-forge-cli**: Direct CLI command execution
- **Blueprint Management**: Debian-specific blueprint creation
- **Image Building**: CLI-based image generation
### Composer Integration
- **debian-forge-composer**: Web service integration
- **API Communication**: RESTful API interactions
- **Build Orchestration**: Centralized build management

View file

@ -1,96 +0,0 @@
# Debian Forge User Guide
*Generated on: 2025-08-23 09:39:21*
## Getting Started
### Installation
1. Clone the repository: `git clone <repository-url>`
2. Navigate to the project directory: `cd debian-forge`
3. Install dependencies: `pip install -r requirements.txt`
4. Initialize the system: `python3 -m debian_forge.init`
### Quick Start
1. **Start the system**: `python3 main.py`
2. **Access web interface**: Open browser to `http://localhost:8080`
3. **Create your first blueprint**: Use the web interface or CLI
4. **Build your first image**: Submit a build request
## User Interface
### Web Interface
- **Dashboard**: System overview and status
- **Blueprint Management**: Create and manage image blueprints
- **Build Management**: Monitor and control build processes
- **User Management**: Manage user accounts and permissions
### Command Line Interface
- **Image Building**: `debian-forge-cli build-image <blueprint>`
- **Blueprint Management**: `debian-forge-cli blueprint <command>`
- **System Status**: `debian-forge-cli status`
## Blueprint Creation
### Basic Blueprint Structure
```json
{
"name": "debian-server",
"description": "Debian server image",
"version": "1.0.0",
"packages": [
"openssh-server",
"nginx",
"postgresql"
],
"customizations": {
"user": {
"name": "admin",
"password": "secure_password"
}
}
}
```
### Debian-Specific Features
- **Package Management**: APT-based package installation
- **Repository Configuration**: Debian repository management
- **Debian Variants**: Support for different Debian flavors
## Image Building
### Build Process
1. **Blueprint Submission**: Submit blueprint to the system
2. **Build Queuing**: Build request enters the queue
3. **Build Execution**: System processes the build request
4. **Image Generation**: OSBuild stages create the final image
5. **Result Delivery**: Download or access the generated image
### Build Types
- **Raw Images**: Direct disk images for virtualization
- **Container Images**: Docker/OCI compatible images
- **Cloud Images**: Cloud provider specific formats
- **Live Images**: Bootable ISO images
## User Management
### User Roles
- **Administrator**: Full system access and control
- **Builder**: Can create and manage blueprints and builds
- **Viewer**: Read-only access to system information
### Authentication
- **User Registration**: Self-service user creation
- **Password Management**: Secure password policies
- **Session Management**: Secure session handling
## Troubleshooting
### Common Issues
- **Build Failures**: Check blueprint syntax and dependencies
- **Authentication Issues**: Verify user credentials and permissions
- **Performance Issues**: Monitor system resources and queue length
### Getting Help
- **System Logs**: Check application logs for errors
- **Documentation**: Refer to technical documentation
- **Community**: Join Debian Forge community forums

View file

@ -1,24 +0,0 @@
{
"name": "debian-atomic-base",
"version": "1.0.0",
"description": "Debian atomic blueprint for debian-atomic-base",
"packages": [
"bash",
"coreutils",
"systemd",
"apt",
"dpkg"
],
"modules": [],
"groups": [],
"customizations": {
"debian": {
"repositories": [
{
"name": "debian-main",
"baseurl": "http://deb.debian.org/debian"
}
]
}
}
}

View file

@ -1,22 +0,0 @@
{
"name": "test-cli",
"version": "1.0.0",
"description": "Debian atomic blueprint for test-cli",
"packages": [
"bash",
"coreutils"
],
"modules": [],
"groups": [],
"customizations": {
"debian": {
"repositories": [
{
"name": "debian-main",
"baseurl": "http://deb.debian.org/debian",
"enabled": true
}
]
}
}
}

View file

@ -1,21 +0,0 @@
{
"name": "valid-test",
"version": "1.0.0",
"description": "Debian atomic blueprint for valid-test",
"packages": [
"bash"
],
"modules": [],
"groups": [],
"customizations": {
"debian": {
"repositories": [
{
"name": "debian-main",
"baseurl": "http://deb.debian.org/debian",
"enabled": true
}
]
}
}
}

1
runners/org.osbuild.bazzite Symbolic link
View file

@ -0,0 +1 @@
org.osbuild.debian