01562657fb
Some checks are pending
Checks / Spelling (push) Waiting to run
Checks / Python Linters (push) Waiting to run
Checks / Shell Linters (push) Waiting to run
Checks / 📦 Packit config lint (push) Waiting to run
Checks / 🔍 Check for valid snapshot urls (push) Waiting to run
Checks / 🔍 Check JSON files for formatting consistency (push) Waiting to run
Generate / Documentation (push) Waiting to run
Generate / Test Data (push) Waiting to run
Tests / Unittest (push) Waiting to run
Tests / Assembler test (legacy) (push) Waiting to run
Tests / Smoke run: unittest as normal user on default runner (push) Waiting to run
- Add comprehensive build lifecycle test script - Create build environment management system with isolation - Implement host health monitoring and resource tracking - Add automatic environment cleanup and reuse policies - Create OSBuild integration module for pipeline management - Fix attribute references in integration code - All components tested and working
338 lines
12 KiB
Python
338 lines
12 KiB
Python
#!/usr/bin/python3
|
|
"""
|
|
Debian Forge OSBuild Integration
|
|
|
|
Integrates modified OSBuild with the build orchestration system.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import json
|
|
import subprocess
|
|
import tempfile
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional, Any, Tuple
|
|
from build_orchestrator import BuildOrchestrator, BuildStatus
|
|
from build_environment import BuildEnvironmentManager
|
|
from artifact_manager import ArtifactManager
|
|
|
|
|
|
class OSBuildIntegration:
|
|
"""Integrates OSBuild with Debian Forge build orchestration"""
|
|
|
|
def __init__(self, osbuild_path: str = "python3 -m osbuild"):
|
|
self.osbuild_path = osbuild_path
|
|
self.orchestrator = BuildOrchestrator()
|
|
self.env_manager = BuildEnvironmentManager()
|
|
self.artifact_manager = ArtifactManager()
|
|
|
|
def submit_osbuild_pipeline(self, manifest_path: str, priority: int = 5,
|
|
resource_requirements: Optional[Dict[str, Any]] = None,
|
|
environment_id: Optional[str] = None) -> str:
|
|
"""Submit an OSBuild pipeline for execution"""
|
|
|
|
# Validate manifest
|
|
if not self._validate_manifest(manifest_path):
|
|
raise ValueError(f"Invalid manifest: {manifest_path}")
|
|
|
|
# Create build environment if specified
|
|
if environment_id:
|
|
if not self.env_manager.get_environment(environment_id):
|
|
env_path = self.env_manager.create_environment(environment_id)
|
|
print(f"Created build environment: {environment_id}")
|
|
|
|
# Submit build to orchestrator
|
|
build_id = self.orchestrator.submit_build(
|
|
manifest_path,
|
|
priority=priority,
|
|
resource_requirements=resource_requirements or {},
|
|
metadata={
|
|
"type": "osbuild_pipeline",
|
|
"environment_id": environment_id,
|
|
"manifest_path": manifest_path
|
|
}
|
|
)
|
|
|
|
print(f"Submitted OSBuild pipeline: {build_id}")
|
|
return build_id
|
|
|
|
def execute_pipeline(self, manifest_path: str, output_dir: str,
|
|
environment_id: Optional[str] = None) -> Tuple[bool, Optional[str]]:
|
|
"""Execute an OSBuild pipeline directly"""
|
|
|
|
# Create output directory
|
|
os.makedirs(output_dir, exist_ok=True)
|
|
|
|
# Set up environment if specified
|
|
env_vars = {}
|
|
if environment_id:
|
|
env = self.env_manager.get_environment(environment_id)
|
|
if env:
|
|
self.env_manager.use_environment(environment_id)
|
|
env_vars["OSBUILD_ENV_PATH"] = env.base_path
|
|
try:
|
|
# Execute OSBuild
|
|
result = self._run_osbuild(manifest_path, output_dir, env_vars)
|
|
return result
|
|
finally:
|
|
self.env_manager.release_environment(environment_id)
|
|
else:
|
|
return False, f"Environment {environment_id} not found"
|
|
else:
|
|
# Execute without specific environment
|
|
return self._run_osbuild(manifest_path, output_dir, env_vars)
|
|
|
|
def _run_osbuild(self, manifest_path: str, output_dir: str, env_vars: Dict[str, str]) -> Tuple[bool, Optional[str]]:
|
|
"""Run OSBuild command"""
|
|
|
|
# Build OSBuild command
|
|
cmd = [
|
|
"python3", "-m", "osbuild",
|
|
"--libdir", ".",
|
|
"--output-dir", output_dir,
|
|
manifest_path
|
|
]
|
|
|
|
print(f"Executing OSBuild: {' '.join(cmd)}")
|
|
|
|
try:
|
|
# Run OSBuild
|
|
result = subprocess.run(
|
|
cmd,
|
|
capture_output=True,
|
|
text=True,
|
|
cwd=os.getcwd(),
|
|
env={**os.environ, **env_vars}
|
|
)
|
|
|
|
if result.returncode == 0:
|
|
print("✅ OSBuild pipeline completed successfully")
|
|
return True, None
|
|
else:
|
|
error_msg = f"OSBuild failed with return code {result.returncode}"
|
|
if result.stderr:
|
|
error_msg += f"\nStderr: {result.stderr}"
|
|
return False, error_msg
|
|
|
|
except Exception as e:
|
|
error_msg = f"Failed to execute OSBuild: {str(e)}"
|
|
return False, error_msg
|
|
|
|
def _validate_manifest(self, manifest_path: str) -> bool:
|
|
"""Validate OSBuild manifest"""
|
|
try:
|
|
with open(manifest_path, 'r') as f:
|
|
manifest = json.load(f)
|
|
|
|
# Check basic structure
|
|
if "version" not in manifest:
|
|
print("❌ Manifest missing version")
|
|
return False
|
|
|
|
if "pipelines" not in manifest:
|
|
print("❌ Manifest missing pipelines")
|
|
return False
|
|
|
|
# Validate pipelines
|
|
for pipeline in manifest["pipelines"]:
|
|
if "name" not in pipeline:
|
|
print("❌ Pipeline missing name")
|
|
return False
|
|
|
|
if "runner" not in pipeline:
|
|
print("❌ Pipeline missing runner")
|
|
return False
|
|
|
|
if "stages" not in pipeline:
|
|
print("❌ Pipeline missing stages")
|
|
return False
|
|
|
|
# Validate stages
|
|
for stage in pipeline["stages"]:
|
|
if "name" not in stage:
|
|
print("❌ Stage missing name")
|
|
return False
|
|
|
|
print("✅ Manifest validation passed")
|
|
return True
|
|
|
|
except FileNotFoundError:
|
|
print(f"❌ Manifest file not found: {manifest_path}")
|
|
return False
|
|
except json.JSONDecodeError as e:
|
|
print(f"❌ Invalid JSON in manifest: {e}")
|
|
return False
|
|
except Exception as e:
|
|
print(f"❌ Manifest validation error: {e}")
|
|
return False
|
|
|
|
def get_pipeline_status(self, build_id: str) -> Optional[Dict[str, Any]]:
|
|
"""Get pipeline execution status"""
|
|
build_status = self.orchestrator.get_build_status(build_id)
|
|
if not build_status:
|
|
return None
|
|
|
|
# Get artifacts for this build
|
|
artifacts = self.artifact_manager.get_build_artifacts(build_id)
|
|
|
|
# Get environment info if available
|
|
environment_info = None
|
|
if build_status.metadata and "environment_id" in build_status.metadata:
|
|
env_id = build_status.metadata["environment_id"]
|
|
env = self.env_manager.get_environment(env_id)
|
|
if env:
|
|
environment_info = env.to_dict()
|
|
|
|
return {
|
|
"build_id": build_id,
|
|
"status": build_status.status.value,
|
|
"progress": build_status.progress,
|
|
"submitted_at": build_status.submitted_at.isoformat(),
|
|
"started_at": build_status.started_at.isoformat() if build_status.started_at else None,
|
|
"completed_at": build_status.completed_at.isoformat() if build_status.completed_at else None,
|
|
"error_message": build_status.error_message,
|
|
"artifacts": [a.to_dict() for a in artifacts],
|
|
"environment": environment_info,
|
|
"metadata": build_status.metadata
|
|
}
|
|
|
|
def list_pipelines(self) -> Dict[str, List[Dict[str, Any]]]:
|
|
"""List all pipeline builds"""
|
|
builds = self.orchestrator.list_builds()
|
|
|
|
result = {}
|
|
for status, build_list in builds.items():
|
|
result[status] = []
|
|
for build in build_list:
|
|
if build.metadata and build.metadata.get("type") == "osbuild_pipeline":
|
|
result[status].append({
|
|
"build_id": build.id,
|
|
"manifest_path": build.manifest_path,
|
|
"priority": build.priority,
|
|
"status": build.status.value,
|
|
"submitted_at": build.submitted_at.isoformat()
|
|
})
|
|
|
|
return result
|
|
|
|
def cancel_pipeline(self, build_id: str) -> bool:
|
|
"""Cancel a pipeline execution"""
|
|
return self.orchestrator.cancel_build(build_id)
|
|
|
|
def get_pipeline_logs(self, build_id: str) -> List[str]:
|
|
"""Get logs for a pipeline execution"""
|
|
return self.orchestrator.get_build_logs(build_id)
|
|
|
|
|
|
def create_test_debian_manifest() -> Dict[str, Any]:
|
|
"""Create a test Debian manifest for integration testing"""
|
|
return {
|
|
"version": "2",
|
|
"pipelines": [
|
|
{
|
|
"name": "debian-base-system",
|
|
"runner": "org.osbuild.linux",
|
|
"stages": [
|
|
{
|
|
"name": "org.osbuild.mkdir",
|
|
"options": {
|
|
"paths": ["/tmp/debian-test"]
|
|
}
|
|
},
|
|
{
|
|
"name": "org.osbuild.copy",
|
|
"options": {
|
|
"paths": [
|
|
{
|
|
"from": "test-debian-manifest.json",
|
|
"to": "/tmp/debian-test/manifest.json"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "org.osbuild.shell",
|
|
"options": {
|
|
"script": "echo 'Debian pipeline test completed' > /tmp/debian-test/status.txt"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
|
|
|
|
def test_osbuild_integration():
|
|
"""Test OSBuild integration functionality"""
|
|
print("Testing OSBuild Integration...")
|
|
|
|
integration = OSBuildIntegration()
|
|
|
|
# Create test manifest
|
|
test_manifest = create_test_debian_manifest()
|
|
manifest_path = "test-osbuild-integration.json"
|
|
|
|
with open(manifest_path, 'w') as f:
|
|
json.dump(test_manifest, f, indent=2)
|
|
|
|
try:
|
|
# Test manifest validation
|
|
print("\n1. Testing manifest validation...")
|
|
if integration._validate_manifest(manifest_path):
|
|
print("✅ Manifest validation passed")
|
|
else:
|
|
print("❌ Manifest validation failed")
|
|
return False
|
|
|
|
# Test pipeline submission
|
|
print("\n2. Testing pipeline submission...")
|
|
build_id = integration.submit_osbuild_pipeline(
|
|
manifest_path,
|
|
priority=5,
|
|
resource_requirements={"cpu_percent": 10, "memory_gb": 1, "storage_gb": 1}
|
|
)
|
|
print(f"✅ Pipeline submitted: {build_id}")
|
|
|
|
# Test pipeline status
|
|
print("\n3. Testing pipeline status...")
|
|
status = integration.get_pipeline_status(build_id)
|
|
if status:
|
|
print(f"✅ Pipeline status retrieved: {status['status']}")
|
|
else:
|
|
print("❌ Failed to get pipeline status")
|
|
return False
|
|
|
|
# Test pipeline listing
|
|
print("\n4. Testing pipeline listing...")
|
|
pipelines = integration.list_pipelines()
|
|
if pipelines and "pending" in pipelines and len(pipelines["pending"]) > 0:
|
|
print(f"✅ Pipeline listing working: {len(pipelines['pending'])} pending")
|
|
else:
|
|
print("❌ Pipeline listing failed")
|
|
return False
|
|
|
|
print("\n🎉 All OSBuild integration tests passed!")
|
|
return True
|
|
|
|
except Exception as e:
|
|
print(f"❌ OSBuild integration test failed: {e}")
|
|
return False
|
|
finally:
|
|
# Cleanup
|
|
if os.path.exists(manifest_path):
|
|
os.remove(manifest_path)
|
|
|
|
|
|
def main():
|
|
"""Main function for OSBuild integration testing"""
|
|
print("Debian Forge OSBuild Integration")
|
|
print("=" * 40)
|
|
|
|
if test_osbuild_integration():
|
|
return 0
|
|
else:
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|