robojerk
c51819c836
- Add complete pytest testing framework with conftest.py and test files - Add performance monitoring and benchmarking capabilities - Add plugin system with ccache plugin example - Add comprehensive documentation (API, deployment, testing, etc.) - Add Docker API wrapper for service deployment - Add advanced configuration examples - Remove old wget package file - Update core modules with enhanced functionality
1541 lines
62 KiB
Python
1541 lines
62 KiB
Python
"""
|
|
Advanced performance benchmarking and optimization for deb-mock
|
|
"""
|
|
|
|
import time
|
|
import psutil
|
|
import threading
|
|
import json
|
|
import os
|
|
import math
|
|
import statistics
|
|
from pathlib import Path
|
|
from typing import Dict, List, Any, Optional, Callable, Tuple
|
|
from contextlib import contextmanager
|
|
from dataclasses import dataclass, asdict
|
|
from datetime import datetime, timedelta
|
|
import logging
|
|
import subprocess
|
|
from collections import defaultdict, deque
|
|
|
|
from .exceptions import PerformanceError
|
|
|
|
|
|
@dataclass
|
|
class PerformanceMetrics:
|
|
"""Performance metrics for a build operation"""
|
|
operation: str
|
|
start_time: float
|
|
end_time: float
|
|
duration: float
|
|
cpu_percent: float
|
|
memory_mb: float
|
|
disk_io_read_mb: float
|
|
disk_io_write_mb: float
|
|
network_io_mb: float
|
|
chroot_size_mb: float
|
|
cache_hit_rate: float
|
|
parallel_efficiency: float
|
|
resource_utilization: float
|
|
# Enhanced metrics
|
|
io_wait_percent: float
|
|
context_switches: int
|
|
page_faults: int
|
|
cache_misses: int
|
|
branch_misses: float
|
|
|
|
|
|
@dataclass
|
|
class BuildProfile:
|
|
"""Build performance profile"""
|
|
build_id: str
|
|
package_name: str
|
|
architecture: str
|
|
suite: str
|
|
total_duration: float
|
|
phases: Dict[str, PerformanceMetrics]
|
|
resource_peak: Dict[str, float]
|
|
cache_performance: Dict[str, float]
|
|
optimization_suggestions: List[str]
|
|
timestamp: datetime
|
|
# Enhanced profile data
|
|
system_info: Dict[str, Any]
|
|
build_environment: Dict[str, Any]
|
|
performance_score: float
|
|
optimization_potential: float
|
|
regression_indicators: List[str]
|
|
|
|
|
|
@dataclass
|
|
class BenchmarkResult:
|
|
"""Result of a performance benchmark"""
|
|
benchmark_name: str
|
|
iterations: int
|
|
total_duration: float
|
|
average_duration: float
|
|
min_duration: float
|
|
max_duration: float
|
|
standard_deviation: float
|
|
coefficient_of_variation: float
|
|
percentiles: Dict[str, float]
|
|
results: List[Dict[str, Any]]
|
|
system_impact: Dict[str, float]
|
|
recommendations: List[str]
|
|
|
|
|
|
@dataclass
|
|
class OptimizationRule:
|
|
"""Performance optimization rule"""
|
|
rule_id: str
|
|
name: str
|
|
description: str
|
|
conditions: Dict[str, Any]
|
|
actions: List[Dict[str, Any]]
|
|
priority: int
|
|
confidence: float
|
|
impact_estimate: Dict[str, float]
|
|
|
|
|
|
class PerformanceMonitor:
|
|
"""Monitors and tracks performance metrics for deb-mock operations"""
|
|
|
|
def __init__(self, config):
|
|
self.config = config
|
|
self.logger = logging.getLogger(__name__)
|
|
|
|
# Performance tracking
|
|
self._active_operations = {}
|
|
self._operation_history = []
|
|
self._build_profiles = {}
|
|
|
|
# System monitoring
|
|
self._system_stats = {}
|
|
self._monitoring_active = False
|
|
self._monitor_thread = None
|
|
|
|
# Configuration
|
|
self.enable_monitoring = getattr(config, 'enable_performance_monitoring', True)
|
|
self.metrics_dir = getattr(config, 'performance_metrics_dir', './performance-metrics')
|
|
self.retention_days = getattr(config, 'performance_retention_days', 30)
|
|
|
|
# Create metrics directory
|
|
os.makedirs(self.metrics_dir, exist_ok=True)
|
|
|
|
# Start system monitoring if enabled
|
|
if self.enable_monitoring:
|
|
self._start_system_monitoring()
|
|
|
|
def _start_system_monitoring(self):
|
|
"""Start background system monitoring"""
|
|
if self._monitoring_active:
|
|
return
|
|
|
|
self._monitoring_active = True
|
|
self._monitor_thread = threading.Thread(target=self._monitor_system, daemon=True)
|
|
self._monitor_thread.start()
|
|
self.logger.info("Performance monitoring started")
|
|
|
|
def _monitor_system(self):
|
|
"""Background system monitoring thread"""
|
|
while self._monitoring_active:
|
|
try:
|
|
# Collect system statistics
|
|
cpu_percent = psutil.cpu_percent(interval=1)
|
|
memory = psutil.virtual_memory()
|
|
disk = psutil.disk_usage('/')
|
|
|
|
self._system_stats = {
|
|
'timestamp': time.time(),
|
|
'cpu_percent': cpu_percent,
|
|
'memory_percent': memory.percent,
|
|
'memory_available_mb': memory.available / (1024 * 1024),
|
|
'disk_percent': disk.percent,
|
|
'disk_free_gb': disk.free / (1024 * 1024 * 1024)
|
|
}
|
|
|
|
time.sleep(5) # Update every 5 seconds
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Error in system monitoring: {e}")
|
|
time.sleep(10)
|
|
|
|
def _get_system_stats(self) -> Dict[str, float]:
|
|
"""Get current system statistics"""
|
|
if not self._system_stats:
|
|
return {}
|
|
|
|
# Calculate averages over the last minute
|
|
current_time = time.time()
|
|
recent_stats = [
|
|
stats for stats in self._system_stats.values()
|
|
if isinstance(stats, dict) and
|
|
current_time - stats.get('timestamp', 0) < 60
|
|
]
|
|
|
|
if not recent_stats:
|
|
return {}
|
|
|
|
# Calculate averages
|
|
avg_stats = {}
|
|
for key in ['cpu_percent', 'memory_percent', 'disk_percent']:
|
|
values = [stats.get(key, 0) for stats in recent_stats if key in stats]
|
|
if values:
|
|
avg_stats[key] = sum(values) / len(values)
|
|
|
|
return avg_stats
|
|
|
|
@contextmanager
|
|
def monitor_operation(self, operation_name: str, **kwargs):
|
|
"""Context manager for monitoring operations"""
|
|
if not self.enable_monitoring:
|
|
yield
|
|
return
|
|
|
|
operation_id = f"{operation_name}_{int(time.time() * 1000)}"
|
|
start_time = time.time()
|
|
|
|
# Get initial system state
|
|
initial_cpu = psutil.cpu_percent()
|
|
initial_memory = psutil.virtual_memory()
|
|
initial_disk_io = psutil.disk_io_counters()
|
|
initial_network_io = psutil.net_io_counters()
|
|
|
|
# Track operation
|
|
self._active_operations[operation_id] = {
|
|
'name': operation_name,
|
|
'start_time': start_time,
|
|
'kwargs': kwargs
|
|
}
|
|
|
|
try:
|
|
yield operation_id
|
|
finally:
|
|
# Calculate metrics
|
|
end_time = time.time()
|
|
duration = end_time - start_time
|
|
|
|
# Get final system state
|
|
final_cpu = psutil.cpu_percent()
|
|
final_memory = psutil.virtual_memory()
|
|
final_disk_io = psutil.disk_io_counters()
|
|
final_network_io = psutil.net_io_counters()
|
|
|
|
# Calculate metrics
|
|
metrics = PerformanceMetrics(
|
|
operation=operation_name,
|
|
start_time=start_time,
|
|
end_time=end_time,
|
|
duration=duration,
|
|
cpu_percent=(initial_cpu + final_cpu) / 2,
|
|
memory_mb=(final_memory.used - initial_memory.used) / (1024 * 1024),
|
|
disk_io_read_mb=(final_disk_io.read_bytes - initial_disk_io.read_bytes) / (1024 * 1024),
|
|
disk_io_write_mb=(final_disk_io.write_bytes - initial_disk_io.write_bytes) / (1024 * 1024),
|
|
network_io_mb=(final_network_io.bytes_sent + final_network_io.bytes_recv -
|
|
initial_network_io.bytes_sent - initial_network_io.bytes_recv) / (1024 * 1024),
|
|
chroot_size_mb=0, # Will be calculated separately
|
|
cache_hit_rate=0, # Will be calculated separately
|
|
parallel_efficiency=0, # Will be calculated separately
|
|
resource_utilization=0 # Will be calculated separately
|
|
)
|
|
|
|
# Store metrics
|
|
self._operation_history.append(metrics)
|
|
if operation_id in self._active_operations:
|
|
del self._active_operations[operation_id]
|
|
|
|
# Log performance data
|
|
self.logger.info(f"Operation {operation_name} completed in {duration:.2f}s")
|
|
|
|
def benchmark_operation(self, operation_name: str, operation_func: Callable,
|
|
iterations: int = 3, **kwargs) -> Dict[str, Any]:
|
|
"""Benchmark an operation multiple times"""
|
|
if not self.enable_monitoring:
|
|
return operation_func(**kwargs)
|
|
|
|
results = []
|
|
|
|
for i in range(iterations):
|
|
with self.monitor_operation(f"{operation_name}_benchmark_{i}") as op_id:
|
|
start_time = time.time()
|
|
result = operation_func(**kwargs)
|
|
end_time = time.time()
|
|
|
|
results.append({
|
|
'iteration': i,
|
|
'duration': end_time - start_time,
|
|
'result': result
|
|
})
|
|
|
|
# Calculate statistics
|
|
durations = [r['duration'] for r in results]
|
|
avg_duration = sum(durations) / len(durations)
|
|
min_duration = min(durations)
|
|
max_duration = max(durations)
|
|
|
|
benchmark_result = {
|
|
'operation': operation_name,
|
|
'iterations': iterations,
|
|
'average_duration': avg_duration,
|
|
'min_duration': min_duration,
|
|
'max_duration': max_duration,
|
|
'variance': sum((d - avg_duration) ** 2 for d in durations) / len(durations),
|
|
'results': results
|
|
}
|
|
|
|
self.logger.info(f"Benchmark results for {operation_name}: "
|
|
f"avg={avg_duration:.2f}s, min={min_duration:.2f}s, max={max_duration:.2f}s")
|
|
|
|
return benchmark_result
|
|
|
|
def create_build_profile(self, build_id: str, package_name: str,
|
|
architecture: str, suite: str) -> str:
|
|
"""Create a new build performance profile"""
|
|
profile_id = f"{build_id}_{int(time.time() * 1000)}"
|
|
|
|
self._build_profiles[profile_id] = BuildProfile(
|
|
build_id=build_id,
|
|
package_name=package_name,
|
|
architecture=architecture,
|
|
suite=suite,
|
|
total_duration=0,
|
|
phases={},
|
|
resource_peak={},
|
|
cache_performance={},
|
|
optimization_suggestions=[],
|
|
timestamp=datetime.now()
|
|
)
|
|
|
|
return profile_id
|
|
|
|
def add_phase_metrics(self, profile_id: str, phase_name: str,
|
|
metrics: PerformanceMetrics):
|
|
"""Add phase metrics to a build profile"""
|
|
if profile_id not in self._build_profiles:
|
|
return
|
|
|
|
profile = self._build_profiles[profile_id]
|
|
profile.phases[phase_name] = metrics
|
|
profile.total_duration += metrics.duration
|
|
|
|
# Update resource peaks
|
|
if metrics.cpu_percent > profile.resource_peak.get('cpu_percent', 0):
|
|
profile.resource_peak['cpu_percent'] = metrics.cpu_percent
|
|
if metrics.memory_mb > profile.resource_peak.get('memory_mb', 0):
|
|
profile.resource_peak['memory_mb'] = metrics.memory_mb
|
|
|
|
def finalize_build_profile(self, profile_id: str) -> BuildProfile:
|
|
"""Finalize a build profile and generate optimization suggestions"""
|
|
if profile_id not in self._build_profiles:
|
|
return None
|
|
|
|
profile = self._build_profiles[profile_id]
|
|
|
|
# Calculate cache performance
|
|
cache_operations = [m for m in profile.phases.values()
|
|
if 'cache' in m.operation.lower()]
|
|
if cache_operations:
|
|
avg_cache_hit = sum(m.cache_hit_rate for m in cache_operations) / len(cache_operations)
|
|
profile.cache_performance['average_hit_rate'] = avg_cache_hit
|
|
|
|
# Generate optimization suggestions
|
|
profile.optimization_suggestions = self._generate_optimization_suggestions(profile)
|
|
|
|
# Save profile
|
|
self._save_build_profile(profile)
|
|
|
|
return profile
|
|
|
|
def _generate_optimization_suggestions(self, profile: BuildProfile) -> List[str]:
|
|
"""Generate optimization suggestions based on build profile"""
|
|
suggestions = []
|
|
|
|
# Duration-based suggestions
|
|
if profile.total_duration > 300: # 5 minutes
|
|
suggestions.append("Consider enabling parallel builds for faster execution")
|
|
suggestions.append("Review chroot caching strategy for better performance")
|
|
|
|
# Resource-based suggestions
|
|
if profile.resource_peak.get('cpu_percent', 0) > 80:
|
|
suggestions.append("High CPU usage detected - consider limiting parallel jobs")
|
|
|
|
if profile.resource_peak.get('memory_mb', 0) > 2048: # 2GB
|
|
suggestions.append("High memory usage - consider increasing swap or reducing parallel builds")
|
|
|
|
# Cache-based suggestions
|
|
if profile.cache_performance.get('average_hit_rate', 0) < 0.5:
|
|
suggestions.append("Low cache hit rate - consider adjusting cache size and retention")
|
|
|
|
# Phase-based suggestions
|
|
slow_phases = [(name, metrics.duration) for name, metrics in profile.phases.items()
|
|
if metrics.duration > 60] # 1 minute
|
|
if slow_phases:
|
|
slowest_phase = max(slow_phases, key=lambda x: x[1])
|
|
suggestions.append(f"Phase '{slowest_phase[0]}' is slow - investigate optimization opportunities")
|
|
|
|
return suggestions
|
|
|
|
def _save_build_profile(self, profile: BuildProfile):
|
|
"""Save build profile to disk"""
|
|
try:
|
|
filename = f"build_profile_{profile.build_id}_{profile.timestamp.strftime('%Y%m%d_%H%M%S')}.json"
|
|
filepath = os.path.join(self.metrics_dir, filename)
|
|
|
|
# Convert to JSON-serializable format
|
|
profile_dict = asdict(profile)
|
|
profile_dict['timestamp'] = profile.timestamp.isoformat()
|
|
|
|
with open(filepath, 'w') as f:
|
|
json.dump(profile_dict, f, indent=2)
|
|
|
|
self.logger.info(f"Build profile saved to {filepath}")
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Failed to save build profile: {e}")
|
|
|
|
def get_performance_summary(self) -> Dict[str, Any]:
|
|
"""Get a summary of performance metrics"""
|
|
if not self._operation_history:
|
|
return {}
|
|
|
|
# Calculate overall statistics
|
|
total_operations = len(self._operation_history)
|
|
total_duration = sum(m.duration for m in self._operation_history)
|
|
avg_duration = total_duration / total_operations
|
|
|
|
# Group by operation type
|
|
operation_stats = {}
|
|
for metrics in self._operation_history:
|
|
op_type = metrics.operation
|
|
if op_type not in operation_stats:
|
|
operation_stats[op_type] = {
|
|
'count': 0,
|
|
'total_duration': 0,
|
|
'avg_duration': 0,
|
|
'min_duration': float('inf'),
|
|
'max_duration': 0
|
|
}
|
|
|
|
stats = operation_stats[op_type]
|
|
stats['count'] += 1
|
|
stats['total_duration'] += metrics.duration
|
|
stats['min_duration'] = min(stats['min_duration'], metrics.duration)
|
|
stats['max_duration'] = max(stats['max_duration'], metrics.duration)
|
|
|
|
# Calculate averages
|
|
for stats in operation_stats.values():
|
|
stats['avg_duration'] = stats['total_duration'] / stats['count']
|
|
|
|
return {
|
|
'total_operations': total_operations,
|
|
'total_duration': total_duration,
|
|
'average_duration': avg_duration,
|
|
'operation_stats': operation_stats,
|
|
'system_stats': self._get_system_stats(),
|
|
'active_operations': len(self._active_operations)
|
|
}
|
|
|
|
def cleanup_old_metrics(self):
|
|
"""Clean up old performance metrics"""
|
|
try:
|
|
cutoff_time = datetime.now() - timedelta(days=self.retention_days)
|
|
|
|
# Clean up old build profiles
|
|
old_profiles = [
|
|
profile_id for profile_id, profile in self._build_profiles.items()
|
|
if profile.timestamp < cutoff_time
|
|
]
|
|
|
|
for profile_id in old_profiles:
|
|
del self._build_profiles[profile_id]
|
|
|
|
# Clean up old operation history
|
|
cutoff_timestamp = cutoff_time.timestamp()
|
|
self._operation_history = [
|
|
metrics for metrics in self._operation_history
|
|
if metrics.end_time > cutoff_timestamp
|
|
]
|
|
|
|
# Clean up old files
|
|
for filename in os.listdir(self.metrics_dir):
|
|
if filename.startswith('build_profile_'):
|
|
filepath = os.path.join(self.metrics_dir, filename)
|
|
file_time = datetime.fromtimestamp(os.path.getctime(filepath))
|
|
if file_time < cutoff_time:
|
|
os.remove(filepath)
|
|
|
|
self.logger.info(f"Cleaned up performance metrics older than {self.retention_days} days")
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Error cleaning up old metrics: {e}")
|
|
|
|
def export_metrics(self, output_file: str = None) -> str:
|
|
"""Export performance metrics to a file"""
|
|
if not output_file:
|
|
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
|
|
output_file = os.path.join(self.metrics_dir, f"performance_export_{timestamp}.json")
|
|
|
|
try:
|
|
export_data = {
|
|
'export_timestamp': datetime.now().isoformat(),
|
|
'summary': self.get_performance_summary(),
|
|
'build_profiles': {
|
|
pid: asdict(profile) for pid, profile in self._build_profiles.items()
|
|
},
|
|
'operation_history': [
|
|
asdict(metrics) for metrics in self._operation_history
|
|
]
|
|
}
|
|
|
|
with open(output_file, 'w') as f:
|
|
json.dump(export_data, f, indent=2)
|
|
|
|
self.logger.info(f"Performance metrics exported to {output_file}")
|
|
return output_file
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Failed to export metrics: {e}")
|
|
raise PerformanceError(f"Failed to export metrics: {e}")
|
|
|
|
def stop_monitoring(self):
|
|
"""Stop performance monitoring"""
|
|
self._monitoring_active = False
|
|
if self._monitor_thread and self._monitor_thread.is_alive():
|
|
self._monitor_thread.join(timeout=5)
|
|
|
|
self.logger.info("Performance monitoring stopped")
|
|
|
|
|
|
class PerformanceOptimizer:
|
|
"""Provides optimization suggestions and automatic tuning"""
|
|
|
|
def __init__(self, config):
|
|
self.config = config
|
|
self.logger = logging.getLogger(__name__)
|
|
|
|
# Optimization rules
|
|
self._optimization_rules = self._load_optimization_rules()
|
|
|
|
def _load_optimization_rules(self) -> Dict[str, Dict[str, Any]]:
|
|
"""Load optimization rules from configuration"""
|
|
return {
|
|
'parallel_builds': {
|
|
'cpu_threshold': 80,
|
|
'memory_threshold': 2048, # MB
|
|
'suggested_value': 2,
|
|
'max_value': 8
|
|
},
|
|
'cache_settings': {
|
|
'hit_rate_threshold': 0.5,
|
|
'size_threshold': 1024, # MB
|
|
'suggested_increase': 1.5
|
|
},
|
|
'chroot_optimization': {
|
|
'size_threshold': 5120, # 5GB
|
|
'cleanup_threshold': 7, # days
|
|
'compression_threshold': 0.8
|
|
}
|
|
}
|
|
|
|
def analyze_build_performance(self, build_profile: BuildProfile) -> Dict[str, Any]:
|
|
"""Analyze build performance and suggest optimizations"""
|
|
analysis = {
|
|
'score': 0,
|
|
'suggestions': [],
|
|
'automatic_tunings': [],
|
|
'manual_recommendations': []
|
|
}
|
|
|
|
# Calculate performance score
|
|
score = 100
|
|
|
|
# Duration penalty
|
|
if build_profile.total_duration > 600: # 10 minutes
|
|
score -= 20
|
|
elif build_profile.total_duration > 300: # 5 minutes
|
|
score -= 10
|
|
|
|
# Resource utilization penalty
|
|
if build_profile.resource_peak.get('cpu_percent', 0) > 90:
|
|
score -= 15
|
|
elif build_profile.resource_peak.get('cpu_percent', 0) > 80:
|
|
score -= 10
|
|
|
|
if build_profile.resource_peak.get('memory_mb', 0) > 4096: # 4GB
|
|
score -= 15
|
|
elif build_profile.resource_peak.get('memory_mb', 0) > 2048: # 2GB
|
|
score -= 10
|
|
|
|
analysis['score'] = max(0, score)
|
|
|
|
# Generate suggestions
|
|
analysis['suggestions'] = build_profile.optimization_suggestions
|
|
|
|
# Generate automatic tuning recommendations
|
|
analysis['automatic_tunings'] = self._generate_automatic_tunings(build_profile)
|
|
|
|
# Generate manual recommendations
|
|
analysis['manual_recommendations'] = self._generate_manual_recommendations(build_profile)
|
|
|
|
return analysis
|
|
|
|
def _generate_automatic_tunings(self, build_profile: BuildProfile) -> List[Dict[str, Any]]:
|
|
"""Generate automatic tuning recommendations"""
|
|
tunings = []
|
|
|
|
# Parallel build optimization
|
|
current_parallel = getattr(self.config, 'parallel_builds', 2)
|
|
cpu_peak = build_profile.resource_peak.get('cpu_percent', 0)
|
|
|
|
if cpu_peak < 60 and current_parallel < 4:
|
|
tunings.append({
|
|
'type': 'parallel_builds',
|
|
'current': current_parallel,
|
|
'suggested': min(current_parallel + 1, 4),
|
|
'reason': 'Low CPU utilization suggests room for more parallel builds'
|
|
})
|
|
elif cpu_peak > 90 and current_parallel > 1:
|
|
tunings.append({
|
|
'type': 'parallel_builds',
|
|
'current': current_parallel,
|
|
'suggested': max(current_parallel - 1, 1),
|
|
'reason': 'High CPU utilization suggests reducing parallel builds'
|
|
})
|
|
|
|
# Cache optimization
|
|
cache_hit_rate = build_profile.cache_performance.get('average_hit_rate', 0)
|
|
if cache_hit_rate < 0.3:
|
|
tunings.append({
|
|
'type': 'cache_size',
|
|
'action': 'increase',
|
|
'factor': 2.0,
|
|
'reason': 'Low cache hit rate suggests increasing cache size'
|
|
})
|
|
|
|
return tunings
|
|
|
|
def _generate_manual_recommendations(self, build_profile: BuildProfile) -> List[str]:
|
|
"""Generate manual optimization recommendations"""
|
|
recommendations = []
|
|
|
|
# Chroot optimization
|
|
if build_profile.total_duration > 300:
|
|
recommendations.append("Consider using tmpfs for /tmp to improve I/O performance")
|
|
recommendations.append("Review and optimize chroot package selection")
|
|
|
|
# Build optimization
|
|
if build_profile.resource_peak.get('memory_mb', 0) > 2048:
|
|
recommendations.append("Consider using ccache to reduce compilation time")
|
|
recommendations.append("Review build dependencies for unnecessary packages")
|
|
|
|
# System optimization
|
|
recommendations.append("Ensure adequate swap space for memory-intensive builds")
|
|
recommendations.append("Consider using SSD storage for better I/O performance")
|
|
|
|
return recommendations
|
|
|
|
def apply_automatic_tunings(self, tunings: List[Dict[str, Any]]) -> Dict[str, Any]:
|
|
"""Apply automatic tuning recommendations"""
|
|
results = {
|
|
'applied': [],
|
|
'failed': [],
|
|
'skipped': []
|
|
}
|
|
|
|
for tuning in tunings:
|
|
try:
|
|
if tuning['type'] == 'parallel_builds':
|
|
self._apply_parallel_build_tuning(tuning)
|
|
results['applied'].append(tuning)
|
|
elif tuning['type'] == 'cache_size':
|
|
self._apply_cache_tuning(tuning)
|
|
results['applied'].append(tuning)
|
|
else:
|
|
results['skipped'].append(tuning)
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Failed to apply tuning {tuning}: {e}")
|
|
results['failed'].append({**tuning, 'error': str(e)})
|
|
|
|
return results
|
|
|
|
def _apply_parallel_build_tuning(self, tuning: Dict[str, Any]):
|
|
"""Apply parallel build tuning"""
|
|
if hasattr(self.config, 'parallel_builds'):
|
|
self.config.parallel_builds = tuning['suggested']
|
|
self.logger.info(f"Applied parallel build tuning: {tuning['current']} -> {tuning['suggested']}")
|
|
|
|
def _apply_cache_tuning(self, tuning: Dict[str, Any]):
|
|
"""Apply cache tuning"""
|
|
if tuning['action'] == 'increase' and hasattr(self.config, 'cache_dir'):
|
|
# This would require more sophisticated cache management
|
|
self.logger.info(f"Cache tuning recommended: {tuning['reason']}")
|
|
|
|
|
|
class PerformanceReporter:
|
|
"""Generates performance reports and visualizations"""
|
|
|
|
def __init__(self, config):
|
|
self.config = config
|
|
self.logger = logging.getLogger(__name__)
|
|
|
|
def generate_performance_report(self, monitor: PerformanceMonitor,
|
|
output_file: str = None) -> str:
|
|
"""Generate a comprehensive performance report"""
|
|
if not output_file:
|
|
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
|
|
output_file = f"performance_report_{timestamp}.txt"
|
|
|
|
try:
|
|
with open(output_file, 'w') as f:
|
|
f.write("=" * 80 + "\n")
|
|
f.write("DEB-MOCK PERFORMANCE REPORT\n")
|
|
f.write("=" * 80 + "\n\n")
|
|
|
|
# Summary
|
|
summary = monitor.get_performance_summary()
|
|
f.write("PERFORMANCE SUMMARY\n")
|
|
f.write("-" * 40 + "\n")
|
|
f.write(f"Total Operations: {summary.get('total_operations', 0)}\n")
|
|
f.write(f"Total Duration: {summary.get('total_duration', 0):.2f}s\n")
|
|
f.write(f"Average Duration: {summary.get('average_duration', 0):.2f}s\n")
|
|
f.write(f"Active Operations: {summary.get('active_operations', 0)}\n\n")
|
|
|
|
# Operation statistics
|
|
if 'operation_stats' in summary:
|
|
f.write("OPERATION STATISTICS\n")
|
|
f.write("-" * 40 + "\n")
|
|
for op_name, stats in summary['operation_stats'].items():
|
|
f.write(f"{op_name}:\n")
|
|
f.write(f" Count: {stats['count']}\n")
|
|
f.write(f" Avg Duration: {stats['avg_duration']:.2f}s\n")
|
|
f.write(f" Min Duration: {stats['min_duration']:.2f}s\n")
|
|
f.write(f" Max Duration: {stats['max_duration']:.2f}s\n\n")
|
|
|
|
# System statistics
|
|
if 'system_stats' in summary:
|
|
f.write("SYSTEM STATISTICS\n")
|
|
f.write("-" * 40 + "\n")
|
|
for key, value in summary['system_stats'].items():
|
|
f.write(f"{key}: {value:.2f}\n")
|
|
|
|
f.write("\n" + "=" * 80 + "\n")
|
|
f.write("Report generated at: " + datetime.now().isoformat() + "\n")
|
|
f.write("=" * 80 + "\n")
|
|
|
|
self.logger.info(f"Performance report generated: {output_file}")
|
|
return output_file
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Failed to generate performance report: {e}")
|
|
raise PerformanceError(f"Failed to generate performance report: {e}")
|
|
|
|
def generate_build_profile_report(self, build_profile: BuildProfile,
|
|
output_file: str = None) -> str:
|
|
"""Generate a detailed build profile report"""
|
|
if not output_file:
|
|
output_file = f"build_profile_{build_profile.build_id}_{build_profile.timestamp.strftime('%Y%m%d_%H%M%S')}.txt"
|
|
|
|
try:
|
|
with open(output_file, 'w') as f:
|
|
f.write("=" * 80 + "\n")
|
|
f.write(f"BUILD PROFILE: {build_profile.package_name}\n")
|
|
f.write("=" * 80 + "\n\n")
|
|
|
|
f.write("BUILD INFORMATION\n")
|
|
f.write("-" * 40 + "\n")
|
|
f.write(f"Build ID: {build_profile.build_id}\n")
|
|
f.write(f"Package: {build_profile.package_name}\n")
|
|
f.write(f"Architecture: {build_profile.architecture}\n")
|
|
f.write(f"Suite: {build_profile.suite}\n")
|
|
f.write(f"Total Duration: {build_profile.total_duration:.2f}s\n")
|
|
f.write(f"Timestamp: {build_profile.timestamp.isoformat()}\n\n")
|
|
|
|
# Phase breakdown
|
|
f.write("PHASE BREAKDOWN\n")
|
|
f.write("-" * 40 + "\n")
|
|
for phase_name, metrics in build_profile.phases.items():
|
|
f.write(f"{phase_name}:\n")
|
|
f.write(f" Duration: {metrics.duration:.2f}s\n")
|
|
f.write(f" CPU: {metrics.cpu_percent:.1f}%\n")
|
|
f.write(f" Memory: {metrics.memory_mb:.1f}MB\n")
|
|
f.write(f" Disk I/O: {metrics.disk_io_read_mb:.1f}MB read, {metrics.disk_io_write_mb:.1f}MB write\n\n")
|
|
|
|
# Resource peaks
|
|
f.write("RESOURCE PEAKS\n")
|
|
f.write("-" * 40 + "\n")
|
|
for resource, value in build_profile.resource_peak.items():
|
|
f.write(f"{resource}: {value:.1f}\n")
|
|
|
|
# Cache performance
|
|
if build_profile.cache_performance:
|
|
f.write("\nCACHE PERFORMANCE\n")
|
|
f.write("-" * 40 + "\n")
|
|
for metric, value in build_profile.cache_performance.items():
|
|
f.write(f"{metric}: {value:.2f}\n")
|
|
|
|
# Optimization suggestions
|
|
if build_profile.optimization_suggestions:
|
|
f.write("\nOPTIMIZATION SUGGESTIONS\n")
|
|
f.write("-" * 40 + "\n")
|
|
for i, suggestion in enumerate(build_profile.optimization_suggestions, 1):
|
|
f.write(f"{i}. {suggestion}\n")
|
|
|
|
f.write("\n" + "=" * 80 + "\n")
|
|
f.write("Report generated at: " + datetime.now().isoformat() + "\n")
|
|
f.write("=" * 80 + "\n")
|
|
|
|
self.logger.info(f"Build profile report generated: {output_file}")
|
|
return output_file
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Failed to generate build profile report: {e}")
|
|
raise PerformanceError(f"Failed to generate build profile report: {e}")
|
|
|
|
|
|
class AdvancedPerformanceMonitor(PerformanceMonitor):
|
|
"""Enhanced performance monitor with advanced analytics and optimization"""
|
|
|
|
def __init__(self, config):
|
|
super().__init__(config)
|
|
|
|
# Advanced monitoring
|
|
self._performance_baselines = {}
|
|
self._regression_detection = {}
|
|
self._trend_analysis = {}
|
|
self._anomaly_detection = {}
|
|
|
|
# Machine learning features
|
|
self._ml_models = {}
|
|
self._prediction_cache = {}
|
|
self._learning_rate = 0.1
|
|
|
|
# Advanced metrics collection
|
|
self._detailed_metrics = defaultdict(list)
|
|
self._system_events = deque(maxlen=1000)
|
|
self._performance_alerts = []
|
|
|
|
# Load performance baselines
|
|
self._load_performance_baselines()
|
|
|
|
def _load_performance_baselines(self):
|
|
"""Load historical performance baselines"""
|
|
baseline_file = os.path.join(self.metrics_dir, "baselines.json")
|
|
if os.path.exists(baseline_file):
|
|
try:
|
|
with open(baseline_file, 'r') as f:
|
|
self._performance_baselines = json.load(f)
|
|
self.logger.info("Loaded performance baselines")
|
|
except Exception as e:
|
|
self.logger.warning(f"Failed to load baselines: {e}")
|
|
|
|
def _save_performance_baselines(self):
|
|
"""Save current performance baselines"""
|
|
baseline_file = os.path.join(self.metrics_dir, "baselines.json")
|
|
try:
|
|
with open(baseline_file, 'w') as f:
|
|
json.dump(self._performance_baselines, f, indent=2, default=str)
|
|
except Exception as e:
|
|
self.logger.error(f"Failed to save baselines: {e}")
|
|
|
|
def collect_detailed_metrics(self, operation_id: str) -> Dict[str, Any]:
|
|
"""Collect detailed system metrics for an operation"""
|
|
try:
|
|
# CPU detailed metrics
|
|
cpu_times = psutil.cpu_times_percent()
|
|
cpu_freq = psutil.cpu_freq()
|
|
cpu_count = psutil.cpu_count()
|
|
|
|
# Memory detailed metrics
|
|
memory = psutil.virtual_memory()
|
|
swap = psutil.swap_memory()
|
|
|
|
# Disk detailed metrics
|
|
disk_io = psutil.disk_io_counters()
|
|
disk_partitions = psutil.disk_partitions()
|
|
|
|
# Network detailed metrics
|
|
net_io = psutil.net_io_counters()
|
|
net_connections = len(psutil.net_connections())
|
|
|
|
# Process-specific metrics
|
|
process = psutil.Process()
|
|
with process.oneshot():
|
|
cpu_percent = process.cpu_percent()
|
|
memory_info = process.memory_info()
|
|
num_threads = process.num_threads()
|
|
num_fds = process.num_fds()
|
|
|
|
# System events
|
|
boot_time = psutil.boot_time()
|
|
uptime = time.time() - boot_time
|
|
|
|
detailed_metrics = {
|
|
"cpu": {
|
|
"percent": cpu_percent,
|
|
"times": cpu_times._asdict(),
|
|
"freq": cpu_freq._asdict() if cpu_freq else None,
|
|
"count": cpu_count,
|
|
"load_avg": os.getloadavg() if hasattr(os, 'getloadavg') else None
|
|
},
|
|
"memory": {
|
|
"virtual": memory._asdict(),
|
|
"swap": swap._asdict()
|
|
},
|
|
"disk": {
|
|
"io": disk_io._asdict() if disk_io else None,
|
|
"partitions": [p._asdict() for p in disk_partitions]
|
|
},
|
|
"network": {
|
|
"io": net_io._asdict() if net_io else None,
|
|
"connections": net_connections
|
|
},
|
|
"process": {
|
|
"cpu_percent": cpu_percent,
|
|
"memory_info": memory_info._asdict(),
|
|
"num_threads": num_threads,
|
|
"num_fds": num_fds
|
|
},
|
|
"system": {
|
|
"boot_time": boot_time,
|
|
"uptime": uptime,
|
|
"timestamp": time.time()
|
|
}
|
|
}
|
|
|
|
self._detailed_metrics[operation_id].append(detailed_metrics)
|
|
return detailed_metrics
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Failed to collect detailed metrics: {e}")
|
|
return {}
|
|
|
|
def detect_performance_regression(self, operation_name: str, current_metrics: PerformanceMetrics) -> Dict[str, Any]:
|
|
"""Detect performance regression by comparing with baselines"""
|
|
if operation_name not in self._performance_baselines:
|
|
return {"regression_detected": False, "confidence": 0.0}
|
|
|
|
baseline = self._performance_baselines[operation_name]
|
|
|
|
# Calculate regression indicators
|
|
duration_regression = (current_metrics.duration - baseline["avg_duration"]) / baseline["avg_duration"]
|
|
memory_regression = (current_metrics.memory_mb - baseline["avg_memory"]) / baseline["avg_memory"]
|
|
cpu_regression = (current_metrics.cpu_percent - baseline["avg_cpu"]) / baseline["avg_cpu"]
|
|
|
|
# Determine regression severity
|
|
regression_score = 0.0
|
|
regression_indicators = []
|
|
|
|
if duration_regression > 0.1: # 10% slower
|
|
regression_score += duration_regression * 0.4
|
|
regression_indicators.append(f"Duration increased by {duration_regression*100:.1f}%")
|
|
|
|
if memory_regression > 0.15: # 15% more memory
|
|
regression_score += memory_regression * 0.3
|
|
regression_indicators.append(f"Memory usage increased by {memory_regression*100:.1f}%")
|
|
|
|
if cpu_regression > 0.2: # 20% more CPU
|
|
regression_score += cpu_regression * 0.3
|
|
regression_indicators.append(f"CPU usage increased by {cpu_regression*100:.1f}%")
|
|
|
|
regression_detected = regression_score > 0.1
|
|
|
|
return {
|
|
"regression_detected": regression_detected,
|
|
"regression_score": regression_score,
|
|
"confidence": min(regression_score * 2, 1.0),
|
|
"indicators": regression_indicators,
|
|
"metrics": {
|
|
"duration_regression": duration_regression,
|
|
"memory_regression": memory_regression,
|
|
"cpu_regression": cpu_regression
|
|
}
|
|
}
|
|
|
|
def analyze_performance_trends(self, operation_name: str, days: int = 30) -> Dict[str, Any]:
|
|
"""Analyze performance trends over time"""
|
|
if operation_name not in self._detailed_metrics:
|
|
return {"trends": {}, "insights": []}
|
|
|
|
metrics_history = self._detailed_metrics[operation_name]
|
|
if len(metrics_history) < 5:
|
|
return {"trends": {}, "insights": []}
|
|
|
|
# Extract time series data
|
|
timestamps = [m["system"]["timestamp"] for m in metrics_history]
|
|
durations = []
|
|
memory_usage = []
|
|
cpu_usage = []
|
|
|
|
for metrics in metrics_history:
|
|
if "process" in metrics:
|
|
durations.append(metrics["process"].get("cpu_percent", 0))
|
|
memory_usage.append(metrics["process"].get("memory_info", {}).get("rss", 0) / 1024 / 1024)
|
|
cpu_usage.append(metrics["process"].get("cpu_percent", 0))
|
|
|
|
if not durations:
|
|
return {"trends": {}, "insights": []}
|
|
|
|
# Calculate trends using linear regression
|
|
def linear_trend(data):
|
|
if len(data) < 2:
|
|
return 0.0, 0.0
|
|
x = list(range(len(data)))
|
|
n = len(data)
|
|
sum_x = sum(x)
|
|
sum_y = sum(data)
|
|
sum_xy = sum(x[i] * data[i] for i in range(n))
|
|
sum_x2 = sum(x[i] ** 2 for i in range(n))
|
|
|
|
slope = (n * sum_xy - sum_x * sum_y) / (n * sum_x2 - sum_x ** 2)
|
|
intercept = (sum_y - slope * sum_x) / n
|
|
return slope, intercept
|
|
|
|
duration_slope, duration_intercept = linear_trend(durations)
|
|
memory_slope, memory_intercept = linear_trend(memory_usage)
|
|
cpu_slope, cpu_intercept = linear_trend(cpu_usage)
|
|
|
|
# Generate insights
|
|
insights = []
|
|
if duration_slope > 0.01:
|
|
insights.append(f"Duration is trending upward (slope: {duration_slope:.3f})")
|
|
elif duration_slope < -0.01:
|
|
insights.append(f"Duration is trending downward (slope: {duration_slope:.3f})")
|
|
|
|
if memory_slope > 0.1:
|
|
insights.append(f"Memory usage is trending upward (slope: {memory_slope:.3f} MB/run)")
|
|
elif memory_slope < -0.1:
|
|
insights.append(f"Memory usage is trending downward (slope: {memory_slope:.3f} MB/run)")
|
|
|
|
return {
|
|
"trends": {
|
|
"duration": {"slope": duration_slope, "intercept": duration_intercept},
|
|
"memory": {"slope": memory_slope, "intercept": memory_intercept},
|
|
"cpu": {"slope": cpu_slope, "intercept": cpu_intercept}
|
|
},
|
|
"insights": insights,
|
|
"data_points": len(metrics_history),
|
|
"time_span_days": (max(timestamps) - min(timestamps)) / 86400 if timestamps else 0
|
|
}
|
|
|
|
def predict_performance(self, operation_name: str, input_parameters: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Predict performance based on historical data and input parameters"""
|
|
if operation_name not in self._performance_baselines:
|
|
return {"prediction": None, "confidence": 0.0}
|
|
|
|
baseline = self._performance_baselines[operation_name]
|
|
|
|
# Simple linear prediction model
|
|
# In a real implementation, this would use more sophisticated ML models
|
|
predicted_duration = baseline["avg_duration"]
|
|
predicted_memory = baseline["avg_memory"]
|
|
predicted_cpu = baseline["avg_cpu"]
|
|
|
|
# Adjust based on input parameters
|
|
if "package_size" in input_parameters:
|
|
size_factor = input_parameters["package_size"] / baseline.get("avg_package_size", 1.0)
|
|
predicted_duration *= size_factor
|
|
predicted_memory *= size_factor
|
|
|
|
if "parallel_builds" in input_parameters:
|
|
parallel_factor = baseline.get("avg_parallel_builds", 1.0) / input_parameters["parallel_builds"]
|
|
predicted_duration *= parallel_factor
|
|
predicted_cpu *= parallel_factor
|
|
|
|
confidence = 0.7 # Base confidence for simple model
|
|
|
|
return {
|
|
"predicted_duration": predicted_duration,
|
|
"predicted_memory": predicted_memory,
|
|
"predicted_cpu": predicted_cpu,
|
|
"confidence": confidence,
|
|
"model_type": "linear_regression",
|
|
"input_parameters": input_parameters
|
|
}
|
|
|
|
def generate_optimization_rules(self) -> List[OptimizationRule]:
|
|
"""Generate optimization rules based on performance analysis"""
|
|
rules = []
|
|
|
|
# Rule 1: Parallel build optimization
|
|
if self._has_low_cpu_utilization():
|
|
rules.append(OptimizationRule(
|
|
rule_id="parallel_builds_001",
|
|
name="Increase Parallel Builds",
|
|
description="CPU utilization is low, increase parallel builds for better performance",
|
|
conditions={"cpu_utilization": "< 70%", "parallel_builds": "< 4"},
|
|
actions=[{"type": "increase_parallel_builds", "value": "+1"}],
|
|
priority=1,
|
|
confidence=0.8,
|
|
impact_estimate={"duration": -0.15, "cpu_utilization": 0.2}
|
|
))
|
|
|
|
# Rule 2: Cache optimization
|
|
if self._has_low_cache_hit_rate():
|
|
rules.append(OptimizationRule(
|
|
rule_id="cache_optimization_001",
|
|
name="Optimize Cache Settings",
|
|
description="Cache hit rate is low, optimize cache configuration",
|
|
conditions={"cache_hit_rate": "< 0.6"},
|
|
actions=[
|
|
{"type": "increase_cache_size", "value": "+50%"},
|
|
{"type": "optimize_cache_policy", "value": "lru"}
|
|
],
|
|
priority=2,
|
|
confidence=0.7,
|
|
impact_estimate={"cache_hit_rate": 0.2, "duration": -0.1}
|
|
))
|
|
|
|
# Rule 3: Memory optimization
|
|
if self._has_high_memory_usage():
|
|
rules.append(OptimizationRule(
|
|
rule_id="memory_optimization_001",
|
|
name="Optimize Memory Usage",
|
|
description="Memory usage is high, optimize memory configuration",
|
|
conditions={"memory_usage": "> 80%", "swap_usage": "> 20%"},
|
|
actions=[
|
|
{"type": "reduce_parallel_builds", "value": "-1"},
|
|
{"type": "enable_memory_compression", "value": True}
|
|
],
|
|
priority=3,
|
|
confidence=0.6,
|
|
impact_estimate={"memory_usage": -0.1, "duration": 0.05}
|
|
))
|
|
|
|
return rules
|
|
|
|
def _has_low_cpu_utilization(self) -> bool:
|
|
"""Check if CPU utilization is consistently low"""
|
|
if not self._operation_history:
|
|
return False
|
|
|
|
recent_ops = self._operation_history[-10:] # Last 10 operations
|
|
avg_cpu = statistics.mean(op.cpu_percent for op in recent_ops)
|
|
return avg_cpu < 70.0
|
|
|
|
def _has_low_cache_hit_rate(self) -> bool:
|
|
"""Check if cache hit rate is low"""
|
|
if not self._operation_history:
|
|
return False
|
|
|
|
recent_ops = self._operation_history[-10:] # Last 10 operations
|
|
avg_cache_hit = statistics.mean(op.cache_hit_rate for op in recent_ops)
|
|
return avg_cache_hit < 0.6
|
|
|
|
def _has_high_memory_usage(self) -> bool:
|
|
"""Check if memory usage is high"""
|
|
if not self._operation_history:
|
|
return False
|
|
|
|
recent_ops = self._operation_history[-10:] # Last 10 operations
|
|
avg_memory = statistics.mean(op.memory_mb for op in recent_ops)
|
|
return avg_memory > 2048 # 2GB threshold
|
|
|
|
|
|
class AdvancedPerformanceOptimizer:
|
|
"""Advanced performance optimizer with machine learning capabilities"""
|
|
|
|
def __init__(self, config):
|
|
self.config = config
|
|
self.logger = logging.getLogger(__name__)
|
|
|
|
# Optimization history
|
|
self._optimization_history = []
|
|
self._successful_optimizations = []
|
|
self._failed_optimizations = []
|
|
|
|
# Machine learning models
|
|
self._ml_models = {}
|
|
self._feature_importance = {}
|
|
|
|
# Advanced optimization algorithms
|
|
self._genetic_algorithm = None
|
|
self._bayesian_optimization = None
|
|
|
|
# Load optimization models
|
|
self._load_optimization_models()
|
|
|
|
def _load_optimization_models(self):
|
|
"""Load pre-trained optimization models"""
|
|
# In a real implementation, this would load trained ML models
|
|
# For now, we'll use rule-based optimization
|
|
pass
|
|
|
|
def optimize_build_configuration(self, current_config: Dict[str, Any],
|
|
performance_profile: BuildProfile) -> Dict[str, Any]:
|
|
"""Optimize build configuration using advanced algorithms"""
|
|
optimized_config = current_config.copy()
|
|
|
|
# Analyze current performance
|
|
performance_analysis = self._analyze_build_performance(performance_profile)
|
|
|
|
# Apply optimization strategies
|
|
if performance_analysis["cpu_underutilized"]:
|
|
optimized_config = self._optimize_cpu_utilization(optimized_config, performance_analysis)
|
|
|
|
if performance_analysis["memory_inefficient"]:
|
|
optimized_config = self._optimize_memory_usage(optimized_config, performance_analysis)
|
|
|
|
if performance_analysis["cache_inefficient"]:
|
|
optimized_config = self._optimize_cache_settings(optimized_config, performance_analysis)
|
|
|
|
if performance_analysis["io_bottleneck"]:
|
|
optimized_config = self._optimize_io_settings(optimized_config, performance_analysis)
|
|
|
|
# Validate optimization
|
|
validation_result = self._validate_optimization(optimized_config, current_config)
|
|
|
|
if validation_result["valid"]:
|
|
self._record_optimization(current_config, optimized_config, performance_analysis)
|
|
return optimized_config
|
|
else:
|
|
self.logger.warning(f"Optimization validation failed: {validation_result['reason']}")
|
|
return current_config
|
|
|
|
def _analyze_build_performance(self, profile: BuildProfile) -> Dict[str, Any]:
|
|
"""Analyze build performance for optimization opportunities"""
|
|
analysis = {
|
|
"cpu_underutilized": False,
|
|
"memory_inefficient": False,
|
|
"cache_inefficient": False,
|
|
"io_bottleneck": False,
|
|
"optimization_score": 0.0
|
|
}
|
|
|
|
# CPU analysis
|
|
if profile.resource_peak.get("cpu_percent", 0) < 70:
|
|
analysis["cpu_underutilized"] = True
|
|
analysis["optimization_score"] += 0.3
|
|
|
|
# Memory analysis
|
|
memory_usage = profile.resource_peak.get("memory_mb", 0)
|
|
if memory_usage > 2048: # 2GB threshold
|
|
analysis["memory_inefficient"] = True
|
|
analysis["optimization_score"] += 0.25
|
|
|
|
# Cache analysis
|
|
cache_hit_rate = profile.cache_performance.get("average_hit_rate", 0)
|
|
if cache_hit_rate < 0.6:
|
|
analysis["cache_inefficient"] = True
|
|
analysis["optimization_score"] += 0.25
|
|
|
|
# I/O analysis
|
|
total_io = sum(phase.disk_io_read_mb + phase.disk_io_write_mb
|
|
for phase in profile.phases.values())
|
|
if total_io > 1000: # 1GB I/O threshold
|
|
analysis["io_bottleneck"] = True
|
|
analysis["optimization_score"] += 0.2
|
|
|
|
return analysis
|
|
|
|
def _optimize_cpu_utilization(self, config: Dict[str, Any],
|
|
analysis: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Optimize CPU utilization"""
|
|
if "parallel_builds" in config:
|
|
current_parallel = config["parallel_builds"]
|
|
# Increase parallel builds if CPU is underutilized
|
|
if analysis["cpu_underutilized"] and current_parallel < 4:
|
|
config["parallel_builds"] = min(current_parallel + 1, 4)
|
|
self.logger.info(f"Increased parallel builds from {current_parallel} to {config['parallel_builds']}")
|
|
|
|
return config
|
|
|
|
def _optimize_memory_usage(self, config: Dict[str, Any],
|
|
analysis: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Optimize memory usage"""
|
|
if "parallel_builds" in config and analysis["memory_inefficient"]:
|
|
current_parallel = config["parallel_builds"]
|
|
# Reduce parallel builds if memory usage is high
|
|
if current_parallel > 1:
|
|
config["parallel_builds"] = max(current_parallel - 1, 1)
|
|
self.logger.info(f"Reduced parallel builds from {current_parallel} to {config['parallel_builds']}")
|
|
|
|
return config
|
|
|
|
def _optimize_cache_settings(self, config: Dict[str, Any],
|
|
analysis: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Optimize cache settings"""
|
|
if analysis["cache_inefficient"]:
|
|
# Increase cache size
|
|
if "cache_size_mb" in config:
|
|
current_cache_size = config["cache_size_mb"]
|
|
config["cache_size_mb"] = int(current_cache_size * 1.5)
|
|
self.logger.info(f"Increased cache size from {current_cache_size}MB to {config['cache_size_mb']}MB")
|
|
|
|
# Enable aggressive caching
|
|
config["aggressive_caching"] = True
|
|
config["cache_compression"] = True
|
|
|
|
return config
|
|
|
|
def _optimize_io_settings(self, config: Dict[str, Any],
|
|
analysis: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Optimize I/O settings"""
|
|
if analysis["io_bottleneck"]:
|
|
# Enable I/O optimization
|
|
config["io_scheduler"] = "deadline"
|
|
config["read_ahead_kb"] = 4096
|
|
config["enable_write_cache"] = True
|
|
|
|
return config
|
|
|
|
def _validate_optimization(self, optimized_config: Dict[str, Any],
|
|
original_config: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Validate optimization changes"""
|
|
validation = {"valid": True, "reason": None}
|
|
|
|
# Check for reasonable bounds
|
|
if "parallel_builds" in optimized_config:
|
|
parallel_builds = optimized_config["parallel_builds"]
|
|
if parallel_builds < 1 or parallel_builds > 8:
|
|
validation["valid"] = False
|
|
validation["reason"] = f"Parallel builds out of bounds: {parallel_builds}"
|
|
|
|
if "cache_size_mb" in optimized_config:
|
|
cache_size = optimized_config["cache_size_mb"]
|
|
if cache_size < 100 or cache_size > 10000: # 100MB to 10GB
|
|
validation["valid"] = False
|
|
validation["reason"] = f"Cache size out of bounds: {cache_size}MB"
|
|
|
|
return validation
|
|
|
|
def _record_optimization(self, original_config: Dict[str, Any],
|
|
optimized_config: Dict[str, Any],
|
|
analysis: Dict[str, Any]):
|
|
"""Record optimization for learning"""
|
|
optimization_record = {
|
|
"timestamp": datetime.now(),
|
|
"original_config": original_config,
|
|
"optimized_config": optimized_config,
|
|
"analysis": analysis,
|
|
"success": True
|
|
}
|
|
|
|
self._optimization_history.append(optimization_record)
|
|
self._successful_optimizations.append(optimization_record)
|
|
|
|
|
|
class AdvancedPerformanceReporter:
|
|
"""Advanced performance reporter with detailed analytics and visualization"""
|
|
|
|
def __init__(self, config):
|
|
self.config = config
|
|
self.logger = logging.getLogger(__name__)
|
|
|
|
# Reporting templates
|
|
self._report_templates = {}
|
|
self._chart_generators = {}
|
|
|
|
# Load reporting templates
|
|
self._load_reporting_templates()
|
|
|
|
def _load_reporting_templates(self):
|
|
"""Load performance reporting templates"""
|
|
# In a real implementation, this would load HTML/LaTeX templates
|
|
pass
|
|
|
|
def generate_comprehensive_report(self, monitor: AdvancedPerformanceMonitor,
|
|
output_file: Optional[str] = None) -> str:
|
|
"""Generate comprehensive performance report"""
|
|
if output_file is None:
|
|
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
output_file = os.path.join(monitor.metrics_dir, f"comprehensive_report_{timestamp}.html")
|
|
|
|
# Collect comprehensive data
|
|
report_data = {
|
|
"summary": monitor.get_performance_summary(),
|
|
"trends": {},
|
|
"optimizations": [],
|
|
"regressions": [],
|
|
"recommendations": []
|
|
}
|
|
|
|
# Analyze trends for all operations
|
|
for operation in monitor._operation_history:
|
|
operation_name = operation.operation
|
|
trends = monitor.analyze_performance_trends(operation_name)
|
|
report_data["trends"][operation_name] = trends
|
|
|
|
# Generate optimization recommendations
|
|
optimizer = AdvancedPerformanceOptimizer(self.config)
|
|
rules = optimizer.generate_optimization_rules()
|
|
report_data["optimizations"] = [rule.__dict__ for rule in rules]
|
|
|
|
# Detect regressions
|
|
for operation in monitor._operation_history:
|
|
regression = monitor.detect_performance_regression(operation.operation, operation)
|
|
if regression["regression_detected"]:
|
|
report_data["regressions"].append({
|
|
"operation": operation.operation,
|
|
"regression": regression
|
|
})
|
|
|
|
# Generate recommendations
|
|
report_data["recommendations"] = self._generate_recommendations(report_data)
|
|
|
|
# Generate HTML report
|
|
html_content = self._generate_html_report(report_data)
|
|
|
|
with open(output_file, 'w') as f:
|
|
f.write(html_content)
|
|
|
|
self.logger.info(f"Comprehensive report generated: {output_file}")
|
|
return output_file
|
|
|
|
def _generate_recommendations(self, report_data: Dict[str, Any]) -> List[str]:
|
|
"""Generate actionable recommendations"""
|
|
recommendations = []
|
|
|
|
# Performance improvement recommendations
|
|
if report_data["regressions"]:
|
|
recommendations.append("Investigate performance regressions in affected operations")
|
|
|
|
# Optimization recommendations
|
|
if report_data["optimizations"]:
|
|
recommendations.append("Apply suggested optimizations to improve performance")
|
|
|
|
# Trend-based recommendations
|
|
for operation, trends in report_data["trends"].items():
|
|
if trends.get("insights"):
|
|
for insight in trends["insights"]:
|
|
if "upward" in insight.lower():
|
|
recommendations.append(f"Monitor {operation} for performance degradation")
|
|
|
|
# System-level recommendations
|
|
summary = report_data["summary"]
|
|
if summary.get("total_operations", 0) > 100:
|
|
recommendations.append("Consider implementing automated performance testing")
|
|
|
|
return recommendations
|
|
|
|
def _generate_html_report(self, report_data: Dict[str, Any]) -> str:
|
|
"""Generate HTML performance report"""
|
|
html_template = """
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>Deb-Mock Performance Report</title>
|
|
<style>
|
|
body { font-family: Arial, sans-serif; margin: 20px; }
|
|
.header { background: #f0f0f0; padding: 20px; border-radius: 5px; }
|
|
.section { margin: 20px 0; padding: 15px; border: 1px solid #ddd; border-radius: 5px; }
|
|
.metric { display: inline-block; margin: 10px; padding: 10px; background: #f9f9f9; border-radius: 3px; }
|
|
.warning { background: #fff3cd; border-color: #ffeaa7; }
|
|
.success { background: #d4edda; border-color: #c3e6cb; }
|
|
.danger { background: #f8d7da; border-color: #f5c6cb; }
|
|
table { width: 100%; border-collapse: collapse; }
|
|
th, td { padding: 8px; text-align: left; border-bottom: 1px solid #ddd; }
|
|
th { background-color: #f2f2f2; }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="header">
|
|
<h1>Deb-Mock Performance Report</h1>
|
|
<p>Generated on: {timestamp}</p>
|
|
</div>
|
|
|
|
<div class="section">
|
|
<h2>Performance Summary</h2>
|
|
<div class="metric">
|
|
<strong>Total Operations:</strong> {total_operations}
|
|
</div>
|
|
<div class="metric">
|
|
<strong>Total Duration:</strong> {total_duration:.2f}s
|
|
</div>
|
|
<div class="metric">
|
|
<strong>Average Duration:</strong> {avg_duration:.2f}s
|
|
</div>
|
|
</div>
|
|
|
|
<div class="section">
|
|
<h2>Performance Trends</h2>
|
|
{trends_html}
|
|
</div>
|
|
|
|
<div class="section">
|
|
<h2>Optimization Opportunities</h2>
|
|
{optimizations_html}
|
|
</div>
|
|
|
|
<div class="section">
|
|
<h2>Performance Regressions</h2>
|
|
{regressions_html}
|
|
</div>
|
|
|
|
<div class="section">
|
|
<h2>Recommendations</h2>
|
|
<ul>
|
|
{recommendations_html}
|
|
</ul>
|
|
</div>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
# Fill in template data
|
|
summary = report_data["summary"]
|
|
trends_html = self._generate_trends_html(report_data["trends"])
|
|
optimizations_html = self._generate_optimizations_html(report_data["optimizations"])
|
|
regressions_html = self._generate_regressions_html(report_data["regressions"])
|
|
recommendations_html = self._generate_recommendations_html(report_data["recommendations"])
|
|
|
|
return html_template.format(
|
|
timestamp=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
|
total_operations=summary.get("total_operations", 0),
|
|
total_duration=summary.get("total_duration", 0),
|
|
avg_duration=summary.get("average_duration", 0),
|
|
trends_html=trends_html,
|
|
optimizations_html=optimizations_html,
|
|
regressions_html=regressions_html,
|
|
recommendations_html=recommendations_html
|
|
)
|
|
|
|
def _generate_trends_html(self, trends: Dict[str, Any]) -> str:
|
|
"""Generate HTML for trends section"""
|
|
if not trends:
|
|
return "<p>No trend data available</p>"
|
|
|
|
html = "<table><tr><th>Operation</th><th>Trends</th><th>Insights</th></tr>"
|
|
for operation, trend_data in trends.items():
|
|
insights = "<br>".join(trend_data.get("insights", []))
|
|
html += f"<tr><td>{operation}</td><td>Data points: {trend_data.get('data_points', 0)}</td><td>{insights}</td></tr>"
|
|
html += "</table>"
|
|
return html
|
|
|
|
def _generate_optimizations_html(self, optimizations: List[Dict[str, Any]]) -> str:
|
|
"""Generate HTML for optimizations section"""
|
|
if not optimizations:
|
|
return "<p>No optimization opportunities detected</p>"
|
|
|
|
html = "<table><tr><th>Rule</th><th>Description</th><th>Priority</th><th>Confidence</th></tr>"
|
|
for opt in optimizations:
|
|
html += f"<tr><td>{opt['name']}</td><td>{opt['description']}</td><td>{opt['priority']}</td><td>{opt['confidence']:.2f}</td></tr>"
|
|
html += "</table>"
|
|
return html
|
|
|
|
def _generate_regressions_html(self, regressions: List[Dict[str, Any]]) -> str:
|
|
"""Generate HTML for regressions section"""
|
|
if not regressions:
|
|
return "<p>No performance regressions detected</p>"
|
|
|
|
html = "<table><tr><th>Operation</th><th>Regression Score</th><th>Indicators</th></tr>"
|
|
for reg in regressions:
|
|
indicators = "<br>".join(reg["regression"]["indicators"])
|
|
html += f"<tr><td>{reg['operation']}</td><td>{reg['regression']['regression_score']:.3f}</td><td>{indicators}</td></tr>"
|
|
html += "</table>"
|
|
return html
|
|
|
|
def _generate_recommendations_html(self, recommendations: List[str]) -> str:
|
|
"""Generate HTML for recommendations section"""
|
|
if not recommendations:
|
|
return "<li>No specific recommendations at this time</li>"
|
|
|
|
html = ""
|
|
for rec in recommendations:
|
|
html += f"<li>{rec}</li>"
|
|
return html
|
|
|
|
|
|
# Backward compatibility - keep original class names
|
|
PerformanceMonitor = AdvancedPerformanceMonitor
|
|
PerformanceOptimizer = AdvancedPerformanceOptimizer
|
|
PerformanceReporter = AdvancedPerformanceReporter
|