"""
Advanced performance benchmarking and optimization for deb-mock
"""
import time
import psutil
import threading
import json
import os
import math
import statistics
from pathlib import Path
from typing import Dict, List, Any, Optional, Callable, Tuple
from contextlib import contextmanager
from dataclasses import dataclass, asdict
from datetime import datetime, timedelta
import logging
import subprocess
from collections import defaultdict, deque
from .exceptions import PerformanceError
@dataclass
class PerformanceMetrics:
"""Performance metrics for a build operation"""
operation: str
start_time: float
end_time: float
duration: float
cpu_percent: float
memory_mb: float
disk_io_read_mb: float
disk_io_write_mb: float
network_io_mb: float
chroot_size_mb: float
cache_hit_rate: float
parallel_efficiency: float
resource_utilization: float
# Enhanced metrics
io_wait_percent: float
context_switches: int
page_faults: int
cache_misses: int
branch_misses: float
@dataclass
class BuildProfile:
"""Build performance profile"""
build_id: str
package_name: str
architecture: str
suite: str
total_duration: float
phases: Dict[str, PerformanceMetrics]
resource_peak: Dict[str, float]
cache_performance: Dict[str, float]
optimization_suggestions: List[str]
timestamp: datetime
# Enhanced profile data
system_info: Dict[str, Any]
build_environment: Dict[str, Any]
performance_score: float
optimization_potential: float
regression_indicators: List[str]
@dataclass
class BenchmarkResult:
"""Result of a performance benchmark"""
benchmark_name: str
iterations: int
total_duration: float
average_duration: float
min_duration: float
max_duration: float
standard_deviation: float
coefficient_of_variation: float
percentiles: Dict[str, float]
results: List[Dict[str, Any]]
system_impact: Dict[str, float]
recommendations: List[str]
@dataclass
class OptimizationRule:
"""Performance optimization rule"""
rule_id: str
name: str
description: str
conditions: Dict[str, Any]
actions: List[Dict[str, Any]]
priority: int
confidence: float
impact_estimate: Dict[str, float]
class PerformanceMonitor:
"""Monitors and tracks performance metrics for deb-mock operations"""
def __init__(self, config):
self.config = config
self.logger = logging.getLogger(__name__)
# Performance tracking
self._active_operations = {}
self._operation_history = []
self._build_profiles = {}
# System monitoring
self._system_stats = {}
self._monitoring_active = False
self._monitor_thread = None
# Configuration
self.enable_monitoring = getattr(config, 'enable_performance_monitoring', True)
self.metrics_dir = getattr(config, 'performance_metrics_dir', './performance-metrics')
self.retention_days = getattr(config, 'performance_retention_days', 30)
# Create metrics directory
os.makedirs(self.metrics_dir, exist_ok=True)
# Start system monitoring if enabled
if self.enable_monitoring:
self._start_system_monitoring()
def _start_system_monitoring(self):
"""Start background system monitoring"""
if self._monitoring_active:
return
self._monitoring_active = True
self._monitor_thread = threading.Thread(target=self._monitor_system, daemon=True)
self._monitor_thread.start()
self.logger.info("Performance monitoring started")
def _monitor_system(self):
"""Background system monitoring thread"""
while self._monitoring_active:
try:
# Collect system statistics
cpu_percent = psutil.cpu_percent(interval=1)
memory = psutil.virtual_memory()
disk = psutil.disk_usage('/')
self._system_stats = {
'timestamp': time.time(),
'cpu_percent': cpu_percent,
'memory_percent': memory.percent,
'memory_available_mb': memory.available / (1024 * 1024),
'disk_percent': disk.percent,
'disk_free_gb': disk.free / (1024 * 1024 * 1024)
}
time.sleep(5) # Update every 5 seconds
except Exception as e:
self.logger.error(f"Error in system monitoring: {e}")
time.sleep(10)
def _get_system_stats(self) -> Dict[str, float]:
"""Get current system statistics"""
if not self._system_stats:
return {}
# Calculate averages over the last minute
current_time = time.time()
recent_stats = [
stats for stats in self._system_stats.values()
if isinstance(stats, dict) and
current_time - stats.get('timestamp', 0) < 60
]
if not recent_stats:
return {}
# Calculate averages
avg_stats = {}
for key in ['cpu_percent', 'memory_percent', 'disk_percent']:
values = [stats.get(key, 0) for stats in recent_stats if key in stats]
if values:
avg_stats[key] = sum(values) / len(values)
return avg_stats
@contextmanager
def monitor_operation(self, operation_name: str, **kwargs):
"""Context manager for monitoring operations"""
if not self.enable_monitoring:
yield
return
operation_id = f"{operation_name}_{int(time.time() * 1000)}"
start_time = time.time()
# Get initial system state
initial_cpu = psutil.cpu_percent()
initial_memory = psutil.virtual_memory()
initial_disk_io = psutil.disk_io_counters()
initial_network_io = psutil.net_io_counters()
# Track operation
self._active_operations[operation_id] = {
'name': operation_name,
'start_time': start_time,
'kwargs': kwargs
}
try:
yield operation_id
finally:
# Calculate metrics
end_time = time.time()
duration = end_time - start_time
# Get final system state
final_cpu = psutil.cpu_percent()
final_memory = psutil.virtual_memory()
final_disk_io = psutil.disk_io_counters()
final_network_io = psutil.net_io_counters()
# Calculate metrics
metrics = PerformanceMetrics(
operation=operation_name,
start_time=start_time,
end_time=end_time,
duration=duration,
cpu_percent=(initial_cpu + final_cpu) / 2,
memory_mb=(final_memory.used - initial_memory.used) / (1024 * 1024),
disk_io_read_mb=(final_disk_io.read_bytes - initial_disk_io.read_bytes) / (1024 * 1024),
disk_io_write_mb=(final_disk_io.write_bytes - initial_disk_io.write_bytes) / (1024 * 1024),
network_io_mb=(final_network_io.bytes_sent + final_network_io.bytes_recv -
initial_network_io.bytes_sent - initial_network_io.bytes_recv) / (1024 * 1024),
chroot_size_mb=0, # Will be calculated separately
cache_hit_rate=0, # Will be calculated separately
parallel_efficiency=0, # Will be calculated separately
resource_utilization=0 # Will be calculated separately
)
# Store metrics
self._operation_history.append(metrics)
if operation_id in self._active_operations:
del self._active_operations[operation_id]
# Log performance data
self.logger.info(f"Operation {operation_name} completed in {duration:.2f}s")
def benchmark_operation(self, operation_name: str, operation_func: Callable,
iterations: int = 3, **kwargs) -> Dict[str, Any]:
"""Benchmark an operation multiple times"""
if not self.enable_monitoring:
return operation_func(**kwargs)
results = []
for i in range(iterations):
with self.monitor_operation(f"{operation_name}_benchmark_{i}") as op_id:
start_time = time.time()
result = operation_func(**kwargs)
end_time = time.time()
results.append({
'iteration': i,
'duration': end_time - start_time,
'result': result
})
# Calculate statistics
durations = [r['duration'] for r in results]
avg_duration = sum(durations) / len(durations)
min_duration = min(durations)
max_duration = max(durations)
benchmark_result = {
'operation': operation_name,
'iterations': iterations,
'average_duration': avg_duration,
'min_duration': min_duration,
'max_duration': max_duration,
'variance': sum((d - avg_duration) ** 2 for d in durations) / len(durations),
'results': results
}
self.logger.info(f"Benchmark results for {operation_name}: "
f"avg={avg_duration:.2f}s, min={min_duration:.2f}s, max={max_duration:.2f}s")
return benchmark_result
def create_build_profile(self, build_id: str, package_name: str,
architecture: str, suite: str) -> str:
"""Create a new build performance profile"""
profile_id = f"{build_id}_{int(time.time() * 1000)}"
self._build_profiles[profile_id] = BuildProfile(
build_id=build_id,
package_name=package_name,
architecture=architecture,
suite=suite,
total_duration=0,
phases={},
resource_peak={},
cache_performance={},
optimization_suggestions=[],
timestamp=datetime.now()
)
return profile_id
def add_phase_metrics(self, profile_id: str, phase_name: str,
metrics: PerformanceMetrics):
"""Add phase metrics to a build profile"""
if profile_id not in self._build_profiles:
return
profile = self._build_profiles[profile_id]
profile.phases[phase_name] = metrics
profile.total_duration += metrics.duration
# Update resource peaks
if metrics.cpu_percent > profile.resource_peak.get('cpu_percent', 0):
profile.resource_peak['cpu_percent'] = metrics.cpu_percent
if metrics.memory_mb > profile.resource_peak.get('memory_mb', 0):
profile.resource_peak['memory_mb'] = metrics.memory_mb
def finalize_build_profile(self, profile_id: str) -> BuildProfile:
"""Finalize a build profile and generate optimization suggestions"""
if profile_id not in self._build_profiles:
return None
profile = self._build_profiles[profile_id]
# Calculate cache performance
cache_operations = [m for m in profile.phases.values()
if 'cache' in m.operation.lower()]
if cache_operations:
avg_cache_hit = sum(m.cache_hit_rate for m in cache_operations) / len(cache_operations)
profile.cache_performance['average_hit_rate'] = avg_cache_hit
# Generate optimization suggestions
profile.optimization_suggestions = self._generate_optimization_suggestions(profile)
# Save profile
self._save_build_profile(profile)
return profile
def _generate_optimization_suggestions(self, profile: BuildProfile) -> List[str]:
"""Generate optimization suggestions based on build profile"""
suggestions = []
# Duration-based suggestions
if profile.total_duration > 300: # 5 minutes
suggestions.append("Consider enabling parallel builds for faster execution")
suggestions.append("Review chroot caching strategy for better performance")
# Resource-based suggestions
if profile.resource_peak.get('cpu_percent', 0) > 80:
suggestions.append("High CPU usage detected - consider limiting parallel jobs")
if profile.resource_peak.get('memory_mb', 0) > 2048: # 2GB
suggestions.append("High memory usage - consider increasing swap or reducing parallel builds")
# Cache-based suggestions
if profile.cache_performance.get('average_hit_rate', 0) < 0.5:
suggestions.append("Low cache hit rate - consider adjusting cache size and retention")
# Phase-based suggestions
slow_phases = [(name, metrics.duration) for name, metrics in profile.phases.items()
if metrics.duration > 60] # 1 minute
if slow_phases:
slowest_phase = max(slow_phases, key=lambda x: x[1])
suggestions.append(f"Phase '{slowest_phase[0]}' is slow - investigate optimization opportunities")
return suggestions
def _save_build_profile(self, profile: BuildProfile):
"""Save build profile to disk"""
try:
filename = f"build_profile_{profile.build_id}_{profile.timestamp.strftime('%Y%m%d_%H%M%S')}.json"
filepath = os.path.join(self.metrics_dir, filename)
# Convert to JSON-serializable format
profile_dict = asdict(profile)
profile_dict['timestamp'] = profile.timestamp.isoformat()
with open(filepath, 'w') as f:
json.dump(profile_dict, f, indent=2)
self.logger.info(f"Build profile saved to {filepath}")
except Exception as e:
self.logger.error(f"Failed to save build profile: {e}")
def get_performance_summary(self) -> Dict[str, Any]:
"""Get a summary of performance metrics"""
if not self._operation_history:
return {}
# Calculate overall statistics
total_operations = len(self._operation_history)
total_duration = sum(m.duration for m in self._operation_history)
avg_duration = total_duration / total_operations
# Group by operation type
operation_stats = {}
for metrics in self._operation_history:
op_type = metrics.operation
if op_type not in operation_stats:
operation_stats[op_type] = {
'count': 0,
'total_duration': 0,
'avg_duration': 0,
'min_duration': float('inf'),
'max_duration': 0
}
stats = operation_stats[op_type]
stats['count'] += 1
stats['total_duration'] += metrics.duration
stats['min_duration'] = min(stats['min_duration'], metrics.duration)
stats['max_duration'] = max(stats['max_duration'], metrics.duration)
# Calculate averages
for stats in operation_stats.values():
stats['avg_duration'] = stats['total_duration'] / stats['count']
return {
'total_operations': total_operations,
'total_duration': total_duration,
'average_duration': avg_duration,
'operation_stats': operation_stats,
'system_stats': self._get_system_stats(),
'active_operations': len(self._active_operations)
}
def cleanup_old_metrics(self):
"""Clean up old performance metrics"""
try:
cutoff_time = datetime.now() - timedelta(days=self.retention_days)
# Clean up old build profiles
old_profiles = [
profile_id for profile_id, profile in self._build_profiles.items()
if profile.timestamp < cutoff_time
]
for profile_id in old_profiles:
del self._build_profiles[profile_id]
# Clean up old operation history
cutoff_timestamp = cutoff_time.timestamp()
self._operation_history = [
metrics for metrics in self._operation_history
if metrics.end_time > cutoff_timestamp
]
# Clean up old files
for filename in os.listdir(self.metrics_dir):
if filename.startswith('build_profile_'):
filepath = os.path.join(self.metrics_dir, filename)
file_time = datetime.fromtimestamp(os.path.getctime(filepath))
if file_time < cutoff_time:
os.remove(filepath)
self.logger.info(f"Cleaned up performance metrics older than {self.retention_days} days")
except Exception as e:
self.logger.error(f"Error cleaning up old metrics: {e}")
def export_metrics(self, output_file: str = None) -> str:
"""Export performance metrics to a file"""
if not output_file:
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
output_file = os.path.join(self.metrics_dir, f"performance_export_{timestamp}.json")
try:
export_data = {
'export_timestamp': datetime.now().isoformat(),
'summary': self.get_performance_summary(),
'build_profiles': {
pid: asdict(profile) for pid, profile in self._build_profiles.items()
},
'operation_history': [
asdict(metrics) for metrics in self._operation_history
]
}
with open(output_file, 'w') as f:
json.dump(export_data, f, indent=2)
self.logger.info(f"Performance metrics exported to {output_file}")
return output_file
except Exception as e:
self.logger.error(f"Failed to export metrics: {e}")
raise PerformanceError(f"Failed to export metrics: {e}")
def stop_monitoring(self):
"""Stop performance monitoring"""
self._monitoring_active = False
if self._monitor_thread and self._monitor_thread.is_alive():
self._monitor_thread.join(timeout=5)
self.logger.info("Performance monitoring stopped")
class PerformanceOptimizer:
"""Provides optimization suggestions and automatic tuning"""
def __init__(self, config):
self.config = config
self.logger = logging.getLogger(__name__)
# Optimization rules
self._optimization_rules = self._load_optimization_rules()
def _load_optimization_rules(self) -> Dict[str, Dict[str, Any]]:
"""Load optimization rules from configuration"""
return {
'parallel_builds': {
'cpu_threshold': 80,
'memory_threshold': 2048, # MB
'suggested_value': 2,
'max_value': 8
},
'cache_settings': {
'hit_rate_threshold': 0.5,
'size_threshold': 1024, # MB
'suggested_increase': 1.5
},
'chroot_optimization': {
'size_threshold': 5120, # 5GB
'cleanup_threshold': 7, # days
'compression_threshold': 0.8
}
}
def analyze_build_performance(self, build_profile: BuildProfile) -> Dict[str, Any]:
"""Analyze build performance and suggest optimizations"""
analysis = {
'score': 0,
'suggestions': [],
'automatic_tunings': [],
'manual_recommendations': []
}
# Calculate performance score
score = 100
# Duration penalty
if build_profile.total_duration > 600: # 10 minutes
score -= 20
elif build_profile.total_duration > 300: # 5 minutes
score -= 10
# Resource utilization penalty
if build_profile.resource_peak.get('cpu_percent', 0) > 90:
score -= 15
elif build_profile.resource_peak.get('cpu_percent', 0) > 80:
score -= 10
if build_profile.resource_peak.get('memory_mb', 0) > 4096: # 4GB
score -= 15
elif build_profile.resource_peak.get('memory_mb', 0) > 2048: # 2GB
score -= 10
analysis['score'] = max(0, score)
# Generate suggestions
analysis['suggestions'] = build_profile.optimization_suggestions
# Generate automatic tuning recommendations
analysis['automatic_tunings'] = self._generate_automatic_tunings(build_profile)
# Generate manual recommendations
analysis['manual_recommendations'] = self._generate_manual_recommendations(build_profile)
return analysis
def _generate_automatic_tunings(self, build_profile: BuildProfile) -> List[Dict[str, Any]]:
"""Generate automatic tuning recommendations"""
tunings = []
# Parallel build optimization
current_parallel = getattr(self.config, 'parallel_builds', 2)
cpu_peak = build_profile.resource_peak.get('cpu_percent', 0)
if cpu_peak < 60 and current_parallel < 4:
tunings.append({
'type': 'parallel_builds',
'current': current_parallel,
'suggested': min(current_parallel + 1, 4),
'reason': 'Low CPU utilization suggests room for more parallel builds'
})
elif cpu_peak > 90 and current_parallel > 1:
tunings.append({
'type': 'parallel_builds',
'current': current_parallel,
'suggested': max(current_parallel - 1, 1),
'reason': 'High CPU utilization suggests reducing parallel builds'
})
# Cache optimization
cache_hit_rate = build_profile.cache_performance.get('average_hit_rate', 0)
if cache_hit_rate < 0.3:
tunings.append({
'type': 'cache_size',
'action': 'increase',
'factor': 2.0,
'reason': 'Low cache hit rate suggests increasing cache size'
})
return tunings
def _generate_manual_recommendations(self, build_profile: BuildProfile) -> List[str]:
"""Generate manual optimization recommendations"""
recommendations = []
# Chroot optimization
if build_profile.total_duration > 300:
recommendations.append("Consider using tmpfs for /tmp to improve I/O performance")
recommendations.append("Review and optimize chroot package selection")
# Build optimization
if build_profile.resource_peak.get('memory_mb', 0) > 2048:
recommendations.append("Consider using ccache to reduce compilation time")
recommendations.append("Review build dependencies for unnecessary packages")
# System optimization
recommendations.append("Ensure adequate swap space for memory-intensive builds")
recommendations.append("Consider using SSD storage for better I/O performance")
return recommendations
def apply_automatic_tunings(self, tunings: List[Dict[str, Any]]) -> Dict[str, Any]:
"""Apply automatic tuning recommendations"""
results = {
'applied': [],
'failed': [],
'skipped': []
}
for tuning in tunings:
try:
if tuning['type'] == 'parallel_builds':
self._apply_parallel_build_tuning(tuning)
results['applied'].append(tuning)
elif tuning['type'] == 'cache_size':
self._apply_cache_tuning(tuning)
results['applied'].append(tuning)
else:
results['skipped'].append(tuning)
except Exception as e:
self.logger.error(f"Failed to apply tuning {tuning}: {e}")
results['failed'].append({**tuning, 'error': str(e)})
return results
def _apply_parallel_build_tuning(self, tuning: Dict[str, Any]):
"""Apply parallel build tuning"""
if hasattr(self.config, 'parallel_builds'):
self.config.parallel_builds = tuning['suggested']
self.logger.info(f"Applied parallel build tuning: {tuning['current']} -> {tuning['suggested']}")
def _apply_cache_tuning(self, tuning: Dict[str, Any]):
"""Apply cache tuning"""
if tuning['action'] == 'increase' and hasattr(self.config, 'cache_dir'):
# This would require more sophisticated cache management
self.logger.info(f"Cache tuning recommended: {tuning['reason']}")
class PerformanceReporter:
"""Generates performance reports and visualizations"""
def __init__(self, config):
self.config = config
self.logger = logging.getLogger(__name__)
def generate_performance_report(self, monitor: PerformanceMonitor,
output_file: str = None) -> str:
"""Generate a comprehensive performance report"""
if not output_file:
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
output_file = f"performance_report_{timestamp}.txt"
try:
with open(output_file, 'w') as f:
f.write("=" * 80 + "\n")
f.write("DEB-MOCK PERFORMANCE REPORT\n")
f.write("=" * 80 + "\n\n")
# Summary
summary = monitor.get_performance_summary()
f.write("PERFORMANCE SUMMARY\n")
f.write("-" * 40 + "\n")
f.write(f"Total Operations: {summary.get('total_operations', 0)}\n")
f.write(f"Total Duration: {summary.get('total_duration', 0):.2f}s\n")
f.write(f"Average Duration: {summary.get('average_duration', 0):.2f}s\n")
f.write(f"Active Operations: {summary.get('active_operations', 0)}\n\n")
# Operation statistics
if 'operation_stats' in summary:
f.write("OPERATION STATISTICS\n")
f.write("-" * 40 + "\n")
for op_name, stats in summary['operation_stats'].items():
f.write(f"{op_name}:\n")
f.write(f" Count: {stats['count']}\n")
f.write(f" Avg Duration: {stats['avg_duration']:.2f}s\n")
f.write(f" Min Duration: {stats['min_duration']:.2f}s\n")
f.write(f" Max Duration: {stats['max_duration']:.2f}s\n\n")
# System statistics
if 'system_stats' in summary:
f.write("SYSTEM STATISTICS\n")
f.write("-" * 40 + "\n")
for key, value in summary['system_stats'].items():
f.write(f"{key}: {value:.2f}\n")
f.write("\n" + "=" * 80 + "\n")
f.write("Report generated at: " + datetime.now().isoformat() + "\n")
f.write("=" * 80 + "\n")
self.logger.info(f"Performance report generated: {output_file}")
return output_file
except Exception as e:
self.logger.error(f"Failed to generate performance report: {e}")
raise PerformanceError(f"Failed to generate performance report: {e}")
def generate_build_profile_report(self, build_profile: BuildProfile,
output_file: str = None) -> str:
"""Generate a detailed build profile report"""
if not output_file:
output_file = f"build_profile_{build_profile.build_id}_{build_profile.timestamp.strftime('%Y%m%d_%H%M%S')}.txt"
try:
with open(output_file, 'w') as f:
f.write("=" * 80 + "\n")
f.write(f"BUILD PROFILE: {build_profile.package_name}\n")
f.write("=" * 80 + "\n\n")
f.write("BUILD INFORMATION\n")
f.write("-" * 40 + "\n")
f.write(f"Build ID: {build_profile.build_id}\n")
f.write(f"Package: {build_profile.package_name}\n")
f.write(f"Architecture: {build_profile.architecture}\n")
f.write(f"Suite: {build_profile.suite}\n")
f.write(f"Total Duration: {build_profile.total_duration:.2f}s\n")
f.write(f"Timestamp: {build_profile.timestamp.isoformat()}\n\n")
# Phase breakdown
f.write("PHASE BREAKDOWN\n")
f.write("-" * 40 + "\n")
for phase_name, metrics in build_profile.phases.items():
f.write(f"{phase_name}:\n")
f.write(f" Duration: {metrics.duration:.2f}s\n")
f.write(f" CPU: {metrics.cpu_percent:.1f}%\n")
f.write(f" Memory: {metrics.memory_mb:.1f}MB\n")
f.write(f" Disk I/O: {metrics.disk_io_read_mb:.1f}MB read, {metrics.disk_io_write_mb:.1f}MB write\n\n")
# Resource peaks
f.write("RESOURCE PEAKS\n")
f.write("-" * 40 + "\n")
for resource, value in build_profile.resource_peak.items():
f.write(f"{resource}: {value:.1f}\n")
# Cache performance
if build_profile.cache_performance:
f.write("\nCACHE PERFORMANCE\n")
f.write("-" * 40 + "\n")
for metric, value in build_profile.cache_performance.items():
f.write(f"{metric}: {value:.2f}\n")
# Optimization suggestions
if build_profile.optimization_suggestions:
f.write("\nOPTIMIZATION SUGGESTIONS\n")
f.write("-" * 40 + "\n")
for i, suggestion in enumerate(build_profile.optimization_suggestions, 1):
f.write(f"{i}. {suggestion}\n")
f.write("\n" + "=" * 80 + "\n")
f.write("Report generated at: " + datetime.now().isoformat() + "\n")
f.write("=" * 80 + "\n")
self.logger.info(f"Build profile report generated: {output_file}")
return output_file
except Exception as e:
self.logger.error(f"Failed to generate build profile report: {e}")
raise PerformanceError(f"Failed to generate build profile report: {e}")
class AdvancedPerformanceMonitor(PerformanceMonitor):
"""Enhanced performance monitor with advanced analytics and optimization"""
def __init__(self, config):
super().__init__(config)
# Advanced monitoring
self._performance_baselines = {}
self._regression_detection = {}
self._trend_analysis = {}
self._anomaly_detection = {}
# Machine learning features
self._ml_models = {}
self._prediction_cache = {}
self._learning_rate = 0.1
# Advanced metrics collection
self._detailed_metrics = defaultdict(list)
self._system_events = deque(maxlen=1000)
self._performance_alerts = []
# Load performance baselines
self._load_performance_baselines()
def _load_performance_baselines(self):
"""Load historical performance baselines"""
baseline_file = os.path.join(self.metrics_dir, "baselines.json")
if os.path.exists(baseline_file):
try:
with open(baseline_file, 'r') as f:
self._performance_baselines = json.load(f)
self.logger.info("Loaded performance baselines")
except Exception as e:
self.logger.warning(f"Failed to load baselines: {e}")
def _save_performance_baselines(self):
"""Save current performance baselines"""
baseline_file = os.path.join(self.metrics_dir, "baselines.json")
try:
with open(baseline_file, 'w') as f:
json.dump(self._performance_baselines, f, indent=2, default=str)
except Exception as e:
self.logger.error(f"Failed to save baselines: {e}")
def collect_detailed_metrics(self, operation_id: str) -> Dict[str, Any]:
"""Collect detailed system metrics for an operation"""
try:
# CPU detailed metrics
cpu_times = psutil.cpu_times_percent()
cpu_freq = psutil.cpu_freq()
cpu_count = psutil.cpu_count()
# Memory detailed metrics
memory = psutil.virtual_memory()
swap = psutil.swap_memory()
# Disk detailed metrics
disk_io = psutil.disk_io_counters()
disk_partitions = psutil.disk_partitions()
# Network detailed metrics
net_io = psutil.net_io_counters()
net_connections = len(psutil.net_connections())
# Process-specific metrics
process = psutil.Process()
with process.oneshot():
cpu_percent = process.cpu_percent()
memory_info = process.memory_info()
num_threads = process.num_threads()
num_fds = process.num_fds()
# System events
boot_time = psutil.boot_time()
uptime = time.time() - boot_time
detailed_metrics = {
"cpu": {
"percent": cpu_percent,
"times": cpu_times._asdict(),
"freq": cpu_freq._asdict() if cpu_freq else None,
"count": cpu_count,
"load_avg": os.getloadavg() if hasattr(os, 'getloadavg') else None
},
"memory": {
"virtual": memory._asdict(),
"swap": swap._asdict()
},
"disk": {
"io": disk_io._asdict() if disk_io else None,
"partitions": [p._asdict() for p in disk_partitions]
},
"network": {
"io": net_io._asdict() if net_io else None,
"connections": net_connections
},
"process": {
"cpu_percent": cpu_percent,
"memory_info": memory_info._asdict(),
"num_threads": num_threads,
"num_fds": num_fds
},
"system": {
"boot_time": boot_time,
"uptime": uptime,
"timestamp": time.time()
}
}
self._detailed_metrics[operation_id].append(detailed_metrics)
return detailed_metrics
except Exception as e:
self.logger.error(f"Failed to collect detailed metrics: {e}")
return {}
def detect_performance_regression(self, operation_name: str, current_metrics: PerformanceMetrics) -> Dict[str, Any]:
"""Detect performance regression by comparing with baselines"""
if operation_name not in self._performance_baselines:
return {"regression_detected": False, "confidence": 0.0}
baseline = self._performance_baselines[operation_name]
# Calculate regression indicators
duration_regression = (current_metrics.duration - baseline["avg_duration"]) / baseline["avg_duration"]
memory_regression = (current_metrics.memory_mb - baseline["avg_memory"]) / baseline["avg_memory"]
cpu_regression = (current_metrics.cpu_percent - baseline["avg_cpu"]) / baseline["avg_cpu"]
# Determine regression severity
regression_score = 0.0
regression_indicators = []
if duration_regression > 0.1: # 10% slower
regression_score += duration_regression * 0.4
regression_indicators.append(f"Duration increased by {duration_regression*100:.1f}%")
if memory_regression > 0.15: # 15% more memory
regression_score += memory_regression * 0.3
regression_indicators.append(f"Memory usage increased by {memory_regression*100:.1f}%")
if cpu_regression > 0.2: # 20% more CPU
regression_score += cpu_regression * 0.3
regression_indicators.append(f"CPU usage increased by {cpu_regression*100:.1f}%")
regression_detected = regression_score > 0.1
return {
"regression_detected": regression_detected,
"regression_score": regression_score,
"confidence": min(regression_score * 2, 1.0),
"indicators": regression_indicators,
"metrics": {
"duration_regression": duration_regression,
"memory_regression": memory_regression,
"cpu_regression": cpu_regression
}
}
def analyze_performance_trends(self, operation_name: str, days: int = 30) -> Dict[str, Any]:
"""Analyze performance trends over time"""
if operation_name not in self._detailed_metrics:
return {"trends": {}, "insights": []}
metrics_history = self._detailed_metrics[operation_name]
if len(metrics_history) < 5:
return {"trends": {}, "insights": []}
# Extract time series data
timestamps = [m["system"]["timestamp"] for m in metrics_history]
durations = []
memory_usage = []
cpu_usage = []
for metrics in metrics_history:
if "process" in metrics:
durations.append(metrics["process"].get("cpu_percent", 0))
memory_usage.append(metrics["process"].get("memory_info", {}).get("rss", 0) / 1024 / 1024)
cpu_usage.append(metrics["process"].get("cpu_percent", 0))
if not durations:
return {"trends": {}, "insights": []}
# Calculate trends using linear regression
def linear_trend(data):
if len(data) < 2:
return 0.0, 0.0
x = list(range(len(data)))
n = len(data)
sum_x = sum(x)
sum_y = sum(data)
sum_xy = sum(x[i] * data[i] for i in range(n))
sum_x2 = sum(x[i] ** 2 for i in range(n))
slope = (n * sum_xy - sum_x * sum_y) / (n * sum_x2 - sum_x ** 2)
intercept = (sum_y - slope * sum_x) / n
return slope, intercept
duration_slope, duration_intercept = linear_trend(durations)
memory_slope, memory_intercept = linear_trend(memory_usage)
cpu_slope, cpu_intercept = linear_trend(cpu_usage)
# Generate insights
insights = []
if duration_slope > 0.01:
insights.append(f"Duration is trending upward (slope: {duration_slope:.3f})")
elif duration_slope < -0.01:
insights.append(f"Duration is trending downward (slope: {duration_slope:.3f})")
if memory_slope > 0.1:
insights.append(f"Memory usage is trending upward (slope: {memory_slope:.3f} MB/run)")
elif memory_slope < -0.1:
insights.append(f"Memory usage is trending downward (slope: {memory_slope:.3f} MB/run)")
return {
"trends": {
"duration": {"slope": duration_slope, "intercept": duration_intercept},
"memory": {"slope": memory_slope, "intercept": memory_intercept},
"cpu": {"slope": cpu_slope, "intercept": cpu_intercept}
},
"insights": insights,
"data_points": len(metrics_history),
"time_span_days": (max(timestamps) - min(timestamps)) / 86400 if timestamps else 0
}
def predict_performance(self, operation_name: str, input_parameters: Dict[str, Any]) -> Dict[str, Any]:
"""Predict performance based on historical data and input parameters"""
if operation_name not in self._performance_baselines:
return {"prediction": None, "confidence": 0.0}
baseline = self._performance_baselines[operation_name]
# Simple linear prediction model
# In a real implementation, this would use more sophisticated ML models
predicted_duration = baseline["avg_duration"]
predicted_memory = baseline["avg_memory"]
predicted_cpu = baseline["avg_cpu"]
# Adjust based on input parameters
if "package_size" in input_parameters:
size_factor = input_parameters["package_size"] / baseline.get("avg_package_size", 1.0)
predicted_duration *= size_factor
predicted_memory *= size_factor
if "parallel_builds" in input_parameters:
parallel_factor = baseline.get("avg_parallel_builds", 1.0) / input_parameters["parallel_builds"]
predicted_duration *= parallel_factor
predicted_cpu *= parallel_factor
confidence = 0.7 # Base confidence for simple model
return {
"predicted_duration": predicted_duration,
"predicted_memory": predicted_memory,
"predicted_cpu": predicted_cpu,
"confidence": confidence,
"model_type": "linear_regression",
"input_parameters": input_parameters
}
def generate_optimization_rules(self) -> List[OptimizationRule]:
"""Generate optimization rules based on performance analysis"""
rules = []
# Rule 1: Parallel build optimization
if self._has_low_cpu_utilization():
rules.append(OptimizationRule(
rule_id="parallel_builds_001",
name="Increase Parallel Builds",
description="CPU utilization is low, increase parallel builds for better performance",
conditions={"cpu_utilization": "< 70%", "parallel_builds": "< 4"},
actions=[{"type": "increase_parallel_builds", "value": "+1"}],
priority=1,
confidence=0.8,
impact_estimate={"duration": -0.15, "cpu_utilization": 0.2}
))
# Rule 2: Cache optimization
if self._has_low_cache_hit_rate():
rules.append(OptimizationRule(
rule_id="cache_optimization_001",
name="Optimize Cache Settings",
description="Cache hit rate is low, optimize cache configuration",
conditions={"cache_hit_rate": "< 0.6"},
actions=[
{"type": "increase_cache_size", "value": "+50%"},
{"type": "optimize_cache_policy", "value": "lru"}
],
priority=2,
confidence=0.7,
impact_estimate={"cache_hit_rate": 0.2, "duration": -0.1}
))
# Rule 3: Memory optimization
if self._has_high_memory_usage():
rules.append(OptimizationRule(
rule_id="memory_optimization_001",
name="Optimize Memory Usage",
description="Memory usage is high, optimize memory configuration",
conditions={"memory_usage": "> 80%", "swap_usage": "> 20%"},
actions=[
{"type": "reduce_parallel_builds", "value": "-1"},
{"type": "enable_memory_compression", "value": True}
],
priority=3,
confidence=0.6,
impact_estimate={"memory_usage": -0.1, "duration": 0.05}
))
return rules
def _has_low_cpu_utilization(self) -> bool:
"""Check if CPU utilization is consistently low"""
if not self._operation_history:
return False
recent_ops = self._operation_history[-10:] # Last 10 operations
avg_cpu = statistics.mean(op.cpu_percent for op in recent_ops)
return avg_cpu < 70.0
def _has_low_cache_hit_rate(self) -> bool:
"""Check if cache hit rate is low"""
if not self._operation_history:
return False
recent_ops = self._operation_history[-10:] # Last 10 operations
avg_cache_hit = statistics.mean(op.cache_hit_rate for op in recent_ops)
return avg_cache_hit < 0.6
def _has_high_memory_usage(self) -> bool:
"""Check if memory usage is high"""
if not self._operation_history:
return False
recent_ops = self._operation_history[-10:] # Last 10 operations
avg_memory = statistics.mean(op.memory_mb for op in recent_ops)
return avg_memory > 2048 # 2GB threshold
class AdvancedPerformanceOptimizer:
"""Advanced performance optimizer with machine learning capabilities"""
def __init__(self, config):
self.config = config
self.logger = logging.getLogger(__name__)
# Optimization history
self._optimization_history = []
self._successful_optimizations = []
self._failed_optimizations = []
# Machine learning models
self._ml_models = {}
self._feature_importance = {}
# Advanced optimization algorithms
self._genetic_algorithm = None
self._bayesian_optimization = None
# Load optimization models
self._load_optimization_models()
def _load_optimization_models(self):
"""Load pre-trained optimization models"""
# In a real implementation, this would load trained ML models
# For now, we'll use rule-based optimization
pass
def optimize_build_configuration(self, current_config: Dict[str, Any],
performance_profile: BuildProfile) -> Dict[str, Any]:
"""Optimize build configuration using advanced algorithms"""
optimized_config = current_config.copy()
# Analyze current performance
performance_analysis = self._analyze_build_performance(performance_profile)
# Apply optimization strategies
if performance_analysis["cpu_underutilized"]:
optimized_config = self._optimize_cpu_utilization(optimized_config, performance_analysis)
if performance_analysis["memory_inefficient"]:
optimized_config = self._optimize_memory_usage(optimized_config, performance_analysis)
if performance_analysis["cache_inefficient"]:
optimized_config = self._optimize_cache_settings(optimized_config, performance_analysis)
if performance_analysis["io_bottleneck"]:
optimized_config = self._optimize_io_settings(optimized_config, performance_analysis)
# Validate optimization
validation_result = self._validate_optimization(optimized_config, current_config)
if validation_result["valid"]:
self._record_optimization(current_config, optimized_config, performance_analysis)
return optimized_config
else:
self.logger.warning(f"Optimization validation failed: {validation_result['reason']}")
return current_config
def _analyze_build_performance(self, profile: BuildProfile) -> Dict[str, Any]:
"""Analyze build performance for optimization opportunities"""
analysis = {
"cpu_underutilized": False,
"memory_inefficient": False,
"cache_inefficient": False,
"io_bottleneck": False,
"optimization_score": 0.0
}
# CPU analysis
if profile.resource_peak.get("cpu_percent", 0) < 70:
analysis["cpu_underutilized"] = True
analysis["optimization_score"] += 0.3
# Memory analysis
memory_usage = profile.resource_peak.get("memory_mb", 0)
if memory_usage > 2048: # 2GB threshold
analysis["memory_inefficient"] = True
analysis["optimization_score"] += 0.25
# Cache analysis
cache_hit_rate = profile.cache_performance.get("average_hit_rate", 0)
if cache_hit_rate < 0.6:
analysis["cache_inefficient"] = True
analysis["optimization_score"] += 0.25
# I/O analysis
total_io = sum(phase.disk_io_read_mb + phase.disk_io_write_mb
for phase in profile.phases.values())
if total_io > 1000: # 1GB I/O threshold
analysis["io_bottleneck"] = True
analysis["optimization_score"] += 0.2
return analysis
def _optimize_cpu_utilization(self, config: Dict[str, Any],
analysis: Dict[str, Any]) -> Dict[str, Any]:
"""Optimize CPU utilization"""
if "parallel_builds" in config:
current_parallel = config["parallel_builds"]
# Increase parallel builds if CPU is underutilized
if analysis["cpu_underutilized"] and current_parallel < 4:
config["parallel_builds"] = min(current_parallel + 1, 4)
self.logger.info(f"Increased parallel builds from {current_parallel} to {config['parallel_builds']}")
return config
def _optimize_memory_usage(self, config: Dict[str, Any],
analysis: Dict[str, Any]) -> Dict[str, Any]:
"""Optimize memory usage"""
if "parallel_builds" in config and analysis["memory_inefficient"]:
current_parallel = config["parallel_builds"]
# Reduce parallel builds if memory usage is high
if current_parallel > 1:
config["parallel_builds"] = max(current_parallel - 1, 1)
self.logger.info(f"Reduced parallel builds from {current_parallel} to {config['parallel_builds']}")
return config
def _optimize_cache_settings(self, config: Dict[str, Any],
analysis: Dict[str, Any]) -> Dict[str, Any]:
"""Optimize cache settings"""
if analysis["cache_inefficient"]:
# Increase cache size
if "cache_size_mb" in config:
current_cache_size = config["cache_size_mb"]
config["cache_size_mb"] = int(current_cache_size * 1.5)
self.logger.info(f"Increased cache size from {current_cache_size}MB to {config['cache_size_mb']}MB")
# Enable aggressive caching
config["aggressive_caching"] = True
config["cache_compression"] = True
return config
def _optimize_io_settings(self, config: Dict[str, Any],
analysis: Dict[str, Any]) -> Dict[str, Any]:
"""Optimize I/O settings"""
if analysis["io_bottleneck"]:
# Enable I/O optimization
config["io_scheduler"] = "deadline"
config["read_ahead_kb"] = 4096
config["enable_write_cache"] = True
return config
def _validate_optimization(self, optimized_config: Dict[str, Any],
original_config: Dict[str, Any]) -> Dict[str, Any]:
"""Validate optimization changes"""
validation = {"valid": True, "reason": None}
# Check for reasonable bounds
if "parallel_builds" in optimized_config:
parallel_builds = optimized_config["parallel_builds"]
if parallel_builds < 1 or parallel_builds > 8:
validation["valid"] = False
validation["reason"] = f"Parallel builds out of bounds: {parallel_builds}"
if "cache_size_mb" in optimized_config:
cache_size = optimized_config["cache_size_mb"]
if cache_size < 100 or cache_size > 10000: # 100MB to 10GB
validation["valid"] = False
validation["reason"] = f"Cache size out of bounds: {cache_size}MB"
return validation
def _record_optimization(self, original_config: Dict[str, Any],
optimized_config: Dict[str, Any],
analysis: Dict[str, Any]):
"""Record optimization for learning"""
optimization_record = {
"timestamp": datetime.now(),
"original_config": original_config,
"optimized_config": optimized_config,
"analysis": analysis,
"success": True
}
self._optimization_history.append(optimization_record)
self._successful_optimizations.append(optimization_record)
class AdvancedPerformanceReporter:
"""Advanced performance reporter with detailed analytics and visualization"""
def __init__(self, config):
self.config = config
self.logger = logging.getLogger(__name__)
# Reporting templates
self._report_templates = {}
self._chart_generators = {}
# Load reporting templates
self._load_reporting_templates()
def _load_reporting_templates(self):
"""Load performance reporting templates"""
# In a real implementation, this would load HTML/LaTeX templates
pass
def generate_comprehensive_report(self, monitor: AdvancedPerformanceMonitor,
output_file: Optional[str] = None) -> str:
"""Generate comprehensive performance report"""
if output_file is None:
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
output_file = os.path.join(monitor.metrics_dir, f"comprehensive_report_{timestamp}.html")
# Collect comprehensive data
report_data = {
"summary": monitor.get_performance_summary(),
"trends": {},
"optimizations": [],
"regressions": [],
"recommendations": []
}
# Analyze trends for all operations
for operation in monitor._operation_history:
operation_name = operation.operation
trends = monitor.analyze_performance_trends(operation_name)
report_data["trends"][operation_name] = trends
# Generate optimization recommendations
optimizer = AdvancedPerformanceOptimizer(self.config)
rules = optimizer.generate_optimization_rules()
report_data["optimizations"] = [rule.__dict__ for rule in rules]
# Detect regressions
for operation in monitor._operation_history:
regression = monitor.detect_performance_regression(operation.operation, operation)
if regression["regression_detected"]:
report_data["regressions"].append({
"operation": operation.operation,
"regression": regression
})
# Generate recommendations
report_data["recommendations"] = self._generate_recommendations(report_data)
# Generate HTML report
html_content = self._generate_html_report(report_data)
with open(output_file, 'w') as f:
f.write(html_content)
self.logger.info(f"Comprehensive report generated: {output_file}")
return output_file
def _generate_recommendations(self, report_data: Dict[str, Any]) -> List[str]:
"""Generate actionable recommendations"""
recommendations = []
# Performance improvement recommendations
if report_data["regressions"]:
recommendations.append("Investigate performance regressions in affected operations")
# Optimization recommendations
if report_data["optimizations"]:
recommendations.append("Apply suggested optimizations to improve performance")
# Trend-based recommendations
for operation, trends in report_data["trends"].items():
if trends.get("insights"):
for insight in trends["insights"]:
if "upward" in insight.lower():
recommendations.append(f"Monitor {operation} for performance degradation")
# System-level recommendations
summary = report_data["summary"]
if summary.get("total_operations", 0) > 100:
recommendations.append("Consider implementing automated performance testing")
return recommendations
def _generate_html_report(self, report_data: Dict[str, Any]) -> str:
"""Generate HTML performance report"""
html_template = """
Deb-Mock Performance Report
Performance Summary
Total Operations: {total_operations}
Total Duration: {total_duration:.2f}s
Average Duration: {avg_duration:.2f}s
Performance Trends
{trends_html}
Optimization Opportunities
{optimizations_html}
Performance Regressions
{regressions_html}
"""
# Fill in template data
summary = report_data["summary"]
trends_html = self._generate_trends_html(report_data["trends"])
optimizations_html = self._generate_optimizations_html(report_data["optimizations"])
regressions_html = self._generate_regressions_html(report_data["regressions"])
recommendations_html = self._generate_recommendations_html(report_data["recommendations"])
return html_template.format(
timestamp=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
total_operations=summary.get("total_operations", 0),
total_duration=summary.get("total_duration", 0),
avg_duration=summary.get("average_duration", 0),
trends_html=trends_html,
optimizations_html=optimizations_html,
regressions_html=regressions_html,
recommendations_html=recommendations_html
)
def _generate_trends_html(self, trends: Dict[str, Any]) -> str:
"""Generate HTML for trends section"""
if not trends:
return "No trend data available
"
html = "| Operation | Trends | Insights |
|---|
"
for operation, trend_data in trends.items():
insights = "
".join(trend_data.get("insights", []))
html += f"| {operation} | Data points: {trend_data.get('data_points', 0)} | {insights} |
"
html += "
"
return html
def _generate_optimizations_html(self, optimizations: List[Dict[str, Any]]) -> str:
"""Generate HTML for optimizations section"""
if not optimizations:
return "No optimization opportunities detected
"
html = "| Rule | Description | Priority | Confidence |
|---|
"
for opt in optimizations:
html += f"| {opt['name']} | {opt['description']} | {opt['priority']} | {opt['confidence']:.2f} |
"
html += "
"
return html
def _generate_regressions_html(self, regressions: List[Dict[str, Any]]) -> str:
"""Generate HTML for regressions section"""
if not regressions:
return "No performance regressions detected
"
html = "| Operation | Regression Score | Indicators |
|---|
"
for reg in regressions:
indicators = "
".join(reg["regression"]["indicators"])
html += f"| {reg['operation']} | {reg['regression']['regression_score']:.3f} | {indicators} |
"
html += "
"
return html
def _generate_recommendations_html(self, recommendations: List[str]) -> str:
"""Generate HTML for recommendations section"""
if not recommendations:
return "No specific recommendations at this time"
html = ""
for rec in recommendations:
html += f"{rec}"
return html
# Backward compatibility - keep original class names
PerformanceMonitor = AdvancedPerformanceMonitor
PerformanceOptimizer = AdvancedPerformanceOptimizer
PerformanceReporter = AdvancedPerformanceReporter