spent some time doing research, reconfigure, and testing. New understanding

This commit is contained in:
robojerk 2025-08-17 15:02:27 -07:00
parent ec63937f20
commit f6228e65a5
33 changed files with 5487 additions and 1881 deletions

55
scripts/apt-cacher-ng.sh Executable file
View file

@ -0,0 +1,55 @@
#!/bin/bash
# Define the hostname and port for the new mirror
NEW_HOST="192.168.1.101"
NEW_PORT="3142"
# Path to the sources files (Debian 13+ uses debian.sources)
SOURCES_LIST="/etc/apt/sources.list"
DEBIAN_SOURCES="/etc/apt/sources.list.d/debian.sources"
# Check which sources file exists
if [[ -f "$DEBIAN_SOURCES" ]]; then
SOURCES_FILE="$DEBIAN_SOURCES"
echo "Using Debian 13+ sources format: $DEBIAN_SOURCES"
elif [[ -f "$SOURCES_LIST" ]]; then
SOURCES_FILE="$SOURCES_LIST"
echo "Using traditional sources.list format: $SOURCES_LIST"
else
echo "Error: No sources file found at $SOURCES_LIST or $DEBIAN_SOURCES"
exit 1
fi
# Create a backup of the original sources file
cp "$SOURCES_FILE" "${SOURCES_FILE}.bak"
echo "Backup of $SOURCES_FILE created at ${SOURCES_FILE}.bak"
# Use sed to modify the file
# Explanation of the sed command:
# -i: edit the file in place
# -E: use extended regular expressions
# s|...|...|g: substitute (replace)
#
# The regex breaks down as follows:
# ^: start of line
# (deb(?:-src)?): captures "deb" or "deb-src"
# \s+: one or more spaces
# (https?://)?: captures "http://" or "https://" (optional)
# ([^/]+): captures the hostname, anything that is not a slash
# (.*): captures the rest of the line, including the path
#
# The replacement string:
# \1: the captured "deb" or "deb-src"
# : a space
# http://${NEW_HOST}:${NEW_PORT}/: the new prefix
# \3: the captured protocol (e.g., "http://", "https://"), if it exists. Replaces "https://" with "HTTPS///" to follow the example.
# \4: the captured hostname
# \5: the captured rest of the line
sed -i -E "s|^(deb(?:-src)?)\s+(https?://)?([^/]+)(.*)$|\1 http://${NEW_HOST}:${NEW_PORT}/\3\4|" "$SOURCES_FILE"
# A second sed command to handle the "HTTPS///" case as per the example
# It replaces "https://" with "HTTPS///"
sed -i -E "s|http://${NEW_HOST}:${NEW_PORT}/https://|http://${NEW_HOST}:${NEW_PORT}/HTTPS///|" "$SOURCES_FILE"
echo "The file $SOURCES_FILE has been successfully modified."
echo "Please verify the changes and run 'sudo apt update' to refresh the package list."

263
scripts/apt-ostree-report.sh Executable file
View file

@ -0,0 +1,263 @@
#!/bin/bash
# apt-ostree Reporting Script
# Comprehensive reporting for Debian Atomic apt-ostree testing
set -e
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Configuration
REPORT_FILE="${REPORT_FILE:-apt-ostree-report.txt}"
VERBOSE="${VERBOSE:-false}"
# Helper functions
log_info() {
echo -e "${BLUE}[INFO]${NC} $1"
}
log_success() {
echo -e "${GREEN}[SUCCESS]${NC} $1"
}
log_warning() {
echo -e "${YELLOW}[WARNING]${NC} $1"
}
log_error() {
echo -e "${RED}[ERROR]${NC} $1"
}
# Test functions
test_apt_ostree_version() {
log_info "Testing apt-ostree version..."
if apt-ostree --version >/dev/null 2>&1; then
local version=$(apt-ostree --version)
log_success "apt-ostree version: $version"
echo "VERSION: $version" >> "$REPORT_FILE"
return 0
else
log_error "apt-ostree --version failed"
echo "VERSION: FAILED" >> "$REPORT_FILE"
return 1
fi
}
test_apt_ostree_help() {
log_info "Testing apt-ostree help..."
if apt-ostree --help >/dev/null 2>&1; then
log_success "apt-ostree --help works"
echo "HELP: SUCCESS" >> "$REPORT_FILE"
return 0
else
log_error "apt-ostree --help failed"
echo "HELP: FAILED" >> "$REPORT_FILE"
return 1
fi
}
test_apt_ostree_status() {
log_info "Testing apt-ostree status..."
if apt-ostree status >/dev/null 2>&1; then
local status_output=$(apt-ostree status)
log_success "apt-ostree status works"
echo "STATUS: SUCCESS" >> "$REPORT_FILE"
if [ "$VERBOSE" = "true" ]; then
echo "STATUS_OUTPUT:" >> "$REPORT_FILE"
echo "$status_output" >> "$REPORT_FILE"
fi
return 0
else
log_error "apt-ostree status failed"
echo "STATUS: FAILED" >> "$REPORT_FILE"
return 1
fi
}
test_apt_ostree_list() {
log_info "Testing apt-ostree list..."
if apt-ostree list >/dev/null 2>&1; then
local list_output=$(apt-ostree list)
log_success "apt-ostree list works"
echo "LIST: SUCCESS" >> "$REPORT_FILE"
if [ "$VERBOSE" = "true" ]; then
echo "LIST_OUTPUT:" >> "$REPORT_FILE"
echo "$list_output" >> "$REPORT_FILE"
fi
return 0
else
log_error "apt-ostree list failed"
echo "LIST: FAILED" >> "$REPORT_FILE"
return 1
fi
}
test_apt_ostree_search() {
log_info "Testing apt-ostree search..."
if apt-ostree search htop >/dev/null 2>&1; then
local search_output=$(apt-ostree search htop)
log_success "apt-ostree search works"
echo "SEARCH: SUCCESS" >> "$REPORT_FILE"
if [ "$VERBOSE" = "true" ]; then
echo "SEARCH_OUTPUT:" >> "$REPORT_FILE"
echo "$search_output" >> "$REPORT_FILE"
fi
return 0
else
log_error "apt-ostree search failed"
echo "SEARCH: FAILED" >> "$REPORT_FILE"
return 1
fi
}
test_apt_ostree_info() {
log_info "Testing apt-ostree info..."
if apt-ostree info apt-ostree >/dev/null 2>&1; then
local info_output=$(apt-ostree info apt-ostree)
log_success "apt-ostree info works"
echo "INFO: SUCCESS" >> "$REPORT_FILE"
if [ "$VERBOSE" = "true" ]; then
echo "INFO_OUTPUT:" >> "$REPORT_FILE"
echo "$info_output" >> "$REPORT_FILE"
fi
return 0
else
log_error "apt-ostree info failed"
echo "INFO: FAILED" >> "$REPORT_FILE"
return 1
fi
}
test_apt_ostree_install() {
log_info "Testing apt-ostree install..."
if apt-ostree install htop >/dev/null 2>&1; then
log_success "apt-ostree install works"
echo "INSTALL: SUCCESS" >> "$REPORT_FILE"
return 0
else
log_error "apt-ostree install failed"
echo "INSTALL: FAILED" >> "$REPORT_FILE"
return 1
fi
}
test_apt_ostree_remove() {
log_info "Testing apt-ostree remove..."
if apt-ostree remove htop >/dev/null 2>&1; then
log_success "apt-ostree remove works"
echo "REMOVE: SUCCESS" >> "$REPORT_FILE"
return 0
else
log_error "apt-ostree remove failed"
echo "REMOVE: FAILED" >> "$REPORT_FILE"
return 1
fi
}
# Main reporting function
generate_report() {
log_info "Starting apt-ostree comprehensive testing..."
# Initialize report file
echo "=== apt-ostree Testing Report ===" > "$REPORT_FILE"
echo "Generated: $(date)" >> "$REPORT_FILE"
echo "Environment: $(uname -a)" >> "$REPORT_FILE"
echo "" >> "$REPORT_FILE"
# Test counter
local total_tests=0
local passed_tests=0
local failed_tests=0
# Run all tests
local tests=(
"test_apt_ostree_version"
"test_apt_ostree_help"
"test_apt_ostree_status"
"test_apt_ostree_list"
"test_apt_ostree_search"
"test_apt_ostree_info"
"test_apt_ostree_install"
"test_apt_ostree_remove"
)
for test in "${tests[@]}"; do
total_tests=$((total_tests + 1))
if $test; then
passed_tests=$((passed_tests + 1))
else
failed_tests=$((failed_tests + 1))
fi
done
# Summary
echo "" >> "$REPORT_FILE"
echo "=== Test Summary ===" >> "$REPORT_FILE"
echo "Total Tests: $total_tests" >> "$REPORT_FILE"
echo "Passed: $passed_tests" >> "$REPORT_FILE"
echo "Failed: $failed_tests" >> "$REPORT_FILE"
echo "Success Rate: $((passed_tests * 100 / total_tests))%" >> "$REPORT_FILE"
# Display summary
echo ""
log_info "=== Testing Complete ==="
log_info "Total Tests: $total_tests"
log_success "Passed: $passed_tests"
if [ $failed_tests -gt 0 ]; then
log_error "Failed: $failed_tests"
fi
log_info "Success Rate: $((passed_tests * 100 / total_tests))%"
log_info "Report saved to: $REPORT_FILE"
return $failed_tests
}
# Main execution
main() {
# Parse command line arguments
while [[ $# -gt 0 ]]; do
case $1 in
-v|--verbose)
VERBOSE="true"
shift
;;
-o|--output)
REPORT_FILE="$2"
shift 2
;;
-h|--help)
echo "Usage: $0 [OPTIONS]"
echo "Options:"
echo " -v, --verbose Enable verbose output"
echo " -o, --output Specify output file (default: apt-ostree-report.txt)"
echo " -h, --help Show this help message"
exit 0
;;
*)
echo "Unknown option: $1"
exit 1
;;
esac
done
# Check if apt-ostree is available
if ! command -v apt-ostree >/dev/null 2>&1; then
log_error "apt-ostree not found. Please ensure it's installed and in PATH."
exit 1
fi
# Generate report
generate_report
local exit_code=$?
# Exit with number of failed tests
exit $exit_code
}
# Run main function
main "$@"

View file

@ -1,104 +1,42 @@
#!/usr/bin/env python3
"""
Debian Atomic Comps Sync Script
Fedora comps-sync.py equivalent for Debian package groups
This script syncs Debian tasks (package groups) with Debian Atomic variant configurations,
ensuring variants stay updated with the Debian package ecosystem.
Usage:
./comps-sync.py /path/to/debian-tasks
./comps-sync.py --save /path/to/debian-tasks
Debian Atomic Package Group Synchronization
Simplified version for current treefile structure
"""
import argparse
import yaml
import os
import sys
import yaml
import xml.etree.ElementTree as ET
import argparse
from pathlib import Path
from typing import Dict, List, Set
class DebianAtomicCompsSync:
def __init__(self, repo_path: str):
def __init__(self, repo_path: str = "."):
self.repo_path = Path(repo_path)
self.variants_dir = self.repo_path / "variants"
self.treefiles_dir = self.repo_path / "treefiles"
# Ensure directories exist
self.treefiles_dir.mkdir(exist_ok=True)
# Variant configurations - Fedora Atomic 1:1 parallel
self.variants = {
"base": {
"description": "Base OSTree system",
"packages": [],
"groups": ["base", "system"]
},
"workstation": {
"description": "Debian Atomic Workstation (Fedora Silverblue equivalent)",
"packages": [],
"groups": ["desktop", "gnome", "office", "productivity"]
},
"kde": {
"description": "Debian Atomic KDE (Fedora Kinoite equivalent)",
"packages": [],
"groups": ["desktop", "kde", "office", "productivity"]
},
"sway": {
"description": "Debian Atomic Sway (Fedora Sway Atomic equivalent)",
"packages": [],
"groups": ["desktop", "sway", "wayland", "minimal"]
},
"server": {
"description": "Debian Atomic Server (Fedora CoreOS equivalent)",
"packages": [],
"groups": ["server", "enterprise", "monitoring", "container"]
}
}
self.variants = ["base", "workstation", "server", "testing", "debian-bootc-base"]
def parse_debian_tasks(self, tasks_file: str) -> Dict[str, List[str]]:
"""Parse Debian tasks file for package groups"""
print(f"Parsing Debian tasks file: {tasks_file}")
# This is a simplified parser - in practice you'd want to parse
# actual Debian tasks files or use debian-policy package
tasks = {}
def parse_debian_tasks(self, tasks_file: str) -> dict:
"""Parse Debian tasks file"""
try:
# For now, we'll create example package groups
# In a real implementation, you'd parse the actual tasks file
tasks = {
"base": [
"systemd", "ostree", "grub2", "linux-image-amd64",
"initramfs-tools", "bash", "coreutils", "vim"
],
"server": [
"openssh-server", "nginx", "postgresql", "monitoring-plugins",
"logrotate", "cron", "rsyslog"
],
"gaming": [
"steam", "wine", "lutris", "gamemode", "mangohud",
"nvidia-driver", "mesa-utils", "pulseaudio"
],
"development": [
"build-essential", "git", "python3", "nodejs", "rustc",
"docker.io", "vscode", "eclipse"
],
"desktop": [
"firefox", "libreoffice", "gimp", "vlc", "thunderbird",
"file-roller", "gnome-tweaks"
]
}
with open(tasks_file, 'r') as f:
data = yaml.safe_load(f)
print(f"Parsed {len(tasks)} package groups")
return tasks
# Extract package groups from tasks
package_groups = {}
if 'tasks' in data:
for task in data['tasks']:
if 'name' in task and 'packages' in task:
package_groups[task['name']] = task['packages']
print(f"Parsed {len(package_groups)} package groups")
return package_groups
except Exception as e:
print(f"Error parsing tasks file: {e}")
return {}
def load_variant_configs(self) -> Dict[str, Dict]:
def load_variant_configs(self) -> dict:
"""Load existing variant configurations"""
configs = {}
@ -116,82 +54,53 @@ class DebianAtomicCompsSync:
return configs
def update_variant_packages(self, variant_name: str, package_groups: Dict[str, List[str]]) -> Dict:
def update_variant_packages(self, variant_name: str, package_groups: dict) -> dict:
"""Update variant with new package groups"""
variant = self.variants[variant_name]
updated_packages = []
# Add packages from relevant groups
# Find matching package group for this variant
matching_group = None
for group_name, packages in package_groups.items():
if any(group in variant["groups"] for group in [group_name]):
updated_packages.extend(packages)
if variant_name in group_name:
matching_group = packages
break
# Remove duplicates and sort
updated_packages = sorted(list(set(updated_packages)))
if not matching_group:
print(f" No matching package group found for {variant_name}")
return {"packages": []}
# Create updated configuration
config = {
"include": "common.yaml",
"ref": f"particle-os/{variant_name}",
"packages": updated_packages,
"packages": matching_group,
"metadata": {
"variant": variant_name,
"description": variant["description"],
"groups": variant["groups"]
"description": f"Debian Atomic {variant_name} variant"
}
}
return config
def generate_common_config(self) -> Dict:
"""Generate common configuration for all variants"""
return {
"repos": ["debian-stable", "debian-security"],
"packages": [
"systemd", "ostree", "grub2", "bash", "coreutils",
"network-manager", "podman", "skopeo"
],
"metadata": {
"project": "Particle-OS",
"type": "atomic",
"base": "debian"
}
}
def save_configs(self, configs: Dict[str, Dict], dry_run: bool = True):
def save_configs(self, configs: dict, dry_run: bool = True):
"""Save variant configurations to treefiles"""
if dry_run:
print("\n=== DRY RUN - No files will be modified ===")
# Save common configuration
common_config = self.generate_common_config()
common_file = self.treefiles_dir / "common.yaml"
if not dry_run:
with open(common_file, 'w') as f:
yaml.dump(common_config, f, default_flow_style=False, indent=2)
print(f"Saved: {common_file}")
else:
print(f"Would save: {common_file}")
print("Content:")
print(yaml.dump(common_config, default_flow_style=False, indent=2))
# Save variant configurations
for variant_name, config in configs.items():
config_file = self.treefiles_dir / f"{variant_name}.yaml"
if not dry_run:
with open(config_file, 'w') as f:
yaml.dump(config, f, default_flow_style=False, indent=2)
print(f"Saved: {config_file}")
else:
print(f"\nWould save: {config_file}")
print("Content:")
print(yaml.dump(config, default_flow_style=False, indent=2))
if config.get("packages"): # Only save if we have packages
config_file = self.treefiles_dir / f"{variant_name}.yaml"
if not dry_run:
with open(config_file, 'w') as f:
yaml.dump(config, f, default_flow_style=False, indent=2)
print(f"Saved: {config_file}")
else:
print(f"\nWould save: {config_file}")
print("Content:")
print(yaml.dump(config, default_flow_style=False, indent=2))
def sync_packages(self, tasks_file: str, save: bool = False):
"""Main sync function"""
print("Particle-OS Comps Sync")
print("Debian Atomic Comps Sync")
print("======================")
# Parse Debian tasks
@ -212,18 +121,21 @@ class DebianAtomicCompsSync:
)
# Show changes
old_packages = existing_configs.get(variant_name, {}).get("packages", [])
new_packages = updated_configs[variant_name]["packages"]
old_packages = existing_configs.get(variant_name, {}).get("packages", []) or []
new_packages = updated_configs[variant_name].get("packages", []) or []
added = set(new_packages) - set(old_packages)
removed = set(old_packages) - set(new_packages)
if added:
print(f" Added packages: {', '.join(sorted(added))}")
if removed:
print(f" Removed packages: {', '.join(sorted(removed))}")
if not added and not removed:
print(" No changes")
if new_packages:
added = set(new_packages) - set(old_packages)
removed = set(old_packages) - set(new_packages)
if added:
print(f" Added packages: {', '.join(sorted(added))}")
if removed:
print(f" Removed packages: {', '.join(sorted(removed))}")
if not added and not removed:
print(" No changes")
else:
print(" No packages to add")
# Save configurations
self.save_configs(updated_configs, dry_run=not save)
@ -240,7 +152,7 @@ class DebianAtomicCompsSync:
def main():
parser = argparse.ArgumentParser(
description="Sync Debian package groups with Particle-OS variants"
description="Sync Debian package groups with Debian Atomic variants"
)
parser.add_argument(
"tasks_file",
@ -254,7 +166,7 @@ def main():
parser.add_argument(
"--repo-path",
default=".",
help="Path to Particle-OS repository (default: current directory)"
help="Path to Debian Atomic repository (default: current directory)"
)
args = parser.parse_args()
@ -264,8 +176,8 @@ def main():
print(f"Error: Tasks file not found: {args.tasks_file}")
sys.exit(1)
# Initialize sync
sync = DebianAtomicCompsSync(args.repo_path)
# Initialize sync
sync = DebianAtomicCompsSync(args.repo_path)
# Perform sync
try: