From eb18f1a51451883b8cb2953a0a4b04b461561229 Mon Sep 17 00:00:00 2001 From: robojerk Date: Tue, 26 Aug 2025 15:52:43 -0700 Subject: [PATCH] Add dynamic apt-cacher-ng configuration system for collaborators --- .gitignore | 8 +- config/README.md | 164 ++++++++++++++++++++++++++++++ config/debian-forge.conf | 42 ++++++++ osbuild/config_manager.py | 140 +++++++++++++++++++++++++ stages/org.osbuild.apt.py | 43 ++++++-- stages/org.osbuild.debootstrap.py | 51 +++++++--- tools/debian-forge-config | 147 ++++++++++++++++++++++++++ 7 files changed, 570 insertions(+), 25 deletions(-) create mode 100644 config/README.md create mode 100644 config/debian-forge.conf create mode 100644 osbuild/config_manager.py create mode 100755 tools/debian-forge-config diff --git a/.gitignore b/.gitignore index f03edc60..fa37020e 100644 --- a/.gitignore +++ b/.gitignore @@ -131,9 +131,11 @@ cache/ .env.local .env.*.local -# Local configuration -config.local.* -*.local +# Local configuration files (collaborator-specific) +config/*.local.conf +config/*.local.* +*.local.conf +*.local.* # Backup files *.bak diff --git a/config/README.md b/config/README.md new file mode 100644 index 00000000..40aa3d1e --- /dev/null +++ b/config/README.md @@ -0,0 +1,164 @@ +# Debian Forge Configuration + +This directory contains configuration files for Debian Forge. The system is designed to be collaborative-friendly, allowing each developer to have their own settings without affecting others. + +## Configuration Files + +### `debian-forge.conf` (Default Configuration) +- **Purpose**: Default configuration values for the project +- **Status**: Tracked in git (shared by all collaborators) +- **Usage**: Contains sensible defaults and examples + +### `debian-forge.local.conf.example` (Template) +- **Purpose**: Template for local configuration +- **Status**: Tracked in git (shared template) +- **Usage**: Copy this file to create your local configuration + +### `debian-forge.local.conf` (Local Configuration) +- **Purpose**: Your personal configuration settings +- **Status**: **NOT tracked in git** (personal to each collaborator) +- **Usage**: Customize this file for your environment + +## Quick Start + +### 1. Set up your local configuration +```bash +# Create your local configuration file +./tools/debian-forge-config setup + +# This copies the example template to config/debian-forge.local.conf +# Edit this file to customize your settings +``` + +### 2. Configure apt-cacher-ng proxy +```bash +# Set your proxy URL +./tools/debian-forge-config apt-proxy http://localhost:3142 + +# Or disable proxy +./tools/debian-forge-config apt-proxy none + +# Or set via environment variable +export DEBIAN_FORGE_APT_PROXY=http://localhost:3142 +``` + +### 3. View current configuration +```bash +./tools/debian-forge-config show +``` + +## Configuration Options + +### apt-cacher-ng Section +```ini +[apt-cacher-ng] +# Set to your proxy URL, or leave empty to disable +url = http://localhost:3142 +``` + +**Examples:** +- `url = http://localhost:3142` - Local proxy +- `url = http://192.168.1.100:3142` - Network proxy +- `url = http://apt-cache.company.local:3142` - Company proxy +- `url = ` - No proxy (leave empty) + +### build Section +```ini +[build] +default_suite = bookworm +default_arch = amd64 +timeout = 3600 +jobs = 4 +``` + +### stages Section +```ini +[stages] +apt_update = true +apt_recommends = false +apt_unauthenticated = false +``` + +## Priority Order + +Configuration values are loaded in this priority order (highest to lowest): + +1. **Stage options** (passed directly to stages) +2. **Environment variables** (e.g., `DEBIAN_FORGE_APT_PROXY`) +3. **Local configuration** (`debian-forge.local.conf`) +4. **Default configuration** (`debian-forge.conf`) + +## Environment Variables + +You can override any configuration setting using environment variables: + +```bash +# Override apt-cacher-ng proxy +export DEBIAN_FORGE_APT_PROXY=http://localhost:3142 + +# Override default suite +export DEBIAN_FORGE_DEFAULT_SUITE=trixie + +# Override default architecture +export DEBIAN_FORGE_DEFAULT_ARCH=arm64 +``` + +## CLI Tool Usage + +The `debian-forge-config` tool provides several commands: + +```bash +# Set up local configuration +./tools/debian-forge-config setup + +# Show current configuration +./tools/debian-forge-config show + +# Set apt-cacher-ng proxy +./tools/debian-forge-config apt-proxy http://localhost:3142 + +# Set any configuration value +./tools/debian-forge-config set build default_suite trixie +``` + +## Collaboration Workflow + +### For New Collaborators: +1. Clone the repository +2. Run `./tools/debian-forge-config setup` +3. Edit `config/debian-forge.local.conf` with your settings +4. Your local settings won't affect others + +### For Existing Collaborators: +1. Your `debian-forge.local.conf` is already configured +2. Update settings as needed using the CLI tool +3. Your changes remain local + +### For Project Updates: +1. Default configuration changes are tracked in git +2. Your local overrides are preserved +3. You can merge new default settings as needed + +## File Locations + +- **Default config**: `config/debian-forge.conf` +- **Your local config**: `config/debian-forge.local.conf` (create this) +- **Template**: `config/debian-forge.local.conf.example` +- **CLI tool**: `tools/debian-forge-config` + +## Troubleshooting + +### Configuration not loading? +- Check that `config/debian-forge.local.conf` exists +- Verify file permissions +- Use `./tools/debian-forge-config show` to debug + +### Proxy not working? +- Verify your proxy URL is correct +- Check that apt-cacher-ng is running +- Use environment variable override for testing + +### Settings not taking effect? +- Remember the priority order +- Stage options override configuration files +- Use `./tools/debian-forge-config show` to see current values diff --git a/config/debian-forge.conf b/config/debian-forge.conf new file mode 100644 index 00000000..1bbea06d --- /dev/null +++ b/config/debian-forge.conf @@ -0,0 +1,42 @@ +# Debian Forge Configuration File +# This file contains configuration options that can be customized per environment +# Copy this file to config/debian-forge.local.conf and modify as needed +# The .local.conf file is ignored by git, so each collaborator can have their own settings + +[apt-cacher-ng] +# apt-cacher-ng proxy configuration +# Set to empty string or comment out to disable +# Examples: +# url = http://localhost:3142 +# url = http://192.168.1.100:3142 +# url = http://apt-cache.company.local:3142 +url = + +# Alternative: use environment variable +# Set DEBIAN_FORGE_APT_PROXY environment variable to override this setting +# export DEBIAN_FORGE_APT_PROXY=http://localhost:3142 + +[build] +# Build environment settings +# Default Debian suite to use +default_suite = bookworm + +# Default architecture +default_arch = amd64 + +# Build timeout (in seconds) +timeout = 3600 + +# Parallel build jobs +jobs = 4 + +[stages] +# Stage-specific settings +apt_update = true +apt_recommends = false +apt_unauthenticated = false + +[logging] +# Logging configuration +level = INFO +file = .osbuild/debian-forge.log diff --git a/osbuild/config_manager.py b/osbuild/config_manager.py new file mode 100644 index 00000000..427a4074 --- /dev/null +++ b/osbuild/config_manager.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +""" +Configuration Manager for Debian Forge +Handles reading configuration from multiple sources with priority order +""" + +import os +import configparser +from pathlib import Path +from typing import Optional, Dict, Any + + +class DebianForgeConfig: + """Configuration manager for Debian Forge settings""" + + def __init__(self, config_dir: str = "config"): + self.config_dir = Path(config_dir) + self.config = configparser.ConfigParser() + self._load_config() + + def _load_config(self): + """Load configuration from multiple sources with priority order""" + # Priority order (highest to lowest): + # 1. Environment variables + # 2. Local config file (debian-forge.local.conf) + # 3. Default config file (debian-forge.conf) + + # Load default config + default_config = self.config_dir / "debian-forge.conf" + if default_config.exists(): + self.config.read(default_config) + + # Load local config (overrides defaults) + local_config = self.config_dir / "debian-forge.local.conf" + if local_config.exists(): + self.config.read(local_config) + + # Environment variables override everything + self._apply_environment_overrides() + + def _apply_environment_overrides(self): + """Apply environment variable overrides""" + # apt-cacher-ng proxy + env_proxy = os.environ.get("DEBIAN_FORGE_APT_PROXY") + if env_proxy is not None: + if "apt-cacher-ng" not in self.config: + self.config.add_section("apt-cacher-ng") + self.config["apt-cacher-ng"]["url"] = env_proxy + + # Build settings + env_suite = os.environ.get("DEBIAN_FORGE_DEFAULT_SUITE") + if env_suite is not None: + if "build" not in self.config: + self.config.add_section("build") + self.config["build"]["default_suite"] = env_suite + + env_arch = os.environ.get("DEBIAN_FORGE_DEFAULT_ARCH") + if env_arch is not None: + if "build" not in self.config: + self.config.add_section("build") + self.config["build"]["default_arch"] = env_arch + + def get_apt_proxy(self) -> Optional[str]: + """Get apt-cacher-ng proxy URL if configured""" + try: + url = self.config.get("apt-cacher-ng", "url", fallback="") + return url.strip() if url.strip() else None + except (configparser.NoSectionError, configparser.NoOptionError): + return None + + def get_build_setting(self, key: str, default: Any = None) -> Any: + """Get a build setting value""" + try: + return self.config.get("build", key, fallback=default) + except (configparser.NoSectionError, configparser.NoOptionError): + return default + + def get_stage_setting(self, key: str, default: Any = None) -> Any: + """Get a stage setting value""" + try: + return self.config.get("stages", key, fallback=default) + except (configparser.NoSectionError, configparser.NoOptionError): + return default + + def get_logging_setting(self, key: str, default: Any = None) -> Any: + """Get a logging setting value""" + try: + return self.config.get("logging", key, fallback=default) + except (configparser.NoSectionError, configparser.NoOptionError): + return default + + def is_apt_proxy_enabled(self) -> bool: + """Check if apt-cacher-ng proxy is enabled""" + return self.get_apt_proxy() is not None + + def get_all_settings(self) -> Dict[str, Dict[str, Any]]: + """Get all configuration settings as a dictionary""" + result = {} + for section in self.config.sections(): + result[section] = dict(self.config[section]) + return result + + def print_config(self): + """Print current configuration for debugging""" + print("Debian Forge Configuration:") + print("=" * 40) + for section, settings in self.get_all_settings().items(): + print(f"\n[{section}]") + for key, value in settings.items(): + print(f" {key} = {value}") + + # Show environment overrides + env_proxy = os.environ.get("DEBIAN_FORGE_APT_PROXY") + if env_proxy: + print(f"\nEnvironment Override:") + print(f" DEBIAN_FORGE_APT_PROXY = {env_proxy}") + + +# Global configuration instance +config = DebianForgeConfig() + + +def get_apt_proxy() -> Optional[str]: + """Get apt-cacher-ng proxy URL (convenience function)""" + return config.get_apt_proxy() + + +def is_apt_proxy_enabled() -> bool: + """Check if apt-cacher-ng proxy is enabled (convenience function)""" + return config.is_apt_proxy_enabled() + + +def get_build_setting(key: str, default: Any = None) -> Any: + """Get a build setting (convenience function)""" + return config.get_build_setting(key, default) + + +def get_stage_setting(key: str, default: Any = None) -> Any: + """Get a stage setting (convenience function)""" + return config.get_stage_setting(key, default) diff --git a/stages/org.osbuild.apt.py b/stages/org.osbuild.apt.py index f2d2e1fa..6df90930 100644 --- a/stages/org.osbuild.apt.py +++ b/stages/org.osbuild.apt.py @@ -6,6 +6,17 @@ import json import osbuild.api +# Import our configuration manager +try: + from osbuild.config_manager import get_apt_proxy, is_apt_proxy_enabled +except ImportError: + # Fallback if config manager is not available + def get_apt_proxy(): + return None + + def is_apt_proxy_enabled(): + return False + def run_apt_command(tree, command, env=None): """Run apt command in the target filesystem""" @@ -32,6 +43,21 @@ def run_apt_command(tree, command, env=None): return True +def configure_apt_proxy(tree, proxy_url): + """Configure apt proxy if specified""" + if not proxy_url or not proxy_url.strip(): + return + + print(f"Configuring apt proxy: {proxy_url}") + proxy_config = f"""Acquire::http::Proxy "{proxy_url}"; +Acquire::https::Proxy "{proxy_url}"; +""" + proxy_file = os.path.join(tree, "etc/apt/apt.conf.d/99proxy") + os.makedirs(os.path.dirname(proxy_file), exist_ok=True) + with open(proxy_file, "w") as f: + f.write(proxy_config) + + def main(tree, options): """Main function for apt stage""" @@ -40,22 +66,21 @@ def main(tree, options): recommends = options.get("recommends", False) unauthenticated = options.get("unauthenticated", False) update = options.get("update", True) + + # Get apt proxy from multiple sources (priority order): + # 1. Stage options (highest priority) + # 2. Configuration file + # 3. Environment variable apt_proxy = options.get("apt_proxy") + if not apt_proxy: + apt_proxy = get_apt_proxy() if not packages: print("No packages specified for installation") return 1 # Configure apt proxy if specified - if apt_proxy: - print(f"Configuring apt proxy: {apt_proxy}") - proxy_config = f"""Acquire::http::Proxy "{apt_proxy}"; -Acquire::https::Proxy "{apt_proxy}"; -""" - proxy_file = os.path.join(tree, "etc/apt/apt.conf.d/99proxy") - os.makedirs(os.path.dirname(proxy_file), exist_ok=True) - with open(proxy_file, "w") as f: - f.write(proxy_config) + configure_apt_proxy(tree, apt_proxy) # Update package lists if requested if update: diff --git a/stages/org.osbuild.debootstrap.py b/stages/org.osbuild.debootstrap.py index 95680855..26414912 100644 --- a/stages/org.osbuild.debootstrap.py +++ b/stages/org.osbuild.debootstrap.py @@ -25,6 +25,17 @@ from osbuild import api from osbuild.util.mnt import mount from osbuild.util.runners import create_machine_id_if_needed +# Import our configuration manager +try: + from osbuild.config_manager import get_apt_proxy, get_build_setting +except ImportError: + # Fallback if config manager is not available + def get_apt_proxy(): + return None + + def get_build_setting(key, default=None): + return default + def run_debootstrap(suite, target, mirror, arch=None, variant=None, extra_packages=None, apt_proxy=None): """Run debootstrap to create base Debian filesystem""" @@ -77,6 +88,21 @@ deb {mirror} {suite}-security main print(f"Created sources.list for {suite}") +def configure_apt_proxy(tree, proxy_url): + """Configure apt proxy if specified""" + if not proxy_url or not proxy_url.strip(): + return + + print(f"Configuring apt proxy: {proxy_url}") + proxy_config = f"""Acquire::http::Proxy "{proxy_url}"; +Acquire::https::Proxy "{proxy_url}"; +""" + proxy_file = os.path.join(tree, "etc/apt/apt.conf.d/99proxy") + os.makedirs(os.path.dirname(proxy_file), exist_ok=True) + with open(proxy_file, "w") as f: + f.write(proxy_config) + + def generate_base_metadata(tree, suite, arch): """Generate metadata about the created base system""" @@ -128,13 +154,20 @@ def generate_base_metadata(tree, suite, arch): def main(tree, options): """Main function for debootstrap stage""" - # Get options - suite = options.get("suite", "bookworm") + # Get options with fallbacks to configuration + suite = options.get("suite", get_build_setting("default_suite", "bookworm")) mirror = options.get("mirror", "http://deb.debian.org/debian") - arch = options.get("arch") + arch = options.get("arch", get_build_setting("default_arch")) variant = options.get("variant", "minbase") extra_packages = options.get("extra_packages", []) + + # Get apt proxy from multiple sources (priority order): + # 1. Stage options (highest priority) + # 2. Configuration file + # 3. Environment variable apt_proxy = options.get("apt_proxy") + if not apt_proxy: + apt_proxy = get_apt_proxy() if not suite: print("No suite specified for debootstrap") @@ -154,15 +187,7 @@ def main(tree, options): setup_apt_sources(tree, suite, mirror) # Configure apt proxy if specified - if apt_proxy: - print(f"Configuring apt proxy: {apt_proxy}") - proxy_config = f"""Acquire::http::Proxy "{apt_proxy}"; -Acquire::https::Proxy "{apt_proxy}"; -""" - proxy_file = os.path.join(tree, "etc/apt/apt.conf.d/99proxy") - os.makedirs(os.path.dirname(proxy_file), exist_ok=True) - with open(proxy_file, "w") as f: - f.write(proxy_config) + configure_apt_proxy(tree, apt_proxy) # Mount essential filesystems for package operations (similar to rpm stage) for source in ("/dev", "/sys", "/proc"): @@ -189,6 +214,6 @@ Acquire::https::Proxy "{apt_proxy}"; if __name__ == '__main__': - args = osbuild.api.arguments() + args = api.arguments() r = main(args["tree"], args["options"]) sys.exit(r) diff --git a/tools/debian-forge-config b/tools/debian-forge-config new file mode 100755 index 00000000..4367e5df --- /dev/null +++ b/tools/debian-forge-config @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +""" +Debian Forge Configuration Manager CLI +Helps collaborators manage their local configuration settings +""" + +import os +import sys +import argparse +from pathlib import Path + +# Add the project root to the path so we can import our modules +project_root = Path(__file__).parent.parent +sys.path.insert(0, str(project_root)) + +try: + from osbuild.config_manager import config +except ImportError: + print("Error: Could not import configuration manager") + sys.exit(1) + + +def setup_local_config(): + """Set up local configuration file for the user""" + config_dir = Path("config") + local_config = config_dir / "debian-forge.local.conf" + example_config = config_dir / "debian-forge.local.conf.example" + + if local_config.exists(): + print(f"Local configuration already exists at {local_config}") + return + + if not example_config.exists(): + print(f"Example configuration not found at {example_config}") + return + + # Copy example to local config + import shutil + shutil.copy(example_config, local_config) + print(f"Created local configuration at {local_config}") + print("Please edit this file to customize your settings") + + +def show_config(): + """Show current configuration""" + config.print_config() + + +def set_apt_proxy(proxy_url): + """Set apt-cacher-ng proxy URL""" + config_dir = Path("config") + local_config = config_dir / "debian-forge.local.conf" + + # Create local config if it doesn't exist + if not local_config.exists(): + setup_local_config() + + # Read current config + import configparser + cfg = configparser.ConfigParser() + cfg.read(local_config) + + # Ensure apt-cacher-ng section exists + if "apt-cacher-ng" not in cfg: + cfg.add_section("apt-cacher-ng") + + # Set the proxy URL + if proxy_url.lower() in ["none", "disable", ""]: + cfg["apt-cacher-ng"]["url"] = "" + print("apt-cacher-ng proxy disabled") + else: + cfg["apt-cacher-ng"]["url"] = proxy_url + print(f"apt-cacher-ng proxy set to: {proxy_url}") + + # Write back to file + with open(local_config, 'w') as f: + cfg.write(f) + + +def set_build_setting(section, key, value): + """Set a build setting""" + config_dir = Path("config") + local_config = config_dir / "debian-forge.local.conf" + + # Create local config if it doesn't exist + if not local_config.exists(): + setup_local_config() + + # Read current config + import configparser + cfg = configparser.ConfigParser() + cfg.read(local_config) + + # Ensure section exists + if section not in cfg: + cfg.add_section(section) + + # Set the value + cfg[section][key] = value + print(f"Set {section}.{key} = {value}") + + # Write back to file + with open(local_config, 'w') as f: + cfg.write(f) + + +def main(): + parser = argparse.ArgumentParser(description="Debian Forge Configuration Manager") + subparsers = parser.add_subparsers(dest="command", help="Available commands") + + # Setup command + setup_parser = subparsers.add_parser("setup", help="Set up local configuration") + + # Show command + show_parser = subparsers.add_parser("show", help="Show current configuration") + + # Set apt-proxy command + apt_proxy_parser = subparsers.add_parser("apt-proxy", help="Set apt-cacher-ng proxy") + apt_proxy_parser.add_argument("url", nargs="?", default="", + help="Proxy URL (or 'none' to disable)") + + # Set build setting command + set_parser = subparsers.add_parser("set", help="Set a configuration value") + set_parser.add_argument("section", help="Configuration section (e.g., build, stages)") + set_parser.add_argument("key", help="Configuration key") + set_parser.add_argument("value", help="Configuration value") + + args = parser.parse_args() + + if not args.command: + parser.print_help() + return + + if args.command == "setup": + setup_local_config() + elif args.command == "show": + show_config() + elif args.command == "apt-proxy": + set_apt_proxy(args.url) + elif args.command == "set": + set_build_setting(args.section, args.key, args.value) + else: + parser.print_help() + + +if __name__ == "__main__": + main()