Fix YAML linting issues and update system requirements to Debian 13+

- Fix trailing spaces and blank lines in Forgejo workflows - Update system requirements from Ubuntu Jammy/Bookworm to Debian 13+ (Trixie) - Update test treefile to use Debian Trixie instead of Ubuntu Jammy - Update documentation to reflect modern system requirements - Fix yamllint errors for CI/CD functionality - Ensure compatibility with modern OSTree and libapt versions
2025-08-18 11:39:58 -07:00 · 2025-08-18 11:39:58 -07:00 · 3dec23f8f7
commit 3dec23f8f7
parent ec0da91864
85 changed files with 12569 additions and 1088 deletions
--- a/docs/apt-ostree-daemon-plan/development-commands-implementation.md
+++ b/docs/apt-ostree-daemon-plan/development-commands-implementation.md
@ -0,0 +1,687 @@
+# Development Commands Implementation Guide
+
+## Technical Implementation Details
+
+This document provides detailed technical specifications for implementing the missing development commands from rpm-ostree into apt-ostree.
+
+## 1. testutils Command Implementation
+
+### Command Structure
+```rust
+#[derive(Subcommand)]
+pub enum TestutilsSubcommands {
+    /// Inject package list metadata into OSTree commits
+    InjectPkglist(InjectPkglistArgs),
+    
+    /// Run scripts in bubblewrap containers
+    ScriptShell(ScriptShellArgs),
+    
+    /// Generate synthetic OS updates by modifying ELF files
+    GenerateSyntheticUpgrade(GenerateSyntheticUpgradeArgs),
+    
+    /// Run integration tests on booted machine
+    IntegrationReadOnly,
+    
+    /// Run C unit tests
+    CUnits,
+    
+    /// Test command for development verification
+    Moo,
+}
+```
+
+### Argument Structures
+```rust
+#[derive(Args)]
+pub struct InjectPkglistArgs {
+    /// Repository path
+    pub repo: String,
+    
+    /// OSTree reference
+    pub refspec: String,
+}
+
+#[derive(Args)]
+pub struct ScriptShellArgs {
+    /// Root path for script execution
+    #[arg(default_value = "/")]
+    pub rootpath: String,
+}
+
+#[derive(Args)]
+pub struct GenerateSyntheticUpgradeArgs {
+    /// Repository path
+    #[arg(long)]
+    pub repo: String,
+    
+    /// Source reference
+    #[arg(long = "srcref")]
+    pub src_ref: Option<String>,
+    
+    /// OSTree reference
+    #[arg(long = "ref")]
+    pub ostref: String,
+    
+    /// Percentage of binaries to modify
+    #[arg(long, default_value = "30")]
+    pub percentage: u32,
+    
+    /// Commit version
+    #[arg(long)]
+    pub commit_version: Option<String>,
+}
+```
+
+### Core Implementation Functions
+
+#### inject_pkglist
+```rust
+impl TestutilsCommand {
+    fn inject_pkglist(&self, args: &InjectPkglistArgs) -> AptOstreeResult<()> {
+        // 1. Parse refspec into remote and ref
+        let (remote, ref_name) = self.parse_refspec(&args.refspec)?;
+        
+        // 2. Open OSTree repository
+        let repo = OstreeRepo::open_at(AT_FDCWD, &args.repo)?;
+        
+        // 3. Resolve reference to commit
+        let checksum = repo.resolve_rev(&args.refspec, false)?;
+        
+        // 4. Load existing commit
+        let commit = repo.load_commit(&checksum)?;
+        
+        // 5. Check if pkglist already exists
+        if self.has_pkglist_metadata(&commit) {
+            println!("Refspec '{}' already has pkglist metadata; exiting.", args.refspec);
+            return Ok(());
+        }
+        
+        // 6. Create APT package list
+        let pkglist = self.create_apt_pkglist_variant(&repo, &checksum)?;
+        
+        // 7. Create new commit with pkglist metadata
+        let new_meta = self.add_pkglist_to_metadata(&commit, &pkglist)?;
+        
+        // 8. Write new commit
+        let new_checksum = self.write_new_commit(&repo, &checksum, &new_meta)?;
+        
+        // 9. Update reference
+        repo.set_ref_immediate(&remote, &ref_name, &new_checksum)?;
+        
+        println!("{} => {}", args.refspec, new_checksum);
+        Ok(())
+    }
+    
+    fn create_apt_pkglist_variant(&self, repo: &OstreeRepo, commit: &str) -> AptOstreeResult<GVariant> {
+        // Create APT package list from commit
+        // This replaces the RPM-specific logic with APT equivalents
+        let apt_manager = AptManager::new();
+        let packages = apt_manager.get_packages_from_commit(repo, commit)?;
+        
+        // Convert to GVariant format compatible with OSTree
+        self.packages_to_gvariant(&packages)
+    }
+}
+```
+
+#### script_shell
+```rust
+impl TestutilsCommand {
+    fn script_shell(&self, args: &ScriptShellArgs) -> AptOstreeResult<()> {
+        // 1. Open root filesystem directory
+        let rootfs_dfd = self.open_rootfs_dir(&args.rootpath)?;
+        
+        // 2. Run script in bubblewrap container
+        self.run_script_in_bwrap_container(
+            rootfs_dfd,
+            None,
+            true,
+            "testscript",
+            None,
+            None,
+            None,
+            None,
+            STDIN_FILENO,
+        )
+    }
+    
+    fn run_script_in_bwrap_container(
+        &self,
+        rootfs_dfd: i32,
+        env: Option<&[String]>,
+        read_only: bool,
+        script_name: &str,
+        user: Option<&str>,
+        group: Option<&str>,
+        cwd: Option<&str>,
+        extra_args: Option<&[String]>,
+        stdin_fd: i32,
+    ) -> AptOstreeResult<()> {
+        // Implement bubblewrap container execution
+        // This provides safe script execution environment
+        let mut cmd = Command::new("bwrap");
+        
+        // Add bubblewrap arguments for isolation
+        cmd.args(&[
+            "--dev-bind", "/", "/",
+            "--proc", "/proc",
+            "--tmpfs", "/tmp",
+        ]);
+        
+        if read_only {
+            cmd.arg("--ro-bind");
+        }
+        
+        // Execute script
+        cmd.arg("bash")
+            .arg("-c")
+            .arg(script_name)
+            .stdin(unsafe { std::os::unix::io::FromRawFd::from_raw_fd(stdin_fd) });
+        
+        let status = cmd.status()?;
+        if !status.success() {
+            return Err(AptOstreeError::System("Script execution failed".to_string()));
+        }
+        
+        Ok(())
+    }
+}
+```
+
+#### generate_synthetic_upgrade
+```rust
+impl TestutilsCommand {
+    fn generate_synthetic_upgrade(&self, args: &GenerateSyntheticUpgradeArgs) -> AptOstreeResult<()> {
+        // 1. Remount sysroot as read-write
+        self.remount_sysroot_rw()?;
+        
+        // 2. Create temporary directory
+        let tempdir = tempfile::tempdir_in(Path::new(&args.repo).join("tmp"))?;
+        let tmp_rootfs = tempdir.path().join("rootfs");
+        fs::create_dir(&tmp_rootfs)?;
+        
+        // 3. Create note file
+        let notepath = tempdir.path().join("note");
+        fs::write(&notepath, "Synthetic upgrade")?;
+        
+        // 4. Check for objcopy availability
+        let have_objcopy = Path::new("/usr/bin/objcopy").exists();
+        
+        // 5. Mutate executables
+        let mutated = self.mutate_executables(
+            &tmp_rootfs,
+            args.percentage,
+            &notepath,
+            have_objcopy,
+        )?;
+        
+        // 6. Create new OSTree commit
+        self.create_synthetic_commit(&args.repo, &args.ostref, &tmp_rootfs, &args.src_ref)?;
+        
+        println!("Mutated ELF files: {}", mutated);
+        Ok(())
+    }
+    
+    fn mutate_executables(
+        &self,
+        dest: &Path,
+        percentage: u32,
+        notepath: &Path,
+        have_objcopy: bool,
+    ) -> AptOstreeResult<u32> {
+        let mut mutated = 0;
+        let binary_dirs = &["usr/bin", "usr/lib", "usr/lib64"];
+        
+        for binary_dir in binary_dirs {
+            let src_path = Path::new("/").join(binary_dir);
+            if src_path.exists() {
+                let dest_path = dest.join(binary_dir);
+                fs::create_dir_all(&dest_path)?;
+                
+                mutated += self.mutate_executables_in_dir(
+                    &src_path,
+                    &dest_path,
+                    percentage,
+                    notepath,
+                    have_objcopy,
+                )?;
+            }
+        }
+        
+        Ok(mutated)
+    }
+    
+    fn mutate_executables_in_dir(
+        &self,
+        src: &Path,
+        dest: &Path,
+        percentage: u32,
+        notepath: &Path,
+        have_objcopy: bool,
+    ) -> AptOstreeResult<u32> {
+        let mut mutated = 0;
+        
+        for entry in fs::read_dir(src)? {
+            let entry = entry?;
+            let path = entry.path();
+            
+            if path.is_file() && self.is_elf_executable(&path)? {
+                if self.should_mutate(percentage) {
+                    self.mutate_one_executable(&path, dest, notepath, have_objcopy)?;
+                    mutated += 1;
+                }
+            }
+        }
+        
+        Ok(mutated)
+    }
+    
+    fn is_elf_executable(&self, path: &Path) -> AptOstreeResult<bool> {
+        let mut file = fs::File::open(path)?;
+        let mut buf = [0; 5];
+        
+        file.read_exact(&mut buf)?;
+        
+        Ok(buf[0] == 0x7F && &buf[1..4] == b"ELF")
+    }
+    
+    fn should_mutate(&self, percentage: u32) -> bool {
+        let mut rng = rand::thread_rng();
+        rng.gen_range(1..=100) <= percentage
+    }
+}
+```
+
+## 2. shlib-backend Command Implementation
+
+### Command Structure
+```rust
+#[derive(Subcommand)]
+pub enum ShlibBackendSubcommands {
+    /// Get base architecture
+    GetBasearch,
+    
+    /// Variable substitution for architecture
+    VarsubstBasearch {
+        /// Source string for substitution
+        source: String,
+    },
+    
+    /// Extract package list from OSTree commit
+    PackagelistFromCommit {
+        /// Commit hash
+        commit: String,
+    },
+}
+```
+
+### Core Implementation
+```rust
+impl ShlibBackendCommand {
+    fn handle_subcommand(&self, subcommand: &ShlibBackendSubcommands) -> AptOstreeResult<()> {
+        // 1. Create IPC socket
+        let ipc_sock = self.create_ipc_socket()?;
+        
+        // 2. Handle subcommand
+        let result = match subcommand {
+            ShlibBackendSubcommands::GetBasearch => self.get_basearch(),
+            ShlibBackendSubcommands::VarsubstBasearch { source } => {
+                self.varsubst_basearch(source)
+            }
+            ShlibBackendSubcommands::PackagelistFromCommit { commit } => {
+                self.packagelist_from_commit(commit)
+            }
+        }?;
+        
+        // 3. Send result via IPC
+        self.send_memfd_result(&ipc_sock, result)?;
+        
+        Ok(())
+    }
+    
+    fn get_basearch(&self) -> AptOstreeResult<GVariant> {
+        // Get base architecture using APT
+        let apt_manager = AptManager::new();
+        let arch = apt_manager.get_base_architecture()?;
+        
+        Ok(GVariant::new_string(arch))
+    }
+    
+    fn varsubst_basearch(&self, source: &str) -> AptOstreeResult<GVariant> {
+        // Get APT variable substitutions
+        let apt_manager = AptManager::new();
+        let varsubsts = apt_manager.get_variable_substitutions()?;
+        
+        // Perform variable substitution
+        let result = self.substitute_variables(source, &varsubsts)?;
+        
+        Ok(GVariant::new_string(result))
+    }
+    
+    fn packagelist_from_commit(&self, commit: &str) -> AptOstreeResult<GVariant> {
+        // 1. Open OSTree repository
+        let repo = OstreeRepo::open_at(AT_FDCWD, ".")?;
+        
+        // 2. Get package list from commit
+        let packages = self.get_packages_from_commit(&repo, commit)?;
+        
+        // 3. Convert to GVariant format
+        let pkglist = self.packages_to_gvariant(&packages)?;
+        
+        Ok(GVariant::new_maybe(
+            "aptostree.shlib.ipc.pkglist",
+            Some(&pkglist),
+        ))
+    }
+    
+    fn create_ipc_socket(&self) -> AptOstreeResult<GSocket> {
+        // Create IPC socket using file descriptor
+        let fd = std::env::var("APT_OSTREE_SHLIB_IPC_FD")
+            .ok()
+            .and_then(|s| s.parse::<i32>().ok())
+            .ok_or_else(|| {
+                AptOstreeError::System("APT_OSTREE_SHLIB_IPC_FD environment variable not set".to_string())
+            })?;
+        
+        GSocket::new_from_fd(fd)
+    }
+    
+    fn send_memfd_result(&self, ipc_sock: &GSocket, data: GVariant) -> AptOstreeResult<()> {
+        // 1. Create sealed memfd
+        let memfd = self.create_sealed_memfd("apt-ostree-shlib-backend", &data)?;
+        
+        // 2. Send via Unix domain socket
+        let fdarray = [memfd, -1];
+        let list = GUnixFDList::new_from_array(&fdarray, 1);
+        let message = GUnixFDMessage::new_with_fd_list(&list);
+        
+        let buffer = [0xFF];
+        let ov = GOutputVector {
+            buffer: &buffer,
+            size: buffer.len(),
+        };
+        
+        let sent = ipc_sock.send_message(
+            None,
+            &[ov],
+            &[&message],
+            GSocketMsgFlags::NONE,
+        )?;
+        
+        if sent != 1 {
+            return Err(AptOstreeError::System("Failed to send IPC message".to_string()));
+        }
+        
+        Ok(())
+    }
+}
+```
+
+## 3. internals Command Implementation
+
+### Command Structure
+```rust
+#[derive(Subcommand)]
+pub enum InternalsSubcommands {
+    /// Internal system diagnostics
+    Diagnostics,
+    
+    /// System state validation
+    ValidateState,
+    
+    /// Debug information dump
+    DebugDump,
+}
+```
+
+### Core Implementation
+```rust
+impl InternalsCommand {
+    fn handle_subcommand(&self, subcommand: &InternalsSubcommands) -> AptOstreeResult<()> {
+        match subcommand {
+            InternalsSubcommands::Diagnostics => self.run_diagnostics(),
+            InternalsSubcommands::ValidateState => self.validate_system_state(),
+            InternalsSubcommands::DebugDump => self.dump_debug_info(),
+        }
+    }
+    
+    fn run_diagnostics(&self) -> AptOstreeResult<()> {
+        println!("🔍 Running Internal Diagnostics");
+        println!("===============================");
+        
+        // Check system components
+        self.check_ostree_system()?;
+        self.check_apt_system()?;
+        self.check_daemon_status()?;
+        self.check_file_permissions()?;
+        
+        println!("Diagnostics completed successfully");
+        Ok(())
+    }
+    
+    fn validate_system_state(&self) -> AptOstreeResult<()> {
+        println!("✅ Validating System State");
+        println!("===========================");
+        
+        // Validate OSTree state
+        let ostree_manager = OstreeManager::new();
+        if ostree_manager.is_available() {
+            println!("OSTree: Available");
+            self.validate_ostree_state(&ostree_manager)?;
+        } else {
+            println!("OSTree: Not available");
+        }
+        
+        // Validate APT state
+        let apt_manager = AptManager::new();
+        self.validate_apt_state(&apt_manager)?;
+        
+        println!("System state validation completed");
+        Ok(())
+    }
+    
+    fn dump_debug_info(&self) -> AptOstreeResult<()> {
+        println!("🐛 Debug Information Dump");
+        println!("=========================");
+        
+        // System information
+        self.dump_system_info()?;
+        
+        // OSTree information
+        self.dump_ostree_info()?;
+        
+        // APT information
+        self.dump_apt_info()?;
+        
+        // Daemon information
+        self.dump_daemon_info()?;
+        
+        println!("Debug information dump completed");
+        Ok(())
+    }
+}
+```
+
+## 4. CLI Integration
+
+### Hidden Command Support
+```rust
+// Add to src/cli.rs
+#[derive(Subcommand)]
+pub enum Commands {
+    // ... existing commands ...
+    
+    /// Development and debugging tools (hidden)
+    #[command(hide = true)]
+    Testutils(TestutilsArgs),
+    
+    /// Shared library backend (hidden)
+    #[command(hide = true)]
+    ShlibBackend(ShlibBackendArgs),
+    
+    /// Internal system commands (hidden)
+    #[command(hide = true)]
+    Internals(InternalsArgs),
+}
+
+#[derive(Args)]
+pub struct TestutilsArgs {
+    #[command(subcommand)]
+    pub subcommand: TestutilsSubcommands,
+}
+
+#[derive(Args)]
+pub struct ShlibBackendArgs {
+    #[command(subcommand)]
+    pub subcommand: ShlibBackendSubcommands,
+}
+
+#[derive(Args)]
+pub struct InternalsArgs {
+    #[command(subcommand)]
+    pub subcommand: InternalsSubcommands,
+}
+```
+
+### Command Registration
+```rust
+// Add to src/commands/mod.rs
+pub mod testutils;
+pub mod shlib_backend;
+pub mod internals;
+
+// In register_commands function
+self.register(Box::new(testutils::TestutilsCommand::new()));
+self.register(Box::new(shlib_backend::ShlibBackendCommand::new()));
+self.register(Box::new(internals::InternalsCommand::new()));
+```
+
+### Main Dispatch
+```rust
+// Add to src/main.rs match statement
+cli::Commands::Testutils(args) => {
+    let args_vec = vec!["testutils".to_string()];
+    commands::testutils::TestutilsCommand::new().execute(&args_vec)
+},
+cli::Commands::ShlibBackend(args) => {
+    let args_vec = vec!["shlib-backend".to_string()];
+    commands::shlib_backend::ShlibBackendCommand::new().execute(&args_vec)
+},
+cli::Commands::Internals(args) => {
+    let args_vec = vec!["internals".to_string()];
+    commands::internals::InternalsCommand::new().execute(&args_vec)
+},
+```
+
+## 5. Dependencies and Features
+
+### Cargo.toml Additions
+```toml
+[dependencies]
+# For bubblewrap integration
+bubblewrap = "0.1"
+# For ELF file manipulation
+goblin = "0.8"
+# For random number generation
+rand = "0.8"
+# For temporary directories
+tempfile = "3.0"
+# For file operations
+cap-std = "1.0"
+cap-std-ext = "1.0"
+# For system calls
+libc = "0.2"
+```
+
+### Feature Flags
+```toml
+[features]
+# Development commands (hidden by default)
+development = ["bubblewrap", "goblin", "rand", "tempfile"]
+# Full development support
+dev-full = ["development", "cap-std", "cap-std-ext"]
+```
+
+## 6. Testing and Validation
+
+### Unit Tests
+```rust
+#[cfg(test)]
+mod tests {
+    use super::*;
+    
+    #[test]
+    fn test_inject_pkglist() {
+        // Test package list injection
+    }
+    
+    #[test]
+    fn test_script_shell() {
+        // Test script execution
+    }
+    
+    #[test]
+    fn test_synthetic_upgrade() {
+        // Test synthetic upgrade generation
+    }
+    
+    #[test]
+    fn test_shlib_backend() {
+        // Test shared library backend
+    }
+    
+    #[test]
+    fn test_internals() {
+        // Test internal commands
+    }
+}
+```
+
+### Integration Tests
+```rust
+#[cfg(test)]
+mod integration_tests {
+    use super::*;
+    
+    #[test]
+    fn test_full_development_workflow() {
+        // Test complete development workflow
+    }
+    
+    #[test]
+    fn test_debugging_tools() {
+        // Test debugging capabilities
+    }
+    
+    #[test]
+    fn test_system_validation() {
+        // Test system validation tools
+    }
+}
+```
+
+## 7. Security Considerations
+
+### Bubblewrap Integration
+- **Isolation**: Scripts run in isolated containers
+- **Resource Limits**: Memory and process constraints
+- **File Access**: Controlled filesystem access
+- **Network Access**: Restricted network access
+
+### IPC Security
+- **File Descriptors**: Secure descriptor passing
+- **Memory Protection**: Sealed memfd for data transfer
+- **Access Control**: Proper permission checking
+- **Input Validation**: Validate all IPC inputs
+
+### Package Operations
+- **Signature Verification**: Verify package signatures
+- **Repository Validation**: Validate repository sources
+- **Permission Checking**: Check operation permissions
+- **Audit Logging**: Log all package operations
+
+## Conclusion
+
+This implementation guide provides comprehensive technical specifications for integrating the missing development commands from rpm-ostree into apt-ostree. The commands maintain the same logical structure and behavior while adapting to APT-specific package management and Debian system conventions.
+
+The implementation includes proper security measures, comprehensive testing, and integration with the existing apt-ostree architecture. These development tools will significantly enhance the development, testing, and debugging capabilities of apt-ostree.