deb-bootc-image-builder/test/integration/test_full_pipeline.py

#!/usr/bin/env python3
"""
Integration Tests for Debian bootc-image-builder

This module contains comprehensive integration tests that test all Debian osbuild stages
together in a complete pipeline to ensure they work correctly together.

Author: Debian bootc-image-builder team
License: Same as original bootc-image-builder
"""

import unittest
import tempfile
import os
import json
import shutil
import subprocess
from unittest.mock import Mock, patch, MagicMock
import sys

# Add the osbuild-stages directory to the path
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', 'osbuild-stages'))

from apt_stage import AptStage
from debian_kernel_stage import DebianKernelStage
from debian_grub_stage import DebianGrubStage
from debian_filesystem_stage import DebianFilesystemStage


class TestFullPipeline(unittest.TestCase):
    """Integration tests for the complete Debian osbuild pipeline."""
    
    def setUp(self):
        """Set up test fixtures."""
        self.temp_dir = tempfile.mkdtemp()
        self.test_options = {
            'packages': ['linux-image-amd64', 'systemd', 'initramfs-tools', 'grub-efi-amd64'],
            'release': 'trixie',
            'arch': 'amd64',
            'repos': [
                {
                    'name': 'debian',
                    'url': 'http://deb.debian.org/debian',
                    'suite': 'trixie',
                    'components': ['main', 'contrib']
                }
            ]
        }
        
        # Create a mock context
        self.mock_context = Mock()
        self.mock_context.root = self.temp_dir
        
        # Mock the context.run method
        self.mock_context.run.return_value = Mock(
            returncode=0,
            stdout='',
            stderr=''
        )
    
    def tearDown(self):
        """Clean up test fixtures."""
        shutil.rmtree(self.temp_dir)
    
    def test_complete_pipeline_execution(self):
        """Test the complete pipeline execution with all stages."""
        logger.info("Testing complete pipeline execution")
        
        # Step 1: Filesystem Stage
        logger.info("Step 1: Executing Debian Filesystem Stage")
        filesystem_stage = DebianFilesystemStage({
            'rootfs_type': 'ext4',
            'ostree_integration': True,
            'home_symlink': True
        })
        filesystem_stage.run(self.mock_context)
        
        # Verify filesystem structure was created
        self.assertTrue(os.path.exists(os.path.join(self.temp_dir, 'etc')))
        self.assertTrue(os.path.exists(os.path.join(self.temp_dir, 'usr')))
        self.assertTrue(os.path.exists(os.path.join(self.temp_dir, 'var')))
        self.assertTrue(os.path.exists(os.path.join(self.temp_dir, 'home')))
        
        # Verify OSTree integration points
        self.assertTrue(os.path.exists(os.path.join(self.temp_dir, 'ostree')))
        self.assertTrue(os.path.exists(os.path.join(self.temp_dir, 'usr', 'lib', 'ostree-boot')))
        self.assertTrue(os.path.exists(os.path.join(self.temp_dir, 'etc', 'ostree')))
        
        # Verify home symlink
        home_path = os.path.join(self.temp_dir, 'home')
        self.assertTrue(os.path.islink(home_path))
        self.assertEqual(os.readlink(home_path), '../var/home')
        
        logger.info("✓ Filesystem stage completed successfully")
        
        # Step 2: APT Stage
        logger.info("Step 2: Executing APT Stage")
        apt_stage = AptStage(self.test_options)
        apt_stage.run(self.mock_context)
        
        # Verify APT configuration was created
        self.assertTrue(os.path.exists(os.path.join(self.temp_dir, 'etc', 'apt', 'apt.conf.d', '99osbuild')))
        self.assertTrue(os.path.exists(os.path.join(self.temp_dir, 'etc', 'apt', 'sources.list.d', 'debian.list')))
        
        logger.info("✓ APT stage completed successfully")
        
        # Step 3: Kernel Stage
        logger.info("Step 3: Executing Debian Kernel Stage")
        kernel_stage = DebianKernelStage({
            'kernel_package': 'linux-image-amd64',
            'initramfs_tools': True,
            'ostree_integration': True,
            'modules_autoload': True
        })
        kernel_stage.run(self.mock_context)
        
        # Verify kernel stage setup
        self.assertTrue(os.path.exists(os.path.join(self.temp_dir, 'etc', 'initramfs-tools')))
        self.assertTrue(os.path.exists(os.path.join(self.temp_dir, 'etc', 'modules-load.d')))
        
        logger.info("✓ Kernel stage completed successfully")
        
        # Step 4: GRUB Stage
        logger.info("Step 4: Executing Debian GRUB Stage")
        grub_stage = DebianGrubStage({
            'ostree_integration': True,
            'uefi': True,
            'secure_boot': False,
            'timeout': 5,
            'default_entry': 0
        })
        grub_stage.run(self.mock_context)
        
        # Verify GRUB configuration was created
        self.assertTrue(os.path.exists(os.path.join(self.temp_dir, 'etc', 'default', 'grub')))
        self.assertTrue(os.path.exists(os.path.join(self.temp_dir, 'etc', 'grub.d', '10_ostree')))
        
        logger.info("✓ GRUB stage completed successfully")
        
        logger.info("✓ Complete pipeline execution successful")
    
    def test_stage_dependencies(self):
        """Test that stages handle dependencies correctly."""
        logger.info("Testing stage dependencies")
        
        # Test that filesystem stage creates required directories for other stages
        filesystem_stage = DebianFilesystemStage({
            'rootfs_type': 'ext4',
            'ostree_integration': True,
            'home_symlink': True
        })
        filesystem_stage.run(self.mock_context)
        
        # Verify that APT stage can find its required directories
        apt_stage = AptStage(self.test_options)
        apt_stage.run(self.mock_context)
        
        # Verify that kernel stage can find its required directories
        kernel_stage = DebianKernelStage({
            'kernel_package': 'linux-image-amd64',
            'initramfs_tools': True,
            'ostree_integration': True
        })
        kernel_stage.run(self.mock_context)
        
        # Verify that GRUB stage can find its required directories
        grub_stage = DebianGrubStage({
            'ostree_integration': True,
            'uefi': True
        })
        grub_stage.run(self.mock_context)
        
        logger.info("✓ Stage dependencies handled correctly")
    
    def test_ostree_integration_consistency(self):
        """Test that OSTree integration is consistent across all stages."""
        logger.info("Testing OSTree integration consistency")
        
        # Execute all stages with OSTree integration
        filesystem_stage = DebianFilesystemStage({
            'ostree_integration': True,
            'home_symlink': True
        })
        filesystem_stage.run(self.mock_context)
        
        apt_stage = AptStage(self.test_options)
        apt_stage.run(self.mock_context)
        
        kernel_stage = DebianKernelStage({
            'ostree_integration': True,
            'initramfs_tools': True
        })
        kernel_stage.run(self.mock_context)
        
        grub_stage = DebianGrubStage({
            'ostree_integration': True,
            'uefi': True
        })
        grub_stage.run(self.mock_context)
        
        # Verify OSTree configuration consistency
        ostree_conf = os.path.join(self.temp_dir, 'etc', 'ostree', 'ostree.conf')
        self.assertTrue(os.path.exists(ostree_conf))
        
        # Verify OSTree boot directory structure
        ostree_boot = os.path.join(self.temp_dir, 'usr', 'lib', 'ostree-boot')
        self.assertTrue(os.path.exists(ostree_boot))
        
        # Verify GRUB OSTree configuration
        grub_ostree = os.path.join(self.temp_dir, 'etc', 'grub.d', '10_ostree')
        self.assertTrue(os.path.exists(grub_ostree))
        
        logger.info("✓ OSTree integration consistent across all stages")
    
    def test_error_handling(self):
        """Test error handling in the pipeline."""
        logger.info("Testing error handling")
        
        # Test with invalid options
        with self.assertRaises(ValueError):
            invalid_apt_stage = AptStage({})  # No packages specified
        
        # Test with missing context
        with self.assertRaises(AttributeError):
            filesystem_stage = DebianFilesystemStage({'ostree_integration': True})
            filesystem_stage.run(None)
        
        logger.info("✓ Error handling working correctly")
    
    def test_performance_optimization(self):
        """Test that the pipeline is optimized for performance."""
        logger.info("Testing performance optimization")
        
        import time
        
        # Measure execution time for each stage
        start_time = time.time()
        
        filesystem_stage = DebianFilesystemStage({
            'ostree_integration': True,
            'home_symlink': True
        })
        filesystem_stage.run(self.mock_context)
        filesystem_time = time.time() - start_time
        
        start_time = time.time()
        apt_stage = AptStage(self.test_options)
        apt_stage.run(self.mock_context)
        apt_time = time.time() - start_time
        
        start_time = time.time()
        kernel_stage = DebianKernelStage({
            'ostree_integration': True,
            'initramfs_tools': True
        })
        kernel_stage.run(self.mock_context)
        kernel_time = time.time() - start_time
        
        start_time = time.time()
        grub_stage = DebianGrubStage({
            'ostree_integration': True,
            'uefi': True
        })
        grub_stage.run(self.mock_context)
        grub_time = time.time() - start_time
        
        total_time = filesystem_time + apt_time + kernel_time + grub_time
        
        logger.info(f"Stage execution times:")
        logger.info(f"  Filesystem: {filesystem_time:.3f}s")
        logger.info(f"  APT: {apt_time:.3f}s")
        logger.info(f"  Kernel: {kernel_time:.3f}s")
        logger.info(f"  GRUB: {grub_time:.3f}s")
        logger.info(f"  Total: {total_time:.3f}s")
        
        # Verify reasonable performance (should complete in under 5 seconds for mock)
        self.assertLess(total_time, 5.0)
        
        logger.info("✓ Performance optimization verified")
    
    def test_filesystem_permissions(self):
        """Test that filesystem permissions are set correctly."""
        logger.info("Testing filesystem permissions")
        
        filesystem_stage = DebianFilesystemStage({
            'ostree_integration': True,
            'home_symlink': True
        })
        filesystem_stage.run(self.mock_context)
        
        # Test critical permission settings
        permissions_to_test = [
            ('/etc/passwd', 0o644),
            ('/etc/group', 0o644),
            ('/etc/shadow', 0o640),
            ('/root', 0o700),
            ('/tmp', 0o1777),
            ('/var/tmp', 0o1777)
        ]
        
        for path, expected_mode in permissions_to_test:
            full_path = os.path.join(self.temp_dir, path.lstrip('/'))
            if os.path.exists(full_path):
                actual_mode = oct(os.stat(full_path).st_mode)[-3:]
                expected_mode_str = oct(expected_mode)[-3:]
                self.assertEqual(actual_mode, expected_mode_str, 
                               f"Permission mismatch for {path}: expected {expected_mode_str}, got {actual_mode}")
        
        logger.info("✓ Filesystem permissions set correctly")
    
    def test_user_and_group_setup(self):
        """Test that users and groups are set up correctly."""
        logger.info("Testing user and group setup")
        
        filesystem_stage = DebianFilesystemStage({
            'ostree_integration': True,
            'home_symlink': True
        })
        filesystem_stage.run(self.mock_context)
        
        # Test passwd file
        passwd_file = os.path.join(self.temp_dir, 'etc', 'passwd')
        self.assertTrue(os.path.exists(passwd_file))
        
        with open(passwd_file, 'r') as f:
            passwd_content = f.read()
            self.assertIn('root:x:0:0:root:/root:/bin/bash', passwd_content)
            self.assertIn('debian:x:1000:1000:Debian User:/home/debian:/bin/bash', passwd_content)
        
        # Test group file
        group_file = os.path.join(self.temp_dir, 'etc', 'group')
        self.assertTrue(os.path.exists(group_file))
        
        with open(group_file, 'r') as f:
            group_content = f.read()
            self.assertIn('root:x:0:', group_content)
            self.assertIn('users:x:100:', group_content)
        
        # Test home directories
        root_home = os.path.join(self.temp_dir, 'var', 'home', 'root')
        debian_home = os.path.join(self.temp_dir, 'var', 'home', 'debian')
        
        self.assertTrue(os.path.exists(root_home))
        self.assertTrue(os.path.exists(debian_home))
        
        logger.info("✓ User and group setup correct")


class TestPipelineConfiguration(unittest.TestCase):
    """Tests for pipeline configuration and options."""
    
    def test_distribution_definition_parsing(self):
        """Test that distribution definitions can be parsed correctly."""
        logger.info("Testing distribution definition parsing")
        
        # Test parsing of debian-13.yaml
        yaml_file = os.path.join(os.path.dirname(__file__), '..', '..', 'bib', 'data', 'defs', 'debian-13.yaml')
        
        if os.path.exists(yaml_file):
            import yaml
            with open(yaml_file, 'r') as f:
                config = yaml.safe_load(f)
            
            # Verify basic structure
            self.assertIn('qcow2', config)
            self.assertIn('desktop', config)
            self.assertIn('server', config)
            self.assertIn('development', config)
            
            # Verify qcow2 configuration
            qcow2_config = config['qcow2']
            self.assertIn('packages', qcow2_config)
            self.assertIn('stages', qcow2_config)
            
            # Verify packages list
            packages = qcow2_config['packages']
            self.assertIn('linux-image-amd64', packages)
            self.assertIn('systemd', packages)
            self.assertIn('initramfs-tools', packages)
            
            logger.info("✓ Distribution definition parsing successful")
        else:
            logger.warning("Distribution definition file not found, skipping test")
    
    def test_stage_option_validation(self):
        """Test that stage options are validated correctly."""
        logger.info("Testing stage option validation")
        
        # Test APT stage validation
        with self.assertRaises(ValueError):
            AptStage({})  # No packages
        
        # Test valid APT stage
        valid_apt = AptStage({'packages': ['linux-image-amd64']})
        self.assertEqual(valid_apt.packages, ['linux-image-amd64'])
        
        # Test kernel stage validation
        valid_kernel = DebianKernelStage({
            'kernel_package': 'linux-image-amd64',
            'ostree_integration': True
        })
        self.assertEqual(valid_kernel.kernel_package, 'linux-image-amd64')
        self.assertTrue(valid_kernel.ostree_integration)
        
        # Test GRUB stage validation
        valid_grub = DebianGrubStage({
            'ostree_integration': True,
            'uefi': True
        })
        self.assertTrue(valid_grub.ostree_integration)
        self.assertTrue(valid_grub.uefi)
        
        # Test filesystem stage validation
        valid_fs = DebianFilesystemStage({
            'ostree_integration': True,
            'home_symlink': True
        })
        self.assertTrue(valid_fs.ostree_integration)
        self.assertTrue(valid_fs.home_symlink)
        
        logger.info("✓ Stage option validation working correctly")


if __name__ == '__main__':
    # Configure logging for tests
    import logging
    logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
    logger = logging.getLogger(__name__)
    
    unittest.main()