From d6b7d5a1dee03e4c467e3d8a387938456ab15178 Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 29 Aug 2025 17:59:43 -0700 Subject: [PATCH] Refactor CI workflow and add build script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ✅ Refactored CI workflow to use external build script - ✅ Added comprehensive .gitignore for build artifacts - ✅ Created scripts/build-debian-packages.sh for clean package building - ✅ Fixed YAML syntax issues and removed embedded heredocs - ✅ Added proper build dependencies (python3-setuptools) - ✅ Script successfully builds all 9 Debian packages locally - ✅ Ready for CI/CD pipeline testing --- .forgejo/workflows/ci.yml | 282 +------------------------------ .gitignore | 16 ++ scripts/build-debian-packages.sh | 279 ++++++++++++++++++++++++++++++ 3 files changed, 304 insertions(+), 273 deletions(-) create mode 100755 scripts/build-debian-packages.sh diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 6270bd96..013e73dd 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -72,7 +72,7 @@ jobs: git curl pkg-config build-essential gnupg wget \ python3-dev python3-pip python3-setuptools python3-wheel \ python3-venv python3-pytest python3-tox python3-coverage \ - devscripts debhelper dh-python python3-all \ + devscripts debhelper dh-python python3-all python3-setuptools \ libapt-pkg-dev libapt-pkg7.0 libostree-dev \ libssl-dev libdbus-1-dev libglib2.0-dev \ libpolkit-gobject-1-dev libzstd-dev \ @@ -109,10 +109,13 @@ jobs: # Install development dependencies pip install pytest pytest-cov tox black flake8 mypy + # Install additional build dependencies + pip install --upgrade setuptools wheel + - name: Run tests run: | source venv/bin/activate - + # Run Python tests echo "Running Python tests..." python -m pytest test/ -v --cov=osbuild --cov-report=html @@ -125,279 +128,12 @@ jobs: echo "Running integration tests..." python -m pytest test/ -k "integration" -v - - name: Build Debian package + - name: Build Debian packages run: | - echo "Building Debian package..." + echo "Building Debian packages using external script..." - # Get build information for versioning - BUILD_NUMBER="${FORGEJO_RUN_NUMBER:-${GITEA_RUN_NUMBER:-$(date +%Y%m%d%H%M%S)}}" - COMMIT_HASH=$(git rev-parse HEAD 2>/dev/null || echo "unknown") - SHORT_COMMIT=$(echo "$COMMIT_HASH" | cut -c1-10) - - # Extract version from setup.py or setup.cfg - extract_version() { - local version="" - - # Try setup.cfg first - if [ -f "setup.cfg" ]; then - version=$(grep "^version" setup.cfg | cut -d'=' -f2 | tr -d ' ') - [ -n "$version" ] && echo "$version" && return 0 - fi - - # Try setup.py - if [ -f "setup.py" ]; then - version=$(grep "version=" setup.py | sed 's/.*version="\([^"]*\)".*/\1/') - [ -n "$version" ] && echo "$version" && return 0 - fi - - # Try debian/changelog - if [ -f "debian/changelog" ]; then - version=$(sed -nE 's/.*\(([^)]+)\).*/\1/p' debian/changelog | head -n1) - [ -n "$version" ] && echo "$version" && return 0 - fi - - # Ultimate fallback - echo "0.1.0" - } - - PROJECT_VERSION=$(extract_version) - BUILD_VERSION="${PROJECT_VERSION}+build${BUILD_NUMBER}.${SHORT_COMMIT}" - - echo "Build Version: $BUILD_VERSION" - echo "Project Version: $PROJECT_VERSION" - echo "Build Number: $BUILD_NUMBER" - echo "Commit Hash: $SHORT_COMMIT" - - # Create debian directory structure if it doesn't exist - if [ ! -d "debian" ]; then - echo "Creating debian directory structure..." - mkdir -p debian - fi - - # Create control file for main package and sub-packages - cat > debian/control << EOF -Source: debian-forge -Section: admin -Priority: optional -Maintainer: Particle OS -Build-Depends: debhelper (>= 13), dh-python, python3-all, python3-setuptools -Standards-Version: 4.6.2 - -Package: debian-forge -Architecture: all -Depends: \${python3:Depends}, \${misc:Depends}, - python3-debian-forge (= \${binary:Version}), - debian-forge-tools (= \${binary:Version}) -Description: Debian-specific fork of osbuild for Debian Atomic systems - Debian Forge is a 1:1 implementation of osbuild with Debian-specific - optimizations and support. It provides the core engine for parsing - build manifests and executing build stages in the correct order. - . - This package contains the main debian-forge command (via Python entry point) - and core functionality. - -Package: python3-debian-forge -Architecture: all -Depends: \${python3:Depends}, \${misc:Depends}, - jsonschema, requests, psutil -Description: Python library for debian-forge - This package contains the Python library that constitutes the core - of the debian-forge project. It provides the main API and utilities - for building Debian-based system images. - -Package: debian-forge-depsolve-deb -Architecture: all -Depends: \${python3:Depends}, \${misc:Depends}, - python3-debian-forge (= \${binary:Version}), - libapt-pkg-dev -Description: Dependency solver for Debian packages - This package provides the "Dependency Solver" stage that integrates - with apt to resolve package dependencies and create complete lists - of all DEBs needed for image builds. - -Package: debian-forge-ostree -Architecture: all -Depends: \${python3:Depends}, \${misc:Depends}, - python3-debian-forge (= \${binary:Version}), - ostree -Description: OSTree support for debian-forge - This package provides stages necessary to interact with OSTree, - including creating OSTree repositories, committing filesystem trees, - and configuring images for OSTree deployment. - -Package: debian-forge-luks2 -Architecture: all -Depends: \${python3:Depends}, \${misc:Depends}, - python3-debian-forge (= \${binary:Version}), - cryptsetup -Description: LUKS2 encryption support for debian-forge - This package adds support for creating encrypted disk images using - the LUKS2 standard, including partitioning, formatting, and setting - up LUKS2 containers. - -Package: debian-forge-lvm2 -Architecture: all -Depends: \${python3:Depends}, \${misc:Depends}, - python3-debian-forge (= \${binary:Version}), - lvm2 -Description: LVM2 support for debian-forge - This package provides support for Logical Volume Management (LVM), - including stages to create physical volumes, volume groups, and - logical volumes within disk images. - -Package: debian-forge-selinux -Architecture: all -Depends: \${python3:Depends}, \${misc:Depends}, - python3-debian-forge (= \${binary:Version}), - selinux-policy-default -Description: SELinux support for debian-forge - This package provides tools and policies to correctly set and manage - SELinux labels during the build process, ensuring proper security - context for all files in the resulting image. - -Package: debian-forge-apparmor -Architecture: all -Depends: \${python3:Depends}, \${misc:Depends}, - python3-debian-forge (= \${binary:Version}), - apparmor-utils, apparmor-profiles -Description: AppArmor support for debian-forge - This package provides tools and profiles to correctly set and manage - AppArmor security policies during the build process, ensuring proper - security context for all files in the resulting image. - AppArmor is the preferred security framework for Debian systems. - -Package: debian-forge-tools -Architecture: all -Depends: \${python3:Depends}, \${misc:Depends}, - python3-debian-forge (= \${binary:Version}) -Description: Helper tools for debian-forge - This package contains miscellaneous helper tools and utilities used - internally by the debian-forge project, packaged separately for - modularity and optional installation. -EOF - - # Create rules file - cat > debian/rules << 'EOF' -#!/usr/bin/make -f - -%: - dh $@ --with python3 - -override_dh_auto_install: - dh_auto_install - # Install main binary (Python entry point) - # The debian-forge command is created via Python console_scripts entry point - # No need to manually copy binary - dh_python3 handles this - - # Install Python package - mkdir -p debian/python3-debian-forge/usr/lib/python3/dist-packages - cp -r osbuild debian/python3-debian-forge/usr/lib/python3/dist-packages/ - - # Install sub-package specific files - # OSTree stages - mkdir -p debian/debian-forge-ostree/usr/lib/python3/dist-packages/osbuild/stages - cp stages/org.osbuild.ostree.* debian/debian-forge-ostree/usr/lib/python3/dist-packages/osbuild/stages/ 2>/dev/null || true - - # LUKS2 stages - mkdir -p debian/debian-forge-luks2/usr/lib/python3/dist-packages/osbuild/stages - cp stages/org.osbuild.luks2.* debian/debian-forge-luks2/usr/lib/python3/dist-packages/osbuild/stages/ 2>/dev/null || true - - # LVM2 stages - mkdir -p debian/debian-forge-lvm2/usr/lib/python3/dist-packages/osbuild/stages - cp stages/org.osbuild.lvm2.* debian/debian-forge-lvm2/usr/lib/python3/dist-packages/osbuild/stages/ 2>/dev/null || true - - # SELinux stages - mkdir -p debian/debian-forge-selinux/usr/lib/python3/dist-packages/osbuild/stages - cp stages/org.osbuild.selinux.* debian/debian-forge-selinux/usr/lib/python3/dist-packages/osbuild/stages/ 2>/dev/null || true - - # AppArmor stages - mkdir -p debian/debian-forge-apparmor/usr/lib/python3/dist-packages/osbuild/stages - cp stages/org.osbuild.apparmor.* debian/debian-forge-apparmor/usr/lib/python3/dist-packages/osbuild/stages/ 2>/dev/null || true - - # Dependency solver - mkdir -p debian/debian-forge-depsolve-deb/usr/lib/python3/dist-packages/osbuild/stages - cp stages/org.osbuild.apt.* debian/debian-forge-depsolve-deb/usr/lib/python3/dist-packages/osbuild/stages/ 2>/dev/null || true - - # Tools - mkdir -p debian/debian-forge-tools/usr/bin - cp tools/* debian/debian-forge-tools/usr/bin/ 2>/dev/null || true - chmod +x debian/debian-forge-tools/usr/bin/* 2>/dev/null || true - -override_dh_auto_test: - # Skip tests during package build - true -EOF - - chmod +x debian/rules - - # Create changelog - cat > debian/changelog << EOF -debian-forge ($BUILD_VERSION) unstable; urgency=medium - - * CI Build #$BUILD_NUMBER from commit $COMMIT_HASH - * Automated build with comprehensive sub-package support - * Includes: core, ostree, luks2, lvm2, selinux, apparmor, depsolve-deb, and tools packages - - -- CI Bot $(date -R) -EOF - - # Create compat file - echo "13" > debian/compat - - # Create copyright file - cat > debian/copyright << 'EOF' -Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Upstream-Name: debian-forge -Source: https://git.raines.xyz/particle-os/debian-forge - -Files: * -Copyright: 2024 Particle OS -License: Apache-2.0 - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - . - http://www.apache.org/licenses/LICENSE-2.0 - . - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -Files: osbuild/* -Copyright: 2024 Red Hat, Inc. -License: Apache-2.0 - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - . - http://www.apache.org/licenses/LICENSE-2.0 - . - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -EOF - - # Build packages - echo "Building Debian packages..." - dpkg-buildpackage -b -us -uc - - # Check if packages were created - if ls ../*.deb >/dev/null 2>&1; then - echo "✅ Debian packages created successfully" - ls -la ../*.deb - - # Copy packages to current directory - cp ../*.deb . - echo "✅ Packages copied to current directory" - ls -la *.deb - else - echo "❌ No Debian packages found" - exit 1 - fi + # Run the external build script + ./scripts/build-debian-packages.sh - name: Test built packages run: | diff --git a/.gitignore b/.gitignore index fa37020e..7efa1352 100644 --- a/.gitignore +++ b/.gitignore @@ -69,6 +69,22 @@ build/ dist/ *.deb +# Debian build artifacts +debian/ +debian-*/ +*.buildinfo +*.changes +*.dsc +*.tar.xz +*.tar.gz + +# CI/CD artifacts +artifacts/ +build-logs/ +build-environments/ +*.tar.gz +*.zip + # Test and coverage files test-output/ coverage/ diff --git a/scripts/build-debian-packages.sh b/scripts/build-debian-packages.sh new file mode 100755 index 00000000..a7767253 --- /dev/null +++ b/scripts/build-debian-packages.sh @@ -0,0 +1,279 @@ +#!/bin/bash +# Debian Forge Package Building Script +# This script creates the debian directory structure and builds packages + +set -e + +echo "Building Debian packages..." + +# Get build information for versioning +BUILD_NUMBER="${FORGEJO_RUN_NUMBER:-${GITEA_RUN_NUMBER:-$(date +%Y%m%d%H%M%S)}}" +COMMIT_HASH=$(git rev-parse HEAD 2>/dev/null || echo "unknown") +SHORT_COMMIT=$(echo "$COMMIT_HASH" | cut -c1-10) + +# Extract version from setup.py or setup.cfg +extract_version() { + local version="" + + # Try setup.cfg first + if [ -f "setup.cfg" ]; then + version=$(grep "^version" setup.cfg | cut -d'=' -f2 | tr -d ' ') + [ -n "$version" ] && echo "$version" && return 0 + fi + + # Try setup.py + if [ -f "setup.py" ]; then + version=$(grep "version=" setup.py | sed 's/.*version="\([^"]*\)".*/\1/') + [ -n "$version" ] && echo "$version" && return 0 + fi + + # Try debian/changelog + if [ -f "debian/changelog" ]; then + version=$(sed -nE 's/.*\(([^)]+)\).*/\1/p' debian/changelog | head -n1) + [ -n "$version" ] && echo "$version" && return 0 + fi + + # Ultimate fallback + echo "0.1.0" +} + +PROJECT_VERSION=$(extract_version) +BUILD_VERSION="${PROJECT_VERSION}+build${BUILD_NUMBER}.${SHORT_COMMIT}" + +echo "Build Version: $BUILD_VERSION" +echo "Project Version: $PROJECT_VERSION" +echo "Build Number: $BUILD_NUMBER" +echo "Commit Hash: $SHORT_COMMIT" + +# Create debian directory structure if it doesn't exist +if [ ! -d "debian" ]; then + echo "Creating debian directory structure..." + mkdir -p debian +fi + +# Create control file for main package and sub-packages +cat > debian/control << 'EOF' +Source: debian-forge +Section: admin +Priority: optional +Maintainer: Particle OS +Build-Depends: debhelper (>= 13), dh-python, python3-all, python3-setuptools +Standards-Version: 4.6.2 + +Package: debian-forge +Architecture: all +Depends: ${python3:Depends}, ${misc:Depends}, + python3-debian-forge (= ${binary:Version}), + debian-forge-tools (= ${binary:Version}) +Description: Debian-specific fork of osbuild for Debian Atomic systems + Debian Forge is a 1:1 implementation of osbuild with Debian-specific + optimizations and support. It provides the core engine for parsing + build manifests and executing build stages in the correct order. + . + This package contains the main debian-forge command (via Python entry point) + and core functionality. + +Package: python3-debian-forge +Architecture: all +Depends: ${python3:Depends}, ${misc:Depends}, + jsonschema, requests, psutil +Description: Python library for debian-forge + This package contains the Python library that constitutes the core + of the debian-forge project. It provides the main API and utilities + for building Debian-based system images. + +Package: debian-forge-depsolve-deb +Architecture: all +Depends: ${python3:Depends}, ${misc:Depends}, + python3-debian-forge (= ${binary:Version}), + libapt-pkg-dev +Description: Dependency solver for Debian packages + This package provides the "Dependency Solver" stage that integrates + with apt to resolve package dependencies and create complete lists + of all DEBs needed for image builds. + +Package: debian-forge-ostree +Architecture: all +Depends: ${python3:Depends}, ${misc:Depends}, + python3-debian-forge (= ${binary:Version}), + ostree +Description: OSTree support for debian-forge + This package provides stages necessary to interact with OSTree, + including creating OSTree repositories, committing filesystem trees, + and configuring images for OSTree deployment. + +Package: debian-forge-luks2 +Architecture: all +Depends: ${python3:Depends}, ${misc:Depends}, + python3-debian-forge (= ${binary:Version}), + cryptsetup +Description: LUKS2 encryption support for debian-forge + This package adds support for creating encrypted disk images using + the LUKS2 standard, including partitioning, formatting, and setting + up LUKS2 containers. + +Package: debian-forge-lvm2 +Architecture: all +Depends: ${python3:Depends}, ${misc:Depends}, + python3-debian-forge (= ${binary:Version}), + lvm2 +Description: LVM2 support for debian-forge + This package provides support for Logical Volume Management (LVM), + including stages to create physical volumes, volume groups, and + logical volumes within disk images. + +Package: debian-forge-selinux +Architecture: all +Depends: ${python3:Depends}, ${misc:Depends}, + python3-debian-forge (= ${binary:Version}), + selinux-policy-default +Description: SELinux support for debian-forge + This package provides tools and policies to correctly set and manage + SELinux labels during the build process, ensuring proper security + context for all files in the resulting image. + +Package: debian-forge-apparmor +Architecture: all +Depends: ${python3:Depends}, ${misc:Depends}, + python3-debian-forge (= ${binary:Version}), + apparmor-utils, apparmor-profiles +Description: AppArmor support for debian-forge + This package provides tools and profiles to correctly set and manage + AppArmor security policies during the build process, ensuring proper + security context for all files in the resulting image. + AppArmor is the preferred security framework for Debian systems. + +Package: debian-forge-tools +Architecture: all +Depends: ${python3:Depends}, ${misc:Depends}, + python3-debian-forge (= ${binary:Version}) +Description: Helper tools for debian-forge + This package contains miscellaneous helper tools and utilities used + internally by the debian-forge project, packaged separately for + modularity and optional installation. +EOF + +# Create rules file +cat > debian/rules << 'EOF' +#!/usr/bin/make -f + +%: + dh $@ --with python3 + +override_dh_auto_install: + dh_auto_install + # Install main binary (Python entry point) + # The debian-forge command is created via Python console_scripts entry point + # No need to manually copy binary - dh_python3 handles this + + # Install Python package + mkdir -p debian/python3-debian-forge/usr/lib/python3/dist-packages + cp -r osbuild debian/python3-debian-forge/usr/lib/python3/dist-packages/ + + # Install sub-package specific files + # OSTree stages + mkdir -p debian/debian-forge-ostree/usr/lib/python3/dist-packages/osbuild/stages + cp stages/org.osbuild.ostree.* debian/debian-forge-ostree/usr/lib/python3/dist-packages/osbuild/stages/ 2>/dev/null || true + + # LUKS2 stages + mkdir -p debian/debian-forge-luks2/usr/lib/python3/dist-packages/osbuild/stages + cp stages/org.osbuild.luks2.* debian/debian-forge-luks2/usr/lib/python3/dist-packages/osbuild/stages/ 2>/dev/null || true + + # LVM2 stages + mkdir -p debian/debian-forge-lvm2/usr/lib/python3/dist-packages/osbuild/stages + cp stages/org.osbuild.lvm2.* debian/debian-forge-lvm2/usr/lib/python3/dist-packages/osbuild/stages/ 2>/dev/null || true + + # SELinux stages + mkdir -p debian/debian-forge-selinux/usr/lib/python3/dist-packages/osbuild/stages + cp stages/org.osbuild.selinux.* debian/debian-forge-selinux/usr/lib/python3/dist-packages/osbuild/stages/ 2>/dev/null || true + + # AppArmor stages + mkdir -p debian/debian-forge-apparmor/usr/lib/python3/dist-packages/osbuild/stages + cp stages/org.osbuild.apparmor.* debian/debian-forge-apparmor/usr/lib/python3/dist-packages/osbuild/stages/ 2>/dev/null || true + + # Dependency solver + mkdir -p debian/debian-forge-depsolve-deb/usr/lib/python3/dist-packages/osbuild/stages + cp stages/org.osbuild.apt.* debian/debian-forge-depsolve-deb/usr/lib/python3/dist-packages/osbuild/stages/ 2>/dev/null || true + + # Tools + mkdir -p debian/debian-forge-tools/usr/bin + cp tools/* debian/debian-forge-tools/usr/bin/ 2>/dev/null || true + chmod +x debian/debian-forge-tools/usr/bin/* 2>/dev/null || true + +override_dh_auto_test: + # Skip tests during package build + true +EOF + +chmod +x debian/rules + +# Create changelog +cat > debian/changelog << EOF +debian-forge ($BUILD_VERSION) unstable; urgency=medium + + * CI Build #$BUILD_NUMBER from commit $COMMIT_HASH + * Automated build with comprehensive sub-package support + * Includes: core, ostree, luks2, lvm2, selinux, apparmor, depsolve-deb, and tools packages + + -- CI Bot $(date -R) +EOF + +# Create compat file +echo "13" > debian/compat + +# Create copyright file +cat > debian/copyright << 'EOF' +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: debian-forge +Source: https://git.raines.xyz/particle-os/debian-forge + +Files: * +Copyright: 2024 Particle OS +License: Apache-2.0 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + http://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Files: osbuild/* +Copyright: 2024 Red Hat, Inc. +License: Apache-2.0 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + http://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +EOF + +# Build packages +echo "Building Debian packages..." +dpkg-buildpackage -b -us -uc + +# Check if packages were created +if ls ../*.deb >/dev/null 2>&1; then + echo "✅ Debian packages created successfully" + ls -la ../*.deb + + # Copy packages to current directory + cp ../*.deb . + echo "✅ Packages copied to current directory" + ls -la *.deb +else + echo "❌ No Debian packages found" + exit 1 +fi + +echo "✅ Package build completed successfully!"