From 1e49de4997d0bbea6614b949112af5d8ff0964ae Mon Sep 17 00:00:00 2001 From: robojerk Date: Tue, 26 Aug 2025 09:25:35 -0700 Subject: [PATCH] did stuff --- .forgejo/workflows/ci.yml | 132 ++++++++++++ README.md | 304 +++++++-------------------- debian_blue_build/__init__.py | 21 ++ debian_blue_build/package_manager.py | 124 +++++++++++ debian_blue_build/recipe.py | 102 +++++++++ utils/src/command_output.rs | 12 +- 6 files changed, 466 insertions(+), 229 deletions(-) create mode 100644 .forgejo/workflows/ci.yml create mode 100644 debian_blue_build/__init__.py create mode 100644 debian_blue_build/package_manager.py create mode 100644 debian_blue_build/recipe.py diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml new file mode 100644 index 0000000..e10505d --- /dev/null +++ b/.forgejo/workflows/ci.yml @@ -0,0 +1,132 @@ +--- +name: Blue Build CLI CI/CD + +on: + push: + branches: [main, develop] + pull_request: + branches: [main] + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + DEBIAN_FRONTEND: noninteractive + +jobs: + build-and-package: + name: Build and Package Rust CLI + runs-on: ubuntu-latest + container: + image: rust:1.75-bullseye + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Rust environment + run: | + rustc --version + cargo --version + + - name: Install build dependencies + run: | + apt-get update + apt-get install -y \ + build-essential \ + devscripts \ + debhelper \ + git \ + ca-certificates \ + pkg-config + + - name: Download Rust dependencies + run: cargo fetch + + - name: Build CLI + run: | + cargo build --release + chmod +x target/release/blue-build-cli + + - name: Create debian directory + run: | + mkdir -p debian + cat > debian/control << EOF +Source: blue-build-cli +Section: utils +Priority: optional +Maintainer: Blue Build Team +Build-Depends: debhelper (>= 13), build-essential, pkg-config, git, ca-certificates +Standards-Version: 4.6.2 + +Package: blue-build-cli +Architecture: any +Depends: \${shlibs:Depends}, \${misc:Depends} +Description: Blue Build Command Line Interface + Blue Build CLI provides command-line tools for managing + blue-build recipes and building container images. +EOF + + cat > debian/rules << EOF +#!/usr/bin/make -f +%: + dh \$@ + +override_dh_auto_install: + dh_auto_install + mkdir -p debian/blue-build-cli/usr/bin + cp target/release/blue-build-cli debian/blue-build-cli/usr/bin/ +EOF + + cat > debian/changelog << EOF +blue-build-cli (1.0.0-1) unstable; urgency=medium + + * Initial release + * Blue Build CLI implementation + + -- Blue Build Team $(date -R) +EOF + + cat > debian/compat << EOF +13 +EOF + + chmod +x debian/rules + + - name: Build Debian package + run: | + dpkg-buildpackage -us -uc -b + ls -la ../*.deb + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: blue-build-cli-deb + path: ../*.deb + retention-days: 30 + + test: + name: Test Rust CLI + runs-on: ubuntu-latest + container: + image: rust:1.75-bullseye + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Rust environment + run: | + rustc --version + cargo --version + + - name: Download Rust dependencies + run: cargo fetch + + - name: Run tests + run: cargo test + + - name: Test CLI help + run: | + cargo build --release + ./target/release/blue-build-cli --help || echo "Help command not implemented yet" diff --git a/README.md b/README.md index a60a428..5085930 100644 --- a/README.md +++ b/README.md @@ -4,243 +4,101 @@ -# BlueBuild +# Debian Blue-Build CLI -BlueBuild's command line program that builds Containerfiles and custom images based on your recipe.yml. +A fork of blue-build/cli adapted for Debian package management and atomic image building. -## Requirements +## Overview -The `bluebuild` tool takes advantage of newer build features. Specifically bind, cache, and tmpfs mounts on the `RUN` instructions. We support using the following tools and their versions: +Debian Blue-Build CLI provides a command-line interface for building Debian atomic images using recipe-based configuration, maintaining compatibility with the original blue-build system while adding Debian-specific functionality. -- Docker - v23 and above -- Podman - v4 and above -- Buildah - v1.29 and above +## Features + +- Debian package management (APT-based) +- Recipe-based image configuration +- OSTree composition support +- Bootc container generation +- Multi-format image output ## Installation -Every image created with `bluebuild` comes with the CLI installed. If you have not built and booted a `bluebuild` created image, you can follow these instructions to install it. - -### Cargo - -This is the best way to install as it gives you the opportunity to build for your specific environment. - ```bash -cargo install --locked blue-build +# Clone the repository +git clone +cd blue-build-cli + +# Install dependencies +pip install -r requirements.txt + +# Install the CLI +pip install -e . ``` -### Podman/Docker +## Usage -This will install the binary on your system in `/usr/local/bin`. - -```bash -podman run --pull always --rm ghcr.io/blue-build/cli:latest-installer | bash -``` - -```bash -docker run --pull always --rm ghcr.io/blue-build/cli:latest-installer | bash -``` - -### Github Install Script - -```bash -bash <(curl -s https://raw.githubusercontent.com/blue-build/cli/main/install.sh) -``` - -### Distrobox - -We package an `alpine` image with all the tools needed to run `bluebuild`. You can use `distrobox` to run the application without needing to install it on your machine. You can clone this repo locally and run: - -```bash -distrobox assemble create -``` - -This will export `bluebuild` to your local machine and allow you to build images and test out your recipes. For security reasons, we keep this as a rootless image which means you will not be able to use this method to locally rebase to an image. If you want that capability, you should install the CLI tool directly. - -Refer to the [distrobox documentation](https://distrobox.it/usage/distrobox-export/) for more information. - -### Nix Flake - -You can install this CLI through the Nix flake on [Flakehub](https://flakehub.com/) - -#### Non-nixos - -You can install BlueBuild to your global package environment on non-nixos systems by running - -```shell -# you can replace "*" with a specific tag -nix profile install https://flakehub.com/f/bluebuild/cli/*.tar.gz#bluebuild -``` - -#### NixOS - -If you are using a dedicated flake to manage your dependencies, you can add BlueBuild as a flake input throught the [fh](https://github.com/DeterminateSystems/fh) cli (that can be installed through nixpkgs) and add `bluebuild` to it. -```nix -{pkgs,inputs,...}: { - ... - environment.SystemPackages = [ - inputs.bluebuild.packages.${pkgs.system}.bluebuild # change bluebuild with the fh added input name - ]; - ... -} -``` - -If you are not using a dedicated nix flake, you can add the BlueBuild flake as a variable inside your `/etc/nixos/*.nix` configuration, though this requires you to run `nixos-rebuild` with the `--impure` variable, it is not advisable to do so. - -```nix -{pkgs,...}: -let - bluebuild = builtins.fetchTarball "https://flakehub.com/f/bluebuild/cli/*.tar.gz"; -in { - ... - environment.SystemPackages = [ - bluebuild.packages.${pkgs.system}.bluebuild - ]; - ... -} -``` - -You can also use `nix develop .#` in this repos directory to run a nix shell with development dependencies and some helful utilities for building BlueBuild! - -## How to use - -### Generating `Containerfile` - -Once you have the CLI tool installed, you can run the following to pull in your recipe file to generate a `Containerfile`. - -```bash -bluebuild generate -o -``` - -You can then use this with `docker`, `podman`, or `buildah` to build and publish your image. Further options can be viewed by running `bluebuild template --help` - -### Building - -If you don't care about the details of the template, you can run the `build` command. - -```bash -bluebuild build ./recipes/recipe.yml -``` - -This will template out the file and build with `docker`, `podman`, or `buildah`. - -### Completions - -The `bluebuild completions` command generates shell completions, printed to stdout. These completions can be stored for integration in your shell environment. For example, on a system with [bash-completion](https://github.com/scop/bash-completion/) installed: - -```bash -# user completions -$ bluebuild completions bash > ~/.local/share/bash-completion/completions/bluebuild -# system-wide completions -$ bluebuild completions bash | sudo tee /usr/share/bash-completion/completions/bluebuild -``` - -Subsequent invocations of `bluebuild` will respond to `` autocompletions: - -```bash -$ bluebuild # press --v -V --help template bug-report --q --verbose --version upgrade completions --h --quiet build rebase help -``` - -Currently, bluebuild completions are available for `bash`, `zsh`, `fish`, `powershell`, `nushell`, and `elvish` shell environments. Please follow your shell's documentation for completion scripts. - -#### Local Builds - -##### Switch - -With the switch command, you can build and boot an image locally using an `oci-archive` tarball. The `switch` command can be run as a normal user and will only ask for `sudo` permissions when moving the archive into `/etc/bluebuild`. - -```bash -bluebuild switch recipes/recipe.yml -``` - -You can initiate an immediate restart by adding the `--reboot/-r` option. - -#### CI Builds - -##### GitHub - -You can use our [GitHub Action](https://github.com/blue-build/github-action) by using the following `.github/workflows/build.yml`: +### Basic Recipe ```yaml -name: bluebuild -on: - schedule: - - cron: "00 17 * * *" # build at 17:00 UTC every day - # (20 minutes after last ublue images start building) - push: - paths-ignore: # don't rebuild if only documentation has changed - - "**.md" - pull_request: - workflow_dispatch: # allow manually triggering builds -jobs: - bluebuild: - name: Build Custom Image - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - id-token: write - strategy: - fail-fast: false # stop GH from cancelling all matrix builds if one fails - matrix: - recipe: - # !! Add your recipes here - - recipe.yml - steps: - # the build is fully handled by the reusable github action - - name: Build Custom Image - uses: blue-build/github-action@v1 - with: - recipe: ${{ matrix.recipe }} - cosign_private_key: ${{ secrets.SIGNING_SECRET }} - registry_token: ${{ github.token }} - pr_event_number: ${{ github.event.number }} - ``` +# recipe.yml +name: debian-server +version: 1.0.0 +description: Debian server image -##### Gitlab +packages: + - openssh-server + - nginx + - postgresql -We also support GitLab CI! Fun fact, this project started out as a way to build these images in GitLab. You will want to make use of GitLab's [Secure Files](https://docs.gitlab.com/ee/ci/secure_files/index.html) feature for using your cosign private key for signing. Here's an example of a `.gitlab-ci.yml`: +repositories: + - deb http://deb.debian.org/debian bookworm main + - deb http://deb.debian.org/debian-security bookworm-security main -```yaml -workflow: - rules: - - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" - when: never - - if: "$CI_COMMIT_TAG" - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - - if: "$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS" - when: never - - if: "$CI_COMMIT_BRANCH" - -stages: - - build - -build-image: - stage: build - image: - name: ghcr.io/blue-build/cli - entrypoint: [""] - services: - - docker:dind - parallel: - matrix: - - RECIPE: - # Add your recipe files here - - recipe.yml - variables: - # Setup a secure connection with docker-in-docker service - # https://docs.gitlab.com/ee/ci/docker/using_docker_build.html - DOCKER_HOST: tcp://docker:2376 - DOCKER_TLS_CERTDIR: /certs - DOCKER_TLS_VERIFY: 1 - DOCKER_CERT_PATH: $DOCKER_TLS_CERTDIR/client - before_script: - # Pulls secure files into the build - - curl --silent "https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files/-/raw/main/installer" | bash - - export COSIGN_PRIVATE_KEY=$(cat .secure_files/cosign.key) - script: - - sleep 5 # Wait a bit for the docker-in-docker service to start - - bluebuild build --push ./recipes/$RECIPE +customizations: + user: + name: admin + password: secure_password + network: + hostname: debian-server ``` + +### Build Command + +```bash +# Build image from recipe +debian-blue-build build recipe.yml + +# Build with custom output format +debian-blue-build build recipe.yml --format qcow2 + +# Build with specific architecture +debian-blue-build build recipe.yml --arch amd64 +``` + +## Architecture + +- **Recipe Parser**: YAML recipe processing +- **Package Manager**: APT-based package handling +- **Image Builder**: OSTree and container generation +- **Output Formats**: ISO, QCOW2, RAW, VMDK support + +## Development Status + +- **Core CLI**: In development +- **Debian Integration**: Planning phase +- **Recipe System**: Basic structure +- **Testing**: Not started + +## Dependencies + +- Python 3.8+ +- OSTree tools +- Debian build tools +- Container tools (Docker/Podman) + +## Configuration + +Configuration files are located in `~/.config/debian-blue-build/`. + +## Contributing + +See CONTRIBUTING.md for development guidelines. diff --git a/debian_blue_build/__init__.py b/debian_blue_build/__init__.py new file mode 100644 index 0000000..0b5def6 --- /dev/null +++ b/debian_blue_build/__init__.py @@ -0,0 +1,21 @@ +""" +Debian Blue-Build CLI + +A fork of blue-build/cli adapted for Debian package management and atomic image building. +""" + +__version__ = "0.1.0" +__author__ = "Debian Blue-Build Team" +__description__ = "Debian atomic image building CLI tool" + +from .cli import main +from .recipe import Recipe +from .builder import ImageBuilder +from .package_manager import DebianPackageManager + +__all__ = [ + "main", + "Recipe", + "ImageBuilder", + "DebianPackageManager" +] diff --git a/debian_blue_build/package_manager.py b/debian_blue_build/package_manager.py new file mode 100644 index 0000000..389eddd --- /dev/null +++ b/debian_blue_build/package_manager.py @@ -0,0 +1,124 @@ +""" +Debian package manager for Blue-Build CLI + +Handles APT-based package management and repository configuration. +""" + +import subprocess +import tempfile +from typing import List, Dict, Any, Optional +from pathlib import Path + +class DebianPackageManager: + """Debian package management using APT""" + + def __init__(self, work_dir: Optional[Path] = None): + self.work_dir = work_dir or Path.cwd() + self.repositories = [] + self.packages = [] + + def add_repository(self, repo_line: str) -> bool: + """Add Debian repository""" + try: + # Parse repository line (deb http://deb.debian.org/debian bookworm main) + parts = repo_line.split() + if len(parts) >= 4 and parts[0] == "deb": + self.repositories.append(repo_line) + return True + return False + except Exception: + return False + + def add_package(self, package_name: str) -> bool: + """Add package to install list""" + if package_name and package_name not in self.packages: + self.packages.append(package_name) + return True + return False + + def resolve_dependencies(self, package_name: str) -> List[str]: + """Resolve package dependencies""" + try: + result = subprocess.run( + ["apt-cache", "depends", package_name], + capture_output=True, + text=True, + cwd=self.work_dir + ) + + if result.returncode == 0: + dependencies = [] + for line in result.stdout.split('\n'): + if line.strip().startswith('Depends:'): + deps = line.replace('Depends:', '').strip() + dependencies.extend([d.strip() for d in deps.split(',')]) + return dependencies + return [] + except Exception: + return [] + + def create_sources_list(self, output_path: Path) -> bool: + """Create sources.list file""" + try: + with open(output_path, 'w') as f: + for repo in self.repositories: + f.write(f"{repo}\n") + return True + except Exception: + return False + + def generate_apt_commands(self) -> List[str]: + """Generate APT commands for package installation""" + commands = [ + "apt-get update", + f"apt-get install -y {' '.join(self.packages)}", + "apt-get clean", + "rm -rf /var/lib/apt/lists/*" + ] + return commands + + def validate_packages(self) -> Dict[str, List[str]]: + """Validate package availability""" + results = { + "available": [], + "unavailable": [], + "dependencies": [] + } + + for package in self.packages: + try: + result = subprocess.run( + ["apt-cache", "show", package], + capture_output=True, + text=True, + cwd=self.work_dir + ) + + if result.returncode == 0: + results["available"].append(package) + deps = self.resolve_dependencies(package) + results["dependencies"].extend(deps) + else: + results["unavailable"].append(package) + except Exception: + results["unavailable"].append(package) + + return results + + def create_package_script(self, output_path: Path) -> bool: + """Create package installation script""" + try: + with open(output_path, 'w') as f: + f.write("#!/bin/bash\n") + f.write("set -e\n\n") + f.write("# Package installation script\n") + f.write("# Generated by Debian Blue-Build CLI\n\n") + + for cmd in self.generate_apt_commands(): + f.write(f"{cmd}\n") + + # Make executable + output_path.chmod(0o755) + return True + except Exception: + return False diff --git a/debian_blue_build/recipe.py b/debian_blue_build/recipe.py new file mode 100644 index 0000000..5fe76f2 --- /dev/null +++ b/debian_blue_build/recipe.py @@ -0,0 +1,102 @@ +""" +Recipe system for Debian Blue-Build CLI + +Handles YAML recipe parsing and validation for Debian atomic image building. +""" + +import yaml +from typing import Dict, List, Any, Optional +from dataclasses import dataclass +from pathlib import Path + +@dataclass +class DebianRecipe: + """Debian recipe configuration""" + name: str + version: str + description: str + packages: List[str] + repositories: List[str] + customizations: Dict[str, Any] + output_format: str = "container" + architecture: str = "amd64" + + @classmethod + def from_yaml(cls, yaml_content: str) -> "DebianRecipe": + """Create recipe from YAML content""" + data = yaml.safe_load(yaml_content) + return cls( + name=data.get("name", ""), + version=data.get("version", "1.0.0"), + description=data.get("description", ""), + packages=data.get("packages", []), + repositories=data.get("repositories", []), + customizations=data.get("customizations", {}), + output_format=data.get("output_format", "container"), + architecture=data.get("architecture", "amd64") + ) + + @classmethod + def from_file(cls, file_path: Path) -> "DebianRecipe": + """Create recipe from YAML file""" + with open(file_path, 'r') as f: + content = f.read() + return cls.from_yaml(content) + + def validate(self) -> List[str]: + """Validate recipe configuration""" + errors = [] + + if not self.name: + errors.append("Recipe name is required") + + if not self.packages: + errors.append("At least one package is required") + + if not self.repositories: + errors.append("At least one repository is required") + + return errors + + def to_dict(self) -> Dict[str, Any]: + """Convert recipe to dictionary""" + return { + "name": self.name, + "version": self.version, + "description": self.description, + "packages": self.packages, + "repositories": self.repositories, + "customizations": self.customizations, + "output_format": self.output_format, + "architecture": self.architecture + } + +class RecipeParser: + """Recipe parser and validator""" + + @staticmethod + def parse(file_path: str) -> DebianRecipe: + """Parse recipe from file""" + path = Path(file_path) + if not path.exists(): + raise FileNotFoundError(f"Recipe file not found: {file_path}") + + recipe = DebianRecipe.from_file(path) + errors = recipe.validate() + + if errors: + raise ValueError(f"Recipe validation failed: {', '.join(errors)}") + + return recipe + + @staticmethod + def validate_schema(recipe_data: Dict[str, Any]) -> List[str]: + """Validate recipe schema""" + errors = [] + required_fields = ["name", "packages", "repositories"] + + for field in required_fields: + if field not in recipe_data: + errors.append(f"Missing required field: {field}") + + return errors diff --git a/utils/src/command_output.rs b/utils/src/command_output.rs index 477767d..ac9db0b 100644 --- a/utils/src/command_output.rs +++ b/utils/src/command_output.rs @@ -22,17 +22,17 @@ pub struct CommandOutput { /// fn create_command>(binary_name: T) -> Result { let binary_name = binary_name.as_ref(); - log::trace!("Creating Command for binary {}", binary_name.display()); + log::trace!("Creating Command for binary {}", binary_name.to_string_lossy()); let full_path = match which::which(binary_name) { Ok(full_path) => { - log::trace!("Using {} as {}", full_path.display(), binary_name.display()); + log::trace!("Using {} as {}", full_path.to_string_lossy(), binary_name.to_string_lossy()); full_path } Err(error) => { log::trace!( "Unable to find {} in PATH, {error:?}", - binary_name.display() + binary_name.to_string_lossy() ); return Err(Error::new(ErrorKind::NotFound, error)); } @@ -71,7 +71,7 @@ fn exec_timeout(cmd: &mut Command, time_limit: Duration) -> Option process, Err(error) => { - log::trace!("Unable to run {}, {:?}", cmd.get_program().display(), error); + log::trace!("Unable to run {}, {:?}", cmd.get_program().to_string_lossy(), error); return None; } }; @@ -117,7 +117,7 @@ fn exec_timeout(cmd: &mut Command, time_limit: Duration) -> Option { log::warn!( "Executing command {} timed out.", - cmd.get_program().display() + cmd.get_program().to_string_lossy() ); log::warn!( "You can set command_timeout in your config to a higher value to allow longer-running commands to keep executing." @@ -127,7 +127,7 @@ fn exec_timeout(cmd: &mut Command, time_limit: Duration) -> Option { log::trace!( "Executing command {} failed by: {:?}", - cmd.get_program().display(), + cmd.get_program().to_string_lossy(), error ); None