From 6945915d4efa677e094f83993a1dabbde9da5b47 Mon Sep 17 00:00:00 2001 From: Gerald Pinder Date: Tue, 4 Jun 2024 14:03:00 -0400 Subject: [PATCH] feat: Add arm support (#191) This adds support for ARM based images and binaries. --- .github/workflows/build-pr.yml | 141 +++++++++++++++++++++++---------- .github/workflows/build.yml | 137 ++++++++++++++++++++++++-------- Earthfile | 100 ++++++++++++++++++----- 3 files changed, 279 insertions(+), 99 deletions(-) diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml index c8d0f24..186b07b 100644 --- a/.github/workflows/build-pr.yml +++ b/.github/workflows/build-pr.yml @@ -11,9 +11,94 @@ env: FORCE_COLOR: 1 jobs: + arm64-prebuild: + timeout-minutes: 60 + runs-on: ubuntu-latest + + steps: + - name: Maximize build space + uses: ublue-os/remove-unwanted-software@v6 + env: + EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} + if: env.EARTHLY_SAT_TOKEN == null + + - uses: earthly/actions-setup@v1 + + - name: Earthly login + env: + EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} + if: env.EARTHLY_SAT_TOKEN != null + run: | + earthly account login --token ${{ secrets.EARTHLY_SAT_TOKEN }} >> /dev/null + earthly org s blue-build + earthly sat s arm + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + if: github.token != null + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Run build + id: build + run: | + earthly --ci --push -P +prebuild + + amd64-prebuild: + timeout-minutes: 60 + runs-on: ubuntu-latest + + steps: + - name: Maximize build space + uses: ublue-os/remove-unwanted-software@v6 + env: + EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} + if: env.EARTHLY_SAT_TOKEN == null + + - uses: earthly/actions-setup@v1 + + - name: Earthly login + env: + EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} + if: env.EARTHLY_SAT_TOKEN != null + run: | + earthly account login --token ${{ secrets.EARTHLY_SAT_TOKEN }} >> /dev/null + earthly org s blue-build + earthly sat s amd + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + if: github.token != null + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Run build + id: build + run: | + earthly --ci --push -P +prebuild + build: timeout-minutes: 60 runs-on: ubuntu-latest + needs: + - arm64-prebuild + - amd64-prebuild outputs: push: ${{ steps.build.outputs.push }} @@ -53,10 +138,10 @@ jobs: id: build run: | if [ -n "${{ github.token }}" ]; then - earthly --ci --push +build + earthly --ci --push -P +build echo "push=true" >> "$GITHUB_OUTPUT" else - earthly --ci +build + earthly --ci -P +build fi integration-tests: @@ -102,21 +187,13 @@ jobs: uses: ublue-os/remove-unwanted-software@v6 - uses: sigstore/cosign-installer@v3.3.0 - - uses: earthly/actions-setup@v1 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 with: install: true - - name: Earthly login - env: - EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} - if: env.EARTHLY_SAT_TOKEN != null - run: | - earthly account login --token ${{ secrets.EARTHLY_SAT_TOKEN }} >> /dev/null - earthly org s blue-build - earthly sat s pr + - uses: actions-rust-lang/setup-rust-toolchain@v1 - uses: actions/checkout@v4 with: @@ -126,7 +203,7 @@ jobs: - name: Install bluebuild run: | - earthly -a +installer/bluebuild /usr/local/bin/bluebuild + cargo install --path . --debug - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v3 @@ -159,7 +236,6 @@ jobs: uses: ublue-os/remove-unwanted-software@v6 - uses: sigstore/cosign-installer@v3.3.0 - - uses: earthly/actions-setup@v1 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -173,22 +249,17 @@ jobs: username: ${{ github.actor }} password: ${{ github.token }} - - name: Earthly login - env: - EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} - if: env.EARTHLY_SAT_TOKEN != null - run: | - earthly account login --token ${{ secrets.EARTHLY_SAT_TOKEN }} >> /dev/null - earthly org s blue-build - earthly sat s main + - uses: actions-rust-lang/setup-rust-toolchain@v1 - uses: actions/checkout@v4 with: - ref: main + fetch-depth: 0 + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} - name: Install bluebuild run: | - earthly -a +installer/bluebuild /usr/local/bin/bluebuild + cargo install --path . --debug - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v3 @@ -220,7 +291,6 @@ jobs: uses: ublue-os/remove-unwanted-software@v6 - uses: sigstore/cosign-installer@v3.3.0 - - uses: earthly/actions-setup@v1 - name: Setup Podman shell: bash @@ -234,14 +304,7 @@ jobs: sudo apt-get update sudo apt-get install -y podman - - name: Earthly login - env: - EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} - if: env.EARTHLY_SAT_TOKEN != null - run: | - earthly account login --token ${{ secrets.EARTHLY_SAT_TOKEN }} >> /dev/null - earthly org s blue-build - earthly sat s pr + - uses: actions-rust-lang/setup-rust-toolchain@v1 - uses: actions/checkout@v4 with: @@ -251,7 +314,7 @@ jobs: - name: Install bluebuild run: | - earthly -a +installer/bluebuild /usr/local/bin/bluebuild + cargo install --path . --debug - name: Run Build env: @@ -280,7 +343,6 @@ jobs: uses: ublue-os/remove-unwanted-software@v6 - uses: sigstore/cosign-installer@v3.3.0 - - uses: earthly/actions-setup@v1 - name: Setup Buildah shell: bash @@ -294,14 +356,7 @@ jobs: sudo apt-get update sudo apt-get install -y buildah - - name: Earthly login - env: - EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} - if: env.EARTHLY_SAT_TOKEN != null - run: | - earthly account login --token ${{ secrets.EARTHLY_SAT_TOKEN }} >> /dev/null - earthly org s blue-build - earthly sat s pr + - uses: actions-rust-lang/setup-rust-toolchain@v1 - uses: actions/checkout@v4 with: @@ -311,7 +366,7 @@ jobs: - name: Install bluebuild run: | - earthly -a +installer/bluebuild /usr/local/bin/bluebuild + cargo install --path . --debug - name: Run Build env: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7cc6ee9..cb5c5a3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,11 +14,96 @@ env: FORCE_COLOR: 1 jobs: + arm64-prebuild: + timeout-minutes: 60 + runs-on: ubuntu-latest + if: github.repository == 'blue-build/cli' + + steps: + - name: Maximize build space + uses: ublue-os/remove-unwanted-software@v6 + env: + EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} + if: env.EARTHLY_SAT_TOKEN == null + + - uses: earthly/actions-setup@v1 + + - name: Earthly login + env: + EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} + if: env.EARTHLY_SAT_TOKEN != null + run: | + earthly account login --token ${{ secrets.EARTHLY_SAT_TOKEN }} >> /dev/null + earthly org s blue-build + earthly sat s arm + + # Setup repo and add caching + - uses: actions/checkout@v4 + with: + ref: main + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + if: github.token != null + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Run build + id: build + run: | + earthly --ci --push -P +prebuild + + amd64-prebuild: + timeout-minutes: 60 + runs-on: ubuntu-latest + if: github.repository == 'blue-build/cli' + + steps: + - name: Maximize build space + uses: ublue-os/remove-unwanted-software@v6 + env: + EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} + if: env.EARTHLY_SAT_TOKEN == null + + - uses: earthly/actions-setup@v1 + + - name: Earthly login + env: + EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} + if: env.EARTHLY_SAT_TOKEN != null + run: | + earthly account login --token ${{ secrets.EARTHLY_SAT_TOKEN }} >> /dev/null + earthly org s blue-build + earthly sat s amd + + # Setup repo and add caching + - uses: actions/checkout@v4 + with: + ref: main + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + if: github.token != null + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Run build + id: build + run: | + earthly --ci --push -P +prebuild + build: permissions: packages: write timeout-minutes: 60 runs-on: ubuntu-latest + needs: + - arm64-prebuild + - amd64-prebuild steps: - name: Maximize build space @@ -56,7 +141,7 @@ jobs: - name: Run build fork if: github.repository != 'blue-build/cli' - run: earthly --ci +build + run: earthly --ci -P +build integration-tests: permissions: @@ -100,29 +185,23 @@ jobs: uses: ublue-os/remove-unwanted-software@v6 - uses: sigstore/cosign-installer@v3.3.0 - - uses: earthly/actions-setup@v1 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 with: install: true - - name: Earthly login - env: - EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} - if: env.EARTHLY_SAT_TOKEN != null - run: | - earthly account login --token ${{ secrets.EARTHLY_SAT_TOKEN }} >> /dev/null - earthly org s blue-build - earthly sat s main + - uses: actions-rust-lang/setup-rust-toolchain@v1 - uses: actions/checkout@v4 with: - ref: main + fetch-depth: 0 + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} - name: Install bluebuild run: | - earthly -a +installer/bluebuild /usr/local/bin/bluebuild + cargo install --path . --debug - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v3 @@ -155,7 +234,6 @@ jobs: uses: ublue-os/remove-unwanted-software@v6 - uses: sigstore/cosign-installer@v3.3.0 - - uses: earthly/actions-setup@v1 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -169,22 +247,17 @@ jobs: username: ${{ github.actor }} password: ${{ github.token }} - - name: Earthly login - env: - EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} - if: env.EARTHLY_SAT_TOKEN != null - run: | - earthly account login --token ${{ secrets.EARTHLY_SAT_TOKEN }} >> /dev/null - earthly org s blue-build - earthly sat s main + - uses: actions-rust-lang/setup-rust-toolchain@v1 - uses: actions/checkout@v4 with: - ref: main + fetch-depth: 0 + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} - name: Install bluebuild run: | - earthly -a +installer/bluebuild /usr/local/bin/bluebuild + cargo install --path . --debug - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v3 @@ -216,7 +289,6 @@ jobs: uses: ublue-os/remove-unwanted-software@v6 - uses: sigstore/cosign-installer@v3.3.0 - - uses: earthly/actions-setup@v1 - name: Setup Podman shell: bash @@ -230,22 +302,17 @@ jobs: sudo apt-get update sudo apt-get install -y podman - - name: Earthly login - env: - EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} - if: env.EARTHLY_SAT_TOKEN != null - run: | - earthly account login --token ${{ secrets.EARTHLY_SAT_TOKEN }} >> /dev/null - earthly org s blue-build - earthly sat s main + - uses: actions-rust-lang/setup-rust-toolchain@v1 - uses: actions/checkout@v4 with: - ref: main + fetch-depth: 0 + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} - name: Install bluebuild run: | - earthly -a +installer/bluebuild /usr/local/bin/bluebuild + cargo install --path . --debug - name: Run Build env: @@ -303,7 +370,7 @@ jobs: - name: Install bluebuild run: | - earthly -a +installer/bluebuild /usr/local/bin/bluebuild + earthly -a -P +installer/bluebuild /usr/local/bin/bluebuild - name: Run Build env: diff --git a/Earthfile b/Earthfile index 285d7d1..e084b97 100644 --- a/Earthfile +++ b/Earthfile @@ -6,19 +6,32 @@ IMPORT github.com/earthly/lib/rust AS rust ARG --global IMAGE=ghcr.io/blue-build/cli all: + WAIT + BUILD --platform=linux/amd64 --platform=linux/arm64 +prebuild + END BUILD +build BUILD ./integration-tests+all build: WAIT - BUILD +build-scripts + BUILD --platform=linux/amd64 --platform=linux/arm64 +build-scripts END + BUILD +run-checks + BUILD --platform=linux/amd64 --platform=linux/arm64 +build-images + +run-checks: BUILD +lint BUILD +test + +build-images: BUILD +blue-build-cli BUILD +blue-build-cli-alpine BUILD +installer +prebuild: + BUILD +blue-build-cli-prebuild + BUILD +blue-build-cli-alpine-prebuild + lint: FROM +common DO rust+CARGO --args="clippy -- -D warnings" @@ -35,7 +48,7 @@ install: FROM +common ARG --required BUILD_TARGET - DO rust+CARGO --args="build --release --target $BUILD_TARGET" --output="$BUILD_TARGET/release/[^\./]+" + DO rust+CROSS --target="$BUILD_TARGET" --output="$BUILD_TARGET/release/[^\./]+" SAVE ARTIFACT target/$BUILD_TARGET/release/bluebuild @@ -43,12 +56,12 @@ install-all-features: FROM +common ARG --required BUILD_TARGET - DO rust+CARGO --args="build --all-features --release --target $BUILD_TARGET" --output="$BUILD_TARGET/release/[^\./]+" + DO rust+CROSS --args="build --all-features --release" --target="$BUILD_TARGET" --output="$BUILD_TARGET/release/[^\./]+" SAVE ARTIFACT target/$BUILD_TARGET/release/bluebuild common: - FROM ghcr.io/blue-build/earthly-lib/cargo-builder + FROM --platform=native ghcr.io/blue-build/earthly-lib/cargo-builder WORKDIR /app COPY --keep-ts --dir src/ template/ recipe/ utils/ /app @@ -62,8 +75,13 @@ common: DO rust+INIT --keep_fingerprints=true build-scripts: - FROM alpine - LABEL org.opencontainers.image.source="https://github.com/blue-build/cli" + ARG BASE_IMAGE="alpine" + FROM $BASE_IMAGE + + COPY (+digest/base-image-digest --BASE_IMAGE=$BASE_IMAGE) /base-image-digest + LABEL org.opencontainers.image.base.name="$BASE_IMAGE" + LABEL org.opencontainers.image.base.digest="$(cat /base-image-digest)" + COPY --dir scripts/ / FOR script IN "$(ls /scripts | grep -e '.*\.sh$')" RUN echo "Making ${script} executable" && \ @@ -72,15 +90,17 @@ build-scripts: DO --pass-args +SAVE_IMAGE --SUFFIX="-build-scripts" -blue-build-cli: +blue-build-cli-prebuild: ARG BASE_IMAGE="registry.fedoraproject.org/fedora-toolbox" FROM $BASE_IMAGE + + COPY (+digest/base-image-digest --BASE_IMAGE=$BASE_IMAGE) /base-image-digest LABEL org.opencontainers.image.base.name="$BASE_IMAGE" + LABEL org.opencontainers.image.base.digest="$(cat /base-image-digest)" RUN dnf -y install dnf-plugins-core \ && dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo \ && dnf install --refresh -y \ - jq \ docker-ce \ docker-ce-cli \ containerd.io \ @@ -90,11 +110,21 @@ blue-build-cli: podman \ skopeo - LABEL org.opencontainers.image.base.digest="$(skopeo inspect "docker://$BASE_IMAGE" | jq -r '.Digest')" - COPY +cosign/cosign /usr/bin/cosign + ARG EARTHLY_GIT_HASH + ARG TARGETARCH + SAVE IMAGE --push "$IMAGE:$EARTHLY_GIT_HASH-prebuild-$TARGETARCH" - DO --pass-args +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="x86_64-unknown-linux-gnu" +blue-build-cli: + ARG EARTHLY_GIT_HASH + ARG TARGETARCH + FROM "$IMAGE:$EARTHLY_GIT_HASH-prebuild-$TARGETARCH" + + IF [ "$TARGETARCH" = "arm64" ] + DO --pass-args +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="aarch64-unknown-linux-gnu" + ELSE + DO --pass-args +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="x86_64-unknown-linux-gnu" + END RUN mkdir -p /bluebuild WORKDIR /bluebuild @@ -102,17 +132,32 @@ blue-build-cli: DO --pass-args +SAVE_IMAGE -blue-build-cli-alpine: +blue-build-cli-alpine-prebuild: ARG BASE_IMAGE="alpine" FROM $BASE_IMAGE + + COPY (+digest/base-image-digest --BASE_IMAGE=$BASE_IMAGE) /base-image-digest LABEL org.opencontainers.image.base.name="$BASE_IMAGE" + LABEL org.opencontainers.image.base.digest="$(cat /base-image-digest)" - RUN apk update && apk add buildah podman skopeo fuse-overlayfs jq - - LABEL org.opencontainers.image.base.digest="$(skopeo inspect "docker://$BASE_IMAGE" | jq -r '.Digest')" + RUN apk update && apk add buildah podman skopeo fuse-overlayfs COPY +cosign/cosign /usr/bin/cosign - DO --pass-args +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="x86_64-unknown-linux-musl" + + ARG EARTHLY_GIT_HASH + ARG TARGETARCH + SAVE IMAGE --push "$IMAGE:$EARTHLY_GIT_HASH-alpine-prebuild-$TARGETARCH" + +blue-build-cli-alpine: + ARG EARTHLY_GIT_HASH + ARG TARGETARCH + FROM "$IMAGE:$EARTHLY_GIT_HASH-alpine-prebuild-$TARGETARCH" + + IF [ "$TARGETARCH" = "arm64" ] + DO --pass-args +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="aarch64-unknown-linux-musl" + ELSE + DO --pass-args +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="x86_64-unknown-linux-musl" + END RUN mkdir -p /bluebuild WORKDIR /bluebuild @@ -123,13 +168,18 @@ blue-build-cli-alpine: installer: ARG BASE_IMAGE="alpine" FROM $BASE_IMAGE + + COPY (+digest/base-image-digest --BASE_IMAGE=$BASE_IMAGE) /base-image-digest LABEL org.opencontainers.image.base.name="$BASE_IMAGE" + LABEL org.opencontainers.image.base.digest="$(cat /base-image-digest)" - RUN apk update && apk add skopeo jq + ARG TARGETARCH + IF [ "$TARGETARCH" = "arm64" ] + DO --pass-args +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="aarch64-unknown-linux-musl" + ELSE + DO --pass-args +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="x86_64-unknown-linux-musl" + END - LABEL org.opencontainers.image.base.digest="$(skopeo inspect "docker://$BASE_IMAGE" | jq -r '.Digest')" - - DO --pass-args +INSTALL --OUT_DIR="/out/" --BUILD_TARGET="x86_64-unknown-linux-musl" COPY install.sh /install.sh CMD ["cat", "/install.sh"] @@ -141,6 +191,14 @@ cosign: FROM gcr.io/projectsigstore/cosign SAVE ARTIFACT /ko-app/cosign +digest: + FROM alpine + RUN apk update && apk add skopeo jq + + ARG --required BASE_IMAGE + RUN skopeo inspect "docker://$BASE_IMAGE" | jq -r '.Digest' > /base-image-digest + SAVE ARTIFACT /base-image-digest + version: FROM rust @@ -171,7 +229,7 @@ SAVE_IMAGE: ARG SUFFIX="" ARG TAGGED="false" - COPY +version/version / + COPY --platform=native +version/version / ARG VERSION="$(cat /version)" ARG MAJOR_VERSION="$(echo "$VERSION" | cut -d'.' -f1)" ARG MINOR_VERSION="$(echo "$VERSION" | cut -d'.' -f2)"