feat: support versioning modules (#240)

* feat: add nu script that generates build matrix for github action

* chore: push initial version of new github action for testing

* fix: properly close github template sequence

* chore: screw this i'll write the whole build in nu (nu build script started, continue later)

* fix: individual misspellings and such

(yeah, i'm tired)

* fix: containerfile path

* fix: docker arg syntax

* fix: docker build path?

* feat: code structure, buildx, pushing to registry

* fix: tag image properly

* fix: split arguments properly

* fix: use registry properly

* refactor: move docker build to a separate par-each

* feat: correctly tag versioned modules

* feat: cosign signing, better logging

* fix: cosign syntax

* fix: differentiate log types with more expressive colors

* chore: fix cosign syntax in logs

* fix: capture errors, colocate logs while running paraller

* chore: partly revert "fix: capture errors, colocate logs while running paraller"

This reverts commit 9238a0f1d68183e712b567fc50849964cc964c78.

* chore: bring back capture errors, keep order in logs

apparently do --capture-errors is required for nushell to catch external commands errors

* chore: Revert "chore: bring back capture errors, keep order in logs"

These changes didn't do anything...
This reverts commit 020b9a1bce9456f2167397b49aa24a65f6bec8e6.

* fix: properly tag images in PRs and secondary branches

* fix: use tags-variable instead of "tags" string

* chore: change default-flatpaks module folder structure to be versioned

* fix: also log generated tags for versioned modules

* fix: don't add tags meant for latest image for every version of versioned module

* fix: better logging and inline docs

* fix: better logging pt2

* feat: build-unified for building the legacy modules container with just the latest versions

* fix: correct workflow names

* fix: add missing ansi resets

* chore: add nushell extension to recommendations

* fix: update unified job name

Co-authored-by: Gerald Pinder <gmpinder@gmail.com>

* chore: remove matrix output left over from a previous version

---------

Co-authored-by: Gerald Pinder <gmpinder@gmail.com>

.github/workflows/build-individual.yml

@@ -0,0 +1,38 @@
+name: build-individual
+on:
+  push:
+    paths-ignore: # don't rebuild if only documentation has changed
+      - "**.md"
+  pull_request:
+  merge_group:
+  workflow_dispatch:
+
+jobs:
+  build-individual:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: hustcer/setup-nu@v3.10
+        with:
+          version: v0.93
+
+      - uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      
+      - uses: sigstore/cosign-installer@v3.5.0
+
+      - name: Run build and push script
+        run: nu ./build-individual.nu
+        env:
+          REGISTRY: ghcr.io/${{ github.repository_owner }}
+          COSIGN_PRIVATE_KEY: ${{ secrets.SIGNING_SECRET }}
+          GH_EVENT_NAME: ${{ github.event_name }}
+          GH_PR_NUMBER: ${{ github.event.number }}
+          GH_BRANCH: ${{ github.ref_name }}

.github/workflows/build-unified.yml

@@ -0,0 +1,38 @@
+name: build-unified
+on:
+  push:
+    paths-ignore: # don't rebuild if only documentation has changed
+      - "**.md"
+  pull_request:
+  merge_group:
+  workflow_dispatch:
+
+jobs:
+  build-unified:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: hustcer/setup-nu@v3.10
+        with:
+          version: v0.93
+
+      - uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      
+      - uses: sigstore/cosign-installer@v3.5.0
+
+      - name: Run build and push script
+        run: nu ./build-unified.nu
+        env:
+          REGISTRY: ghcr.io/${{ github.repository_owner }}
+          COSIGN_PRIVATE_KEY: ${{ secrets.SIGNING_SECRET }}
+          GH_EVENT_NAME: ${{ github.event_name }}
+          GH_PR_NUMBER: ${{ github.event.number }}
+          GH_BRANCH: ${{ github.ref_name }}

.github/workflows/build.yml

@@ -1,107 +0,0 @@
-name: publish-modules
-on:
-  push:
-    paths-ignore: # don't rebuild if only documentation has changed
-      - "**.md"
-  pull_request:
-  merge_group:
-  workflow_dispatch:
-env:
-  IMAGE_NAME: modules
-  IMAGE_REGISTRY: ghcr.io/${{ github.repository_owner }}
-
-jobs:
-  push-ghcr:
-    name: Build and push image
-    runs-on: ubuntu-22.04
-    permissions:
-      contents: read
-      packages: write
-      id-token: write
-    steps:
-      # Checkout push-to-registry action GitHub repository
-      - name: Checkout Push to Registry action
-        uses: actions/checkout@v4
-
-      - name: Generate tags
-        id: generate-tags
-        shell: bash
-        run: |
-          echo "sha_short=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT
-          alias_tags=()
-          # Only perform the follow code when the action is spawned from a Pull Request
-          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
-            alias_tags+=("pr-${{ github.event.number }}")
-          elif [[ "${{ github.ref_name }}" == "main" ]]; then
-            # The following is run when the timer is triggered or a merge/push to main
-            echo "date=$(date +%Y%m%d)" >> $GITHUB_OUTPUT
-            alias_tags+=("latest" "main")
-          else
-            alias_tags+=("${{ github.ref_name }}")
-          fi
-          echo "alias_tags=${alias_tags[*]}" >> $GITHUB_OUTPUT
-
-      # Build image using Buildah action
-      - name: Build Image
-        id: build_image
-        uses: redhat-actions/buildah-build@v2
-        with:
-          containerfiles: |
-            ./Containerfile
-          image: ${{ env.IMAGE_NAME }}
-          tags: |
-            ${{ steps.generate-tags.outputs.alias_tags }}
-            ${{ steps.generate-tags.outputs.date }}
-            ${{ steps.generate-tags.outputs.sha_short }}
-          oci: true
-
-      # Workaround bug where capital letters in your GitHub username make it impossible to push to GHCR.
-      # https://github.com/macbre/push-to-ghcr/issues/12
-      - name: Lowercase Registry
-        id: registry_case
-        uses: ASzc/change-string-case-action@v6
-        with:
-          string: ${{ env.IMAGE_REGISTRY }}
-
-      # Push the image to GHCR (Image Registry)
-      - name: Push To GHCR
-        uses: redhat-actions/push-to-registry@v2
-        id: push
-        if: github.event_name != 'pull_request'
-        env:
-          REGISTRY_USER: ${{ github.actor }}
-          REGISTRY_PASSWORD: ${{ github.token }}
-        with:
-          image: ${{ steps.build_image.outputs.image }}
-          tags: ${{ steps.build_image.outputs.tags }}
-          registry: ${{ steps.registry_case.outputs.lowercase }}
-          username: ${{ env.REGISTRY_USER }}
-          password: ${{ env.REGISTRY_PASSWORD }}
-          extra-args: |
-            --disable-content-trust
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        if: github.event_name != 'pull_request'
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      # Sign container
-      - uses: sigstore/cosign-installer@v3.5.0
-        if: github.event_name != 'pull_request'
-
-      - name: Sign container image
-        if: github.event_name != 'pull_request'
-        run: |
-          cosign sign -y --key env://COSIGN_PRIVATE_KEY ${{ steps.registry_case.outputs.lowercase }}/${{ env.IMAGE_NAME }}@${TAGS}
-        env:
-          TAGS: ${{ steps.push.outputs.digest }}
-          COSIGN_EXPERIMENTAL: false
-          COSIGN_PRIVATE_KEY: ${{ secrets.SIGNING_SECRET }}
-
-      - name: Echo outputs
-        if: github.event_name != 'pull_request'
-        run: |
-          echo "${{ toJSON(steps.push.outputs) }}"

.gitignore

@@ -1 +1,3 @@
 cosign.key
+
+/modules-latest

.mise.toml

@@ -0,0 +1,2 @@
+[tools]
+act = "latest"

.vscode/extensions.json

@@ -1,5 +1,6 @@
 {
     "recommendations": [
-        "timonwong.shellcheck"
+        "timonwong.shellcheck",
+        "thenuprojectcontributors.vscode-nushell-lang"
     ]
 }

Containerfile

@@ -1,3 +0,0 @@
-FROM scratch
-
-COPY modules /modules

build-individual.nu

@@ -0,0 +1,97 @@
+#!/usr/bin/env nu
+# build separate images for each module in the repo
+
+print $"(ansi green_bold)Gathering images"
+
+let images = ls modules | each { |moduleDir|
+    cd $moduleDir.name
+
+    # module is unversioned
+    if ($"($moduleDir.name | path basename).sh" | path exists) {
+
+        print $"(ansi cyan)Found(ansi reset) (ansi cyan_bold)unversioned(ansi reset) (ansi cyan)module:(ansi reset) ($moduleDir.name | path basename)"
+
+        let tags = (
+            if ($env.GH_EVENT_NAME != "pull_request" and $env.GH_BRANCH == "main") {
+                ["latest", "v1"]
+            } else if ($env.GH_EVENT_NAME != "pull_request") {
+                [$env.GH_BRANCH, $"v1-($env.GH_BRANCH)"]
+            } else {
+                [$"pr-($env.GH_PR_NUMBER)", $"v1-pr-($env.GH_PR_NUMBER)"]
+            }
+        )
+        print $"(ansi cyan)Generated tags:(ansi reset) ($tags | str join ' ')"
+
+        {
+            name: ($moduleDir.name | path basename)
+            directory: ($moduleDir.name)
+            tags: $tags
+        }
+
+    } else { # module is versioned
+
+        print $"(ansi cyan)Found(ansi reset) (ansi blue_bold)versioned(ansi reset) (ansi cyan)module:(ansi reset) ($moduleDir.name | path basename)"
+
+        let versioned = ls v*/
+            | get name | str substring 1.. | into int | sort # sort versions properly
+            | each {|version|
+                let tags = (
+                    if ($env.GH_EVENT_NAME != "pull_request" and $env.GH_BRANCH == "main") {
+                        [$"v($version)"]
+                    } else if ($env.GH_EVENT_NAME != "pull_request") {
+                        [$"v($version)-($env.GH_BRANCH)"]
+                    } else {
+                        [$"v($version)-pr-($env.GH_PR_NUMBER)"]
+                    }
+                )
+                print $"(ansi cyan)Generated tags:(ansi reset) ($tags | str join ' ')"
+
+                {
+                    name: ($moduleDir.name | path basename)
+                    directory: $"($moduleDir.name)/v($version)"
+                    tags: $tags
+                }
+        }
+
+        let latest_tag = (
+            if ($env.GH_EVENT_NAME != "pull_request" and $env.GH_BRANCH == "main") {
+                "latest"
+            } else if ($env.GH_EVENT_NAME != "pull_request") {
+                $env.GH_BRANCH
+            } else {
+                $"pr-($env.GH_PR_NUMBER)"
+            }
+        )
+        print $"(ansi cyan)Extra tag for latest image:(ansi reset) ($latest_tag)"
+        let latest = ($versioned | last)
+        ($versioned
+            | update (($versioned | length) - 1) # update the last / latest item in list
+            ($latest | update "tags" ($latest.tags | append latest_tag)) # append tag which should only be given to the latest version
+        )
+
+    }
+} | flatten directory
+
+print $"(ansi green_bold)Starting image build(ansi reset)"
+
+$images | par-each { |img|
+
+    print $"(ansi cyan)Building image:(ansi reset) modules/($img.name)"
+    (docker build .
+        -f ./individual.Containerfile
+        ...($img.tags | each { |tag| ["-t", $"($env.REGISTRY)/modules/($img.name):($tag)"] } | flatten) # generate and spread list of tags
+        --build-arg $"DIRECTORY=($img.directory)"
+        --build-arg $"NAME=($img.name)")
+
+    print $"(ansi cyan)Pushing image:(ansi reset) ($env.REGISTRY)/modules/($img.name)"
+    let digest = (
+        docker push --all-tags $"($env.REGISTRY)/modules/($img.name)"
+            | split row "\n"  | last | split row " " | get 2 # parse push output to get digest for signing
+    )
+
+    print $"(ansi cyan)Signing image:(ansi reset) ($env.REGISTRY)/modules/($img.name)@($digest)"
+    cosign sign -y --key env://COSIGN_PRIVATE_KEY $"($env.REGISTRY)/modules/($img.name)@($digest)"
+
+}
+
+print $"(ansi green_bold)DONE!(ansi reset)"

build-unified.nu

@@ -0,0 +1,59 @@
+#!/usr/bin/env nu
+# generates modules-latest directory with only latest versions of modules and builds the Containerfile
+
+print $"(ansi green_bold)Gathering images(ansi reset)"
+
+rm -rf ./modules-latest
+mkdir ./modules-latest
+
+ls modules | each { |moduleDir|
+
+    # module is unversioned
+    if ($"($moduleDir.name)/($moduleDir.name | path basename).sh" | path exists) {
+
+        print $"(ansi cyan)Found(ansi reset) (ansi cyan_bold)unversioned(ansi reset) (ansi cyan)module:(ansi reset) ($moduleDir.name | path basename)"
+
+        cp --recursive ($moduleDir.name) $"./modules-latest/($moduleDir.name | path basename)"
+
+    } else { # module is versioned
+
+        print -n $"(ansi cyan)Found(ansi reset) (ansi blue_bold)versioned(ansi reset) (ansi cyan)module:(ansi reset) ($moduleDir.name | path basename), "
+
+        let latest = glob $"./($moduleDir.name)/v*" | last # the glob result is already orderer such that the last value is the biggest
+
+        print $"(ansi blue_bold)Latest version:(ansi reset) ($latest | path basename)"
+
+        cp --recursive ($latest) $"./modules-latest/($moduleDir.name | path basename)"
+
+    }
+}
+
+print $"(ansi green_bold)Starting image build(ansi reset)"
+
+let tags = (
+    if ($env.GH_EVENT_NAME != "pull_request" and $env.GH_BRANCH == "main") {
+        ["latest"]
+    } else if ($env.GH_EVENT_NAME != "pull_request") {
+        [$env.GH_BRANCH]
+    } else {
+        [$"pr-($env.GH_PR_NUMBER)"]
+    }
+)
+
+print $"(ansi green_bold)Generated tags for image:(ansi reset) ($tags | str join ' ')"
+
+(docker build .
+    -f ./unified.Containerfile
+    ...($tags | each { |tag| ["-t", $"($env.REGISTRY)/modules:($tag)"] } | flatten) # generate and spread list of tags
+)
+
+print $"(ansi cyan)Pushing image:(ansi reset) ($env.REGISTRY)/modules"
+let digest = (
+    docker push --all-tags $"($env.REGISTRY)/modules"
+        | split row "\n"  | last | split row " " | get 2 # parse push output to get digest for signing
+)
+
+print $"(ansi cyan)Signing image:(ansi reset) ($env.REGISTRY)/modules@($digest)"
+cosign sign -y --key env://COSIGN_PRIVATE_KEY $"($env.REGISTRY)/modules@($digest)"
+
+print $"(ansi green_bold)DONE!(ansi reset)"

individual.Containerfile

@@ -0,0 +1,6 @@
+FROM scratch
+
+ARG DIRECTORY
+ARG NAME
+
+COPY ${DIRECTORY} /modules/${NAME}

unified.Containerfile

@@ -0,0 +1,4 @@
+FROM scratch
+
+# modules-latest is generated by build-unified.nu
+COPY modules-latest /modules