Merge pull request #38 from hasura/djh/multi-arch-build

Multi-arch build

.github/workflows/nix-deploy.yaml

@@ -1,16 +1,27 @@
-name: SQLServer NDC build and deploy with Nix
+name: multi-architecture docker build
 
 on:
   push:
     branches:
       - main
+      - djh/multi-arch-build
     tags:
-      - 'v*'
+      - "v*"
 
 jobs:
-  binary:
-    name: deploy::binary
+  build_and_deploy:
+    name: build and deploy
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        connector:
+          - ndc-sqlserver
+      fail-fast: false
+    permissions:
+      contents: read
+      id-token: write
+      packages: write
+
     steps:
       - name: Checkout 🛎️
         uses: actions/checkout@v4
@@ -21,47 +32,24 @@ jobs:
       - name: Run the Magic Nix Cache 🔌
         uses: DeterminateSystems/magic-nix-cache-action@v2
 
-      - name: Login to GitHub Container Registry 📦
-        uses: docker/login-action@v3
+      # skip until GCP credentials sorted
+      - id: gcloud-auth
+        if: false
+        name: Authenticate to Google Cloud 🔑
+        uses: google-github-actions/auth@v1
         with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+          token_format: access_token
+          service_account: "[email protected]"
+          workload_identity_provider: "projects/1025009031284/locations/global/workloadIdentityPools/hasura-ddn/providers/github"
 
-      - name: build the crate using nix 🔨
-        run: nix build --print-build-logs
-
-      - name: Create release 🚀
-        uses: actions/upload-artifact@v3
+      # skip until GCP credentials sorted
+      - name: Login to Google Container Registry 📦
+        if: false
+        uses: "docker/login-action@v3"
         with:
-          name: hasura-sqlserver-agent-rs
-          path: result/bin/ndc-sqlserver
-
-      # scream into Slack if something goes wrong
-      - name: Report Status
-        if: always()
-        uses: ravsamhq/notify-slack-action@v2
-        with:
-          status: ${{ job.status }}
-          notify_when: failure
-          notification_title: '😧 Error on <{repo_url}|{repo}>'
-          message_format: '🐴 *{workflow}* {status_message} for <{repo_url}|{repo}>'
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.BROKEN_BUILD_SLACK_WEBHOOK_URL }}
-
-  docker:
-    name: deploy::docker
-    needs: binary
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout 🛎️
-        uses: actions/checkout@v4
-
-      - name: Install Nix ❄
-        uses: DeterminateSystems/nix-installer-action@v6
-
-      - name: Run the Magic Nix Cache 🔌
-        uses: DeterminateSystems/magic-nix-cache-action@v2
+          registry: "us-docker.pkg.dev"
+          username: "oauth2accesstoken"
+          password: "${{ steps.gcloud-auth.outputs.access_token }}"
 
       - name: Login to GitHub Container Registry 📦
         uses: docker/login-action@v3
@@ -70,8 +58,13 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Deploy 🚀
-        run: nix run .#publish-docker-image ${{ github.ref }}
+      # skip until GCP credentials sorted
+      - name: Build and deploy Docker images to Google Container Registry 🚀
+        if: false
+        run: nix run .#publish-docker-image '${{ github.ref }}' '${{ matrix.connector }}' 'us-docker.pkg.dev/hasura-ddn/ddn/${{ matrix.connector }}'
+
+      - name: Build and deploy Docker images to GitHub Packages 🚀
+        run: nix run .#publish-docker-image '${{ github.ref }}' '${{ matrix.connector }}' 'ghcr.io/hasura/${{ matrix.connector }}'
 
       # scream into Slack if something goes wrong
       - name: Report Status
@@ -80,7 +73,7 @@ jobs:
         with:
           status: ${{ job.status }}
           notify_when: failure
-          notification_title: '😧 Error on <{repo_url}|{repo}>'
-          message_format: '🐴 *{workflow}* {status_message} for <{repo_url}|{repo}>'
+          notification_title: "😧 Error on <{repo_url}|{repo}>"
+          message_format: "🐴 *{workflow}* {status_message} for <{repo_url}|{repo}>"
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.BROKEN_BUILD_SLACK_WEBHOOK_URL }}

ci/deploy.sh

@@ -6,15 +6,40 @@
 #
 set -euo pipefail
 
-IMAGE_PATH=ghcr.io/hasura/sqlserver-agent-rs
+DRY_RUN=false
+if [[ "${1:-}" == '-n' || "${1:-}" == '--dry-run' ]]; then
+  DRY_RUN=true
+  echo "$(tput bold)$(tput setaf 1)DRY RUN; some steps will be skipped$(tput sgr0)"
+  shift
+fi
 
-if [ -z "${1+x}" ]; then
-    echo "Expected argument of the form refs/heads/<branch name> or refs/tags/<tag name>."
-    echo "(In a Github workflow the variable github.ref has this format)"
+if [[ $# -ne 3 ]]; then
+    echo >&2 "Usage: ${0} [-n|--dry-run] REF BINARY IMAGE"
+    echo >&2
+    echo >&2 '    REF should be in the form "refs/heads/<branch>" or "refs/tags/<tag>"'
+    echo >&2 '      (in a Github workflow the variable "github.ref" has this format)'
+    echo >&2
+    echo >&2 '    BINARY is the name of the binary, e.g. "ndc-postgres"'
+    echo >&2
+    echo >&2 '    IMAGE is the path of the Docker image, e.g. "ghcr.io/hasura/ndc-postgres"'
+    echo >&2
+    echo >&2 '    "--dry-run" will not push anything, but it will still build'
     exit 1
 fi
 
 github_ref="$1"
+binary_image_name="$2"
+image_path="$3"
+
+# Runs the given command, unless `--dry-run` was set.
+function run {
+  if "$DRY_RUN"; then
+    echo "$(tput bold)$(tput setaf 1)not running:$(tput sgr0) $*"
+  else
+    echo "$(tput bold)$(tput setaf 2)running:$(tput sgr0) $*"
+    "$@"
+  fi
+}
 
 # Assumes that the given ref is a branch name. Sets a tag for a docker image of
 # the form:
@@ -56,7 +81,8 @@ function set_dev_tags {
 #
 # If the input is a branch, set docker tags via `set_dev_tags`.
 function set_docker_tags {
-    local input="$1"
+    local input
+    input="$1"
     if [[ $input =~ ^refs/tags/(v.*)$ ]]; then
         local tag="${BASH_REMATCH[1]}"
         export docker_tags=("$tag")
@@ -68,27 +94,51 @@ function set_docker_tags {
     fi
 }
 
-function maybe_publish {
-    local input="$1"
+function publish_multi_arch {
+    local input
+    local image_archive
+    local image_path_for_arch
+
+    architectures=('aarch64' 'x86_64')
+
+    input="$1"
     set_docker_tags "$input"
+
+    # do nothing if no tags found
     if [[ ${#docker_tags[@]} == 0 ]]; then
-        echo "The given ref, $input, was not a release tag or a branch - will not publish a docker image"
+        echo "The given ref, ${input}, was not a release tag or a branch - will not publish a docker image"
         exit
     fi
 
-    echo "Will publish docker image with tags: ${docker_tags[*]}"
+    # build and push the individual images for each architecture
+    for arch in "${architectures[@]}"; do
+      # build the docker image
+      image_archive="docker-archive://$(nix build --print-out-paths ".#${binary_image_name}-docker-${arch}-linux")"
 
-    nix build .#docker --print-build-logs # writes a tar file to ./result
-    ls -lh result
-    local image_archive
-    image_archive=docker-archive://"$(readlink -f result)"
-    skopeo inspect "$image_archive"
+      echo "Will publish docker image with tags: ${docker_tags[*]}"
+      skopeo inspect "$image_archive"
+
+      image_path_for_arch="${image_path}-${arch}" 
+      for tag in "${docker_tags[@]}"; do
+          echo
+          echo "Pushing docker://${image_path_for_arch}:${tag}"
+          run skopeo copy "$image_archive" "docker://${image_path_for_arch}:${tag}"
+      done
+    done
 
+    # now create and push the manifest
     for tag in "${docker_tags[@]}"; do
-        echo
-        echo "Pushing docker://$IMAGE_PATH:$tag"
-        skopeo copy "$image_archive" docker://"$IMAGE_PATH:$tag"
+      echo "Creating manifest for $image_path:$tag"
+      # create a manifest referencing both architectures
+      # i did not use a loop here, forgive me
+      run docker manifest create \
+          "$image_path:$tag" \
+           --amend "${image_path}-aarch64:$tag" \
+           --amend "${image_path}-x86_64:$tag"
+
+      # push manifest as the main image
+      run docker manifest push "$image_path:$tag"
     done
 }
 
-maybe_publish "$github_ref"
+publish_multi_arch "$github_ref"