Remove abstractions over binaries in the build processes. (#135)

### What Both CI and the Nix flake abstracted over binaries so we could build more than one of them. This is now pointless as we only have one. I have removed the abstractions to make working in these areas easier. ### How I inlined references to "ndc-postgres" in both the Nix flake and the GitHub Actions workflows.
hasura · Nov 3, 2023 · c05e230 · c05e230
1 parent ccd4446
commit c05e230
Show file tree

Hide file tree

Showing 8 changed files with 97 additions and 92 deletions.
diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml
@@ -27,7 +27,7 @@ jobs:
 
       - name: Build the Docker image 🔨
         run: |
-          docker load < $(nix build --no-link --print-out-paths '.#ndc-postgres-docker')
+          docker load < $(nix build --no-link --print-out-paths '.#docker')
 
       - name: Start dependencies ▶️
         run: |

diff --git a/.github/workflows/build-images-and-deploy.yaml b/.github/workflows/build-images-and-deploy.yaml
@@ -11,11 +11,6 @@ jobs:
   build_and_deploy:
     name: build and deploy
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        connector:
-          - ndc-postgres
-      fail-fast: false
     permissions:
       contents: read
       id-token: write
@@ -59,13 +54,13 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build and deploy Docker images to Google Container Registry 🚀
-        run: nix run .#publish-docker-image '${{ github.ref }}' '${{ matrix.connector }}' 'us-docker.pkg.dev/hasura-ddn/ddn/${{ matrix.connector }}'
+        run: nix run .#publish-docker-image '${{ github.ref }}' 'us-docker.pkg.dev/hasura-ddn/ddn/ndc-postgres'
 
       - name: Build and deploy Docker images to GitHub Packages 🚀
-        run: nix run .#publish-docker-image '${{ github.ref }}' '${{ matrix.connector }}' 'ghcr.io/hasura/${{ matrix.connector }}'
+        run: nix run .#publish-docker-image '${{ github.ref }}' 'ghcr.io/hasura/ndc-postgres'
 
       - name: Deploy to staging
-        if: ${{ matrix.connector == 'ndc-postgres' && github.ref == 'refs/heads/main' }}
+        if: github.ref == 'refs/heads/main'
         env:
           BUILDKITE_AUTH_TOKEN: ${{ secrets.BUILDKITE_AUTH_TOKEN }}
         run: |

diff --git a/.github/workflows/check-format.yaml b/.github/workflows/check-format.yaml
@@ -41,9 +41,16 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: DeterminateSystems/nix-installer-action@v6
+      - name: Install Nix ❄
+        uses: cachix/install-nix-action@v22
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
 
-      - uses: DeterminateSystems/magic-nix-cache-action@v2
+      - name: Set up the Nix Cache 🔌
+        uses: cachix/cachix-action@v12
+        with:
+          name: hasura-v3-dev
+          authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
 
       - name: check formatting
         run: |
@@ -55,9 +62,16 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: DeterminateSystems/nix-installer-action@v6
+      - name: Install Nix ❄
+        uses: cachix/install-nix-action@v22
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
 
-      - uses: DeterminateSystems/magic-nix-cache-action@v2
+      - name: Set up the Nix Cache 🔌
+        uses: cachix/cachix-action@v12
+        with:
+          name: hasura-v3-dev
+          authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
 
       - name: check formatting
         run: |

diff --git a/.github/workflows/nix-check.yaml b/.github/workflows/nix-check.yaml
@@ -8,11 +8,19 @@ jobs:
     name: nix flake check
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout 🛎️
+        uses: actions/checkout@v4
 
-      - uses: DeterminateSystems/nix-installer-action@v6
+      - name: Install Nix ❄
+        uses: cachix/install-nix-action@v22
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
 
-      - uses: DeterminateSystems/magic-nix-cache-action@v2
+      - name: Set up the Nix Cache 🔌
+        uses: cachix/cachix-action@v12
+        with:
+          name: hasura-v3-dev
+          authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
 
       - name: nix flake check
         run: |
@@ -22,11 +30,19 @@ jobs:
     name: nix develop
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-
-      - uses: DeterminateSystems/nix-installer-action@v6
-
-      - uses: DeterminateSystems/magic-nix-cache-action@v2
+      - name: Checkout 🛎️
+        uses: actions/checkout@v4
+
+      - name: Install Nix ❄
+        uses: cachix/install-nix-action@v22
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up the Nix Cache 🔌
+        uses: cachix/cachix-action@v12
+        with:
+          name: hasura-v3-dev
+          authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
 
       - name: nix develop --command true
         run: |

diff --git a/ci/deploy.sh b/ci/deploy.sh
@@ -2,7 +2,7 @@
 #
 # To get the skopeo dependency automatically, run with:
 #
-#     $ nix run .#publish-docker-image <github-ref>
+#     $ nix run .#publish-docker-image <github-ref> <image>
 #
 set -euo pipefail
 
@@ -14,22 +14,19 @@ if [[ "${1:-}" == '-n' || "${1:-}" == '--dry-run' ]]; then
 fi
 
 if [[ $# -ne 3 ]]; then
-    echo >&2 "Usage: ${0} [-n|--dry-run] REF BINARY IMAGE"
+    echo >&2 "Usage: ${0} [-n|--dry-run] REF IMAGE"
     echo >&2
     echo >&2 '    REF should be in the form "refs/heads/<branch>" or "refs/tags/<tag>"'
     echo >&2 '      (in a Github workflow the variable "github.ref" has this format)'
     echo >&2
-    echo >&2 '    BINARY is the name of the binary, e.g. "ndc-postgres"'
-    echo >&2
     echo >&2 '    IMAGE is the path of the Docker image, e.g. "ghcr.io/hasura/ndc-postgres"'
     echo >&2
     echo >&2 '    "--dry-run" will not push anything, but it will still build'
     exit 1
 fi
 
 github_ref="$1"
-binary_image_name="$2"
-image_path="$3"
+image="$2"
 
 # Runs the given command, unless `--dry-run` was set.
 function run {
@@ -110,12 +107,12 @@ function publish_multi_arch {
     # build and push the individual images for each architecture
     for arch in "${architectures[@]}"; do
       # build the docker image
-      image_archive="docker-archive://$(nix build --print-out-paths ".#${binary_image_name}-docker-${arch}-linux")"
+      image_archive="docker-archive://$(nix build --print-out-paths ".#docker-${arch}-linux")"
 
       echo "Will publish docker image with tags: ${docker_tags[*]}"
       skopeo inspect "$image_archive"
 
-      image_path_for_arch="${image_path}-${arch}" 
+      image_path_for_arch="${image}-${arch}" 
       for tag in "${docker_tags[@]}"; do
           echo
           echo "Pushing docker://${image_path_for_arch}:${tag}"
@@ -125,16 +122,16 @@ function publish_multi_arch {
 
     # now create and push the manifest
     for tag in "${docker_tags[@]}"; do
-      echo "Creating manifest for $image_path:$tag"
+      echo "Creating manifest for ${image}:${tag}"
       # create a manifest referencing both architectures
       # i did not use a loop here, forgive me
       run docker manifest create \
-          "$image_path:$tag" \
-           --amend "${image_path}-aarch64:$tag" \
-           --amend "${image_path}-x86_64:$tag"
+          "$image:$tag" \
+           --amend "${image}-aarch64:${tag}" \
+           --amend "${image}-x86_64:${tag}"
 
       # push manifest as the main image
-      run docker manifest push "$image_path:$tag"
+      run docker manifest push "${image}:${tag}"
     done
 }
 

diff --git a/docs/production.md b/docs/production.md
@@ -4,12 +4,12 @@ We ship the various connectors as Docker images, built with Nix.
 
 ## Build
 
-You can build each one with `nix build '.#ndc-<flavor>-docker'`, which will build a Docker tarball.
+You can build each one with `nix build '.#docker'`, which will build a Docker tarball.
 
 For example, to build the PostgreSQL image and load it into your Docker image registry, run:
 
 ```
-gunzip < "$(nix build --no-warn-dirty --no-link --print-out-paths '.#ndc-postgres-docker')" | docker load
+gunzip < "$(nix build --no-warn-dirty --no-link --print-out-paths '.#docker')" | docker load
 ```
 
 This will build an image named `ghcr.io/hasura/ndc-postgres:dev`.

diff --git a/flake.nix b/flake.nix
@@ -33,13 +33,19 @@
 
         cargoBuild = import ./nix/cargo-build.nix;
 
-        # create binaries for a given NDC
-        make-binaries = (binary-name: {
-          inherit binary-name;
+        binary-name = "ndc-postgres";
+
+        package = cargoBuild {
+          inherit binary-name crateExpression nixpkgs crane rust-overlay localSystem;
+        };
+
+        inherit (package) rustToolchain;
+      in
+      {
+        packages = rec {
           # a binary for whichever is the local computer
-          local-system = cargoBuild {
-            inherit binary-name crateExpression nixpkgs crane rust-overlay localSystem;
-          };
+          default = package;
+
           # cross compiler an x86_64 linux binary
           x86_64-linux = cargoBuild {
             inherit binary-name crateExpression nixpkgs crane rust-overlay localSystem;
@@ -50,51 +56,28 @@
             inherit binary-name crateExpression nixpkgs crane rust-overlay localSystem;
             crossSystem = "aarch64-linux";
           };
-        });
-
-        # given the binaries, return the flake targets that build Docker etc
-        make-packages =
-          (ndc-binaries:
-            let name = ndc-binaries.binary-name; in {
-              # binary compiled on local system
-              "${name}" = ndc-binaries.local-system;
-              # binary compiled for x86_64-linux
-              "${name}-x86_64-linux" = ndc-binaries.x86_64-linux;
-              # binary compiled for aarch64-linux
-              "${name}-aarch64-linux" = ndc-binaries.aarch64-linux;
-              # docker for local system
-              "${name}-docker" = pkgs.callPackage ./nix/docker.nix {
-                ndc-agent = ndc-binaries.local-system;
-                binary-name = name;
-                image-name = "ghcr.io/hasura/${name}";
-                tag = "dev";
-              };
-              # docker for x86_64-linux
-              "${name}-docker-x86_64-linux" = pkgs.callPackage ./nix/docker.nix {
-                ndc-agent = ndc-binaries.x86_64-linux;
-                architecture = "amd64";
-                binary-name = name;
-                image-name = "ghcr.io/hasura/${name}-x86_64";
-              };
-              # docker for aarch64-linux
-              "${name}-docker-aarch64-linux" = pkgs.callPackage ./nix/docker.nix {
-                ndc-agent = ndc-binaries.aarch64-linux;
-                architecture = "arm64";
-                binary-name = name;
-                image-name = "ghcr.io/hasura/${name}-aarch64";
-              };
-            });
-
-        postgres-binaries = make-binaries "ndc-postgres";
-
-        inherit (postgres-binaries.local-system) cargoArtifacts rustToolchain craneLib buildArgs;
 
-      in
-      {
-        packages = builtins.foldl' (x: y: x // y) { } [
-          (make-packages postgres-binaries)
-        ] // {
-          default = postgres-binaries.local-system;
+          # docker for local system
+          docker = pkgs.callPackage ./nix/docker.nix {
+            inherit binary-name;
+            ndc-agent = default;
+            image-name = "ghcr.io/hasura/ndc-postgres";
+            tag = "dev";
+          };
+          # docker for x86_64-linux
+          docker-x86_64-linux = pkgs.callPackage ./nix/docker.nix {
+            inherit binary-name;
+            ndc-agent = x86_64-linux;
+            architecture = "amd64";
+            image-name = "ghcr.io/hasura/ndc-postgres-x86_64";
+          };
+          # docker for aarch64-linux
+          docker-aarch64-linux = pkgs.callPackage ./nix/docker.nix {
+            inherit binary-name;
+            ndc-agent = aarch64-linux;
+            architecture = "arm64";
+            image-name = "ghcr.io/hasura/ndc-postgres-aarch64";
+          };
 
           publish-docker-image = pkgs.writeShellApplication {
             name = "publish-docker-image";
@@ -105,7 +88,7 @@
 
         checks = {
           # Build the crate as part of `nix flake check`
-          ndc-postgres = postgres-binaries.local-system;
+          ndc-postgres = package;
         };
 
         formatter = pkgs.nixpkgs-fmt;

diff --git a/justfile b/justfile
@@ -311,7 +311,7 @@ find-unused-dependencies:
 
 # check the nix builds work
 build-with-nix:
-  nix build --no-warn-dirty --print-build-logs '.#ndc-postgres'
+  nix build --no-warn-dirty --print-build-logs
 
 # run ndc-postgres-multitenant whilst outputting profile data for massif
 massif-postgres: start-dependencies
@@ -337,14 +337,14 @@ heaptrack-postgres: start-dependencies
 build-docker-with-nix:
   #!/usr/bin/env bash
   if [[ '{{CONNECTOR_IMAGE_TAG}}' == 'dev' ]]; then
-    echo "$(tput bold)nix build .#ndc-postgres-docker | gunzip | docker load$(tput sgr0)"
-    gunzip < "$(nix build --no-warn-dirty --no-link --print-out-paths '.#ndc-postgres-docker')" | docker load
+    echo "$(tput bold)nix build .#docker | gunzip | docker load$(tput sgr0)"
+    gunzip < "$(nix build --no-warn-dirty --no-link --print-out-paths '.#docker')" | docker load
   fi
 
 # check the Postgres arm64 docker build works
 build-aarch64-docker-with-nix:
   #!/usr/bin/env bash
   if [[ '{{CONNECTOR_IMAGE_TAG}}' == 'dev' ]]; then
-    echo "$(tput bold)nix build .#ndc-postgres-docker-aarch64-linux | gunzip | docker load$(tput sgr0)"
-    gunzip < "$(nix build --no-warn-dirty --no-link --print-out-paths --system aarch64-linux '.#ndc-postgres-docker-aarch64-linux')" | docker load
+    echo "$(tput bold)nix build .#docker-aarch64-linux | gunzip | docker load$(tput sgr0)"
+    gunzip < "$(nix build --no-warn-dirty --no-link --print-out-paths --system aarch64-linux '.#docker-aarch64-linux')" | docker load
   fi