diff --git a/.github/workflows/nix-deploy.yaml b/.github/workflows/nix-deploy.yaml index b0bdfb74..320d9a2e 100644 --- a/.github/workflows/nix-deploy.yaml +++ b/.github/workflows/nix-deploy.yaml @@ -1,67 +1,51 @@ -name: SQLServer NDC build and deploy with Nix +name: multi-architecture docker build on: push: branches: - main + - djh/multi-arch-build tags: - - 'v*' + - "v*" jobs: - binary: - name: deploy::binary + build_and_deploy: + name: build and deploy runs-on: ubuntu-latest + strategy: + matrix: + connector: + - ndc-sqlserver + fail-fast: false + permissions: + contents: read + id-token: write + packages: write + steps: - name: Checkout 🛎️ uses: actions/checkout@v4 - name: Install Nix ❄ - uses: DeterminateSystems/nix-installer-action@v4 + uses: DeterminateSystems/nix-installer-action@v6 - name: Run the Magic Nix Cache 🔌 uses: DeterminateSystems/magic-nix-cache-action@v2 - - name: Login to GitHub Container Registry 📦 - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: build the crate using nix 🔨 - run: nix build --print-build-logs - - - name: Create release 🚀 - uses: actions/upload-artifact@v3 + - id: gcloud-auth + name: Authenticate to Google Cloud 🔑 + uses: google-github-actions/auth@v1 with: - name: hasura-sqlserver-agent-rs - path: result/bin/ndc-sqlserver + token_format: access_token + service_account: "hasura-ci-docker-writer@hasura-ddn.iam.gserviceaccount.com" + workload_identity_provider: "projects/1025009031284/locations/global/workloadIdentityPools/hasura-ddn/providers/github" - # scream into Slack if something goes wrong - - name: Report Status - if: always() - uses: ravsamhq/notify-slack-action@v2 + - name: Login to Google Container Registry 📦 + uses: "docker/login-action@v3" with: - status: ${{ job.status }} - notify_when: failure - notification_title: '😧 Error on <{repo_url}|{repo}>' - message_format: '🐴 *{workflow}* {status_message} for <{repo_url}|{repo}>' - env: - SLACK_WEBHOOK_URL: ${{ secrets.BROKEN_BUILD_SLACK_WEBHOOK_URL }} - - docker: - name: deploy::docker - needs: binary - runs-on: ubuntu-latest - steps: - - name: Checkout 🛎️ - uses: actions/checkout@v4 - - - name: Install Nix ❄ - uses: DeterminateSystems/nix-installer-action@v4 - - - name: Run the Magic Nix Cache 🔌 - uses: DeterminateSystems/magic-nix-cache-action@v2 + registry: "us-docker.pkg.dev" + username: "oauth2accesstoken" + password: "${{ steps.gcloud-auth.outputs.access_token }}" - name: Login to GitHub Container Registry 📦 uses: docker/login-action@v3 @@ -70,8 +54,11 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Deploy 🚀 - run: nix run .#publish-docker-image ${{ github.ref }} + - name: Build and deploy Docker images to Google Container Registry 🚀 + run: nix run .#publish-docker-image '${{ github.ref }}' '${{ matrix.connector }}' 'us-docker.pkg.dev/hasura-ddn/ddn/${{ matrix.connector }}' + + - name: Build and deploy Docker images to GitHub Packages 🚀 + run: nix run .#publish-docker-image '${{ github.ref }}' '${{ matrix.connector }}' 'ghcr.io/hasura/${{ matrix.connector }}' # scream into Slack if something goes wrong - name: Report Status @@ -80,7 +67,7 @@ jobs: with: status: ${{ job.status }} notify_when: failure - notification_title: '😧 Error on <{repo_url}|{repo}>' - message_format: '🐴 *{workflow}* {status_message} for <{repo_url}|{repo}>' + notification_title: "😧 Error on <{repo_url}|{repo}>" + message_format: "🐴 *{workflow}* {status_message} for <{repo_url}|{repo}>" env: SLACK_WEBHOOK_URL: ${{ secrets.BROKEN_BUILD_SLACK_WEBHOOK_URL }} diff --git a/ci/deploy.sh b/ci/deploy.sh index c4c1c8f5..6434d058 100755 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -6,15 +6,40 @@ # set -euo pipefail -IMAGE_PATH=ghcr.io/hasura/sqlserver-agent-rs +DRY_RUN=false +if [[ "${1:-}" == '-n' || "${1:-}" == '--dry-run' ]]; then + DRY_RUN=true + echo "$(tput bold)$(tput setaf 1)DRY RUN; some steps will be skipped$(tput sgr0)" + shift +fi -if [ -z "${1+x}" ]; then - echo "Expected argument of the form refs/heads/ or refs/tags/." - echo "(In a Github workflow the variable github.ref has this format)" +if [[ $# -ne 3 ]]; then + echo >&2 "Usage: ${0} [-n|--dry-run] REF BINARY IMAGE" + echo >&2 + echo >&2 ' REF should be in the form "refs/heads/" or "refs/tags/"' + echo >&2 ' (in a Github workflow the variable "github.ref" has this format)' + echo >&2 + echo >&2 ' BINARY is the name of the binary, e.g. "ndc-postgres"' + echo >&2 + echo >&2 ' IMAGE is the path of the Docker image, e.g. "ghcr.io/hasura/ndc-postgres"' + echo >&2 + echo >&2 ' "--dry-run" will not push anything, but it will still build' exit 1 fi github_ref="$1" +binary_image_name="$2" +image_path="$3" + +# Runs the given command, unless `--dry-run` was set. +function run { + if "$DRY_RUN"; then + echo "$(tput bold)$(tput setaf 1)not running:$(tput sgr0) $*" + else + echo "$(tput bold)$(tput setaf 2)running:$(tput sgr0) $*" + "$@" + fi +} # Assumes that the given ref is a branch name. Sets a tag for a docker image of # the form: @@ -56,7 +81,8 @@ function set_dev_tags { # # If the input is a branch, set docker tags via `set_dev_tags`. function set_docker_tags { - local input="$1" + local input + input="$1" if [[ $input =~ ^refs/tags/(v.*)$ ]]; then local tag="${BASH_REMATCH[1]}" export docker_tags=("$tag") @@ -68,27 +94,51 @@ function set_docker_tags { fi } -function maybe_publish { - local input="$1" +function publish_multi_arch { + local input + local image_archive + local image_path_for_arch + + architectures=('aarch64' 'x86_64') + + input="$1" set_docker_tags "$input" + + # do nothing if no tags found if [[ ${#docker_tags[@]} == 0 ]]; then - echo "The given ref, $input, was not a release tag or a branch - will not publish a docker image" + echo "The given ref, ${input}, was not a release tag or a branch - will not publish a docker image" exit fi - echo "Will publish docker image with tags: ${docker_tags[*]}" + # build and push the individual images for each architecture + for arch in "${architectures[@]}"; do + # build the docker image + image_archive="docker-archive://$(nix build --print-out-paths ".#${binary_image_name}-docker-${arch}-linux")" - nix build .#docker --print-build-logs # writes a tar file to ./result - ls -lh result - local image_archive - image_archive=docker-archive://"$(readlink -f result)" - skopeo inspect "$image_archive" + echo "Will publish docker image with tags: ${docker_tags[*]}" + skopeo inspect "$image_archive" + + image_path_for_arch="${image_path}-${arch}" + for tag in "${docker_tags[@]}"; do + echo + echo "Pushing docker://${image_path_for_arch}:${tag}" + run skopeo copy "$image_archive" "docker://${image_path_for_arch}:${tag}" + done + done + # now create and push the manifest for tag in "${docker_tags[@]}"; do - echo - echo "Pushing docker://$IMAGE_PATH:$tag" - skopeo copy "$image_archive" docker://"$IMAGE_PATH:$tag" + echo "Creating manifest for $image_path:$tag" + # create a manifest referencing both architectures + # i did not use a loop here, forgive me + run docker manifest create \ + "$image_path:$tag" \ + --amend "${image_path}-aarch64:$tag" \ + --amend "${image_path}-x86_64:$tag" + + # push manifest as the main image + run docker manifest push "$image_path:$tag" done } -maybe_publish "$github_ref" +publish_multi_arch "$github_ref" diff --git a/flake.lock b/flake.lock index 525f7930..36cd68b0 100644 --- a/flake.lock +++ b/flake.lock @@ -1,21 +1,5 @@ { "nodes": { - "advisory-db": { - "flake": false, - "locked": { - "lastModified": 1695296241, - "narHash": "sha256-t3SBLTqXyMlloAC0qSTHRAlWweiww6rmp0gSotDy1yo=", - "owner": "rustsec", - "repo": "advisory-db", - "rev": "81594d9fd5b32c3eaa199812475498b47fc98742", - "type": "github" - }, - "original": { - "owner": "rustsec", - "repo": "advisory-db", - "type": "github" - } - }, "crane": { "inputs": { "flake-compat": "flake-compat", @@ -30,11 +14,11 @@ ] }, "locked": { - "lastModified": 1695359325, - "narHash": "sha256-HZOraC0Cs2vNhygxRoo76NFeBXh7oZY1ZbGMzOMI68Y=", + "lastModified": 1697596235, + "narHash": "sha256-4VTrrTdoA1u1wyf15krZCFl3c29YLesSNioYEgfb2FY=", "owner": "ipetkov", "repo": "crane", - "rev": "9dae37b4a545f05aa70a2f048428c5196690c5a4", + "rev": "c97a0c0d83bfdf01c29113c5592a3defc27cb315", "type": "github" }, "original": { @@ -46,11 +30,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1673956053, - "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "lastModified": 1696267196, + "narHash": "sha256-AAQ/2sD+0D18bb8hKuEEVpHUYD1GmO2Uh/taFamn6XQ=", "owner": "edolstra", "repo": "flake-compat", - "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "rev": "4f910c9827911b1ec2bf26b5a062cd09f8d89f85", "type": "github" }, "original": { @@ -79,11 +63,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1695145219, - "narHash": "sha256-Eoe9IHbvmo5wEDeJXKFOpKUwxYJIOxKUesounVccNYk=", + "lastModified": 1697456312, + "narHash": "sha256-roiSnrqb5r+ehnKCauPLugoU8S36KgmWraHgRqVYndo=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "5ba549eafcf3e33405e5f66decd1a72356632b96", + "rev": "ca012a02bf8327be9e488546faecae5e05d7d749", "type": "github" }, "original": { @@ -95,7 +79,6 @@ }, "root": { "inputs": { - "advisory-db": "advisory-db", "crane": "crane", "flake-utils": "flake-utils", "nixpkgs": "nixpkgs", @@ -112,11 +95,11 @@ ] }, "locked": { - "lastModified": 1695348673, - "narHash": "sha256-Rtg2h1edMTuNGo9UP0Ync9joT97TS3E1lddx8w8vr3s=", + "lastModified": 1697595136, + "narHash": "sha256-9honwiIeMbBKi7FzfEy89f1ShUiXz/gVxZSS048pKyc=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "9bfad7bee1816a5e171b96fff7c855d5037f902a", + "rev": "a2ccfb2134622b28668a274e403ba6f075ae1223", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 5c942a8f..afa1382f 100644 --- a/flake.nix +++ b/flake.nix @@ -1,5 +1,5 @@ { - description = "PostgreSQL data connector"; + description = "SQLServer data connector"; inputs = { nixpkgs.url = github:NixOS/nixpkgs/nixos-unstable; @@ -19,14 +19,9 @@ flake-utils.follows = "flake-utils"; }; }; - - advisory-db = { - url = "github:rustsec/advisory-db"; - flake = false; - }; }; - outputs = { self, nixpkgs, flake-utils, crane, rust-overlay, advisory-db }: + outputs = { self, nixpkgs, flake-utils, crane, rust-overlay }: flake-utils.lib.eachDefaultSystem (localSystem: let pkgs = import nixpkgs { @@ -34,71 +29,75 @@ overlays = [ rust-overlay.overlays.default ]; }; - # Edit ./nix/sqlserver-agent.nix to adjust library and buildtime + # Edit ./nix/ndc-agent.nix to adjust library and buildtime # dependencies or other build configuration for sqlserver-agent - crateExpression = import ./nix/sqlserver-agent.nix; - cargoBuild = import ./nix/cargo-build.nix; + crateExpression = import ./nix/ndc-agent.nix; - # Build for the architecture and OS that is running the build - sqlserver-agent = cargoBuild { - inherit crateExpression nixpkgs crane rust-overlay localSystem; - }; - - inherit (sqlserver-agent) cargoArtifacts rustToolchain craneLib buildArgs; - - sqlserver-agent-x86_64-linux = cargoBuild { - inherit crateExpression nixpkgs crane rust-overlay localSystem; - crossSystem = "x86_64-linux"; - }; + # cockroachExpression = import ./nix/cockroach-agent.nix; + cargoBuild = import ./nix/cargo-build.nix; - sqlserver-agent-aarch64-linux = cargoBuild { - inherit crateExpression nixpkgs crane rust-overlay localSystem; - crossSystem = "aarch64-linux"; - }; - in - { - checks = { - # Build the crate as part of `nix flake check` - inherit sqlserver-agent; - - crate-clippy = craneLib.cargoClippy (buildArgs // { - inherit cargoArtifacts; - cargoClippyExtraArgs = "--all-targets -- --deny warnings"; - }); - - crate-nextest = craneLib.cargoNextest (buildArgs // { - inherit cargoArtifacts; - partitions = 1; - partitionType = "count"; - }); - - crate-audit = craneLib.cargoAudit { - inherit advisory-db; - inherit (sqlserver-agent) src; + # create binaries for a given NDC + make-binaries = (binary-name: { + inherit binary-name; + # a binary for whichever is the local computer + local-system = cargoBuild { + inherit binary-name crateExpression nixpkgs crane rust-overlay localSystem; }; - }; - - packages = { - default = sqlserver-agent; - sqlserver-agent-x86_64-linux = sqlserver-agent-x86_64-linux; - sqlserver-agent-aarch64-linux = sqlserver-agent-aarch64-linux; - - docker = pkgs.callPackage ./nix/docker.nix { inherit sqlserver-agent; }; - - dockerDev = pkgs.callPackage ./nix/docker.nix { - inherit sqlserver-agent; - tag = "dev"; + # cross compiler an x86_64 linux binary + x86_64-linux = cargoBuild { + inherit binary-name crateExpression nixpkgs crane rust-overlay localSystem; + crossSystem = "x86_64-linux"; }; - - docker-x86_64-linux = pkgs.callPackage ./nix/docker.nix { - sqlserver-agent = sqlserver-agent-x86_64-linux; - architecture = "amd64"; + # cross compile a aarch64 linux binary + aarch64-linux = cargoBuild { + inherit binary-name crateExpression nixpkgs crane rust-overlay localSystem; + crossSystem = "aarch64-linux"; }; + }); + + # given the binaries, return the flake targets that build Docker etc + make-packages = + (ndc-binaries: + let name = ndc-binaries.binary-name; in { + # binary compiled on local system + "${name}" = ndc-binaries.local-system; + # binary compiled for x86_64-linux + "${name}-x86_64-linux" = ndc-binaries.x86_64-linux; + # binary compiled for aarch64-linux + "${name}-aarch64-linux" = ndc-binaries.aarch64-linux; + # docker for local system + "${name}-docker" = pkgs.callPackage ./nix/docker.nix { + ndc-agent = ndc-binaries.local-system; + binary-name = name; + image-name = "ghcr.io/hasura/${name}"; + tag = "dev"; + }; + # docker for x86_64-linux + "${name}-docker-x86_64-linux" = pkgs.callPackage ./nix/docker.nix { + ndc-agent = ndc-binaries.x86_64-linux; + architecture = "amd64"; + binary-name = name; + image-name = "ghcr.io/hasura/${name}-x86_64"; + }; + # docker for aarch64-linux + "${name}-docker-aarch64-linux" = pkgs.callPackage ./nix/docker.nix { + ndc-agent = ndc-binaries.aarch64-linux; + architecture = "arm64"; + binary-name = name; + image-name = "ghcr.io/hasura/${name}-aarch64"; + }; + }); + + sqlserver-binaries = make-binaries "ndc-sqlserver"; + + inherit (sqlserver-binaries.local-system) cargoArtifacts rustToolchain craneLib buildArgs; - docker-aarch64-linux = pkgs.callPackage ./nix/docker.nix { - sqlserver-agent = sqlserver-agent-aarch64-linux; - architecture = "arm64"; - }; + in + { + packages = builtins.foldl' (x: y: x // y) { } [ + (make-packages sqlserver-binaries) + ] // { + default = sqlserver-binaries.local-system; publish-docker-image = pkgs.writeShellApplication { name = "publish-docker-image"; @@ -107,6 +106,11 @@ }; }; + checks = { + # Build the crate as part of `nix flake check` + ndc-sqlserver = sqlserver-binaries.local-system; + }; + formatter = pkgs.nixpkgs-fmt; devShells.default = pkgs.mkShell { @@ -120,13 +124,26 @@ pkgs.cargo-flamegraph pkgs.cargo-insta pkgs.cargo-machete + pkgs.cargo-nextest pkgs.cargo-watch pkgs.just pkgs.k6 + pkgs.nixpkgs-fmt + pkgs.nodePackages.prettier pkgs.pkg-config pkgs.rnix-lsp + pkgs.skopeo + pkgs.nodePackages.prettier rustToolchain - ]; + ] ++ ( + pkgs.lib.optionals + pkgs.stdenv.isLinux + [ + pkgs.heaptrack + pkgs.linuxPackages_latest.perf + pkgs.valgrind + ] + ); }; }); } diff --git a/nix/cargo-build.nix b/nix/cargo-build.nix index b935c753..b4b64967 100644 --- a/nix/cargo-build.nix +++ b/nix/cargo-build.nix @@ -22,6 +22,7 @@ , crane , rust-overlay , localSystem +, binary-name , crossSystem ? localSystem }: @@ -85,7 +86,7 @@ let # Call the given crateExpression to build the crate - or rather to get # a derivation that will build the crate. crate = pkgs.callPackage crateExpression { - inherit craneLib staticallyLinked; + inherit craneLib staticallyLinked binary-name; }; in # Override the derivation to add cross-compilation and static linking environment variables. diff --git a/nix/docker.nix b/nix/docker.nix index dc53cb96..a7a03d08 100644 --- a/nix/docker.nix +++ b/nix/docker.nix @@ -1,21 +1,22 @@ # This is a function that returns a derivation for a docker image. -{ sqlserver-agent +{ ndc-agent +, binary-name , dockerTools , lib , architecture ? null -, name ? "ghcr.io/hasura/sqlserver-agent-rs" +, image-name , tag ? null # defaults to the output hash , extraConfig ? { } # see config options at: https://github.com/moby/moby/blob/master/image/spec/v1.2.md#image-json-field-descriptions }: let args = { - inherit name; + name = image-name; created = "now"; - + contents = [ ndc-agent ]; config = { Entrypoint = [ - "${sqlserver-agent}/bin/ndc-sqlserver" + "/bin/${binary-name}" ]; ExposedPorts = { "8100/tcp" = { }; }; } // extraConfig; diff --git a/nix/sqlserver-agent.nix b/nix/ndc-agent.nix similarity index 93% rename from nix/sqlserver-agent.nix rename to nix/ndc-agent.nix index 56e90063..41a9f891 100644 --- a/nix/sqlserver-agent.nix +++ b/nix/ndc-agent.nix @@ -1,4 +1,4 @@ -# Dependencies and build configuration for the sqlserver-agent crate. +# Dependencies and build configuration for an ndc-agent crate. # # To add runtime library dependencies, add packge names to the argument set # here, and add the same name to the `buildInputs` list below. @@ -20,6 +20,7 @@ , protobuf , stdenv , pkgsStatic +, binary-name }: let @@ -37,7 +38,7 @@ let buildArgs = { inherit src; - pname = "sqlserver-ndc"; + pname = binary-name; buildInputs = [ openssl @@ -69,6 +70,7 @@ let (buildArgs // { inherit cargoArtifacts; doCheck = false; + cargoExtraArgs = "--locked --bin ${binary-name}"; }); in crate.overrideAttrs (prev: {