From 9f702b46edbab4218d4fe2c384c8e1ed31217800 Mon Sep 17 00:00:00 2001 From: Eric Promislow Date: Fri, 17 May 2024 12:57:27 -0700 Subject: [PATCH 1/4] Migrate webhook CI to GHA - Misc. changes to CI caught during further testing: 1. There should be two shell variables, HELM_VERSION for pulling down the right version of helm, and HELM_CHART_VERSION, for identifying generated helm charts. 2. Similarly, `TAG` and `IMAGE_TAG` are two different things. 3. Stop using make/dapper: env vars get lost 4. Skip the RKE Machine config unit test on CI/arm64 - Written up in issue 45837 --- .drone.yml | 191 ------------------ .github/workflows/ci.yaml | 88 ++++++++ .github/workflows/publish.yaml | 55 +++++ .github/workflows/release.yaml | 95 +++++++++ .github/workflows/scripts/install-k3d.sh | 18 ++ .github/workflows/scripts/integration-test-ci | 82 ++++++++ .github/workflows/scripts/setup-cluster.sh | 72 +++++++ .github/workflows/scripts/start-rancher.sh | 27 +++ .github/workflows/scripts/try.sh | 59 ++++++ .gitignore | 3 +- Dockerfile.dapper | 2 +- scripts/build | 2 +- scripts/ci | 1 + scripts/integration-test | 88 +++----- scripts/package | 47 ++--- scripts/package-helm | 14 +- scripts/test-helm | 1 - scripts/version | 21 +- tests/integration/rkeMachineConfig_test.go | 8 + 19 files changed, 573 insertions(+), 301 deletions(-) delete mode 100644 .drone.yml create mode 100644 .github/workflows/ci.yaml create mode 100644 .github/workflows/publish.yaml create mode 100644 .github/workflows/release.yaml create mode 100755 .github/workflows/scripts/install-k3d.sh create mode 100755 .github/workflows/scripts/integration-test-ci create mode 100755 .github/workflows/scripts/setup-cluster.sh create mode 100755 .github/workflows/scripts/start-rancher.sh create mode 100755 .github/workflows/scripts/try.sh diff --git a/.drone.yml b/.drone.yml deleted file mode 100644 index 3f141cfbf..000000000 --- a/.drone.yml +++ /dev/null @@ -1,191 +0,0 @@ ---- -kind: pipeline -name: amd64 -type: docker - -platform: - os: linux - arch: amd64 - -steps: - - name: build - image: rancher/dapper:v0.6.0 - commands: - - dapper ci - volumes: - - name: docker - path: /var/run/docker.sock - - - name: integration-test - image: rancher/rancher:v2.9-head - pull: always - privileged: true - commands: - - curl --cacert /etc/ssl/ca-bundle.pem -sL https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz | tar xvzf - -C /usr/local/bin --strip-components=1 - - scripts/integration-test - - - name: github_binary_release - image: plugins/github-release - settings: - api_key: - from_secret: github_token - prerelease: true - checksum: - - sha256 - checksum_file: CHECKSUMsum-amd64.txt - checksum_flatten: true - files: - - "dist/artifacts/*" - when: - instance: - - drone-publish.rancher.io - ref: - - refs/head/master - - refs/tags/* - event: - - tag - - - name: docker-publish - image: plugins/docker - settings: - dockerfile: package/Dockerfile - password: - from_secret: docker_password - repo: "rancher/rancher-webhook" - tag: "${DRONE_TAG}-amd64" - username: - from_secret: docker_username - when: - instance: - - drone-publish.rancher.io - ref: - - refs/head/master - - refs/tags/* - event: - - tag -volumes: - - name: docker - host: - path: /var/run/docker.sock - ---- -kind: pipeline -name: arm64 -type: docker - -platform: - os: linux - arch: arm64 - -steps: - - name: build - image: rancher/dapper:v0.6.0 - commands: - - dapper ci - volumes: - - name: docker - path: /var/run/docker.sock - - - name: integration-test - image: rancher/rancher:v2.9-head - pull: always - privileged: true - commands: - - curl --cacert /etc/ssl/ca-bundle.pem -sL https://get.helm.sh/helm-v3.13.3-linux-arm64.tar.gz | tar xvzf - -C /usr/local/bin --strip-components=1 - - scripts/integration-test - - - name: github_binary_release - image: plugins/github-release - settings: - api_key: - from_secret: github_token - prerelease: true - checksum: - - sha256 - checksum_file: CHECKSUMsum-arm64.txt - checksum_flatten: true - files: - - "dist/artifacts/*" - when: - instance: - - drone-publish.rancher.io - ref: - - refs/head/master - - refs/tags/* - event: - - tag - - - name: docker-publish - image: plugins/docker - settings: - dockerfile: package/Dockerfile - password: - from_secret: docker_password - repo: "rancher/rancher-webhook" - tag: "${DRONE_TAG}-arm64" - username: - from_secret: docker_username - when: - instance: - - drone-publish.rancher.io - ref: - - refs/head/master - - refs/tags/* - event: - - tag - -volumes: - - name: docker - host: - path: /var/run/docker.sock - ---- -kind: pipeline -name: manifest -type: docker - -platform: - os: linux - arch: amd64 - -steps: - - name: manifest - image: plugins/manifest:1.4.0 - settings: - username: - from_secret: docker_username - password: - from_secret: docker_password - platforms: - - linux/amd64 - - linux/arm64 - target: "rancher/rancher-webhook:${DRONE_TAG}" - template: "rancher/rancher-webhook:${DRONE_TAG}-ARCH" - when: - instance: - - drone-publish.rancher.io - ref: - - refs/head/master - - refs/tags/* - event: - - tag - -depends_on: - - amd64 - - arm64 - ---- -kind: pipeline -name: fossa -type: docker - -steps: - - name: fossa - image: rancher/drone-fossa:latest - failure: ignore - settings: - api_key: - from_secret: FOSSA_API_KEY - when: - instance: - - drone-publish.rancher.io diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 000000000..9cea8796d --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,88 @@ +# Add a comment to force a change. + +name: Webhook CI + +on: + push: + branches: + - release/v* + paths-ignore: + - '.gitignore' + - 'CODEOWNERS' + - 'LICENSE' + pull_request: + paths-ignore: + - '.gitignore' + - 'CODEOWNERS' + - 'LICENSE' + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + name: CI + strategy: + matrix: + archBox: + - { arch: amd64, vmArch: x64 } + - { arch: arm64, vmArch: arm64 } + runs-on: runs-on,runner=1cpu-linux-${{ matrix.archBox.vmArch }},run-id=${{ github.run_id }} + steps: + - name : Checkout repository + # https://github.com/actions/checkout/releases/tag/v4.1.1 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Setup Go + # https://github.com/actions/setup-go/releases/tag/v5.0.0 + uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + with: + go-version-file: 'go.mod' + + # TODO: Pull this next one out once there's a helm-release for rancher 2.9 + - name: Checkout rancher/rancher and build the chart + run: | + mkdir -p "${{ runner.temp}}" + pushd "${{ runner.temp}}" + git clone --depth 1 -b release/v2.9 https://github.com/rancher/rancher.git rancherDir + cd rancherDir + ./scripts/chart/build chart + tar cfz "${{ runner.temp }}/rancher.tgz" -C build/chart/rancher . + popd + + - name: install K3d + run: ./.github/workflows/scripts/install-k3d.sh + env: + K3D_VERSION: latest + + - name: ci + run: make ci + + - name: setup cluster + run: ./.github/workflows/scripts/setup-cluster.sh + env: + CLUSTER_NAME: webhook + K3S_VERSION: v1.28.9-k3s1 + ARCH: "${{ matrix.archBox.arch }}" + + - name: import image + run: k3d image import dist/rancher-webhook-image.tar -c webhook + + - name: start rancher + run: ./.github/workflows/scripts/start-rancher.sh + env: + CHART_PATH: "${{ runner.temp }}/rancher.tgz" + RANCHER_IMAGE_TAG: "v2.9-head" + VERSION: "2.9" + + - name: get vars + run: cat dist/image_tag >> $GITHUB_ENV + + - name: Run integration tests + run: ./.github/workflows/scripts/integration-test-ci + env: + ARCH: "${{ matrix.archBox.arch }}" + CLUSTER_NAME: webhook + IMAGE_REPO: rancher/webhook + IMAGE_TAG: "${{ env.IMAGE_TAG }}" diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 000000000..d8cce88d0 --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,55 @@ +name: Publish Webhook Images + +on: + push: + tags: + - v* + workflow_dispatch: + +env: + REGISTRY: docker.io + REPO: rancher + +permissions: + contents: read + +jobs: + push: + permissions: + contents: read + id-token: write + name: Build and push Webhook images + runs-on: ubuntu-latest + steps: + - name: "Read vault secrets" + uses: rancher-eio/read-vault-secrets@main + with: + secrets: | + secret/data/github/repo/${{ github.repository }}/dockerhub/rancher/credentials username | DOCKER_USERNAME ; + secret/data/github/repo/${{ github.repository }}/dockerhub/rancher/credentials password | DOCKER_PASSWORD + - name : Checkout repository + # https://github.com/actions/checkout/releases/tag/v4.1.1 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Set up Docker Buildx + # https://github.com/docker/setup-buildx-action/commit/d70bba72b1f3fd22344832f00baa16ece964efeb + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb + - name: Log in to the Container registry + # https://github.com/docker/login-action/commit/0d4c9c5ea7693da7b068278f7b52bda2a190a446 + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 + with: + registry: ${{ env.REGISTRY }} + username: ${{ env.DOCKER_USERNAME }} + password: ${{ env.DOCKER_PASSWORD }} + # setup tag name + - if: ${{ startsWith(github.ref, 'refs/tags/') }} + run: | + echo TAG_NAME=$(echo $GITHUB_REF | sed -e "s|refs/tags/||") >> $GITHUB_ENV + - name: Build and push the webhook image + # https://github.com/docker/build-push-action/commit/ca052bb54ab0790a636c9b5f226502c73d547a25 + uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 + with: + context: . + file: ./package/Dockerfile + push: true + tags: ${{ env.REGISTRY }}/${{ env.REPO }}/rancher-webhook:${{ env.TAG_NAME }} + platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 000000000..10d19104e --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,95 @@ +name: release + +on: + push: + tags: + - v* + workflow_dispatch: + +permissions: + contents: write + +jobs: + build: + name: build and package + runs-on: ubuntu-latest + strategy: + matrix: + arch: + - amd64 + - arm64 + steps: + + - name : Checkout repository + # https://github.com/actions/checkout/releases/tag/v4.1.1 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Setup Go + # https://github.com/actions/setup-go/releases/tag/v5.0.0 + uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + with: + go-version-file: 'go.mod' + + - name: Build and package + run: | + ./scripts/build + mkdir -p dist/artifacts + cp bin/webhook dist/artifacts/webhook-linux-${{ matrix.arch }} + env: + ARCH: "${{ matrix.arch}}" + GOARCH: "${{ matrix.arch}}" + + - name: Generate checksum files + run: | + ls -lR dist + cd dist/artifacts + sha256sum webhook-linux-${{ matrix.arch }} > sha256sum-${{ matrix.arch }}.txt + + - name: Upload artifacts + # https://github.com/actions/upload-artifact/commit/65462800fd760344b1a7b4382951275a0abb4808 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 + with: + name: webhook-artifacts-${{ matrix.arch }} + path: | + dist/artifacts/webhook-linux-${{ matrix.arch }} + dist/artifacts/sha256sum-${{ matrix.arch }}.txt + dist/artifacts/rancher-webhook-*.tgz + + release: + needs: build + runs-on: ubuntu-latest + steps: + + - name : Checkout repository + # https://github.com/actions/checkout/releases/tag/v4.1.1 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: package-helm + run: ./scripts/package-helm + + - name: Download the amd64 artifacts + # https://github.com/actions/download-artifact/commit/65a9edc5881444af0b9093a5e628f2fe47ea3b2e + uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e + with: + name: webhook-artifacts-amd64 + path: dist/artifacts + + - name: Download the arm64 artifacts + # https://github.com/actions/download-artifact/commit/65a9edc5881444af0b9093a5e628f2fe47ea3b2e + uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e + with: + name: webhook-artifacts-arm64 + path: dist/artifacts + + - name: Get the version + run: | + source ./scripts/version + echo "TAG=$(echo $TAG | sed 's/-amd64$//')" >> $GITHUB_ENV + + - name: Upload the files + run: | + ls -lR dist + cd dist/artifacts + gh --repo "${{ github.repository }}" release create ${{ github.ref_name }} --prerelease --verify-tag --generate-notes webhook-linux-* sha256sum-*.txt rancher-webhook*.tgz + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/scripts/install-k3d.sh b/.github/workflows/scripts/install-k3d.sh new file mode 100755 index 000000000..6993f3312 --- /dev/null +++ b/.github/workflows/scripts/install-k3d.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -eu + +REPO_URL=https://github.com/rancher/k3d +K3D_URL=https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh + +install_k3d(){ + if [ -z "${K3D_VERSION:-}" -o "${K3D_VERSION:-}" = "latest" ] ; then + K3D_VERSION=$(curl -Ls -o /dev/null -w %{url_effective} "${REPO_URL}/releases/latest" | grep -oE "[^/]+$") + fi + echo -e "Downloading k3d@${K3D_VERSION} from ${K3D_URL}" + curl --silent --fail ${K3D_URL} | TAG=${K3D_VERSION} bash +} + +install_k3d + +k3d version diff --git a/.github/workflows/scripts/integration-test-ci b/.github/workflows/scripts/integration-test-ci new file mode 100755 index 000000000..5451b3f24 --- /dev/null +++ b/.github/workflows/scripts/integration-test-ci @@ -0,0 +1,82 @@ +#!/bin/bash +set -eu + +cd $(dirname $0)/../../.. +DIST_DIR="$PWD"/dist + +source ./scripts/version +# Source tags file to get the last built tags +source ./dist/tags + +set -o pipefail + +source ./.github/workflows/scripts/try.sh + +set +e + +# Wait for rancher to start up +try --waitmsg "Waiting for rancher to start" --failmsg "No rancher here" kubectl rollout status --watch=true --timeout=10s -n cattle-system deploy/rancher +echo "Rancher deployed" + +# Wait for the system to spawn a rancher-webhook deployment +webhook_deployment_created() { + kubectl get deployments -n cattle-system | grep rancher-webhook +} +try --max 48 --delay 5 --waitmsg "Waiting for a rancher-webhook deployment to be created" --failmsg "Deployment creation failed" webhook_deployment_created + +try --waitmsg "Waiting for rancher/webhook to be deployed" --failmsg "No rancher/webhook here" kubectl rollout status --watch=true --timeout=10s -n cattle-system deploy/rancher-webhook +echo "Rancher deployed" + +webhook_deployed() { + status=$(kubectl get apps.catalog.cattle.io -n cattle-system rancher-webhook -o jsonpath="{@.status.summary.state}") && [[ "$status" == "deployed" ]] +} + +# Wait for Rancher to deploy rancher-webhook. +try --waitmsg "Waiting for webhook to be deployed (2)" webhook_deployed +echo "Webhook deployed" + +# Shut down the core rancher part, but leave the rest of the rancher environment running +set -e +echo "Shutting down core rancher" +kubectl scale deploy rancher -n cattle-system --replicas=0 +kubectl wait pods -l app=rancher --for=delete -n cattle-system + +echo "Uploading new webhook image" + +# Install the webhook chart we just built. +upgrade_rancher_webhook() { + helm upgrade rancher-webhook ./dist/artifacts/rancher-webhook-${HELM_CHART_VERSION}.tgz -n cattle-system \ + --wait --timeout=120s --set image.repository="${IMAGE_REPO}" --set image.tag="${IMAGE_TAG}" --reuse-values --debug +} + +set +e + +try --max 3 --delay 2 --waitmsg "Upgrading Webhook" --failmsg "Failed to upgrade webhook" upgrade_rancher_webhook + +x=$(kubectl get pods -n cattle-system -l app=rancher-webhook --no-headers --field-selector='status.phase!=RUNNING' --output custom-columns=NODE:.metadata.name | head -n 1) +if [ -n "$x" ] ; then + echo "Logs for failed rancher-webhook $x ": + kubectl get pods -n cattle-system -l app=rancher-webhook + kubectl get pod "$x" -n cattle-system + kubectl logs pod/"$x" -n cattle-system + kubectl describe pod/"$x" -n cattle-system + try --max 4 --failmsg "Couldn't helm upgrade rancher-webhook" upgrade_rancher_webhook +fi + +# Done trying things, so reinstate 'set -e' +set -e + +./bin/rancher-webhook-integration.test -test.v -test.run IntegrationTest + +# Install the webhook chart with new ports. +helm upgrade rancher-webhook ./dist/artifacts/rancher-webhook-${HELM_CHART_VERSION}.tgz -n cattle-system \ + --wait --reuse-values --set port=443 + +# Test that the ports are set as expected and run a single integration test to verify the webhook is still accessible. +./bin/rancher-webhook-integration.test -test.v -test.run PortTest +./bin/rancher-webhook-integration.test -test.v -test.run IntegrationTest -testify.m TestGlobalRole + +# Scale down rancher-webhook so that we can run tests on the FailurePolicy. +kubectl scale deploy rancher-webhook -n cattle-system --replicas=0 +kubectl wait pods -l app=rancher-webhook --for=delete -n cattle-system +./bin/rancher-webhook-integration.test -test.v -test.run FailurePolicyTest diff --git a/.github/workflows/scripts/setup-cluster.sh b/.github/workflows/scripts/setup-cluster.sh new file mode 100755 index 000000000..2930e775e --- /dev/null +++ b/.github/workflows/scripts/setup-cluster.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +set -e + +source ./scripts/version +set -x + +echo $TAG + +if [ -z "$CLUSTER_NAME" ]; then + echo "CLUSTER_NAME must be specified when setting up a cluster" + exit 1 +fi + +if [ -z "$K3S_VERSION" ]; then + echo "K3S_VERSION must be specified when setting up a cluster, use $(k3d version list k3s) to find valid versions" + exit 1 +fi + +# waits until all nodes are ready +wait_for_nodes(){ + timeout=120 + start_time=$(date +%s) + echo "wait until all agents are ready" + while : + do + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + if [ $elapsed_time -ge $timeout ]; then + echo "Timeout reached, exiting..." + exit 1 + fi + + readyNodes=1 + statusList=$(kubectl get nodes --no-headers | awk '{ print $2}') + # shellcheck disable=SC2162 + while read status + do + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + if [ $elapsed_time -ge $timeout ]; then + echo "Timeout reached, exiting..." + exit 1 + fi + if [ "$status" == "NotReady" ] || [ "$status" == "" ] + then + readyNodes=0 + break + fi + done <<< "$(echo -e "$statusList")" + # all nodes are ready; exit + if [[ $readyNodes == 1 ]] + then + break + fi + sleep 1 + done +} + +k3d registry create gha -p 42765 +k3d cluster create $CLUSTER_NAME --servers 1 --agents 1 \ + --registry-use gha:42765 \ + --image "rancher/k3s:${K3S_VERSION}" --api-port 6550 + +wait_for_nodes + +echo "k3d cluster $CLUSTER_NAME is ready" + +kubectl cluster-info --context k3d-${CLUSTER_NAME} +kubectl config use-context k3d-${CLUSTER_NAME} +kubectl get nodes -o wide +kubectl get pods -A diff --git a/.github/workflows/scripts/start-rancher.sh b/.github/workflows/scripts/start-rancher.sh new file mode 100755 index 000000000..8f41b7014 --- /dev/null +++ b/.github/workflows/scripts/start-rancher.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +flip() { + echo $1 + exit 1 +} + +kubectl get ns | grep -s cattle && flip 'rancher already installed?' + +set -exu + +set +e +helm repo add cert-manager https://charts.jetstack.io +helm repo add rancher-latest https://releases.rancher.com/server-charts/latest +helm repo add jetstack https://charts.jetstack.io +set -e + +helm repo update + +helm upgrade --install cert-manager --namespace cert-manager cert-manager/cert-manager --set installCRDs=true --create-namespace --wait --timeout=10m + +# kubectl get pods --namespace cert-manager +kubectl rollout status --namespace cert-manager deploy/cert-manager --timeout 1m + +# Chart based + +helm upgrade --install rancher "$CHART_PATH" --namespace cattle-system --set replicas=1 --set hostname=localhost --wait --timeout=10m --create-namespace --version "$VERSION" --set rancherImage=rancher/rancher --set rancherImageTag="$RANCHER_IMAGE_TAG" diff --git a/.github/workflows/scripts/try.sh b/.github/workflows/scripts/try.sh new file mode 100755 index 000000000..1abc265e0 --- /dev/null +++ b/.github/workflows/scripts/try.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# Usage: try [--max [30] --delay [2] --waitmsg 'retrying' --failmsg 'retrying' ] command... + +try() { + local max=30 + local delay=2 + local waitmsg="retrying" + local failmsg="" + while [[ $# -gt 0 ]] && [[ $1 == -* ]]; do + case "$1" in + --max) + max=$2 + shift + ;; + --delay) + delay=$2 + shift + ;; + --waitmsg) + waitmsg=$2 + shift + ;; + --failmsg) + failmsg=$2 + shift + ;; + --) + shift + break + ;; + *) + printf "Usage error: unknown flag '%s'" "$1" >&2 + return 1 + ;; + esac + shift + done + + local count=0 + while true; do + $* + status=$? + count=$(expr $count + 1) + if [[ $status -eq 0 ]]; then + break + elif [[ $count -ge $max ]]; then + if [ -n "$failmsg" ] ; then + echo $failmsg + else + echo "Failed to run <$*>" + fi + exit 1 + break + fi + echo "$waitmsg on try $count/$max" + sleep $delay + done +} diff --git a/.gitignore b/.gitignore index ddefc16be..9ee9cc60e 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ *.swp .idea .vscode -/webhook \ No newline at end of file +/webhook +*~ diff --git a/Dockerfile.dapper b/Dockerfile.dapper index 62303aef6..9f5306abf 100644 --- a/Dockerfile.dapper +++ b/Dockerfile.dapper @@ -17,7 +17,7 @@ RUN if [ "${ARCH}" = "amd64" ]; then \ RUN GOBIN=/usr/local/bin go install github.com/golang/mock/mockgen@v1.6.0 -ENV DAPPER_ENV REPO TAG DRONE_TAG CROSS +ENV DAPPER_ENV REPO TAG CROSS ENV DAPPER_SOURCE /go/src/github.com/rancher/webhook/ ENV DAPPER_OUTPUT ./bin ./dist ENV DAPPER_DOCKER_SOCKET true diff --git a/scripts/build b/scripts/build index 0e2b0cd27..fd6df0ce2 100755 --- a/scripts/build +++ b/scripts/build @@ -14,5 +14,5 @@ LINKFLAGS="-X main.GitCommit=$COMMIT $LINKFLAGS" CGO_ENABLED=0 go build -ldflags "$LINKFLAGS $OTHER_LINKFLAGS" -o bin/webhook if [ "$CROSS" = "true" ] && [ "$ARCH" = "amd64" ]; then GOOS=darwin go build -ldflags "$LINKFLAGS" -o bin/webhook-darwin - GOOS=windows go build -ldflags "$LINKFLAGS" -o bin/webhook-windows + GOOS=windows go build -ldflags "$LINKFLAGS" -o bin/webhook-windows-amd64.exe fi diff --git a/scripts/ci b/scripts/ci index 51fe8e966..fad2e9e83 100755 --- a/scripts/ci +++ b/scripts/ci @@ -8,4 +8,5 @@ cd $(dirname $0) ./validate ./validate-ci ./package +./package-helm ./test-helm diff --git a/scripts/integration-test b/scripts/integration-test index 1fd719375..3c73322cc 100755 --- a/scripts/integration-test +++ b/scripts/integration-test @@ -1,83 +1,44 @@ #!/bin/bash -set -e -export KUBECONFIG= -export CATTLE_DEV_MODE=yes -export CATTLE_SERVER_URL="https://$(ip route get 8.8.8.8 | awk '{print $7}'):443" -export CATTLE_BOOTSTRAP_PASSWORD="admin" -export CATTLE_FEATURES="harvester=false" +set -exu cd $(dirname $0)/../ -echo "Starting Rancher Server" -entrypoint.sh >./rancher.log 2>&1 & -RANCHER_PID=$! +source ./scripts/try.sh -echo "Waiting for Rancher health check..." -while ! curl -sf http://localhost:80/healthz >/dev/null 2>&1; do - echo "Waiting for Rancher's /healthz endpoint to become available" - sleep 2 -done +# Wait for rancher to start up +try --delay 2 --max 30 --waitmsg "Waiting for rancher to start" --failmsg "No rancher here" kubectl rollout status --watch=true --timeout=10s -n cattle-system deploy/rancher +echo "Rancher deployed" -# Tail the rancher logs if rancher fails to deploy the webhook after 5 minutes. -bash -c "sleep 300 && echo 'Rancher has not deployed webhook after 5m tailing logs' && tail -f ./rancher.log" & -# Get PID of the tail command so we can kill it if needed -TAIL_PID=$! +# Wait for the rancher webhook to start up +try --delay 2 --max 30 --waitmsg "Waiting for rancher/webhook to be deployed" --failmsg "No webhook here" kubectl rollout status --watch=true --timeout=10s -n cattle-system deploy/rancher-webhook +echo "Webhook deployed" + +webhook_deployed() { + status=$(kubectl get apps.catalog.cattle.io -n cattle-system rancher-webhook -o jsonpath="{@.status.summary.state}") && [[ "$status" == "deployed" ]] +} # Wait for Rancher to deploy rancher-webhook. -while ! kubectl rollout status -w -n cattle-system deploy/rancher-webhook >/dev/null 2>&1; do - echo "Waiting for rancher to deploy rancher-webhook..." - sleep 2 -done +try --delay 2 --max 30 --waitmsg "Waiting for webhook to be deployed (2)" webhook_deployed echo "Webhook deployed" -# After rancher deploys webhook kill the bash command running tail. -kill ${TAIL_PID} - -# Wait for helm operation to complete and save rancher-webhook release info before we kill rancher and the cluster. -while - status=$(kubectl get apps.catalog.cattle.io -n cattle-system rancher-webhook -o jsonpath="{@.status.summary.state}") - [[ "$status" != "deployed" ]] -do - echo "Waiting for helm operation to finish, current status $status" - sleep 2 -done - -# Kill Rancher since we only need the CRDs and the initial webhook values. -# We do not want Rancher to reconcile an older version of the webhook during test. -kill ${RANCHER_PID} - -echo "Rancher has been stopped starting K3s." -# Start Cluster without Rancher. -k3s server --cluster-init --disable=traefik,servicelb,metrics-server,local-storage --node-name=local-node --log=./k3s.log >/dev/null 2>&1 & -KUBECONFIG=/etc/rancher/k3s/k3s.yaml - -# Wait for cluster to start. -while ! kubectl version >/dev/null 2>&1; do - echo "Waiting for cluster to start" - sleep 5 -done +# Shut down the core rancher part, but leave the rest of the rancher environment running -echo "Uploading new webhook image" +kubectl scale deploy rancher -n cattle-system --replicas=0 --timeout=10m +kubectl wait pods -l app=rancher-webhook --for=delete --namespace cattle-system --timeout=10m -###### Upload the newly created webhook image to containerd, then install the webhook chart using the new image -IMAGE_FILE=./dist/rancher-webhook-image.tar -# import image to containerd and get the image name -WEBHOOK_REPO=$(ctr image import ${IMAGE_FILE} | cut -d ' ' -f 2 | cut -d ':' -f 1) +echo "Rancher has been stopped." + +echo "Uploading new webhook image" # Source tags file to get the last built tags source ./dist/tags # Install the webhook chart we just built. -# This command can fail since it is so close to the cluster start so we will give it 3 retires. -RETRIES=0 -while ! helm upgrade rancher-webhook ./dist/artifacts/rancher-webhook-${HELM_VERSION}.tgz -n cattle-system \ - --wait --set image.repository=${WEBHOOK_REPO} --set image.tag=${TAG} --reuse-values; do - if [ "$RETRIES" -ge 3 ]; then - exit 1 - fi - RETRIES=$((RETRIES + 1)) - sleep 2 -done +upgrade_rancher_webhook() { + helm upgrade rancher-webhook ./dist/artifacts/rancher-webhook-${HELM_VERSION}.tgz -n cattle-system \ + --wait --set image.repository="${IMAGE_REPO}" --set image.tag="${IMAGE_TAG}" --reuse-values --debug +} +try --delay 2 --max 4 --failmsg "Couldn't helm upgrade rancher-webhook" upgrade_rancher_webhook ./bin/rancher-webhook-integration.test -test.v -test.run IntegrationTest @@ -92,4 +53,5 @@ helm upgrade rancher-webhook ./dist/artifacts/rancher-webhook-${HELM_VERSION}.tg # Scale down rancher-webhook so that we can run tests on the FailurePolicy. kubectl scale deploy rancher-webhook -n cattle-system --replicas=0 kubectl wait pods -l app=rancher-webhook --for=delete -n cattle-system + ./bin/rancher-webhook-integration.test -test.v -test.run FailurePolicyTest diff --git a/scripts/package b/scripts/package index 4f2ca314c..6ba0f840c 100755 --- a/scripts/package +++ b/scripts/package @@ -1,40 +1,37 @@ #!/bin/bash -set -e +set -eu source $(dirname $0)/version cd $(dirname $0)/.. -function build-image() { - IMAGE=${REPO}/${1}:${TAG} - DOCKERFILE=package/Dockerfile${2} - if [ -e ${DOCKERFILE}.${ARCH} ]; then - DOCKERFILE=${DOCKERFILE}.${ARCH} - fi - - docker build -f ${DOCKERFILE} -t ${IMAGE} . - echo Built ${IMAGE} - - docker save -o dist/rancher-webhook-image.tar ${IMAGE} - - if [ "${PUSH}" = "true" ]; then - docker push ${IMAGE} - fi - -} +echo Running package mkdir -p dist/artifacts cp bin/webhook dist/artifacts/webhook-linux${SUFFIX} for i in bin/webhook-*; do if [ -e "$i" ]; then - if [ "$i" = webhook-windows-amd64 ]; then - cp $i dist/artifacts/webhook-windows-amd64.exe - else - cp $i dist/artifacts - fi + cp $i dist/artifacts fi done +REPO=rancher + +IMAGE=${REPO}/webhook:${TAG} +DOCKERFILE=./package/Dockerfile +if [ -e ${DOCKERFILE}.${ARCH} ]; then + DOCKERFILE=${DOCKERFILE}.${ARCH} +fi + +if [[ ${USE_DOCKER_BUILDX:-0} -eq 1 ]]; then + docker buildx build --platform linux/amd64 -f ${DOCKERFILE} . -t ${IMAGE} +else + docker build -f ${DOCKERFILE} -t ${IMAGE} . +fi +echo Built ${IMAGE} -build-image rancher-webhook +docker save -o dist/rancher-webhook-image.tar ${IMAGE} +echo IMAGE_TAG="${TAG}" > dist/image_tag -./scripts/package-helm +if [ "${PUSH:-}" = "true" ]; then + docker push ${IMAGE} +fi diff --git a/scripts/package-helm b/scripts/package-helm index 45d5c4e2f..3f3a94b90 100755 --- a/scripts/package-helm +++ b/scripts/package-helm @@ -1,27 +1,25 @@ #!/bin/bash -set -e - -if ! hash helm 2>/dev/null; then - exit 0 -fi +set -eu cd $(dirname $0)/.. . ./scripts/version +echo Running package-helm + rm -rf build/charts mkdir -p build dist/artifacts cp -rf charts build/ # must use sed -i''` for GNU and OSX compatibility sed -i'.bkp' \ - -e 's/^version:.*/version: '${HELM_VERSION}'/' \ - -e 's/appVersion:.*/appVersion: '${HELM_VERSION}'/' \ + -e 's/^version:.*/version: '${HELM_CHART_VERSION}'/' \ + -e 's/appVersion:.*/appVersion: '${HELM_CHART_VERSION}'/' \ build/charts/rancher-webhook/Chart.yaml sed -i'.bkb' \ -e 's/tag: latest/tag: '${HELM_TAG}'/' \ build/charts/rancher-webhook/values.yaml -rm build/charts/rancher-webhook/Chart.yaml.bkp build/charts/rancher-webhook/values.yaml.bkb +rm -f build/charts/rancher-webhook/Chart.yaml.bkp build/charts/rancher-webhook/values.yaml.bkb helm package -d ./dist/artifacts ./build/charts/rancher-webhook diff --git a/scripts/test-helm b/scripts/test-helm index a1e95d1e6..d5d61b696 100755 --- a/scripts/test-helm +++ b/scripts/test-helm @@ -2,7 +2,6 @@ set -e cd $(dirname $0)/.. -./scripts/package-helm echo Running helm lint helm lint ./charts/rancher-webhook # Check for unittest plugin diff --git a/scripts/version b/scripts/version index 1c84b8da7..3d25739f4 100755 --- a/scripts/version +++ b/scripts/version @@ -1,11 +1,12 @@ #!/bin/bash +DIRTY= if [ -n "$(git status --porcelain --untracked-files=no)" ]; then DIRTY="-dirty" fi -COMMIT=$(git rev-parse --short HEAD) -GIT_TAG=${DRONE_TAG:-$(git tag -l --contains HEAD | head -n 1)} +COMMIT=${COMMIT:-$(git rev-parse --short HEAD)} +GIT_TAG=${GIT_TAG:-$(git tag -l --contains HEAD | head -n 1)} if [[ -z "$DIRTY" && -n "$GIT_TAG" ]]; then VERSION=$GIT_TAG @@ -13,23 +14,23 @@ else VERSION="0.0.0-${COMMIT}${DIRTY}" fi -if [ -z "$ARCH" ]; then +if [ -z "${ARCH:-}" ] ; then ARCH=$(go env GOHOSTARCH) fi SUFFIX="-${ARCH}" -HELM_TAG=${TAG:-${VERSION}} -HELM_VERSION=${HELM_TAG/v/} -TAG=${TAG:-${VERSION}${SUFFIX}} -REPO=${REPO:-rancher} +HELM_TAG="${HELM_TAG:-${TAG:-${VERSION}}}" +HELM_CHART_VERSION="${HELM_CHART_VERSION:-${HELM_TAG/v/}}" +TAG="${TAG:-${VERSION}${SUFFIX}}" +REPO="${REPO:-rancher}" if echo $TAG | grep -q dirty; then TAG=dev HELM_TAG=dev - HELM_VERSION=0.0.0-dev + HELM_CHART_VERSION=0.0.0-dev fi -DIST_DIR="$(dirname $0)/../dist/" +DIST_DIR="${DIST_DIR:-$(dirname $0)/../dist/}" mkdir -p ${DIST_DIR} -echo "export TAG=${TAG}; export HELM_TAG=${HELM_TAG}; export HELM_VERSION=${HELM_VERSION};" >${DIST_DIR}/tags +echo "export TAG=${TAG}; export HELM_TAG=${HELM_TAG}; export HELM_CHART_VERSION=${HELM_CHART_VERSION};" >${DIST_DIR}/tags diff --git a/tests/integration/rkeMachineConfig_test.go b/tests/integration/rkeMachineConfig_test.go index 7c8ae72c1..47a6a4921 100644 --- a/tests/integration/rkeMachineConfig_test.go +++ b/tests/integration/rkeMachineConfig_test.go @@ -1,12 +1,20 @@ package integration_test import ( + "os" + "runtime" + "github.com/rancher/webhook/pkg/auth" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" ) func (m *IntegrationSuite) TestRKEMachineConfig() { + if runtime.GOARCH == "arm64" && os.Getenv("CI") != "" { + // Temporarily workaround https://github.com/rancher/rancher/issues/45837 : + // Not all CRDs are built in GHA/arm64 + m.T().Skip("Skipping the RKE Machine-Config test on arm64 in CI -- machine info not available") + } objGVK := schema.GroupVersionKind{ Group: "rke-machine-config.cattle.io", Version: "v1", From 198ac611f1cfdb41c9a2fbb1c5b868164665a6ad Mon Sep 17 00:00:00 2001 From: Eric Promislow Date: Tue, 2 Jul 2024 11:37:49 -0700 Subject: [PATCH 2/4] Bring in Tom Lebreux's changes on image publishing. --- .github/workflows/publish.yaml | 55 --------------- .github/workflows/release.yaml | 125 +++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 55 deletions(-) delete mode 100644 .github/workflows/publish.yaml diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml deleted file mode 100644 index d8cce88d0..000000000 --- a/.github/workflows/publish.yaml +++ /dev/null @@ -1,55 +0,0 @@ -name: Publish Webhook Images - -on: - push: - tags: - - v* - workflow_dispatch: - -env: - REGISTRY: docker.io - REPO: rancher - -permissions: - contents: read - -jobs: - push: - permissions: - contents: read - id-token: write - name: Build and push Webhook images - runs-on: ubuntu-latest - steps: - - name: "Read vault secrets" - uses: rancher-eio/read-vault-secrets@main - with: - secrets: | - secret/data/github/repo/${{ github.repository }}/dockerhub/rancher/credentials username | DOCKER_USERNAME ; - secret/data/github/repo/${{ github.repository }}/dockerhub/rancher/credentials password | DOCKER_PASSWORD - - name : Checkout repository - # https://github.com/actions/checkout/releases/tag/v4.1.1 - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Set up Docker Buildx - # https://github.com/docker/setup-buildx-action/commit/d70bba72b1f3fd22344832f00baa16ece964efeb - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb - - name: Log in to the Container registry - # https://github.com/docker/login-action/commit/0d4c9c5ea7693da7b068278f7b52bda2a190a446 - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 - with: - registry: ${{ env.REGISTRY }} - username: ${{ env.DOCKER_USERNAME }} - password: ${{ env.DOCKER_PASSWORD }} - # setup tag name - - if: ${{ startsWith(github.ref, 'refs/tags/') }} - run: | - echo TAG_NAME=$(echo $GITHUB_REF | sed -e "s|refs/tags/||") >> $GITHUB_ENV - - name: Build and push the webhook image - # https://github.com/docker/build-push-action/commit/ca052bb54ab0790a636c9b5f226502c73d547a25 - uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 - with: - context: . - file: ./package/Dockerfile - push: true - tags: ${{ env.REGISTRY }}/${{ env.REPO }}/rancher-webhook:${{ env.TAG_NAME }} - platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 10d19104e..ea290d522 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,6 +9,10 @@ on: permissions: contents: write +env: + REGISTRY: docker.io + REPO: rancher + jobs: build: name: build and package @@ -93,3 +97,124 @@ jobs: gh --repo "${{ github.repository }}" release create ${{ github.ref_name }} --prerelease --verify-tag --generate-notes webhook-linux-* sha256sum-*.txt rancher-webhook*.tgz env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + image: + permissions: + contents: read + id-token: write + strategy: + matrix: + arch: + - amd64 + - arm64 + name: Build and push Webhook images + runs-on: ubuntu-latest + needs: build + steps: + - name : Checkout repository + # https://github.com/actions/checkout/releases/tag/v4.1.1 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Download the artifacts + # https://github.com/actions/download-artifact/commit/65a9edc5881444af0b9093a5e628f2fe47ea3b2e + uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e + with: + name: webhook-artifacts-${{ matrix.arch }} + path: dist/artifacts + + - name: Move binary to bin/ + run: | + mkdir -p bin/ + cp -v dist/artifacts/webhook-linux-${{ matrix.arch }} bin/webhook + chmod +x bin/webhook + + - name: "Read vault secrets" + uses: rancher-eio/read-vault-secrets@main + with: + secrets: | + secret/data/github/repo/${{ github.repository }}/dockerhub/rancher/credentials username | DOCKER_USERNAME ; + secret/data/github/repo/${{ github.repository }}/dockerhub/rancher/credentials password | DOCKER_PASSWORD + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + # https://github.com/docker/setup-buildx-action/commit/d70bba72b1f3fd22344832f00baa16ece964efeb + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb + + - name: Log in to the Container registry + # https://github.com/docker/login-action/commit/0d4c9c5ea7693da7b068278f7b52bda2a190a446 + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 + with: + registry: ${{ env.REGISTRY }} + username: ${{ env.DOCKER_USERNAME }} + password: ${{ env.DOCKER_PASSWORD }} + + # setup tag name + - if: ${{ startsWith(github.ref, 'refs/tags/') }} + run: | + echo TAG_NAME=$(echo $GITHUB_REF | sed -e "s|refs/tags/||") >> $GITHUB_ENV + - name: Build and push the webhook image + id: build + # https://github.com/docker/build-push-action/commit/ca052bb54ab0790a636c9b5f226502c73d547a25 + uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 + with: + context: . + file: ./package/Dockerfile + platforms: "linux/${{ matrix.arch }}" + outputs: type=image,name=${{ env.REPO }}/rancher-webhook,push-by-digest=true,name-canonical=true,push=true + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.arch }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + runs-on: ubuntu-latest + needs: image + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: "${{ env.REPO }}/rancher-webhook" + + - name: "Read vault secrets" + uses: rancher-eio/read-vault-secrets@main + with: + secrets: | + secret/data/github/repo/${{ github.repository }}/dockerhub/rancher/credentials username | DOCKER_USERNAME ; + secret/data/github/repo/${{ github.repository }}/dockerhub/rancher/credentials password | DOCKER_PASSWORD + + - name: Log in to the Container registry + # https://github.com/docker/login-action/commit/0d4c9c5ea7693da7b068278f7b52bda2a190a446 + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 + with: + registry: ${{ env.REGISTRY }} + username: ${{ env.DOCKER_USERNAME }} + password: ${{ env.DOCKER_PASSWORD }} + + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.REPO }}/rancher-webhook@sha256:%s ' *) From 91e7f1854ec8f8150e68bd8aa8f50192f7764b2b Mon Sep 17 00:00:00 2001 From: Eric Promislow Date: Tue, 2 Jul 2024 11:46:07 -0700 Subject: [PATCH 3/4] Add id-token permission for vault --- .github/workflows/release.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ea290d522..f1ff8579f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -179,6 +179,8 @@ jobs: retention-days: 1 merge: + permissions: + id-token: write runs-on: ubuntu-latest needs: image steps: From 9ff5cceabda84f811391e4c006488b7884f090dc Mon Sep 17 00:00:00 2001 From: Eric Promislow Date: Tue, 2 Jul 2024 13:34:41 -0700 Subject: [PATCH 4/4] Pull in Tom's changes to use tag instead of docker metadata. --- .github/workflows/release.yaml | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f1ff8579f..842537537 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -150,10 +150,6 @@ jobs: username: ${{ env.DOCKER_USERNAME }} password: ${{ env.DOCKER_PASSWORD }} - # setup tag name - - if: ${{ startsWith(github.ref, 'refs/tags/') }} - run: | - echo TAG_NAME=$(echo $GITHUB_REF | sed -e "s|refs/tags/||") >> $GITHUB_ENV - name: Build and push the webhook image id: build # https://github.com/docker/build-push-action/commit/ca052bb54ab0790a636c9b5f226502c73d547a25 @@ -194,12 +190,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: "${{ env.REPO }}/rancher-webhook" - - name: "Read vault secrets" uses: rancher-eio/read-vault-secrets@main with: @@ -215,8 +205,13 @@ jobs: username: ${{ env.DOCKER_USERNAME }} password: ${{ env.DOCKER_PASSWORD }} + # setup tag name + - if: ${{ startsWith(github.ref, 'refs/tags/') }} + run: | + echo TAG_NAME=$(echo $GITHUB_REF | sed -e "s|refs/tags/||") >> $GITHUB_ENV + - name: Create manifest list and push working-directory: /tmp/digests run: | - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + docker buildx imagetools create -t ${{ env.REGISTRY }}/${{ env.REPO }}/rancher-webhook:${{ env.TAG_NAME }} \ $(printf '${{ env.REPO }}/rancher-webhook@sha256:%s ' *)