conformance.yml

name: Conformance Testing

on:
  workflow_call:
    inputs:
      image:
        required: true
        type: string
      k8s-version:
        required: true
        type: string
      enable-experimental:
        required: true
        type: boolean

defaults:
  run:
    shell: bash

env:
  PLUS_USAGE_ENDPOINT: ${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}
  ENABLE_EXPERIMENTAL: ${{ inputs.enable-experimental }}

permissions:
  contents: read

jobs:
  conformance-tests:
    name: Run Tests
    runs-on: ubuntu-24.04
    if: ${{ !github.event.pull_request.head.repo.fork || inputs.image != 'plus' }}
    permissions:
      contents: write # needed for uploading release artifacts
    env:
      DOCKER_BUILD_SUMMARY: false
    steps:
      - name: Checkout Repository
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          fetch-depth: 0

      - name: Setup Golang Environment
        uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
        with:
          go-version: stable

      - name: Set GOPATH
        run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV

      - name: Docker Buildx
        uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3.8.0

      - name: NGF Docker meta
        id: ngf-meta
        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
        with:
          images: |
            name=ghcr.io/nginxinc/nginx-gateway-fabric
          tags: |
            type=semver,pattern={{version}}
            type=edge
            type=schedule
            type=ref,event=pr
            type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}

      - name: NGINX Docker meta
        id: nginx-meta
        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
        with:
          images: |
            name=ghcr.io/nginxinc/nginx-gateway-fabric/${{ inputs.image == 'plus' && 'nginx-plus' || inputs.image }}
          tags: |
            type=semver,pattern={{version}}
            type=edge
            type=schedule
            type=ref,event=pr
            type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}

      - name: Generate static deployment
        run: |
          ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
          ngf_tag=${{ steps.ngf-meta.outputs.version }}
          make generate-static-deployment PLUS_ENABLED=${{ inputs.image == 'plus' && 'true' || 'false' }} PREFIX=${ngf_prefix} TAG=${ngf_tag}
        working-directory: ./tests

      - name: Build binary
        uses: goreleaser/goreleaser-action@9ed2f89a662bf1735a48bc8557fd212fa902bebf # v6.1.0
        with:
          version: v2.5.1 # renovate: datasource=github-tags depName=goreleaser/goreleaser
          args: build --single-target --snapshot --clean
        env:
          TELEMETRY_ENDPOINT: "" # disables sending telemetry
          TELEMETRY_ENDPOINT_INSECURE: "false"

      - name: Build NGF Docker Image
        uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 # v6.10.0
        with:
          file: build/Dockerfile
          tags: ${{ steps.ngf-meta.outputs.tags }}
          context: "."
          target: goreleaser
          load: true
          cache-from: type=gha,scope=ngf
          pull: true

      - name: Build NGINX Docker Image
        uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 # v6.10.0
        with:
          file: build/Dockerfile${{ inputs.image == 'nginx' && '.nginx' || '' }}${{ inputs.image == 'plus' && '.nginxplus' || ''}}
          tags: ${{ steps.nginx-meta.outputs.tags }}
          context: "."
          load: true
          cache-from: type=gha,scope=${{ inputs.image }}
          pull: true
          build-args: |
            NJS_DIR=internal/mode/static/nginx/modules/src
            NGINX_CONF_DIR=internal/mode/static/nginx/conf
            BUILD_AGENT=gha

      - name: Update Go Modules
        if: ${{ github.event_name == 'schedule' }}
        run: make update-go-modules
        working-directory: ./tests

      - name: Build Test Docker Image
        uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 # v6.10.0
        with:
          file: tests/conformance/Dockerfile
          tags: conformance-test-runner:${{ github.sha }}
          context: "tests"
          load: true
          cache-from: type=gha
          cache-to: type=gha,mode=max
          pull: true

      - name: Deploy Kubernetes
        id: k8s
        run: |
          kind create cluster --name ${{ github.run_id }} --image=kindest/node:${{ inputs.k8s-version }}
          kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }}

      - name: Setup license file for plus
        if: ${{ inputs.image == 'plus' }}
        env:
          PLUS_LICENSE: ${{ secrets.JWT_PLUS_EXCEPTION_REPORTING }}
        run: echo "${PLUS_LICENSE}" > license.jwt

      - name: Setup conformance tests
        run: |
          ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
          ngf_tag=${{ steps.ngf-meta.outputs.version }}
          if [ ${{ github.event_name }} == "schedule" ]; then export GW_API_VERSION=main; fi
          make helm-install-local${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag}
          make deploy-updated-provisioner PREFIX=${ngf_prefix} TAG=${ngf_tag}
        working-directory: ./tests

      - name: Run conformance tests
        run: |
          make run-conformance-tests CONFORMANCE_TAG=${{ github.sha }} NGF_VERSION=${{ github.ref_name }} CLUSTER_NAME=${{ github.run_id }}
          core_result=$(cat conformance-profile.yaml | yq '.profiles[0].core.result')
          extended_result=$(cat conformance-profile.yaml | yq '.profiles[0].extended.result')
          if [ "${core_result}" == "failure" ] || [ "${extended_result}" == "failure" ]; then echo "Conformance test failed, see above for details." && exit 2; fi
        working-directory: ./tests

      - name: Upload profile to GitHub
        if: ${{ inputs.enable-experimental }}
        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
        with:
          name: conformance-profile-${{ inputs.image }}-${{ inputs.k8s-version }}
          path: ./tests/conformance-profile.yaml

      - name: Upload profile to release
        if: ${{ startsWith(github.ref, 'refs/tags/') && inputs.enable-experimental }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: gh release upload ${{ github.ref_name }} conformance-profile.yaml --clobber
        working-directory: ./tests