system-tests-k8s.yml

name: System Tests K8s

# Run specific system test:
# gh workflow run system-tests-k8s.yml \
#   --ref marko-system-tests-k8s \
#   --field targets="//rs/tests/nns:node_removal_from_registry_test" \
#   --field jobs="32"

on:
  schedule:
    - cron: "0 1 * * *" # Run at 1 AM
    - cron: "0 3 * * *" # Run at 3 AM
    - cron: "0 5 * * *" # Run at 5 AM
    - cron: "0 7 * * *" # Run at 7 AM
  pull_request:
    paths:
      - '.github/workflows/system-tests-k8s.yml'
  workflow_dispatch:
    inputs:
      targets:
        description: 'System Tests / Bazel Targets'
        required: false
        default: '//rs/tests/nns:node_removal_from_registry_test'
      jobs:
        description: 'Concurrent Bazel Test Jobs'
        required: false
        default: '10'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name != 'schedule' }}

env:
  TARGETS: |
    ${{ github.event_name == 'schedule' && '//...' ||
    github.event_name == 'workflow_dispatch' && github.event.inputs.targets ||
    '//rs/tests/nns:node_removal_from_registry_test' }}
  JOBS: |
    ${{ github.event_name == 'schedule' && '10' ||
    github.event_name == 'workflow_dispatch' && github.event.inputs.jobs ||
    '10' }}
  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
  CI_COMMIT_SHA: ${{ github.sha }}
  CI_JOB_NAME: ${{ github.job }}
  CI_PIPELINE_SOURCE: ${{ github.event_name }}
  CI_PROJECT_DIR: ${{ github.workspace }}
  CI_RUN_ID: ${{ github.run_id }}
  BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }}
  BUILDEVENT_DATASET: "github-ci-dfinity"

jobs:
  system-tests-k8s:
    name: System Tests K8s
    runs-on:
      group: ln1
      labels: dind-large
    container:
      image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d
      options: >-
        -e NODE_NAME -e KUBECONFIG
        --privileged --cgroupns host
        -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp
    timeout-minutes: 120
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Before script
        id: before-script
        shell: bash
        run: |
          [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME"

      - name: Set KUBECONFIG
        shell: bash
        run: |
          echo "$TNET_KUBECONFIG" > /tmp/kubeconfig
          echo "KUBECONFIG=/tmp/kubeconfig" >> $GITHUB_ENV
        env:
          TNET_KUBECONFIG: ${{ secrets.TNET_KUBECONFIG }}

      - name: Run System Tests on K8s
        id: bazel-test-all
        uses: ./.github/actions/bazel-test-all/
        env:
          AWS_SHARED_CREDENTIALS_CONTENT: ${{ secrets.AWS_SHARED_CREDENTIALS_FILE }}
        with:
          BAZEL_COMMAND: "test"
          BAZEL_TARGETS: "${{ env.TARGETS }}"
          BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel"
          BAZEL_EXTRA_ARGS: "--local_test_jobs=${{ env.JOBS }} --test_tag_filters=k8s,-manual,-colocated,-long_test,-system_test_hourly,-system_test_nightly --k8s"
          BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }}

      - name: Upload bazel-bep
        uses: actions/upload-artifact@v4
        if: success() || failure()
        with:
          name: ${{ github.job }}-bep
          retention-days: 14
          if-no-files-found: ignore
          compression-level: 9
          path: |
            bazel-bep.pb
            profile.json

  system-tests-k8s-hourly:
    name: System Tests K8s Hourly
    runs-on:
      group: ln1
      labels: dind-large
    container:
      image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d
      options: >-
        -e NODE_NAME -e KUBECONFIG
        --privileged --cgroupns host
        -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp
    timeout-minutes: 150
    # always run after 'system-tests-k8s' job only on manual dispatch and schedule
    if: ${{ always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }}
    needs: [system-tests-k8s]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set KUBECONFIG
        shell: bash
        run: |
          echo "$TNET_KUBECONFIG" > /tmp/kubeconfig
          echo "KUBECONFIG=/tmp/kubeconfig" >> $GITHUB_ENV
        env:
          TNET_KUBECONFIG: ${{ secrets.TNET_KUBECONFIG }}

      - name: Get Bazel Target List
        shell: bash
        run: |
          # Query Bazel for targets: system tests with k8s flag AND (long_test flag OR system_test_hourly flag)
          T=$(bazel query 'attr(tags, "k8s", tests(attr(tags, "long_test|system_test_hourly", //...))) except attr(tags, "colocated|manual|system_test_nightly", //...)')

          # Handle empty target list
          if [[ -z "$T" ]]; then
            echo "No Bazel targets found matching the criteria."
            echo "TARGETS=" >> $GITHUB_ENV
            exit 1
          fi

          # Convert to space-separated list and trim
          T=$(echo "$T" | tr '\n' ' ' | sed -e 's/,$//' -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')

          # Export to GitHub environment
          echo "TARGETS=$T" >> $GITHUB_ENV

      - name: Run System Tests on K8s
        id: bazel-test-all
        uses: ./.github/actions/bazel-test-all/
        env:
          AWS_SHARED_CREDENTIALS_CONTENT: ${{ secrets.AWS_SHARED_CREDENTIALS_FILE }}
        with:
          BAZEL_COMMAND: "test"
          BAZEL_TARGETS: "${{ env.TARGETS }}"
          BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel"
          BAZEL_EXTRA_ARGS: "--local_test_jobs=${{ env.JOBS }} --test_tag_filters=k8s --k8s --flaky_test_attempts=3"
          BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }}

      - name: Upload bazel-bep
        uses: actions/upload-artifact@v4
        if: success() || failure()
        with:
          name: ${{ github.job }}-bep
          retention-days: 14
          if-no-files-found: ignore
          compression-level: 9
          path: |
            bazel-bep.pb
            profile.json