run-tests.yml

name: Run Tests

# We run full CI on push builds to main and on all pull requests
#
# Manual builds (workflow_dispatch) to the main branch are also published
#
# To maximize bug-catching changes while keeping CI times reasonable, we run:
# - All tests on Linux/Java17
# - Fewer tests on Linux/Java11 and Windows/Java17
# - Fewest tests on Windows/Java11

on:
  push:
  pull_request:
  workflow_dispatch:

# cancel older runs of a pull request;
# this will not cancel anything for normal git pushes
concurrency:
  group: cancel-old-pr-runs-${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  # Jobs are listed in rough order of priority: if multiple jobs fail, the first job
  # in the list should be the one that's most worth looking into
  build-linux:
    uses: ./.github/workflows/pre-build.yml
    with:
      os: ubuntu-latest

  build-windows:
    uses: ./.github/workflows/pre-build.yml
    with:
      os: windows-latest

  test-docs:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with: { fetch-depth: 0 }

      - run: ./mill -i docs.githubPages + docs.checkBrokenLinks

  linux:
    needs: build-linux
    strategy:
      fail-fast: false
      matrix:

        include:
          # For most tests, run them arbitrarily on Java 11 or Java 17 on Linux, and
          # on the opposite version on Windows below, so we get decent coverage of
          # each test on each Java version and each operating system
          # We also try to group tests together to manually balance out the runtimes of each jobs
          - java-version: 17
            millargs: "'{main,scalalib,testrunner,bsp,testkit}.__.test'"
            install-android-sdk: false

          - java-version: 11
            millargs: "'{scalajslib,scalanativelib,kotlinlib,pythonlib,javascriptlib}.__.test'"
            install-android-sdk: false

          - java-version: 17
            millargs: "contrib.__.test"
            install-android-sdk: false

          - java-version: 17
            millargs: "'example.javalib.__.local.server.test'"
            install-android-sdk: false

          - java-version: 17
            millargs: "'example.scalalib.__.local.server.test'"
            install-android-sdk: false

          - java-version: 17
            millargs: "'example.kotlinlib.__.local.server.test'"
            install-android-sdk: false

          - java-version: 17
            millargs: "'example.android.__.local.server.test'"
            install-android-sdk: true

          - java-version: 17
            millargs: "'example.{pythonlib,javascriptlib}.__.local.server.test'"
            install-android-sdk: false

          - java-version: 11
            millargs: "'example.thirdparty[{mockito,acyclic,commons-io}].local.server.test'"
            install-android-sdk: false

          - java-version: 17
            millargs: "'example.thirdparty[{fansi,jimfs,netty,gatling}].local.server.test'"
            install-android-sdk: false

          - java-version: '17'
            millargs: "'example.thirdparty[arrow].local.server.test'"
            install-android-sdk: false

          - java-version: 11
            millargs: "'example.{cli,fundamentals,depth,extending}.__.local.server.test'"
            install-android-sdk: false
            # Most of these integration tests should not depend on which mode they
            # are run in, so just run them in `local`
          - java-version: '17'
            millargs: "'integration.{failure,feature,ide}.__.local.server.test'"
            install-android-sdk: false
            # These invalidation tests need to be exercised in both execution modes
            # to make sure they work with and without -i/--no-server being passed
          - java-version: 17
            millargs: "'integration.invalidation.__.packaged.fork.test'"
            install-android-sdk: false

          - java-version: 17
            millargs: "'integration.invalidation.__.packaged.server.test'"
            install-android-sdk: false

          # Run some smoketests with Graal native image launcher
          - java-version: 17
            millargs: "'example.javalib.__.native.server.test'"
            install-android-sdk: false

          - java-version: '17'
            millargs: "'integration.{failure,feature,ide}.__.native.server.test'"
            install-android-sdk: false

    uses: ./.github/workflows/post-build-selective.yml
    with:
      install-android-sdk: ${{ matrix.install-android-sdk }}
      java-version: ${{ matrix.java-version }}
      millargs: ${{ matrix.millargs }}

  windows:
    needs: build-windows
    strategy:
      fail-fast: false
      matrix:
        include:
          # just run a subset of examples/ on Windows, because for some reason running
          # the whole suite can take hours on windows v.s. half an hour on linux
          - java-version: 11
            millargs: '"{main,scalalib,bsp}.__.test"'
          - java-version: 11
            millargs: '"example.scalalib.basic.__.packaged.fork.test"'
          - java-version: 17
            millargs: "'integration.{feature,failure}.__.packaged.fork.test'"
          - java-version: 11
            millargs: "'integration.invalidation.__.packaged.server.test'"
          - java-version: 11
            millargs: "'integration.invalidation.__.native.server.test'"
          - java-version: 11
            millargs: "contrib.__.test"

    uses: ./.github/workflows/post-build-selective.yml
    with:
      os: windows-latest
      java-version: ${{ matrix.java-version }}
      millargs: ${{ matrix.millargs }}

  itest:
    needs: build-linux
    strategy:
      fail-fast: false
      matrix:
        include:
          # bootstrap tests
          - java-version: 11 # Have one job on oldest JVM
            buildcmd: ci/test-mill-dev.sh && ci/test-mill-release.sh && ./mill -i -k __.ivyDepsTree && ./mill -i -k __.ivyDepsTree --withRuntime
          - java-version: 17 # Have one job on default JVM
            buildcmd: ci/test-mill-bootstrap.sh

    uses: ./.github/workflows/post-build-raw.yml
    with:
      java-version: ${{ matrix.java-version }}
      buildcmd: ${{ matrix.buildcmd }}

  # Scalafmt, Mima, and Scalafix job runs last because it's the least important:
  # usually just an automated or mechanical manual fix to do before merging
  lint-autofix:
    needs: build-linux
    uses: ./.github/workflows/post-build-raw.yml
    with:
      java-version: '17'
      buildcmd: ./mill -i mill.scalalib.scalafmt.ScalafmtModule/checkFormatAll __.sources + __.mimaReportBinaryIssues + __.fix --check + mill.javalib.palantirformat.PalantirFormatModule/ --check + mill.kotlinlib.ktlint.KtlintModule/checkFormatAll