CI Main #1355
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Main | |
| on: | |
| merge_group: | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| schedule: # Todo: switch to push on master and remove schedule once we have more macos runners | |
| - cron: "0 * * * *" | |
| pull_request: | |
| push: | |
| branches: | |
| - 'hotfix-*-rc--*' | |
| - 'rc--*' | |
| - 'dev-gh-*' | |
| # runs for the same workflow are cancelled on PRs but not on master | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} | |
| cancel-in-progress: true | |
| permissions: read-all | |
| env: | |
| CI_COMMIT_SHA: ${{ github.sha }} | |
| CI_COMMIT_REF_PROTECTED: ${{ github.ref_protected }} | |
| CI_JOB_NAME: ${{ github.job }} | |
| CI_JOB_ID: ${{ github.job }} # github does not expose this variable https://github.com/orgs/community/discussions/8945 | |
| CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| CI_PIPELINE_SOURCE: ${{ github.event_name }} | |
| CI_PROJECT_DIR: ${{ github.workspace }} | |
| CI_MERGE_REQUEST_TARGET_BRANCH_NAME: ${{ github.event.pull_request.base.ref }} | |
| BRANCH_NAME: ${{ github.head_ref || github.ref_name }} | |
| ROOT_PIPELINE_ID: ${{ github.run_id }} | |
| BAZEL_STARTUP_ARGS: "--output_base=/var/tmp/bazel-output/" | |
| RUSTFLAGS: "--remap-path-prefix=${CI_PROJECT_DIR}=/ic" | |
| AWS_SHARED_CREDENTIALS_CONTENT: ${{ secrets.AWS_SHARED_CREDENTIALS_FILE }} | |
| DOCKER_HUB_USER: ${{ secrets.DOCKER_HUB_USER }} | |
| DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }} | |
| CI_MERGE_REQUEST_TITLE: ${{ github.event.pull_request.title }} | |
| BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }} | |
| BUILDEVENT_DATASET: "github-ci-dfinity" | |
| jobs: | |
| bazel-test-all: | |
| name: Bazel Test All | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:26cc347efa50935342742acddfb5d710fae1982d401911013ad8750f0603c590 | |
| options: >- | |
| -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info | |
| timeout-minutes: 90 | |
| runs-on: | |
| group: zh1 | |
| labels: dind-large | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }} | |
| - name: Before script | |
| id: before-script | |
| shell: bash | |
| run: ./gitlab-ci/src/ci-scripts/before-script.sh | |
| - name: Set BAZEL_EXTRA_ARGS_RULES | |
| shell: bash | |
| run: | | |
| set -xeuo pipefail | |
| if [[ "${{ github.event_name }}" == 'merge_group' ]]; then | |
| echo "BAZEL_EXTRA_ARGS_RULES=--test_timeout_filters=short,moderate --flaky_test_attempts=3" >> $GITHUB_ENV | |
| fi | |
| if [[ $BRANCH_NAME =~ hotfix-.*-rc--.* ]]; then | |
| echo "BAZEL_EXTRA_ARGS_RULES=--test_timeout_filters=short,moderate" >> $GITHUB_ENV | |
| fi | |
| - name: Run Bazel Test All | |
| id: bazel-test-all | |
| uses: ./.github/actions/bazel-test-all/ | |
| with: | |
| BAZEL_COMMAND: "test" | |
| BAZEL_TARGETS: "//... --deleted_packages=gitlab-ci/src/gitlab_config" | |
| BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel" | |
| # check if PR title contains release and set timeout filters accordingly | |
| BAZEL_EXTRA_ARGS_RULES: ${{ env.BAZEL_EXTRA_ARGS_RULES || '' }} | |
| # run on diff only if it is a pull request, otherwise run all targets | |
| RUN_ON_DIFF_ONLY: ${{ contains(github.event_name, 'pull_request') && 'true' || 'false'}} | |
| HONEYCOMB_API_TOKEN: ${{ secrets.HONEYCOMB_API_TOKEN }} | |
| - name: Upload bazel-targets | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: bazel-targets | |
| retention-days: 1 | |
| if-no-files-found: error | |
| path: | | |
| bazel-targets | |
| bazel-build-all-config-check: | |
| runs-on: | |
| labels: dind-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:26cc347efa50935342742acddfb5d710fae1982d401911013ad8750f0603c590 | |
| options: >- | |
| -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info | |
| timeout-minutes: 90 | |
| name: Bazel Build All Config Check | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }} | |
| - name: Before script | |
| id: before-script | |
| shell: bash | |
| run: ./gitlab-ci/src/ci-scripts/before-script.sh | |
| - name: Run bazel build --config=check //rs/... | |
| id: bazel-build-config-check | |
| uses: ./.github/actions/bazel-test-all/ | |
| with: | |
| BAZEL_COMMAND: "build" | |
| BAZEL_TARGETS: "//rs/..." | |
| BAZEL_CI_CONFIG: "--config=check --config=ci --keep_going" | |
| # run on diff only if it is a pull request, otherwise run all targets | |
| #RUN_ON_DIFF_ONLY: ${{ contains(github.event_name, 'pull_request') && 'true' || 'false'}} | |
| # TODO: disabling until the following issue is resolved | |
| # https://github.com/dfinity/ic/actions/runs/9699415138/job/26768332602 | |
| RUN_ON_DIFF_ONLY: false | |
| bazel-test-darwin-x86-64: | |
| name: Bazel Test Darwin x86-64 | |
| timeout-minutes: 120 | |
| runs-on: | |
| labels: macOS | |
| if: ${{ github.event_name != 'merge_group' }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }} | |
| - name: Filter Relevant Files | |
| uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 | |
| id: filter | |
| if: ${{ github.event_name == 'pull_request' }} | |
| with: | |
| filters: | | |
| bazel-test-darwin-x86-64: | |
| - '.github/workflows/ci-main.yml' | |
| - '.bazelrc' | |
| - '.bazelversion' | |
| - '**/*.bazel' | |
| - '**/*.bzl' | |
| - '**/*.lock' | |
| - '**/*.rs' | |
| - '**/*.toml' | |
| - name: Set PATH | |
| run: | | |
| echo "/usr/local/bin" >> $GITHUB_PATH | |
| echo "$HOME/.cargo/bin:" >> $GITHUB_PATH | |
| - name: Before script | |
| id: before-script | |
| shell: bash | |
| run: ./gitlab-ci/src/ci-scripts/before-script.sh | |
| - name: Run Bazel Test Darwin x86-64 | |
| id: bazel-test-darwin-x86-64 | |
| # TODO: remove when we flakiness is resolved or when we move to hosted runners | |
| # settting to true to allow a job to pass when this step fails | |
| continue-on-error: true | |
| if: steps.filter.outputs.bazel-test-darwin-x86-64 == 'true' || github.event_name == 'schedule' | |
| uses: ./.github/actions/bazel-test-all/ | |
| with: | |
| BAZEL_CI_CONFIG: "--config=ci --config macos_ci" | |
| BAZEL_COMMAND: test | |
| BAZEL_EXTRA_ARGS: ${{ github.event_name == 'schedule' && '--test_tag_filters=test_macos,test_macos_master' || '--test_tag_filters=test_macos' }} | |
| BAZEL_STARTUP_ARGS: "--output_base /var/tmp/bazel-output//${ROOT_PIPELINE_ID}" | |
| BAZEL_TARGETS: "//rs/... //publish/binaries/..." | |
| HONEYCOMB_API_TOKEN: ${{ secrets.HONEYCOMB_API_TOKEN }} | |
| - name: Purge Bazel Output | |
| if: always() | |
| shell: bash | |
| run: | | |
| sudo rm -rf /private/var/tmp/bazel-output | |
| bazel-build-fuzzers: | |
| name: Bazel Build Fuzzers | |
| runs-on: | |
| labels: dind-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:26cc347efa50935342742acddfb5d710fae1982d401911013ad8750f0603c590 | |
| options: >- | |
| -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info | |
| timeout-minutes: 90 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }} | |
| - name: Before script | |
| id: before-script | |
| shell: bash | |
| run: ./gitlab-ci/src/ci-scripts/before-script.sh | |
| - name: Run Bazel Build Fuzzers | |
| id: bazel-build-fuzzers | |
| uses: ./.github/actions/bazel-test-all/ | |
| with: | |
| BAZEL_COMMAND: "build" | |
| BAZEL_TARGETS: "//rs/..." | |
| BAZEL_EXTRA_ARGS: "--keep_going --config=fuzzing --build_tag_filters=libfuzzer" | |
| bazel-build-fuzzers-afl: | |
| name: Bazel Build Fuzzers AFL | |
| runs-on: | |
| labels: dind-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:26cc347efa50935342742acddfb5d710fae1982d401911013ad8750f0603c590 | |
| options: >- | |
| -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info | |
| timeout-minutes: 90 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }} | |
| - name: Before script | |
| id: before-script | |
| shell: bash | |
| run: ./gitlab-ci/src/ci-scripts/before-script.sh | |
| - name: Run Bazel Build Fuzzers AFL | |
| id: bazel-build-fuzzers-afl | |
| uses: ./.github/actions/bazel-test-all/ | |
| with: | |
| BAZEL_COMMAND: "build" | |
| BAZEL_TARGETS: "//rs/..." | |
| BAZEL_EXTRA_ARGS: "--keep_going --config=afl" | |
| python-ci-tests: | |
| name: Python CI Tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Run Python CI Tests | |
| id: python-ci-tests | |
| shell: bash | |
| run: | | |
| set -xeuo pipefail | |
| export PYTHONPATH=$PWD/gitlab-ci/src:$PWD/gitlab-ci/src/dependencies | |
| # Todo: remove once dependency scanner tests don't reference name of repo | |
| REPO_NAME="${GITHUB_REPOSITORY##*/}" | |
| ADDITIONAL_FLAGS="" | |
| if [[ $REPO_NAME == "ic-private" ]]; then | |
| ADDITIONAL_FLAGS="--ignore=dependencies/scanner/" | |
| fi | |
| pip3 install --ignore-installed -r requirements.txt | |
| cd gitlab-ci/src | |
| pytest --ignore=gitlab_config/ --ignore=git_changes/ $ADDITIONAL_FLAGS -v -o junit_family=xunit1 \ | |
| --junitxml=../../test_report.xml --cov=. --cov-report=term \ | |
| --cov-report=term-missing --cov-report=html --cov-branch | |
| build-ic: | |
| name: Build IC | |
| runs-on: | |
| labels: dind-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:26cc347efa50935342742acddfb5d710fae1982d401911013ad8750f0603c590 | |
| options: >- | |
| -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info | |
| timeout-minutes: 90 | |
| if: ${{ github.event_name != 'merge_group' }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }} | |
| - name: Before script | |
| id: before-script | |
| shell: bash | |
| run: ./gitlab-ci/src/ci-scripts/before-script.sh | |
| - name: Run Build IC | |
| id: build-ic | |
| shell: bash | |
| run: | | |
| set -eExuo pipefail | |
| REPO_NAME="${GITHUB_REPOSITORY##*/}" | |
| rm -rf "/cache/job/${CI_JOB_ID}/${ROOT_PIPELINE_ID}" | |
| mkdir -p "/cache/job/${CI_JOB_ID}/${ROOT_PIPELINE_ID}/artifacts" | |
| ln -s "/cache/job/${CI_JOB_ID}/${ROOT_PIPELINE_ID}/artifacts" /__w/$REPO_NAME/$REPO_NAME/artifacts | |
| buildevents cmd "$ROOT_PIPELINE_ID" "$CI_JOB_ID" build-command -- \ | |
| "$CI_PROJECT_DIR"/gitlab-ci/src/ci-scripts/build-ic.sh | |
| rm -rf "/cache/job/${CI_JOB_ID}/${ROOT_PIPELINE_ID}" | |
| env: | |
| RUN_ON_DIFF_ONLY: "true" | |
| BAZEL_COMMAND: "build" | |
| - name: Upload build-ic.tar | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: build-ic | |
| retention-days: 1 | |
| if-no-files-found: error | |
| path: | | |
| build-ic.tar | |
| build-determinism: | |
| name: Build Determinism | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| needs: [build-ic, bazel-test-all] | |
| strategy: | |
| matrix: | |
| include: | |
| - TARGET: "//publish/binaries:upload" | |
| PATH0: "release" | |
| PATH1: "build-ic/release" | |
| SETUPOS_FLAG: "false" | |
| - TARGET: "//publish/canisters:upload" | |
| PATH0: "canisters" | |
| PATH1: "build-ic/canisters" | |
| SETUPOS_FLAG: "false" | |
| - TARGET: "//ic-os/guestos/envs/prod:upload_disk-img" | |
| PATH0: "guest-os/update-img" | |
| PATH1: "build-ic/icos/guestos" | |
| SETUPOS_FLAG: "false" | |
| - TARGET: "//ic-os/hostos/envs/prod:upload_update-img" | |
| PATH0: "host-os/update-img" | |
| PATH1: "build-ic/icos/hostos" | |
| SETUPOS_FLAG: "false" | |
| - TARGET: "//ic-os/setupos/envs/prod:upload_disk-img" | |
| PATH0: "setup-os/disk-img" | |
| PATH1: "build-ic/icos/setupos" | |
| SETUPOS_FLAG: "true" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }} | |
| - name: Download bazel-targets [bazel-test-all] | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: bazel-targets | |
| - name: Download build-ic.tar [build-ic] | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: build-ic | |
| - name: Build Determinism Test | |
| id: build-determinism | |
| shell: bash | |
| run: | | |
| set -eExuo pipefail | |
| sudo apt update && sudo apt install -y curl | |
| "$CI_PROJECT_DIR"/gitlab-ci/src/ci-scripts/build-determinism.sh | |
| env: | |
| TARGET: ${{ matrix.TARGET }} | |
| PATH0: ${{ matrix.PATH0 }} | |
| PATH1: ${{ matrix.PATH1 }} | |
| SETUPOS_FLAG: ${{ matrix.SETUPOS_FLAG }} | |
| cargo-clippy-linux: | |
| name: Cargo Clippy Linux | |
| runs-on: | |
| labels: dind-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:26cc347efa50935342742acddfb5d710fae1982d401911013ad8750f0603c590 | |
| options: >- | |
| -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info | |
| timeout-minutes: 90 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }} | |
| - name: Filter Rust Files [*.{rs,toml,lock}] | |
| uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 | |
| id: filter | |
| if: | | |
| github.event_name == 'pull_request' || | |
| github.event_name == 'merge_group' | |
| with: | |
| filters: | | |
| cargo: | |
| - "**/*.rs" | |
| - "**/*.toml" | |
| - "**/*.lock" | |
| - name: Run Cargo Clippy Linux | |
| id: cargo-clippy-linux | |
| if: | | |
| steps.filter.outputs.cargo == 'true' || | |
| github.event_name == 'schedule' || | |
| github.event_name == 'workflow_dispatch' | |
| shell: bash | |
| run: | | |
| set -eExuo pipefail | |
| buildevents cmd "$ROOT_PIPELINE_ID" "$CI_JOB_ID" build-command -- \ | |
| "$CI_PROJECT_DIR"/gitlab-ci/src/ci-scripts/rust-lint.sh | |
| cargo-build-release-linux: | |
| name: Cargo Build Release Linux | |
| runs-on: | |
| labels: dind-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:26cc347efa50935342742acddfb5d710fae1982d401911013ad8750f0603c590 | |
| options: >- | |
| -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info | |
| timeout-minutes: 90 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }} | |
| - name: Filter Rust Files [*.{rs,toml,lock}] | |
| uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 | |
| id: filter | |
| if: | | |
| github.event_name == 'pull_request' || | |
| github.event_name == 'merge_group' | |
| with: | |
| filters: | | |
| cargo: | |
| - "**/*.rs" | |
| - "**/*.toml" | |
| - "**/*.lock" | |
| - name: Run Cargo Build Release Linux | |
| id: cargo-build-release-linux | |
| if: | | |
| steps.filter.outputs.cargo == 'true' || | |
| github.event_name == 'schedule' || | |
| github.event_name == 'workflow_dispatch' | |
| shell: bash | |
| run: | | |
| set -eExuo pipefail | |
| buildevents cmd "$ROOT_PIPELINE_ID" "$CI_JOB_ID" build-command -- \ | |
| cargo build --release |