From ec00dfd2e70c67b816994669c085cd520e88b3aa Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 16:50:32 +0100 Subject: [PATCH 001/191] remove some workflows temporarily --- .github/workflows/analysis.yml | 145 --------------------------------- .github/workflows/checks.yml | 105 ------------------------ .github/workflows/docs.yml | 61 -------------- 3 files changed, 311 deletions(-) delete mode 100644 .github/workflows/analysis.yml delete mode 100644 .github/workflows/checks.yml delete mode 100644 .github/workflows/docs.yml diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml deleted file mode 100644 index f33bb927f40..00000000000 --- a/.github/workflows/analysis.yml +++ /dev/null @@ -1,145 +0,0 @@ -name: Analysis - -on: - push: - branches: - - main - pull_request: - branches: - - main - - 'release/**' - - 'develop/**' - paths-ignore: - - "docs/**" - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -env: - CTEST_OUTPUT_ON_FAILURE: 1 - CCACHE_DIR: ${{ github.workspace }}/ccache - CCACHE_MAXSIZE: 1.25G - CCACHE_KEY_SUFFIX: r2 - ACTS_LOG_FAILURE_THRESHOLD: WARNING - DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.v5.tar.zst - -# NOTE this only builds core unittests to reduce the output size. if we -# found a way to have Github actions not fail regularly with this job -# all unit tests should be reactivated. -jobs: - build_debug: - runs-on: ubuntu-latest - container: ghcr.io/acts-project/ubuntu2404:63 - steps: - - - uses: actions/checkout@v4 - - - name: Install dependencies - run: CI/dependencies.sh - - - name: Cache build - uses: actions/cache@v4 - with: - path: ${{ env.CCACHE_DIR }} - key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} - restore-keys: | - ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - - - name: Configure - run: > - ccache -z && - cmake -B build -S . - --preset=github-ci - -DCMAKE_BUILD_TYPE=Debug - -DCMAKE_CXX_FLAGS="-Werror --coverage -g -gz -g1" - -DPython_EXECUTABLE=$(which python3) - -DACTS_BUILD_ODD=OFF - - name: Build - run: cmake --build build - - name: ccache stats - run: ccache -s - - name: Unit tests - run: ctest --test-dir build -j$(nproc) - - - name: Remove .o files - run: > - du -sh build - && find build -name *.o -delete - && du -sh build - - name: Coverage - run: > - pip3 install gcovr==7.2 - && cd build - && /usr/bin/python3 ../CI/test_coverage.py - - - uses: actions/upload-artifact@v4 - with: - name: coverage-build - path: build - - - name: Save PR number to file - if: github.event_name == 'pull_request' - run: echo ${{ github.event.number }} > PR_NUMBER.txt - - - name: Persist PR number - if: github.event_name == 'pull_request' - uses: actions/upload-artifact@v4 - with: - name: PR_NUMBER - path: PR_NUMBER.txt - - build_performance: - runs-on: ubuntu-latest - container: ghcr.io/acts-project/ubuntu2404:63 - if: github.ref == 'refs/heads/main' - steps: - - uses: actions/checkout@v4 - - - name: Install dependencies - run: CI/dependencies.sh - - - name: Install dependencies - run: pip3 install git+https://github.com/paulgessinger/cmakeperf.git@2a409b5 - - name: Configure - run: > - cmake -B build -S . - --preset=github-ci - -DCMAKE_BUILD_TYPE=RelWithDebInfo - -DCMAKE_CXX_FLAGS="-Werror" - -DPython_EXECUTABLE=$(which python3) - -DACTS_BUILD_ODD=OFF - - name: Measure - run: cmakeperf collect build/compile_commands.json -o perf.csv - - name: Results - run: cmakeperf print perf.csv - - uses: actions/upload-artifact@v4 - with: - name: cmakeperf - path: perf.csv - - # metric_tracking: - # runs-on: ubuntu-latest - # needs: build_performance - # if: github.ref == 'refs/heads/main' - # steps: - # - uses: actions/checkout@v4 - # - name: Install dependencies - # run: pip3 install git+https://github.com/paulgessinger/headwind.git@eeeaa80 - # - uses: actions/download-artifact@v4 - # with: - # name: cmakeperf - # - name: Run collection - # env: - # SSH_AUTH_SOCK: /tmp/ssh_agent.sock - # run: | - # ssh-agent -a $SSH_AUTH_SOCK > /dev/null - # ssh-add - <<< "${{ secrets.METRIC_DEPLOY_SSH_KEY }}" - # git config --global user.email "action@github.com" - # git config --global user.name "GitHub Action" - # git clone git@github.com:acts-project/metrics.git - # hdw collect CI/headwind.yml --commit $(git log --pretty=format:'%H' -1) - # cd metrics - # git add -A - # git commit -m"update metrics" - # git push diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml deleted file mode 100644 index 59ba76bb00a..00000000000 --- a/.github/workflows/checks.yml +++ /dev/null @@ -1,105 +0,0 @@ -name: Checks - -on: - push: - pull_request: - branches: - - main - - 'release/**' - - 'develop/**' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - lint: - runs-on: ubuntu-latest - env: - PRE_COMMIT_HOME: '/tmp/pre-commit' - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - uses: actions/cache@v4 - with: - path: | - ${{ env.PRE_COMMIT_HOME }} - key: ${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }} - - - name: Install pre-commit - run: pip install pre-commit - - - name: Run pre-commit - run: pre-commit run --all-files --show-diff-on-failure - - smearing_config: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Check - run: > - CI/check_smearing_config.py . - - missing_includes: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Install clang - run: > - sudo apt-get install -y clang libeigen3-dev libboost-dev - - name: Check - run: > - CI/missing_include_check.sh - - fpe_masks: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install dependencies - run: > - pip install -r CI/fpe_masks/requirements.txt - - name: Check - run: > - CI/check_fpe_masks.py --token ${{ secrets.GITHUB_TOKEN }} - - unused_files: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Check - run: > - CI/check_unused_files.py - - codegen: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install dependencies - run: > - pip install -r codegen/requirements.txt - - name: Check - run: > - CI/check_codegen - - uses: actions/upload-artifact@v4 - if: failure() - with: - name: changed - path: changed diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index dfa5b303ee2..00000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Docs - -on: - push: - pull_request: - branches: - - main - - 'release/**' - - 'develop/**' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - docs: - runs-on: ubuntu-latest - env: - DOXYGEN_WARN_AS_ERROR: FAIL_ON_WARNINGS - DOXYGEN_VERSION: 1.9.8 - steps: - - uses: actions/checkout@v4 - - - name: Cache doxygen - id: cache-doxygen - uses: actions/cache@v4 - with: - path: /usr/local/bin/doxygen - key: doxygen_${{ env.DOXYGEN_VERSION }} - - - name: Install doxygen - if: steps.cache-doxygen.outputs.cache-hit != 'true' - run: > - curl -SL https://sourceforge.net/projects/doxygen/files/rel-${{ env.DOXYGEN_VERSION }}/doxygen-${{ env.DOXYGEN_VERSION }}.linux.bin.tar.gz/download | tar -xzC . - && mv doxygen-${{ env.DOXYGEN_VERSION }}/bin/doxygen /usr/local/bin/doxygen - - - name: Install dependencies - run: > - pip3 install --upgrade pip - && pip install -r docs/requirements.txt - - - name: Build documentation - run: > - cd docs - && sphinx-build - -b html - -d _build/doctrees/ - -j auto - -W - --keep-going - -t run_doxygen - -t lazy_autodoc - -t white_papers - -b linkcheck - . _build/html/ - - - - uses: actions/upload-artifact@v4 - with: - name: acts-docs - path: docs/_build/html/ From 1a7fd8cb6d64bb7c5f0f0878a7f2ef39fdd7ce21 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 16:59:21 +0100 Subject: [PATCH 002/191] test --- .github/actions/dependencies/action.yml | 33 ++ .github/actions/dependencies/opengl.sh | 33 ++ .github/workflows/builds.yml | 750 ++++++++++++------------ 3 files changed, 442 insertions(+), 374 deletions(-) create mode 100644 .github/actions/dependencies/action.yml create mode 100755 .github/actions/dependencies/opengl.sh diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml new file mode 100644 index 00000000000..44e45828e51 --- /dev/null +++ b/.github/actions/dependencies/action.yml @@ -0,0 +1,33 @@ +name: "Dependencies action" +description: "Reusable action that sets up dependencies for the ACTS CI build" +# inputs: +# example_input: +# description: "An example input" +# required: true +# default: "Hello, World!" +# outputs: +# example_output: +# description: "An example output" +runs: + using: "composite" + steps: + - name: Set up Spack + uses: spack/setup-spack@v2 + + - name: Apply spack patch + shell: bash + working-directory: spack + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am + + - name: Locate OpenGL + shell: bash + run: ./opengl.sh + + # - name: Setup Xcode version + # if: startsWith(inputs.os, 'macos') + # uses: maxim-lobanov/setup-xcode@v1 + # with: + # xcode-version: "${{ inputs.xcode }}" diff --git a/.github/actions/dependencies/opengl.sh b/.github/actions/dependencies/opengl.sh new file mode 100755 index 00000000000..d3943c4d114 --- /dev/null +++ b/.github/actions/dependencies/opengl.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -u +set -e + +source detect_os.sh + +packages_file=${GITHUB_WORKSPACE}/spack/etc/spack/packages.yaml + +if [ "$os" == "ubuntu" ]; then + sudo apt-get update + sudo apt-get install -y libgl1-mesa-dev +cat < "$packages_file" +packages: + opengl: + buildable: false + externals: + - prefix: /usr/ + spec: opengl@4.5 +EOF +cat "$packages_file" +elif [ "$os" == "almalinux" ]; then + dnf install -y mesa-libGLU +cat < "$packages_file" +packages: + opengl: + buildable: false + externals: + - prefix: /usr/ + spec: opengl@4.6 +EOF +cat "$packages_file" +fi diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 2d565795d9f..fb70b81dd3e 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -36,377 +36,379 @@ jobs: submodules: true lfs: true - - name: Set dependencies URL - run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV - - - name: Install dependencies - run: CI/dependencies.sh - - - name: Restore ccache - uses: actions/cache/restore@v4 - id: ccache-restore - with: - path: ${{ env.CCACHE_DIR }} - key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} - restore-keys: | - ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - - - name: Configure - # setting CMAKE_CXX_STANDARD=20 is a workaround for a bug in the - # dd4hep CMake configuration that gets triggered on recent CMake - # versions - # Need to set git user & email for patching to work (GeoModel plugin) - run: > - git config --global user.name 'CI' && - git config --global user.email '<>' && - ccache -z && - cmake -B build -S . - --preset=github-ci - -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" - -DPython_EXECUTABLE=$(which python3) - -DACTS_BUILD_PLUGIN_ONNX=ON - - - name: Build - run: cmake --build build - - - name: ccache stats - run: ccache -s - - - name: Save ccache - uses: actions/cache/save@v4 - if: always() - with: - path: ${{ github.workspace }}/ccache - key: ${{ steps.ccache-restore.outputs.cache-primary-key }} - - - name: Unit tests - run: ctest --test-dir build -j$(nproc) - - - name: Integration tests - run: cmake --build build --target integrationtests - - - name: Install - run: cmake --build build --target install - - - name: Package build - run: tar czf build.tar.gz -C build --exclude "*.o" --exclude "bin/ActsUnitTest*" --exclude "bin/ActsIntegrationTest*" . - - - uses: actions/upload-artifact@v4 - with: - name: acts-linux-ubuntu - path: build.tar.gz - - - name: Downstream configure - run: > - cmake -B build-downstream -S Tests/DownstreamProject - -GNinja - -DCMAKE_BUILD_TYPE=Release - -DCMAKE_CXX_FLAGS=-Werror - -DCMAKE_CXX_STANDARD=20 - -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" - - - name: Downstream build - run: cmake --build build-downstream - - - name: Downstream run - run: ./build-downstream/bin/ShowActsVersion - - linux_examples_test: - runs-on: ubuntu-latest - container: ghcr.io/acts-project/ubuntu2404:63 - needs: [linux_ubuntu] - env: - ACTS_SEQUENCER_DISABLE_FPEMON: true - - steps: - - uses: actions/checkout@v4 - with: - submodules: true - lfs: true - - - name: Set dependencies URL - run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV - - - name: Install dependencies - run: CI/dependencies.sh - - - uses: actions/download-artifact@v4 - with: - name: acts-linux-ubuntu - - - name: Unpack build - run: mkdir build && tar xf build.tar.gz -C build - - - name: Python level tests - shell: bash - env: - PYTEST_MD_REPORT: true - PYTEST_MD_REPORT_VERBOSE: 0 - PYTEST_MD_REPORT_OUTPUT: pytest.md - run: > - geant4-config --install-datasets - && source build/this_acts_withdeps.sh - && python3 -m pip install -r Examples/Python/tests/requirements.txt - && python3 -m pip install pytest-md-report - && pytest -rFsv -k "not exatrkx" -v - && cat ${PYTEST_MD_REPORT_OUTPUT} >> $GITHUB_STEP_SUMMARY - - linux_physmon: - runs-on: ubuntu-latest - container: ghcr.io/acts-project/ubuntu2404:63 - needs: [linux_ubuntu] - env: - ACTS_SEQUENCER_DISABLE_FPEMON: true - - steps: - - uses: actions/checkout@v4 - with: - submodules: true - lfs: true - - - run: apt-get update && apt-get install -y time - - - name: Set dependencies URL - run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV - - - name: Install dependencies - run: CI/dependencies.sh - - - uses: actions/download-artifact@v4 - with: - name: acts-linux-ubuntu - - - name: Unpack build - run: mkdir build && tar xf build.tar.gz -C build - - - name: Save PR number - if: ${{ github.event_name == 'pull_request' }} - run: | - mkdir -p physmon - echo ${{ github.event.number }} > physmon/pr_number - echo ${{ github.event.pull_request.head.sha }} > physmon/sha - - - name: Physics performance checks - shell: bash - run: > - echo "::group::Dependencies" - && git config --global safe.directory "$GITHUB_WORKSPACE" - && python3 -m pip install histcmp==0.6.8 matplotlib - && python3 -m pip install -r Examples/Scripts/requirements.txt - && geant4-config --install-datasets - && venv_python=$(which python3) - && echo $venv_python - && source build/this_acts_withdeps.sh - && export PATH=$(dirname $venv_python):$PATH - && echo $PATH - && which python3 - && echo "::endgroup::" - && CI/physmon/phys_perf_mon.sh all physmon - - - name: Post step summary - if: always() - run: cat physmon/summary.md >> $GITHUB_STEP_SUMMARY - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: physmon - path: physmon - - linux_physmon_perf_report: - needs: [linux_physmon] - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' - steps: - - name: Install dependencies - run: pip3 install spyral-cli==1.1.2 - - - uses: actions/download-artifact@v4 - with: - name: physmon - path: physmon - - - name: Store metrics - env: - SSH_AUTH_SOCK: /tmp/ssh_agent.sock - run: | - ssh-agent -a $SSH_AUTH_SOCK > /dev/null - ssh-add - <<< "${{ secrets.RUNTIME_METRIC_DEPLOY_SSH_KEY }}" - git config --global user.email "action@github.com" - git config --global user.name "GitHub Action" - git clone git@github.com:acts-project/runtime_metrics.git - spyral maxima runtime_metrics/metrics.csv physmon/memory/*.csv -e $(date +%Y-%m-%dT%H-%M-%S) -e ${GITHUB_REF_NAME} -e ${GITHUB_REF} -e ${GITHUB_SHA} - cd runtime_metrics - git add -A - git commit -m"update metrics" - git push - - linux_ubuntu_extra: - runs-on: ubuntu-latest - strategy: - matrix: - include: - - image: ubuntu2204 - std: 20 - - image: ubuntu2204_clang - std: 20 - container: ghcr.io/acts-project/${{ matrix.image }}:63 - env: - INSTALL_DIR: ${{ github.workspace }}/install - ACTS_LOG_FAILURE_THRESHOLD: WARNING - - steps: - - uses: actions/checkout@v4 - with: - submodules: true - lfs: true - - - name: Set dependencies URL - run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV - - - name: Install dependencies - run: CI/dependencies.sh - - - name: Restore ccache - uses: actions/cache/restore@v4 - id: ccache-restore - with: - path: ${{ env.CCACHE_DIR }} - key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} - restore-keys: | - ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - - - - name: Configure - # setting CMAKE_CXX_STANDARD=20 is a workaround for a bug in the - # dd4hep CMake configuration that gets triggered on recent CMake - # versions - run: > - ccache -z && - cmake -B build -S . - --preset=github-ci - -DCMAKE_CXX_STANDARD=${{ matrix.std }} - -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" - -DPython_EXECUTABLE=$(which python3) - - - name: Build - run: cmake --build build - - - name: ccache stats - run: ccache -s - - - name: Save ccache - uses: actions/cache/save@v4 - if: always() - with: - path: ${{ github.workspace }}/ccache - key: ${{ steps.ccache-restore.outputs.cache-primary-key }} - - - name: Unit tests - run: cmake --build build --target test - - - name: Integration tests - run: cmake --build build --target integrationtests - - - name: Install - run: cmake --build build --target install - - - name: Downstream configure - run: > - cmake -B build-downstream -S Tests/DownstreamProject - -GNinja - -DCMAKE_BUILD_TYPE=Release - -DCMAKE_CXX_FLAGS=-Werror - -DCMAKE_CXX_STANDARD=${{ matrix.std }} - -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" - - - name: Downstream build - run: cmake --build build-downstream - - - name: Downstream run - run: ./build-downstream/bin/ShowActsVersion - - macos: - runs-on: macos-14 - env: - INSTALL_DIR: ${{ github.workspace }}/install_acts - ACTS_LOG_FAILURE_THRESHOLD: WARNING - - steps: - - uses: actions/checkout@v4 - with: - submodules: true - lfs: true - - - name: Print architecture - run: uname -p - - - name: Set dependencies URL - run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/macos-14/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV - - - name: Install dependencies - run: > - brew install cmake ninja ccache xerces-c - && CI/dependencies.sh - - - name: Restore ccache - uses: actions/cache/restore@v4 - id: ccache-restore - with: - path: ${{ env.CCACHE_DIR }} - key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} - restore-keys: | - ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - - - name: Configure - run: > - ccache -z - && cmake -B build -S . - --preset=github-ci - -DCMAKE_PREFIX_PATH="${{ env.DEPENDENCY_DIR }}" - -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" - -DPython_EXECUTABLE=$(which python3) - - - name: Build - run: cmake --build build - - - name: ccache stats - run: ccache -s - - - name: Save ccache - uses: actions/cache/save@v4 - if: always() - with: - path: ${{ github.workspace }}/ccache - key: ${{ steps.ccache-restore.outputs.cache-primary-key }} - - - name: Unit tests - run: cmake --build build --target test - - - name: Integration tests - run: cmake --build build --target integrationtests - - - name: Install - run: cmake --build build --target install - - - uses: actions/upload-artifact@v4 - with: - name: acts-macos - path: ${{ env.INSTALL_DIR }} - - - name: Downstream configure - run: > - cmake -B build-downstream -S Tests/DownstreamProject - -GNinja - -DCMAKE_BUILD_TYPE=Release - -DCMAKE_CXX_FLAGS=-Werror - -DCMAKE_CXX_STANDARD=20 - -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" - - - name: Downstream build - run: cmake --build build-downstream - - - name: Downstream run - run: > - PATH="${{ env.DEPENDENCY_DIR }}/bin:$PATH" - && ./build-downstream/bin/ShowActsVersion + - uses: ./.github/actions/dependencies + + # - name: Set dependencies URL + # run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV + # + # - name: Install dependencies + # run: CI/dependencies.sh + # + # - name: Restore ccache + # uses: actions/cache/restore@v4 + # id: ccache-restore + # with: + # path: ${{ env.CCACHE_DIR }} + # key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} + # restore-keys: | + # ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- + # + # - name: Configure + # # setting CMAKE_CXX_STANDARD=20 is a workaround for a bug in the + # # dd4hep CMake configuration that gets triggered on recent CMake + # # versions + # # Need to set git user & email for patching to work (GeoModel plugin) + # run: > + # git config --global user.name 'CI' && + # git config --global user.email '<>' && + # ccache -z && + # cmake -B build -S . + # --preset=github-ci + # -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + # -DPython_EXECUTABLE=$(which python3) + # -DACTS_BUILD_PLUGIN_ONNX=ON + # + # - name: Build + # run: cmake --build build + # + # - name: ccache stats + # run: ccache -s + # + # - name: Save ccache + # uses: actions/cache/save@v4 + # if: always() + # with: + # path: ${{ github.workspace }}/ccache + # key: ${{ steps.ccache-restore.outputs.cache-primary-key }} + # + # - name: Unit tests + # run: ctest --test-dir build -j$(nproc) + # + # - name: Integration tests + # run: cmake --build build --target integrationtests + # + # - name: Install + # run: cmake --build build --target install + # + # - name: Package build + # run: tar czf build.tar.gz -C build --exclude "*.o" --exclude "bin/ActsUnitTest*" --exclude "bin/ActsIntegrationTest*" . + # + # - uses: actions/upload-artifact@v4 + # with: + # name: acts-linux-ubuntu + # path: build.tar.gz + # + # - name: Downstream configure + # run: > + # cmake -B build-downstream -S Tests/DownstreamProject + # -GNinja + # -DCMAKE_BUILD_TYPE=Release + # -DCMAKE_CXX_FLAGS=-Werror + # -DCMAKE_CXX_STANDARD=20 + # -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" + # + # - name: Downstream build + # run: cmake --build build-downstream + # + # - name: Downstream run + # run: ./build-downstream/bin/ShowActsVersion + + # linux_examples_test: + # runs-on: ubuntu-latest + # container: ghcr.io/acts-project/ubuntu2404:63 + # needs: [linux_ubuntu] + # env: + # ACTS_SEQUENCER_DISABLE_FPEMON: true + # + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: true + # lfs: true + # + # - name: Set dependencies URL + # run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV + # + # - name: Install dependencies + # run: CI/dependencies.sh + # + # - uses: actions/download-artifact@v4 + # with: + # name: acts-linux-ubuntu + # + # - name: Unpack build + # run: mkdir build && tar xf build.tar.gz -C build + # + # - name: Python level tests + # shell: bash + # env: + # PYTEST_MD_REPORT: true + # PYTEST_MD_REPORT_VERBOSE: 0 + # PYTEST_MD_REPORT_OUTPUT: pytest.md + # run: > + # geant4-config --install-datasets + # && source build/this_acts_withdeps.sh + # && python3 -m pip install -r Examples/Python/tests/requirements.txt + # && python3 -m pip install pytest-md-report + # && pytest -rFsv -k "not exatrkx" -v + # && cat ${PYTEST_MD_REPORT_OUTPUT} >> $GITHUB_STEP_SUMMARY + # + # linux_physmon: + # runs-on: ubuntu-latest + # container: ghcr.io/acts-project/ubuntu2404:63 + # needs: [linux_ubuntu] + # env: + # ACTS_SEQUENCER_DISABLE_FPEMON: true + # + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: true + # lfs: true + # + # - run: apt-get update && apt-get install -y time + # + # - name: Set dependencies URL + # run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV + # + # - name: Install dependencies + # run: CI/dependencies.sh + # + # - uses: actions/download-artifact@v4 + # with: + # name: acts-linux-ubuntu + # + # - name: Unpack build + # run: mkdir build && tar xf build.tar.gz -C build + # + # - name: Save PR number + # if: ${{ github.event_name == 'pull_request' }} + # run: | + # mkdir -p physmon + # echo ${{ github.event.number }} > physmon/pr_number + # echo ${{ github.event.pull_request.head.sha }} > physmon/sha + # + # - name: Physics performance checks + # shell: bash + # run: > + # echo "::group::Dependencies" + # && git config --global safe.directory "$GITHUB_WORKSPACE" + # && python3 -m pip install histcmp==0.6.8 matplotlib + # && python3 -m pip install -r Examples/Scripts/requirements.txt + # && geant4-config --install-datasets + # && venv_python=$(which python3) + # && echo $venv_python + # && source build/this_acts_withdeps.sh + # && export PATH=$(dirname $venv_python):$PATH + # && echo $PATH + # && which python3 + # && echo "::endgroup::" + # && CI/physmon/phys_perf_mon.sh all physmon + # + # - name: Post step summary + # if: always() + # run: cat physmon/summary.md >> $GITHUB_STEP_SUMMARY + # + # - uses: actions/upload-artifact@v4 + # if: always() + # with: + # name: physmon + # path: physmon + # + # linux_physmon_perf_report: + # needs: [linux_physmon] + # runs-on: ubuntu-latest + # if: github.ref == 'refs/heads/main' + # steps: + # - name: Install dependencies + # run: pip3 install spyral-cli==1.1.2 + # + # - uses: actions/download-artifact@v4 + # with: + # name: physmon + # path: physmon + # + # - name: Store metrics + # env: + # SSH_AUTH_SOCK: /tmp/ssh_agent.sock + # run: | + # ssh-agent -a $SSH_AUTH_SOCK > /dev/null + # ssh-add - <<< "${{ secrets.RUNTIME_METRIC_DEPLOY_SSH_KEY }}" + # git config --global user.email "action@github.com" + # git config --global user.name "GitHub Action" + # git clone git@github.com:acts-project/runtime_metrics.git + # spyral maxima runtime_metrics/metrics.csv physmon/memory/*.csv -e $(date +%Y-%m-%dT%H-%M-%S) -e ${GITHUB_REF_NAME} -e ${GITHUB_REF} -e ${GITHUB_SHA} + # cd runtime_metrics + # git add -A + # git commit -m"update metrics" + # git push + # + # linux_ubuntu_extra: + # runs-on: ubuntu-latest + # strategy: + # matrix: + # include: + # - image: ubuntu2204 + # std: 20 + # - image: ubuntu2204_clang + # std: 20 + # container: ghcr.io/acts-project/${{ matrix.image }}:63 + # env: + # INSTALL_DIR: ${{ github.workspace }}/install + # ACTS_LOG_FAILURE_THRESHOLD: WARNING + # + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: true + # lfs: true + # + # - name: Set dependencies URL + # run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV + # + # - name: Install dependencies + # run: CI/dependencies.sh + # + # - name: Restore ccache + # uses: actions/cache/restore@v4 + # id: ccache-restore + # with: + # path: ${{ env.CCACHE_DIR }} + # key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} + # restore-keys: | + # ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- + # + # + # - name: Configure + # # setting CMAKE_CXX_STANDARD=20 is a workaround for a bug in the + # # dd4hep CMake configuration that gets triggered on recent CMake + # # versions + # run: > + # ccache -z && + # cmake -B build -S . + # --preset=github-ci + # -DCMAKE_CXX_STANDARD=${{ matrix.std }} + # -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + # -DPython_EXECUTABLE=$(which python3) + # + # - name: Build + # run: cmake --build build + # + # - name: ccache stats + # run: ccache -s + # + # - name: Save ccache + # uses: actions/cache/save@v4 + # if: always() + # with: + # path: ${{ github.workspace }}/ccache + # key: ${{ steps.ccache-restore.outputs.cache-primary-key }} + # + # - name: Unit tests + # run: cmake --build build --target test + # + # - name: Integration tests + # run: cmake --build build --target integrationtests + # + # - name: Install + # run: cmake --build build --target install + # + # - name: Downstream configure + # run: > + # cmake -B build-downstream -S Tests/DownstreamProject + # -GNinja + # -DCMAKE_BUILD_TYPE=Release + # -DCMAKE_CXX_FLAGS=-Werror + # -DCMAKE_CXX_STANDARD=${{ matrix.std }} + # -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" + # + # - name: Downstream build + # run: cmake --build build-downstream + # + # - name: Downstream run + # run: ./build-downstream/bin/ShowActsVersion + # + # macos: + # runs-on: macos-14 + # env: + # INSTALL_DIR: ${{ github.workspace }}/install_acts + # ACTS_LOG_FAILURE_THRESHOLD: WARNING + # + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: true + # lfs: true + # + # - name: Print architecture + # run: uname -p + # + # - name: Set dependencies URL + # run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/macos-14/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV + # + # - name: Install dependencies + # run: > + # brew install cmake ninja ccache xerces-c + # && CI/dependencies.sh + # + # - name: Restore ccache + # uses: actions/cache/restore@v4 + # id: ccache-restore + # with: + # path: ${{ env.CCACHE_DIR }} + # key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} + # restore-keys: | + # ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- + # + # - name: Configure + # run: > + # ccache -z + # && cmake -B build -S . + # --preset=github-ci + # -DCMAKE_PREFIX_PATH="${{ env.DEPENDENCY_DIR }}" + # -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" + # -DPython_EXECUTABLE=$(which python3) + # + # - name: Build + # run: cmake --build build + # + # - name: ccache stats + # run: ccache -s + # + # - name: Save ccache + # uses: actions/cache/save@v4 + # if: always() + # with: + # path: ${{ github.workspace }}/ccache + # key: ${{ steps.ccache-restore.outputs.cache-primary-key }} + # + # - name: Unit tests + # run: cmake --build build --target test + # + # - name: Integration tests + # run: cmake --build build --target integrationtests + # + # - name: Install + # run: cmake --build build --target install + # + # - uses: actions/upload-artifact@v4 + # with: + # name: acts-macos + # path: ${{ env.INSTALL_DIR }} + # + # - name: Downstream configure + # run: > + # cmake -B build-downstream -S Tests/DownstreamProject + # -GNinja + # -DCMAKE_BUILD_TYPE=Release + # -DCMAKE_CXX_FLAGS=-Werror + # -DCMAKE_CXX_STANDARD=20 + # -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" + # + # - name: Downstream build + # run: cmake --build build-downstream + # + # - name: Downstream run + # run: > + # PATH="${{ env.DEPENDENCY_DIR }}/bin:$PATH" + # && ./build-downstream/bin/ShowActsVersion From 81f61ace16d96e683daedf3187651b315baf2dfd Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:05:24 +0100 Subject: [PATCH 003/191] use correct folder --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 44e45828e51..c947ecdef74 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -24,7 +24,7 @@ runs: - name: Locate OpenGL shell: bash - run: ./opengl.sh + run: "${GITHUB_ACTION_PATH}/opengl.sh" # - name: Setup Xcode version # if: startsWith(inputs.os, 'macos') From 5cf1d76332d421943b8e117fe8b9d19bf4b38e10 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:17:24 +0100 Subject: [PATCH 004/191] prog --- .github/actions/dependencies/action.yml | 22 +++++++++++++++++----- .github/workflows/builds.yml | 3 ++- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index c947ecdef74..edf892be396 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -1,10 +1,10 @@ name: "Dependencies action" description: "Reusable action that sets up dependencies for the ACTS CI build" -# inputs: -# example_input: -# description: "An example input" -# required: true -# default: "Hello, World!" +inputs: + xcode-version: + description: "Which Xcode version to use (only for macOS)" + required: true + default: "" # outputs: # example_output: # description: "An example output" @@ -26,6 +26,18 @@ runs: shell: bash run: "${GITHUB_ACTION_PATH}/opengl.sh" + - name: Get spack lock file name + shell: bash + run: | + arch=$(spack arch --family) + if [[ $arch == darwin* ]]; then + lock_file="spack-${arch}-${{ inputs.xcode }}.lock" + else + lock_file="spack-${arch}.lock" + fi + echo "Lock file: ${lock_file}" + echo "SPACK_LOCK_FILE=${lock_file}" >> $GITHUB_ENV + # - name: Setup Xcode version # if: startsWith(inputs.os, 'macos') # uses: maxim-lobanov/setup-xcode@v1 diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index fb70b81dd3e..b90566eda12 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -36,7 +36,8 @@ jobs: submodules: true lfs: true - - uses: ./.github/actions/dependencies + - name: Install dependencies + uses: ./.github/actions/dependencies # - name: Set dependencies URL # run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV From bc75425765a053eb0a6f3084d46d1e3e8acf1e19 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:18:44 +0100 Subject: [PATCH 005/191] add missing file --- .github/actions/dependencies/detect_os.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/actions/dependencies/detect_os.sh diff --git a/.github/actions/dependencies/detect_os.sh b/.github/actions/dependencies/detect_os.sh new file mode 100644 index 00000000000..b943539cbe1 --- /dev/null +++ b/.github/actions/dependencies/detect_os.sh @@ -0,0 +1,17 @@ +if [ $(uname) == "Linux" ]; then + os_name=$(cat /etc/os-release | grep -e "^PRETTY_NAME=" | sed 's/PRETTY_NAME="\(.*\)"/\1/g') + if [[ $os_name == *"Ubuntu"* ]]; then + os="ubuntu" + elif [[ $os_name == *"AlmaLinux"* ]]; then + os="almalinux" + fi +elif [ $(uname) == "Darwin" ]; then + os_name="$(sw_vers -productName) $(sw_vers -productVersion)" + os="macos" +else + echo "Only Ubuntu, AlmaLinux and macOS are supported. Exiting." + exit 1 +fi + +export os +export os_name From f12851380752e5843a309a75432dbf3741202d5a Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:20:10 +0100 Subject: [PATCH 006/191] script_dir --- .github/actions/dependencies/opengl.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/dependencies/opengl.sh b/.github/actions/dependencies/opengl.sh index d3943c4d114..4c7240a60c5 100755 --- a/.github/actions/dependencies/opengl.sh +++ b/.github/actions/dependencies/opengl.sh @@ -3,7 +3,9 @@ set -u set -e -source detect_os.sh +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +source ${SCRIPT_DIR}/detect_os.sh packages_file=${GITHUB_WORKSPACE}/spack/etc/spack/packages.yaml From d394c3b4c05b396ba27ea279e0be53d3b84c029d Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:22:45 +0100 Subject: [PATCH 007/191] maybe sudo --- .github/actions/dependencies/opengl.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/actions/dependencies/opengl.sh b/.github/actions/dependencies/opengl.sh index 4c7240a60c5..bb6f68dc747 100755 --- a/.github/actions/dependencies/opengl.sh +++ b/.github/actions/dependencies/opengl.sh @@ -9,9 +9,17 @@ source ${SCRIPT_DIR}/detect_os.sh packages_file=${GITHUB_WORKSPACE}/spack/etc/spack/packages.yaml +if ! command -v sudo &> /dev/null +then + SUDO="" +else + SUDO="sudo" +fi + + if [ "$os" == "ubuntu" ]; then - sudo apt-get update - sudo apt-get install -y libgl1-mesa-dev + ${SUDO} apt-get update + ${SUDO} apt-get install -y libgl1-mesa-dev cat < "$packages_file" packages: opengl: @@ -22,7 +30,7 @@ packages: EOF cat "$packages_file" elif [ "$os" == "almalinux" ]; then - dnf install -y mesa-libGLU + ${SUDO} dnf install -y mesa-libGLU cat < "$packages_file" packages: opengl: From 84ab8aaf0cb9a4304798380ca6d683cb325119eb Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:27:38 +0100 Subject: [PATCH 008/191] download lock file --- .github/actions/dependencies/action.yml | 26 +++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index edf892be396..61646ed6252 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -5,9 +5,10 @@ inputs: description: "Which Xcode version to use (only for macOS)" required: true default: "" -# outputs: -# example_output: -# description: "An example output" + dependency-tag: + description: "Which dependency tag to use" + required: true + default: "v6-test" runs: using: "composite" steps: @@ -38,8 +39,17 @@ runs: echo "Lock file: ${lock_file}" echo "SPACK_LOCK_FILE=${lock_file}" >> $GITHUB_ENV - # - name: Setup Xcode version - # if: startsWith(inputs.os, 'macos') - # uses: maxim-lobanov/setup-xcode@v1 - # with: - # xcode-version: "${{ inputs.xcode }}" + - name: Setup Xcode version + if: startsWith(runner.os, 'macos') + uses: maxim-lobanov/setup-xcode@v1 + with: + xcode-version: "${{ inputs.xcode-version }}" + + - name: Get spack lock file from tag + shell: bash + run: | + url="https://github.com/acts-project/ci-dependencies/releases/download/${{ inputs.dependency-tag }}/${SPACK_LOCK_FILE}" + echo "URL: ${url}" + curl -L -o spack.lock $url + + ls -al From 75867aa4e50ea4b5762ad7539541419dc624d9da Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:30:30 +0100 Subject: [PATCH 009/191] run the install --- .github/actions/dependencies/action.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 61646ed6252..9c306eea78a 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -52,4 +52,9 @@ runs: echo "URL: ${url}" curl -L -o spack.lock $url - ls -al + - name: Create spack environment + shell: bash + run: | + spack env create -d spack_env spack.lock + spack env activate -d spack-env + spack install --use-buildcache only From 9daaddd46c80e0c498de4c68b54b836a0f4d8e0c Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:32:58 +0100 Subject: [PATCH 010/191] don't activate --- .github/actions/dependencies/action.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 9c306eea78a..4374861559a 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -55,6 +55,5 @@ runs: - name: Create spack environment shell: bash run: | - spack env create -d spack_env spack.lock - spack env activate -d spack-env - spack install --use-buildcache only + spack env create -d . spack.lock + spack -e . install --use-buildcache only From d308cf6e3e7035adb9f09ecfdc98fcc57da89e8e Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:35:23 +0100 Subject: [PATCH 011/191] named --- .github/actions/dependencies/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 4374861559a..8ad7eeb7ed0 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -55,5 +55,5 @@ runs: - name: Create spack environment shell: bash run: | - spack env create -d . spack.lock - spack -e . install --use-buildcache only + spack env create ci spack.lock + spack -e ci install --use-buildcache only From 4d044f3f8fadcd014e94e32fae8506e567d976fd Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:39:41 +0100 Subject: [PATCH 012/191] add the build cache --- .github/actions/dependencies/action.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 8ad7eeb7ed0..e714a493cfb 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -23,6 +23,12 @@ runs: git config user.email 41898282+github-actions[bot]@users.noreply.github.com curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am + + - name: Add spack buildcache mirror + shell: bash + run: | + spack mirror add acts-spack-buildcache oci://ghcr.io/acts-project/spack-buildcache --unsigned + - name: Locate OpenGL shell: bash run: "${GITHUB_ACTION_PATH}/opengl.sh" From 963c13c493d1d7fc613d6968dc63d5881908fe9a Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:44:35 +0100 Subject: [PATCH 013/191] add some other jobs --- .github/workflows/builds.yml | 94 ++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 53 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index b90566eda12..fd7689f512a 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -39,12 +39,6 @@ jobs: - name: Install dependencies uses: ./.github/actions/dependencies - # - name: Set dependencies URL - # run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV - # - # - name: Install dependencies - # run: CI/dependencies.sh - # # - name: Restore ccache # uses: actions/cache/restore@v4 # id: ccache-restore @@ -244,31 +238,28 @@ jobs: # git commit -m"update metrics" # git push # - # linux_ubuntu_extra: - # runs-on: ubuntu-latest - # strategy: - # matrix: - # include: - # - image: ubuntu2204 - # std: 20 - # - image: ubuntu2204_clang - # std: 20 - # container: ghcr.io/acts-project/${{ matrix.image }}:63 - # env: - # INSTALL_DIR: ${{ github.workspace }}/install - # ACTS_LOG_FAILURE_THRESHOLD: WARNING - # - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # lfs: true - # - # - name: Set dependencies URL - # run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV - # - # - name: Install dependencies - # run: CI/dependencies.sh + linux_ubuntu_extra: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - image: ubuntu2204 + std: 20 + - image: ubuntu2204_clang + std: 20 + container: ghcr.io/acts-project/${{ matrix.image }}:63 + env: + INSTALL_DIR: ${{ github.workspace }}/install + ACTS_LOG_FAILURE_THRESHOLD: WARNING + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + lfs: true + + - name: Install dependencies + uses: ./.github/actions/dependencies # # - name: Restore ccache # uses: actions/cache/restore@v4 @@ -329,28 +320,25 @@ jobs: # - name: Downstream run # run: ./build-downstream/bin/ShowActsVersion # - # macos: - # runs-on: macos-14 - # env: - # INSTALL_DIR: ${{ github.workspace }}/install_acts - # ACTS_LOG_FAILURE_THRESHOLD: WARNING - # - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # lfs: true - # - # - name: Print architecture - # run: uname -p - # - # - name: Set dependencies URL - # run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/macos-14/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV - # - # - name: Install dependencies - # run: > - # brew install cmake ninja ccache xerces-c - # && CI/dependencies.sh + macos: + runs-on: macos-14 + env: + INSTALL_DIR: ${{ github.workspace }}/install_acts + ACTS_LOG_FAILURE_THRESHOLD: WARNING + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + lfs: true + + # - name: Install dependencies + # run: > + # brew install cmake ninja ccache xerces-c + # && CI/dependencies.sh + + - name: Install dependencies + uses: ./.github/actions/dependencies # # - name: Restore ccache # uses: actions/cache/restore@v4 From 7414ac9869dc8cb8ac308e4c1c8a9797e69974e4 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:46:32 +0100 Subject: [PATCH 014/191] correct xcode version --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index e714a493cfb..a4b19ca93b0 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -38,7 +38,7 @@ runs: run: | arch=$(spack arch --family) if [[ $arch == darwin* ]]; then - lock_file="spack-${arch}-${{ inputs.xcode }}.lock" + lock_file="spack-${arch}-${{ inputs.xcode-version }}.lock" else lock_file="spack-${arch}.lock" fi From e1aa09eaadbd9529d12d294c9794a08deec56ba2 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:48:39 +0100 Subject: [PATCH 015/191] lock to xcode 16.1.0 --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index a4b19ca93b0..4a38b333f97 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -4,7 +4,7 @@ inputs: xcode-version: description: "Which Xcode version to use (only for macOS)" required: true - default: "" + default: "16.0.0" dependency-tag: description: "Which dependency tag to use" required: true From 6514dd9db2298c18e9be0a8c0b3d6a26de3b4207 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:50:24 +0100 Subject: [PATCH 016/191] another error --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 4a38b333f97..8a852c2433e 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -38,7 +38,7 @@ runs: run: | arch=$(spack arch --family) if [[ $arch == darwin* ]]; then - lock_file="spack-${arch}-${{ inputs.xcode-version }}.lock" + lock_file="spack-${arch}-xcode${{ inputs.xcode-version }}.lock" else lock_file="spack-${arch}.lock" fi From 5b9ac2126e32ebf85c6d4edf9d312e8e49077665 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 09:46:58 +0100 Subject: [PATCH 017/191] print folder, create env --- .github/actions/dependencies/action.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 8a852c2433e..f7b707326cb 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -62,4 +62,11 @@ runs: shell: bash run: | spack env create ci spack.lock + spack -e ci spec + spack -e ci find spack -e ci install --use-buildcache only + pwd + spack -e view enable dependencies + ls -al + + # - name: Install Geant4 data From 04e302d6832d3ac810890ef4f6fce96ad47c7296 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 09:57:03 +0100 Subject: [PATCH 018/191] add view creation --- .github/actions/dependencies/action.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index f7b707326cb..72b17e4de9e 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -61,12 +61,10 @@ runs: - name: Create spack environment shell: bash run: | - spack env create ci spack.lock + spack env create ci spack.lock --with-view $PWD/dependencies spack -e ci spec spack -e ci find spack -e ci install --use-buildcache only - pwd - spack -e view enable dependencies ls -al # - name: Install Geant4 data From 527c06fa74f8c19858373d6baf5ae97d5292ad4a Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 11:03:20 +0100 Subject: [PATCH 019/191] get g4 data --- .github/actions/dependencies/action.yml | 7 ++++++- .github/actions/dependencies/with_spack_env.sh | 11 +++++++++++ .github/workflows/builds.yml | 3 +++ 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100755 .github/actions/dependencies/with_spack_env.sh diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 72b17e4de9e..545565b6c83 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -67,4 +67,9 @@ runs: spack -e ci install --use-buildcache only ls -al - # - name: Install Geant4 data + - name: Install Geant4 data + shell: bash + run: | + spack -e ci install --add geant4-data + "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci env | grep G4 | awk '{print "export " $0}' > geant4_data.sh + source geant4_data.sh diff --git a/.github/actions/dependencies/with_spack_env.sh b/.github/actions/dependencies/with_spack_env.sh new file mode 100755 index 00000000000..13cefd50977 --- /dev/null +++ b/.github/actions/dependencies/with_spack_env.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e +set -u + +source "${SPACK_ROOT}/share/spack/setup-env.sh" + +env=$1 +shift +spack env activate "$env" + +"$@" diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index fd7689f512a..29c42aa3ee7 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -39,6 +39,9 @@ jobs: - name: Install dependencies uses: ./.github/actions/dependencies + - run: | + env | grep G4 + # - name: Restore ccache # uses: actions/cache/restore@v4 # id: ccache-restore From a9611476e6bd79335072f855e02a76e471e97d43 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 11:15:15 +0100 Subject: [PATCH 020/191] different root detection mechanism --- .github/actions/dependencies/with_spack_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/with_spack_env.sh b/.github/actions/dependencies/with_spack_env.sh index 13cefd50977..ae0de83d4b4 100755 --- a/.github/actions/dependencies/with_spack_env.sh +++ b/.github/actions/dependencies/with_spack_env.sh @@ -2,7 +2,7 @@ set -e set -u -source "${SPACK_ROOT}/share/spack/setup-env.sh" +source "$(spack location -r)/share/spack/setup-env.sh" env=$1 shift From 2826fe63349e15ec741e33806b1b7e58c708bdf4 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 11:25:50 +0100 Subject: [PATCH 021/191] other env mechanism --- .github/actions/dependencies/action.yml | 5 ++--- .github/workflows/builds.yml | 5 ++++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 545565b6c83..806c5080ad7 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -43,7 +43,7 @@ runs: lock_file="spack-${arch}.lock" fi echo "Lock file: ${lock_file}" - echo "SPACK_LOCK_FILE=${lock_file}" >> $GITHUB_ENV + echo "SPACK_LOCK_FILE=${lock_file}" >> "$GITHUB_ENV" - name: Setup Xcode version if: startsWith(runner.os, 'macos') @@ -71,5 +71,4 @@ runs: shell: bash run: | spack -e ci install --add geant4-data - "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci env | grep G4 | awk '{print "export " $0}' > geant4_data.sh - source geant4_data.sh + "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci env | grep G4 >> "$GITHUB_ENV" diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 29c42aa3ee7..81cf0d799c0 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -40,7 +40,10 @@ jobs: uses: ./.github/actions/dependencies - run: | - env | grep G4 + env + ls -al + source geant4_data.sh + env # - name: Restore ccache # uses: actions/cache/restore@v4 From 005638ce471523ed42fd151ca14d5172e4976413 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 11:45:22 +0100 Subject: [PATCH 022/191] get python going --- .github/actions/dependencies/action.yml | 1 + .github/workflows/builds.yml | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 806c5080ad7..7688184d7a4 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -66,6 +66,7 @@ runs: spack -e ci find spack -e ci install --use-buildcache only ls -al + echo "PATH=$PWD/dependencies/bin:$PATH" >> "$GITHUB_ENV" - name: Install Geant4 data shell: bash diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 81cf0d799c0..892d4ee0c7e 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -39,11 +39,8 @@ jobs: - name: Install dependencies uses: ./.github/actions/dependencies - - run: | - env - ls -al - source geant4_data.sh - env + - run: env + - run: python3 -m pip install jinja2 pyyaml # - name: Restore ccache # uses: actions/cache/restore@v4 From 0b691bb365a1bb3ec148d5e08fd0f61e247eca99 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 12:45:56 +0100 Subject: [PATCH 023/191] paths --- .github/actions/dependencies/action.yml | 4 +++- .github/workflows/builds.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 7688184d7a4..f02b3a12fc4 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -66,7 +66,9 @@ runs: spack -e ci find spack -e ci install --use-buildcache only ls -al - echo "PATH=$PWD/dependencies/bin:$PATH" >> "$GITHUB_ENV" + venv_dir="$PWD/dependencies/venv" + "$PWD/dependencies/bin/python3" -m venv "$venv_dir" + echo "PATH=$PWD/dependencies/venv/bin/:$PWD/dependencies/bin:$PATH" >> "$GITHUB_ENV" - name: Install Geant4 data shell: bash diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 892d4ee0c7e..74506668703 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -40,7 +40,9 @@ jobs: uses: ./.github/actions/dependencies - run: env - - run: python3 -m pip install jinja2 pyyaml + - run: | + which python3 + python3 -m pip install jinja2 pyyaml # - name: Restore ccache # uses: actions/cache/restore@v4 From 2f9ecbaeacf69478d9ef9578b07a81bd7e4aedcd Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 13:02:01 +0100 Subject: [PATCH 024/191] debug --- .github/actions/dependencies/action.yml | 1 + .github/workflows/builds.yml | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index f02b3a12fc4..6acaf0c8985 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -69,6 +69,7 @@ runs: venv_dir="$PWD/dependencies/venv" "$PWD/dependencies/bin/python3" -m venv "$venv_dir" echo "PATH=$PWD/dependencies/venv/bin/:$PWD/dependencies/bin:$PATH" >> "$GITHUB_ENV" + ls -al "${venv_dir}/bin" - name: Install Geant4 data shell: bash diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 74506668703..17b6dbb006b 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -40,7 +40,11 @@ jobs: uses: ./.github/actions/dependencies - run: env - - run: | + - run: | + ls -al + ls -al dependencies + ls -al dependencies/venv + ls -al dependencies/venv/bin which python3 python3 -m pip install jinja2 pyyaml From 8d3ce2c37bc6d3b95868bbc9fcd0f8e15b145e54 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 13:13:57 +0100 Subject: [PATCH 025/191] make python stuff later --- .github/actions/dependencies/action.yml | 14 +++++++++----- .github/workflows/builds.yml | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 6acaf0c8985..0442b97dbf0 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -65,14 +65,18 @@ runs: spack -e ci spec spack -e ci find spack -e ci install --use-buildcache only - ls -al - venv_dir="$PWD/dependencies/venv" - "$PWD/dependencies/bin/python3" -m venv "$venv_dir" - echo "PATH=$PWD/dependencies/venv/bin/:$PWD/dependencies/bin:$PATH" >> "$GITHUB_ENV" - ls -al "${venv_dir}/bin" - name: Install Geant4 data shell: bash run: | spack -e ci install --add geant4-data "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci env | grep G4 >> "$GITHUB_ENV" + + - name: Prepare python environment + shell: bash + run: | + ls -al + venv_dir="$PWD/dependencies/venv" + "$PWD/dependencies/bin/python3" -m venv "$venv_dir" + echo "PATH=$PWD/dependencies/venv/bin/:$PWD/dependencies/bin:$PATH" >> "$GITHUB_ENV" + ls -al "${venv_dir}/bin" diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 17b6dbb006b..96ee9febb6d 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -42,7 +42,7 @@ jobs: - run: env - run: | ls -al - ls -al dependencies + ls -al dependencies/ ls -al dependencies/venv ls -al dependencies/venv/bin which python3 From 46a3f86cb98d95920e094bd351676261a38dd6e1 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 13:54:49 +0100 Subject: [PATCH 026/191] more debug --- .github/actions/dependencies/action.yml | 2 +- .github/workflows/builds.yml | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 0442b97dbf0..9161989f456 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -78,5 +78,5 @@ runs: ls -al venv_dir="$PWD/dependencies/venv" "$PWD/dependencies/bin/python3" -m venv "$venv_dir" - echo "PATH=$PWD/dependencies/venv/bin/:$PWD/dependencies/bin:$PATH" >> "$GITHUB_ENV" + echo "PATH=${venv_dir}/bin/:$PWD/dependencies/bin:$PATH" >> "$GITHUB_ENV" ls -al "${venv_dir}/bin" diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 96ee9febb6d..4ae0462fe6c 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -40,7 +40,12 @@ jobs: uses: ./.github/actions/dependencies - run: env + - run: | + pwd + ls -al + ls -al /__w/acts/acts/dependencies/venv/bin/ - run: | + echo $PATH ls -al ls -al dependencies/ ls -al dependencies/venv @@ -49,8 +54,8 @@ jobs: python3 -m pip install jinja2 pyyaml # - name: Restore ccache + # id: ccache-restore # uses: actions/cache/restore@v4 - # id: ccache-restore # with: # path: ${{ env.CCACHE_DIR }} # key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} From c90864ba5c00e5283ae9b2d4420e337f60f4fdfe Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 15:18:28 +0100 Subject: [PATCH 027/191] different way of setting PATH --- .github/actions/dependencies/action.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 9161989f456..016292a4842 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -65,6 +65,7 @@ runs: spack -e ci spec spack -e ci find spack -e ci install --use-buildcache only + echo "$PWD/dependencies/bin" >> "$GITHUB_PATH" - name: Install Geant4 data shell: bash @@ -77,6 +78,6 @@ runs: run: | ls -al venv_dir="$PWD/dependencies/venv" - "$PWD/dependencies/bin/python3" -m venv "$venv_dir" - echo "PATH=${venv_dir}/bin/:$PWD/dependencies/bin:$PATH" >> "$GITHUB_ENV" + python3 -m venv "$venv_dir" + echo "${venv_dir}/bin/" >> "$GITHUB_PATH" ls -al "${venv_dir}/bin" From 02f7f8953246514cc91d5c83f585c585dd8d86f3 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 15:27:05 +0100 Subject: [PATCH 028/191] enable build --- .github/workflows/builds.yml | 136 +++++++++++++++++------------------ 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 4ae0462fe6c..b2fd3d9d0dd 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -53,74 +53,74 @@ jobs: which python3 python3 -m pip install jinja2 pyyaml - # - name: Restore ccache - # id: ccache-restore - # uses: actions/cache/restore@v4 - # with: - # path: ${{ env.CCACHE_DIR }} - # key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} - # restore-keys: | - # ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - # - # - name: Configure - # # setting CMAKE_CXX_STANDARD=20 is a workaround for a bug in the - # # dd4hep CMake configuration that gets triggered on recent CMake - # # versions - # # Need to set git user & email for patching to work (GeoModel plugin) - # run: > - # git config --global user.name 'CI' && - # git config --global user.email '<>' && - # ccache -z && - # cmake -B build -S . - # --preset=github-ci - # -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" - # -DPython_EXECUTABLE=$(which python3) - # -DACTS_BUILD_PLUGIN_ONNX=ON - # - # - name: Build - # run: cmake --build build - # - # - name: ccache stats - # run: ccache -s - # - # - name: Save ccache - # uses: actions/cache/save@v4 - # if: always() - # with: - # path: ${{ github.workspace }}/ccache - # key: ${{ steps.ccache-restore.outputs.cache-primary-key }} - # - # - name: Unit tests - # run: ctest --test-dir build -j$(nproc) - # - # - name: Integration tests - # run: cmake --build build --target integrationtests - # - # - name: Install - # run: cmake --build build --target install - # - # - name: Package build - # run: tar czf build.tar.gz -C build --exclude "*.o" --exclude "bin/ActsUnitTest*" --exclude "bin/ActsIntegrationTest*" . - # - # - uses: actions/upload-artifact@v4 - # with: - # name: acts-linux-ubuntu - # path: build.tar.gz - # - # - name: Downstream configure - # run: > - # cmake -B build-downstream -S Tests/DownstreamProject - # -GNinja - # -DCMAKE_BUILD_TYPE=Release - # -DCMAKE_CXX_FLAGS=-Werror - # -DCMAKE_CXX_STANDARD=20 - # -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" - # - # - name: Downstream build - # run: cmake --build build-downstream - # - # - name: Downstream run - # run: ./build-downstream/bin/ShowActsVersion + - name: Restore ccache + id: ccache-restore + uses: actions/cache/restore@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} + restore-keys: | + ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- + + - name: Configure + # setting CMAKE_CXX_STANDARD=20 is a workaround for a bug in the + # dd4hep CMake configuration that gets triggered on recent CMake + # versions + # Need to set git user & email for patching to work (GeoModel plugin) + run: > + git config --global user.name 'CI' && + git config --global user.email '<>' && + ccache -z && + cmake -B build -S . + --preset=github-ci + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + -DPython_EXECUTABLE=$(which python3) + -DACTS_BUILD_PLUGIN_ONNX=ON + + - name: Build + run: cmake --build build + + - name: ccache stats + run: ccache -s + + - name: Save ccache + uses: actions/cache/save@v4 + if: always() + with: + path: ${{ github.workspace }}/ccache + key: ${{ steps.ccache-restore.outputs.cache-primary-key }} + + - name: Unit tests + run: ctest --test-dir build -j$(nproc) + + - name: Integration tests + run: cmake --build build --target integrationtests + + - name: Install + run: cmake --build build --target install + + - name: Package build + run: tar czf build.tar.gz -C build --exclude "*.o" --exclude "bin/ActsUnitTest*" --exclude "bin/ActsIntegrationTest*" . + + - uses: actions/upload-artifact@v4 + with: + name: acts-linux-ubuntu + path: build.tar.gz + + - name: Downstream configure + run: > + cmake -B build-downstream -S Tests/DownstreamProject + -GNinja + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_CXX_FLAGS=-Werror + -DCMAKE_CXX_STANDARD=20 + -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" + + - name: Downstream build + run: cmake --build build-downstream + + - name: Downstream run + run: ./build-downstream/bin/ShowActsVersion # linux_examples_test: # runs-on: ubuntu-latest From edc202eceb4c7d3234b9533936df717d3bd5f227 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 16:07:08 +0100 Subject: [PATCH 029/191] set CMake prefix path --- .github/actions/dependencies/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 016292a4842..2c096b79be4 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -66,6 +66,7 @@ runs: spack -e ci find spack -e ci install --use-buildcache only echo "$PWD/dependencies/bin" >> "$GITHUB_PATH" + echo "CMAKE_PREFIX_PATH=$PWD/dependencies" >> "$GITHUB_ENV" - name: Install Geant4 data shell: bash @@ -79,5 +80,5 @@ runs: ls -al venv_dir="$PWD/dependencies/venv" python3 -m venv "$venv_dir" - echo "${venv_dir}/bin/" >> "$GITHUB_PATH" + echo "${venv_dir}/bin" >> "$GITHUB_PATH" ls -al "${venv_dir}/bin" From 54f3ec98161d23608d4162d240c5d751eb5a3190 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 16:08:09 +0100 Subject: [PATCH 030/191] additional variables --- .github/actions/dependencies/action.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 2c096b79be4..eb429b1f682 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -67,6 +67,12 @@ runs: spack -e ci install --use-buildcache only echo "$PWD/dependencies/bin" >> "$GITHUB_PATH" echo "CMAKE_PREFIX_PATH=$PWD/dependencies" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${destination}/lib" >> "$GITHUB_ENV" + echo "ROOT_INCLUDE_PATH=${destination}/include" >> "$GITHUB_ENV" + # # Geant4 puts CLHEP in a subdirectory + # set_env ROOT_INCLUDE_PATH "${destination}/include/Geant4" + # Pythia8 looks for settings in this directory + echo "PYTHIA8DATA=${destination}/share/Pythia8/xmldoc" >> "$GITHUB_ENV" - name: Install Geant4 data shell: bash From f6bafc62e3050fbd8325a246ef5cf476cd08f2f7 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 18:05:49 +0100 Subject: [PATCH 031/191] use more CPUs to install --- .github/actions/dependencies/action.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index eb429b1f682..f60a161b477 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -61,10 +61,16 @@ runs: - name: Create spack environment shell: bash run: | + if [[ $(spack arch --platform) == darwin ]]; then + NCPUS=$(sysctl -n hw.ncpu) + else + NCPUS=$(nproc) + fi + spack env create ci spack.lock --with-view $PWD/dependencies spack -e ci spec spack -e ci find - spack -e ci install --use-buildcache only + spack -n$NCPUS -e ci install --use-buildcache only echo "$PWD/dependencies/bin" >> "$GITHUB_PATH" echo "CMAKE_PREFIX_PATH=$PWD/dependencies" >> "$GITHUB_ENV" echo "LD_LIBRARY_PATH=${destination}/lib" >> "$GITHUB_ENV" From f56a378e40eb91f245465bfd81116c5e4857aa39 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 18:11:40 +0100 Subject: [PATCH 032/191] add timing --- .github/actions/dependencies/action.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index f60a161b477..4e23c16cf41 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -67,10 +67,10 @@ runs: NCPUS=$(nproc) fi - spack env create ci spack.lock --with-view $PWD/dependencies - spack -e ci spec - spack -e ci find - spack -n$NCPUS -e ci install --use-buildcache only + time spack env create ci spack.lock --with-view $PWD/dependencies + time spack -e ci spec + time spack -e ci find + time spack -n$NCPUS -e ci install --use-buildcache only echo "$PWD/dependencies/bin" >> "$GITHUB_PATH" echo "CMAKE_PREFIX_PATH=$PWD/dependencies" >> "$GITHUB_ENV" echo "LD_LIBRARY_PATH=${destination}/lib" >> "$GITHUB_ENV" @@ -83,7 +83,7 @@ runs: - name: Install Geant4 data shell: bash run: | - spack -e ci install --add geant4-data + time spack -e ci install --add geant4-data "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci env | grep G4 >> "$GITHUB_ENV" - name: Prepare python environment From 2e59eb2211ccb4a4f659b8c6fb99e9077fc5d4b5 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 09:40:13 +0100 Subject: [PATCH 033/191] try better parallel --- .github/actions/dependencies/action.yml | 5 +++-- .github/actions/dependencies/parallel.sh | 28 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 .github/actions/dependencies/parallel.sh diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 4e23c16cf41..abfd84fdf4a 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -70,7 +70,7 @@ runs: time spack env create ci spack.lock --with-view $PWD/dependencies time spack -e ci spec time spack -e ci find - time spack -n$NCPUS -e ci install --use-buildcache only + time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci install --use-buildcache only echo "$PWD/dependencies/bin" >> "$GITHUB_PATH" echo "CMAKE_PREFIX_PATH=$PWD/dependencies" >> "$GITHUB_ENV" echo "LD_LIBRARY_PATH=${destination}/lib" >> "$GITHUB_ENV" @@ -83,7 +83,8 @@ runs: - name: Install Geant4 data shell: bash run: | - time spack -e ci install --add geant4-data + time spack -e ci add geant4-data + time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci install "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci env | grep G4 >> "$GITHUB_ENV" - name: Prepare python environment diff --git a/.github/actions/dependencies/parallel.sh b/.github/actions/dependencies/parallel.sh new file mode 100644 index 00000000000..4ec4ece0f1f --- /dev/null +++ b/.github/actions/dependencies/parallel.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +N=$1 +shift +declare -a pids=() + +# Start all processes in the background +for i in $(seq 1 "$N"); do + # Replace `sleep 10` with the actual command you want to run. + # For demonstration, we are using a command that sleeps for 10 seconds. + # Make sure it runs in the background with '&'. + "$@" & + pids+=($!) +done + +# Wait for all processes to finish, if any fails, kill them all +for pid in "${pids[@]}"; do + if ! wait "$pid"; then + echo "Process $pid failed. Terminating all remaining processes..." + # Kill all started processes + kill "${pids[@]}" 2>/dev/null || true + exit 1 + fi +done + +echo "All processes completed successfully." +exit 0 From d3742329cfff2190ee20380414bd84f77daf1c4d Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 09:40:45 +0100 Subject: [PATCH 034/191] number --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index abfd84fdf4a..8dde3859934 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -84,7 +84,7 @@ runs: shell: bash run: | time spack -e ci add geant4-data - time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci install + time ${GITHUB_ACTION_PATH}/parallel.sh 10 spack -e ci install "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci env | grep G4 >> "$GITHUB_ENV" - name: Prepare python environment From 3c27432b45437e1072a9ba24223a3615a1c9d5a5 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 09:55:35 +0100 Subject: [PATCH 035/191] make parallel script executable --- .github/actions/dependencies/parallel.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .github/actions/dependencies/parallel.sh diff --git a/.github/actions/dependencies/parallel.sh b/.github/actions/dependencies/parallel.sh old mode 100644 new mode 100755 From 75bb1088ea63183b699516d2d72af7eb64f8258d Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 10:04:39 +0100 Subject: [PATCH 036/191] higher count on install from buildcache --- .github/actions/dependencies/action.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 8dde3859934..73d83d933f8 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -61,11 +61,12 @@ runs: - name: Create spack environment shell: bash run: | - if [[ $(spack arch --platform) == darwin ]]; then - NCPUS=$(sysctl -n hw.ncpu) - else - NCPUS=$(nproc) - fi + # if [[ $(spack arch --platform) == darwin ]]; then + # NCPUS=$(sysctl -n hw.ncpu) + # else + # NCPUS=$(nproc) + # fi + NCPUS=16 # we're only downloading time spack env create ci spack.lock --with-view $PWD/dependencies time spack -e ci spec From a2180e792192ec7d458d37044a3b5a51e2aa42aa Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 11:17:41 +0100 Subject: [PATCH 037/191] separate concretize --- .github/actions/dependencies/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 73d83d933f8..b30f4da5288 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -85,6 +85,7 @@ runs: shell: bash run: | time spack -e ci add geant4-data + time spack -e ci concretize time ${GITHUB_ACTION_PATH}/parallel.sh 10 spack -e ci install "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci env | grep G4 >> "$GITHUB_ENV" From 7c8fa52a98b03fcf28f94aceb365838428bf3a16 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 14:43:14 +0100 Subject: [PATCH 038/191] install combo --- .github/actions/dependencies/action.yml | 38 ++++++++++++++----------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index b30f4da5288..c1747b40278 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -39,11 +39,16 @@ runs: arch=$(spack arch --family) if [[ $arch == darwin* ]]; then lock_file="spack-${arch}-xcode${{ inputs.xcode-version }}.lock" + lock_file_g4="spack-${arch}-xcode${{ inputs.xcode-version }}-geant4-data.lock" else lock_file="spack-${arch}.lock" + lock_file_g4="spack-${arch}-geant4-data.lock" fi + echo "Lock file: ${lock_file}" + echo "Lock file G4: ${lock_file_g4}" echo "SPACK_LOCK_FILE=${lock_file}" >> "$GITHUB_ENV" + echo "SPACK_LOCK_FILE_G4=${lock_file_g4}" >> "$GITHUB_ENV" - name: Setup Xcode version if: startsWith(runner.os, 'macos') @@ -51,27 +56,36 @@ runs: with: xcode-version: "${{ inputs.xcode-version }}" - - name: Get spack lock file from tag + - name: Get spack lock files from tag shell: bash run: | url="https://github.com/acts-project/ci-dependencies/releases/download/${{ inputs.dependency-tag }}/${SPACK_LOCK_FILE}" echo "URL: ${url}" curl -L -o spack.lock $url + g4url="https://github.com/acts-project/ci-dependencies/releases/download/${{ inputs.dependency-tag }}/${SPACK_LOCK_FILE_G4}" + echo "URL: ${g4url}" + curl -L -o spack-g4.lock $g4url - name: Create spack environment shell: bash run: | - # if [[ $(spack arch --platform) == darwin ]]; then - # NCPUS=$(sysctl -n hw.ncpu) - # else - # NCPUS=$(nproc) - # fi - NCPUS=16 # we're only downloading + NCPUS=32 # we're only downloading + + time spack env create ci spack.lock --with-view $PWD/dependencies & + time spack env create ci-g4 spack-g4.lock & + wait - time spack env create ci spack.lock --with-view $PWD/dependencies time spack -e ci spec time spack -e ci find + time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci install --use-buildcache only + time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci-g4 install + + "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci-g4 env | grep G4 >> "$GITHUB_ENV" + + - name: Configure downstream environment + shell: bash + run: | echo "$PWD/dependencies/bin" >> "$GITHUB_PATH" echo "CMAKE_PREFIX_PATH=$PWD/dependencies" >> "$GITHUB_ENV" echo "LD_LIBRARY_PATH=${destination}/lib" >> "$GITHUB_ENV" @@ -81,14 +95,6 @@ runs: # Pythia8 looks for settings in this directory echo "PYTHIA8DATA=${destination}/share/Pythia8/xmldoc" >> "$GITHUB_ENV" - - name: Install Geant4 data - shell: bash - run: | - time spack -e ci add geant4-data - time spack -e ci concretize - time ${GITHUB_ACTION_PATH}/parallel.sh 10 spack -e ci install - "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci env | grep G4 >> "$GITHUB_ENV" - - name: Prepare python environment shell: bash run: | From faeea3881de8fe075adc9363611a6600749c27d9 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 14:47:41 +0100 Subject: [PATCH 039/191] correct tag --- .github/actions/dependencies/action.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index c1747b40278..cac415bd90e 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -8,7 +8,7 @@ inputs: dependency-tag: description: "Which dependency tag to use" required: true - default: "v6-test" + default: "v6-test3" runs: using: "composite" steps: @@ -78,8 +78,9 @@ runs: time spack -e ci spec time spack -e ci find - time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci install --use-buildcache only - time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci-g4 install + time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci install --use-buildcache only & + time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci-g4 install & + wait "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci-g4 env | grep G4 >> "$GITHUB_ENV" From 2f82d79369554220cffb5842884af50b201ce218 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 14:56:34 +0100 Subject: [PATCH 040/191] don't overlap the install --- .github/actions/dependencies/action.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index cac415bd90e..557162049f3 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -72,15 +72,17 @@ runs: NCPUS=32 # we're only downloading time spack env create ci spack.lock --with-view $PWD/dependencies & + pid1=$! time spack env create ci-g4 spack-g4.lock & - wait + pid2=$! + wait $pid1 + wait $pid2 time spack -e ci spec time spack -e ci find - time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci install --use-buildcache only & - time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci-g4 install & - wait + time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci install --use-buildcache only + time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci-g4 install "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci-g4 env | grep G4 >> "$GITHUB_ENV" From 8ddd0e1daba45bca5a4cd9974837dedb40cc93fb Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 15:02:43 +0100 Subject: [PATCH 041/191] i don't understand the spec --- .github/actions/dependencies/action.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 557162049f3..faf6a2bb6e6 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -82,6 +82,10 @@ runs: time spack -e ci find time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci install --use-buildcache only + + time spack -e ci-g4 spec + time spack -e ci-g4 find + time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci-g4 install "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci-g4 env | grep G4 >> "$GITHUB_ENV" From 288fde5081762f7345313f898bf5dbb6957a66d7 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 16:17:14 +0100 Subject: [PATCH 042/191] move all to a script --- .github/actions/dependencies/action.yml | 112 -------------- .github/actions/dependencies/detect_os.sh | 17 --- .../actions/dependencies/with_spack_env.sh | 11 -- .github/workflows/builds.yml | 6 +- CI/dependencies.sh | 103 ------------- .../actions => CI}/dependencies/opengl.sh | 11 +- .../actions => CI}/dependencies/parallel.sh | 0 CI/dependencies/setup.sh | 137 ++++++++++++++++++ 8 files changed, 144 insertions(+), 253 deletions(-) delete mode 100644 .github/actions/dependencies/action.yml delete mode 100644 .github/actions/dependencies/detect_os.sh delete mode 100755 .github/actions/dependencies/with_spack_env.sh delete mode 100755 CI/dependencies.sh rename {.github/actions => CI}/dependencies/opengl.sh (68%) rename {.github/actions => CI}/dependencies/parallel.sh (100%) create mode 100755 CI/dependencies/setup.sh diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml deleted file mode 100644 index faf6a2bb6e6..00000000000 --- a/.github/actions/dependencies/action.yml +++ /dev/null @@ -1,112 +0,0 @@ -name: "Dependencies action" -description: "Reusable action that sets up dependencies for the ACTS CI build" -inputs: - xcode-version: - description: "Which Xcode version to use (only for macOS)" - required: true - default: "16.0.0" - dependency-tag: - description: "Which dependency tag to use" - required: true - default: "v6-test3" -runs: - using: "composite" - steps: - - name: Set up Spack - uses: spack/setup-spack@v2 - - - name: Apply spack patch - shell: bash - working-directory: spack - run: | - git config user.name github-actions[bot] - git config user.email 41898282+github-actions[bot]@users.noreply.github.com - curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am - - - - name: Add spack buildcache mirror - shell: bash - run: | - spack mirror add acts-spack-buildcache oci://ghcr.io/acts-project/spack-buildcache --unsigned - - - name: Locate OpenGL - shell: bash - run: "${GITHUB_ACTION_PATH}/opengl.sh" - - - name: Get spack lock file name - shell: bash - run: | - arch=$(spack arch --family) - if [[ $arch == darwin* ]]; then - lock_file="spack-${arch}-xcode${{ inputs.xcode-version }}.lock" - lock_file_g4="spack-${arch}-xcode${{ inputs.xcode-version }}-geant4-data.lock" - else - lock_file="spack-${arch}.lock" - lock_file_g4="spack-${arch}-geant4-data.lock" - fi - - echo "Lock file: ${lock_file}" - echo "Lock file G4: ${lock_file_g4}" - echo "SPACK_LOCK_FILE=${lock_file}" >> "$GITHUB_ENV" - echo "SPACK_LOCK_FILE_G4=${lock_file_g4}" >> "$GITHUB_ENV" - - - name: Setup Xcode version - if: startsWith(runner.os, 'macos') - uses: maxim-lobanov/setup-xcode@v1 - with: - xcode-version: "${{ inputs.xcode-version }}" - - - name: Get spack lock files from tag - shell: bash - run: | - url="https://github.com/acts-project/ci-dependencies/releases/download/${{ inputs.dependency-tag }}/${SPACK_LOCK_FILE}" - echo "URL: ${url}" - curl -L -o spack.lock $url - g4url="https://github.com/acts-project/ci-dependencies/releases/download/${{ inputs.dependency-tag }}/${SPACK_LOCK_FILE_G4}" - echo "URL: ${g4url}" - curl -L -o spack-g4.lock $g4url - - - name: Create spack environment - shell: bash - run: | - NCPUS=32 # we're only downloading - - time spack env create ci spack.lock --with-view $PWD/dependencies & - pid1=$! - time spack env create ci-g4 spack-g4.lock & - pid2=$! - wait $pid1 - wait $pid2 - - time spack -e ci spec - time spack -e ci find - - time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci install --use-buildcache only - - time spack -e ci-g4 spec - time spack -e ci-g4 find - - time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci-g4 install - - "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci-g4 env | grep G4 >> "$GITHUB_ENV" - - - name: Configure downstream environment - shell: bash - run: | - echo "$PWD/dependencies/bin" >> "$GITHUB_PATH" - echo "CMAKE_PREFIX_PATH=$PWD/dependencies" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${destination}/lib" >> "$GITHUB_ENV" - echo "ROOT_INCLUDE_PATH=${destination}/include" >> "$GITHUB_ENV" - # # Geant4 puts CLHEP in a subdirectory - # set_env ROOT_INCLUDE_PATH "${destination}/include/Geant4" - # Pythia8 looks for settings in this directory - echo "PYTHIA8DATA=${destination}/share/Pythia8/xmldoc" >> "$GITHUB_ENV" - - - name: Prepare python environment - shell: bash - run: | - ls -al - venv_dir="$PWD/dependencies/venv" - python3 -m venv "$venv_dir" - echo "${venv_dir}/bin" >> "$GITHUB_PATH" - ls -al "${venv_dir}/bin" diff --git a/.github/actions/dependencies/detect_os.sh b/.github/actions/dependencies/detect_os.sh deleted file mode 100644 index b943539cbe1..00000000000 --- a/.github/actions/dependencies/detect_os.sh +++ /dev/null @@ -1,17 +0,0 @@ -if [ $(uname) == "Linux" ]; then - os_name=$(cat /etc/os-release | grep -e "^PRETTY_NAME=" | sed 's/PRETTY_NAME="\(.*\)"/\1/g') - if [[ $os_name == *"Ubuntu"* ]]; then - os="ubuntu" - elif [[ $os_name == *"AlmaLinux"* ]]; then - os="almalinux" - fi -elif [ $(uname) == "Darwin" ]; then - os_name="$(sw_vers -productName) $(sw_vers -productVersion)" - os="macos" -else - echo "Only Ubuntu, AlmaLinux and macOS are supported. Exiting." - exit 1 -fi - -export os -export os_name diff --git a/.github/actions/dependencies/with_spack_env.sh b/.github/actions/dependencies/with_spack_env.sh deleted file mode 100755 index ae0de83d4b4..00000000000 --- a/.github/actions/dependencies/with_spack_env.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -set -e -set -u - -source "$(spack location -r)/share/spack/setup-env.sh" - -env=$1 -shift -spack env activate "$env" - -"$@" diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index b2fd3d9d0dd..06fd96ba9ad 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -37,7 +37,7 @@ jobs: lfs: true - name: Install dependencies - uses: ./.github/actions/dependencies + uses: CI/dependencies/setup.sh - run: env - run: | @@ -273,7 +273,7 @@ jobs: lfs: true - name: Install dependencies - uses: ./.github/actions/dependencies + uses: CI/dependencies/setup.sh # # - name: Restore ccache # uses: actions/cache/restore@v4 @@ -352,7 +352,7 @@ jobs: # && CI/dependencies.sh - name: Install dependencies - uses: ./.github/actions/dependencies + uses: CI/dependencies/setup.sh # # - name: Restore ccache # uses: actions/cache/restore@v4 diff --git a/CI/dependencies.sh b/CI/dependencies.sh deleted file mode 100755 index 7c8879b499e..00000000000 --- a/CI/dependencies.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash - -function run() { - set -x - "$@" - { set +x; } 2> /dev/null -} - -function set_env { - key="$1" - value="$2" - - echo "=> ${key}=${value}" - - if [ -n "${GITHUB_ACTIONS:-}" ]; then - echo "${key}=${value}" >> $GITHUB_ENV - else - export ${key}=${value} - fi -} - -url=${1:-${DEPENDENCY_URL:-}} - -if [ -n "${GITHUB_ACTIONS:-}" ]; then - destination="${GITHUB_WORKSPACE}/dependencies" -elif [ -n "${GITLAB_CI:-}" ];then - destination="${CI_PROJECT_DIR}/dependencies" -else - destination=${2} -fi - -set_env DEPENDENCY_DIR "${destination}" - -if [ -z "${url}" ]; then - echo "url is not set" - exit 1 -fi - -echo "URL: $url" -echo "DESTINATION: $destination" - -# check curl location -CURL=$(command -v curl) -if [ -z "$CURL" ]; then - echo "curl is not available" - exit 1 -fi - -UNZSTD=$(command -v unzstd) -if [ -z "$UNZSTD" ]; then - echo "unzstd is not available" - exit 1 -fi - -TAR=$(command -v tar) -if [ -z "$TAR" ]; then - echo "tar is not available" - exit 1 -fi - -run mkdir -p "${destination}" - -run $CURL \ - --retry 5 \ - --connect-timeout 2 \ - --location $url \ - | unzstd \ - | tar \ - -x \ - --strip-components=1 \ - --directory "${destination}" - -# Patch up geant4-config data install script -out=$(${destination}/bin/geant4-config --datasets) -line=$(echo "$out" | head -n1) -orig_share=$(echo "$line" | perl -pe 's|.*?(\/.*)\/share.*|\1|') -orig_share_escaped=$(echo $orig_share|perl -pe 's|/|\\/|g') -destination_escaped=$(echo "$destination"|perl -pe 's|/|\\/|g') -perl -pi.bak -e "s/$orig_share_escaped/$destination_escaped/g" ${destination}/bin/geant4-config - -if [ -n "${GITHUB_ACTIONS:-}" ]; then - echo "Running in GitHub Actions" - venv="${GITHUB_WORKSPACE}/venv" -fi - -if [ -n "${GITLAB_CI:-}" ];then - echo "Running in GitLab CI" - venv="${CI_PROJECT_DIR}/venv" -fi - -if [ -n "${CI:-}" ];then - run "${destination}/bin/python3" -m venv "${venv}" - run "${venv}/bin/python3" -m pip install pyyaml jinja2 - set_env PATH "${venv}/bin:${destination}/bin/:${PATH}" -fi - -set_env CMAKE_PREFIX_PATH "${destination}" -set_env LD_LIBRARY_PATH "${destination}/lib" -set_env ROOT_INCLUDE_PATH "${destination}/include" -# Geant4 puts CLHEP in a subdirectory -set_env ROOT_INCLUDE_PATH "${destination}/include/Geant4" -# Pythia8 looks for settings in this directory -set_env PYTHIA8DATA "${destination}/share/Pythia8/xmldoc" diff --git a/.github/actions/dependencies/opengl.sh b/CI/dependencies/opengl.sh similarity index 68% rename from .github/actions/dependencies/opengl.sh rename to CI/dependencies/opengl.sh index bb6f68dc747..10e3eda6544 100755 --- a/.github/actions/dependencies/opengl.sh +++ b/CI/dependencies/opengl.sh @@ -3,11 +3,7 @@ set -u set -e -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - -source ${SCRIPT_DIR}/detect_os.sh - -packages_file=${GITHUB_WORKSPACE}/spack/etc/spack/packages.yaml +packages_file=$(spack location -r)/spack/etc/spack/packages.yaml if ! command -v sudo &> /dev/null then @@ -16,8 +12,9 @@ else SUDO="sudo" fi +os=$(spack arch --platform) -if [ "$os" == "ubuntu" ]; then +if [ "$os" == ubuntu* ]; then ${SUDO} apt-get update ${SUDO} apt-get install -y libgl1-mesa-dev cat < "$packages_file" @@ -29,7 +26,7 @@ packages: spec: opengl@4.5 EOF cat "$packages_file" -elif [ "$os" == "almalinux" ]; then +elif [ "$os" == almalinux* ]; then ${SUDO} dnf install -y mesa-libGLU cat < "$packages_file" packages: diff --git a/.github/actions/dependencies/parallel.sh b/CI/dependencies/parallel.sh similarity index 100% rename from .github/actions/dependencies/parallel.sh rename to CI/dependencies/parallel.sh diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh new file mode 100755 index 00000000000..f0f0a14b5f6 --- /dev/null +++ b/CI/dependencies/setup.sh @@ -0,0 +1,137 @@ +#!/bin/bash +set -e +set -u + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +export SPACK_COLOR=always + +function set_env { + key="$1" + value="$2" + + echo "=> ${key}=${value}" + + if [ -n "${GITHUB_ACTIONS:-}" ]; then + echo "${key}=${value}" >> "$GITHUB_ENV" + else + export "${key}"="${value}" + fi +} + +function start_section() { + local section_name="$1" + if [ -n "${GITHUB_ACTIONS:-}" ]; then + echo "::group::${section_name}" + else + echo "${section_name}" + fi +} + +function end_section() { + if [ -n "${GITHUB_ACTIONS:-}" ]; then + echo "::endgroup::" + fi +} + +tag=${1:-${DEPENDENCY_TAG}} +if [ -n "${GITHUB_ACTIONS:-}" ]; then + destination="${GITHUB_WORKSPACE}/dependencies" +elif [ -n "${GITLAB_CI:-}" ];then + destination="${CI_PROJECT_DIR}/dependencies" +else + destination=${2} +fi + +echo "Install tag: $tag" +echo "Install destination: $destination" + +mkdir -p ${destination} + + +start_section "Install spack if not already installed" +if ! command -v spack &> /dev/null; then + git clone -c feature.manyFiles=true https://github.com/spack/spack.git + pushd spack > /dev/null + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + # Apply patch for spack improvements + curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am + popd > /dev/null +fi +end_section + +if [ -n "${CI:-}" ]; then +start_section "Add buildcache mirror" +mirror_name="acts-spack-buildcache" +mirror_url="oci://ghcr.io/acts-project/spack-buildcache" +spack mirror add ${mirror_name} ${mirror_url} --unsigned +end_section + +start_section "Locate OpenGL" +"${SCRIPT_DIR}/opengl.sh" +end_section +fi + +start_section "Get spack lock file name" +arch=$(spack arch --family) +lock_file="spack-${arch}.lock" +end_section + + +url="https://github.com/acts-project/ci-dependencies/releases/download/${tag}/${lock_file}" +echo "URL: $url" + +env_dir="${destination}/env" +view_dir="${destination}/view" +mkdir -p ${env_dir} +lock_file_path="${destination}/spack.lock" +start_section "Get spack lock files from tag" +curl -fL -o $lock_file_path $url +end_section + + + +start_section "Create spack environment" +time spack env create -d ${env_dir} ${lock_file_path} --with-view $view_dir +time spack -e ${env_dir} spec +time spack -e ${env_dir} find +end_section + +start_section "Install spack packages" +NCPUS=32 +time ${SCRIPT_DIR}/parallel.sh $NCPUS spack -e ${env_dir} install --use-buildcache only | tee install.log +end_section + +start_section "Patch up Geant4 data directory" +# ${SCRIPT_DIR}/with_spack_env.sh ${env_dir} geant4-config --install-datasets +geant4_dir=$(spack -e ${env_dir} location -i geant4) +# Prepare the folder for G4 data, and symlink it to where G4 will look for it +mkdir -p ${geant4_dir}/share/Geant4 +ln -s ${geant4_dir}/share/Geant4/data ${view_dir}/share/Geant4/data +end_section + +start_section "Set environment variables" +set_env CMAKE_PREFIX_PATH "${view_dir}" +set_env LD_LIBRARY_PATH "${view_dir}/lib" +set_env ROOT_INCLUDE_PATH "${view_dir}/include" +# Geant4 puts CLHEP in a subdirectory +set_env ROOT_INCLUDE_PATH "${view_dir}/include/Geant4" +end_section + +start_section "Prepare python environment" +ls -al +venv_dir="${view_dir}/venv" +python3 -m venv "$venv_dir" + +if [ -n "${GITHUB_ACTIONS:-}" ]; then + echo "${venv_dir}/bin" >> "$GITHUB_PATH" + echo "${view_dir}/bin" >> "$GITHUB_PATH" +fi +set_env PATH "${venv_dir}/bin:${view_dir}/bin/:${PATH}" +end_section + + + +# Pythia8 looks for settings in this directory +# set_env PYTHIA8DATA "${destination}/share/Pythia8/xmldoc" From 1e7b4fe19a943eb7a9ff14512743087f5068d977 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 16:18:44 +0100 Subject: [PATCH 043/191] run --- .github/workflows/builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 06fd96ba9ad..debe1878670 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -37,7 +37,7 @@ jobs: lfs: true - name: Install dependencies - uses: CI/dependencies/setup.sh + run: CI/dependencies/setup.sh - run: env - run: | From 4e9e43c4d396d07d40118a15fca9447efa3dab2a Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 16:19:26 +0100 Subject: [PATCH 044/191] all --- .github/workflows/builds.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index debe1878670..acabc98dacf 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -273,7 +273,7 @@ jobs: lfs: true - name: Install dependencies - uses: CI/dependencies/setup.sh + run: CI/dependencies/setup.sh # # - name: Restore ccache # uses: actions/cache/restore@v4 @@ -352,7 +352,7 @@ jobs: # && CI/dependencies.sh - name: Install dependencies - uses: CI/dependencies/setup.sh + run: CI/dependencies/setup.sh # # - name: Restore ccache # uses: actions/cache/restore@v4 From c87a1dcfcdcc223c6f845c882db26374c98717fb Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 16:21:52 +0100 Subject: [PATCH 045/191] set it up if we just installed it --- CI/dependencies/setup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index f0f0a14b5f6..65e825a3ddf 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -57,6 +57,7 @@ if ! command -v spack &> /dev/null; then git config user.email 41898282+github-actions[bot]@users.noreply.github.com # Apply patch for spack improvements curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am + source "$(pwd)/share/spack/setup-env.sh" popd > /dev/null fi end_section From d402b93721a423519403dd8738840cfd004b9bb0 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 16:23:48 +0100 Subject: [PATCH 046/191] correct tag --- .github/workflows/builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index acabc98dacf..42104415706 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -20,7 +20,7 @@ env: CCACHE_DIR: ${{ github.workspace }}/ccache CCACHE_MAXSIZE: 500M CCACHE_KEY_SUFFIX: r2 - DEPENDENCY_TAG: v5 + DEPENDENCY_TAG: v6-test3 jobs: linux_ubuntu: From b83fe5a907816cf466daedbbedad719cca926e4d Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 16:28:02 +0100 Subject: [PATCH 047/191] diagnostics --- .github/workflows/builds.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 42104415706..630f0004b57 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -48,8 +48,9 @@ jobs: echo $PATH ls -al ls -al dependencies/ - ls -al dependencies/venv - ls -al dependencies/venv/bin + ls -al dependencies/view + ls -al dependencies/view/venv + ls -al dependencies/view/venv/bin which python3 python3 -m pip install jinja2 pyyaml From b0cbe89ea15a716739fb22e7cf0bd3aa4cab715e Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 16:28:42 +0100 Subject: [PATCH 048/191] use the right python --- CI/dependencies/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index 65e825a3ddf..1bc4016d671 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -123,7 +123,7 @@ end_section start_section "Prepare python environment" ls -al venv_dir="${view_dir}/venv" -python3 -m venv "$venv_dir" +${view_dir}/bin/python3 -m venv "$venv_dir" if [ -n "${GITHUB_ACTIONS:-}" ]; then echo "${venv_dir}/bin" >> "$GITHUB_PATH" From 8ca7c8d2886d93bd8530f7cb044cecf5a504e287 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 16:35:48 +0100 Subject: [PATCH 049/191] diag --- .github/workflows/builds.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 630f0004b57..766cdf71502 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -40,10 +40,6 @@ jobs: run: CI/dependencies/setup.sh - run: env - - run: | - pwd - ls -al - ls -al /__w/acts/acts/dependencies/venv/bin/ - run: | echo $PATH ls -al From dd83cfbe955f484af2bee61d0a3d6c024092d8be Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 16:40:54 +0100 Subject: [PATCH 050/191] output filter --- CI/dependencies/setup.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index 1bc4016d671..fd106289b7e 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -101,7 +101,9 @@ end_section start_section "Install spack packages" NCPUS=32 -time ${SCRIPT_DIR}/parallel.sh $NCPUS spack -e ${env_dir} install --use-buildcache only | tee install.log +time ${SCRIPT_DIR}/parallel.sh $NCPUS spack -e ${env_dir} install --use-buildcache only \ + | tee install.log \ + | grep -v "^Waiting\|^\[+\]" end_section start_section "Patch up Geant4 data directory" From 13975e32b0404551070d772fef523c07208f9fda Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 16:55:23 +0100 Subject: [PATCH 051/191] limit parallel --- CI/dependencies/setup.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index fd106289b7e..787ef237018 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -100,7 +100,11 @@ time spack -e ${env_dir} find end_section start_section "Install spack packages" -NCPUS=32 +if [ "$(uname)" = "Darwin" ]; then + NCPUS=$(sysctl -n hw.ncpu) +else + NCPUS=$(nproc) +fi time ${SCRIPT_DIR}/parallel.sh $NCPUS spack -e ${env_dir} install --use-buildcache only \ | tee install.log \ | grep -v "^Waiting\|^\[+\]" From 2b4fc825c8e7cfd30eab5bd934778f949f266fae Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 17:10:37 +0100 Subject: [PATCH 052/191] update tag --- .github/workflows/builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 766cdf71502..38b990013f2 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -20,7 +20,7 @@ env: CCACHE_DIR: ${{ github.workspace }}/ccache CCACHE_MAXSIZE: 500M CCACHE_KEY_SUFFIX: r2 - DEPENDENCY_TAG: v6-test3 + DEPENDENCY_TAG: v7_b1 jobs: linux_ubuntu: From f9eab1f4a502b999ec1c789a838adee114e1d60d Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 18:02:34 +0100 Subject: [PATCH 053/191] quotes --- CI/dependencies/setup.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index 787ef237018..c389ce5f8c2 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -94,9 +94,9 @@ end_section start_section "Create spack environment" -time spack env create -d ${env_dir} ${lock_file_path} --with-view $view_dir -time spack -e ${env_dir} spec -time spack -e ${env_dir} find +time spack env create -d "${env_dir}" "${lock_file_path}" --with-view "$view_dir" +time spack -e "${env_dir}" spec +time spack -e "${env_dir}" find end_section start_section "Install spack packages" @@ -105,17 +105,17 @@ if [ "$(uname)" = "Darwin" ]; then else NCPUS=$(nproc) fi -time ${SCRIPT_DIR}/parallel.sh $NCPUS spack -e ${env_dir} install --use-buildcache only \ +time "${SCRIPT_DIR}"/parallel.sh "$NCPUS" spack -e "${env_dir}" install --use-buildcache only \ | tee install.log \ | grep -v "^Waiting\|^\[+\]" end_section start_section "Patch up Geant4 data directory" # ${SCRIPT_DIR}/with_spack_env.sh ${env_dir} geant4-config --install-datasets -geant4_dir=$(spack -e ${env_dir} location -i geant4) +geant4_dir=$(spack -e "${env_dir}" location -i geant4) # Prepare the folder for G4 data, and symlink it to where G4 will look for it -mkdir -p ${geant4_dir}/share/Geant4 -ln -s ${geant4_dir}/share/Geant4/data ${view_dir}/share/Geant4/data +mkdir -p "${geant4_dir}"/share/Geant4 +ln -s "${geant4_dir}"/share/Geant4/data ${view_dir}/share/Geant4/data end_section start_section "Set environment variables" @@ -129,7 +129,7 @@ end_section start_section "Prepare python environment" ls -al venv_dir="${view_dir}/venv" -${view_dir}/bin/python3 -m venv "$venv_dir" +"${view_dir}"/bin/python3 -m venv "$venv_dir" if [ -n "${GITHUB_ACTIONS:-}" ]; then echo "${venv_dir}/bin" >> "$GITHUB_PATH" From c9b8a2611ea6716d18c3cd5ee1b88dfc19b9df7f Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 18:02:41 +0100 Subject: [PATCH 054/191] long spec output --- CI/dependencies/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index c389ce5f8c2..372d1e687dd 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -95,7 +95,7 @@ end_section start_section "Create spack environment" time spack env create -d "${env_dir}" "${lock_file_path}" --with-view "$view_dir" -time spack -e "${env_dir}" spec +time spack -e "${env_dir}" spec -l time spack -e "${env_dir}" find end_section From 0b17997f520d536fbd92b689441434379edde53d Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 18:04:45 +0100 Subject: [PATCH 055/191] see if the other builds work --- .github/workflows/builds.yml | 263 +++++++++++++++++------------------ 1 file changed, 124 insertions(+), 139 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 38b990013f2..98365cbeedb 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -39,17 +39,6 @@ jobs: - name: Install dependencies run: CI/dependencies/setup.sh - - run: env - - run: | - echo $PATH - ls -al - ls -al dependencies/ - ls -al dependencies/view - ls -al dependencies/view/venv - ls -al dependencies/view/venv/bin - which python3 - python3 -m pip install jinja2 pyyaml - - name: Restore ccache id: ccache-restore uses: actions/cache/restore@v4 @@ -271,66 +260,66 @@ jobs: - name: Install dependencies run: CI/dependencies/setup.sh - # - # - name: Restore ccache - # uses: actions/cache/restore@v4 - # id: ccache-restore - # with: - # path: ${{ env.CCACHE_DIR }} - # key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} - # restore-keys: | - # ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - # - # - # - name: Configure - # # setting CMAKE_CXX_STANDARD=20 is a workaround for a bug in the - # # dd4hep CMake configuration that gets triggered on recent CMake - # # versions - # run: > - # ccache -z && - # cmake -B build -S . - # --preset=github-ci - # -DCMAKE_CXX_STANDARD=${{ matrix.std }} - # -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" - # -DPython_EXECUTABLE=$(which python3) - # - # - name: Build - # run: cmake --build build - # - # - name: ccache stats - # run: ccache -s - # - # - name: Save ccache - # uses: actions/cache/save@v4 - # if: always() - # with: - # path: ${{ github.workspace }}/ccache - # key: ${{ steps.ccache-restore.outputs.cache-primary-key }} - # - # - name: Unit tests - # run: cmake --build build --target test - # - # - name: Integration tests - # run: cmake --build build --target integrationtests - # - # - name: Install - # run: cmake --build build --target install - # - # - name: Downstream configure - # run: > - # cmake -B build-downstream -S Tests/DownstreamProject - # -GNinja - # -DCMAKE_BUILD_TYPE=Release - # -DCMAKE_CXX_FLAGS=-Werror - # -DCMAKE_CXX_STANDARD=${{ matrix.std }} - # -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" - # - # - name: Downstream build - # run: cmake --build build-downstream - # - # - name: Downstream run - # run: ./build-downstream/bin/ShowActsVersion - # + + - name: Restore ccache + uses: actions/cache/restore@v4 + id: ccache-restore + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} + restore-keys: | + ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- + + + - name: Configure + # setting CMAKE_CXX_STANDARD=20 is a workaround for a bug in the + # dd4hep CMake configuration that gets triggered on recent CMake + # versions + run: > + ccache -z && + cmake -B build -S . + --preset=github-ci + -DCMAKE_CXX_STANDARD=${{ matrix.std }} + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + -DPython_EXECUTABLE=$(which python3) + + - name: Build + run: cmake --build build + + - name: ccache stats + run: ccache -s + + - name: Save ccache + uses: actions/cache/save@v4 + if: always() + with: + path: ${{ github.workspace }}/ccache + key: ${{ steps.ccache-restore.outputs.cache-primary-key }} + + - name: Unit tests + run: cmake --build build --target test + + - name: Integration tests + run: cmake --build build --target integrationtests + + - name: Install + run: cmake --build build --target install + + - name: Downstream configure + run: > + cmake -B build-downstream -S Tests/DownstreamProject + -GNinja + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_CXX_FLAGS=-Werror + -DCMAKE_CXX_STANDARD=${{ matrix.std }} + -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" + + - name: Downstream build + run: cmake --build build-downstream + + - name: Downstream run + run: ./build-downstream/bin/ShowActsVersion + macos: runs-on: macos-14 env: @@ -343,72 +332,68 @@ jobs: submodules: true lfs: true - # - name: Install dependencies - # run: > - # brew install cmake ninja ccache xerces-c - # && CI/dependencies.sh - - name: Install dependencies - run: CI/dependencies/setup.sh - # - # - name: Restore ccache - # uses: actions/cache/restore@v4 - # id: ccache-restore - # with: - # path: ${{ env.CCACHE_DIR }} - # key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} - # restore-keys: | - # ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - # - # - name: Configure - # run: > - # ccache -z - # && cmake -B build -S . - # --preset=github-ci - # -DCMAKE_PREFIX_PATH="${{ env.DEPENDENCY_DIR }}" - # -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" - # -DPython_EXECUTABLE=$(which python3) - # - # - name: Build - # run: cmake --build build - # - # - name: ccache stats - # run: ccache -s - # - # - name: Save ccache - # uses: actions/cache/save@v4 - # if: always() - # with: - # path: ${{ github.workspace }}/ccache - # key: ${{ steps.ccache-restore.outputs.cache-primary-key }} - # - # - name: Unit tests - # run: cmake --build build --target test - # - # - name: Integration tests - # run: cmake --build build --target integrationtests - # - # - name: Install - # run: cmake --build build --target install - # - # - uses: actions/upload-artifact@v4 - # with: - # name: acts-macos - # path: ${{ env.INSTALL_DIR }} - # - # - name: Downstream configure - # run: > - # cmake -B build-downstream -S Tests/DownstreamProject - # -GNinja - # -DCMAKE_BUILD_TYPE=Release - # -DCMAKE_CXX_FLAGS=-Werror - # -DCMAKE_CXX_STANDARD=20 - # -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" - # - # - name: Downstream build - # run: cmake --build build-downstream - # - # - name: Downstream run - # run: > - # PATH="${{ env.DEPENDENCY_DIR }}/bin:$PATH" - # && ./build-downstream/bin/ShowActsVersion + run: > + brew install ninja ccache + && CI/dependencies/setup.sh + + - name: Restore ccache + uses: actions/cache/restore@v4 + id: ccache-restore + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} + restore-keys: | + ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- + + - name: Configure + run: > + ccache -z + && cmake -B build -S . + --preset=github-ci + -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" + -DPython_EXECUTABLE=$(which python3) + + - name: Build + run: cmake --build build + + - name: ccache stats + run: ccache -s + + - name: Save ccache + uses: actions/cache/save@v4 + if: always() + with: + path: ${{ github.workspace }}/ccache + key: ${{ steps.ccache-restore.outputs.cache-primary-key }} + + - name: Unit tests + run: cmake --build build --target test + + - name: Integration tests + run: cmake --build build --target integrationtests + + - name: Install + run: cmake --build build --target install + + - uses: actions/upload-artifact@v4 + with: + name: acts-macos + path: ${{ env.INSTALL_DIR }} + + - name: Downstream configure + run: > + cmake -B build-downstream -S Tests/DownstreamProject + -GNinja + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_CXX_FLAGS=-Werror + -DCMAKE_CXX_STANDARD=20 + -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" + + - name: Downstream build + run: cmake --build build-downstream + + - name: Downstream run + run: > + PATH="${{ env.DEPENDENCY_DIR }}/bin:$PATH" + && ./build-downstream/bin/ShowActsVersion From 8403c02d230fb6229676a4322f87d83f0c09fb22 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 18:49:12 +0100 Subject: [PATCH 056/191] fix: Use toUnderlying to avoid cross-enum comparison --- Core/include/Acts/EventData/TrackStateProxy.hpp | 3 ++- Core/include/Acts/EventData/VectorMultiTrajectory.hpp | 3 ++- .../include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Core/include/Acts/EventData/TrackStateProxy.hpp b/Core/include/Acts/EventData/TrackStateProxy.hpp index 4338c434f77..9e7f8f57392 100644 --- a/Core/include/Acts/EventData/TrackStateProxy.hpp +++ b/Core/include/Acts/EventData/TrackStateProxy.hpp @@ -783,7 +783,8 @@ class TrackStateProxy { void allocateCalibrated(const Eigen::DenseBase& val, const Eigen::DenseBase& cov) requires(Eigen::PlainObjectBase::RowsAtCompileTime > 0 && - Eigen::PlainObjectBase::RowsAtCompileTime <= eBoundSize && + Eigen::PlainObjectBase::RowsAtCompileTime <= + toUnderlying(eBoundSize) && Eigen::PlainObjectBase::RowsAtCompileTime == Eigen::PlainObjectBase::RowsAtCompileTime && Eigen::PlainObjectBase::RowsAtCompileTime == diff --git a/Core/include/Acts/EventData/VectorMultiTrajectory.hpp b/Core/include/Acts/EventData/VectorMultiTrajectory.hpp index e13498cea2c..e7d1e77a711 100644 --- a/Core/include/Acts/EventData/VectorMultiTrajectory.hpp +++ b/Core/include/Acts/EventData/VectorMultiTrajectory.hpp @@ -497,7 +497,8 @@ class VectorMultiTrajectory final const Eigen::DenseBase& cov) requires(Eigen::PlainObjectBase::RowsAtCompileTime > 0 && - Eigen::PlainObjectBase::RowsAtCompileTime <= eBoundSize && + Eigen::PlainObjectBase::RowsAtCompileTime <= + toUnderlying(eBoundSize) && Eigen::PlainObjectBase::RowsAtCompileTime == Eigen::PlainObjectBase::RowsAtCompileTime && Eigen::PlainObjectBase::RowsAtCompileTime == diff --git a/Plugins/Podio/include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp b/Plugins/Podio/include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp index 53eebf0e9b4..0f66b9c1c4b 100644 --- a/Plugins/Podio/include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp +++ b/Plugins/Podio/include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp @@ -621,7 +621,8 @@ class MutablePodioTrackStateContainer final const Eigen::DenseBase& cov) requires(Eigen::PlainObjectBase::RowsAtCompileTime > 0 && - Eigen::PlainObjectBase::RowsAtCompileTime <= eBoundSize && + Eigen::PlainObjectBase::RowsAtCompileTime <= + toUnderlying(eBoundSize) && Eigen::PlainObjectBase::RowsAtCompileTime == Eigen::PlainObjectBase::RowsAtCompileTime && Eigen::PlainObjectBase::RowsAtCompileTime == From ac6c351d11ce756023c5a92c701a977665cba25a Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 20:01:20 +0100 Subject: [PATCH 057/191] Revert "fix: Use toUnderlying to avoid cross-enum comparison" This reverts commit 197c57de389c623aba42c9f4960500b332904825. --- .../include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Plugins/Podio/include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp b/Plugins/Podio/include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp index 0f66b9c1c4b..53eebf0e9b4 100644 --- a/Plugins/Podio/include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp +++ b/Plugins/Podio/include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp @@ -621,8 +621,7 @@ class MutablePodioTrackStateContainer final const Eigen::DenseBase& cov) requires(Eigen::PlainObjectBase::RowsAtCompileTime > 0 && - Eigen::PlainObjectBase::RowsAtCompileTime <= - toUnderlying(eBoundSize) && + Eigen::PlainObjectBase::RowsAtCompileTime <= eBoundSize && Eigen::PlainObjectBase::RowsAtCompileTime == Eigen::PlainObjectBase::RowsAtCompileTime && Eigen::PlainObjectBase::RowsAtCompileTime == From fc6f99bf2b972495f73bce07c23d8424c9ae29d1 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 20:03:14 +0100 Subject: [PATCH 058/191] update concepts with toUnderlying --- Core/include/Acts/Utilities/EigenConcepts.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Core/include/Acts/Utilities/EigenConcepts.hpp b/Core/include/Acts/Utilities/EigenConcepts.hpp index 4151dd6e237..77718dda92f 100644 --- a/Core/include/Acts/Utilities/EigenConcepts.hpp +++ b/Core/include/Acts/Utilities/EigenConcepts.hpp @@ -37,16 +37,16 @@ concept eigen_base_is_square = eigen_base_is_fixed_size && template concept eigen_bases_have_same_num_rows = eigen_base_is_fixed_size && eigen_base_is_fixed_size && - static_cast(Eigen::PlainObjectBase::RowsAtCompileTime) == - static_cast(Eigen::PlainObjectBase::RowsAtCompileTime); + toUnderlying(Eigen::PlainObjectBase::RowsAtCompileTime) == + toUnderlying(Eigen::PlainObjectBase::RowsAtCompileTime); /// @brief Concept that is true iff T1 and T2 have the same, known at compile /// time, number of columns. template concept eigen_bases_have_same_num_cols = eigen_base_is_fixed_size && eigen_base_is_fixed_size && - static_cast(Eigen::PlainObjectBase::ColsAtCompileTime) == - static_cast(Eigen::PlainObjectBase::ColsAtCompileTime); + toUnderlying(Eigen::PlainObjectBase::ColsAtCompileTime) == + toUnderlying(Eigen::PlainObjectBase::ColsAtCompileTime); /// @brief Concept that is true iff T1 and T2 have the same, known at compile /// time, size. From 8ae3da1291fefc65ec9fa2d347aa116cf6285bae Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Sat, 14 Dec 2024 12:29:58 +0100 Subject: [PATCH 059/191] another compile fix --- Core/include/Acts/EventData/VectorMultiTrajectory.hpp | 2 +- .../Acts/Plugins/Podio/PodioTrackStateContainer.hpp | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Core/include/Acts/EventData/VectorMultiTrajectory.hpp b/Core/include/Acts/EventData/VectorMultiTrajectory.hpp index 8f416dd6969..6c9fe454aea 100644 --- a/Core/include/Acts/EventData/VectorMultiTrajectory.hpp +++ b/Core/include/Acts/EventData/VectorMultiTrajectory.hpp @@ -500,7 +500,7 @@ class VectorMultiTrajectory final Concepts::eigen_bases_have_same_num_rows && Concepts::eigen_base_is_square && Eigen::PlainObjectBase::RowsAtCompileTime <= - static_cast>(eBoundSize)) + toUnderlying(eBoundSize)) { constexpr std::size_t measdim = val_t::RowsAtCompileTime; diff --git a/Plugins/Podio/include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp b/Plugins/Podio/include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp index 53eebf0e9b4..af1a2a88412 100644 --- a/Plugins/Podio/include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp +++ b/Plugins/Podio/include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp @@ -620,12 +620,11 @@ class MutablePodioTrackStateContainer final const Eigen::DenseBase& val, const Eigen::DenseBase& cov) - requires(Eigen::PlainObjectBase::RowsAtCompileTime > 0 && - Eigen::PlainObjectBase::RowsAtCompileTime <= eBoundSize && - Eigen::PlainObjectBase::RowsAtCompileTime == - Eigen::PlainObjectBase::RowsAtCompileTime && - Eigen::PlainObjectBase::RowsAtCompileTime == - Eigen::PlainObjectBase::ColsAtCompileTime) + requires(Concepts::eigen_base_is_fixed_size && + Concepts::eigen_bases_have_same_num_rows && + Concepts::eigen_base_is_square && + Eigen::PlainObjectBase::RowsAtCompileTime <= + toUnderlying(eBoundSize)) { constexpr std::size_t measdim = val_t::RowsAtCompileTime; From 763798e6fae203d45b7e418bb76425f0a82843cc Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Sun, 15 Dec 2024 14:44:54 +0100 Subject: [PATCH 060/191] be smarter about selecting a compiler --- .github/workflows/builds.yml | 16 ++- CI/dependencies/select_lockfile.py | 175 +++++++++++++++++++++++++++++ CI/dependencies/setup.sh | 99 +++++++++++++--- 3 files changed, 272 insertions(+), 18 deletions(-) create mode 100755 CI/dependencies/select_lockfile.py diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 98365cbeedb..c66730b3ff0 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -20,7 +20,7 @@ env: CCACHE_DIR: ${{ github.workspace }}/ccache CCACHE_MAXSIZE: 500M CCACHE_KEY_SUFFIX: r2 - DEPENDENCY_TAG: v7_b1 + DEPENDENCY_TAG: v7_b4 jobs: linux_ubuntu: @@ -37,6 +37,8 @@ jobs: lfs: true - name: Install dependencies + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: CI/dependencies/setup.sh - name: Restore ccache @@ -245,9 +247,11 @@ jobs: include: - image: ubuntu2204 std: 20 - - image: ubuntu2204_clang + compiler: g++ + - image: ubuntu2204 std: 20 - container: ghcr.io/acts-project/${{ matrix.image }}:63 + compiler: clang++ + container: ghcr.io/acts-project/${{ matrix.image }}:71 env: INSTALL_DIR: ${{ github.workspace }}/install ACTS_LOG_FAILURE_THRESHOLD: WARNING @@ -259,7 +263,9 @@ jobs: lfs: true - name: Install dependencies - run: CI/dependencies/setup.sh + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: CI/dependencies/setup.sh -c ${{ matrix.compiler }} - name: Restore ccache uses: actions/cache/restore@v4 @@ -333,6 +339,8 @@ jobs: lfs: true - name: Install dependencies + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: > brew install ninja ccache && CI/dependencies/setup.sh diff --git a/CI/dependencies/select_lockfile.py b/CI/dependencies/select_lockfile.py new file mode 100755 index 00000000000..e23df66f360 --- /dev/null +++ b/CI/dependencies/select_lockfile.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 + +import os +import argparse +import json +import urllib.request +import urllib.error +import re +import subprocess +from typing import Tuple, Dict + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--tag", type=str, required=True, help="Tag to use") + parser.add_argument("--arch", type=str, required=True, help="Architecture to use") + parser.add_argument( + "--compiler-binary", + type=str, + default=os.environ.get("CXX"), + help="Compiler to use (defaults to CXX environment variable if set)", + ) + parser.add_argument( + "--compiler", + type=str, + default=None, + help="Compiler to use (defaults to compiler binary if set)", + ) + parser.add_argument( + "--output", + type=str, + default=None, + help="Output file to write lockfile to", + ) + args = parser.parse_args() + + print("Fetching lockfiles for tag:", args.tag) + print("Architecture:", args.arch) + if args.compiler_binary is not None: + compiler = determine_compiler_version(args.compiler_binary) + print("Compiler:", args.compiler_binary, f"{compiler}") + elif args.compiler is not None: + if not re.match(r"^([\w-]+)@(\d+\.\d+\.\d+)$", args.compiler): + print(f"Invalid compiler format: {args.compiler}") + exit(1) + compiler = args.compiler + print("Compiler:", f"{compiler}") + else: + print("No compiler specified") + exit(1) + + base_url = f"https://api.github.com/repos/acts-project/ci-dependencies/releases/tags/{args.tag}" + + data = json.loads(fetch_github(base_url)) + + lockfiles = parse_assets(data) + + print("Available lockfiles:") + for arch, compilers in lockfiles.items(): + print(f"> {arch}:") + for c, (n, _) in compilers.items(): + print(f" - {c}: {n}") + + if args.arch not in lockfiles: + print(f"No lockfile found for architecture {args.arch}") + exit(1) + + lockfile = select_lockfile(lockfiles, args.arch, compiler) + + print("Selected lockfile:", lockfile) + + if args.output: + with open(args.output, "wb") as f: + f.write(fetch_github(lockfile)) + + +def parse_assets(data: Dict) -> Dict[str, Dict[str, Tuple[str, str]]]: + lockfiles: Dict[str, Dict[str, Tuple[str, str]]] = {} + + for asset in data["assets"]: + url = asset["browser_download_url"] + + name = asset["name"] + if not name.endswith(".lock") or not name.startswith("spack_"): + continue + + m = re.match(r"spack_(.*(?:aarch64|x86_64))(?:_(.*))?\.lock", name) + if m is None: + continue + + arch, compiler = m.groups() + compiler = compiler if compiler else "default" + lockfiles.setdefault(arch, {})[compiler] = (name, url) + + return lockfiles + + +def select_lockfile( + lockfiles: Dict[str, Dict[str, Tuple[str, str]]], arch: str, compiler: str +): + # Default to the default lockfile + _, lockfile = lockfiles[arch]["default"] + + # Extract compiler family and version + compiler_family = compiler.split("@")[0] + + # Find all matching compiler families + matching_compilers = { + comp: ver + for comp, ver in lockfiles[arch].items() + if comp != "default" and comp.split("@")[0] == compiler_family + } + + if matching_compilers: + if compiler in matching_compilers: + # Exact match found + _, lockfile = matching_compilers[compiler] + else: + # Find highest version of same compiler family + highest_version = max( + matching_compilers.keys(), + key=lambda x: [int(v) for v in x.split("@")[1].split(".")], + ) + _, lockfile = matching_compilers[highest_version] + + return lockfile + + +def determine_compiler_version(binary: str): + try: + result = subprocess.run([binary, "--version"], capture_output=True, text=True) + + line = result.stdout.split("\n", 1)[0] + print(line) + if "clang" in line: + compiler = "clang" + if "Apple" in line: + compiler = "apple-clang" + elif "gcc" in line or "GCC" in line: + compiler = "gcc" + else: + print(f"Unknown compiler: {binary}") + exit(1) + + m = re.search(r"(\d+\.\d+\.\d+)", line) + if m is None: + print(f"Failed to determine version for compiler: {binary}") + exit(1) + (version,) = m.groups() + return f"{compiler}@{version}" + + except (subprocess.SubprocessError, FileNotFoundError): + print(f"Failed to determine version for compiler: {binary}") + exit(1) + + +def fetch_github(base_url): + headers = {} + if "GITHUB_TOKEN" in os.environ: + headers["Authorization"] = f"token {os.environ['GITHUB_TOKEN']}" + + try: + req = urllib.request.Request(base_url, headers=headers) + with urllib.request.urlopen(req) as response: + return response.read() + except urllib.error.URLError as e: + print(f"Failed to fetch from {base_url}: {e}") + exit(1) + except json.JSONDecodeError as e: + print(f"Failed to parse JSON response: {e}") + exit(1) + + +if __name__ == "__main__": + main() diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index 372d1e687dd..18e979d00ef 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -34,15 +34,88 @@ function end_section() { fi } -tag=${1:-${DEPENDENCY_TAG}} -if [ -n "${GITHUB_ACTIONS:-}" ]; then +# if [ -n "${1:-${DEPENDENCY_TAG:-}}" ]; then +# tag=${1:-${DEPENDENCY_TAG}} +# else +# echo "Usage: $0 " +# exit 1 +# fi + +# if [ -n "${GITHUB_ACTIONS:-}" ]; then +# destination="${GITHUB_WORKSPACE}/dependencies" +# elif [ -n "${GITLAB_CI:-}" ];then +# destination="${CI_PROJECT_DIR}/dependencies" +# else +# if [ -n "${2:-}" ]; then +# destination=${2} +# else +# echo "Usage: $0 " +# exit 1 +# fi +# fi + +# Parse command line arguments +while getopts "c:t:d:h" opt; do + case ${opt} in + c ) + compiler=$OPTARG + ;; + t ) + tag=$OPTARG + ;; + d ) + destination=$OPTARG + ;; + h ) + echo "Usage: $0 [-c compiler] [-t tag] [-d destination]" + echo "Options:" + echo " -c Specify compiler (defaults to CXX env var)" + echo " -t Specify dependency tag (defaults to DEPENDENCY_TAG env var)" + echo " -d Specify install destination (defaults based on CI environment)" + echo " -h Show this help message" + exit 0 + ;; + \? ) + echo "Invalid option: -$OPTARG" 1>&2 + exit 1 + ;; + : ) + echo "Option -$OPTARG requires an argument" 1>&2 + exit 1 + ;; + esac +done + +# Set defaults if not specified +if [ -z "${compiler:-}" ]; then + compiler="${CXX:-}" + if [ -z "${compiler:-}" ]; then + echo "No compiler specified via -c or CXX environment variable" + exit 1 + fi +fi + +if [ -z "${tag:-}" ]; then + tag="${DEPENDENCY_TAG:-}" + if [ -z "${tag:-}" ]; then + echo "No tag specified via -t or DEPENDENCY_TAG environment variable" + exit 1 + fi +fi + +if [ -z "${destination:-}" ]; then + if [ -n "${GITHUB_ACTIONS:-}" ]; then destination="${GITHUB_WORKSPACE}/dependencies" -elif [ -n "${GITLAB_CI:-}" ];then + elif [ -n "${GITLAB_CI:-}" ]; then destination="${CI_PROJECT_DIR}/dependencies" -else - destination=${2} + else + echo "No destination specified via -d and not running in CI" + exit 1 + fi fi + + echo "Install tag: $tag" echo "Install destination: $destination" @@ -74,21 +147,19 @@ start_section "Locate OpenGL" end_section fi -start_section "Get spack lock file name" +start_section "Get spack lock file" arch=$(spack arch --family) -lock_file="spack-${arch}.lock" -end_section - - -url="https://github.com/acts-project/ci-dependencies/releases/download/${tag}/${lock_file}" -echo "URL: $url" env_dir="${destination}/env" view_dir="${destination}/view" mkdir -p ${env_dir} + lock_file_path="${destination}/spack.lock" -start_section "Get spack lock files from tag" -curl -fL -o $lock_file_path $url +"${SCRIPT_DIR}/select_lockfile.py" \ + --tag "${tag}" \ + --arch "${arch}"\ + --output "${lock_file_path}" \ + --compiler-binary "${compiler}" end_section From ea4b66b64f4112a514dac75ac80b5950638dace9 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Sun, 15 Dec 2024 14:45:45 +0100 Subject: [PATCH 061/191] use the compiler in cmake --- .github/workflows/builds.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index c66730b3ff0..7244039ad35 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -286,6 +286,7 @@ jobs: cmake -B build -S . --preset=github-ci -DCMAKE_CXX_STANDARD=${{ matrix.std }} + -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DPython_EXECUTABLE=$(which python3) From a9251b3731177476eee26945f27c280cd9bf9d72 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Sun, 15 Dec 2024 14:51:43 +0100 Subject: [PATCH 062/191] handle empty argument --- CI/dependencies/select_lockfile.py | 10 ++++++---- CI/dependencies/setup.sh | 24 ++++++++++++++---------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/CI/dependencies/select_lockfile.py b/CI/dependencies/select_lockfile.py index e23df66f360..cc44207e878 100755 --- a/CI/dependencies/select_lockfile.py +++ b/CI/dependencies/select_lockfile.py @@ -7,7 +7,7 @@ import urllib.error import re import subprocess -from typing import Tuple, Dict +from typing import Tuple, Dict, Optional def main(): @@ -46,8 +46,7 @@ def main(): compiler = args.compiler print("Compiler:", f"{compiler}") else: - print("No compiler specified") - exit(1) + compiler = None base_url = f"https://api.github.com/repos/acts-project/ci-dependencies/releases/tags/{args.tag}" @@ -96,11 +95,14 @@ def parse_assets(data: Dict) -> Dict[str, Dict[str, Tuple[str, str]]]: def select_lockfile( - lockfiles: Dict[str, Dict[str, Tuple[str, str]]], arch: str, compiler: str + lockfiles: Dict[str, Dict[str, Tuple[str, str]]], arch: str, compiler: Optional[str] ): # Default to the default lockfile _, lockfile = lockfiles[arch]["default"] + if compiler is None: + return lockfile + # Extract compiler family and version compiler_family = compiler.split("@")[0] diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index 18e979d00ef..75bb1bf054d 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -88,11 +88,7 @@ done # Set defaults if not specified if [ -z "${compiler:-}" ]; then - compiler="${CXX:-}" - if [ -z "${compiler:-}" ]; then - echo "No compiler specified via -c or CXX environment variable" - exit 1 - fi + compiler="${CXX:-default}" fi if [ -z "${tag:-}" ]; then @@ -155,11 +151,19 @@ view_dir="${destination}/view" mkdir -p ${env_dir} lock_file_path="${destination}/spack.lock" -"${SCRIPT_DIR}/select_lockfile.py" \ - --tag "${tag}" \ - --arch "${arch}"\ - --output "${lock_file_path}" \ - --compiler-binary "${compiler}" +cmd=( + "${SCRIPT_DIR}/select_lockfile.py" + "--tag" "${tag}" + "--arch" "${arch}" + "--output" "${lock_file_path}" +) + +if [ "${compiler}" != "default" ]; then + cmd+=("--compiler-binary" "${compiler}") +fi + +"${cmd[@]}" + end_section From e897c5b470dca065509c818efdbebfb5f6577b70 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Sun, 15 Dec 2024 14:54:51 +0100 Subject: [PATCH 063/191] detect g++ --- CI/dependencies/select_lockfile.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/CI/dependencies/select_lockfile.py b/CI/dependencies/select_lockfile.py index cc44207e878..732b482f307 100755 --- a/CI/dependencies/select_lockfile.py +++ b/CI/dependencies/select_lockfile.py @@ -36,17 +36,6 @@ def main(): print("Fetching lockfiles for tag:", args.tag) print("Architecture:", args.arch) - if args.compiler_binary is not None: - compiler = determine_compiler_version(args.compiler_binary) - print("Compiler:", args.compiler_binary, f"{compiler}") - elif args.compiler is not None: - if not re.match(r"^([\w-]+)@(\d+\.\d+\.\d+)$", args.compiler): - print(f"Invalid compiler format: {args.compiler}") - exit(1) - compiler = args.compiler - print("Compiler:", f"{compiler}") - else: - compiler = None base_url = f"https://api.github.com/repos/acts-project/ci-dependencies/releases/tags/{args.tag}" @@ -64,6 +53,18 @@ def main(): print(f"No lockfile found for architecture {args.arch}") exit(1) + if args.compiler_binary is not None: + compiler = determine_compiler_version(args.compiler_binary) + print("Compiler:", args.compiler_binary, f"{compiler}") + elif args.compiler is not None: + if not re.match(r"^([\w-]+)@(\d+\.\d+\.\d+)$", args.compiler): + print(f"Invalid compiler format: {args.compiler}") + exit(1) + compiler = args.compiler + print("Compiler:", f"{compiler}") + else: + compiler = None + lockfile = select_lockfile(lockfiles, args.arch, compiler) print("Selected lockfile:", lockfile) @@ -138,7 +139,7 @@ def determine_compiler_version(binary: str): compiler = "clang" if "Apple" in line: compiler = "apple-clang" - elif "gcc" in line or "GCC" in line: + elif "gcc" in line or "GCC" in line or "g++" in line: compiler = "gcc" else: print(f"Unknown compiler: {binary}") From 9b87e4b468ece232a7b6fba3ba004f162a04d013 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 16 Dec 2024 09:24:14 +0100 Subject: [PATCH 064/191] b5 --- .github/workflows/builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 7244039ad35..5966d803827 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -20,7 +20,7 @@ env: CCACHE_DIR: ${{ github.workspace }}/ccache CCACHE_MAXSIZE: 500M CCACHE_KEY_SUFFIX: r2 - DEPENDENCY_TAG: v7_b4 + DEPENDENCY_TAG: v7_b5 jobs: linux_ubuntu: From fa0e3672ec8dcf70fb121e497bb3c8dde79fa19d Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 16 Dec 2024 09:29:09 +0100 Subject: [PATCH 065/191] fix opengl detection --- CI/dependencies/opengl.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CI/dependencies/opengl.sh b/CI/dependencies/opengl.sh index 10e3eda6544..4cec4cda4e4 100755 --- a/CI/dependencies/opengl.sh +++ b/CI/dependencies/opengl.sh @@ -14,7 +14,7 @@ fi os=$(spack arch --platform) -if [ "$os" == ubuntu* ]; then +if [[ "$os" == ubuntu* ]]; then ${SUDO} apt-get update ${SUDO} apt-get install -y libgl1-mesa-dev cat < "$packages_file" @@ -26,7 +26,7 @@ packages: spec: opengl@4.5 EOF cat "$packages_file" -elif [ "$os" == almalinux* ]; then +elif [[ "$os" == almalinux* ]]; then ${SUDO} dnf install -y mesa-libGLU cat < "$packages_file" packages: From c85073c88c10d5a7f5e4c12a1d368b2fb30ef58b Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 16 Dec 2024 09:34:37 +0100 Subject: [PATCH 066/191] fix glob --- CI/dependencies/opengl.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CI/dependencies/opengl.sh b/CI/dependencies/opengl.sh index 4cec4cda4e4..fe8c94e0120 100755 --- a/CI/dependencies/opengl.sh +++ b/CI/dependencies/opengl.sh @@ -14,7 +14,7 @@ fi os=$(spack arch --platform) -if [[ "$os" == ubuntu* ]]; then +if [[ "$os" == *ubuntu* ]]; then ${SUDO} apt-get update ${SUDO} apt-get install -y libgl1-mesa-dev cat < "$packages_file" @@ -26,7 +26,7 @@ packages: spec: opengl@4.5 EOF cat "$packages_file" -elif [[ "$os" == almalinux* ]]; then +elif [[ "$os" == *almalinux* ]]; then ${SUDO} dnf install -y mesa-libGLU cat < "$packages_file" packages: @@ -37,4 +37,6 @@ packages: spec: opengl@4.6 EOF cat "$packages_file" +else [[ "$os" == *darwin* ]] + echo "Nothing to do on Darwin" fi From c39268667d9dd49b10c3c881c373a1040761cd56 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 16 Dec 2024 09:43:22 +0100 Subject: [PATCH 067/191] family --- CI/dependencies/opengl.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CI/dependencies/opengl.sh b/CI/dependencies/opengl.sh index fe8c94e0120..5a1c83c4293 100755 --- a/CI/dependencies/opengl.sh +++ b/CI/dependencies/opengl.sh @@ -12,7 +12,7 @@ else SUDO="sudo" fi -os=$(spack arch --platform) +os=$(spack arch --family) if [[ "$os" == *ubuntu* ]]; then ${SUDO} apt-get update From 3503bb85f0f8d95e414a275db6b3af839096b37d Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 16 Dec 2024 11:32:59 +0100 Subject: [PATCH 068/191] opengl --- CI/dependencies/opengl.sh | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/CI/dependencies/opengl.sh b/CI/dependencies/opengl.sh index 5a1c83c4293..b7db00bbcda 100755 --- a/CI/dependencies/opengl.sh +++ b/CI/dependencies/opengl.sh @@ -3,7 +3,9 @@ set -u set -e -packages_file=$(spack location -r)/spack/etc/spack/packages.yaml +packages_file=$(spack location -r)/etc/spack/packages.yaml +echo "Packages file: $packages_file" +stat "$packages_file" || true if ! command -v sudo &> /dev/null then @@ -14,16 +16,28 @@ fi os=$(spack arch --family) +echo "OS: $os" + if [[ "$os" == *ubuntu* ]]; then ${SUDO} apt-get update ${SUDO} apt-get install -y libgl1-mesa-dev + +if [[ "$os" == *ubuntu24* ]]; then + version="4.6" +elif [[ "$os" == *ubuntu20* ]]; then + version="4.5" +else + echo "Unknown OS version, default OpenGL version" + version="4.5" +fi + cat < "$packages_file" packages: opengl: buildable: false externals: - prefix: /usr/ - spec: opengl@4.5 + spec: opengl@${version} EOF cat "$packages_file" elif [[ "$os" == *almalinux* ]]; then From e7e95dffa3e478d81d15318fd306e88633de16a6 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 16 Dec 2024 11:39:37 +0100 Subject: [PATCH 069/191] wrap script in action --- .github/actions/dependencies/action.yml | 28 +++++++++++++++++++++++++ .github/workflows/builds.yml | 16 +++++--------- 2 files changed, 33 insertions(+), 11 deletions(-) create mode 100644 .github/actions/dependencies/action.yml diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml new file mode 100644 index 00000000000..286925f4245 --- /dev/null +++ b/.github/actions/dependencies/action.yml @@ -0,0 +1,28 @@ +name: 'Dependencies' +description: 'Install dependencies from lockfile' + +inputs: + compiler: + description: 'Compiler to use' + required: false + default: '' + +runs: + using: 'composite' + steps: + - name: Prerequisites + if: startsWith(inputs.os, 'macos') + shell: bash + run: | + brew install ninja ccache + + - name: Install dependencies + shell: run + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if [ -n "${{ inputs.compiler }}" ]; then + CI/dependencies/setup.sh -c "${{ inputs.compiler }}" + else + CI/dependencies/setup.sh + fi diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 5966d803827..5c428f1cfb5 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -37,9 +37,7 @@ jobs: lfs: true - name: Install dependencies - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: CI/dependencies/setup.sh + uses: ./.github/actions/dependencies - name: Restore ccache id: ccache-restore @@ -263,9 +261,9 @@ jobs: lfs: true - name: Install dependencies - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: CI/dependencies/setup.sh -c ${{ matrix.compiler }} + uses: ./.github/actions/dependencies + with: + compiler: ${{ matrix.compiler }} - name: Restore ccache uses: actions/cache/restore@v4 @@ -340,11 +338,7 @@ jobs: lfs: true - name: Install dependencies - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: > - brew install ninja ccache - && CI/dependencies/setup.sh + uses: ./.github/actions/dependencies - name: Restore ccache uses: actions/cache/restore@v4 From e61e05647f1f6e231ca93a3785e357fdbda3cf68 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 16 Dec 2024 11:55:41 +0100 Subject: [PATCH 070/191] explicit token --- .github/actions/dependencies/action.yml | 6 +++++- .github/workflows/builds.yml | 5 +++++ CI/dependencies/select_lockfile.py | 5 +++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 286925f4245..2507f9f8181 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -6,6 +6,10 @@ inputs: description: 'Compiler to use' required: false default: '' + GITHUB_TOKEN: + description: 'Secrets to pass to the action' + required: false + default: '' runs: using: 'composite' @@ -19,7 +23,7 @@ runs: - name: Install dependencies shell: run env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN }} run: | if [ -n "${{ inputs.compiler }}" ]; then CI/dependencies/setup.sh -c "${{ inputs.compiler }}" diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 5c428f1cfb5..24e157c0667 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -38,6 +38,8 @@ jobs: - name: Install dependencies uses: ./.github/actions/dependencies + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Restore ccache id: ccache-restore @@ -264,6 +266,7 @@ jobs: uses: ./.github/actions/dependencies with: compiler: ${{ matrix.compiler }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Restore ccache uses: actions/cache/restore@v4 @@ -339,6 +342,8 @@ jobs: - name: Install dependencies uses: ./.github/actions/dependencies + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Restore ccache uses: actions/cache/restore@v4 diff --git a/CI/dependencies/select_lockfile.py b/CI/dependencies/select_lockfile.py index 732b482f307..5bf3b4aa568 100755 --- a/CI/dependencies/select_lockfile.py +++ b/CI/dependencies/select_lockfile.py @@ -159,8 +159,9 @@ def determine_compiler_version(binary: str): def fetch_github(base_url): headers = {} - if "GITHUB_TOKEN" in os.environ: - headers["Authorization"] = f"token {os.environ['GITHUB_TOKEN']}" + token = os.environ.get("GITHUB_TOKEN") + if token is not None and token != "": + headers["Authorization"] = f"token {token}" try: req = urllib.request.Request(base_url, headers=headers) From d947b2ff6b33e064dffcdcf9d7d25a0b0729a852 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 16 Dec 2024 14:08:38 +0100 Subject: [PATCH 071/191] action --- .github/actions/dependencies/action.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 2507f9f8181..2690caf309e 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -25,8 +25,11 @@ runs: env: GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN }} run: | + pwd if [ -n "${{ inputs.compiler }}" ]; then + echo "With compiler" CI/dependencies/setup.sh -c "${{ inputs.compiler }}" else + echo "Without compiler" CI/dependencies/setup.sh fi From ae1a2277d110a0126c29a58075e67723195c652b Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 16 Dec 2024 14:11:35 +0100 Subject: [PATCH 072/191] shell --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 2690caf309e..05aa72a905c 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -21,7 +21,7 @@ runs: brew install ninja ccache - name: Install dependencies - shell: run + shell: bash env: GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN }} run: | From 520435012ee8c4d998aa1e718cf43bff448f99b8 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 16 Dec 2024 14:42:23 +0100 Subject: [PATCH 073/191] use runner os --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 05aa72a905c..17787e7035b 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -15,7 +15,7 @@ runs: using: 'composite' steps: - name: Prerequisites - if: startsWith(inputs.os, 'macos') + if: startsWith(runner.os, 'macos') shell: bash run: | brew install ninja ccache From 8c9cdf90326b74f5179bb6db761f8648de5c4744 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 15:44:01 +0100 Subject: [PATCH 074/191] update tag --- .github/workflows/builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 24e157c0667..c57a7209c7b 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -20,7 +20,7 @@ env: CCACHE_DIR: ${{ github.workspace }}/ccache CCACHE_MAXSIZE: 500M CCACHE_KEY_SUFFIX: r2 - DEPENDENCY_TAG: v7_b5 + DEPENDENCY_TAG: v7_b6 jobs: linux_ubuntu: From b59c846520719ce62e2bb1543283d2598bd65fab Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:09:21 +0100 Subject: [PATCH 075/191] dedupe matrix cache keys --- .github/workflows/builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index c57a7209c7b..7925acdd1a8 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -273,7 +273,7 @@ jobs: id: ccache-restore with: path: ${{ env.CCACHE_DIR }} - key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} + key: ccache-${{ runner.os }}-${{ github.job }}-${{ matrix.image }}-${{ matrix.std }}-${{ matrix.compiler }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} restore-keys: | ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- From 0920ebda59d79a03d27f4b6f9912b8684b217dcf Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:09:41 +0100 Subject: [PATCH 076/191] remove remnant --- .github/workflows/builds.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 7925acdd1a8..4f5c8e9d8b8 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -279,9 +279,6 @@ jobs: - name: Configure - # setting CMAKE_CXX_STANDARD=20 is a workaround for a bug in the - # dd4hep CMake configuration that gets triggered on recent CMake - # versions run: > ccache -z && cmake -B build -S . From b9ffa07fffb8aad223730f1677166a1652843ae0 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:10:15 +0100 Subject: [PATCH 077/191] see if we still need to supply the python interpreter --- .github/workflows/builds.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 4f5c8e9d8b8..9d36cfc85d3 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -62,7 +62,6 @@ jobs: cmake -B build -S . --preset=github-ci -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" - -DPython_EXECUTABLE=$(which python3) -DACTS_BUILD_PLUGIN_ONNX=ON - name: Build @@ -101,7 +100,6 @@ jobs: -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS=-Werror - -DCMAKE_CXX_STANDARD=20 -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" - name: Downstream build @@ -286,7 +284,6 @@ jobs: -DCMAKE_CXX_STANDARD=${{ matrix.std }} -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" - -DPython_EXECUTABLE=$(which python3) - name: Build run: cmake --build build @@ -357,7 +354,6 @@ jobs: && cmake -B build -S . --preset=github-ci -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" - -DPython_EXECUTABLE=$(which python3) - name: Build run: cmake --build build From 980f8ce5d9b5de4bc0c37b68c16fce385803bf01 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:14:47 +0100 Subject: [PATCH 078/191] change path ordering --- CI/dependencies/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index 75bb1bf054d..c6174ea0661 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -207,8 +207,8 @@ venv_dir="${view_dir}/venv" "${view_dir}"/bin/python3 -m venv "$venv_dir" if [ -n "${GITHUB_ACTIONS:-}" ]; then - echo "${venv_dir}/bin" >> "$GITHUB_PATH" echo "${view_dir}/bin" >> "$GITHUB_PATH" + echo "${venv_dir}/bin" >> "$GITHUB_PATH" fi set_env PATH "${venv_dir}/bin:${view_dir}/bin/:${PATH}" end_section From 111b4dc3ac08460fd5ce491bf2c1282493a2d8fa Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:16:58 +0100 Subject: [PATCH 079/191] we need it in macos at least --- .github/workflows/builds.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 9d36cfc85d3..468446d4325 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -354,6 +354,7 @@ jobs: && cmake -B build -S . --preset=github-ci -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" + -DPython_EXECUTABLE=$(which python3) - name: Build run: cmake --build build From f4d3e576803cc9deb7e7affc84457bc8470fb95f Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:26:17 +0100 Subject: [PATCH 080/191] let's try to cache the spack checkout --- .github/actions/dependencies/action.yml | 29 ++++++++++++++++ CI/dependencies/setup.sh | 45 ++++++++++--------------- 2 files changed, 46 insertions(+), 28 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 17787e7035b..12366030395 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -20,6 +20,22 @@ runs: run: | brew install ninja ccache + - name: Restore spack cache + id: cache-spack-restore + uses: actions/cache/restore@v4 + with: + path: spack.tar.zst + key: ${{ env.CACHE_KEY }}-spack + + - name: Extract spack cache if found + if: steps.cache-spack-restore.outputs.cache-hit == 'true' + shell: bash + run: | + echo "Cache hit, extracting spack..." + zstd -d spack.tar.zst -o spack.tar + tar xf spack.tar + rm spack.tar + - name: Install dependencies shell: bash env: @@ -33,3 +49,16 @@ runs: echo "Without compiler" CI/dependencies/setup.sh fi + + - name: Compress spack directory for caching + shell: bash + run: | + # Compress spack directory for caching + echo "Compressing spack directory..." + tar cf spack.tar spack/ + zstd -T0 -10 spack.tar -o spack.tar.zst + + uses: actions/cache/save@v4 + with: + path: spack.tar.zst + key: ${{ env.CACHE_KEY }}-spack diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index c6174ea0661..b0690919d56 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -34,26 +34,6 @@ function end_section() { fi } -# if [ -n "${1:-${DEPENDENCY_TAG:-}}" ]; then -# tag=${1:-${DEPENDENCY_TAG}} -# else -# echo "Usage: $0 " -# exit 1 -# fi - -# if [ -n "${GITHUB_ACTIONS:-}" ]; then -# destination="${GITHUB_WORKSPACE}/dependencies" -# elif [ -n "${GITLAB_CI:-}" ];then -# destination="${CI_PROJECT_DIR}/dependencies" -# else -# if [ -n "${2:-}" ]; then -# destination=${2} -# else -# echo "Usage: $0 " -# exit 1 -# fi -# fi - # Parse command line arguments while getopts "c:t:d:h" opt; do case ${opt} in @@ -120,14 +100,23 @@ mkdir -p ${destination} start_section "Install spack if not already installed" if ! command -v spack &> /dev/null; then - git clone -c feature.manyFiles=true https://github.com/spack/spack.git - pushd spack > /dev/null - git config user.name github-actions[bot] - git config user.email 41898282+github-actions[bot]@users.noreply.github.com - # Apply patch for spack improvements - curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am - source "$(pwd)/share/spack/setup-env.sh" - popd > /dev/null + if [ ! -d "spack" ]; then + echo "Cloning spack" + git clone -c feature.manyFiles=true https://github.com/spack/spack.git + pushd spack > /dev/null + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + # Apply patch for spack improvements + curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am + popd > /dev/null + else + echo "Updating spack" + pushd spack > /dev/null + git pull --rebase + popd > /dev/null + fi + + source "$(pwd)/spack/share/spack/setup-env.sh" fi end_section From cd631d753104fda890b3f18525440fe62311dec0 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:27:20 +0100 Subject: [PATCH 081/191] yaml error --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 12366030395..13559991df7 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -58,7 +58,7 @@ runs: tar cf spack.tar spack/ zstd -T0 -10 spack.tar -o spack.tar.zst - uses: actions/cache/save@v4 + - uses: actions/cache/save@v4 with: path: spack.tar.zst key: ${{ env.CACHE_KEY }}-spack From dde89b248dfb6e07621566d60839ccab30ebb3f2 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:33:22 +0100 Subject: [PATCH 082/191] compress before installing, rebase --- .github/actions/dependencies/action.yml | 26 ++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 13559991df7..6625c0d8e68 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -35,6 +35,19 @@ runs: zstd -d spack.tar.zst -o spack.tar tar xf spack.tar rm spack.tar + git -C spack pull --rebase + + - name: Compress spack directory for caching if we cloned + if: steps.cache-spack-restore.outputs.cache-hit == 'false' + shell: bash + run: | + tar cf spack.tar spack/ + zstd -T0 -10 spack.tar -o spack.tar.zst + + - uses: actions/cache/save@v4 + with: + path: spack.tar.zst + key: ${{ env.CACHE_KEY }}-spack - name: Install dependencies shell: bash @@ -49,16 +62,3 @@ runs: echo "Without compiler" CI/dependencies/setup.sh fi - - - name: Compress spack directory for caching - shell: bash - run: | - # Compress spack directory for caching - echo "Compressing spack directory..." - tar cf spack.tar spack/ - zstd -T0 -10 spack.tar -o spack.tar.zst - - - uses: actions/cache/save@v4 - with: - path: spack.tar.zst - key: ${{ env.CACHE_KEY }}-spack From 6c8b4b70970e20540d47eea6cf129535d0890bf5 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:34:07 +0100 Subject: [PATCH 083/191] better compression --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 6625c0d8e68..84cc0085615 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -42,7 +42,7 @@ runs: shell: bash run: | tar cf spack.tar spack/ - zstd -T0 -10 spack.tar -o spack.tar.zst + zstd -T0 -19 spack.tar -o spack.tar.zst - uses: actions/cache/save@v4 with: From 94d431805e1a782e9f94d209ccdf01f5e6bfeed1 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:36:15 +0100 Subject: [PATCH 084/191] repack? --- .github/actions/dependencies/action.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 84cc0085615..36481206a38 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -25,7 +25,7 @@ runs: uses: actions/cache/restore@v4 with: path: spack.tar.zst - key: ${{ env.CACHE_KEY }}-spack + key: spack-clone - name: Extract spack cache if found if: steps.cache-spack-restore.outputs.cache-hit == 'true' @@ -37,8 +37,7 @@ runs: rm spack.tar git -C spack pull --rebase - - name: Compress spack directory for caching if we cloned - if: steps.cache-spack-restore.outputs.cache-hit == 'false' + - name: Compress spack directory for caching shell: bash run: | tar cf spack.tar spack/ @@ -47,7 +46,7 @@ runs: - uses: actions/cache/save@v4 with: path: spack.tar.zst - key: ${{ env.CACHE_KEY }}-spack + key: spack-clone - name: Install dependencies shell: bash From ec206749b9447185ad9b769898417f9fd20d747c Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:38:42 +0100 Subject: [PATCH 085/191] actually, set up with update --- .github/actions/dependencies/action.yml | 7 +++++-- CI/dependencies/setup.sh | 20 +------------------- CI/dependencies/setup_spack.sh | 23 +++++++++++++++++++++++ 3 files changed, 29 insertions(+), 21 deletions(-) create mode 100644 CI/dependencies/setup_spack.sh diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 36481206a38..4b99eb2b341 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -31,11 +31,14 @@ runs: if: steps.cache-spack-restore.outputs.cache-hit == 'true' shell: bash run: | - echo "Cache hit, extracting spack..." zstd -d spack.tar.zst -o spack.tar tar xf spack.tar rm spack.tar - git -C spack pull --rebase + + - name: Install spack if not found + shell: bash + run: | + CI/dependencies/setup_spack.sh - name: Compress spack directory for caching shell: bash diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index b0690919d56..9cef587dd8c 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -99,25 +99,7 @@ mkdir -p ${destination} start_section "Install spack if not already installed" -if ! command -v spack &> /dev/null; then - if [ ! -d "spack" ]; then - echo "Cloning spack" - git clone -c feature.manyFiles=true https://github.com/spack/spack.git - pushd spack > /dev/null - git config user.name github-actions[bot] - git config user.email 41898282+github-actions[bot]@users.noreply.github.com - # Apply patch for spack improvements - curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am - popd > /dev/null - else - echo "Updating spack" - pushd spack > /dev/null - git pull --rebase - popd > /dev/null - fi - - source "$(pwd)/spack/share/spack/setup-env.sh" -fi +source "${SCRIPT_DIR}/install_spack.sh" end_section if [ -n "${CI:-}" ]; then diff --git a/CI/dependencies/setup_spack.sh b/CI/dependencies/setup_spack.sh new file mode 100644 index 00000000000..1ebc3a896d8 --- /dev/null +++ b/CI/dependencies/setup_spack.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -e +set -u + +if ! command -v spack &> /dev/null; then + if [ ! -d "spack" ]; then + echo "Cloning spack" + git clone -c feature.manyFiles=true https://github.com/spack/spack.git + pushd spack > /dev/null + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + # Apply patch for spack improvements + curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am + popd > /dev/null + else + echo "Updating spack" + pushd spack > /dev/null + git pull --rebase + popd > /dev/null + fi + + source "$(pwd)/spack/share/spack/setup-env.sh" +fi From 7c5ef9826b3510e2ae81b04d117936fd11690223 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:40:29 +0100 Subject: [PATCH 086/191] shuffle --- .github/actions/dependencies/action.yml | 11 ++++++----- CI/dependencies/setup_spack.sh | 0 2 files changed, 6 insertions(+), 5 deletions(-) mode change 100644 => 100755 CI/dependencies/setup_spack.sh diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 4b99eb2b341..fd92196d53b 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -14,11 +14,6 @@ inputs: runs: using: 'composite' steps: - - name: Prerequisites - if: startsWith(runner.os, 'macos') - shell: bash - run: | - brew install ninja ccache - name: Restore spack cache id: cache-spack-restore @@ -51,6 +46,12 @@ runs: path: spack.tar.zst key: spack-clone + - name: Prerequisites + if: startsWith(runner.os, 'macos') + shell: bash + run: | + brew install ninja ccache + - name: Install dependencies shell: bash env: diff --git a/CI/dependencies/setup_spack.sh b/CI/dependencies/setup_spack.sh old mode 100644 new mode 100755 From 2acd2ab552508ff67cc4205e27e2a98f30bc4ef7 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:43:51 +0100 Subject: [PATCH 087/191] need to do it after we did the environment --- .github/actions/dependencies/action.yml | 22 +++++++++++----------- CI/dependencies/setup.sh | 20 +++++++++++++++++++- CI/dependencies/setup_spack.sh | 23 ----------------------- 3 files changed, 30 insertions(+), 35 deletions(-) delete mode 100755 CI/dependencies/setup_spack.sh diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index fd92196d53b..928c4d78499 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -35,17 +35,6 @@ runs: run: | CI/dependencies/setup_spack.sh - - name: Compress spack directory for caching - shell: bash - run: | - tar cf spack.tar spack/ - zstd -T0 -19 spack.tar -o spack.tar.zst - - - uses: actions/cache/save@v4 - with: - path: spack.tar.zst - key: spack-clone - - name: Prerequisites if: startsWith(runner.os, 'macos') shell: bash @@ -65,3 +54,14 @@ runs: echo "Without compiler" CI/dependencies/setup.sh fi + + - name: Compress spack directory for caching + shell: bash + run: | + tar --exclude='opt' cf spack.tar spack/ + zstd -T0 -19 spack.tar -o spack.tar.zst + + - uses: actions/cache/save@v4 + with: + path: spack.tar.zst + key: spack-clone diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index 9cef587dd8c..b0690919d56 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -99,7 +99,25 @@ mkdir -p ${destination} start_section "Install spack if not already installed" -source "${SCRIPT_DIR}/install_spack.sh" +if ! command -v spack &> /dev/null; then + if [ ! -d "spack" ]; then + echo "Cloning spack" + git clone -c feature.manyFiles=true https://github.com/spack/spack.git + pushd spack > /dev/null + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + # Apply patch for spack improvements + curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am + popd > /dev/null + else + echo "Updating spack" + pushd spack > /dev/null + git pull --rebase + popd > /dev/null + fi + + source "$(pwd)/spack/share/spack/setup-env.sh" +fi end_section if [ -n "${CI:-}" ]; then diff --git a/CI/dependencies/setup_spack.sh b/CI/dependencies/setup_spack.sh deleted file mode 100755 index 1ebc3a896d8..00000000000 --- a/CI/dependencies/setup_spack.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -set -e -set -u - -if ! command -v spack &> /dev/null; then - if [ ! -d "spack" ]; then - echo "Cloning spack" - git clone -c feature.manyFiles=true https://github.com/spack/spack.git - pushd spack > /dev/null - git config user.name github-actions[bot] - git config user.email 41898282+github-actions[bot]@users.noreply.github.com - # Apply patch for spack improvements - curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am - popd > /dev/null - else - echo "Updating spack" - pushd spack > /dev/null - git pull --rebase - popd > /dev/null - fi - - source "$(pwd)/spack/share/spack/setup-env.sh" -fi From 8b6b8830c764a4bef06102e14563c24b3d126488 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:46:03 +0100 Subject: [PATCH 088/191] exclude tarball --- .github/actions/dependencies/action.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 928c4d78499..a0e87891c36 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -58,7 +58,11 @@ runs: - name: Compress spack directory for caching shell: bash run: | - tar --exclude='opt' cf spack.tar spack/ + tar \ + --exclude='opt' \ + --exclude='var/spack/cache' \ + --exclude='var/spack/environments' \ + cf spack.tar spack/ zstd -T0 -19 spack.tar -o spack.tar.zst - uses: actions/cache/save@v4 From 4c0a7bbd67bec0ef14c7e60f41dab1430363ee76 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:55:10 +0100 Subject: [PATCH 089/191] drop the caching again --- .github/actions/dependencies/action.yml | 36 ------------------------- 1 file changed, 36 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index a0e87891c36..17787e7035b 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -14,27 +14,6 @@ inputs: runs: using: 'composite' steps: - - - name: Restore spack cache - id: cache-spack-restore - uses: actions/cache/restore@v4 - with: - path: spack.tar.zst - key: spack-clone - - - name: Extract spack cache if found - if: steps.cache-spack-restore.outputs.cache-hit == 'true' - shell: bash - run: | - zstd -d spack.tar.zst -o spack.tar - tar xf spack.tar - rm spack.tar - - - name: Install spack if not found - shell: bash - run: | - CI/dependencies/setup_spack.sh - - name: Prerequisites if: startsWith(runner.os, 'macos') shell: bash @@ -54,18 +33,3 @@ runs: echo "Without compiler" CI/dependencies/setup.sh fi - - - name: Compress spack directory for caching - shell: bash - run: | - tar \ - --exclude='opt' \ - --exclude='var/spack/cache' \ - --exclude='var/spack/environments' \ - cf spack.tar spack/ - zstd -T0 -19 spack.tar -o spack.tar.zst - - - uses: actions/cache/save@v4 - with: - path: spack.tar.zst - key: spack-clone From 7e01b2bd417ab659026618f1b8e7f26195b02773 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 08:18:47 +0100 Subject: [PATCH 090/191] path python macos --- .github/workflows/builds.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 468446d4325..31181743721 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -349,9 +349,11 @@ jobs: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - name: Configure - run: > + run: | ccache -z - && cmake -B build -S . + echo $PATH + which python3 + cmake -B build -S . --preset=github-ci -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" -DPython_EXECUTABLE=$(which python3) From 49f24af0234caebe5b1b458bc1aa1b24a786a55b Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 08:19:57 +0100 Subject: [PATCH 091/191] tmate --- .github/workflows/builds.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 31181743721..4cefc0be04d 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -348,6 +348,9 @@ jobs: restore-keys: | ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- + - uses: mxschmitt/action-tmate@v3 + # if: failure() + - name: Configure run: | ccache -z From 760891ffcb395694afcd8e1c7407d62b62cf198d Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 08:20:14 +0100 Subject: [PATCH 092/191] other tmate --- .github/workflows/builds.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 4cefc0be04d..f3351c4be16 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -50,6 +50,9 @@ jobs: restore-keys: | ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- + - uses: mxschmitt/action-tmate@v3 + # if: failure() + - name: Configure # setting CMAKE_CXX_STANDARD=20 is a workaround for a bug in the # dd4hep CMake configuration that gets triggered on recent CMake @@ -348,8 +351,6 @@ jobs: restore-keys: | ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - - uses: mxschmitt/action-tmate@v3 - # if: failure() - name: Configure run: | From c18a483caf605785af704a3c6e1d0573cae554c2 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 08:33:06 +0100 Subject: [PATCH 093/191] point cmakeprefixpath at venv dir --- CI/dependencies/setup.sh | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index b0690919d56..8c64bf86a6c 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -178,17 +178,10 @@ start_section "Patch up Geant4 data directory" # ${SCRIPT_DIR}/with_spack_env.sh ${env_dir} geant4-config --install-datasets geant4_dir=$(spack -e "${env_dir}" location -i geant4) # Prepare the folder for G4 data, and symlink it to where G4 will look for it -mkdir -p "${geant4_dir}"/share/Geant4 -ln -s "${geant4_dir}"/share/Geant4/data ${view_dir}/share/Geant4/data +mkdir -p "${geant4_dir}/share/Geant4" +ln -s "${geant4_dir}/share/Geant4/data" "${view_dir}/share/Geant4/data" end_section -start_section "Set environment variables" -set_env CMAKE_PREFIX_PATH "${view_dir}" -set_env LD_LIBRARY_PATH "${view_dir}/lib" -set_env ROOT_INCLUDE_PATH "${view_dir}/include" -# Geant4 puts CLHEP in a subdirectory -set_env ROOT_INCLUDE_PATH "${view_dir}/include/Geant4" -end_section start_section "Prepare python environment" ls -al @@ -202,7 +195,13 @@ fi set_env PATH "${venv_dir}/bin:${view_dir}/bin/:${PATH}" end_section - +start_section "Set environment variables" +set_env CMAKE_PREFIX_PATH "${venv_dir}:${view_dir}" +set_env LD_LIBRARY_PATH "${view_dir}/lib" +set_env ROOT_INCLUDE_PATH "${view_dir}/include" +# Geant4 puts CLHEP in a subdirectory +set_env ROOT_INCLUDE_PATH "${view_dir}/include/Geant4" +end_section # Pythia8 looks for settings in this directory # set_env PYTHIA8DATA "${destination}/share/Pythia8/xmldoc" From 9783a07d009bfbd912cb493dac1e7c4f9331456c Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 08:33:12 +0100 Subject: [PATCH 094/191] chain updates --- .github/workflows/builds.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index f3351c4be16..51356421ff3 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -50,7 +50,7 @@ jobs: restore-keys: | ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - - uses: mxschmitt/action-tmate@v3 + # - uses: mxschmitt/action-tmate@v3 # if: failure() - name: Configure @@ -59,8 +59,6 @@ jobs: # versions # Need to set git user & email for patching to work (GeoModel plugin) run: > - git config --global user.name 'CI' && - git config --global user.email '<>' && ccache -z && cmake -B build -S . --preset=github-ci @@ -353,14 +351,11 @@ jobs: - name: Configure - run: | - ccache -z - echo $PATH - which python3 + run: > + ccache -z && cmake -B build -S . --preset=github-ci -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" - -DPython_EXECUTABLE=$(which python3) - name: Build run: cmake --build build From 13a3fe71089762b06d24aadbed810cec10b8e51e Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 08:39:55 +0100 Subject: [PATCH 095/191] system site packages? --- CI/dependencies/setup.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index 8c64bf86a6c..8cb8768319d 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -186,7 +186,9 @@ end_section start_section "Prepare python environment" ls -al venv_dir="${view_dir}/venv" -"${view_dir}"/bin/python3 -m venv "$venv_dir" +"${view_dir}"/bin/python3 -m venv \ + --system-site-packages \ + "$venv_dir" if [ -n "${GITHUB_ACTIONS:-}" ]; then echo "${view_dir}/bin" >> "$GITHUB_PATH" From 09fb061f49f6930b11fa2220d7b0787892c8fc44 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 08:50:55 +0100 Subject: [PATCH 096/191] install base python packages --- CI/dependencies/setup.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index 8cb8768319d..a90765625c0 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -190,6 +190,8 @@ venv_dir="${view_dir}/venv" --system-site-packages \ "$venv_dir" +"${venv_dir}/bin/python3" -m pip install pyyaml jinja2 + if [ -n "${GITHUB_ACTIONS:-}" ]; then echo "${view_dir}/bin" >> "$GITHUB_PATH" echo "${venv_dir}/bin" >> "$GITHUB_PATH" From e8b0a9d6f4e6ad12b3f57457058fd7f8fb6331ba Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 09:56:46 +0100 Subject: [PATCH 097/191] tmae macos --- .github/workflows/builds.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 51356421ff3..c20c53413b6 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -349,6 +349,9 @@ jobs: restore-keys: | ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- + - uses: mxschmitt/action-tmate@v3 + # if: failure() + - name: Configure run: > From baad99b85d36f37ecb0737b10cdc597c0a879455 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 10:18:03 +0100 Subject: [PATCH 098/191] add ptyhon find framework directive --- CMakePresets.json | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakePresets.json b/CMakePresets.json index 2aef0f3fe94..2c781e1c807 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -47,6 +47,7 @@ "cacheVariables": { "CMAKE_BUILD_TYPE": "Release", "CMAKE_COMPILE_WARNING_AS_ERROR": "ON", + "Python_FIND_FRAMEWORK": "LAST", "ACTS_FORCE_ASSERTIONS": "ON", "ACTS_ENABLE_LOG_FAILURE_THRESHOLD": "ON", "ACTS_BUILD_BENCHMARKS": "ON", From 6beabe8ae37149468c479b0707c390f52a83f9f0 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 10:18:52 +0100 Subject: [PATCH 099/191] no tmate --- .github/workflows/builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index c20c53413b6..917f6570042 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -349,7 +349,7 @@ jobs: restore-keys: | ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - - uses: mxschmitt/action-tmate@v3 + # - uses: mxschmitt/action-tmate@v3 # if: failure() From b7250a8c3f093e67f586f85bc3b7492ec47edb74 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 10:24:36 +0100 Subject: [PATCH 100/191] location strategy --- CMakePresets.json | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakePresets.json b/CMakePresets.json index 2c781e1c807..b6cb70bfa91 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -48,6 +48,7 @@ "CMAKE_BUILD_TYPE": "Release", "CMAKE_COMPILE_WARNING_AS_ERROR": "ON", "Python_FIND_FRAMEWORK": "LAST", + "Python_FIND_STRATEGY": "LOCATION", "ACTS_FORCE_ASSERTIONS": "ON", "ACTS_ENABLE_LOG_FAILURE_THRESHOLD": "ON", "ACTS_BUILD_BENCHMARKS": "ON", From 110ecc149eb2215790d97e01c008281e57ad1c23 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 10:31:24 +0100 Subject: [PATCH 101/191] see if policy also works --- CMakeLists.txt | 2 ++ CMakePresets.json | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 04fb58d6825..b8a53a90c4b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -258,6 +258,8 @@ if(POLICY CMP0135) cmake_policy(SET CMP0135 NEW) endif() +cmake_policy(SET CMP0094 NEW) + # required packages if(ACTS_SETUP_BOOST) if(POLICY CMP0167) diff --git a/CMakePresets.json b/CMakePresets.json index b6cb70bfa91..2c781e1c807 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -48,7 +48,6 @@ "CMAKE_BUILD_TYPE": "Release", "CMAKE_COMPILE_WARNING_AS_ERROR": "ON", "Python_FIND_FRAMEWORK": "LAST", - "Python_FIND_STRATEGY": "LOCATION", "ACTS_FORCE_ASSERTIONS": "ON", "ACTS_ENABLE_LOG_FAILURE_THRESHOLD": "ON", "ACTS_BUILD_BENCHMARKS": "ON", From 841de097989244516904f698d4533f5b008a52c7 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 10:46:19 +0100 Subject: [PATCH 102/191] more general CMAKE_FIND_FRAMEWORK --- CMakePresets.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakePresets.json b/CMakePresets.json index 2c781e1c807..fc7275cbd4e 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -47,7 +47,7 @@ "cacheVariables": { "CMAKE_BUILD_TYPE": "Release", "CMAKE_COMPILE_WARNING_AS_ERROR": "ON", - "Python_FIND_FRAMEWORK": "LAST", + "CMAKE_FIND_FRAMEWORK": "LAST", "ACTS_FORCE_ASSERTIONS": "ON", "ACTS_ENABLE_LOG_FAILURE_THRESHOLD": "ON", "ACTS_BUILD_BENCHMARKS": "ON", From 9122a38f7bafba750622cea3de6470f1867e34ee Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 10:52:44 +0100 Subject: [PATCH 103/191] collect all policy settings in one place --- CMakeLists.txt | 52 +++++++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b8a53a90c4b..de68185d7a7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,6 +13,36 @@ include(ActsRetrieveVersion) project(Acts VERSION ${_acts_version} LANGUAGES CXX) +# policy settings + +# Steers how project() handles the VERSION option +cmake_policy(SET CMP0048 NEW) + +# the `_VERSION` variables set by `setup(... VERSION ...)` have +# only local scope, i.e. they are not accessible her for dependencies added +# via `add_subdirectory`. this overrides the `project(...)` function for +# sub-projects such that the resulting `_VERSION` has +# global scope and is accessible within the main project later on. +macro(project) + _project(${ARGN}) + set(${PROJECT_NAME}_VERSION "${${PROJECT_NAME}_VERSION}" CACHE INTERNAL "") +endmacro() + +# Controls the way python is located +if(POLICY CMP0094) + cmake_policy(SET CMP0094 NEW) +endif() + +# Controls behavior of DOWNLOAD_EXTRACT_TIMESTAMP +if(POLICY CMP0135) + cmake_policy(SET CMP0135 NEW) +endif() + +# Use boost's cmake config files +if(POLICY CMP0167) + cmake_policy(SET CMP0167 NEW) +endif() + # build options # all options and compile-time parameters must be defined here for clear @@ -253,19 +283,8 @@ set(_acts_boost_recommended_version 1.78.0) # Include the sources for the external dependencies. include(ActsExternSources) -# Controls behavior of DOWNLOAD_EXTRACT_TIMESTAMP -if(POLICY CMP0135) - cmake_policy(SET CMP0135 NEW) -endif() - -cmake_policy(SET CMP0094 NEW) - # required packages if(ACTS_SETUP_BOOST) - if(POLICY CMP0167) - cmake_policy(SET CMP0167 NEW) - endif() - # Enable both program_options and unit_test_framework to reduce complexity # Also Cuda tests seem to use program_options if( @@ -313,17 +332,6 @@ endif() find_package(Filesystem REQUIRED) -# the `_VERSION` variables set by `setup(... VERSION ...)` have -# only local scope, i.e. they are not accessible her for dependencies added -# via `add_subdirectory`. this overrides the `project(...)` function for -# sub-projects such that the resulting `_VERSION` has -# global scope and is accessible within the main project later on. -cmake_policy(SET CMP0048 NEW) -macro(project) - _project(${ARGN}) - set(${PROJECT_NAME}_VERSION "${${PROJECT_NAME}_VERSION}" CACHE INTERNAL "") -endmacro() - # CUDA settings are collected here in a macro, so that they can be reused by different plugins macro(enable_cuda) enable_language(CUDA) From bbe98445036968654dc9fa3dc45002493b66e5ac Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 11:47:30 +0100 Subject: [PATCH 104/191] run downstream jobs --- .github/workflows/builds.yml | 207 +++++++++++++++++------------------ 1 file changed, 100 insertions(+), 107 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 917f6570042..047d431599f 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -109,107 +109,105 @@ jobs: - name: Downstream run run: ./build-downstream/bin/ShowActsVersion - # linux_examples_test: - # runs-on: ubuntu-latest - # container: ghcr.io/acts-project/ubuntu2404:63 - # needs: [linux_ubuntu] - # env: - # ACTS_SEQUENCER_DISABLE_FPEMON: true - # - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # lfs: true - # - # - name: Set dependencies URL - # run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV - # - # - name: Install dependencies - # run: CI/dependencies.sh - # - # - uses: actions/download-artifact@v4 - # with: - # name: acts-linux-ubuntu - # - # - name: Unpack build - # run: mkdir build && tar xf build.tar.gz -C build - # - # - name: Python level tests - # shell: bash - # env: - # PYTEST_MD_REPORT: true - # PYTEST_MD_REPORT_VERBOSE: 0 - # PYTEST_MD_REPORT_OUTPUT: pytest.md - # run: > - # geant4-config --install-datasets - # && source build/this_acts_withdeps.sh - # && python3 -m pip install -r Examples/Python/tests/requirements.txt - # && python3 -m pip install pytest-md-report - # && pytest -rFsv -k "not exatrkx" -v - # && cat ${PYTEST_MD_REPORT_OUTPUT} >> $GITHUB_STEP_SUMMARY - # - # linux_physmon: - # runs-on: ubuntu-latest - # container: ghcr.io/acts-project/ubuntu2404:63 - # needs: [linux_ubuntu] - # env: - # ACTS_SEQUENCER_DISABLE_FPEMON: true - # - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # lfs: true - # - # - run: apt-get update && apt-get install -y time - # - # - name: Set dependencies URL - # run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV - # - # - name: Install dependencies - # run: CI/dependencies.sh - # - # - uses: actions/download-artifact@v4 - # with: - # name: acts-linux-ubuntu - # - # - name: Unpack build - # run: mkdir build && tar xf build.tar.gz -C build - # - # - name: Save PR number - # if: ${{ github.event_name == 'pull_request' }} - # run: | - # mkdir -p physmon - # echo ${{ github.event.number }} > physmon/pr_number - # echo ${{ github.event.pull_request.head.sha }} > physmon/sha - # - # - name: Physics performance checks - # shell: bash - # run: > - # echo "::group::Dependencies" - # && git config --global safe.directory "$GITHUB_WORKSPACE" - # && python3 -m pip install histcmp==0.6.8 matplotlib - # && python3 -m pip install -r Examples/Scripts/requirements.txt - # && geant4-config --install-datasets - # && venv_python=$(which python3) - # && echo $venv_python - # && source build/this_acts_withdeps.sh - # && export PATH=$(dirname $venv_python):$PATH - # && echo $PATH - # && which python3 - # && echo "::endgroup::" - # && CI/physmon/phys_perf_mon.sh all physmon - # - # - name: Post step summary - # if: always() - # run: cat physmon/summary.md >> $GITHUB_STEP_SUMMARY - # - # - uses: actions/upload-artifact@v4 - # if: always() - # with: - # name: physmon - # path: physmon + linux_examples_test: + runs-on: ubuntu-latest + container: ghcr.io/acts-project/ubuntu2404:63 + needs: [linux_ubuntu] + env: + ACTS_SEQUENCER_DISABLE_FPEMON: true + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + lfs: true + + - name: Install dependencies + uses: ./.github/actions/dependencies + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/download-artifact@v4 + with: + name: acts-linux-ubuntu + + - name: Unpack build + run: mkdir build && tar xf build.tar.gz -C build + + - name: Python level tests + shell: bash + env: + PYTEST_MD_REPORT: true + PYTEST_MD_REPORT_VERBOSE: 0 + PYTEST_MD_REPORT_OUTPUT: pytest.md + run: > + geant4-config --install-datasets + && source build/this_acts_withdeps.sh + && python3 -m pip install -r Examples/Python/tests/requirements.txt + && python3 -m pip install pytest-md-report + && pytest -rFsv -k "not exatrkx" -v + && cat ${PYTEST_MD_REPORT_OUTPUT} >> $GITHUB_STEP_SUMMARY + + linux_physmon: + runs-on: ubuntu-latest + container: ghcr.io/acts-project/ubuntu2404:63 + needs: [linux_ubuntu] + env: + ACTS_SEQUENCER_DISABLE_FPEMON: true + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + lfs: true + + - run: apt-get update && apt-get install -y time + + - name: Install dependencies + uses: ./.github/actions/dependencies + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/download-artifact@v4 + with: + name: acts-linux-ubuntu + + - name: Unpack build + run: mkdir build && tar xf build.tar.gz -C build + + - name: Save PR number + if: ${{ github.event_name == 'pull_request' }} + run: | + mkdir -p physmon + echo ${{ github.event.number }} > physmon/pr_number + echo ${{ github.event.pull_request.head.sha }} > physmon/sha + + - name: Physics performance checks + shell: bash + run: > + echo "::group::Dependencies" + && git config --global safe.directory "$GITHUB_WORKSPACE" + && python3 -m pip install histcmp==0.6.8 matplotlib + && python3 -m pip install -r Examples/Scripts/requirements.txt + && geant4-config --install-datasets + && venv_python=$(which python3) + && echo $venv_python + && source build/this_acts_withdeps.sh + && export PATH=$(dirname $venv_python):$PATH + && echo $PATH + && which python3 + && echo "::endgroup::" + && CI/physmon/phys_perf_mon.sh all physmon + + - name: Post step summary + if: always() + run: cat physmon/summary.md >> $GITHUB_STEP_SUMMARY + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: physmon + path: physmon # # linux_physmon_perf_report: # needs: [linux_physmon] @@ -272,7 +270,7 @@ jobs: id: ccache-restore with: path: ${{ env.CCACHE_DIR }} - key: ccache-${{ runner.os }}-${{ github.job }}-${{ matrix.image }}-${{ matrix.std }}-${{ matrix.compiler }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} + key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ matrix.image }}-${{ matrix.std }}-${{ matrix.compiler }}-${{ github.sha }} restore-keys: | ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- @@ -349,10 +347,6 @@ jobs: restore-keys: | ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - # - uses: mxschmitt/action-tmate@v3 - # if: failure() - - - name: Configure run: > ccache -z && @@ -393,7 +387,6 @@ jobs: -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS=-Werror - -DCMAKE_CXX_STANDARD=20 -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" - name: Downstream build From debddb0d20048bd66a82ed0b9dd053e4e6b5c118 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 12:58:36 +0100 Subject: [PATCH 105/191] update hashes due to G4 version change --- Examples/Python/tests/root_file_hashes.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Examples/Python/tests/root_file_hashes.txt b/Examples/Python/tests/root_file_hashes.txt index 07d42c66ad1..3b16b9e63a0 100644 --- a/Examples/Python/tests/root_file_hashes.txt +++ b/Examples/Python/tests/root_file_hashes.txt @@ -1,8 +1,8 @@ test_pythia8__particles.root: 125182a9647ef3cec71afbc1b9e1676e40c13826c8333d6704345dd5133d5e91 test_fatras__particles_simulation.root: bc970873fef0c2efd86ed5413623802353d2cd04abea72de14e8cdfc0e40076f test_fatras__hits.root: 6e4beb045fa1712c4d14c280ba33c3fa13e4aff9de88d55c3e32f62ad226f724 -test_geant4__particles_simulation.root: 49926c71a9b54e13aa1cc7596d3302baf3c87d8e2c1d0267cb4523f6abdc0ac2 -test_geant4__hits.root: 4c9e704a75f47ed2e61652679a1d6f18fa4d9cf53faa8f8f5bbf7995634207aa +test_geant4__particles_simulation.root: fd10da8000715f7f515001670e713efd8df21501b32e9b2635f0152d63eecaf4 +test_geant4__hits.root: d8efffea763de487454c23c48bde26548df59f2342807b3b486eb98bdc6a1559 test_seeding__estimatedparams.root: 6759004f945cabe03098c94b3eea7e3323acd9f37edfa71641797007336643c8 test_seeding__performance_seeding.root: 992f9c611d30dde0d3f3ab676bab19ada61ab6a4442828e27b65ec5e5b7a2880 test_seeding__particles.root: c423bc666df3674f1a1140dec68ea13f44173232b8057e8a02572aee4f3e7d5b From f67fe6a634bf2d17f1e69a2ba057797696ace59f Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 13:06:17 +0100 Subject: [PATCH 106/191] include more in the cache key --- .github/workflows/builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 047d431599f..5bc26b27618 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -272,7 +272,7 @@ jobs: path: ${{ env.CCACHE_DIR }} key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ matrix.image }}-${{ matrix.std }}-${{ matrix.compiler }}-${{ github.sha }} restore-keys: | - ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- + ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ matrix.image }}-${{ matrix.std }}-${{ matrix.compiler }}- - name: Configure From 901b8749ee2bc0bb08b563c95cd256203406ddeb Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 13:25:30 +0100 Subject: [PATCH 107/191] try spack caching again --- .github/actions/dependencies/action.yml | 38 ++++++++++++++++++++++++- CI/dependencies/setup.sh | 20 ++----------- CI/dependencies/setup_spack.sh | 20 +++++++++++++ 3 files changed, 60 insertions(+), 18 deletions(-) create mode 100755 CI/dependencies/setup_spack.sh diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 17787e7035b..d9702eed9bf 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -10,6 +10,10 @@ inputs: description: 'Secrets to pass to the action' required: false default: '' + spack_version: + description: 'Version of Spack to use' + required: false + default: '2024-11-13' runs: using: 'composite' @@ -18,7 +22,39 @@ runs: if: startsWith(runner.os, 'macos') shell: bash run: | - brew install ninja ccache + brew install ninja ccache zstd + + - name: Restore cache + id: cache-restore + uses: actions/cache/restore@v3 + with: + path: spack.tar.zst + key: spack-${{ runner.os }}-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} + + - name: Extract cached Spack + if: steps.cache-restore.outputs.cache-hit == 'true' + shell: bash + run: | + zstd -d spack.tar.zst -f + tar xf spack.tar + rm spack.tar + + - name: Setup Spack from scratch + if: steps.cache-restore.outputs.cache-hit != 'true' + shell: bash + env: + SPACK_VERSION: ${{ inputs.spack_version }} + run: | + CI/dependencies/setup_spack.sh + tar cf spack.tar spack/ + zstd -19 --rm spack.tar + + - name: Save cache + if: steps.cache-restore.outputs.cache-hit != 'true' + uses: actions/cache/save@v3 + with: + path: spack.tar.zst + key: spack-${{ runner.os }}-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} - name: Install dependencies shell: bash diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index a90765625c0..80d439b5296 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -100,26 +100,12 @@ mkdir -p ${destination} start_section "Install spack if not already installed" if ! command -v spack &> /dev/null; then - if [ ! -d "spack" ]; then - echo "Cloning spack" - git clone -c feature.manyFiles=true https://github.com/spack/spack.git - pushd spack > /dev/null - git config user.name github-actions[bot] - git config user.email 41898282+github-actions[bot]@users.noreply.github.com - # Apply patch for spack improvements - curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am - popd > /dev/null - else - echo "Updating spack" - pushd spack > /dev/null - git pull --rebase - popd > /dev/null - fi - - source "$(pwd)/spack/share/spack/setup-env.sh" + "${SCRIPT_DIR}/setup_spack.sh" fi end_section +source "$(pwd)/spack/share/spack/setup-env.sh" + if [ -n "${CI:-}" ]; then start_section "Add buildcache mirror" mirror_name="acts-spack-buildcache" diff --git a/CI/dependencies/setup_spack.sh b/CI/dependencies/setup_spack.sh new file mode 100755 index 00000000000..3fc5a644f34 --- /dev/null +++ b/CI/dependencies/setup_spack.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +_spack_version=${SPACK_VERSION:-develop} + +if [ ! -d "spack" ]; then + echo "Cloning spack" + git clone -c feature.manyFiles=true https://github.com/spack/spack.git -b ${_spack_version} + pushd spack > /dev/null + git config user.name CI + git config user.email <> + echo "Applying patch for spack improvements" + curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am + curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/48236.patch | git am + + rm -rf .git + + echo "Populating the repository index" + bin/spack list > /dev/null + popd > /dev/null +fi From ae4856f75d935c57391d4aa2abc1254f78572735 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 13:27:51 +0100 Subject: [PATCH 108/191] syntax --- CI/dependencies/setup_spack.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CI/dependencies/setup_spack.sh b/CI/dependencies/setup_spack.sh index 3fc5a644f34..268d718bc03 100755 --- a/CI/dependencies/setup_spack.sh +++ b/CI/dependencies/setup_spack.sh @@ -6,8 +6,8 @@ if [ ! -d "spack" ]; then echo "Cloning spack" git clone -c feature.manyFiles=true https://github.com/spack/spack.git -b ${_spack_version} pushd spack > /dev/null - git config user.name CI - git config user.email <> + git config user.name 'CI' + git config user.email '<>' echo "Applying patch for spack improvements" curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/48236.patch | git am From 3f19f021be281312e9524989cf2e28da8eb65ca0 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 13:33:14 +0100 Subject: [PATCH 109/191] tag fix --- .github/actions/dependencies/action.yml | 2 +- CI/dependencies/setup_spack.sh | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index d9702eed9bf..7c5f23571ad 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -13,7 +13,7 @@ inputs: spack_version: description: 'Version of Spack to use' required: false - default: '2024-11-13' + default: 'develop-2024-12-15' runs: using: 'composite' diff --git a/CI/dependencies/setup_spack.sh b/CI/dependencies/setup_spack.sh index 268d718bc03..58e9abca3bc 100755 --- a/CI/dependencies/setup_spack.sh +++ b/CI/dependencies/setup_spack.sh @@ -1,10 +1,12 @@ #!/bin/bash +set -e +set -u _spack_version=${SPACK_VERSION:-develop} if [ ! -d "spack" ]; then echo "Cloning spack" - git clone -c feature.manyFiles=true https://github.com/spack/spack.git -b ${_spack_version} + git clone --branch ${_spack_version} -c feature.manyFiles=true https://github.com/spack/spack.git pushd spack > /dev/null git config user.name 'CI' git config user.email '<>' From d2dfaffe93464e8f256ae4684c7fe1a17dc8f7f3 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 13:40:24 +0100 Subject: [PATCH 110/191] switch repo, no patches --- CI/dependencies/setup_spack.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/CI/dependencies/setup_spack.sh b/CI/dependencies/setup_spack.sh index 58e9abca3bc..dba3939d282 100755 --- a/CI/dependencies/setup_spack.sh +++ b/CI/dependencies/setup_spack.sh @@ -6,13 +6,10 @@ _spack_version=${SPACK_VERSION:-develop} if [ ! -d "spack" ]; then echo "Cloning spack" - git clone --branch ${_spack_version} -c feature.manyFiles=true https://github.com/spack/spack.git + git clone --branch ${_spack_version} -c feature.manyFiles=true https://github.com/acts-project/spack.git pushd spack > /dev/null git config user.name 'CI' git config user.email '<>' - echo "Applying patch for spack improvements" - curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am - curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/48236.patch | git am rm -rf .git From 976552d4800de9e3e83a2620e5d5efbe84111042 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 13:42:10 +0100 Subject: [PATCH 111/191] special branch --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 7c5f23571ad..4a42c13c287 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -13,7 +13,7 @@ inputs: spack_version: description: 'Version of Spack to use' required: false - default: 'develop-2024-12-15' + default: 'develop-acts' runs: using: 'composite' From eae5a6aa93d9aae804e4e17b8b152128428cb950 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 13:50:14 +0100 Subject: [PATCH 112/191] single cache key --- .github/actions/dependencies/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 4a42c13c287..1195e236b26 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -29,7 +29,7 @@ runs: uses: actions/cache/restore@v3 with: path: spack.tar.zst - key: spack-${{ runner.os }}-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} + key: spack-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} - name: Extract cached Spack if: steps.cache-restore.outputs.cache-hit == 'true' @@ -54,7 +54,7 @@ runs: uses: actions/cache/save@v3 with: path: spack.tar.zst - key: spack-${{ runner.os }}-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} + key: spack-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} - name: Install dependencies shell: bash From ed30c34d07b1026f5f8c79f5c14fa178a240a003 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 14:07:49 +0100 Subject: [PATCH 113/191] let's keep the .spack directory as well --- .github/actions/dependencies/action.yml | 9 +++++---- CI/dependencies/setup.sh | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 1195e236b26..106542a7904 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -29,15 +29,16 @@ runs: uses: actions/cache/restore@v3 with: path: spack.tar.zst - key: spack-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} + key: spack-v2-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} - name: Extract cached Spack if: steps.cache-restore.outputs.cache-hit == 'true' shell: bash run: | - zstd -d spack.tar.zst -f tar xf spack.tar + tar xf -C ~ spack_cache.tar rm spack.tar + rm spack_cache.tar - name: Setup Spack from scratch if: steps.cache-restore.outputs.cache-hit != 'true' @@ -47,14 +48,14 @@ runs: run: | CI/dependencies/setup_spack.sh tar cf spack.tar spack/ - zstd -19 --rm spack.tar + tar cf spack_cache.tar -C ~ .spack - name: Save cache if: steps.cache-restore.outputs.cache-hit != 'true' uses: actions/cache/save@v3 with: path: spack.tar.zst - key: spack-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} + key: spack-v2-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} - name: Install dependencies shell: bash diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index 80d439b5296..a922c009e82 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -178,14 +178,14 @@ venv_dir="${view_dir}/venv" "${venv_dir}/bin/python3" -m pip install pyyaml jinja2 +end_section + +start_section "Set environment variables" if [ -n "${GITHUB_ACTIONS:-}" ]; then echo "${view_dir}/bin" >> "$GITHUB_PATH" echo "${venv_dir}/bin" >> "$GITHUB_PATH" fi set_env PATH "${venv_dir}/bin:${view_dir}/bin/:${PATH}" -end_section - -start_section "Set environment variables" set_env CMAKE_PREFIX_PATH "${venv_dir}:${view_dir}" set_env LD_LIBRARY_PATH "${view_dir}/lib" set_env ROOT_INCLUDE_PATH "${view_dir}/include" From 6e03ab4ee0eb3d81f4790e3b7726f71704bbc7ad Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 14:10:27 +0100 Subject: [PATCH 114/191] namwes, shuffle --- .github/actions/dependencies/action.yml | 6 +++++- CI/dependencies/setup_spack.sh | 5 ----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 106542a7904..10ef86a8daf 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -47,6 +47,8 @@ runs: SPACK_VERSION: ${{ inputs.spack_version }} run: | CI/dependencies/setup_spack.sh + bin/spack list > /dev/null + rm -rf spack/.git tar cf spack.tar spack/ tar cf spack_cache.tar -C ~ .spack @@ -54,7 +56,9 @@ runs: if: steps.cache-restore.outputs.cache-hit != 'true' uses: actions/cache/save@v3 with: - path: spack.tar.zst + path: | + spack.tar + spack_cache.tar key: spack-v2-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} - name: Install dependencies diff --git a/CI/dependencies/setup_spack.sh b/CI/dependencies/setup_spack.sh index dba3939d282..4cb0772e01e 100755 --- a/CI/dependencies/setup_spack.sh +++ b/CI/dependencies/setup_spack.sh @@ -10,10 +10,5 @@ if [ ! -d "spack" ]; then pushd spack > /dev/null git config user.name 'CI' git config user.email '<>' - - rm -rf .git - - echo "Populating the repository index" - bin/spack list > /dev/null popd > /dev/null fi From cdbc1ae2f7178cb2cba3658219ce78e8955ad840 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 14:12:02 +0100 Subject: [PATCH 115/191] binary path --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 10ef86a8daf..68fa9e50887 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -47,7 +47,7 @@ runs: SPACK_VERSION: ${{ inputs.spack_version }} run: | CI/dependencies/setup_spack.sh - bin/spack list > /dev/null + spack/bin/spack list > /dev/null rm -rf spack/.git tar cf spack.tar spack/ tar cf spack_cache.tar -C ~ .spack From 25dfad2f64d56f9e51f686c9747b96fd1a9e9138 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 14:29:03 +0100 Subject: [PATCH 116/191] no zstd install --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 68fa9e50887..8664480cd6e 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -22,7 +22,7 @@ runs: if: startsWith(runner.os, 'macos') shell: bash run: | - brew install ninja ccache zstd + brew install ninja ccache - name: Restore cache id: cache-restore From 30485e8fbfbceb3ae40c9902ca0b950057c3e6d7 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 14:30:26 +0100 Subject: [PATCH 117/191] key issue? --- .github/actions/dependencies/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 8664480cd6e..83c72aeefcf 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -29,7 +29,7 @@ runs: uses: actions/cache/restore@v3 with: path: spack.tar.zst - key: spack-v2-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} + key: spack-r1-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} - name: Extract cached Spack if: steps.cache-restore.outputs.cache-hit == 'true' @@ -59,7 +59,7 @@ runs: path: | spack.tar spack_cache.tar - key: spack-v2-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} + key: spack-r1-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} - name: Install dependencies shell: bash From 3cb1f223df78439cb0de7d632b037eed90c17500 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 15:01:36 +0100 Subject: [PATCH 118/191] trigger From 65617b64e8355bfb57346613d7bdf8c62cab4dd8 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 15:08:29 +0100 Subject: [PATCH 119/191] restore key? --- .github/actions/dependencies/action.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 83c72aeefcf..a3c305e1126 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -30,6 +30,8 @@ runs: with: path: spack.tar.zst key: spack-r1-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} + restore-keys: | + spack-r1-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} - name: Extract cached Spack if: steps.cache-restore.outputs.cache-hit == 'true' From 77f8be8636e5d9e1aef71e4feb4427be9660425c Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 15:15:21 +0100 Subject: [PATCH 120/191] keys? --- .github/actions/dependencies/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index a3c305e1126..6dc4ee1bc9b 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -29,7 +29,7 @@ runs: uses: actions/cache/restore@v3 with: path: spack.tar.zst - key: spack-r1-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} + key: spack-r1-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }}-${{ github.sha }} restore-keys: | spack-r1-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} @@ -61,7 +61,7 @@ runs: path: | spack.tar spack_cache.tar - key: spack-r1-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} + key: spack-r1-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }}-${{ github.sha }} - name: Install dependencies shell: bash From 47c1297638b8bad07b33856a4b6b68381172d417 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 6 Jan 2025 13:41:25 +0100 Subject: [PATCH 121/191] debug examples failure --- .github/workflows/builds.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 5bc26b27618..02e4b511280 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -148,6 +148,11 @@ jobs: && pytest -rFsv -k "not exatrkx" -v && cat ${PYTEST_MD_REPORT_OUTPUT} >> $GITHUB_STEP_SUMMARY + - uses: mxschmitt/action-tmate@v3 + if: failure() + + + linux_physmon: runs-on: ubuntu-latest container: ghcr.io/acts-project/ubuntu2404:63 From a127c69dc9b618c91c60615c769d4e0b9ef44590 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 6 Jan 2025 13:54:50 +0100 Subject: [PATCH 122/191] try runner specific to reduce race condition likelihood --- .github/actions/dependencies/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 6dc4ee1bc9b..f3eff6d1aa8 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -29,9 +29,9 @@ runs: uses: actions/cache/restore@v3 with: path: spack.tar.zst - key: spack-r1-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }}-${{ github.sha }} + key: spack-r1-${{ inputs.spack_version }}-${{ runner.os }}-${{ github.job }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} restore-keys: | - spack-r1-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} + spack-r1-${{ inputs.spack_version }}-${{ runner.os }}-${{ github.job }}- - name: Extract cached Spack if: steps.cache-restore.outputs.cache-hit == 'true' From 606ecb0be96f2cc1fed1901c22a90496b3e18c3e Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 6 Jan 2025 14:01:04 +0100 Subject: [PATCH 123/191] save with correct key --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index f3eff6d1aa8..f7566d0258c 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -61,7 +61,7 @@ runs: path: | spack.tar spack_cache.tar - key: spack-r1-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }}-${{ github.sha }} + key: ${{ steps.cache-restore.outputs.cache-primary-key }} - name: Install dependencies shell: bash From fb54b01047fcb460f928d182b8dcb359df33a764 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Tue, 7 Jan 2025 13:48:03 +0100 Subject: [PATCH 124/191] bump cache action --- .github/actions/dependencies/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index f7566d0258c..156330325ef 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -26,7 +26,7 @@ runs: - name: Restore cache id: cache-restore - uses: actions/cache/restore@v3 + uses: actions/cache/restore@v4 with: path: spack.tar.zst key: spack-r1-${{ inputs.spack_version }}-${{ runner.os }}-${{ github.job }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} @@ -56,7 +56,7 @@ runs: - name: Save cache if: steps.cache-restore.outputs.cache-hit != 'true' - uses: actions/cache/save@v3 + uses: actions/cache/save@v4 with: path: | spack.tar From 8ce198a97f9c4c6b8b4fe66cafb4310f2f982168 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Tue, 7 Jan 2025 13:48:09 +0100 Subject: [PATCH 125/191] bump dependency tag --- .github/workflows/builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 02e4b511280..21bf80387b7 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -20,7 +20,7 @@ env: CCACHE_DIR: ${{ github.workspace }}/ccache CCACHE_MAXSIZE: 500M CCACHE_KEY_SUFFIX: r2 - DEPENDENCY_TAG: v7_b6 + DEPENDENCY_TAG: v7_b7 jobs: linux_ubuntu: From 76f2c35b1a6cb85829b770035331071a69a8b9ba Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Tue, 7 Jan 2025 16:41:44 +0100 Subject: [PATCH 126/191] set compiler family on default job --- .github/workflows/builds.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 21bf80387b7..39d4e0a0629 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -39,6 +39,7 @@ jobs: - name: Install dependencies uses: ./.github/actions/dependencies with: + compiler: g++ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Restore ccache From 9d26de752d86bd0cf3ee687e51da118dfa6ca9c5 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 8 Jan 2025 10:33:39 +0100 Subject: [PATCH 127/191] debug root access --- .github/workflows/builds.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 39d4e0a0629..d4391f53503 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -196,14 +196,20 @@ jobs: && python3 -m pip install histcmp==0.6.8 matplotlib && python3 -m pip install -r Examples/Scripts/requirements.txt && geant4-config --install-datasets - && venv_python=$(which python3) - && echo $venv_python + # && venv_python=$(which python3) + # && echo $venv_python && source build/this_acts_withdeps.sh - && export PATH=$(dirname $venv_python):$PATH - && echo $PATH + # && export PATH=$(dirname $venv_python):$PATH + # && echo $PATH && which python3 + && echo $PATH + && cat build/this_acts_withdeps.sh && echo "::endgroup::" - && CI/physmon/phys_perf_mon.sh all physmon + # && CI/physmon/phys_perf_mon.sh all physmon + + + - uses: mxschmitt/action-tmate@v3 + - name: Post step summary if: always() From 818d4074de95301e8037a718b6d49dd06ce45daa Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 8 Jan 2025 11:43:55 +0100 Subject: [PATCH 128/191] output --- .github/workflows/builds.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index d4391f53503..58f8f5dc5b8 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -190,22 +190,22 @@ jobs: - name: Physics performance checks shell: bash - run: > - echo "::group::Dependencies" - && git config --global safe.directory "$GITHUB_WORKSPACE" - && python3 -m pip install histcmp==0.6.8 matplotlib - && python3 -m pip install -r Examples/Scripts/requirements.txt - && geant4-config --install-datasets - # && venv_python=$(which python3) - # && echo $venv_python - && source build/this_acts_withdeps.sh - # && export PATH=$(dirname $venv_python):$PATH - # && echo $PATH - && which python3 - && echo $PATH - && cat build/this_acts_withdeps.sh - && echo "::endgroup::" - # && CI/physmon/phys_perf_mon.sh all physmon + run: | + # echo "::group::Dependencies" + git config --global safe.directory "$GITHUB_WORKSPACE" + python3 -m pip install histcmp==0.6.8 matplotlib + python3 -m pip install -r Examples/Scripts/requirements.txt + # geant4-config --install-datasets + # venv_python=$(which python3) + # echo $venv_python + source build/this_acts_withdeps.sh + # export PATH=$(dirname $venv_python):$PATH + # echo $PATH + which python3 + echo $PATH + cat build/this_acts_withdeps.sh + # echo "::endgroup::" + # CI/physmon/phys_perf_mon.sh all physmon - uses: mxschmitt/action-tmate@v3 From 571978f3fbec2aa667d977759f2ae2d849c6af03 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 8 Jan 2025 12:54:40 +0100 Subject: [PATCH 129/191] use relative path for root setup script --- cmake/setup_withdeps.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/setup_withdeps.sh.in b/cmake/setup_withdeps.sh.in index ce1dc3506e7..3ff3772d87a 100644 --- a/cmake/setup_withdeps.sh.in +++ b/cmake/setup_withdeps.sh.in @@ -44,7 +44,7 @@ export DYLD_LIBRARY_PATH="$script_dir/lib:${DYLD_LIBRARY_PATH}" # activate dependencies if present if [[ -d "@ROOT_DIR@" ]]; then - . @ROOT_BINDIR@/thisroot.sh + . @ROOT_DIR@/../../../thisroot.sh fi if [[ -d "@Geant4_DIR@" ]]; then . @Geant4_INCLUDE_DIR@/../../bin/geant4.sh From dd40048068e839444f12a7eb75d47d1224975871 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 8 Jan 2025 13:03:50 +0100 Subject: [PATCH 130/191] fix compiler in macos --- .github/workflows/builds.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 58f8f5dc5b8..750bdbf1cda 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -349,6 +349,7 @@ jobs: uses: ./.github/actions/dependencies with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + compiler: clang++ - name: Restore ccache uses: actions/cache/restore@v4 From d13fd0e11e53627494486ddae3830a2764e31e87 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 8 Jan 2025 13:04:10 +0100 Subject: [PATCH 131/191] fixes --- .github/actions/dependencies/action.yml | 1 - .github/workflows/builds.yml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 156330325ef..84aaeca2d28 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -68,7 +68,6 @@ runs: env: GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN }} run: | - pwd if [ -n "${{ inputs.compiler }}" ]; then echo "With compiler" CI/dependencies/setup.sh -c "${{ inputs.compiler }}" diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 750bdbf1cda..7b82a86b1f3 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -209,6 +209,7 @@ jobs: - uses: mxschmitt/action-tmate@v3 + if: failure() - name: Post step summary From 97086920b5ee1dce824a3cef77a65eaa4725c405 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 8 Jan 2025 13:04:46 +0100 Subject: [PATCH 132/191] disable extra builds --- .github/workflows/builds.yml | 166 +++++++++++++++++------------------ 1 file changed, 83 insertions(+), 83 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 7b82a86b1f3..1f622bfcb6e 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -250,89 +250,89 @@ jobs: # git commit -m"update metrics" # git push # - linux_ubuntu_extra: - runs-on: ubuntu-latest - strategy: - matrix: - include: - - image: ubuntu2204 - std: 20 - compiler: g++ - - image: ubuntu2204 - std: 20 - compiler: clang++ - container: ghcr.io/acts-project/${{ matrix.image }}:71 - env: - INSTALL_DIR: ${{ github.workspace }}/install - ACTS_LOG_FAILURE_THRESHOLD: WARNING - - steps: - - uses: actions/checkout@v4 - with: - submodules: true - lfs: true - - - name: Install dependencies - uses: ./.github/actions/dependencies - with: - compiler: ${{ matrix.compiler }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Restore ccache - uses: actions/cache/restore@v4 - id: ccache-restore - with: - path: ${{ env.CCACHE_DIR }} - key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ matrix.image }}-${{ matrix.std }}-${{ matrix.compiler }}-${{ github.sha }} - restore-keys: | - ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ matrix.image }}-${{ matrix.std }}-${{ matrix.compiler }}- - - - - name: Configure - run: > - ccache -z && - cmake -B build -S . - --preset=github-ci - -DCMAKE_CXX_STANDARD=${{ matrix.std }} - -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} - -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" - - - name: Build - run: cmake --build build - - - name: ccache stats - run: ccache -s - - - name: Save ccache - uses: actions/cache/save@v4 - if: always() - with: - path: ${{ github.workspace }}/ccache - key: ${{ steps.ccache-restore.outputs.cache-primary-key }} - - - name: Unit tests - run: cmake --build build --target test - - - name: Integration tests - run: cmake --build build --target integrationtests - - - name: Install - run: cmake --build build --target install - - - name: Downstream configure - run: > - cmake -B build-downstream -S Tests/DownstreamProject - -GNinja - -DCMAKE_BUILD_TYPE=Release - -DCMAKE_CXX_FLAGS=-Werror - -DCMAKE_CXX_STANDARD=${{ matrix.std }} - -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" - - - name: Downstream build - run: cmake --build build-downstream - - - name: Downstream run - run: ./build-downstream/bin/ShowActsVersion + # linux_ubuntu_extra: + # runs-on: ubuntu-latest + # strategy: + # matrix: + # include: + # - image: ubuntu2204 + # std: 20 + # compiler: g++ + # - image: ubuntu2204 + # std: 20 + # compiler: clang++ + # container: ghcr.io/acts-project/${{ matrix.image }}:71 + # env: + # INSTALL_DIR: ${{ github.workspace }}/install + # ACTS_LOG_FAILURE_THRESHOLD: WARNING + # + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: true + # lfs: true + # + # - name: Install dependencies + # uses: ./.github/actions/dependencies + # with: + # compiler: ${{ matrix.compiler }} + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # + # - name: Restore ccache + # uses: actions/cache/restore@v4 + # id: ccache-restore + # with: + # path: ${{ env.CCACHE_DIR }} + # key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ matrix.image }}-${{ matrix.std }}-${{ matrix.compiler }}-${{ github.sha }} + # restore-keys: | + # ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ matrix.image }}-${{ matrix.std }}-${{ matrix.compiler }}- + # + # + # - name: Configure + # run: > + # ccache -z && + # cmake -B build -S . + # --preset=github-ci + # -DCMAKE_CXX_STANDARD=${{ matrix.std }} + # -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} + # -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + # + # - name: Build + # run: cmake --build build + # + # - name: ccache stats + # run: ccache -s + # + # - name: Save ccache + # uses: actions/cache/save@v4 + # if: always() + # with: + # path: ${{ github.workspace }}/ccache + # key: ${{ steps.ccache-restore.outputs.cache-primary-key }} + # + # - name: Unit tests + # run: cmake --build build --target test + # + # - name: Integration tests + # run: cmake --build build --target integrationtests + # + # - name: Install + # run: cmake --build build --target install + # + # - name: Downstream configure + # run: > + # cmake -B build-downstream -S Tests/DownstreamProject + # -GNinja + # -DCMAKE_BUILD_TYPE=Release + # -DCMAKE_CXX_FLAGS=-Werror + # -DCMAKE_CXX_STANDARD=${{ matrix.std }} + # -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" + # + # - name: Downstream build + # run: cmake --build build-downstream + # + # - name: Downstream run + # run: ./build-downstream/bin/ShowActsVersion macos: runs-on: macos-14 From 0aaa5b5183824471e337fbd2918365b4ed006d4b Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 8 Jan 2025 13:04:51 +0100 Subject: [PATCH 133/191] change cache key --- .github/actions/dependencies/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 84aaeca2d28..1fc6a5e5cad 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -29,9 +29,9 @@ runs: uses: actions/cache/restore@v4 with: path: spack.tar.zst - key: spack-r1-${{ inputs.spack_version }}-${{ runner.os }}-${{ github.job }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} + key: spack-r2-${{ inputs.spack_version }}-${{ runner.os }}-${{ github.job }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} restore-keys: | - spack-r1-${{ inputs.spack_version }}-${{ runner.os }}-${{ github.job }}- + spack-r2-${{ inputs.spack_version }}-${{ runner.os }}-${{ github.job }}- - name: Extract cached Spack if: steps.cache-restore.outputs.cache-hit == 'true' From aebf0302ae6e2c4300195140dabcd5c1231a7578 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 8 Jan 2025 13:13:42 +0100 Subject: [PATCH 134/191] missing bin --- cmake/setup_withdeps.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/setup_withdeps.sh.in b/cmake/setup_withdeps.sh.in index 3ff3772d87a..badf09eafb9 100644 --- a/cmake/setup_withdeps.sh.in +++ b/cmake/setup_withdeps.sh.in @@ -44,7 +44,7 @@ export DYLD_LIBRARY_PATH="$script_dir/lib:${DYLD_LIBRARY_PATH}" # activate dependencies if present if [[ -d "@ROOT_DIR@" ]]; then - . @ROOT_DIR@/../../../thisroot.sh + . @ROOT_DIR@/../../../bin/thisroot.sh fi if [[ -d "@Geant4_DIR@" ]]; then . @Geant4_INCLUDE_DIR@/../../bin/geant4.sh From 2b7eebb5c488b1ec2c0404ae66e5b2e6ad8ffa02 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 8 Jan 2025 13:29:33 +0100 Subject: [PATCH 135/191] test root import --- .github/workflows/builds.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 1f622bfcb6e..72e554a0e48 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -195,15 +195,15 @@ jobs: git config --global safe.directory "$GITHUB_WORKSPACE" python3 -m pip install histcmp==0.6.8 matplotlib python3 -m pip install -r Examples/Scripts/requirements.txt - # geant4-config --install-datasets - # venv_python=$(which python3) - # echo $venv_python + geant4-config --install-datasets + venv_python=$(which python3) + echo $venv_python source build/this_acts_withdeps.sh - # export PATH=$(dirname $venv_python):$PATH - # echo $PATH + export PATH=$(dirname $venv_python):$PATH which python3 echo $PATH - cat build/this_acts_withdeps.sh + # cat build/this_acts_withdeps.sh + python -c "import ROOT" # echo "::endgroup::" # CI/physmon/phys_perf_mon.sh all physmon From 9f0d9e47b45f57510f071c4b93bbc2a01f7394e5 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 8 Jan 2025 13:58:35 +0100 Subject: [PATCH 136/191] restore builds --- .github/workflows/builds.yml | 174 +++++++++++++++++------------------ 1 file changed, 86 insertions(+), 88 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 72e554a0e48..fe77709b46f 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -191,7 +191,7 @@ jobs: - name: Physics performance checks shell: bash run: | - # echo "::group::Dependencies" + echo "::group::Dependencies" git config --global safe.directory "$GITHUB_WORKSPACE" python3 -m pip install histcmp==0.6.8 matplotlib python3 -m pip install -r Examples/Scripts/requirements.txt @@ -202,10 +202,8 @@ jobs: export PATH=$(dirname $venv_python):$PATH which python3 echo $PATH - # cat build/this_acts_withdeps.sh - python -c "import ROOT" - # echo "::endgroup::" - # CI/physmon/phys_perf_mon.sh all physmon + echo "::endgroup::" + CI/physmon/phys_perf_mon.sh all physmon - uses: mxschmitt/action-tmate@v3 @@ -250,89 +248,89 @@ jobs: # git commit -m"update metrics" # git push # - # linux_ubuntu_extra: - # runs-on: ubuntu-latest - # strategy: - # matrix: - # include: - # - image: ubuntu2204 - # std: 20 - # compiler: g++ - # - image: ubuntu2204 - # std: 20 - # compiler: clang++ - # container: ghcr.io/acts-project/${{ matrix.image }}:71 - # env: - # INSTALL_DIR: ${{ github.workspace }}/install - # ACTS_LOG_FAILURE_THRESHOLD: WARNING - # - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # lfs: true - # - # - name: Install dependencies - # uses: ./.github/actions/dependencies - # with: - # compiler: ${{ matrix.compiler }} - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # - # - name: Restore ccache - # uses: actions/cache/restore@v4 - # id: ccache-restore - # with: - # path: ${{ env.CCACHE_DIR }} - # key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ matrix.image }}-${{ matrix.std }}-${{ matrix.compiler }}-${{ github.sha }} - # restore-keys: | - # ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ matrix.image }}-${{ matrix.std }}-${{ matrix.compiler }}- - # - # - # - name: Configure - # run: > - # ccache -z && - # cmake -B build -S . - # --preset=github-ci - # -DCMAKE_CXX_STANDARD=${{ matrix.std }} - # -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} - # -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" - # - # - name: Build - # run: cmake --build build - # - # - name: ccache stats - # run: ccache -s - # - # - name: Save ccache - # uses: actions/cache/save@v4 - # if: always() - # with: - # path: ${{ github.workspace }}/ccache - # key: ${{ steps.ccache-restore.outputs.cache-primary-key }} - # - # - name: Unit tests - # run: cmake --build build --target test - # - # - name: Integration tests - # run: cmake --build build --target integrationtests - # - # - name: Install - # run: cmake --build build --target install - # - # - name: Downstream configure - # run: > - # cmake -B build-downstream -S Tests/DownstreamProject - # -GNinja - # -DCMAKE_BUILD_TYPE=Release - # -DCMAKE_CXX_FLAGS=-Werror - # -DCMAKE_CXX_STANDARD=${{ matrix.std }} - # -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" - # - # - name: Downstream build - # run: cmake --build build-downstream - # - # - name: Downstream run - # run: ./build-downstream/bin/ShowActsVersion + linux_ubuntu_extra: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - image: ubuntu2204 + std: 20 + compiler: g++ + - image: ubuntu2204 + std: 20 + compiler: clang++ + container: ghcr.io/acts-project/${{ matrix.image }}:71 + env: + INSTALL_DIR: ${{ github.workspace }}/install + ACTS_LOG_FAILURE_THRESHOLD: WARNING + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + lfs: true + + - name: Install dependencies + uses: ./.github/actions/dependencies + with: + compiler: ${{ matrix.compiler }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Restore ccache + uses: actions/cache/restore@v4 + id: ccache-restore + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ matrix.image }}-${{ matrix.std }}-${{ matrix.compiler }}-${{ github.sha }} + restore-keys: | + ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ matrix.image }}-${{ matrix.std }}-${{ matrix.compiler }}- + + + - name: Configure + run: > + ccache -z && + cmake -B build -S . + --preset=github-ci + -DCMAKE_CXX_STANDARD=${{ matrix.std }} + -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + + - name: Build + run: cmake --build build + + - name: ccache stats + run: ccache -s + + - name: Save ccache + uses: actions/cache/save@v4 + if: always() + with: + path: ${{ github.workspace }}/ccache + key: ${{ steps.ccache-restore.outputs.cache-primary-key }} + + - name: Unit tests + run: cmake --build build --target test + + - name: Integration tests + run: cmake --build build --target integrationtests + + - name: Install + run: cmake --build build --target install + + - name: Downstream configure + run: > + cmake -B build-downstream -S Tests/DownstreamProject + -GNinja + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_CXX_FLAGS=-Werror + -DCMAKE_CXX_STANDARD=${{ matrix.std }} + -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" + + - name: Downstream build + run: cmake --build build-downstream + + - name: Downstream run + run: ./build-downstream/bin/ShowActsVersion macos: runs-on: macos-14 From 00f3ede3490ed6c9dd52ee4650ca05836b1cabd6 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 8 Jan 2025 15:34:21 +0100 Subject: [PATCH 137/191] hopefully fix cache --- .github/actions/dependencies/action.yml | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 1fc6a5e5cad..25d67c5f584 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -28,19 +28,15 @@ runs: id: cache-restore uses: actions/cache/restore@v4 with: - path: spack.tar.zst - key: spack-r2-${{ inputs.spack_version }}-${{ runner.os }}-${{ github.job }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} - restore-keys: | - spack-r2-${{ inputs.spack_version }}-${{ runner.os }}-${{ github.job }}- + path: spack_cache + key: spack-r2-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} - name: Extract cached Spack if: steps.cache-restore.outputs.cache-hit == 'true' shell: bash run: | - tar xf spack.tar - tar xf -C ~ spack_cache.tar - rm spack.tar - rm spack_cache.tar + mv spack_cache/spack . + mv spack_cache/.spack ~ - name: Setup Spack from scratch if: steps.cache-restore.outputs.cache-hit != 'true' @@ -51,16 +47,16 @@ runs: CI/dependencies/setup_spack.sh spack/bin/spack list > /dev/null rm -rf spack/.git - tar cf spack.tar spack/ - tar cf spack_cache.tar -C ~ .spack + mkdir spack_cache + cp -r spack spack_cache/ + cp -r ~/.spack spack_cache/ - name: Save cache if: steps.cache-restore.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: path: | - spack.tar - spack_cache.tar + spack_cache key: ${{ steps.cache-restore.outputs.cache-primary-key }} - name: Install dependencies From 0d7ddff2056ff744374bba6c7229b7c69328136b Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 8 Jan 2025 16:14:31 +0100 Subject: [PATCH 138/191] disable GHA builds (to save resources) --- .github/workflows/{builds.yml => builds.yml.disabled} | 8 -------- 1 file changed, 8 deletions(-) rename .github/workflows/{builds.yml => builds.yml.disabled} (98%) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml.disabled similarity index 98% rename from .github/workflows/builds.yml rename to .github/workflows/builds.yml.disabled index fe77709b46f..13b1ba545c8 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml.disabled @@ -51,9 +51,6 @@ jobs: restore-keys: | ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - # - uses: mxschmitt/action-tmate@v3 - # if: failure() - - name: Configure # setting CMAKE_CXX_STANDARD=20 is a workaround for a bug in the # dd4hep CMake configuration that gets triggered on recent CMake @@ -149,11 +146,6 @@ jobs: && pytest -rFsv -k "not exatrkx" -v && cat ${PYTEST_MD_REPORT_OUTPUT} >> $GITHUB_STEP_SUMMARY - - uses: mxschmitt/action-tmate@v3 - if: failure() - - - linux_physmon: runs-on: ubuntu-latest container: ghcr.io/acts-project/ubuntu2404:63 From 6fccdb845d081beee799af3485072e9928664596 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 8 Jan 2025 16:46:30 +0100 Subject: [PATCH 139/191] enable gitlab --- .gitlab-ci.yml | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 25aff340111..2618d576738 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,14 +29,13 @@ clang_tidy: && ln -sf /usr/bin/clang-18 /usr/bin/clang && ln -sf /usr/bin/clang-tidy-18 /usr/bin/clang-tidy - - source CI/dependencies.sh + - source CI/dependencies/setup.sh - > cmake -B build -S . --preset=gitlab-ci-clangtidy -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang - -DPython_EXECUTABLE=$(which python3) # Main clang-tidy run during cmake compilation - CI/clang_tidy/run_clang_tidy.sh clang-tidy build @@ -83,14 +82,13 @@ build_exatrkx_cpu: - git clone $CLONE_URL src - cd src - git checkout $HEAD_SHA - - source CI/dependencies.sh + - source CI/dependencies/setup.sh - cd .. - mkdir build # Here we only do a minimal build without examples to save resources - > cmake -B build -S src --preset=gitlab-ci-exatrkx - -DPython_EXECUTABLE=$(which python3) -DACTS_EXATRKX_ENABLE_CUDA=OFF - ccache -z @@ -129,13 +127,12 @@ build_exatrkx: - git clone $CLONE_URL src - cd src - git checkout $HEAD_SHA - - source CI/dependencies.sh + - source CI/dependencies/setup.sh - cd .. - mkdir build - > cmake -B build -S src --preset=gitlab-ci-exatrkx - -DPython_EXECUTABLE=$(which python3) -DCMAKE_CUDA_ARCHITECTURES="75;86" - ccache -z @@ -157,7 +154,7 @@ test_exatrkx_unittests: - git clone $CLONE_URL src - cd src - git checkout $HEAD_SHA - - source CI/dependencies.sh + - source CI/dependencies/setup.sh - cd .. - ctest --test-dir build -R ExaTrkX @@ -176,7 +173,7 @@ test_exatrkx_python: - cd src - git checkout $HEAD_SHA - nvidia-smi - - source CI/dependencies.sh + - source CI/dependencies/setup.sh - source ../build/this_acts_withdeps.sh - python3 -m pip install -r Examples/Python/tests/requirements.txt - echo $PYTHONPATH @@ -217,14 +214,13 @@ build_linux_ubuntu: - git checkout $HEAD_SHA - git submodule init - git submodule update - - source CI/dependencies.sh + - source CI/dependencies/setup.sh - cd .. - mkdir build - > cmake -B build -S src --preset=gitlab-ci - -DPython_EXECUTABLE=$(which python3) -DACTS_BUILD_PLUGIN_ONNX=ON - ccache -z @@ -246,7 +242,7 @@ linux_test_examples: - git checkout $HEAD_SHA - git submodule init - git submodule update - - source CI/dependencies.sh + - source CI/dependencies/setup.sh - cd .. - geant4-config --install-datasets @@ -276,7 +272,7 @@ linux_physmon: - git checkout $HEAD_SHA - git submodule init - git submodule update - - source CI/dependencies.sh + - source CI/dependencies/setup.sh - cd .. - git config --global safe.directory "$GITHUB_WORKSPACE" @@ -324,7 +320,6 @@ linux_physmon: - > cmake -B build -S src --preset=gitlab-ci - -DPython_EXECUTABLE=$(which python3) -DCMAKE_CXX_STANDARD=${CXXSTD} - ccache -z @@ -421,7 +416,6 @@ linux_ubuntu_2204_clang: cmake -B build -S src --preset=gitlab-ci -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" - -DPython_EXECUTABLE=$(which python3) -DACTS_BUILD_PLUGIN_GEOMODEL=OFF # GeoModel is not in LCG at this point - ccache -z From 5d5527d6f2a5d1296b4ef200410b01ef9b246644 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 9 Jan 2025 10:21:48 +0100 Subject: [PATCH 140/191] update tag in gitlab --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2618d576738..d01a144aeb7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,7 @@ variables: CCACHE_KEY_SUFFIX: r2 CTEST_OUTPUT_ON_FAILURE: 1 - DEPENDENCY_TAG: v5 + DEPENDENCY_TAG: v7_b7 clang_tidy: stage: build From c93e082bd57ea81b685a17093216115a76b6e3df Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 9 Jan 2025 10:28:29 +0100 Subject: [PATCH 141/191] fix ubuntu extra --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d01a144aeb7..0d6cf3ab9d8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -313,7 +313,7 @@ linux_physmon: - git checkout $HEAD_SHA - git submodule init - git submodule update - - source CI/dependencies.sh + - source CI/dependencies/setup.sh - cd .. - mkdir build From 92aa7c760aa2f8239cd67ddfb08326c9acd5578d Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 9 Jan 2025 10:38:13 +0100 Subject: [PATCH 142/191] use cern mirror on gitlab ci --- CI/dependencies/setup.sh | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index a922c009e82..7ed14ed6a6b 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -107,15 +107,19 @@ end_section source "$(pwd)/spack/share/spack/setup-env.sh" if [ -n "${CI:-}" ]; then -start_section "Add buildcache mirror" -mirror_name="acts-spack-buildcache" -mirror_url="oci://ghcr.io/acts-project/spack-buildcache" -spack mirror add ${mirror_name} ${mirror_url} --unsigned -end_section + start_section "Add buildcache mirror" + mirror_name="acts-spack-buildcache" + mirror_url="oci://ghcr.io/acts-project/spack-buildcache" + if [ -n "${GITLAB_CI:-}" ]; then + # Use CERN mirror for non-Github Actions + mirror_url="oci://registry.cern.ch/ghcr.io/acts-project/spack-buildcache" + fi + spack mirror add ${mirror_name} ${mirror_url} --unsigned + end_section -start_section "Locate OpenGL" -"${SCRIPT_DIR}/opengl.sh" -end_section + start_section "Locate OpenGL" + "${SCRIPT_DIR}/opengl.sh" + end_section fi start_section "Get spack lock file" From 9006f67fba9acfa539bcd0e5185e4c961b3fb00f Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 9 Jan 2025 10:39:24 +0100 Subject: [PATCH 143/191] disable unrelated jobs for now --- .gitlab-ci.yml | 522 ++++++++++++++++++++++++------------------------- 1 file changed, 261 insertions(+), 261 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0d6cf3ab9d8..8d6a2ff3076 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,182 +6,182 @@ variables: DEPENDENCY_TAG: v7_b7 -clang_tidy: - stage: build - image: ghcr.io/acts-project/ubuntu2404:63 - tags: - - large - artifacts: - paths: - - src/clang-tidy/ - when: always - expire_in: 1 week - variables: - DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.$DEPENDENCY_TAG.tar.zst - script: - - git clone $CLONE_URL src - - cd src - - git checkout $HEAD_SHA - - > - apt-get update - && apt-get install -y clang-tidy-18 - && ln -sf /usr/bin/clang++-18 /usr/bin/clang++ - && ln -sf /usr/bin/clang-18 /usr/bin/clang - && ln -sf /usr/bin/clang-tidy-18 /usr/bin/clang-tidy - - - source CI/dependencies/setup.sh - - - > - cmake -B build -S . - --preset=gitlab-ci-clangtidy - -DCMAKE_CXX_COMPILER=clang++ - -DCMAKE_C_COMPILER=clang - - # Main clang-tidy run during cmake compilation - - CI/clang_tidy/run_clang_tidy.sh clang-tidy build - - # Install dependencies for processing scripts - - python3 -m pip install -r CI/clang_tidy/requirements.txt - - # Parse the main clang-tidy run - - > - CI/clang_tidy/parse_clang_tidy.py - clang-tidy/clang-tidy.log - clang-tidy/clang-tidy.json - --exclude "*thirdparty*" - --exclude "*ActsPodioEdm*" - --exclude "*build/_deps/*" - - # Check the combined report against the defined limits - - CI/clang_tidy/check_clang_tidy.py --report clang-tidy/clang-tidy.json --config CI/clang_tidy/limits.yml - - # Generate an html report - - codereport clang-tidy/clang-tidy.json clang-tidy/html - -build_exatrkx_cpu: - stage: build - image: ghcr.io/acts-project/ubuntu2204_exatrkx:63 - variables: - DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst - tags: - - large - - cache: - key: ccache-${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CCACHE_KEY_SUFFIX} - fallback_keys: - - ccache-${CI_JOB_NAME}-${CI_DEFAULT_BRANCH}-${CCACHE_KEY_SUFFIX} - when: always - paths: - - ${CCACHE_DIR} - - script: - - export PATH=/usr/local/sbin:/usr/sbin:/sbin:$PATH - - export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:$PATH - - export TORCH_CUDA_ARCH_LIST="8.0 8.6 8.9 9.0" - - echo $PATH - - git clone $CLONE_URL src - - cd src - - git checkout $HEAD_SHA - - source CI/dependencies/setup.sh - - cd .. - - mkdir build - # Here we only do a minimal build without examples to save resources - - > - cmake -B build -S src - --preset=gitlab-ci-exatrkx - -DACTS_EXATRKX_ENABLE_CUDA=OFF - - - ccache -z - - cmake --build build -- -j6 - - ccache -s - -build_exatrkx: - stage: build - image: ghcr.io/acts-project/ubuntu2204_exatrkx:63 - variables: - DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst - tags: - - large - - cache: - key: ccache-${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CCACHE_KEY_SUFFIX} - fallback_keys: - - ccache-${CI_JOB_NAME}-${CI_DEFAULT_BRANCH}-${CCACHE_KEY_SUFFIX} - when: always - paths: - - ${CCACHE_DIR} - - artifacts: - paths: - - build/ - exclude: - - build/**/*.o - - build/bin/ActsIntegrationTest* - expire_in: 6 hours - - script: - - export PATH=/usr/local/sbin:/usr/sbin:/sbin:$PATH - - export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:$PATH - - export TORCH_CUDA_ARCH_LIST="8.0 8.6 8.9 9.0" - - echo $PATH - - git clone $CLONE_URL src - - cd src - - git checkout $HEAD_SHA - - source CI/dependencies/setup.sh - - cd .. - - mkdir build - - > - cmake -B build -S src - --preset=gitlab-ci-exatrkx - -DCMAKE_CUDA_ARCHITECTURES="75;86" - - - ccache -z - - cmake --build build -- -j6 - - ccache -s - -test_exatrkx_unittests: - stage: test - needs: - - build_exatrkx - image: ghcr.io/acts-project/ubuntu2204_exatrkx:63 - variables: - DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst - tags: - - docker-gpu-nvidia - script: - - - apt-get update -y - - git clone $CLONE_URL src - - cd src - - git checkout $HEAD_SHA - - source CI/dependencies/setup.sh - - cd .. - - ctest --test-dir build -R ExaTrkX - -test_exatrkx_python: - stage: test - needs: - - build_exatrkx - image: ghcr.io/acts-project/ubuntu2204_exatrkx:63 - variables: - DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst - tags: - - docker-gpu-nvidia - script: - - apt-get update -y - - git clone $CLONE_URL src - - cd src - - git checkout $HEAD_SHA - - nvidia-smi - - source CI/dependencies/setup.sh - - source ../build/this_acts_withdeps.sh - - python3 -m pip install -r Examples/Python/tests/requirements.txt - - echo $PYTHONPATH - - which python3 - - python3 --version - - python3 -c "import acts" - - pytest -rFsv -k torch --collect-only - - pytest -rFsv -k gpu-torch # For now only test torch GPU pipeline +# clang_tidy: +# stage: build +# image: ghcr.io/acts-project/ubuntu2404:63 +# tags: +# - large +# artifacts: +# paths: +# - src/clang-tidy/ +# when: always +# expire_in: 1 week +# variables: +# DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.$DEPENDENCY_TAG.tar.zst +# script: +# - git clone $CLONE_URL src +# - cd src +# - git checkout $HEAD_SHA +# - > +# apt-get update +# && apt-get install -y clang-tidy-18 +# && ln -sf /usr/bin/clang++-18 /usr/bin/clang++ +# && ln -sf /usr/bin/clang-18 /usr/bin/clang +# && ln -sf /usr/bin/clang-tidy-18 /usr/bin/clang-tidy +# +# - source CI/dependencies/setup.sh +# +# - > +# cmake -B build -S . +# --preset=gitlab-ci-clangtidy +# -DCMAKE_CXX_COMPILER=clang++ +# -DCMAKE_C_COMPILER=clang +# +# # Main clang-tidy run during cmake compilation +# - CI/clang_tidy/run_clang_tidy.sh clang-tidy build +# +# # Install dependencies for processing scripts +# - python3 -m pip install -r CI/clang_tidy/requirements.txt +# +# # Parse the main clang-tidy run +# - > +# CI/clang_tidy/parse_clang_tidy.py +# clang-tidy/clang-tidy.log +# clang-tidy/clang-tidy.json +# --exclude "*thirdparty*" +# --exclude "*ActsPodioEdm*" +# --exclude "*build/_deps/*" +# +# # Check the combined report against the defined limits +# - CI/clang_tidy/check_clang_tidy.py --report clang-tidy/clang-tidy.json --config CI/clang_tidy/limits.yml +# +# # Generate an html report +# - codereport clang-tidy/clang-tidy.json clang-tidy/html + +# build_exatrkx_cpu: +# stage: build +# image: ghcr.io/acts-project/ubuntu2204_exatrkx:63 +# variables: +# DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst +# tags: +# - large +# +# cache: +# key: ccache-${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CCACHE_KEY_SUFFIX} +# fallback_keys: +# - ccache-${CI_JOB_NAME}-${CI_DEFAULT_BRANCH}-${CCACHE_KEY_SUFFIX} +# when: always +# paths: +# - ${CCACHE_DIR} +# +# script: +# - export PATH=/usr/local/sbin:/usr/sbin:/sbin:$PATH +# - export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:$PATH +# - export TORCH_CUDA_ARCH_LIST="8.0 8.6 8.9 9.0" +# - echo $PATH +# - git clone $CLONE_URL src +# - cd src +# - git checkout $HEAD_SHA +# - source CI/dependencies/setup.sh +# - cd .. +# - mkdir build +# # Here we only do a minimal build without examples to save resources +# - > +# cmake -B build -S src +# --preset=gitlab-ci-exatrkx +# -DACTS_EXATRKX_ENABLE_CUDA=OFF +# +# - ccache -z +# - cmake --build build -- -j6 +# - ccache -s +# +# build_exatrkx: +# stage: build +# image: ghcr.io/acts-project/ubuntu2204_exatrkx:63 +# variables: +# DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst +# tags: +# - large +# +# cache: +# key: ccache-${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CCACHE_KEY_SUFFIX} +# fallback_keys: +# - ccache-${CI_JOB_NAME}-${CI_DEFAULT_BRANCH}-${CCACHE_KEY_SUFFIX} +# when: always +# paths: +# - ${CCACHE_DIR} +# +# artifacts: +# paths: +# - build/ +# exclude: +# - build/**/*.o +# - build/bin/ActsIntegrationTest* +# expire_in: 6 hours +# +# script: +# - export PATH=/usr/local/sbin:/usr/sbin:/sbin:$PATH +# - export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:$PATH +# - export TORCH_CUDA_ARCH_LIST="8.0 8.6 8.9 9.0" +# - echo $PATH +# - git clone $CLONE_URL src +# - cd src +# - git checkout $HEAD_SHA +# - source CI/dependencies/setup.sh +# - cd .. +# - mkdir build +# - > +# cmake -B build -S src +# --preset=gitlab-ci-exatrkx +# -DCMAKE_CUDA_ARCHITECTURES="75;86" +# +# - ccache -z +# - cmake --build build -- -j6 +# - ccache -s +# +# test_exatrkx_unittests: +# stage: test +# needs: +# - build_exatrkx +# image: ghcr.io/acts-project/ubuntu2204_exatrkx:63 +# variables: +# DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst +# tags: +# - docker-gpu-nvidia +# script: +# +# - apt-get update -y +# - git clone $CLONE_URL src +# - cd src +# - git checkout $HEAD_SHA +# - source CI/dependencies/setup.sh +# - cd .. +# - ctest --test-dir build -R ExaTrkX +# +# test_exatrkx_python: +# stage: test +# needs: +# - build_exatrkx +# image: ghcr.io/acts-project/ubuntu2204_exatrkx:63 +# variables: +# DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst +# tags: +# - docker-gpu-nvidia +# script: +# - apt-get update -y +# - git clone $CLONE_URL src +# - cd src +# - git checkout $HEAD_SHA +# - nvidia-smi +# - source CI/dependencies/setup.sh +# - source ../build/this_acts_withdeps.sh +# - python3 -m pip install -r Examples/Python/tests/requirements.txt +# - echo $PYTHONPATH +# - which python3 +# - python3 --version +# - python3 -c "import acts" +# - pytest -rFsv -k torch --collect-only +# - pytest -rFsv -k gpu-torch # For now only test torch GPU pipeline build_linux_ubuntu: stage: build @@ -366,88 +366,88 @@ linux_ubuntu_2204_clang: ### LCG JOB MATRIX ### ###################### -.lcg_base_job: - image: ghcr.io/acts-project/${OS}-base:63 - stage: build - tags: - - cvmfs - - variables: - ACTS_LOG_FAILURE_THRESHOLD: WARNING - INSTALL_DIR: ${{ github.workspace }}/install - - SETUP: - - cache: - key: ccache-${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CCACHE_KEY_SUFFIX} - fallback_keys: - - ccache-${CI_JOB_NAME}-${CI_DEFAULT_BRANCH}-${CCACHE_KEY_SUFFIX} - when: always - paths: - - ${CCACHE_DIR} - - before_script: - - 'echo "LCG_VERSION: ${LCG_VERSION}"' - - 'echo "COMPILER: ${COMPILER}"' - - # Figure out LCG platform name based on version number and OS - - > - if [ "$OS" = "alma9" ]; then - export LCG_PLATFORM="el9" - else - export LCG_PLATFORM="$OS" - fi - - - 'echo "LCG_PLATFORM: ${LCG_PLATFORM}"' - - source /cvmfs/sft.cern.ch/lcg/views/LCG_${LCG_VERSION}/x86_64-${LCG_PLATFORM}-${COMPILER}-opt/setup.sh - - - git clone $CLONE_URL src - - - cd src - - git checkout $HEAD_SHA - - git submodule init - - git submodule update - - cd .. - - - ccache --version - - script: - - > - cmake -B build -S src - --preset=gitlab-ci - -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" - -DACTS_BUILD_PLUGIN_GEOMODEL=OFF # GeoModel is not in LCG at this point - - - ccache -z - - cmake --build build -- -j6 - - ccache -s - - - ctest --test-dir build -j$(nproc) - - -lcg_105: - extends: .lcg_base_job - - variables: - LCG_VERSION: "105" - - parallel: - matrix: - - OS: [alma9] - COMPILER: - - gcc13 - - clang16 - -lcg_106a: - extends: .lcg_base_job - - variables: - LCG_VERSION: "106a" - - parallel: - matrix: - - OS: [alma9] - COMPILER: - - gcc13 - - gcc14 - - clang16 +# .lcg_base_job: +# image: ghcr.io/acts-project/${OS}-base:63 +# stage: build +# tags: +# - cvmfs +# +# variables: +# ACTS_LOG_FAILURE_THRESHOLD: WARNING +# INSTALL_DIR: ${{ github.workspace }}/install +# +# SETUP: +# +# cache: +# key: ccache-${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CCACHE_KEY_SUFFIX} +# fallback_keys: +# - ccache-${CI_JOB_NAME}-${CI_DEFAULT_BRANCH}-${CCACHE_KEY_SUFFIX} +# when: always +# paths: +# - ${CCACHE_DIR} +# +# before_script: +# - 'echo "LCG_VERSION: ${LCG_VERSION}"' +# - 'echo "COMPILER: ${COMPILER}"' +# +# # Figure out LCG platform name based on version number and OS +# - > +# if [ "$OS" = "alma9" ]; then +# export LCG_PLATFORM="el9" +# else +# export LCG_PLATFORM="$OS" +# fi +# +# - 'echo "LCG_PLATFORM: ${LCG_PLATFORM}"' +# - source /cvmfs/sft.cern.ch/lcg/views/LCG_${LCG_VERSION}/x86_64-${LCG_PLATFORM}-${COMPILER}-opt/setup.sh +# +# - git clone $CLONE_URL src +# +# - cd src +# - git checkout $HEAD_SHA +# - git submodule init +# - git submodule update +# - cd .. +# +# - ccache --version +# +# script: +# - > +# cmake -B build -S src +# --preset=gitlab-ci +# -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" +# -DACTS_BUILD_PLUGIN_GEOMODEL=OFF # GeoModel is not in LCG at this point +# +# - ccache -z +# - cmake --build build -- -j6 +# - ccache -s +# +# - ctest --test-dir build -j$(nproc) +# +# +# lcg_105: +# extends: .lcg_base_job +# +# variables: +# LCG_VERSION: "105" +# +# parallel: +# matrix: +# - OS: [alma9] +# COMPILER: +# - gcc13 +# - clang16 +# +# lcg_106a: +# extends: .lcg_base_job +# +# variables: +# LCG_VERSION: "106a" +# +# parallel: +# matrix: +# - OS: [alma9] +# COMPILER: +# - gcc13 +# - gcc14 +# - clang16 From a64c80bf1e6fef898b8699f1516a14d14a4b337b Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 9 Jan 2025 10:46:58 +0100 Subject: [PATCH 144/191] reduce ccache size --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8d6a2ff3076..b704ee5f3ad 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ variables: CCACHE_DIR: ${CI_PROJECT_DIR}/ccache - CCACHE_MAXSIZE: 2G + CCACHE_MAXSIZE: 500M CCACHE_KEY_SUFFIX: r2 CTEST_OUTPUT_ON_FAILURE: 1 From ec2c95aa7a56c4ee9fb39dbe87afafa297d0b275 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 9 Jan 2025 10:47:40 +0100 Subject: [PATCH 145/191] add caching feature to lock file getter --- CI/dependencies/select_lockfile.py | 115 ++++++++++++++++++++++++----- 1 file changed, 95 insertions(+), 20 deletions(-) diff --git a/CI/dependencies/select_lockfile.py b/CI/dependencies/select_lockfile.py index 5bf3b4aa568..e8005567f5c 100755 --- a/CI/dependencies/select_lockfile.py +++ b/CI/dependencies/select_lockfile.py @@ -7,7 +7,88 @@ import urllib.error import re import subprocess +import hashlib +import tempfile +from pathlib import Path from typing import Tuple, Dict, Optional +import contextlib + +# Modify the default cache dir to use a temporary directory +DEFAULT_CACHE_SIZE_LIMIT = 1 * 1024 * 1024 # 1MB + + +def compute_cache_key(url: str) -> str: + """Compute a cache key for a URL""" + return hashlib.sha256(url.encode()).hexdigest() + + +def prune_cache(cache_dir: Optional[Path], size_limit: int): + """Prune the cache to keep it under the size limit""" + if cache_dir is None or not cache_dir.exists(): + return + + # Get all cache files with their modification times + cache_files = [ + (cache_dir / f, (cache_dir / f).stat().st_mtime) + for f in os.listdir(cache_dir) + if (cache_dir / f).is_file() + ] + total_size = sum(f.stat().st_size for f, _ in cache_files) + + if total_size <= size_limit: + return + + # Sort by modification time (oldest first) + cache_files.sort(key=lambda x: x[1]) + + # Remove files until we're under the limit + for file_path, _ in cache_files: + if total_size <= size_limit: + break + total_size -= file_path.stat().st_size + file_path.unlink() + + +def fetch_github(base_url: str, cache_dir: Optional[Path], cache_limit: int) -> bytes: + headers = {} + token = os.environ.get("GITHUB_TOKEN") + if token is not None and token != "": + headers["Authorization"] = f"token {token}" + + with contextlib.ExitStack() as stack: + if cache_dir is not None: + cache_dir.mkdir(parents=True, exist_ok=True) + else: + cache_dir = Path(stack.enter_context(tempfile.TemporaryDirectory())) + + # Check cache first + cache_key = compute_cache_key(base_url) + cache_file = cache_dir / cache_key + + if cache_file.exists(): + print("Cache hit on", base_url) + return cache_file.read_bytes() + else: + print("Cache miss on", base_url) + + try: + req = urllib.request.Request(base_url, headers=headers) + with urllib.request.urlopen(req) as response: + content = response.read() + + # Write to cache + cache_file.write_bytes(content) + + # Prune cache if necessary + prune_cache(cache_dir, cache_limit) + + return content + except urllib.error.URLError as e: + print(f"Failed to fetch from {base_url}: {e}") + exit(1) + except json.JSONDecodeError as e: + print(f"Failed to parse JSON response: {e}") + exit(1) def main(): @@ -32,6 +113,18 @@ def main(): default=None, help="Output file to write lockfile to", ) + parser.add_argument( + "--cache-dir", + type=lambda x: Path(x) if x else None, + default=os.environ.get("LOCKFILE_CACHE_DIR"), + help="Directory to use for caching (defaults to LOCKFILE_CACHE_DIR env var)", + ) + parser.add_argument( + "--cache-limit", + type=int, + default=int(os.environ.get("LOCKFILE_CACHE_LIMIT", DEFAULT_CACHE_SIZE_LIMIT)), + help="Cache size limit in bytes (defaults to LOCKFILE_CACHE_LIMIT env var)", + ) args = parser.parse_args() print("Fetching lockfiles for tag:", args.tag) @@ -39,7 +132,7 @@ def main(): base_url = f"https://api.github.com/repos/acts-project/ci-dependencies/releases/tags/{args.tag}" - data = json.loads(fetch_github(base_url)) + data = json.loads(fetch_github(base_url, args.cache_dir, args.cache_limit)) lockfiles = parse_assets(data) @@ -71,7 +164,7 @@ def main(): if args.output: with open(args.output, "wb") as f: - f.write(fetch_github(lockfile)) + f.write(fetch_github(lockfile, Path(args.cache_dir), args.cache_limit)) def parse_assets(data: Dict) -> Dict[str, Dict[str, Tuple[str, str]]]: @@ -157,23 +250,5 @@ def determine_compiler_version(binary: str): exit(1) -def fetch_github(base_url): - headers = {} - token = os.environ.get("GITHUB_TOKEN") - if token is not None and token != "": - headers["Authorization"] = f"token {token}" - - try: - req = urllib.request.Request(base_url, headers=headers) - with urllib.request.urlopen(req) as response: - return response.read() - except urllib.error.URLError as e: - print(f"Failed to fetch from {base_url}: {e}") - exit(1) - except json.JSONDecodeError as e: - print(f"Failed to parse JSON response: {e}") - exit(1) - - if __name__ == "__main__": main() From 51f0c5b0823094b20451895997de196594a20866 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 9 Jan 2025 10:49:56 +0100 Subject: [PATCH 146/191] let's try caching --- .gitlab-ci.yml | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b704ee5f3ad..1889242bc4d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -185,17 +185,24 @@ variables: build_linux_ubuntu: stage: build - image: ghcr.io/acts-project/ubuntu2404:63 + image: registry.cern.ch/ghcr.io/acts-project/ubuntu2404:63 variables: DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.$DEPENDENCY_TAG.tar.zst cache: - key: ccache-${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CCACHE_KEY_SUFFIX} - fallback_keys: - - ccache-${CI_JOB_NAME}-${CI_DEFAULT_BRANCH}-${CCACHE_KEY_SUFFIX} - when: always - paths: - - ${CCACHE_DIR} + - key: ccache-${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CCACHE_KEY_SUFFIX} + fallback_keys: + - ccache-${CI_JOB_NAME}-${CI_DEFAULT_BRANCH}-${CCACHE_KEY_SUFFIX} + when: always + paths: + - ${CCACHE_DIR} + - key: + files: + - CI/dependencies/setup_spack.sh + prefix: spack_ + paths: + - spack + - ~/.spack artifacts: paths: @@ -229,7 +236,7 @@ build_linux_ubuntu: linux_test_examples: stage: test - image: ghcr.io/acts-project/ubuntu2404:63 + image: registry.cern.ch/ghcr.io/acts-project/ubuntu2404:63 variables: DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.$DEPENDENCY_TAG.tar.zst needs: [build_linux_ubuntu] @@ -253,7 +260,7 @@ linux_test_examples: linux_physmon: stage: test - image: ghcr.io/acts-project/ubuntu2404:63 + image: registry.cern.ch/ghcr.io/acts-project/ubuntu2404:63 variables: DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.$DEPENDENCY_TAG.tar.zst needs: [build_linux_ubuntu] From 7b20950519b272e25b9adbe12e0983998b16da4d Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 9 Jan 2025 10:55:19 +0100 Subject: [PATCH 147/191] cache fix --- CI/dependencies/select_lockfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CI/dependencies/select_lockfile.py b/CI/dependencies/select_lockfile.py index e8005567f5c..a4ea2847f72 100755 --- a/CI/dependencies/select_lockfile.py +++ b/CI/dependencies/select_lockfile.py @@ -164,7 +164,7 @@ def main(): if args.output: with open(args.output, "wb") as f: - f.write(fetch_github(lockfile, Path(args.cache_dir), args.cache_limit)) + f.write(fetch_github(lockfile, args.cache_dir, args.cache_limit)) def parse_assets(data: Dict) -> Dict[str, Dict[str, Tuple[str, str]]]: From d775a42062df230888ff91c9ea0d3cc512a544f7 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 9 Jan 2025 10:56:16 +0100 Subject: [PATCH 148/191] cache config --- .gitlab-ci.yml | 208 +++++++++++++++++++++++++------------------------ 1 file changed, 105 insertions(+), 103 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1889242bc4d..91785d6f97a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,8 @@ variables: CCACHE_KEY_SUFFIX: r2 CTEST_OUTPUT_ON_FAILURE: 1 + SPACK_VERSION: develop-acts + DEPENDENCY_TAG: v7_b7 # clang_tidy: @@ -190,16 +192,16 @@ build_linux_ubuntu: DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.$DEPENDENCY_TAG.tar.zst cache: - - key: ccache-${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CCACHE_KEY_SUFFIX} - fallback_keys: - - ccache-${CI_JOB_NAME}-${CI_DEFAULT_BRANCH}-${CCACHE_KEY_SUFFIX} - when: always - paths: - - ${CCACHE_DIR} + # - key: ccache-${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CCACHE_KEY_SUFFIX} + # fallback_keys: + # - ccache-${CI_JOB_NAME}-${CI_DEFAULT_BRANCH}-${CCACHE_KEY_SUFFIX} + # when: always + # paths: + # - ${CCACHE_DIR} - key: files: - CI/dependencies/setup_spack.sh - prefix: spack_ + prefix: spack_${SPACK_VERSION} paths: - spack - ~/.spack @@ -231,69 +233,69 @@ build_linux_ubuntu: -DACTS_BUILD_PLUGIN_ONNX=ON - ccache -z - - cmake --build build -- -j6 + # - cmake --build build -- -j6 - ccache -s -linux_test_examples: - stage: test - image: registry.cern.ch/ghcr.io/acts-project/ubuntu2404:63 - variables: - DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.$DEPENDENCY_TAG.tar.zst - needs: [build_linux_ubuntu] - - script: - - apt-get update && apt-get install -y git-lfs - - - git clone $CLONE_URL src - - cd src - - git checkout $HEAD_SHA - - git submodule init - - git submodule update - - source CI/dependencies/setup.sh - - cd .. - - - geant4-config --install-datasets - - source build/this_acts_withdeps.sh - - cd src - - python3 -m pip install -r Examples/Python/tests/requirements.txt - - pytest -rFsv -k "not exatrkx" -v -s - -linux_physmon: - stage: test - image: registry.cern.ch/ghcr.io/acts-project/ubuntu2404:63 - variables: - DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.$DEPENDENCY_TAG.tar.zst - needs: [build_linux_ubuntu] - - artifacts: - when: always - paths: - - src/physmon - expire_in: 1 week - - script: - - apt-get update && apt-get install -y git-lfs time - - - git clone $CLONE_URL src - - cd src - - git checkout $HEAD_SHA - - git submodule init - - git submodule update - - source CI/dependencies/setup.sh - - cd .. - - - git config --global safe.directory "$GITHUB_WORKSPACE" - - python3 -m pip install histcmp==0.6.8 matplotlib - - python3 -m pip install -r src/Examples/Scripts/requirements.txt - - geant4-config --install-datasets - - venv_python=$(which python3) - - echo $venv_python - - source build/this_acts_withdeps.sh - - export PATH=$(dirname $venv_python):$PATH - - echo $PATH - - which python3 - - cd src - - CI/physmon/phys_perf_mon.sh all physmon +# linux_test_examples: +# stage: test +# image: registry.cern.ch/ghcr.io/acts-project/ubuntu2404:63 +# variables: +# DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.$DEPENDENCY_TAG.tar.zst +# needs: [build_linux_ubuntu] +# +# script: +# - apt-get update && apt-get install -y git-lfs +# +# - git clone $CLONE_URL src +# - cd src +# - git checkout $HEAD_SHA +# - git submodule init +# - git submodule update +# - source CI/dependencies/setup.sh +# - cd .. +# +# - geant4-config --install-datasets +# - source build/this_acts_withdeps.sh +# - cd src +# - python3 -m pip install -r Examples/Python/tests/requirements.txt +# - pytest -rFsv -k "not exatrkx" -v -s +# +# linux_physmon: +# stage: test +# image: registry.cern.ch/ghcr.io/acts-project/ubuntu2404:63 +# variables: +# DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.$DEPENDENCY_TAG.tar.zst +# needs: [build_linux_ubuntu] +# +# artifacts: +# when: always +# paths: +# - src/physmon +# expire_in: 1 week +# +# script: +# - apt-get update && apt-get install -y git-lfs time +# +# - git clone $CLONE_URL src +# - cd src +# - git checkout $HEAD_SHA +# - git submodule init +# - git submodule update +# - source CI/dependencies/setup.sh +# - cd .. +# +# - git config --global safe.directory "$GITHUB_WORKSPACE" +# - python3 -m pip install histcmp==0.6.8 matplotlib +# - python3 -m pip install -r src/Examples/Scripts/requirements.txt +# - geant4-config --install-datasets +# - venv_python=$(which python3) +# - echo $venv_python +# - source build/this_acts_withdeps.sh +# - export PATH=$(dirname $venv_python):$PATH +# - echo $PATH +# - which python3 +# - cd src +# - CI/physmon/phys_perf_mon.sh all physmon ############################### ### UBUNTU EXTRA JOB MATRIX ### @@ -330,43 +332,43 @@ linux_physmon: -DCMAKE_CXX_STANDARD=${CXXSTD} - ccache -z - - cmake --build build -- -j6 + # - cmake --build build -- -j6 - ccache -s - - ctest --test-dir build -j$(nproc) - - cmake --build build --target integrationtests - - # Install main project - - cmake --install build - - # Downstream configure - - > - cmake -B build-downstream -S src/Tests/DownstreamProject - -GNinja - -DCMAKE_BUILD_TYPE=Release - -DCMAKE_CXX_FLAGS=-Werror - -DCMAKE_CXX_STANDARD=${CXXSTD} - -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" - - # Downstream build - - cmake --build build-downstream - - # Downstream run - - ./build-downstream/bin/ShowActsVersion - -linux_ubuntu_2204: - extends: .linux_ubuntu_extra - variables: - CXXSTD: 20 - DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst - image: ghcr.io/acts-project/ubuntu2204:63 - -linux_ubuntu_2204_clang: - extends: .linux_ubuntu_extra - variables: - CXXSTD: 20 - DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst - image: ghcr.io/acts-project/ubuntu2204_clang:63 + # - ctest --test-dir build -j$(nproc) + # - cmake --build build --target integrationtests + # + # # Install main project + # - cmake --install build + # + # # Downstream configure + # - > + # cmake -B build-downstream -S src/Tests/DownstreamProject + # -GNinja + # -DCMAKE_BUILD_TYPE=Release + # -DCMAKE_CXX_FLAGS=-Werror + # -DCMAKE_CXX_STANDARD=${CXXSTD} + # -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" + # + # # Downstream build + # - cmake --build build-downstream + # + # # Downstream run + # - ./build-downstream/bin/ShowActsVersion + +# linux_ubuntu_2204: +# extends: .linux_ubuntu_extra +# variables: +# CXXSTD: 20 +# DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst +# image: ghcr.io/acts-project/ubuntu2204:63 +# +# linux_ubuntu_2204_clang: +# extends: .linux_ubuntu_extra +# variables: +# CXXSTD: 20 +# DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst +# image: ghcr.io/acts-project/ubuntu2204_clang:63 ###################### From b1370d059662e7539135f92fa0926cac5317418d Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 9 Jan 2025 11:00:11 +0100 Subject: [PATCH 149/191] fix compiler to gcc --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 91785d6f97a..51c0dda9b05 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -223,7 +223,7 @@ build_linux_ubuntu: - git checkout $HEAD_SHA - git submodule init - git submodule update - - source CI/dependencies/setup.sh + - source CI/dependencies/setup.sh -c g++ - cd .. - mkdir build From b83cc3e652909e1dc5e6294974ce19ae0b47375c Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 9 Jan 2025 11:00:48 +0100 Subject: [PATCH 150/191] change cwd for deps install --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 51c0dda9b05..a50289b9e92 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -223,9 +223,11 @@ build_linux_ubuntu: - git checkout $HEAD_SHA - git submodule init - git submodule update - - source CI/dependencies/setup.sh -c g++ - cd .. + + - source CI/dependencies/setup.sh -c g++ + - mkdir build - > cmake -B build -S src From 6f86c5f8609a7f76298144afb9e1dfce51c50138 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 9 Jan 2025 11:07:45 +0100 Subject: [PATCH 151/191] add a global digest for the cache content --- CI/dependencies/select_lockfile.py | 34 ++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/CI/dependencies/select_lockfile.py b/CI/dependencies/select_lockfile.py index a4ea2847f72..c73a9ec2314 100755 --- a/CI/dependencies/select_lockfile.py +++ b/CI/dependencies/select_lockfile.py @@ -22,6 +22,29 @@ def compute_cache_key(url: str) -> str: return hashlib.sha256(url.encode()).hexdigest() +def compute_cache_digest(cache_dir: Path) -> str: + """Compute a digest of all cache files except digest.txt""" + files = sorted( + f + for f in os.listdir(cache_dir) + if (cache_dir / f).is_file() and f != "digest.txt" + ) + + digest = hashlib.sha256() + for fname in files: + fpath = cache_dir / fname + digest.update(fname.encode()) + digest.update(str(fpath.stat().st_size).encode()) + digest.update(fpath.read_bytes()) + return digest.hexdigest() + + +def update_cache_digest(cache_dir: Path): + """Update the cache digest file""" + digest = compute_cache_digest(cache_dir) + (cache_dir / "digest.txt").write_text(digest) + + def prune_cache(cache_dir: Optional[Path], size_limit: int): """Prune the cache to keep it under the size limit""" if cache_dir is None or not cache_dir.exists(): @@ -32,6 +55,7 @@ def prune_cache(cache_dir: Optional[Path], size_limit: int): (cache_dir / f, (cache_dir / f).stat().st_mtime) for f in os.listdir(cache_dir) if (cache_dir / f).is_file() + and f != "digest.txt" # Exclude digest from pruning ] total_size = sum(f.stat().st_size for f, _ in cache_files) @@ -48,6 +72,9 @@ def prune_cache(cache_dir: Optional[Path], size_limit: int): total_size -= file_path.stat().st_size file_path.unlink() + # Update digest after pruning + update_cache_digest(cache_dir) + def fetch_github(base_url: str, cache_dir: Optional[Path], cache_limit: int) -> bytes: headers = {} @@ -79,7 +106,10 @@ def fetch_github(base_url: str, cache_dir: Optional[Path], cache_limit: int) -> # Write to cache cache_file.write_bytes(content) - # Prune cache if necessary + # Update digest after adding new file + update_cache_digest(cache_dir) + + # Prune cache if necessary (this will update digest again if pruning occurs) prune_cache(cache_dir, cache_limit) return content @@ -115,7 +145,7 @@ def main(): ) parser.add_argument( "--cache-dir", - type=lambda x: Path(x) if x else None, + type=lambda x: Path(x).expanduser() if x else None, default=os.environ.get("LOCKFILE_CACHE_DIR"), help="Directory to use for caching (defaults to LOCKFILE_CACHE_DIR env var)", ) From a33c62ce270142a96acf54bc00d85fe88fd0fa89 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 9 Jan 2025 11:07:50 +0100 Subject: [PATCH 152/191] see if lockfile caching works --- .gitlab-ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a50289b9e92..7289eaac24b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,7 @@ variables: CTEST_OUTPUT_ON_FAILURE: 1 SPACK_VERSION: develop-acts + LOCKFILE_CACHE_DIR: ~/spack_lockfile_cache DEPENDENCY_TAG: v7_b7 @@ -205,6 +206,12 @@ build_linux_ubuntu: paths: - spack - ~/.spack + - key: + files: + - ${LOCKFILE_CACHE_DIR}/digest.txt + prefix: spack_lockfile_ + paths: + - ${LOCKFILE_CACHE_DIR} artifacts: paths: From cabe566a83f703239a12ad72451f743bc9efaa59 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 9 Jan 2025 11:08:32 +0100 Subject: [PATCH 153/191] path --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7289eaac24b..39f88afda4a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -233,7 +233,7 @@ build_linux_ubuntu: - cd .. - - source CI/dependencies/setup.sh -c g++ + - source src/CI/dependencies/setup.sh -c g++ - mkdir build - > From c88aedc6750364c367b45696211309f8b5f93866 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 9 Jan 2025 11:17:22 +0100 Subject: [PATCH 154/191] update paths for caching --- .gitlab-ci.yml | 8 ++++---- CI/dependencies/setup.sh | 7 +++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 39f88afda4a..90157414b35 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,7 @@ variables: CTEST_OUTPUT_ON_FAILURE: 1 SPACK_VERSION: develop-acts - LOCKFILE_CACHE_DIR: ~/spack_lockfile_cache + LOCKFILE_CACHE_DIR: ${CI_PROJECT_DIR}/spack_lockfile_cache DEPENDENCY_TAG: v7_b7 @@ -205,13 +205,13 @@ build_linux_ubuntu: prefix: spack_${SPACK_VERSION} paths: - spack - - ~/.spack + - .spack - key: files: - - ${LOCKFILE_CACHE_DIR}/digest.txt + - spack_lockfile_cache/digest.txt prefix: spack_lockfile_ paths: - - ${LOCKFILE_CACHE_DIR} + - spack_lockfile_cache artifacts: paths: diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index 7ed14ed6a6b..7b8ead4e546 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -104,8 +104,15 @@ if ! command -v spack &> /dev/null; then fi end_section +if [ -n "${GITLAB_CI:-}" ]; then + # Use the project spack config for GitLab CI so we can cache it + mkdir -p ${CI_PROJECT_DIR}/.spack + ln -s ${CI_PROJECT_DIR}/.spack ${HOME}/.spack +fi + source "$(pwd)/spack/share/spack/setup-env.sh" + if [ -n "${CI:-}" ]; then start_section "Add buildcache mirror" mirror_name="acts-spack-buildcache" From 83900e01c7f1110cc9234f9caeb660dbc2cdd254 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 9 Jan 2025 11:23:31 +0100 Subject: [PATCH 155/191] prune spack directory before caching --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 90157414b35..e628522e855 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -213,6 +213,12 @@ build_linux_ubuntu: paths: - spack_lockfile_cache + after_script: + - du -sh spack + - rm -rf spack/opt + - rm -rf spack/var/spack/cache + - du -sh spack + artifacts: paths: - build/ From cc65f712a783252d569053418f3cbccdec84d95b Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 9 Jan 2025 11:29:12 +0100 Subject: [PATCH 156/191] more pruning --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e628522e855..25243fa8f31 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -215,9 +215,15 @@ build_linux_ubuntu: after_script: - du -sh spack + - du -sh spack/* + - du -sh spack/opt/spack/* + - du -sh spack/var/spack/* - rm -rf spack/opt - rm -rf spack/var/spack/cache + - find spack -type f -name "*.pyc" -delete - du -sh spack + - du -sh spack/* + - du -sh spack/var/spack/* artifacts: paths: From d498f9d2370e214ded16fb57f31925ab7afafbc2 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 9 Jan 2025 11:32:39 +0100 Subject: [PATCH 157/191] only add mirror if not already there --- CI/dependencies/setup.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index 7b8ead4e546..e850008e6b9 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -116,12 +116,17 @@ source "$(pwd)/spack/share/spack/setup-env.sh" if [ -n "${CI:-}" ]; then start_section "Add buildcache mirror" mirror_name="acts-spack-buildcache" - mirror_url="oci://ghcr.io/acts-project/spack-buildcache" + mirror_url="oci://ghcr.io/acts-project/spack-buildcache" if [ -n "${GITLAB_CI:-}" ]; then - # Use CERN mirror for non-Github Actions + # Use CERN mirror for non-Github Actions mirror_url="oci://registry.cern.ch/ghcr.io/acts-project/spack-buildcache" fi - spack mirror add ${mirror_name} ${mirror_url} --unsigned + + # Check if this buildcache is already configured + if ! spack mirror list | grep -q ${mirror_name}; then + echo "Adding buildcache ${mirror_name}" + spack mirror add ${mirror_name} ${mirror_url} --unsigned + fi end_section start_section "Locate OpenGL" From 5e121a8836284153cab179ecdadecb32f253b0d3 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 9 Jan 2025 12:57:33 +0100 Subject: [PATCH 158/191] delete the git folder too --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 25243fa8f31..25189715889 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -218,6 +218,7 @@ build_linux_ubuntu: - du -sh spack/* - du -sh spack/opt/spack/* - du -sh spack/var/spack/* + - rm -rf spack/.git - rm -rf spack/opt - rm -rf spack/var/spack/cache - find spack -type f -name "*.pyc" -delete From b5923443dba972f0fa29f600bdf950869fd56adf Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 9 Jan 2025 12:57:49 +0100 Subject: [PATCH 159/191] enable build --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 25189715889..81aeffa90a4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -255,7 +255,7 @@ build_linux_ubuntu: -DACTS_BUILD_PLUGIN_ONNX=ON - ccache -z - # - cmake --build build -- -j6 + - cmake --build build -- -j6 - ccache -s # linux_test_examples: From d311fcc31263609e291e3a53a4ee2a31291f8050 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 9 Jan 2025 14:52:49 +0100 Subject: [PATCH 160/191] use ccache again --- .gitlab-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 81aeffa90a4..118a49361b8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -193,12 +193,12 @@ build_linux_ubuntu: DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.$DEPENDENCY_TAG.tar.zst cache: - # - key: ccache-${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CCACHE_KEY_SUFFIX} - # fallback_keys: - # - ccache-${CI_JOB_NAME}-${CI_DEFAULT_BRANCH}-${CCACHE_KEY_SUFFIX} - # when: always - # paths: - # - ${CCACHE_DIR} + - key: ccache-${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CCACHE_KEY_SUFFIX} + fallback_keys: + - ccache-${CI_JOB_NAME}-${CI_DEFAULT_BRANCH}-${CCACHE_KEY_SUFFIX} + when: always + paths: + - ${CCACHE_DIR} - key: files: - CI/dependencies/setup_spack.sh From e4ef4a6aa4c249d6f03a83239b76fa34b35652bc Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 10 Jan 2025 13:24:03 +0100 Subject: [PATCH 161/191] reenable some jobs --- .gitlab-ci.yml | 608 ++++++++++++++++++++++++------------------------- 1 file changed, 304 insertions(+), 304 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 118a49361b8..cd1b7f208e4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -61,130 +61,130 @@ variables: # # Generate an html report # - codereport clang-tidy/clang-tidy.json clang-tidy/html -# build_exatrkx_cpu: -# stage: build -# image: ghcr.io/acts-project/ubuntu2204_exatrkx:63 -# variables: -# DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst -# tags: -# - large -# -# cache: -# key: ccache-${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CCACHE_KEY_SUFFIX} -# fallback_keys: -# - ccache-${CI_JOB_NAME}-${CI_DEFAULT_BRANCH}-${CCACHE_KEY_SUFFIX} -# when: always -# paths: -# - ${CCACHE_DIR} -# -# script: -# - export PATH=/usr/local/sbin:/usr/sbin:/sbin:$PATH -# - export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:$PATH -# - export TORCH_CUDA_ARCH_LIST="8.0 8.6 8.9 9.0" -# - echo $PATH -# - git clone $CLONE_URL src -# - cd src -# - git checkout $HEAD_SHA -# - source CI/dependencies/setup.sh -# - cd .. -# - mkdir build -# # Here we only do a minimal build without examples to save resources -# - > -# cmake -B build -S src -# --preset=gitlab-ci-exatrkx -# -DACTS_EXATRKX_ENABLE_CUDA=OFF -# -# - ccache -z -# - cmake --build build -- -j6 -# - ccache -s -# -# build_exatrkx: -# stage: build -# image: ghcr.io/acts-project/ubuntu2204_exatrkx:63 -# variables: -# DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst -# tags: -# - large -# -# cache: -# key: ccache-${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CCACHE_KEY_SUFFIX} -# fallback_keys: -# - ccache-${CI_JOB_NAME}-${CI_DEFAULT_BRANCH}-${CCACHE_KEY_SUFFIX} -# when: always -# paths: -# - ${CCACHE_DIR} -# -# artifacts: -# paths: -# - build/ -# exclude: -# - build/**/*.o -# - build/bin/ActsIntegrationTest* -# expire_in: 6 hours -# -# script: -# - export PATH=/usr/local/sbin:/usr/sbin:/sbin:$PATH -# - export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:$PATH -# - export TORCH_CUDA_ARCH_LIST="8.0 8.6 8.9 9.0" -# - echo $PATH -# - git clone $CLONE_URL src -# - cd src -# - git checkout $HEAD_SHA -# - source CI/dependencies/setup.sh -# - cd .. -# - mkdir build -# - > -# cmake -B build -S src -# --preset=gitlab-ci-exatrkx -# -DCMAKE_CUDA_ARCHITECTURES="75;86" -# -# - ccache -z -# - cmake --build build -- -j6 -# - ccache -s -# -# test_exatrkx_unittests: -# stage: test -# needs: -# - build_exatrkx -# image: ghcr.io/acts-project/ubuntu2204_exatrkx:63 -# variables: -# DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst -# tags: -# - docker-gpu-nvidia -# script: -# -# - apt-get update -y -# - git clone $CLONE_URL src -# - cd src -# - git checkout $HEAD_SHA -# - source CI/dependencies/setup.sh -# - cd .. -# - ctest --test-dir build -R ExaTrkX -# -# test_exatrkx_python: -# stage: test -# needs: -# - build_exatrkx -# image: ghcr.io/acts-project/ubuntu2204_exatrkx:63 -# variables: -# DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst -# tags: -# - docker-gpu-nvidia -# script: -# - apt-get update -y -# - git clone $CLONE_URL src -# - cd src -# - git checkout $HEAD_SHA -# - nvidia-smi -# - source CI/dependencies/setup.sh -# - source ../build/this_acts_withdeps.sh -# - python3 -m pip install -r Examples/Python/tests/requirements.txt -# - echo $PYTHONPATH -# - which python3 -# - python3 --version -# - python3 -c "import acts" -# - pytest -rFsv -k torch --collect-only -# - pytest -rFsv -k gpu-torch # For now only test torch GPU pipeline +build_exatrkx_cpu: + stage: build + image: ghcr.io/acts-project/ubuntu2204_exatrkx:63 + variables: + DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst + tags: + - large + + cache: + key: ccache-${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CCACHE_KEY_SUFFIX} + fallback_keys: + - ccache-${CI_JOB_NAME}-${CI_DEFAULT_BRANCH}-${CCACHE_KEY_SUFFIX} + when: always + paths: + - ${CCACHE_DIR} + + script: + - export PATH=/usr/local/sbin:/usr/sbin:/sbin:$PATH + - export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:$PATH + - export TORCH_CUDA_ARCH_LIST="8.0 8.6 8.9 9.0" + - echo $PATH + - git clone $CLONE_URL src + - cd src + - git checkout $HEAD_SHA + - source CI/dependencies/setup.sh + - cd .. + - mkdir build + # Here we only do a minimal build without examples to save resources + - > + cmake -B build -S src + --preset=gitlab-ci-exatrkx + -DACTS_EXATRKX_ENABLE_CUDA=OFF + + - ccache -z + - cmake --build build -- -j6 + - ccache -s + +build_exatrkx: + stage: build + image: ghcr.io/acts-project/ubuntu2204_exatrkx:63 + variables: + DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst + tags: + - large + + cache: + key: ccache-${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CCACHE_KEY_SUFFIX} + fallback_keys: + - ccache-${CI_JOB_NAME}-${CI_DEFAULT_BRANCH}-${CCACHE_KEY_SUFFIX} + when: always + paths: + - ${CCACHE_DIR} + + artifacts: + paths: + - build/ + exclude: + - build/**/*.o + - build/bin/ActsIntegrationTest* + expire_in: 6 hours + + script: + - export PATH=/usr/local/sbin:/usr/sbin:/sbin:$PATH + - export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:$PATH + - export TORCH_CUDA_ARCH_LIST="8.0 8.6 8.9 9.0" + - echo $PATH + - git clone $CLONE_URL src + - cd src + - git checkout $HEAD_SHA + - source CI/dependencies/setup.sh + - cd .. + - mkdir build + - > + cmake -B build -S src + --preset=gitlab-ci-exatrkx + -DCMAKE_CUDA_ARCHITECTURES="75;86" + + - ccache -z + - cmake --build build -- -j6 + - ccache -s + +test_exatrkx_unittests: + stage: test + needs: + - build_exatrkx + image: ghcr.io/acts-project/ubuntu2204_exatrkx:63 + variables: + DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst + tags: + - docker-gpu-nvidia + script: + + - apt-get update -y + - git clone $CLONE_URL src + - cd src + - git checkout $HEAD_SHA + - source CI/dependencies/setup.sh + - cd .. + - ctest --test-dir build -R ExaTrkX + +test_exatrkx_python: + stage: test + needs: + - build_exatrkx + image: ghcr.io/acts-project/ubuntu2204_exatrkx:63 + variables: + DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst + tags: + - docker-gpu-nvidia + script: + - apt-get update -y + - git clone $CLONE_URL src + - cd src + - git checkout $HEAD_SHA + - nvidia-smi + - source CI/dependencies/setup.sh + - source ../build/this_acts_withdeps.sh + - python3 -m pip install -r Examples/Python/tests/requirements.txt + - echo $PYTHONPATH + - which python3 + - python3 --version + - python3 -c "import acts" + - pytest -rFsv -k torch --collect-only + - pytest -rFsv -k gpu-torch # For now only test torch GPU pipeline build_linux_ubuntu: stage: build @@ -258,66 +258,66 @@ build_linux_ubuntu: - cmake --build build -- -j6 - ccache -s -# linux_test_examples: -# stage: test -# image: registry.cern.ch/ghcr.io/acts-project/ubuntu2404:63 -# variables: -# DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.$DEPENDENCY_TAG.tar.zst -# needs: [build_linux_ubuntu] -# -# script: -# - apt-get update && apt-get install -y git-lfs -# -# - git clone $CLONE_URL src -# - cd src -# - git checkout $HEAD_SHA -# - git submodule init -# - git submodule update -# - source CI/dependencies/setup.sh -# - cd .. -# -# - geant4-config --install-datasets -# - source build/this_acts_withdeps.sh -# - cd src -# - python3 -m pip install -r Examples/Python/tests/requirements.txt -# - pytest -rFsv -k "not exatrkx" -v -s -# -# linux_physmon: -# stage: test -# image: registry.cern.ch/ghcr.io/acts-project/ubuntu2404:63 -# variables: -# DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.$DEPENDENCY_TAG.tar.zst -# needs: [build_linux_ubuntu] -# -# artifacts: -# when: always -# paths: -# - src/physmon -# expire_in: 1 week -# -# script: -# - apt-get update && apt-get install -y git-lfs time -# -# - git clone $CLONE_URL src -# - cd src -# - git checkout $HEAD_SHA -# - git submodule init -# - git submodule update -# - source CI/dependencies/setup.sh -# - cd .. -# -# - git config --global safe.directory "$GITHUB_WORKSPACE" -# - python3 -m pip install histcmp==0.6.8 matplotlib -# - python3 -m pip install -r src/Examples/Scripts/requirements.txt -# - geant4-config --install-datasets -# - venv_python=$(which python3) -# - echo $venv_python -# - source build/this_acts_withdeps.sh -# - export PATH=$(dirname $venv_python):$PATH -# - echo $PATH -# - which python3 -# - cd src -# - CI/physmon/phys_perf_mon.sh all physmon +linux_test_examples: + stage: test + image: registry.cern.ch/ghcr.io/acts-project/ubuntu2404:63 + variables: + DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.$DEPENDENCY_TAG.tar.zst + needs: [build_linux_ubuntu] + + script: + - apt-get update && apt-get install -y git-lfs + + - git clone $CLONE_URL src + - cd src + - git checkout $HEAD_SHA + - git submodule init + - git submodule update + - source CI/dependencies/setup.sh + - cd .. + + - geant4-config --install-datasets + - source build/this_acts_withdeps.sh + - cd src + - python3 -m pip install -r Examples/Python/tests/requirements.txt + - pytest -rFsv -k "not exatrkx" -v -s + +linux_physmon: + stage: test + image: registry.cern.ch/ghcr.io/acts-project/ubuntu2404:63 + variables: + DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.$DEPENDENCY_TAG.tar.zst + needs: [build_linux_ubuntu] + + artifacts: + when: always + paths: + - src/physmon + expire_in: 1 week + + script: + - apt-get update && apt-get install -y git-lfs time + + - git clone $CLONE_URL src + - cd src + - git checkout $HEAD_SHA + - git submodule init + - git submodule update + - source CI/dependencies/setup.sh + - cd .. + + - git config --global safe.directory "$GITHUB_WORKSPACE" + - python3 -m pip install histcmp==0.6.8 matplotlib + - python3 -m pip install -r src/Examples/Scripts/requirements.txt + - geant4-config --install-datasets + - venv_python=$(which python3) + - echo $venv_python + - source build/this_acts_withdeps.sh + - export PATH=$(dirname $venv_python):$PATH + - echo $PATH + - which python3 + - cd src + - CI/physmon/phys_perf_mon.sh all physmon ############################### ### UBUNTU EXTRA JOB MATRIX ### @@ -354,131 +354,131 @@ build_linux_ubuntu: -DCMAKE_CXX_STANDARD=${CXXSTD} - ccache -z - # - cmake --build build -- -j6 + - cmake --build build -- -j6 - ccache -s - # - ctest --test-dir build -j$(nproc) - # - cmake --build build --target integrationtests - # - # # Install main project - # - cmake --install build - # - # # Downstream configure - # - > - # cmake -B build-downstream -S src/Tests/DownstreamProject - # -GNinja - # -DCMAKE_BUILD_TYPE=Release - # -DCMAKE_CXX_FLAGS=-Werror - # -DCMAKE_CXX_STANDARD=${CXXSTD} - # -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" - # - # # Downstream build - # - cmake --build build-downstream - # - # # Downstream run - # - ./build-downstream/bin/ShowActsVersion - -# linux_ubuntu_2204: -# extends: .linux_ubuntu_extra -# variables: -# CXXSTD: 20 -# DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst -# image: ghcr.io/acts-project/ubuntu2204:63 -# -# linux_ubuntu_2204_clang: -# extends: .linux_ubuntu_extra -# variables: -# CXXSTD: 20 -# DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst -# image: ghcr.io/acts-project/ubuntu2204_clang:63 + - ctest --test-dir build -j$(nproc) + - cmake --build build --target integrationtests + + # Install main project + - cmake --install build + + # Downstream configure + - > + cmake -B build-downstream -S src/Tests/DownstreamProject + -GNinja + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_CXX_FLAGS=-Werror + -DCMAKE_CXX_STANDARD=${CXXSTD} + -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" + + # Downstream build + - cmake --build build-downstream + + # Downstream run + - ./build-downstream/bin/ShowActsVersion + +linux_ubuntu_2204: + extends: .linux_ubuntu_extra + variables: + CXXSTD: 20 + DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst + image: ghcr.io/acts-project/ubuntu2204:63 + +linux_ubuntu_2204_clang: + extends: .linux_ubuntu_extra + variables: + CXXSTD: 20 + DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst + image: ghcr.io/acts-project/ubuntu2204_clang:63 ###################### ### LCG JOB MATRIX ### ###################### -# .lcg_base_job: -# image: ghcr.io/acts-project/${OS}-base:63 -# stage: build -# tags: -# - cvmfs -# -# variables: -# ACTS_LOG_FAILURE_THRESHOLD: WARNING -# INSTALL_DIR: ${{ github.workspace }}/install -# -# SETUP: -# -# cache: -# key: ccache-${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CCACHE_KEY_SUFFIX} -# fallback_keys: -# - ccache-${CI_JOB_NAME}-${CI_DEFAULT_BRANCH}-${CCACHE_KEY_SUFFIX} -# when: always -# paths: -# - ${CCACHE_DIR} -# -# before_script: -# - 'echo "LCG_VERSION: ${LCG_VERSION}"' -# - 'echo "COMPILER: ${COMPILER}"' -# -# # Figure out LCG platform name based on version number and OS -# - > -# if [ "$OS" = "alma9" ]; then -# export LCG_PLATFORM="el9" -# else -# export LCG_PLATFORM="$OS" -# fi -# -# - 'echo "LCG_PLATFORM: ${LCG_PLATFORM}"' -# - source /cvmfs/sft.cern.ch/lcg/views/LCG_${LCG_VERSION}/x86_64-${LCG_PLATFORM}-${COMPILER}-opt/setup.sh -# -# - git clone $CLONE_URL src -# -# - cd src -# - git checkout $HEAD_SHA -# - git submodule init -# - git submodule update -# - cd .. -# -# - ccache --version -# -# script: -# - > -# cmake -B build -S src -# --preset=gitlab-ci -# -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -# -DACTS_BUILD_PLUGIN_GEOMODEL=OFF # GeoModel is not in LCG at this point -# -# - ccache -z -# - cmake --build build -- -j6 -# - ccache -s -# -# - ctest --test-dir build -j$(nproc) -# -# -# lcg_105: -# extends: .lcg_base_job -# -# variables: -# LCG_VERSION: "105" -# -# parallel: -# matrix: -# - OS: [alma9] -# COMPILER: -# - gcc13 -# - clang16 -# -# lcg_106a: -# extends: .lcg_base_job -# -# variables: -# LCG_VERSION: "106a" -# -# parallel: -# matrix: -# - OS: [alma9] -# COMPILER: -# - gcc13 -# - gcc14 -# - clang16 +.lcg_base_job: + image: ghcr.io/acts-project/${OS}-base:63 + stage: build + tags: + - cvmfs + + variables: + ACTS_LOG_FAILURE_THRESHOLD: WARNING + INSTALL_DIR: ${{ github.workspace }}/install + + SETUP: + + cache: + key: ccache-${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CCACHE_KEY_SUFFIX} + fallback_keys: + - ccache-${CI_JOB_NAME}-${CI_DEFAULT_BRANCH}-${CCACHE_KEY_SUFFIX} + when: always + paths: + - ${CCACHE_DIR} + + before_script: + - 'echo "LCG_VERSION: ${LCG_VERSION}"' + - 'echo "COMPILER: ${COMPILER}"' + + # Figure out LCG platform name based on version number and OS + - > + if [ "$OS" = "alma9" ]; then + export LCG_PLATFORM="el9" + else + export LCG_PLATFORM="$OS" + fi + + - 'echo "LCG_PLATFORM: ${LCG_PLATFORM}"' + - source /cvmfs/sft.cern.ch/lcg/views/LCG_${LCG_VERSION}/x86_64-${LCG_PLATFORM}-${COMPILER}-opt/setup.sh + + - git clone $CLONE_URL src + + - cd src + - git checkout $HEAD_SHA + - git submodule init + - git submodule update + - cd .. + + - ccache --version + + script: + - > + cmake -B build -S src + --preset=gitlab-ci + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + -DACTS_BUILD_PLUGIN_GEOMODEL=OFF # GeoModel is not in LCG at this point + + - ccache -z + - cmake --build build -- -j6 + - ccache -s + + - ctest --test-dir build -j$(nproc) + + +lcg_105: + extends: .lcg_base_job + + variables: + LCG_VERSION: "105" + + parallel: + matrix: + - OS: [alma9] + COMPILER: + - gcc13 + - clang16 + +lcg_106a: + extends: .lcg_base_job + + variables: + LCG_VERSION: "106a" + + parallel: + matrix: + - OS: [alma9] + COMPILER: + - gcc13 + - gcc14 + - clang16 From 6b36f01cc6f682bd41743ca1c6c86313c35ce172 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 10 Jan 2025 13:44:53 +0100 Subject: [PATCH 162/191] add spack cache, cleanup to other jobs --- .gitlab-ci.yml | 105 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 79 insertions(+), 26 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1ed6af57862..954bb73dba2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,6 +18,35 @@ variables: paths: - ${CCACHE_DIR} +.spack_cache: + cache: + - key: + files: + - CI/dependencies/setup_spack.sh + prefix: spack_${SPACK_VERSION} + paths: + - spack + - .spack + - key: + files: + - spack_lockfile_cache/digest.txt + prefix: spack_lockfile_ + paths: + - spack_lockfile_cache + +.spack_cleanup: + after_script: + - du -sh spack + - du -sh spack/* + - du -sh spack/opt/spack/* + - du -sh spack/var/spack/* + - rm -rf spack/.git + - rm -rf spack/opt + - rm -rf spack/var/spack/cache + - find spack -type f -name "*.pyc" -delete + - du -sh spack + - du -sh spack/* + - du -sh spack/var/spack/* # clang_tidy: @@ -71,6 +100,9 @@ variables: # # # Generate an html report # - codereport clang-tidy/clang-tidy.json clang-tidy/html + # + # after_script: + # - !reference [.spack_cleanup, after_script] build_exatrkx_cpu: stage: build @@ -82,6 +114,7 @@ build_exatrkx_cpu: cache: - !reference [.ccache_base, cache] + - !reference [.spack_cache, cache] script: - export PATH=/usr/local/sbin:/usr/sbin:/sbin:$PATH @@ -104,6 +137,9 @@ build_exatrkx_cpu: - cmake --build build -- -j6 - ccache -s + after_script: + - !reference [.spack_cleanup, after_script] + build_exatrkx: stage: build image: ghcr.io/acts-project/ubuntu2204_exatrkx:63 @@ -114,6 +150,7 @@ build_exatrkx: cache: - !reference [.ccache_base, cache] + - !reference [.spack_cache, cache] artifacts: paths: @@ -143,6 +180,9 @@ build_exatrkx: - cmake --build build -- -j6 - ccache -s + after_script: + - !reference [.spack_cleanup, after_script] + test_exatrkx_unittests: stage: test needs: @@ -152,6 +192,9 @@ test_exatrkx_unittests: DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst tags: - docker-gpu-nvidia + + cache: + - !reference [.spack_cache, cache] script: - apt-get update -y @@ -162,6 +205,9 @@ test_exatrkx_unittests: - cd .. - ctest --test-dir build -R ExaTrkX + after_script: + - !reference [.spack_cleanup, after_script] + test_exatrkx_python: stage: test needs: @@ -171,6 +217,10 @@ test_exatrkx_python: DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst tags: - docker-gpu-nvidia + + cache: + - !reference [.spack_cache, cache] + script: - apt-get update -y - git clone $CLONE_URL src @@ -187,6 +237,9 @@ test_exatrkx_python: - pytest -rFsv -k torch --collect-only - pytest -rFsv -k gpu-torch # For now only test torch GPU pipeline + after_script: + - !reference [.spack_cleanup, after_script] + build_linux_ubuntu: stage: build image: registry.cern.ch/ghcr.io/acts-project/ubuntu2404:63 @@ -195,32 +248,7 @@ build_linux_ubuntu: cache: - !reference [.ccache_base, cache] - - key: - files: - - CI/dependencies/setup_spack.sh - prefix: spack_${SPACK_VERSION} - paths: - - spack - - .spack - - key: - files: - - spack_lockfile_cache/digest.txt - prefix: spack_lockfile_ - paths: - - spack_lockfile_cache - - after_script: - - du -sh spack - - du -sh spack/* - - du -sh spack/opt/spack/* - - du -sh spack/var/spack/* - - rm -rf spack/.git - - rm -rf spack/opt - - rm -rf spack/var/spack/cache - - find spack -type f -name "*.pyc" -delete - - du -sh spack - - du -sh spack/* - - du -sh spack/var/spack/* + - !reference [.spack_cache, cache] artifacts: paths: @@ -254,6 +282,10 @@ build_linux_ubuntu: - cmake --build build -- -j6 - ccache -s + after_script: + - !reference [.spack_cleanup, after_script] + + linux_test_examples: stage: test image: registry.cern.ch/ghcr.io/acts-project/ubuntu2404:63 @@ -261,6 +293,9 @@ linux_test_examples: DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.$DEPENDENCY_TAG.tar.zst needs: [build_linux_ubuntu] + cache: + - !reference [.spack_cache, cache] + script: - apt-get update && apt-get install -y git-lfs @@ -278,6 +313,9 @@ linux_test_examples: - python3 -m pip install -r Examples/Python/tests/requirements.txt - pytest -rFsv -k "not exatrkx" -v -s + after_script: + - !reference [.spack_cleanup, after_script] + linux_physmon: stage: test image: registry.cern.ch/ghcr.io/acts-project/ubuntu2404:63 @@ -291,6 +329,9 @@ linux_physmon: - src/physmon expire_in: 1 week + cache: + - !reference [.spack_cache, cache] + script: - apt-get update && apt-get install -y git-lfs time @@ -315,6 +356,9 @@ linux_physmon: - cd src - CI/physmon/phys_perf_mon.sh all physmon + after_script: + - !reference [.spack_cleanup, after_script] + ############################### ### UBUNTU EXTRA JOB MATRIX ### ############################### @@ -327,6 +371,7 @@ linux_physmon: cache: - !reference [.ccache_base, cache] + - !reference [.spack_cache, cache] script: - git clone $CLONE_URL src @@ -369,6 +414,10 @@ linux_physmon: # Downstream run - ./build-downstream/bin/ShowActsVersion + after_script: + - !reference [.spack_cleanup, after_script] + - !reference [.spack_cache, cache] + linux_ubuntu_2204: extends: .linux_ubuntu_extra variables: @@ -402,6 +451,7 @@ linux_ubuntu_2204_clang: cache: - !reference [.ccache_base, cache] + - !reference [.spack_cache, cache] before_script: - 'echo "LCG_VERSION: ${LCG_VERSION}"' @@ -441,6 +491,9 @@ linux_ubuntu_2204_clang: - ctest --test-dir build -j$(nproc) + after_script: + - !reference [.spack_cleanup, after_script] + lcg_105: extends: .lcg_base_job From 81e1191386e5f7fac7f0caf2ec47ece4fecdba4a Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 10 Jan 2025 13:50:02 +0100 Subject: [PATCH 163/191] fix syntax --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 954bb73dba2..2df84207482 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,8 +22,8 @@ variables: cache: - key: files: - - CI/dependencies/setup_spack.sh - prefix: spack_${SPACK_VERSION} + - CI/dependencies/setup_spack.sh + prefix: spack_${SPACK_VERSION} paths: - spack - .spack From ac99ab48379ab5e7545ab3367f90d44f6631aecb Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 10 Jan 2025 13:53:32 +0100 Subject: [PATCH 164/191] and another one --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2df84207482..d79153e27b7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,8 +31,8 @@ variables: files: - spack_lockfile_cache/digest.txt prefix: spack_lockfile_ - paths: - - spack_lockfile_cache + paths: + - spack_lockfile_cache .spack_cleanup: after_script: From a002ddd7c0139c7ec165ee20f50dd6a7826e0d9a Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 10 Jan 2025 13:55:58 +0100 Subject: [PATCH 165/191] another one --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d79153e27b7..7e0d3e9e137 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -416,7 +416,6 @@ linux_physmon: after_script: - !reference [.spack_cleanup, after_script] - - !reference [.spack_cache, cache] linux_ubuntu_2204: extends: .linux_ubuntu_extra From f887058b65daf5f8a9f1069900dda0e18fb56646 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 10 Jan 2025 14:56:50 +0100 Subject: [PATCH 166/191] lcg doesn't need spack --- .gitlab-ci.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7e0d3e9e137..c8827dbf285 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -450,7 +450,6 @@ linux_ubuntu_2204_clang: cache: - !reference [.ccache_base, cache] - - !reference [.spack_cache, cache] before_script: - 'echo "LCG_VERSION: ${LCG_VERSION}"' @@ -490,9 +489,6 @@ linux_ubuntu_2204_clang: - ctest --test-dir build -j$(nproc) - after_script: - - !reference [.spack_cleanup, after_script] - lcg_105: extends: .lcg_base_job From 7f41271ba6e73cd54038dd33b3da8dcb168d74d3 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 10 Jan 2025 15:14:47 +0100 Subject: [PATCH 167/191] fix for downstream execeution breaking --- CI/dependencies/setup.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index e850008e6b9..0fab18961d4 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -1,6 +1,9 @@ #!/bin/bash -set -e -set -u +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + # Only set these properties if it's called as a subprocess + set -e + set -u +fi SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) @@ -211,3 +214,5 @@ end_section # Pythia8 looks for settings in this directory # set_env PYTHIA8DATA "${destination}/share/Pythia8/xmldoc" + +set +u From 5600c5e06e2388d2a2ce7a4775e1f39d762d5dad Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Sat, 11 Jan 2025 21:59:35 +0100 Subject: [PATCH 168/191] increase FPE stack trace depth --- Examples/Scripts/Python/event_recording.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Examples/Scripts/Python/event_recording.py b/Examples/Scripts/Python/event_recording.py index 8c31799c768..91f9e25e538 100755 --- a/Examples/Scripts/Python/event_recording.py +++ b/Examples/Scripts/Python/event_recording.py @@ -20,7 +20,9 @@ def runEventRecording(detector, outputDir, s=None): os.mkdir(hepmc_dir) s = s or acts.examples.Sequencer( - events=int(os.environ.get("NEVENTS", 100)), numThreads=1 + events=int(os.environ.get("NEVENTS", 100)), + numThreads=1, + fpeStackTraceLength=20, ) rnd = acts.examples.RandomNumbers(seed=42) From 904336d704a3c00d5c016d42b483d6f08c3b06ef Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Sun, 12 Jan 2025 10:45:52 +0100 Subject: [PATCH 169/191] try to mask G4 FPE --- Examples/Algorithms/Geant4HepMC/src/EventRecording.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Examples/Algorithms/Geant4HepMC/src/EventRecording.cpp b/Examples/Algorithms/Geant4HepMC/src/EventRecording.cpp index 51826af837a..00b9fd7a853 100644 --- a/Examples/Algorithms/Geant4HepMC/src/EventRecording.cpp +++ b/Examples/Algorithms/Geant4HepMC/src/EventRecording.cpp @@ -81,7 +81,7 @@ ProcessCode EventRecording::execute(const AlgorithmContext& context) const { part); // Begin with the simulation - m_runManager->BeamOn(1); + m_runManager->BeamOn(1); // MARK: fpeMask(FLTINV, 1, #4021) // Test if the event was aborted if (Geant4::HepMC3::SteppingAction::instance()->eventAborted()) { From 6cb9639795760417c75e79a8b0ae8d23bd400fd2 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Sun, 12 Jan 2025 10:52:43 +0100 Subject: [PATCH 170/191] restore ccache cache size --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 268663bcb74..4340a292b82 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ variables: CCACHE_DIR: ${CI_PROJECT_DIR}/ccache - CCACHE_MAXSIZE: 500M + CCACHE_MAXSIZE: 2G CCACHE_KEY_SUFFIX: r2 CTEST_OUTPUT_ON_FAILURE: 1 From fd25cb8d395af4345d2e404b3f34fdbc8afa7602 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Sun, 12 Jan 2025 12:04:22 +0100 Subject: [PATCH 171/191] wider mask range --- Examples/Algorithms/Geant4HepMC/src/EventRecording.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Examples/Algorithms/Geant4HepMC/src/EventRecording.cpp b/Examples/Algorithms/Geant4HepMC/src/EventRecording.cpp index 00b9fd7a853..55da5d9c7a7 100644 --- a/Examples/Algorithms/Geant4HepMC/src/EventRecording.cpp +++ b/Examples/Algorithms/Geant4HepMC/src/EventRecording.cpp @@ -81,12 +81,16 @@ ProcessCode EventRecording::execute(const AlgorithmContext& context) const { part); // Begin with the simulation - m_runManager->BeamOn(1); // MARK: fpeMask(FLTINV, 1, #4021) + // MARK: fpeMaskBegin(FLTINV, 1, #4021) + m_runManager->BeamOn(1); // this is where the actual FPE occurs // Test if the event was aborted - if (Geant4::HepMC3::SteppingAction::instance()->eventAborted()) { + if (Geant4::HepMC3::SteppingAction::instance() + ->eventAborted()) { // this is where the FPE is attributed to due + // to inlining continue; } + // MARK: fpeMaskEnd(FLTINV) // Set event start time HepMC3::GenEvent event = Geant4::HepMC3::EventAction::instance()->event(); From 61479f35032f08dc5b440021e5a3ee64857c0f52 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Sun, 12 Jan 2025 17:18:36 +0100 Subject: [PATCH 172/191] cleanup, re-add clang-tidy, explicit compilers --- .gitlab-ci.yml | 140 +++++++++++++++++++++++-------------------------- 1 file changed, 66 insertions(+), 74 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4340a292b82..6503a81d6cd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -36,73 +36,63 @@ variables: .spack_cleanup: after_script: - - du -sh spack - - du -sh spack/* - - du -sh spack/opt/spack/* - - du -sh spack/var/spack/* - - rm -rf spack/.git - - rm -rf spack/opt - - rm -rf spack/var/spack/cache + - rm -rf spack/.git spack/opt spack/var/spack/cache - find spack -type f -name "*.pyc" -delete - - du -sh spack - - du -sh spack/* - - du -sh spack/var/spack/* - - -# clang_tidy: -# stage: build -# image: ghcr.io/acts-project/ubuntu2404:63 -# tags: -# - large -# artifacts: -# paths: -# - src/clang-tidy/ -# when: always -# expire_in: 1 week -# variables: -# DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.$DEPENDENCY_TAG.tar.zst -# script: -# - git clone $CLONE_URL src -# - cd src -# - git checkout $HEAD_SHA -# - > -# apt-get update -# && apt-get install -y clang-tidy-18 -# && ln -sf /usr/bin/clang++-18 /usr/bin/clang++ -# && ln -sf /usr/bin/clang-18 /usr/bin/clang -# && ln -sf /usr/bin/clang-tidy-18 /usr/bin/clang-tidy -# -# - source CI/dependencies/setup.sh -# -# - > -# cmake -B build -S . -# --preset=gitlab-ci-clangtidy -# -DCMAKE_CXX_COMPILER=clang++ -# -DCMAKE_C_COMPILER=clang -# -# # Main clang-tidy run during cmake compilation -# - CI/clang_tidy/run_clang_tidy.sh clang-tidy build -# -# # Install dependencies for processing scripts -# - python3 -m pip install -r CI/clang_tidy/requirements.txt -# -# # Parse the main clang-tidy run -# - > -# CI/clang_tidy/parse_clang_tidy.py -# clang-tidy/clang-tidy.log -# clang-tidy/clang-tidy.json -# --exclude "*thirdparty*" -# --exclude "*ActsPodioEdm*" -# --exclude "*build/_deps/*" -# -# # Check the combined report against the defined limits -# - CI/clang_tidy/check_clang_tidy.py --report clang-tidy/clang-tidy.json --config CI/clang_tidy/limits.yml -# -# # Generate an html report -# - codereport clang-tidy/clang-tidy.json clang-tidy/html - # - # after_script: - # - !reference [.spack_cleanup, after_script] + +clang_tidy: + stage: build + image: ghcr.io/acts-project/ubuntu2404:63 + tags: + - large + artifacts: + paths: + - src/clang-tidy/ + when: always + expire_in: 1 week + variables: + DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.$DEPENDENCY_TAG.tar.zst + script: + - git clone $CLONE_URL src + - cd src + - git checkout $HEAD_SHA + - > + apt-get update + && apt-get install -y clang-tidy-18 + && ln -sf /usr/bin/clang++-18 /usr/bin/clang++ + && ln -sf /usr/bin/clang-18 /usr/bin/clang + && ln -sf /usr/bin/clang-tidy-18 /usr/bin/clang-tidy + + - source CI/dependencies/setup.sh -c clang++ + + - > + cmake -B build -S . + --preset=gitlab-ci-clangtidy + -DCMAKE_CXX_COMPILER=clang++ + -DCMAKE_C_COMPILER=clang + + # Main clang-tidy run during cmake compilation + - CI/clang_tidy/run_clang_tidy.sh clang-tidy build + + # Install dependencies for processing scripts + - python3 -m pip install -r CI/clang_tidy/requirements.txt + + # Parse the main clang-tidy run + - > + CI/clang_tidy/parse_clang_tidy.py + clang-tidy/clang-tidy.log + clang-tidy/clang-tidy.json + --exclude "*thirdparty*" + --exclude "*ActsPodioEdm*" + --exclude "*build/_deps/*" + + # Check the combined report against the defined limits + - CI/clang_tidy/check_clang_tidy.py --report clang-tidy/clang-tidy.json --config CI/clang_tidy/limits.yml + + # Generate an html report + - codereport clang-tidy/clang-tidy.json clang-tidy/html + + after_script: + - !reference [.spack_cleanup, after_script] build_exatrkx_cpu: stage: build @@ -124,7 +114,7 @@ build_exatrkx_cpu: - git clone $CLONE_URL src - cd src - git checkout $HEAD_SHA - - source CI/dependencies/setup.sh + - source CI/dependencies/setup.sh -c g++ - cd .. - mkdir build # Here we only do a minimal build without examples to save resources @@ -168,7 +158,7 @@ build_exatrkx: - git clone $CLONE_URL src - cd src - git checkout $HEAD_SHA - - source CI/dependencies/setup.sh + - source CI/dependencies/setup.sh -c g++ - cd .. - mkdir build - > @@ -201,7 +191,7 @@ test_exatrkx_unittests: - git clone $CLONE_URL src - cd src - git checkout $HEAD_SHA - - source CI/dependencies/setup.sh + - source CI/dependencies/setup.sh -c g++ - cd .. - ctest --test-dir build -R ExaTrkX @@ -227,7 +217,7 @@ test_exatrkx_python: - cd src - git checkout $HEAD_SHA - nvidia-smi - - source CI/dependencies/setup.sh + - source CI/dependencies/setup.sh -c g++ - source ../build/this_acts_withdeps.sh - python3 -m pip install -r Examples/Python/tests/requirements.txt - echo $PYTHONPATH @@ -304,7 +294,7 @@ linux_test_examples: - git checkout $HEAD_SHA - git submodule init - git submodule update - - source CI/dependencies/setup.sh + - source CI/dependencies/setup.sh -c g++ - cd .. - geant4-config --install-datasets @@ -340,7 +330,7 @@ linux_physmon: - git checkout $HEAD_SHA - git submodule init - git submodule update - - source CI/dependencies/setup.sh + - source CI/dependencies/setup.sh -c g++ - cd .. - git config --global safe.directory "$GITHUB_WORKSPACE" @@ -380,7 +370,7 @@ linux_physmon: - git checkout $HEAD_SHA - git submodule init - git submodule update - - source CI/dependencies/setup.sh + - source CI/dependencies/setup.sh -c ${COMPILER} - cd .. - mkdir build @@ -422,6 +412,7 @@ linux_ubuntu_2204: variables: CXXSTD: 20 DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst + COMPILER: g++ image: registry.cern.ch/ghcr.io/acts-project/ubuntu2204:63 linux_ubuntu_2204_clang: @@ -429,7 +420,8 @@ linux_ubuntu_2204_clang: variables: CXXSTD: 20 DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst - image: registry.cern.ch/ghcr.io/acts-project/ubuntu2204_clang:63 + COMPILER: clang++ + image: registry.cern.ch/ghcr.io/acts-project/ubuntu2204:63 ###################### From 4a08350fbca97c690513aa0dfbe445ce7de90d1c Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Sun, 12 Jan 2025 17:21:49 +0100 Subject: [PATCH 173/191] add analysis.yml to debug --- .github/workflows/analysis.yml | 150 +++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 .github/workflows/analysis.yml diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml new file mode 100644 index 00000000000..0e110ae2ad0 --- /dev/null +++ b/.github/workflows/analysis.yml @@ -0,0 +1,150 @@ +name: Analysis + +on: + push: + branches: + - main + pull_request: + branches: + - main + - 'release/**' + - 'develop/**' + paths-ignore: + - "docs/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + CTEST_OUTPUT_ON_FAILURE: 1 + CCACHE_DIR: ${{ github.workspace }}/ccache + CCACHE_MAXSIZE: 1.25G + CCACHE_KEY_SUFFIX: r2 + ACTS_LOG_FAILURE_THRESHOLD: WARNING + DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.v5.tar.zst + +# NOTE this only builds core unittests to reduce the output size. if we +# found a way to have Github actions not fail regularly with this job +# all unit tests should be reactivated. +jobs: + build_debug: + runs-on: ubuntu-latest + container: ghcr.io/acts-project/ubuntu2404:63 + steps: + + - uses: actions/checkout@v4 + + - name: Install dependencies + uses: ./.github/actions/dependencies + with: + compiler: g++ + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Cache build + uses: actions/cache@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} + restore-keys: | + ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- + + - name: Configure + run: > + ccache -z && + cmake -B build -S . + --preset=github-ci + -DCMAKE_BUILD_TYPE=Debug + -DCMAKE_CXX_FLAGS="-Werror --coverage -g -gz -g1" + -DACTS_BUILD_ODD=OFF + - name: Build + run: cmake --build build + - name: ccache stats + run: ccache -s + - name: Unit tests + run: ctest --test-dir build -j$(nproc) + + - name: Remove .o files + run: > + du -sh build + && find build -name *.o -delete + && du -sh build + - name: Coverage + run: > + pip3 install gcovr==7.2 + && cd build + && /usr/bin/python3 ../CI/test_coverage.py + + - uses: actions/upload-artifact@v4 + with: + name: coverage-build + path: build + + - name: Save PR number to file + if: github.event_name == 'pull_request' + run: echo ${{ github.event.number }} > PR_NUMBER.txt + + - name: Persist PR number + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@v4 + with: + name: PR_NUMBER + path: PR_NUMBER.txt + + build_performance: + runs-on: ubuntu-latest + container: ghcr.io/acts-project/ubuntu2404:63 + # if: github.ref == 'refs/heads/main' + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + uses: ./.github/actions/dependencies + with: + compiler: g++ + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Install dependencies + run: pip3 install git+https://github.com/paulgessinger/cmakeperf.git@2a409b5 + - name: Configure + run: > + cmake -B build -S . + --preset=github-ci + -DCMAKE_BUILD_TYPE=RelWithDebInfo + -DCMAKE_CXX_FLAGS="-Werror" + -DPython_EXECUTABLE=$(which python3) + -DACTS_BUILD_ODD=OFF + - name: Measure + run: cmakeperf collect build/compile_commands.json -o perf.csv + - name: Results + run: cmakeperf print perf.csv + - uses: actions/upload-artifact@v4 + with: + name: cmakeperf + path: perf.csv + + # metric_tracking: + # runs-on: ubuntu-latest + # needs: build_performance + # if: github.ref == 'refs/heads/main' + # steps: + # - uses: actions/checkout@v4 + # - name: Install dependencies + # run: pip3 install git+https://github.com/paulgessinger/headwind.git@eeeaa80 + # - uses: actions/download-artifact@v4 + # with: + # name: cmakeperf + # - name: Run collection + # env: + # SSH_AUTH_SOCK: /tmp/ssh_agent.sock + # run: | + # ssh-agent -a $SSH_AUTH_SOCK > /dev/null + # ssh-add - <<< "${{ secrets.METRIC_DEPLOY_SSH_KEY }}" + # git config --global user.email "action@github.com" + # git config --global user.name "GitHub Action" + # git clone git@github.com:acts-project/metrics.git + # hdw collect CI/headwind.yml --commit $(git log --pretty=format:'%H' -1) + # cd metrics + # git add -A + # git commit -m"update metrics" + # git push From 6406e3a5f281df3ecb6d437225f5f586ce69e3d6 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Sun, 12 Jan 2025 19:38:43 +0100 Subject: [PATCH 174/191] bump image for ubuntu clang --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6503a81d6cd..33d163569d1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -413,7 +413,7 @@ linux_ubuntu_2204: CXXSTD: 20 DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst COMPILER: g++ - image: registry.cern.ch/ghcr.io/acts-project/ubuntu2204:63 + image: registry.cern.ch/ghcr.io/acts-project/ubuntu2204:71 linux_ubuntu_2204_clang: extends: .linux_ubuntu_extra @@ -421,7 +421,7 @@ linux_ubuntu_2204_clang: CXXSTD: 20 DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst COMPILER: clang++ - image: registry.cern.ch/ghcr.io/acts-project/ubuntu2204:63 + image: registry.cern.ch/ghcr.io/acts-project/ubuntu2204:71 ###################### From c14e67e1c4cdf64973c770a3145d4e9ba90f815d Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Sun, 12 Jan 2025 19:38:59 +0100 Subject: [PATCH 175/191] missing dependency tag --- .github/workflows/analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml index 0e110ae2ad0..a0a806377a9 100644 --- a/.github/workflows/analysis.yml +++ b/.github/workflows/analysis.yml @@ -22,7 +22,7 @@ env: CCACHE_MAXSIZE: 1.25G CCACHE_KEY_SUFFIX: r2 ACTS_LOG_FAILURE_THRESHOLD: WARNING - DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.v5.tar.zst + DEPENDENCY_TAG: v7_b7 # NOTE this only builds core unittests to reduce the output size. if we # found a way to have Github actions not fail regularly with this job From 2a90a86649a053b25d0e4c1eab992c6c5241a871 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 13 Jan 2025 08:17:23 +0100 Subject: [PATCH 176/191] set to correct compiler in build --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 33d163569d1..75ff9f59618 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -377,6 +377,7 @@ linux_physmon: - > cmake -B build -S src --preset=gitlab-ci + -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_CXX_STANDARD=${CXXSTD} - ccache -z @@ -395,6 +396,7 @@ linux_physmon: -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS=-Werror + -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_CXX_STANDARD=${CXXSTD} -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" From 9ffcf926fafa5a85b48290e7a970f0d153128c98 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 13 Jan 2025 08:17:32 +0100 Subject: [PATCH 177/191] limit pytest to failing test for debug --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 75ff9f59618..00bb1783c64 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -301,7 +301,7 @@ linux_test_examples: - source build/this_acts_withdeps.sh - cd src - python3 -m pip install -r Examples/Python/tests/requirements.txt - - pytest -rFsv -k "not exatrkx" -v -s + - pytest -rFsv -k "test_full_chain_odd_example_pythia_geant4" -v -s after_script: - !reference [.spack_cleanup, after_script] From a52bcc05a07529508f686d190299bd57b0f102f2 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 13 Jan 2025 09:37:50 +0100 Subject: [PATCH 178/191] ignore find failure --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 00bb1783c64..eec501b32b8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,7 +37,7 @@ variables: .spack_cleanup: after_script: - rm -rf spack/.git spack/opt spack/var/spack/cache - - find spack -type f -name "*.pyc" -delete + - find spack -type f -name "*.pyc" -delete || true clang_tidy: stage: build From 5cce4e8dd3f270680f1c852b31abf9dc27f2ba16 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 13 Jan 2025 10:10:38 +0100 Subject: [PATCH 179/191] mask two more FPEs --- Core/include/Acts/Visualization/Interpolation3D.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Core/include/Acts/Visualization/Interpolation3D.hpp b/Core/include/Acts/Visualization/Interpolation3D.hpp index ea8e649869e..96298701591 100644 --- a/Core/include/Acts/Visualization/Interpolation3D.hpp +++ b/Core/include/Acts/Visualization/Interpolation3D.hpp @@ -59,8 +59,13 @@ trajectory_type spline(const trajectory_type& inputsRaw, std::size_t nPoints, for (std::size_t i = 0; i < inputs.size(); ++i) { points.col(i) = inputs[i].transpose(); } + + // MARK: fpeMaskBegin(FLTDIV, 1, #4024) + // MARK: fpeMaskBegin(FLTINV, 1, #4024) Eigen::Spline spline3D = Eigen::SplineFitting>::Interpolate(points, 2); + // MARK: fpeMaskEnd(FLTDIV) + // MARK: fpeMaskEnd(FLTINV) double step = 1. / (nPoints - 1); for (std::size_t i = 0; i < nPoints; ++i) { From 956db090a5ff00df0ad567d0e081860a8f60a26d Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 13 Jan 2025 11:38:40 +0100 Subject: [PATCH 180/191] remove DEPENDENCY_URL, add spack cache to clang-tidy job --- .gitlab-ci.yml | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eec501b32b8..018d4164dbb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -49,8 +49,10 @@ clang_tidy: - src/clang-tidy/ when: always expire_in: 1 week - variables: - DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.$DEPENDENCY_TAG.tar.zst + + cache: + - !reference [.spack_cache, cache] + script: - git clone $CLONE_URL src - cd src @@ -97,8 +99,6 @@ clang_tidy: build_exatrkx_cpu: stage: build image: registry.cern.ch/ghcr.io/acts-project/ubuntu2204_exatrkx:63 - variables: - DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst tags: - large @@ -133,8 +133,6 @@ build_exatrkx_cpu: build_exatrkx: stage: build image: registry.cern.ch/ghcr.io/acts-project/ubuntu2204_exatrkx:63 - variables: - DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst tags: - large @@ -178,8 +176,6 @@ test_exatrkx_unittests: needs: - build_exatrkx image: registry.cern.ch/ghcr.io/acts-project/ubuntu2204_exatrkx:63 - variables: - DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst tags: - docker-gpu-nvidia @@ -203,8 +199,6 @@ test_exatrkx_python: needs: - build_exatrkx image: registry.cern.ch/ghcr.io/acts-project/ubuntu2204_exatrkx:63 - variables: - DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst tags: - docker-gpu-nvidia @@ -233,8 +227,6 @@ test_exatrkx_python: build_linux_ubuntu: stage: build image: registry.cern.ch/ghcr.io/acts-project/ubuntu2404:63 - variables: - DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.$DEPENDENCY_TAG.tar.zst cache: - !reference [.ccache_base, cache] @@ -279,8 +271,6 @@ build_linux_ubuntu: linux_test_examples: stage: test image: registry.cern.ch/ghcr.io/acts-project/ubuntu2404:63 - variables: - DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.$DEPENDENCY_TAG.tar.zst needs: [build_linux_ubuntu] cache: @@ -309,8 +299,6 @@ linux_test_examples: linux_physmon: stage: test image: registry.cern.ch/ghcr.io/acts-project/ubuntu2404:63 - variables: - DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.$DEPENDENCY_TAG.tar.zst needs: [build_linux_ubuntu] artifacts: @@ -413,7 +401,6 @@ linux_ubuntu_2204: extends: .linux_ubuntu_extra variables: CXXSTD: 20 - DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst COMPILER: g++ image: registry.cern.ch/ghcr.io/acts-project/ubuntu2204:71 @@ -421,7 +408,6 @@ linux_ubuntu_2204_clang: extends: .linux_ubuntu_extra variables: CXXSTD: 20 - DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.$DEPENDENCY_TAG.tar.zst COMPILER: clang++ image: registry.cern.ch/ghcr.io/acts-project/ubuntu2204:71 From a1b2dd1cbbdf545df650492cf4c434baa6cdd765 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 13 Jan 2025 13:10:11 +0100 Subject: [PATCH 181/191] Revert "increase FPE stack trace depth" This reverts commit 5600c5e06e2388d2a2ce7a4775e1f39d762d5dad. --- Examples/Scripts/Python/event_recording.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Examples/Scripts/Python/event_recording.py b/Examples/Scripts/Python/event_recording.py index 91f9e25e538..8c31799c768 100755 --- a/Examples/Scripts/Python/event_recording.py +++ b/Examples/Scripts/Python/event_recording.py @@ -20,9 +20,7 @@ def runEventRecording(detector, outputDir, s=None): os.mkdir(hepmc_dir) s = s or acts.examples.Sequencer( - events=int(os.environ.get("NEVENTS", 100)), - numThreads=1, - fpeStackTraceLength=20, + events=int(os.environ.get("NEVENTS", 100)), numThreads=1 ) rnd = acts.examples.RandomNumbers(seed=42) From 3751f6ebd45aa30e38a5d85501693453b8ec5cfb Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 13 Jan 2025 13:10:47 +0100 Subject: [PATCH 182/191] add other github jobs back in --- .../{builds.yml.disabled => builds.yml} | 1 + .github/workflows/checks.yml | 105 ++++++++++++++++++ .github/workflows/docs.yml | 61 ++++++++++ 3 files changed, 167 insertions(+) rename .github/workflows/{builds.yml.disabled => builds.yml} (99%) create mode 100644 .github/workflows/checks.yml create mode 100644 .github/workflows/docs.yml diff --git a/.github/workflows/builds.yml.disabled b/.github/workflows/builds.yml similarity index 99% rename from .github/workflows/builds.yml.disabled rename to .github/workflows/builds.yml index 13b1ba545c8..9d290d3ec85 100644 --- a/.github/workflows/builds.yml.disabled +++ b/.github/workflows/builds.yml @@ -316,6 +316,7 @@ jobs: -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS=-Werror -DCMAKE_CXX_STANDARD=${{ matrix.std }} + -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" - name: Downstream build diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 00000000000..59ba76bb00a --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,105 @@ +name: Checks + +on: + push: + pull_request: + branches: + - main + - 'release/**' + - 'develop/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-latest + env: + PRE_COMMIT_HOME: '/tmp/pre-commit' + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - uses: actions/cache@v4 + with: + path: | + ${{ env.PRE_COMMIT_HOME }} + key: ${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }} + + - name: Install pre-commit + run: pip install pre-commit + + - name: Run pre-commit + run: pre-commit run --all-files --show-diff-on-failure + + smearing_config: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Check + run: > + CI/check_smearing_config.py . + + missing_includes: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install clang + run: > + sudo apt-get install -y clang libeigen3-dev libboost-dev + - name: Check + run: > + CI/missing_include_check.sh + + fpe_masks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: > + pip install -r CI/fpe_masks/requirements.txt + - name: Check + run: > + CI/check_fpe_masks.py --token ${{ secrets.GITHUB_TOKEN }} + + unused_files: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Check + run: > + CI/check_unused_files.py + + codegen: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: > + pip install -r codegen/requirements.txt + - name: Check + run: > + CI/check_codegen + - uses: actions/upload-artifact@v4 + if: failure() + with: + name: changed + path: changed diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000000..dfa5b303ee2 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,61 @@ +name: Docs + +on: + push: + pull_request: + branches: + - main + - 'release/**' + - 'develop/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + docs: + runs-on: ubuntu-latest + env: + DOXYGEN_WARN_AS_ERROR: FAIL_ON_WARNINGS + DOXYGEN_VERSION: 1.9.8 + steps: + - uses: actions/checkout@v4 + + - name: Cache doxygen + id: cache-doxygen + uses: actions/cache@v4 + with: + path: /usr/local/bin/doxygen + key: doxygen_${{ env.DOXYGEN_VERSION }} + + - name: Install doxygen + if: steps.cache-doxygen.outputs.cache-hit != 'true' + run: > + curl -SL https://sourceforge.net/projects/doxygen/files/rel-${{ env.DOXYGEN_VERSION }}/doxygen-${{ env.DOXYGEN_VERSION }}.linux.bin.tar.gz/download | tar -xzC . + && mv doxygen-${{ env.DOXYGEN_VERSION }}/bin/doxygen /usr/local/bin/doxygen + + - name: Install dependencies + run: > + pip3 install --upgrade pip + && pip install -r docs/requirements.txt + + - name: Build documentation + run: > + cd docs + && sphinx-build + -b html + -d _build/doctrees/ + -j auto + -W + --keep-going + -t run_doxygen + -t lazy_autodoc + -t white_papers + -b linkcheck + . _build/html/ + + + - uses: actions/upload-artifact@v4 + with: + name: acts-docs + path: docs/_build/html/ From 57e5bc527c1e0450342fdb8e45bad9ac5d7036f7 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 13 Jan 2025 14:01:02 +0100 Subject: [PATCH 183/191] try to make spack location independent of current directory --- CI/dependencies/setup.sh | 9 +++++++-- CI/dependencies/setup_spack.sh | 8 +++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index 0fab18961d4..52db6caffd2 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -100,10 +100,16 @@ echo "Install destination: $destination" mkdir -p ${destination} +if [ -n "${GITLAB_CI:-}" ]; then + _spack_folder=${CI_PROJECT_DIR}/spack +else + _spack_folder=${PWD}/spack +fi start_section "Install spack if not already installed" if ! command -v spack &> /dev/null; then - "${SCRIPT_DIR}/setup_spack.sh" + "${SCRIPT_DIR}/setup_spack.sh" "${_spack_folder}" + source "${_spack_folder}/spack/share/spack/setup-env.sh" fi end_section @@ -113,7 +119,6 @@ if [ -n "${GITLAB_CI:-}" ]; then ln -s ${CI_PROJECT_DIR}/.spack ${HOME}/.spack fi -source "$(pwd)/spack/share/spack/setup-env.sh" if [ -n "${CI:-}" ]; then diff --git a/CI/dependencies/setup_spack.sh b/CI/dependencies/setup_spack.sh index 4cb0772e01e..b79a9de280b 100755 --- a/CI/dependencies/setup_spack.sh +++ b/CI/dependencies/setup_spack.sh @@ -4,10 +4,12 @@ set -u _spack_version=${SPACK_VERSION:-develop} -if [ ! -d "spack" ]; then +_spack_folder=$1 + +if [ ! -d "${_spack_folder}" ]; then echo "Cloning spack" - git clone --branch ${_spack_version} -c feature.manyFiles=true https://github.com/acts-project/spack.git - pushd spack > /dev/null + git clone --branch ${_spack_version} -c feature.manyFiles=true https://github.com/acts-project/spack.git ${_spack_folder} + pushd ${_spack_folder} > /dev/null git config user.name 'CI' git config user.email '<>' popd > /dev/null From a37a5b9b15d010fc69c9706cd0af9b079214559d Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 13 Jan 2025 14:28:35 +0100 Subject: [PATCH 184/191] github: give spack clone path --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 25d67c5f584..a77f29c02b6 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -44,7 +44,7 @@ runs: env: SPACK_VERSION: ${{ inputs.spack_version }} run: | - CI/dependencies/setup_spack.sh + CI/dependencies/setup_spack.sh ${PWD}/spack spack/bin/spack list > /dev/null rm -rf spack/.git mkdir spack_cache From 6fce37204df9952c314364b3b5253e31e645a39d Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 13 Jan 2025 14:36:04 +0100 Subject: [PATCH 185/191] path fix --- CI/dependencies/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index 52db6caffd2..cde254db6af 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -109,7 +109,7 @@ fi start_section "Install spack if not already installed" if ! command -v spack &> /dev/null; then "${SCRIPT_DIR}/setup_spack.sh" "${_spack_folder}" - source "${_spack_folder}/spack/share/spack/setup-env.sh" + source "${_spack_folder}/share/spack/setup-env.sh" fi end_section From 1c59936b053c5203f328a0c857151d1af3e4ea14 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Tue, 14 Jan 2025 13:17:58 +0100 Subject: [PATCH 186/191] add tests and downstream build to main ubuntu job, run tests single threaded --- .gitlab-ci.yml | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0459296ef82..b3eb50290ec 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -264,6 +264,25 @@ build_linux_ubuntu: - cmake --build build -- -j6 - ccache -s + - ctest --test-dir build #-j$(nproc) + - cmake --build build --target integrationtests + + # Install main project + - cmake --install build + + # Downstream configure + - > + cmake -B build-downstream -S src/Tests/DownstreamProject + -GNinja + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_CXX_FLAGS=-Werror + -DCMAKE_CXX_COMPILER=clang++ + -DCMAKE_CXX_STANDARD=${CXXSTD} + -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" + + # Downstream build + - cmake --build build-downstream + after_script: - !reference [.spack_cleanup, after_script] @@ -372,7 +391,7 @@ linux_physmon: - cmake --build build -- -j6 - ccache -s - - ctest --test-dir build -j$(nproc) + - ctest --test-dir build #-j$(nproc) - cmake --build build --target integrationtests # Install main project @@ -467,7 +486,7 @@ linux_ubuntu_2204_clang: - cmake --build build -- -j6 - ccache -s - - ctest --test-dir build -j$(nproc) + - ctest --test-dir build #-j$(nproc) lcg_105: From aab69b86ea7e41cda80bccf3fd77757bf54962db Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Tue, 14 Jan 2025 13:52:31 +0100 Subject: [PATCH 187/191] remove unneeded compiler setting --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b3eb50290ec..ab4dc5100b6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -276,7 +276,6 @@ build_linux_ubuntu: -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS=-Werror - -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_CXX_STANDARD=${CXXSTD} -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" From 592c74c6a6fa2d6a0e6d395e975e67c5009ac1f5 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Tue, 14 Jan 2025 14:14:35 +0100 Subject: [PATCH 188/191] pipe install log to file --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ab4dc5100b6..846144a1809 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -268,7 +268,7 @@ build_linux_ubuntu: - cmake --build build --target integrationtests # Install main project - - cmake --install build + - cmake --install build > install.log # Downstream configure - > @@ -394,7 +394,7 @@ linux_physmon: - cmake --build build --target integrationtests # Install main project - - cmake --install build + - cmake --install build > install.log # Downstream configure - > From 84ecb53934d4d932b78b2c0c6fa37fcac8005b95 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Tue, 14 Jan 2025 14:18:22 +0100 Subject: [PATCH 189/191] fix --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 846144a1809..30545017934 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -276,7 +276,6 @@ build_linux_ubuntu: -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS=-Werror - -DCMAKE_CXX_STANDARD=${CXXSTD} -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" # Downstream build From 6a348f30c5b241336728053a053d9b668a15a193 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 15 Jan 2025 17:43:16 +0100 Subject: [PATCH 190/191] follow up --- .github/workflows/analysis.yml | 2 +- .gitlab-ci.yml | 4 ++-- CI/dependencies/opengl.sh | 3 +++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml index a0a806377a9..a7cc7444849 100644 --- a/.github/workflows/analysis.yml +++ b/.github/workflows/analysis.yml @@ -94,7 +94,7 @@ jobs: build_performance: runs-on: ubuntu-latest container: ghcr.io/acts-project/ubuntu2404:63 - # if: github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/main' steps: - uses: actions/checkout@v4 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 30545017934..014cd59b746 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -382,7 +382,7 @@ linux_physmon: - > cmake -B build -S src --preset=gitlab-ci - -DCMAKE_CXX_COMPILER=clang++ + -DCMAKE_CXX_COMPILER=${COMPILER} -DCMAKE_CXX_STANDARD=${CXXSTD} - ccache -z @@ -401,7 +401,7 @@ linux_physmon: -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS=-Werror - -DCMAKE_CXX_COMPILER=clang++ + -DCMAKE_CXX_COMPILER=${COMPILER} -DCMAKE_CXX_STANDARD=${CXXSTD} -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" diff --git a/CI/dependencies/opengl.sh b/CI/dependencies/opengl.sh index b7db00bbcda..9e42da472cd 100755 --- a/CI/dependencies/opengl.sh +++ b/CI/dependencies/opengl.sh @@ -1,4 +1,7 @@ #!/bin/bash +# This file locates the the OpenGL library and adds it to the packages.yaml file. +# OpenGL needs to be supplied to spack as an *external*, which means that spack uses +# it in dependency resolution, but does not attempt to build it. set -u set -e From fe3f5168e2c595234d63e4797fbd5831fdbfb18e Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 17 Jan 2025 10:47:22 +0100 Subject: [PATCH 191/191] bump images --- .gitlab-ci.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f643ef7da68..0a8a4abb905 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,7 +41,7 @@ variables: clang_tidy: stage: build - image: registry.cern.ch/ghcr.io/acts-project/ubuntu2404:63 + image: registry.cern.ch/ghcr.io/acts-project/ubuntu2404:71 tags: - large artifacts: @@ -98,7 +98,7 @@ clang_tidy: build_exatrkx_cpu: stage: build - image: registry.cern.ch/ghcr.io/acts-project/ubuntu2204_exatrkx:63 + image: registry.cern.ch/ghcr.io/acts-project/ubuntu2204_exatrkx:71 tags: - large @@ -132,7 +132,7 @@ build_exatrkx_cpu: build_exatrkx: stage: build - image: registry.cern.ch/ghcr.io/acts-project/ubuntu2204_exatrkx:63 + image: registry.cern.ch/ghcr.io/acts-project/ubuntu2204_exatrkx:71 tags: - large @@ -175,7 +175,7 @@ test_exatrkx_unittests: stage: test needs: - build_exatrkx - image: registry.cern.ch/ghcr.io/acts-project/ubuntu2204_exatrkx:63 + image: registry.cern.ch/ghcr.io/acts-project/ubuntu2204_exatrkx:71 tags: - docker-gpu-nvidia @@ -198,7 +198,7 @@ test_exatrkx_python: stage: test needs: - build_exatrkx - image: registry.cern.ch/ghcr.io/acts-project/ubuntu2204_exatrkx:63 + image: registry.cern.ch/ghcr.io/acts-project/ubuntu2204_exatrkx:71 tags: - docker-gpu-nvidia @@ -226,7 +226,7 @@ test_exatrkx_python: build_linux_ubuntu: stage: build - image: registry.cern.ch/ghcr.io/acts-project/ubuntu2404:63 + image: registry.cern.ch/ghcr.io/acts-project/ubuntu2404:71 cache: - !reference [.ccache_base, cache] @@ -289,7 +289,7 @@ build_linux_ubuntu: linux_test_examples: stage: test - image: registry.cern.ch/ghcr.io/acts-project/ubuntu2404:63 + image: registry.cern.ch/ghcr.io/acts-project/ubuntu2404:71 needs: [build_linux_ubuntu] cache: @@ -318,6 +318,7 @@ linux_test_examples: linux_physmon: stage: test needs: [build_linux_ubuntu] + image: registry.cern.ch/ghcr.io/acts-project/ubuntu2404:71 artifacts: when: always @@ -435,7 +436,7 @@ linux_ubuntu_2204_clang: ###################### .lcg_base_job: - image: registry.cern.ch/ghcr.io/acts-project/${OS}-base:63 + image: registry.cern.ch/ghcr.io/acts-project/${OS}-base:71 stage: build tags: - cvmfs