From ed0597bd4d01b97b6089ecb92e16c5cf2372b7db Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Tue, 9 Jan 2024 12:01:22 +0100 Subject: [PATCH 01/22] ci: Fix ccache env variable and use Ninja in Linux/MacOS build --- .../workflows/conda/conda-env-macos-linux.yml | 18 +++++++ .../{conda-env.yml => conda-env-windows.yml} | 0 .github/workflows/macos-linux-conda.yml | 51 +++++++++++-------- .github/workflows/windows-conda-clang.yml | 2 +- .github/workflows/windows-conda-v142.yml | 2 +- 5 files changed, 49 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/conda/conda-env-macos-linux.yml rename .github/workflows/conda/{conda-env.yml => conda-env-windows.yml} (100%) diff --git a/.github/workflows/conda/conda-env-macos-linux.yml b/.github/workflows/conda/conda-env-macos-linux.yml new file mode 100644 index 0000000000..afcc3a8ace --- /dev/null +++ b/.github/workflows/conda/conda-env-macos-linux.yml @@ -0,0 +1,18 @@ +name: pinocchio +channels: + - conda-forge + - nodefaults +dependencies: + - eigen=3.4.0 + - hpp-fcl + - numpy + - boost + - eigenpy + - urdfdom + - python + - mamba + - cmake + - ccache + - llvm-openmp + - cxx-compiler + - pkg-config diff --git a/.github/workflows/conda/conda-env.yml b/.github/workflows/conda/conda-env-windows.yml similarity index 100% rename from .github/workflows/conda/conda-env.yml rename to .github/workflows/conda/conda-env-windows.yml diff --git a/.github/workflows/macos-linux-conda.yml b/.github/workflows/macos-linux-conda.yml index 8e09544c32..216d02f243 100644 --- a/.github/workflows/macos-linux-conda.yml +++ b/.github/workflows/macos-linux-conda.yml @@ -4,10 +4,13 @@ on: [push,pull_request] jobs: pinocchio-conda: - name: Pinocchio on ${{ matrix.os }} in ${{ matrix.build_type }} mode with Conda + name: ${{ matrix.os }} - Python ${{ matrix.python-version }} ${{ matrix.build_type }} with Conda runs-on: ${{ matrix.os }} env: - CCACHE_DIR: ${{ matrix.CCACHE_DIR }} + CCACHE_BASEDIR: "${GITHUB_WORKSPACE}" + CCACHE_DIR: "${GITHUB_WORKSPACE}/.ccache" + CCACHE_COMPRESS: true + CCACHE_COMPRESSLEVEL: 6 BUILD_ADVANCED_TESTING: ${{ matrix.BUILD_ADVANCED_TESTING }} strategy: @@ -15,13 +18,12 @@ jobs: matrix: os: ["ubuntu-latest", "macos-latest"] build_type: [Release, Debug] + python-version: ["3.8", "3.12"] include: - os: ubuntu-latest - CCACHE_DIR: /home/runner/.ccache BUILD_ADVANCED_TESTING: OFF - os: macos-latest - CCACHE_DIR: /Users/runner/.ccache BUILD_ADVANCED_TESTING: ON steps: @@ -31,40 +33,45 @@ jobs: - uses: actions/cache@v3 with: - path: ${{ env.CCACHE_DIR }} - key: ccache-conda-${{ matrix.os }}-${{ matrix.build_type }} + path: .ccache + key: ccache-macos-linux-conda-${{ matrix.os }}-${{ matrix.build_type }}-${{ github.sha }} + restore-keys: ccache-macos-linux-conda-${{ matrix.os }}-${{ matrix.build_type }}- - uses: conda-incubator/setup-miniconda@v3 with: activate-environment: pinocchio auto-update-conda: true environment-file: .github/workflows/conda/conda-env.yml - python-version: 3.8 + python-version: ${{ matrix.python-version }} + auto-activate-base: false - - name: Install cmake and update conda - shell: bash -l {0} - run: | - conda activate pinocchio - conda install cmake ccache llvm-openmp compilers=1.4.2 -c conda-forge - conda list - name: Build Pinocchio - shell: bash -l {0} + shell: bash -el {0} run: | - conda activate pinocchio + conda list echo $CONDA_PREFIX mkdir build cd build - cmake .. -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DBUILD_WITH_COLLISION_SUPPORT=ON -DBUILD_ADVANCED_TESTING=${{ env.BUILD_ADVANCED_TESTING }} -DBUILD_WITH_CASADI_SUPPORT=OFF -DPYTHON_EXECUTABLE=$(which python3) -DBUILD_WITH_OPENMP_SUPPORT=ON -DINSTALL_DOCUMENTATION=ON -DOpenMP_ROOT=$CONDA_PREFIX - make - make build_tests - CTEST_OUTPUT_ON_FAILURE=1 make test - make install + cmake .. \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \ + -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ + -DBUILD_WITH_COLLISION_SUPPORT=ON \ + -DBUILD_ADVANCED_TESTING=${{ env.BUILD_ADVANCED_TESTING }} \ + -DBUILD_WITH_CASADI_SUPPORT=OFF \ + -DPYTHON_EXECUTABLE=$(which python3) \ + -DBUILD_WITH_OPENMP_SUPPORT=ON \ + -DINSTALL_DOCUMENTATION=ON \ + -DOpenMP_ROOT=$CONDA_PREFIX + cmake --build . -j2 + ctest --output-on-failure + cmake --install . - name: Uninstall Pinocchio - shell: bash -l {0} + shell: bash -el {0} run: | cd build - make uninstall + cmake --build . --target uninstall diff --git a/.github/workflows/windows-conda-clang.yml b/.github/workflows/windows-conda-clang.yml index 028bc8f348..db299c53d2 100644 --- a/.github/workflows/windows-conda-clang.yml +++ b/.github/workflows/windows-conda-clang.yml @@ -26,7 +26,7 @@ jobs: ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true' with: activate-environment: pinocchio - environment-file: .github/workflows/conda/conda-env.yml + environment-file: .github/workflows/conda/conda-env-windows.yml python-version: 3.7 - name: Install cmake and update conda run: | diff --git a/.github/workflows/windows-conda-v142.yml b/.github/workflows/windows-conda-v142.yml index 29e8d3d741..f765d5ceba 100644 --- a/.github/workflows/windows-conda-v142.yml +++ b/.github/workflows/windows-conda-v142.yml @@ -25,7 +25,7 @@ jobs: ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true' with: activate-environment: pinocchio - environment-file: .github/workflows/conda/conda-env.yml + environment-file: .github/workflows/conda/conda-env-windows.yml python-version: 3.7 - name: Install cmake and update conda run: | From f5c25e344613a116d3f3cc071285e5a74f921fa1 Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Tue, 9 Jan 2024 12:12:39 +0100 Subject: [PATCH 02/22] ci: Add Python version in cache key --- .github/workflows/macos-linux-conda.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macos-linux-conda.yml b/.github/workflows/macos-linux-conda.yml index 216d02f243..0a25f89cf3 100644 --- a/.github/workflows/macos-linux-conda.yml +++ b/.github/workflows/macos-linux-conda.yml @@ -34,8 +34,8 @@ jobs: - uses: actions/cache@v3 with: path: .ccache - key: ccache-macos-linux-conda-${{ matrix.os }}-${{ matrix.build_type }}-${{ github.sha }} - restore-keys: ccache-macos-linux-conda-${{ matrix.os }}-${{ matrix.build_type }}- + key: ccache-macos-linux-conda-${{ matrix.os }}-${{ matrix.build_type }}-${{ matrix.python-version }}-${{ github.sha }} + restore-keys: ccache-macos-linux-conda-${{ matrix.os }}-${{ matrix.build_type }}-${{ matrix.python-version }}- - uses: conda-incubator/setup-miniconda@v3 with: @@ -75,3 +75,18 @@ jobs: run: | cd build cmake --build . --target uninstall + + check: + if: always() + name: check-macos-linux-conda + + needs: + - pinocchio-conda + + runs-on: Ubuntu-latest + + steps: + - name: Decide whether the needed jobs succeeded or failed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} From 37b3d7428269263302c69a265ce2d05eb56941a8 Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Tue, 9 Jan 2024 12:23:02 +0100 Subject: [PATCH 03/22] ci: Use the right environment --- .github/workflows/macos-linux-conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos-linux-conda.yml b/.github/workflows/macos-linux-conda.yml index 0a25f89cf3..dd1d5fb384 100644 --- a/.github/workflows/macos-linux-conda.yml +++ b/.github/workflows/macos-linux-conda.yml @@ -41,7 +41,7 @@ jobs: with: activate-environment: pinocchio auto-update-conda: true - environment-file: .github/workflows/conda/conda-env.yml + environment-file: .github/workflows/conda/conda-env-macos-linux.yml python-version: ${{ matrix.python-version }} auto-activate-base: false From d95ba2e47282160d17b6d5189c3ec9308a1c6fb6 Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Tue, 9 Jan 2024 15:15:48 +0100 Subject: [PATCH 04/22] ci: Use ccache in Windows build --- ...-linux.yml => environment_macos_linux.yml} | 5 +- ...nv-windows.yml => environment_windows.yml} | 6 +- .github/workflows/macos-linux-conda.yml | 3 +- .github/workflows/windows-conda-clang.yml | 62 +++++++------- .github/workflows/windows-conda-v142.yml | 82 +++++++------------ 5 files changed, 71 insertions(+), 87 deletions(-) rename .github/workflows/conda/{conda-env-macos-linux.yml => environment_macos_linux.yml} (89%) rename .github/workflows/conda/{conda-env-windows.yml => environment_windows.yml} (75%) diff --git a/.github/workflows/conda/conda-env-macos-linux.yml b/.github/workflows/conda/environment_macos_linux.yml similarity index 89% rename from .github/workflows/conda/conda-env-macos-linux.yml rename to .github/workflows/conda/environment_macos_linux.yml index afcc3a8ace..a465fe2ad5 100644 --- a/.github/workflows/conda/conda-env-macos-linux.yml +++ b/.github/workflows/conda/environment_macos_linux.yml @@ -1,7 +1,6 @@ name: pinocchio channels: - conda-forge - - nodefaults dependencies: - eigen=3.4.0 - hpp-fcl @@ -10,9 +9,9 @@ dependencies: - eigenpy - urdfdom - python - - mamba - cmake - ccache + - pkg-config + - ninja - llvm-openmp - cxx-compiler - - pkg-config diff --git a/.github/workflows/conda/conda-env-windows.yml b/.github/workflows/conda/environment_windows.yml similarity index 75% rename from .github/workflows/conda/conda-env-windows.yml rename to .github/workflows/conda/environment_windows.yml index 6d0dc37225..426ee8a438 100644 --- a/.github/workflows/conda/conda-env-windows.yml +++ b/.github/workflows/conda/environment_windows.yml @@ -1,7 +1,6 @@ name: pinocchio channels: - conda-forge - - nodefaults dependencies: - eigen=3.4.0 - hpp-fcl @@ -10,4 +9,7 @@ dependencies: - eigenpy - urdfdom - python - - mamba + - cmake + - ccache + - pkg-config + - ninja diff --git a/.github/workflows/macos-linux-conda.yml b/.github/workflows/macos-linux-conda.yml index dd1d5fb384..7c45fc236a 100644 --- a/.github/workflows/macos-linux-conda.yml +++ b/.github/workflows/macos-linux-conda.yml @@ -41,7 +41,7 @@ jobs: with: activate-environment: pinocchio auto-update-conda: true - environment-file: .github/workflows/conda/conda-env-macos-linux.yml + environment-file: .github/workflows/conda/environment_macos_linux.yml python-version: ${{ matrix.python-version }} auto-activate-base: false @@ -56,6 +56,7 @@ jobs: cd build cmake .. \ + -G "Ninja" \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \ -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ diff --git a/.github/workflows/windows-conda-clang.yml b/.github/workflows/windows-conda-clang.yml index db299c53d2..f9044a4890 100644 --- a/.github/workflows/windows-conda-clang.yml +++ b/.github/workflows/windows-conda-clang.yml @@ -6,45 +6,43 @@ on: jobs: build: runs-on: ${{ matrix.os }} + env: + CCACHE_BASEDIR: "${GITHUB_WORKSPACE}" + CCACHE_DIR: "${GITHUB_WORKSPACE}/.ccache" + CCACHE_COMPRESS: true + CCACHE_COMPRESSLEVEL: 6 + strategy: fail-fast: false matrix: - name: [windows-latest-clang-cl] - - include: - - name: windows-latest-clang-cl - os: windows-2019 - compiler: clang-cl + os: ["windows-2019"] steps: - uses: actions/checkout@v4 - - name: Checkout submodules - run: | - git submodule update --init - - uses: goanpeca/setup-miniconda@v2 - env: - ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true' + with: + submodules: recursive + + - uses: actions/cache@v3 + with: + path: .ccache + key: ccache-windows-conda-clang-${{ matrix.os }}-${{ github.sha }} + restore-keys: ccache-windows-conda-clang-${{ matrix.os }}- + + - uses: conda-incubator/setup-miniconda@v3 with: activate-environment: pinocchio - environment-file: .github/workflows/conda/conda-env-windows.yml - python-version: 3.7 - - name: Install cmake and update conda - run: | - conda install cmake -c main - conda list + auto-update-conda: true + environment-file: .github/workflows/conda/environment_windows.yml + python-version: "3.10" + auto-activate-base: false - name: Build Pinocchio shell: cmd /C CALL {0} - env: - ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true' run: | - :: unset extra Boost envs - set Boost_ROOT= - set BOOST_ROOT_1_69_0= - set BOOST_ROOT_1_72_0= - set PATH=%PATH:C:\hostedtoolcache\windows\Boost\1.72.0;=% - call "%programfiles(x86)%\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" amd64 + :: Tell Ninja to use clang-cl + set CC=clang-cl + set CXX=clang-cl :: Create build directory mkdir build @@ -52,24 +50,28 @@ jobs: :: Configure cmake ^ - -G "Visual Studio 16 2019" -T "ClangCl" -DCMAKE_GENERATOR_PLATFORM=x64 ^ + -G "Ninja" ^ -DCMAKE_INSTALL_PREFIX=%CONDA_PREFIX%\Library ^ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache ^ -DCMAKE_BUILD_TYPE=Release ^ -DPYTHON_SITELIB=%CONDA_PREFIX%\Lib\site-packages ^ -DPYTHON_EXECUTABLE=%CONDA_PREFIX%\python.exe ^ -DBUILD_WITH_URDF_SUPPORT=ON ^ - -DBUILD_PYTHON_INTERFACE=ON ^ + -DBUILD_PYTHON_INTERFACE=OFF ^ -DBUILD_WITH_COLLISION_SUPPORT=ON ^ + -DBUILD_TESTING=ON ^ -DINSTALL_DOCUMENTATION=ON ^ .. - :: Build + :: Build and Install cmake --build . --config Release --target install + if errorlevel 1 exit 1 :: Testing - set PATH=%PATH%;%CONDA_PREFIX%\Lib\site-packages\pinocchio ctest --output-on-failure -C Release -V + if errorlevel 1 exit 1 :: Test Python import cd .. python -c "import pinocchio" + if errorlevel 1 exit 1 diff --git a/.github/workflows/windows-conda-v142.yml b/.github/workflows/windows-conda-v142.yml index f765d5ceba..2f021fd16f 100644 --- a/.github/workflows/windows-conda-v142.yml +++ b/.github/workflows/windows-conda-v142.yml @@ -6,46 +6,39 @@ on: jobs: build: runs-on: ${{ matrix.os }} + env: + CCACHE_BASEDIR: "${GITHUB_WORKSPACE}" + CCACHE_DIR: "${GITHUB_WORKSPACE}/.ccache" + CCACHE_COMPRESS: true + CCACHE_COMPRESSLEVEL: 6 + strategy: fail-fast: false matrix: - name: [windows-latest] - - include: - - name: windows-latest - os: windows-2019 + os: ["windows-2019"] steps: - uses: actions/checkout@v4 - - name: Checkout submodules - run: | - git submodule update --init - - uses: goanpeca/setup-miniconda@v2 - env: - ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true' with: - activate-environment: pinocchio - environment-file: .github/workflows/conda/conda-env-windows.yml - python-version: 3.7 - - name: Install cmake and update conda - run: | - conda install cmake -c main - conda list + submodules: recursive - - name: Display the path - run: echo %cd% - shell: cmd + - uses: actions/cache@v3 + with: + path: .ccache + key: ccache-windows-conda-vs-${{ matrix.os }}-${{ github.sha }} + restore-keys: ccache-windows-conda-vs-${{ matrix.os }}- + + - uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: pinocchio + auto-update-conda: true + environment-file: .github/workflows/conda/environment_windows.yml + python-version: "3.10" + auto-activate-base: false - name: Build Pinocchio shell: cmd /C CALL {0} - env: - ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true' run: | - :: unset extra Boost envs - set Boost_ROOT= - set BOOST_ROOT_1_69_0= - set BOOST_ROOT_1_72_0= - call "%programfiles(x86)%\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" amd64 :: Create build directory @@ -54,8 +47,9 @@ jobs: :: Configure cmake ^ - -G "Visual Studio 16 2019" -T "v142" -DCMAKE_GENERATOR_PLATFORM=x64 ^ + -G "Ninja" ^ -DCMAKE_INSTALL_PREFIX=%CONDA_PREFIX%\Library ^ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache ^ -DCMAKE_BUILD_TYPE=Release ^ -DPYTHON_SITELIB=%CONDA_PREFIX%\Lib\site-packages ^ -DPYTHON_EXECUTABLE=%CONDA_PREFIX%\python.exe ^ @@ -66,29 +60,15 @@ jobs: -DINSTALL_DOCUMENTATION=ON ^ .. - :: Build + :: Build and Install cmake --build . --config Release --target install + if errorlevel 1 exit 1 -# - name: Start SSH session -# uses: luchihoratiu/debug-via-ssh@main -# with: -# NGROK_AUTH_TOKEN: ${{ secrets.NGROK_AUTH_TOKEN }} -# SSH_PASS: ${{ secrets.SSH_PASS }} -# - - name: Testing - shell: cmd /C CALL {0} - env: - ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true' - run: | :: Testing - pushd build - set PATH=%PATH%;%CONDA_PREFIX%\Lib\site-packages\pinocchio - set PATH=%PATH%;%CONDA_PREFIX%\Library\Lib - set PATH=%PATH%;%CONDA_PREFIX%\Library\bin - .\unittest\Release\test-cpp-urdf.exe - .\examples\Release\example-cpp-geometry-models.exe - ctest --output-on-failure -C Release -V --repeat until-pass:1 + ctest --output-on-failure -C Release -V + if errorlevel 1 exit 1 -# :: Test Python import -# cd .. -# python -c "import pinocchio" + :: Test Python import + cd .. + python -c "import pinocchio" + if errorlevel 1 exit 1 From 63f1335139b48624543e004310367fb58025918c Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Tue, 9 Jan 2024 15:43:09 +0100 Subject: [PATCH 05/22] [ci] Deactivate ctest verbose --- .github/workflows/windows-conda-clang.yml | 17 ++++++++++++++++- .github/workflows/windows-conda-v142.yml | 17 ++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows-conda-clang.yml b/.github/workflows/windows-conda-clang.yml index f9044a4890..dc1b46d739 100644 --- a/.github/workflows/windows-conda-clang.yml +++ b/.github/workflows/windows-conda-clang.yml @@ -68,10 +68,25 @@ jobs: if errorlevel 1 exit 1 :: Testing - ctest --output-on-failure -C Release -V + ctest --output-on-failure -C Release if errorlevel 1 exit 1 :: Test Python import cd .. python -c "import pinocchio" if errorlevel 1 exit 1 + + check: + if: always() + name: check-windows-conda-clang + + needs: + - build + + runs-on: Ubuntu-latest + + steps: + - name: Decide whether the needed jobs succeeded or failed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/windows-conda-v142.yml b/.github/workflows/windows-conda-v142.yml index 2f021fd16f..61e6546fd3 100644 --- a/.github/workflows/windows-conda-v142.yml +++ b/.github/workflows/windows-conda-v142.yml @@ -65,10 +65,25 @@ jobs: if errorlevel 1 exit 1 :: Testing - ctest --output-on-failure -C Release -V + ctest --output-on-failure -C Release if errorlevel 1 exit 1 :: Test Python import cd .. python -c "import pinocchio" if errorlevel 1 exit 1 + + check: + if: always() + name: check-windows-conda-v142 + + needs: + - build + + runs-on: Ubuntu-latest + + steps: + - name: Decide whether the needed jobs succeeded or failed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} From 0f419dbabf83ee28d04882464c7482d025600ce4 Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Tue, 9 Jan 2024 15:47:48 +0100 Subject: [PATCH 06/22] ci: Build Python interface in Windows build --- .github/workflows/windows-conda-clang.yml | 2 +- .github/workflows/windows-conda-v142.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows-conda-clang.yml b/.github/workflows/windows-conda-clang.yml index dc1b46d739..669c68cd70 100644 --- a/.github/workflows/windows-conda-clang.yml +++ b/.github/workflows/windows-conda-clang.yml @@ -57,7 +57,7 @@ jobs: -DPYTHON_SITELIB=%CONDA_PREFIX%\Lib\site-packages ^ -DPYTHON_EXECUTABLE=%CONDA_PREFIX%\python.exe ^ -DBUILD_WITH_URDF_SUPPORT=ON ^ - -DBUILD_PYTHON_INTERFACE=OFF ^ + -DBUILD_PYTHON_INTERFACE=ON ^ -DBUILD_WITH_COLLISION_SUPPORT=ON ^ -DBUILD_TESTING=ON ^ -DINSTALL_DOCUMENTATION=ON ^ diff --git a/.github/workflows/windows-conda-v142.yml b/.github/workflows/windows-conda-v142.yml index 61e6546fd3..34e74a2182 100644 --- a/.github/workflows/windows-conda-v142.yml +++ b/.github/workflows/windows-conda-v142.yml @@ -54,7 +54,7 @@ jobs: -DPYTHON_SITELIB=%CONDA_PREFIX%\Lib\site-packages ^ -DPYTHON_EXECUTABLE=%CONDA_PREFIX%\python.exe ^ -DBUILD_WITH_URDF_SUPPORT=ON ^ - -DBUILD_PYTHON_INTERFACE=OFF ^ + -DBUILD_PYTHON_INTERFACE=ON ^ -DBUILD_WITH_COLLISION_SUPPORT=ON ^ -DBUILD_TESTING=ON ^ -DINSTALL_DOCUMENTATION=ON ^ From 28db6ad1b6a84d80914564e09b9e6424bf966fbe Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Wed, 10 Jan 2024 14:21:55 +0100 Subject: [PATCH 07/22] cmake: Fix binding output path on Windows --- bindings/python/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index 05dd837537..d8da7c9f07 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -78,6 +78,8 @@ IF(BUILD_PYTHON_INTERFACE) SUFFIX ${PYTHON_EXT_SUFFIX} OUTPUT_NAME "${PYWRAP}" LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bindings/python/${PROJECT_NAME}" + # On Windows, shared library are treat as binary + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bindings/python/${PROJECT_NAME}" ) IF(UNIX) From f6c8e38b5649e2ba05a0489e901de0731cf4bbb6 Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Wed, 10 Jan 2024 14:53:36 +0100 Subject: [PATCH 08/22] python: Add code to manage dll in non standard directory --- bindings/python/pinocchio/__init__.py | 42 ++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/bindings/python/pinocchio/__init__.py b/bindings/python/pinocchio/__init__.py index 1354d404a4..80a74b49aa 100644 --- a/bindings/python/pinocchio/__init__.py +++ b/bindings/python/pinocchio/__init__.py @@ -3,7 +3,47 @@ # import numpy -from .pinocchio_pywrap import * + +# On Windows, if pinocchio.dll is not in the same directory than +# the .pyd, it will not be loaded. +# We first try to load pinocchio, then, if it fail and we are on Windows: +# 1. We add all paths inside PINOCCHIO_WINDOWS_DLL_PATH to DllDirectory +# 2. If PINOCCHIO_WINDOWS_DLL_PATH we add the relative path from the +# package directory to the bin directory to DllDirectory +# This solution is inspired from: +# - https://github.com/PixarAnimationStudios/OpenUSD/pull/1511/files +# - https://stackoverflow.com/questions/65334494/python-c-extension-packaging-dll-along-with-pyd +# More resources on https://github.com/diffpy/pyobjcryst/issues/33 +try: + from .pinocchio_pywrap import * +except ImportError: + import os + import sys + import platform + if platform.system() == "Windows": + __pinocchio_paths = os.getenv('PINOCCHIO_WINDOWS_DLL_PATH') + if __pinocchio_paths is None: + # Standard site-packages to bin path + RELATIVE_DLL_PATH = "..\\..\\..\\bin" + __dll_paths = [os.path.join(os.path.dirname(__file__), RELATIVE_DLL_PATH)] + else: + __dll_paths = __pinocchio_paths.split(os.pathsep) + if sys.version_info >= (3, 8): + for p in __dll_paths: + # add_dll_directory can fail on relative path and non + # existing path. + # Since we don't know all the fail criterion we just ignore + # thrown exception + try: + os.add_dll_directory(p) + except OSError: + pass + else: + for p in __dll_paths: + os.environ["PATH"] += os.pathsep + p + from .pinocchio_pywrap import * + else: + raise from .pinocchio_pywrap import __version__, __raw_version__ From d23d2335d705602838c8ead4dfc0746473f235b1 Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Wed, 10 Jan 2024 15:49:01 +0100 Subject: [PATCH 09/22] ci: Define PINOCCHIO_WINDOWS_DLL_PATH when running tests --- CMakeLists.txt | 7 +++++++ examples/CMakeLists.txt | 4 +++- unittest/python/CMakeLists.txt | 10 ++++++++-- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0f55803364..b1c924259d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -285,6 +285,13 @@ FUNCTION(PREPEND_PATH_FROM_LIST list_name path_name dest_list) SET(${dest_list} ${list_name_} PARENT_SCOPE) ENDFUNCTION(PREPEND_PATH_FROM_LIST) +# Define PINOCCHIO_WINDOWS_DLL_PATH environment variable on Windows +FUNCTION(ADD_WINDOWS_DLL_PATH_TO_TEST TEST_NAME WINDOWS_DLL_PATH) + GET_TEST_PROPERTY(${TEST_NAME} ENVIRONMENT ENV_VARIABLES) + LIST(APPEND ENV_VARIABLES "PINOCCHIO_WINDOWS_DLL_PATH=${WINDOWS_DLL_PATH}") + SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES ENVIRONMENT "${ENV_VARIABLES}") +ENDFUNCTION() + # Enforce the preprocessed version of boost::list and boost::vector # This information is redundant with the content of include/pinocchio/container/boost-container-limits.hpp # but it avoids any compilation issue. diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 2f498e48f3..0ea6156a7b 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -106,7 +106,9 @@ IF(BUILD_PYTHON_INTERFACE) ENDIF(BUILD_WITH_OPENMP_SUPPORT) FOREACH(EXAMPLE ${${PROJECT_NAME}_PYTHON_EXAMPLES}) - ADD_PYTHON_UNIT_TEST("example-py-${EXAMPLE}" "examples/${EXAMPLE}.py" "bindings/python") + SET(EXAMPLE_NAME "example-py-${EXAMPLE}") + ADD_PYTHON_UNIT_TEST("${EXAMPLE_NAME}" "examples/${EXAMPLE}.py" "bindings/python") + ADD_WINDOWS_DLL_PATH_TO_TEST(${EXAMPLE_NAME} "${CMAKE_CURRENT_BINARY_DIR}/../src") ENDFOREACH(EXAMPLE ${${PROJECT_NAME}_PYTHON_EXAMPLES}) ENDIF(BUILD_PYTHON_INTERFACE) diff --git a/unittest/python/CMakeLists.txt b/unittest/python/CMakeLists.txt index 2fb9fba4cf..2bd91c3637 100644 --- a/unittest/python/CMakeLists.txt +++ b/unittest/python/CMakeLists.txt @@ -77,17 +77,23 @@ SET(${PROJECT_NAME}_PYTHON_MEMORYCHECK_TESTS bindings_build_geom_from_urdf_memorycheck ) +SET(PINOCCHIO_DLL_PATH ${CMAKE_CURRENT_BINARY_DIR}/../../src) + FOREACH(TEST ${${PROJECT_NAME}_PYTHON_TESTS}) - ADD_PYTHON_UNIT_TEST("test-py-${TEST}" "unittest/python/${TEST}.py" "bindings/python") + SET(TEST_NAME "test-py-${TEST}") + ADD_PYTHON_UNIT_TEST(${TEST_NAME} "unittest/python/${TEST}.py" "bindings/python") + ADD_WINDOWS_DLL_PATH_TO_TEST(${TEST_NAME} ${PINOCCHIO_DLL_PATH}) ENDFOREACH(TEST ${${PROJECT_NAME}_PYTHON_TESTS}) MAKE_DIRECTORY("${CMAKE_CURRENT_BINARY_DIR}/serialization-data") IF(MEMORYCHECK_COMMAND AND MEMORYCHECK_COMMAND MATCHES ".*valgrind$") FOREACH(TEST ${${PROJECT_NAME}_PYTHON_MEMORYCHECK_TESTS}) - ADD_PYTHON_MEMORYCHECK_UNIT_TEST("test-py-memory-${TEST}" + SET(TEST_NAME "test-py-memory-${TEST}") + ADD_PYTHON_MEMORYCHECK_UNIT_TEST(${TEST_NAME} "unittest/python/${TEST}.py" "bindings/python") + ADD_WINDOWS_DLL_PATH_TO_TEST(${TEST_NAME} ${PINOCCHIO_DLL_PATH}) ENDFOREACH() ELSE() MESSAGE(STATUS "Valgrind not found, memory checks are disabled") From 8166168b07298df86a761ca867f879cace4dd551 Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Wed, 10 Jan 2024 16:38:56 +0100 Subject: [PATCH 10/22] cmake: Use a generator expression to bind dll directory --- CMakeLists.txt | 6 ++++-- examples/CMakeLists.txt | 2 +- unittest/python/CMakeLists.txt | 6 ++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b1c924259d..5ef06eb8d1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -286,10 +286,12 @@ FUNCTION(PREPEND_PATH_FROM_LIST list_name path_name dest_list) ENDFUNCTION(PREPEND_PATH_FROM_LIST) # Define PINOCCHIO_WINDOWS_DLL_PATH environment variable on Windows -FUNCTION(ADD_WINDOWS_DLL_PATH_TO_TEST TEST_NAME WINDOWS_DLL_PATH) +FUNCTION(ADD_WINDOWS_DLL_PATH_TO_TEST TEST_NAME) + IF(WIN32) GET_TEST_PROPERTY(${TEST_NAME} ENVIRONMENT ENV_VARIABLES) - LIST(APPEND ENV_VARIABLES "PINOCCHIO_WINDOWS_DLL_PATH=${WINDOWS_DLL_PATH}") + LIST(APPEND ENV_VARIABLES "PINOCCHIO_WINDOWS_DLL_PATH=$") SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES ENVIRONMENT "${ENV_VARIABLES}") + ENDIF() ENDFUNCTION() # Enforce the preprocessed version of boost::list and boost::vector diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 0ea6156a7b..cf8da50bea 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -108,7 +108,7 @@ IF(BUILD_PYTHON_INTERFACE) FOREACH(EXAMPLE ${${PROJECT_NAME}_PYTHON_EXAMPLES}) SET(EXAMPLE_NAME "example-py-${EXAMPLE}") ADD_PYTHON_UNIT_TEST("${EXAMPLE_NAME}" "examples/${EXAMPLE}.py" "bindings/python") - ADD_WINDOWS_DLL_PATH_TO_TEST(${EXAMPLE_NAME} "${CMAKE_CURRENT_BINARY_DIR}/../src") + ADD_WINDOWS_DLL_PATH_TO_TEST(${EXAMPLE_NAME}) ENDFOREACH(EXAMPLE ${${PROJECT_NAME}_PYTHON_EXAMPLES}) ENDIF(BUILD_PYTHON_INTERFACE) diff --git a/unittest/python/CMakeLists.txt b/unittest/python/CMakeLists.txt index 2bd91c3637..5ac0adb7d1 100644 --- a/unittest/python/CMakeLists.txt +++ b/unittest/python/CMakeLists.txt @@ -77,12 +77,10 @@ SET(${PROJECT_NAME}_PYTHON_MEMORYCHECK_TESTS bindings_build_geom_from_urdf_memorycheck ) -SET(PINOCCHIO_DLL_PATH ${CMAKE_CURRENT_BINARY_DIR}/../../src) - FOREACH(TEST ${${PROJECT_NAME}_PYTHON_TESTS}) SET(TEST_NAME "test-py-${TEST}") ADD_PYTHON_UNIT_TEST(${TEST_NAME} "unittest/python/${TEST}.py" "bindings/python") - ADD_WINDOWS_DLL_PATH_TO_TEST(${TEST_NAME} ${PINOCCHIO_DLL_PATH}) + ADD_WINDOWS_DLL_PATH_TO_TEST(${TEST_NAME}) ENDFOREACH(TEST ${${PROJECT_NAME}_PYTHON_TESTS}) MAKE_DIRECTORY("${CMAKE_CURRENT_BINARY_DIR}/serialization-data") @@ -93,7 +91,7 @@ IF(MEMORYCHECK_COMMAND AND MEMORYCHECK_COMMAND MATCHES ".*valgrind$") ADD_PYTHON_MEMORYCHECK_UNIT_TEST(${TEST_NAME} "unittest/python/${TEST}.py" "bindings/python") - ADD_WINDOWS_DLL_PATH_TO_TEST(${TEST_NAME} ${PINOCCHIO_DLL_PATH}) + ADD_WINDOWS_DLL_PATH_TO_TEST(${TEST_NAME}) ENDFOREACH() ELSE() MESSAGE(STATUS "Valgrind not found, memory checks are disabled") From bf2626b2dcaf32cf2382a870bb12ffe8ed15c675 Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Wed, 10 Jan 2024 18:20:44 +0100 Subject: [PATCH 11/22] cmake: Modify the PATH to allow python_parser test to run --- unittest/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/unittest/CMakeLists.txt b/unittest/CMakeLists.txt index 795bb33c84..3a6eff92b5 100644 --- a/unittest/CMakeLists.txt +++ b/unittest/CMakeLists.txt @@ -107,10 +107,12 @@ IF(BUILD_WITH_LIBPYTHON) TARGET_LINK_LIBRARIES(test-cpp-python_parser PUBLIC ${PYWRAP}) TARGET_LINK_LIBRARIES(test-cpp-python_parser PUBLIC ${PYTHON_LIBRARIES}) - SET(ENV_VARIABLES "PYTHONPATH=${CMAKE_BINARY_DIR}/bindings/python:$ENV{PYTHONPATH}") + SET(ENV_VARIABLES "PYTHONPATH=${CMAKE_BINARY_DIR}/bindings/python") IF(WIN32) GET_FILENAME_COMPONENT(PYTHONHOME ${PYTHON_EXECUTABLE} PATH) LIST(APPEND ENV_VARIABLES "PYTHONHOME=${PYTHONHOME}") + STRING(REPLACE ";" "\\\;" PATH "$ENV{PATH}") + LIST(APPEND ENV_VARIABLES "PATH=${PATH}\;${CMAKE_BINARY_DIR}/src\;${CMAKE_BINARY_DIR}/bindings/python/pinocchio") UNSET(PYTHONHOME) ENDIF(WIN32) IF(APPLE) From b1041e74d72a6f4bd9da0d530a60448e3c3ac076 Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Thu, 11 Jan 2024 10:09:33 +0100 Subject: [PATCH 12/22] cmake: Fix ; escaping --- unittest/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittest/CMakeLists.txt b/unittest/CMakeLists.txt index 3a6eff92b5..4d6b137475 100644 --- a/unittest/CMakeLists.txt +++ b/unittest/CMakeLists.txt @@ -112,7 +112,7 @@ IF(BUILD_WITH_LIBPYTHON) GET_FILENAME_COMPONENT(PYTHONHOME ${PYTHON_EXECUTABLE} PATH) LIST(APPEND ENV_VARIABLES "PYTHONHOME=${PYTHONHOME}") STRING(REPLACE ";" "\\\;" PATH "$ENV{PATH}") - LIST(APPEND ENV_VARIABLES "PATH=${PATH}\;${CMAKE_BINARY_DIR}/src\;${CMAKE_BINARY_DIR}/bindings/python/pinocchio") + LIST(APPEND ENV_VARIABLES "PATH=${PATH}\\\;${CMAKE_BINARY_DIR}/src\\\;${CMAKE_BINARY_DIR}/bindings/python/pinocchio") UNSET(PYTHONHOME) ENDIF(WIN32) IF(APPLE) From 4ecbed8e9c3d7ec00700e73ac622e569b62b56a3 Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Thu, 11 Jan 2024 10:23:18 +0100 Subject: [PATCH 13/22] cmake: Setup back environment PYTHONPATH when running python parser test --- unittest/CMakeLists.txt | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/unittest/CMakeLists.txt b/unittest/CMakeLists.txt index 4d6b137475..c9df03cddb 100644 --- a/unittest/CMakeLists.txt +++ b/unittest/CMakeLists.txt @@ -107,14 +107,20 @@ IF(BUILD_WITH_LIBPYTHON) TARGET_LINK_LIBRARIES(test-cpp-python_parser PUBLIC ${PYWRAP}) TARGET_LINK_LIBRARIES(test-cpp-python_parser PUBLIC ${PYTHON_LIBRARIES}) - SET(ENV_VARIABLES "PYTHONPATH=${CMAKE_BINARY_DIR}/bindings/python") + SET(ENV_VARIABLES) IF(WIN32) GET_FILENAME_COMPONENT(PYTHONHOME ${PYTHON_EXECUTABLE} PATH) LIST(APPEND ENV_VARIABLES "PYTHONHOME=${PYTHONHOME}") - STRING(REPLACE ";" "\\\;" PATH "$ENV{PATH}") - LIST(APPEND ENV_VARIABLES "PATH=${PATH}\\\;${CMAKE_BINARY_DIR}/src\\\;${CMAKE_BINARY_DIR}/bindings/python/pinocchio") UNSET(PYTHONHOME) - ENDIF(WIN32) + + STRING(REPLACE ";" "\\\;" _PATH "$ENV{PATH}") + LIST(APPEND ENV_VARIABLES "PATH=${_PATH}\\\;${CMAKE_BINARY_DIR}/src\\\;${CMAKE_BINARY_DIR}/bindings/python/pinocchio") + + STRING(REPLACE ";" "\\\;" _PYTHONPATH "$ENV{PYTHONPATH}") + LIST(APPEND ENV_VARIABLES "PYTHONPATH=${_PYTHONPATH}\\\;${CMAKE_BINARY_DIR}/bindings/python") + ELSE() + LIST(APPEND ENV_VARIABLES "PYTHONPATH=$ENV{PYTHONPATH}:${CMAKE_BINARY_DIR}/bindings/python") + ENDIF() IF(APPLE) LIST(APPEND ENV_VARIABLES "LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH}") LIST(APPEND ENV_VARIABLES "DYLD_LIBRARY_PATH=$ENV{DYLD_LIBRARY_PATH}") From 395e783868131dd5b566cc7126fc273866719db4 Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Thu, 11 Jan 2024 11:06:58 +0100 Subject: [PATCH 14/22] ci: Merge Windows build --- .github/workflows/windows-conda-v142.yml | 89 ------------------- ...dows-conda-clang.yml => windows-conda.yml} | 29 +++--- 2 files changed, 18 insertions(+), 100 deletions(-) delete mode 100644 .github/workflows/windows-conda-v142.yml rename .github/workflows/{windows-conda-clang.yml => windows-conda.yml} (75%) diff --git a/.github/workflows/windows-conda-v142.yml b/.github/workflows/windows-conda-v142.yml deleted file mode 100644 index 34e74a2182..0000000000 --- a/.github/workflows/windows-conda-v142.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: CI - Windows (v142) via Conda -on: - pull_request: - push: - -jobs: - build: - runs-on: ${{ matrix.os }} - env: - CCACHE_BASEDIR: "${GITHUB_WORKSPACE}" - CCACHE_DIR: "${GITHUB_WORKSPACE}/.ccache" - CCACHE_COMPRESS: true - CCACHE_COMPRESSLEVEL: 6 - - strategy: - fail-fast: false - matrix: - os: ["windows-2019"] - - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - - uses: actions/cache@v3 - with: - path: .ccache - key: ccache-windows-conda-vs-${{ matrix.os }}-${{ github.sha }} - restore-keys: ccache-windows-conda-vs-${{ matrix.os }}- - - - uses: conda-incubator/setup-miniconda@v3 - with: - activate-environment: pinocchio - auto-update-conda: true - environment-file: .github/workflows/conda/environment_windows.yml - python-version: "3.10" - auto-activate-base: false - - - name: Build Pinocchio - shell: cmd /C CALL {0} - run: | - call "%programfiles(x86)%\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" amd64 - - :: Create build directory - mkdir build - pushd build - - :: Configure - cmake ^ - -G "Ninja" ^ - -DCMAKE_INSTALL_PREFIX=%CONDA_PREFIX%\Library ^ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache ^ - -DCMAKE_BUILD_TYPE=Release ^ - -DPYTHON_SITELIB=%CONDA_PREFIX%\Lib\site-packages ^ - -DPYTHON_EXECUTABLE=%CONDA_PREFIX%\python.exe ^ - -DBUILD_WITH_URDF_SUPPORT=ON ^ - -DBUILD_PYTHON_INTERFACE=ON ^ - -DBUILD_WITH_COLLISION_SUPPORT=ON ^ - -DBUILD_TESTING=ON ^ - -DINSTALL_DOCUMENTATION=ON ^ - .. - - :: Build and Install - cmake --build . --config Release --target install - if errorlevel 1 exit 1 - - :: Testing - ctest --output-on-failure -C Release - if errorlevel 1 exit 1 - - :: Test Python import - cd .. - python -c "import pinocchio" - if errorlevel 1 exit 1 - - check: - if: always() - name: check-windows-conda-v142 - - needs: - - build - - runs-on: Ubuntu-latest - - steps: - - name: Decide whether the needed jobs succeeded or failed - uses: re-actors/alls-green@release/v1 - with: - jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/windows-conda-clang.yml b/.github/workflows/windows-conda.yml similarity index 75% rename from .github/workflows/windows-conda-clang.yml rename to .github/workflows/windows-conda.yml index 669c68cd70..d69a799cc6 100644 --- a/.github/workflows/windows-conda-clang.yml +++ b/.github/workflows/windows-conda.yml @@ -1,10 +1,11 @@ -name: CI - Windows (CLANG) via Conda +name: CI - Windows via Conda on: pull_request: push: jobs: build: + name: ${{ matrix.os }} - ${{ matrix.compiler }} with Conda runs-on: ${{ matrix.os }} env: CCACHE_BASEDIR: "${GITHUB_WORKSPACE}" @@ -16,6 +17,7 @@ jobs: fail-fast: false matrix: os: ["windows-2019"] + compiler: ["cl", clang-cl"] steps: - uses: actions/checkout@v4 @@ -25,8 +27,8 @@ jobs: - uses: actions/cache@v3 with: path: .ccache - key: ccache-windows-conda-clang-${{ matrix.os }}-${{ github.sha }} - restore-keys: ccache-windows-conda-clang-${{ matrix.os }}- + key: ccache-windows-conda-${{ matrix.compiler }}-${{ matrix.os }}-${{ github.sha }} + restore-keys: ccache-windows-conda-${{ matrix.compiler }}-${{ matrix.os }}- - uses: conda-incubator/setup-miniconda@v3 with: @@ -38,17 +40,19 @@ jobs: - name: Build Pinocchio shell: cmd /C CALL {0} + env: + CC: ${{ matrix.compiler }} + CXX: ${{ matrix.compiler }} run: | + call conda list + call "%programfiles(x86)%\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" amd64 - :: Tell Ninja to use clang-cl - set CC=clang-cl - set CXX=clang-cl :: Create build directory mkdir build pushd build - :: Configure + :: Configure (Ninja use CC and CXX to determine the compiler) cmake ^ -G "Ninja" ^ -DCMAKE_INSTALL_PREFIX=%CONDA_PREFIX%\Library ^ @@ -63,14 +67,17 @@ jobs: -DINSTALL_DOCUMENTATION=ON ^ .. - :: Build and Install - cmake --build . --config Release --target install + :: Build + cmake --build . if errorlevel 1 exit 1 :: Testing - ctest --output-on-failure -C Release + ctest --output-on-failure if errorlevel 1 exit 1 + :: Install + cmake --install . + :: Test Python import cd .. python -c "import pinocchio" @@ -78,7 +85,7 @@ jobs: check: if: always() - name: check-windows-conda-clang + name: check-windows-conda needs: - build From 3cad12d00a0a7f3fdfa23b7bf4bd7c2da54ea5bf Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Thu, 11 Jan 2024 11:08:10 +0100 Subject: [PATCH 15/22] ci: Add verbose tests output to debug ros ci --- .github/workflows/ros-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ros-ci.yml b/.github/workflows/ros-ci.yml index 6e57d1abf0..7db4796382 100644 --- a/.github/workflows/ros-ci.yml +++ b/.github/workflows/ros-ci.yml @@ -28,6 +28,7 @@ jobs: # Note, this does not affect projects that do not have pure CMake projects in their upstream_ws. AFTER_RUN_TARGET_TEST: 'ici_with_unset_variables source /root/target_ws/install/setup.bash && cd /root/target_ws/build/pinocchio && make test' IMMEDIATE_TEST_OUTPUT: 1 + VERBOSE_TESTS: true runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From 4d5573a77ab75ff80c82fbc150dea4274287ec2d Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Thu, 11 Jan 2024 11:29:08 +0100 Subject: [PATCH 16/22] ci: Fix a typo --- .github/workflows/windows-conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows-conda.yml b/.github/workflows/windows-conda.yml index d69a799cc6..9f08e8fa5e 100644 --- a/.github/workflows/windows-conda.yml +++ b/.github/workflows/windows-conda.yml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: os: ["windows-2019"] - compiler: ["cl", clang-cl"] + compiler: ["cl", "clang-cl"] steps: - uses: actions/checkout@v4 From 1894f3a420b5eb44c05b01bc3a721ca43d5588f9 Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Thu, 11 Jan 2024 14:37:36 +0100 Subject: [PATCH 17/22] ci: Add PATH to all unit tests and examples on windows --- examples/CMakeLists.txt | 7 +++++++ unittest/CMakeLists.txt | 15 ++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index cf8da50bea..e8f6141724 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -7,6 +7,13 @@ FUNCTION(ADD_PINOCCHIO_CPP_EXAMPLE EXAMPLE) SET(EXAMPLE_NAME "example-cpp-${EXAMPLE_NAME}") ADD_UNIT_TEST(${EXAMPLE_NAME} "${EXAMPLE}.cpp") TARGET_LINK_LIBRARIES(${EXAMPLE_NAME} PUBLIC ${PROJECT_NAME}) + + # There is no RPATH in Windows, then we must use the PATH to find the DLL + IF(WIN32) + STRING(REPLACE ";" "\\\;" _PATH "$ENV{PATH}") + SET(ENV_VARIABLES "PATH=${_PATH}\\\;${CMAKE_BINARY_DIR}/src\\\;${CMAKE_BINARY_DIR}/bindings/python/pinocchio") + SET_TESTS_PROPERTIES(${EXAMPLE_NAME} PROPERTIES ENVIRONMENT "${ENV_VARIABLES}") + ENDIF() ENDFUNCTION() SET(${PROJECT_NAME}_EXAMPLES diff --git a/unittest/CMakeLists.txt b/unittest/CMakeLists.txt index c9df03cddb..cdf30713bc 100644 --- a/unittest/CMakeLists.txt +++ b/unittest/CMakeLists.txt @@ -22,6 +22,14 @@ MACRO(ADD_PINOCCHIO_UNIT_TEST NAME) SET(TEST_NAME "test-cpp-${NAME}") ADD_UNIT_TEST(${TEST_NAME} ${NAME}.cpp) + + # There is no RPATH in Windows, then we must use the PATH to find the DLL + IF(WIN32) + STRING(REPLACE ";" "\\\;" _PATH "$ENV{PATH}") + SET(ENV_VARIABLES "PATH=${_PATH}\\\;${CMAKE_BINARY_DIR}/src\\\;${CMAKE_BINARY_DIR}/bindings/python/pinocchio") + SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES ENVIRONMENT "${ENV_VARIABLES}") + ENDIF() + SET_TARGET_PROPERTIES(${TEST_NAME} PROPERTIES LINKER_LANGUAGE CXX) TARGET_INCLUDE_DIRECTORIES(${TEST_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) @@ -107,15 +115,12 @@ IF(BUILD_WITH_LIBPYTHON) TARGET_LINK_LIBRARIES(test-cpp-python_parser PUBLIC ${PYWRAP}) TARGET_LINK_LIBRARIES(test-cpp-python_parser PUBLIC ${PYTHON_LIBRARIES}) - SET(ENV_VARIABLES) + GET_TEST_PROPERTY(test-cpp-python_parser ENVIRONMENT ENV_VARIABLES) IF(WIN32) GET_FILENAME_COMPONENT(PYTHONHOME ${PYTHON_EXECUTABLE} PATH) LIST(APPEND ENV_VARIABLES "PYTHONHOME=${PYTHONHOME}") UNSET(PYTHONHOME) - STRING(REPLACE ";" "\\\;" _PATH "$ENV{PATH}") - LIST(APPEND ENV_VARIABLES "PATH=${_PATH}\\\;${CMAKE_BINARY_DIR}/src\\\;${CMAKE_BINARY_DIR}/bindings/python/pinocchio") - STRING(REPLACE ";" "\\\;" _PYTHONPATH "$ENV{PYTHONPATH}") LIST(APPEND ENV_VARIABLES "PYTHONPATH=${_PYTHONPATH}\\\;${CMAKE_BINARY_DIR}/bindings/python") ELSE() @@ -125,7 +130,7 @@ IF(BUILD_WITH_LIBPYTHON) LIST(APPEND ENV_VARIABLES "LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH}") LIST(APPEND ENV_VARIABLES "DYLD_LIBRARY_PATH=$ENV{DYLD_LIBRARY_PATH}") ENDIF(APPLE) - SET_PROPERTY(TEST test-cpp-python_parser PROPERTY ENVIRONMENT ${ENV_VARIABLES}) + SET_TESTS_PROPERTIES(test-cpp-python_parser PROPERTIES ENVIRONMENT "${ENV_VARIABLES}") ADD_SUBDIRECTORY(python) ENDIF(BUILD_WITH_LIBPYTHON) From c8ad1a4f3e3fc808e2e90b7cd4ed9ece336962a5 Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Thu, 11 Jan 2024 15:56:37 +0100 Subject: [PATCH 18/22] ci: Try to debug ROS CI --- .github/workflows/ros-ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ros-ci.yml b/.github/workflows/ros-ci.yml index 7db4796382..6a7c50f2a8 100644 --- a/.github/workflows/ros-ci.yml +++ b/.github/workflows/ros-ci.yml @@ -26,9 +26,9 @@ jobs: # The work-around is thus to use catkin_tools for building and sourcing, and to manually specify execution of the test # target after completion of the regular test target. The output of this step does affect the output of the CI process. # Note, this does not affect projects that do not have pure CMake projects in their upstream_ws. - AFTER_RUN_TARGET_TEST: 'ici_with_unset_variables source /root/target_ws/install/setup.bash && cd /root/target_ws/build/pinocchio && make test' + AFTER_RUN_TARGET_TEST: 'ici_with_unset_variables source /root/target_ws/install/setup.bash && cd /root/target_ws/build/pinocchio && ctest --output-on-failure' IMMEDIATE_TEST_OUTPUT: 1 - VERBOSE_TESTS: true + VERBOSE_TESTS: 1 runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -38,7 +38,8 @@ jobs: - uses: actions/cache@v3 with: path: ${{ env.CCACHE_DIR }} - key: ccache-${{ matrix.env.ROS_DISTRO }}-${{ matrix.env.ROS_REPO }} + key: ccache-${{ matrix.env.ROS_DISTRO }}-${{ matrix.env.ROS_REPO }}-${{ github.sha }} + restore-key: ccache-${{ matrix.env.ROS_DISTRO }}-${{ matrix.env.ROS_REPO }}- # Run industrial_ci - uses: 'ros-industrial/industrial_ci@3ed9846c96ed1e0bb36193e8e250632eaac980d0' env: ${{ matrix.env }} From 57c6ef1ba1d468609adc85ac85eb65414dcbac5c Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Thu, 11 Jan 2024 16:40:47 +0100 Subject: [PATCH 19/22] ci: Revert ROS verbose output and fix cache setup --- .github/workflows/ros-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ros-ci.yml b/.github/workflows/ros-ci.yml index 6a7c50f2a8..e5d9e89b87 100644 --- a/.github/workflows/ros-ci.yml +++ b/.github/workflows/ros-ci.yml @@ -26,9 +26,8 @@ jobs: # The work-around is thus to use catkin_tools for building and sourcing, and to manually specify execution of the test # target after completion of the regular test target. The output of this step does affect the output of the CI process. # Note, this does not affect projects that do not have pure CMake projects in their upstream_ws. - AFTER_RUN_TARGET_TEST: 'ici_with_unset_variables source /root/target_ws/install/setup.bash && cd /root/target_ws/build/pinocchio && ctest --output-on-failure' + AFTER_RUN_TARGET_TEST: 'ici_with_unset_variables source /root/target_ws/install/setup.bash && cd /root/target_ws/build/pinocchio && make test' IMMEDIATE_TEST_OUTPUT: 1 - VERBOSE_TESTS: 1 runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -39,7 +38,7 @@ jobs: with: path: ${{ env.CCACHE_DIR }} key: ccache-${{ matrix.env.ROS_DISTRO }}-${{ matrix.env.ROS_REPO }}-${{ github.sha }} - restore-key: ccache-${{ matrix.env.ROS_DISTRO }}-${{ matrix.env.ROS_REPO }}- + restore-keys: ccache-${{ matrix.env.ROS_DISTRO }}-${{ matrix.env.ROS_REPO }}- # Run industrial_ci - uses: 'ros-industrial/industrial_ci@3ed9846c96ed1e0bb36193e8e250632eaac980d0' env: ${{ matrix.env }} From 87b930ca0110c36464fb56abe6f5350b0917735f Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Fri, 12 Jan 2024 14:43:25 +0100 Subject: [PATCH 20/22] python: Restore DllDirectory state after importing Pinocchio --- bindings/python/pinocchio/__init__.py | 46 ++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/bindings/python/pinocchio/__init__.py b/bindings/python/pinocchio/__init__.py index 80a74b49aa..b9d87ac03a 100644 --- a/bindings/python/pinocchio/__init__.py +++ b/bindings/python/pinocchio/__init__.py @@ -16,10 +16,12 @@ # More resources on https://github.com/diffpy/pyobjcryst/issues/33 try: from .pinocchio_pywrap import * + from .pinocchio_pywrap import __version__, __raw_version__ except ImportError: import os import sys import platform + import contextlib if platform.system() == "Windows": __pinocchio_paths = os.getenv('PINOCCHIO_WINDOWS_DLL_PATH') if __pinocchio_paths is None: @@ -28,25 +30,53 @@ __dll_paths = [os.path.join(os.path.dirname(__file__), RELATIVE_DLL_PATH)] else: __dll_paths = __pinocchio_paths.split(os.pathsep) - if sys.version_info >= (3, 8): - for p in __dll_paths: + + # Restore PATH state after importing Pinocchio + class __PathManager(contextlib.AbstractContextManager): + def add_dll_directory(self, dll_dir: str): + os.environ["PATH"] += os.pathsep + dll_dir + + def __enter__(self): + self.old_path = os.environ["PATH"] + return self + + def __exit__(self): + os.environ["PATH"] = self.old_path + + # Restore DllDirectory state after importing Pinocchio + class __DllDirectoryManager(contextlib.AbstractContextManager): + def add_dll_directory(self, dll_dir: str): # add_dll_directory can fail on relative path and non # existing path. # Since we don't know all the fail criterion we just ignore # thrown exception try: - os.add_dll_directory(p) + self.dll_dirs.append(os.add_dll_directory(dll_dir)) except OSError: pass - else: + + def __enter__(self): + self.dll_dirs = [] + return self + + def __exit__(self, *exc_details): + for d in self.dll_dirs: + d.close() + + def __build_directory_manager(): + if sys.version_info >= (3, 8): + return __DllDirectoryManager() + else: + return __PathManager() + + with __build_directory_manager() as dll_dir_manager: for p in __dll_paths: - os.environ["PATH"] += os.pathsep + p - from .pinocchio_pywrap import * + dll_dir_manager.add_dll_directory(p) + from .pinocchio_pywrap import * + from .pinocchio_pywrap import __version__, __raw_version__ else: raise -from .pinocchio_pywrap import __version__, __raw_version__ - from . import utils from .explog import exp, log From 357e215aa607875eaf9bfea8a25f93299cc6f8ea Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Fri, 12 Jan 2024 14:53:55 +0100 Subject: [PATCH 21/22] python: clean __init__.py --- bindings/python/pinocchio/__init__.py | 53 +----------------- .../python/pinocchio/windows_dll_manager.py | 56 +++++++++++++++++++ 2 files changed, 59 insertions(+), 50 deletions(-) create mode 100644 bindings/python/pinocchio/windows_dll_manager.py diff --git a/bindings/python/pinocchio/__init__.py b/bindings/python/pinocchio/__init__.py index b9d87ac03a..4831c8de4e 100644 --- a/bindings/python/pinocchio/__init__.py +++ b/bindings/python/pinocchio/__init__.py @@ -18,59 +18,12 @@ from .pinocchio_pywrap import * from .pinocchio_pywrap import __version__, __raw_version__ except ImportError: - import os - import sys import platform - import contextlib if platform.system() == "Windows": - __pinocchio_paths = os.getenv('PINOCCHIO_WINDOWS_DLL_PATH') - if __pinocchio_paths is None: - # Standard site-packages to bin path - RELATIVE_DLL_PATH = "..\\..\\..\\bin" - __dll_paths = [os.path.join(os.path.dirname(__file__), RELATIVE_DLL_PATH)] - else: - __dll_paths = __pinocchio_paths.split(os.pathsep) + from .windows_dll_manager import get_dll_paths, build_directory_manager - # Restore PATH state after importing Pinocchio - class __PathManager(contextlib.AbstractContextManager): - def add_dll_directory(self, dll_dir: str): - os.environ["PATH"] += os.pathsep + dll_dir - - def __enter__(self): - self.old_path = os.environ["PATH"] - return self - - def __exit__(self): - os.environ["PATH"] = self.old_path - - # Restore DllDirectory state after importing Pinocchio - class __DllDirectoryManager(contextlib.AbstractContextManager): - def add_dll_directory(self, dll_dir: str): - # add_dll_directory can fail on relative path and non - # existing path. - # Since we don't know all the fail criterion we just ignore - # thrown exception - try: - self.dll_dirs.append(os.add_dll_directory(dll_dir)) - except OSError: - pass - - def __enter__(self): - self.dll_dirs = [] - return self - - def __exit__(self, *exc_details): - for d in self.dll_dirs: - d.close() - - def __build_directory_manager(): - if sys.version_info >= (3, 8): - return __DllDirectoryManager() - else: - return __PathManager() - - with __build_directory_manager() as dll_dir_manager: - for p in __dll_paths: + with build_directory_manager() as dll_dir_manager: + for p in get_dll_paths(): dll_dir_manager.add_dll_directory(p) from .pinocchio_pywrap import * from .pinocchio_pywrap import __version__, __raw_version__ diff --git a/bindings/python/pinocchio/windows_dll_manager.py b/bindings/python/pinocchio/windows_dll_manager.py new file mode 100644 index 0000000000..e718970814 --- /dev/null +++ b/bindings/python/pinocchio/windows_dll_manager.py @@ -0,0 +1,56 @@ +import os +import sys +import contextlib + + +def get_dll_paths(): + pinocchio_paths = os.getenv('PINOCCHIO_WINDOWS_DLL_PATH') + if pinocchio_paths is None: + # Standard site-packages to bin path + RELATIVE_DLL_PATH = "..\\..\\..\\bin" + return [os.path.join(os.path.dirname(__file__), RELATIVE_DLL_PATH)] + else: + return pinocchio_paths.split(os.pathsep) + + +class PathManager(contextlib.AbstractContextManager): + """Restore PATH state after importing Python module""" + + def add_dll_directory(self, dll_dir: str): + os.environ["PATH"] += os.pathsep + dll_dir + + def __enter__(self): + self.old_path = os.environ["PATH"] + return self + + def __exit__(self, *exc_details): + os.environ["PATH"] = self.old_path + + +class DllDirectoryManager(contextlib.AbstractContextManager): + """Restore DllDirectory state after importing Python module""" + + def add_dll_directory(self, dll_dir: str): + # add_dll_directory can fail on relative path and non + # existing path. + # Since we don't know all the fail criterion we just ignore + # thrown exception + try: + self.dll_dirs.append(os.add_dll_directory(dll_dir)) + except OSError: + pass + + def __enter__(self): + self.dll_dirs = [] + return self + + def __exit__(self, *exc_details): + for d in self.dll_dirs: + d.close() + + +def build_directory_manager(): + if sys.version_info >= (3, 8): + return DllDirectoryManager() + else: + return PathManager() From 8a1119e37004215fa0366020d9127da16ae9cdc8 Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Fri, 12 Jan 2024 14:58:09 +0100 Subject: [PATCH 22/22] python: clean __init__.py --- bindings/python/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index d8da7c9f07..72c88be6dd 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -109,6 +109,7 @@ IF(BUILD_PYTHON_INTERFACE) romeo_wrapper.py explog.py shortcuts.py + windows_dll_manager.py ) FOREACH(python ${PYTHON_FILES})