From 1451ff8180598cdbfc1109bf61e9500f2dcb2f57 Mon Sep 17 00:00:00 2001 From: Cyrus Harrison Date: Fri, 2 Feb 2024 09:36:35 -0800 Subject: [PATCH] migrate from distuils to pip for hybrid module creation (#1206) * migrate from distuils to pip for hybrid module creation * pip ver alignment * add pip as explicit dep when python is on * appveryor use py 3.8 insead of 2.7 * wire up python module cmake target deps * appveyor python updates --- appveyor.yml | 7 +- .../packages/conduit/package.py | 1 + setup.py | 2 +- src/CMakeLists.txt | 10 +++ src/cmake/thirdparty/SetupPython.cmake | 72 ++++++++++--------- .../cpp_fort_and_py/SetupPython.cmake | 69 +++++++++--------- src/libs/blueprint/python/CMakeLists.txt | 20 ++++++ src/libs/conduit/python/CMakeLists.txt | 3 + src/libs/conduit/python/setup.py | 25 +++---- src/libs/relay/python/CMakeLists.txt | 17 +++++ 10 files changed, 142 insertions(+), 84 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 9769cb35b..04b05b33c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -29,6 +29,8 @@ init: - "ECHO %PYTHON_VERSION% %MINICONDA%" install: + # setup python 3.8 + - set PATH=C:\Python38\bin;%PATH% # setup hdf5 #- set PATH=%PATH%;%HDF5_BINDIR% #- ps: Invoke-WebRequest "https://support.hdfgroup.org/ftp/HDF5/current18/bin/windows/extra/hdf5-1.8.18-win64-vs2015-shared.zip" -OutFile hdf5.zip @@ -39,7 +41,8 @@ install: # check path - "echo %PATH%" - "echo %PYTHONPATH%" - - pip install numpy + - python -c "import sys; print(sys.version); print(sys.executable)" + - python -m pip install numpy # try to import numpy to as conda gut-check - python -c "import numpy as n; print(n.__version__); print(n.get_include());" # try to import numpy to as conda gut-check @@ -52,7 +55,7 @@ install: - msmpisdk.msi /passive - set PATH=C:\Program Files (x86)\Microsoft SDKs\MPI;%PATH% # install mpi4py - - pip install mpi4py + - python -m pip install mpi4py # Install CMake 3.24.3 ############################################################################ diff --git a/scripts/uberenv_configs/packages/conduit/package.py b/scripts/uberenv_configs/packages/conduit/package.py index 91804b659..17b97c8e5 100644 --- a/scripts/uberenv_configs/packages/conduit/package.py +++ b/scripts/uberenv_configs/packages/conduit/package.py @@ -128,6 +128,7 @@ class Conduit(CMakePackage): ####################### depends_on("python", when="+python") extends("python", when="+python") + depends_on("py-pip", when="+python", type=("build", "run")) depends_on("py-numpy", when="+python", type=("build", "run")) depends_on("py-mpi4py", when="+python+mpi", type=("build", "run")) diff --git a/setup.py b/setup.py index 09a5b184b..9ba601ce6 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ from setuptools.command.build_ext import build_ext from distutils.version import LooseVersion -CONDUIT_VERSION = '0.8.7' +CONDUIT_VERSION = '0.8.8' class CMakeExtension(Extension): def __init__(self, name, sourcedir=''): diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d961c5703..856ca7386 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -45,6 +45,16 @@ option(ENABLE_OPENMP "Build OpenMP Support" OFF) # conduit is brought in as a submodule option(CONDUIT_ENABLE_TESTS "Build conduit tests" ON) +###################### +# CMake Policy Selections +###################### + +# allow FindPythonInterp and FindPythonLibs +# https://cmake.org/cmake/help/latest/policy/CMP0148.html +if(POLICY CMP0148) + cmake_policy(SET CMP0148 OLD) +endif() + ################################ # Invoke CMake Fortran setup # if ENABLE_FORTRAN == ON diff --git a/src/cmake/thirdparty/SetupPython.cmake b/src/cmake/thirdparty/SetupPython.cmake index ef1fb66bf..0431875a2 100644 --- a/src/cmake/thirdparty/SetupPython.cmake +++ b/src/cmake/thirdparty/SetupPython.cmake @@ -18,9 +18,11 @@ endif() find_package(PythonInterp REQUIRED) if(PYTHONINTERP_FOUND) - MESSAGE(STATUS "PYTHON_EXECUTABLE ${PYTHON_EXECUTABLE}") + # clear extra python module dirs + set(EXTRA_PYTHON_MODULE_DIRS "") + execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c" "import sys;from distutils.sysconfig import get_config_var; sys.stdout.write(get_config_var('VERSION'))" OUTPUT_VARIABLE PYTHON_CONFIG_VERSION @@ -47,7 +49,9 @@ if(PYTHONINTERP_FOUND) MESSAGE(FATAL_ERROR "Reported PYTHON_SITE_PACKAGES_DIR ${PYTHON_SITE_PACKAGES_DIR} does not exist!") endif() - + # for embedded python, we need to know where the site packages dir is + list(APPEND EXTRA_PYTHON_MODULE_DIRS ${PYTHON_SITE_PACKAGES_DIR}) + # check if we need "-undefined dynamic_lookup" by inspecting LDSHARED flags execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c" "import sys;import sysconfig;sys.stdout.write(sysconfig.get_config_var('LDSHARED'))" @@ -182,7 +186,7 @@ find_package_handle_standard_args(Python DEFAULT_MSG ############################################################################## # Macro to use a pure python distutils setup script ############################################################################## -FUNCTION(PYTHON_ADD_DISTUTILS_SETUP) +FUNCTION(PYTHON_ADD_PIP_SETUP) set(singleValuedArgs NAME DEST_DIR PY_MODULE_DIR PY_SETUP_FILE FOLDER) set(multiValuedArgs PY_SOURCES) @@ -193,52 +197,52 @@ FUNCTION(PYTHON_ADD_DISTUTILS_SETUP) # check req'd args if(NOT DEFINED args_NAME) message(FATAL_ERROR - "PYTHON_ADD_DISTUTILS_SETUP: Missing required argument NAME") + "PYTHON_ADD_PIP_SETUP: Missing required argument NAME") endif() if(NOT DEFINED args_DEST_DIR) message(FATAL_ERROR - "PYTHON_ADD_DISTUTILS_SETUP: Missing required argument DEST_DIR") + "PYTHON_ADD_PIP_SETUP: Missing required argument DEST_DIR") endif() if(NOT DEFINED args_PY_MODULE_DIR) message(FATAL_ERROR - "PYTHON_ADD_DISTUTILS_SETUP: Missing required argument PY_MODULE_DIR") + "PYTHON_ADD_PIP_SETUP: Missing required argument PY_MODULE_DIR") endif() if(NOT DEFINED args_PY_SETUP_FILE) message(FATAL_ERROR - "PYTHON_ADD_DISTUTILS_SETUP: Missing required argument PY_SETUP_FILE") + "PYTHON_ADD_PIP_SETUP: Missing required argument PY_SETUP_FILE") endif() if(NOT DEFINED args_PY_SOURCES) message(FATAL_ERROR - "PYTHON_ADD_DISTUTILS_SETUP: Missing required argument PY_SOURCES") + "PYTHON_ADD_PIP_SETUP: Missing required argument PY_SOURCES") endif() - MESSAGE(STATUS "Configuring python distutils setup: ${args_NAME}") + MESSAGE(STATUS "Configuring python pip setup: ${args_NAME}") # dest for build dir set(abs_dest_path ${CMAKE_BINARY_DIR}/${args_DEST_DIR}) if(WIN32) - # on windows, distutils seems to need standard "\" style paths + # on windows, python seems to need standard "\" style paths string(REGEX REPLACE "/" "\\\\" abs_dest_path ${abs_dest_path}) endif() + # NOTE: With pip, you can't directly control build dir with an arg + # like we were able to do with distutils, you have to use TMPDIR + # TODO: we might want to explore this in the future add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${args_NAME}_build - COMMAND ${CMAKE_COMMAND} -E env SETUPTOOLS_USE_DISTUTILS=stdlib - ${PYTHON_EXECUTABLE} ${args_PY_SETUP_FILE} -v - build - --build-base=${CMAKE_CURRENT_BINARY_DIR}/${args_NAME}_build - install - --install-purelib="${abs_dest_path}" + COMMAND ${PYTHON_EXECUTABLE} -m pip install . -V --upgrade + --disable-pip-version-check --no-warn-script-location + --target "${abs_dest_path}" DEPENDS ${args_PY_SETUP_FILE} ${args_PY_SOURCES} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) add_custom_target(${args_NAME} ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${args_NAME}_build) - # also use distutils for the install ... + # also use pip for the install ... # if PYTHON_MODULE_INSTALL_PREFIX is set, install there if(PYTHON_MODULE_INSTALL_PREFIX) set(py_mod_inst_prefix ${PYTHON_MODULE_INSTALL_PREFIX}) @@ -249,10 +253,9 @@ FUNCTION(PYTHON_ADD_DISTUTILS_SETUP) INSTALL(CODE " EXECUTE_PROCESS(WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - COMMAND ${CMAKE_COMMAND} -E env SETUPTOOLS_USE_DISTUTILS=stdlib - ${PYTHON_EXECUTABLE} ${args_PY_SETUP_FILE} -v - build --build-base=${CMAKE_CURRENT_BINARY_DIR}/${args_NAME}_build_install - install --install-purelib=${py_mod_inst_prefix} + COMMAND ${PYTHON_EXECUTABLE} -m pip install . -V --upgrade + --disable-pip-version-check --no-warn-script-location + --target ${py_mod_inst_prefix} OUTPUT_VARIABLE PY_DIST_UTILS_INSTALL_OUT) MESSAGE(STATUS \"\${PY_DIST_UTILS_INSTALL_OUT}\") ") @@ -261,10 +264,9 @@ FUNCTION(PYTHON_ADD_DISTUTILS_SETUP) INSTALL(CODE " EXECUTE_PROCESS(WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - COMMAND ${CMAKE_COMMAND} -E env SETUPTOOLS_USE_DISTUTILS=stdlib - ${PYTHON_EXECUTABLE} ${args_PY_SETUP_FILE} -v - build --build-base=${CMAKE_CURRENT_BINARY_DIR}/${args_NAME}_build_install - install --install-purelib=\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/${args_DEST_DIR} + COMMAND ${PYTHON_EXECUTABLE} -m pip install . -V --upgrade + --disable-pip-version-check --no-warn-script-location + --target \$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/${args_DEST_DIR} OUTPUT_VARIABLE PY_DIST_UTILS_INSTALL_OUT) MESSAGE(STATUS \"\${PY_DIST_UTILS_INSTALL_OUT}\") ") @@ -275,7 +277,7 @@ FUNCTION(PYTHON_ADD_DISTUTILS_SETUP) blt_set_target_folder(TARGET ${args_NAME} FOLDER ${args_FOLDER}) endif() -ENDFUNCTION(PYTHON_ADD_DISTUTILS_SETUP) +ENDFUNCTION(PYTHON_ADD_PIP_SETUP) ############################################################################## # Macro to create a compiled python module @@ -368,7 +370,7 @@ FUNCTION(PYTHON_ADD_COMPILED_MODULE) ENDFUNCTION(PYTHON_ADD_COMPILED_MODULE) ############################################################################## -# Macro to create a compiled distutils and compiled python module +# Macro to create a pip script and compiled python module ############################################################################## FUNCTION(PYTHON_ADD_HYBRID_MODULE) set(singleValuedArgs NAME DEST_DIR PY_MODULE_DIR PY_SETUP_FILE FOLDER) @@ -411,12 +413,12 @@ FUNCTION(PYTHON_ADD_HYBRID_MODULE) MESSAGE(STATUS "Configuring hybrid python module: ${args_NAME}") - PYTHON_ADD_DISTUTILS_SETUP(NAME "${args_NAME}_py_setup" - DEST_DIR ${args_DEST_DIR} - PY_MODULE_DIR ${args_PY_MODULE_DIR} - PY_SETUP_FILE ${args_PY_SETUP_FILE} - PY_SOURCES ${args_PY_SOURCES} - FOLDER ${args_FOLDER}) + PYTHON_ADD_PIP_SETUP(NAME "${args_NAME}_py_setup" + DEST_DIR ${args_DEST_DIR} + PY_MODULE_DIR ${args_PY_MODULE_DIR} + PY_SETUP_FILE ${args_PY_SETUP_FILE} + PY_SOURCES ${args_PY_SOURCES} + FOLDER ${args_FOLDER}) PYTHON_ADD_COMPILED_MODULE(NAME ${args_NAME} DEST_DIR ${args_DEST_DIR} @@ -424,6 +426,10 @@ FUNCTION(PYTHON_ADD_HYBRID_MODULE) SOURCES ${args_SOURCES} FOLDER ${args_FOLDER}) + # args_NAME depends on "${args_NAME}_py_setup" + add_dependencies( ${args_NAME} "${args_NAME}_py_setup") + ENDFUNCTION(PYTHON_ADD_HYBRID_MODULE) + diff --git a/src/examples/cpp_fort_and_py/SetupPython.cmake b/src/examples/cpp_fort_and_py/SetupPython.cmake index ef1fb66bf..c64d3b93e 100644 --- a/src/examples/cpp_fort_and_py/SetupPython.cmake +++ b/src/examples/cpp_fort_and_py/SetupPython.cmake @@ -18,9 +18,11 @@ endif() find_package(PythonInterp REQUIRED) if(PYTHONINTERP_FOUND) - MESSAGE(STATUS "PYTHON_EXECUTABLE ${PYTHON_EXECUTABLE}") + # clear extra python module dirs + set(EXTRA_PYTHON_MODULE_DIRS "") + execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c" "import sys;from distutils.sysconfig import get_config_var; sys.stdout.write(get_config_var('VERSION'))" OUTPUT_VARIABLE PYTHON_CONFIG_VERSION @@ -47,7 +49,9 @@ if(PYTHONINTERP_FOUND) MESSAGE(FATAL_ERROR "Reported PYTHON_SITE_PACKAGES_DIR ${PYTHON_SITE_PACKAGES_DIR} does not exist!") endif() - + # for embedded python, we need to know where the site packages dir is + list(APPEND EXTRA_PYTHON_MODULE_DIRS ${PYTHON_SITE_PACKAGES_DIR}) + # check if we need "-undefined dynamic_lookup" by inspecting LDSHARED flags execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c" "import sys;import sysconfig;sys.stdout.write(sysconfig.get_config_var('LDSHARED'))" @@ -182,7 +186,7 @@ find_package_handle_standard_args(Python DEFAULT_MSG ############################################################################## # Macro to use a pure python distutils setup script ############################################################################## -FUNCTION(PYTHON_ADD_DISTUTILS_SETUP) +FUNCTION(PYTHON_ADD_PIP_SETUP) set(singleValuedArgs NAME DEST_DIR PY_MODULE_DIR PY_SETUP_FILE FOLDER) set(multiValuedArgs PY_SOURCES) @@ -193,52 +197,52 @@ FUNCTION(PYTHON_ADD_DISTUTILS_SETUP) # check req'd args if(NOT DEFINED args_NAME) message(FATAL_ERROR - "PYTHON_ADD_DISTUTILS_SETUP: Missing required argument NAME") + "PYTHON_ADD_PIP_SETUP: Missing required argument NAME") endif() if(NOT DEFINED args_DEST_DIR) message(FATAL_ERROR - "PYTHON_ADD_DISTUTILS_SETUP: Missing required argument DEST_DIR") + "PYTHON_ADD_PIP_SETUP: Missing required argument DEST_DIR") endif() if(NOT DEFINED args_PY_MODULE_DIR) message(FATAL_ERROR - "PYTHON_ADD_DISTUTILS_SETUP: Missing required argument PY_MODULE_DIR") + "PYTHON_ADD_PIP_SETUP: Missing required argument PY_MODULE_DIR") endif() if(NOT DEFINED args_PY_SETUP_FILE) message(FATAL_ERROR - "PYTHON_ADD_DISTUTILS_SETUP: Missing required argument PY_SETUP_FILE") + "PYTHON_ADD_PIP_SETUP: Missing required argument PY_SETUP_FILE") endif() if(NOT DEFINED args_PY_SOURCES) message(FATAL_ERROR - "PYTHON_ADD_DISTUTILS_SETUP: Missing required argument PY_SOURCES") + "PYTHON_ADD_PIP_SETUP: Missing required argument PY_SOURCES") endif() - MESSAGE(STATUS "Configuring python distutils setup: ${args_NAME}") + MESSAGE(STATUS "Configuring python pip setup: ${args_NAME}") # dest for build dir set(abs_dest_path ${CMAKE_BINARY_DIR}/${args_DEST_DIR}) if(WIN32) - # on windows, distutils seems to need standard "\" style paths + # on windows, python seems to need standard "\" style paths string(REGEX REPLACE "/" "\\\\" abs_dest_path ${abs_dest_path}) endif() + # NOTE: With pip, you can't directly control build dir with an arg + # like we were able to do with distutils, you have to use TMPDIR + # TODO: we might want to explore this in the future add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${args_NAME}_build - COMMAND ${CMAKE_COMMAND} -E env SETUPTOOLS_USE_DISTUTILS=stdlib - ${PYTHON_EXECUTABLE} ${args_PY_SETUP_FILE} -v - build - --build-base=${CMAKE_CURRENT_BINARY_DIR}/${args_NAME}_build - install - --install-purelib="${abs_dest_path}" + COMMAND ${PYTHON_EXECUTABLE} -m pip install . -V --upgrade + --disable-pip-version-check --no-warn-script-location + --target "${abs_dest_path}" DEPENDS ${args_PY_SETUP_FILE} ${args_PY_SOURCES} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) add_custom_target(${args_NAME} ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${args_NAME}_build) - # also use distutils for the install ... + # also use pip for the install ... # if PYTHON_MODULE_INSTALL_PREFIX is set, install there if(PYTHON_MODULE_INSTALL_PREFIX) set(py_mod_inst_prefix ${PYTHON_MODULE_INSTALL_PREFIX}) @@ -249,10 +253,9 @@ FUNCTION(PYTHON_ADD_DISTUTILS_SETUP) INSTALL(CODE " EXECUTE_PROCESS(WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - COMMAND ${CMAKE_COMMAND} -E env SETUPTOOLS_USE_DISTUTILS=stdlib - ${PYTHON_EXECUTABLE} ${args_PY_SETUP_FILE} -v - build --build-base=${CMAKE_CURRENT_BINARY_DIR}/${args_NAME}_build_install - install --install-purelib=${py_mod_inst_prefix} + COMMAND ${PYTHON_EXECUTABLE} -m pip install . -V --upgrade + --disable-pip-version-check --no-warn-script-location + --target ${py_mod_inst_prefix} OUTPUT_VARIABLE PY_DIST_UTILS_INSTALL_OUT) MESSAGE(STATUS \"\${PY_DIST_UTILS_INSTALL_OUT}\") ") @@ -261,10 +264,9 @@ FUNCTION(PYTHON_ADD_DISTUTILS_SETUP) INSTALL(CODE " EXECUTE_PROCESS(WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - COMMAND ${CMAKE_COMMAND} -E env SETUPTOOLS_USE_DISTUTILS=stdlib - ${PYTHON_EXECUTABLE} ${args_PY_SETUP_FILE} -v - build --build-base=${CMAKE_CURRENT_BINARY_DIR}/${args_NAME}_build_install - install --install-purelib=\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/${args_DEST_DIR} + COMMAND ${PYTHON_EXECUTABLE} -m pip install . -V --upgrade + --disable-pip-version-check --no-warn-script-location + --target \$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/${args_DEST_DIR} OUTPUT_VARIABLE PY_DIST_UTILS_INSTALL_OUT) MESSAGE(STATUS \"\${PY_DIST_UTILS_INSTALL_OUT}\") ") @@ -275,7 +277,7 @@ FUNCTION(PYTHON_ADD_DISTUTILS_SETUP) blt_set_target_folder(TARGET ${args_NAME} FOLDER ${args_FOLDER}) endif() -ENDFUNCTION(PYTHON_ADD_DISTUTILS_SETUP) +ENDFUNCTION(PYTHON_ADD_PIP_SETUP) ############################################################################## # Macro to create a compiled python module @@ -368,7 +370,7 @@ FUNCTION(PYTHON_ADD_COMPILED_MODULE) ENDFUNCTION(PYTHON_ADD_COMPILED_MODULE) ############################################################################## -# Macro to create a compiled distutils and compiled python module +# Macro to create a pip script and compiled python module ############################################################################## FUNCTION(PYTHON_ADD_HYBRID_MODULE) set(singleValuedArgs NAME DEST_DIR PY_MODULE_DIR PY_SETUP_FILE FOLDER) @@ -411,12 +413,12 @@ FUNCTION(PYTHON_ADD_HYBRID_MODULE) MESSAGE(STATUS "Configuring hybrid python module: ${args_NAME}") - PYTHON_ADD_DISTUTILS_SETUP(NAME "${args_NAME}_py_setup" - DEST_DIR ${args_DEST_DIR} - PY_MODULE_DIR ${args_PY_MODULE_DIR} - PY_SETUP_FILE ${args_PY_SETUP_FILE} - PY_SOURCES ${args_PY_SOURCES} - FOLDER ${args_FOLDER}) + PYTHON_ADD_PIP_SETUP(NAME "${args_NAME}_py_setup" + DEST_DIR ${args_DEST_DIR} + PY_MODULE_DIR ${args_PY_MODULE_DIR} + PY_SETUP_FILE ${args_PY_SETUP_FILE} + PY_SOURCES ${args_PY_SOURCES} + FOLDER ${args_FOLDER}) PYTHON_ADD_COMPILED_MODULE(NAME ${args_NAME} DEST_DIR ${args_DEST_DIR} @@ -427,3 +429,4 @@ FUNCTION(PYTHON_ADD_HYBRID_MODULE) ENDFUNCTION(PYTHON_ADD_HYBRID_MODULE) + diff --git a/src/libs/blueprint/python/CMakeLists.txt b/src/libs/blueprint/python/CMakeLists.txt index af88de056..4f878335d 100644 --- a/src/libs/blueprint/python/CMakeLists.txt +++ b/src/libs/blueprint/python/CMakeLists.txt @@ -13,6 +13,10 @@ PYTHON_ADD_COMPILED_MODULE(NAME conduit_blueprint_python ${CMAKE_CURRENT_BINARY_DIR}/conduit_blueprint_python_exports.h FOLDER libs/python) +# compiled modules depend on output dir structure created by main module setup +add_dependencies( conduit_blueprint_python conduit_python_py_setup) + + # link with the proper libs (beyond python) target_link_libraries(conduit_blueprint_python conduit conduit_blueprint conduit_python_build) @@ -28,6 +32,10 @@ PYTHON_ADD_COMPILED_MODULE(NAME conduit_blueprint_mcarray_python ${CMAKE_CURRENT_BINARY_DIR}/conduit_blueprint_python_exports.h FOLDER libs/python) +# compiled modules depend on output dir structure created by main module setup +add_dependencies( conduit_blueprint_mcarray_python conduit_python_py_setup) + + # link with the proper libs (beyond python) target_link_libraries(conduit_blueprint_mcarray_python conduit conduit_blueprint conduit_python_build) @@ -39,6 +47,9 @@ PYTHON_ADD_COMPILED_MODULE(NAME conduit_blueprint_mcarray_examples_pyth ${CMAKE_CURRENT_BINARY_DIR}/conduit_blueprint_python_exports.h FOLDER libs/python) +# compiled modules depend on output dir structure created by main module setup +add_dependencies( conduit_blueprint_mcarray_examples_python conduit_python_py_setup) + # link with the proper libs (beyond python) target_link_libraries(conduit_blueprint_mcarray_examples_python conduit conduit_blueprint conduit_python_build) @@ -54,6 +65,9 @@ PYTHON_ADD_COMPILED_MODULE(NAME conduit_blueprint_mesh_python ${CMAKE_CURRENT_BINARY_DIR}/conduit_blueprint_python_exports.h FOLDER libs/python) +# compiled modules depend on output dir structure created by main module setup +add_dependencies( conduit_blueprint_mesh_python conduit_python_py_setup) + # link with the proper libs (beyond python) target_link_libraries(conduit_blueprint_mesh_python conduit conduit_blueprint conduit_python_build) @@ -66,6 +80,9 @@ PYTHON_ADD_COMPILED_MODULE(NAME conduit_blueprint_mesh_examples_python ${CMAKE_CURRENT_BINARY_DIR}/conduit_blueprint_python_exports.h FOLDER libs/python) +# compiled modules depend on output dir structure created by main module setup +add_dependencies( conduit_blueprint_mesh_examples_python conduit_python_py_setup) + # link with the proper libs (beyond python) target_link_libraries(conduit_blueprint_mesh_examples_python conduit conduit_blueprint conduit_python_build) @@ -81,6 +98,9 @@ PYTHON_ADD_COMPILED_MODULE(NAME conduit_blueprint_table_python ${CMAKE_CURRENT_BINARY_DIR}/conduit_blueprint_python_exports.h FOLDER libs/python) +# compiled modules depend on output dir structure created by main module setup +add_dependencies( conduit_blueprint_table_python conduit_python_py_setup) + # link with the proper libs (beyond python) target_link_libraries(conduit_blueprint_table_python conduit conduit_blueprint conduit_python_build) diff --git a/src/libs/conduit/python/CMakeLists.txt b/src/libs/conduit/python/CMakeLists.txt index 91b12405f..c2167acf1 100644 --- a/src/libs/conduit/python/CMakeLists.txt +++ b/src/libs/conduit/python/CMakeLists.txt @@ -59,6 +59,9 @@ PYTHON_ADD_COMPILED_MODULE(NAME conduit_utils_python ${CMAKE_CURRENT_BINARY_DIR}/conduit_python_exports.h FOLDER libs/python) +# compiled modules depend on output dir structure created by main module setup +add_dependencies( conduit_utils_python conduit_python_py_setup) + # link with the proper libs (beyond python) target_link_libraries(conduit_utils_python conduit conduit_python_build) diff --git a/src/libs/conduit/python/setup.py b/src/libs/conduit/python/setup.py index 53708984a..15a8daf85 100644 --- a/src/libs/conduit/python/setup.py +++ b/src/libs/conduit/python/setup.py @@ -1,26 +1,20 @@ # Copyright (c) Lawrence Livermore National Security, LLC and other Conduit # Project developers. See top-level LICENSE AND COPYRIGHT files for dates and # other details. No copyright assignment is required to contribute to Conduit. +############################################################################### + ############################################################################### # file: setup.py -# Purpose: disutils setup for conduit python module. +# Purpose: setuptools setup for conduit python module. # ############################################################################### - -import os import sys import platform -from distutils.core import setup -from distutils.command.install_egg_info import install_egg_info - -# disable install_egg_info -class SkipEggInfo(install_egg_info): - def run(self): - pass +from setuptools import setup # path args fix helper for windows def adjust_windows_args_paths(): - print("[windows detected: normalzing paths]") + print("[windows detected: normalizing paths]") nargs =[] for v in sys.argv: nargs.append(v.replace("/","\\")) @@ -31,9 +25,13 @@ def adjust_windows_args_paths(): if platform.system() == 'Windows': adjust_windows_args_paths() +CONDUIT_VERSION = '0.8.8' + setup (name = 'conduit', description = 'conduit', + version = CONDUIT_VERSION, package_dir = {'conduit':'py_src'}, + zip_safe=False, packages=['conduit', 'conduit.utils', 'conduit.blueprint', @@ -48,7 +46,4 @@ def adjust_windows_args_paths(): 'conduit.relay.io.blueprint', 'conduit.relay.io.silo', 'conduit.relay.mpi', - 'conduit.relay.web'], - cmdclass={'install_egg_info': SkipEggInfo}) - - + 'conduit.relay.web']) diff --git a/src/libs/relay/python/CMakeLists.txt b/src/libs/relay/python/CMakeLists.txt index 40d9d3e94..8223f8dee 100644 --- a/src/libs/relay/python/CMakeLists.txt +++ b/src/libs/relay/python/CMakeLists.txt @@ -13,6 +13,9 @@ PYTHON_ADD_COMPILED_MODULE(NAME conduit_relay_python ${CMAKE_CURRENT_BINARY_DIR}/conduit_relay_python_exports.h FOLDER libs/python) +# compiled modules depend on output dir structure created by main module setup +add_dependencies( conduit_relay_python conduit_python_py_setup) + # link with the proper libs target_link_libraries(conduit_relay_python conduit conduit_relay conduit_python_build) @@ -24,6 +27,8 @@ PYTHON_ADD_COMPILED_MODULE(NAME conduit_relay_io_python ${CMAKE_CURRENT_BINARY_DIR}/conduit_relay_python_exports.h FOLDER libs/python) +# compiled modules depend on output dir structure created by main module setup +add_dependencies( conduit_relay_io_python conduit_python_py_setup) # link with the proper libs (beyond python) target_link_libraries(conduit_relay_io_python conduit conduit_relay conduit_python_build) @@ -36,6 +41,9 @@ PYTHON_ADD_COMPILED_MODULE(NAME conduit_relay_io_blueprint_python ${CMAKE_CURRENT_BINARY_DIR}/conduit_relay_python_exports.h FOLDER libs/python) +# compiled modules depend on output dir structure created by main module setup +add_dependencies( conduit_relay_io_blueprint_python conduit_python_py_setup) + # link with the proper libs (beyond python) target_link_libraries(conduit_relay_io_blueprint_python conduit conduit_relay conduit_blueprint conduit_python_build) @@ -49,6 +57,9 @@ if(SILO_FOUND) ${CMAKE_CURRENT_BINARY_DIR}/conduit_relay_python_exports.h FOLDER libs/python) + # compiled modules depend on output dir structure created by main module setup + add_dependencies( conduit_relay_io_silo_python conduit_python_py_setup) + # link with the proper libs (beyond python) target_link_libraries(conduit_relay_io_silo_python conduit conduit_relay conduit_blueprint conduit_python_build) @@ -63,6 +74,9 @@ if(ENABLE_RELAY_WEBSERVER) ${CMAKE_CURRENT_BINARY_DIR}/conduit_relay_python_exports.h FOLDER libs/python) + # compiled modules depend on output dir structure created by main module setup + add_dependencies( conduit_relay_web_python conduit_python_py_setup) + # link with the proper libs (beyond python) target_link_libraries(conduit_relay_web_python conduit conduit_relay conduit_python_build) endif() @@ -81,6 +95,9 @@ if(MPI_FOUND) ${CMAKE_CURRENT_BINARY_DIR}/conduit_relay_python_exports.h FOLDER libs/python) + # compiled modules depend on output dir structure created by main module setup + add_dependencies( conduit_relay_mpi_python conduit_python_py_setup) + # link with the proper libs (beyond python) target_link_libraries(conduit_relay_mpi_python conduit conduit_relay_mpi conduit_python_build)