migrate from distuils to pip for hybrid module creation (#1206)

* migrate from distuils to pip for hybrid module creation * pip ver alignment * add pip as explicit dep when python is on * appveryor use py 3.8 insead of 2.7 * wire up python module cmake target deps * appveyor python updates
LLNL · Feb 2, 2024 · 1451ff8 · 1451ff8
1 parent dc2509f
commit 1451ff8
Show file tree

Hide file tree

Showing 10 changed files with 142 additions and 84 deletions.
diff --git a/appveyor.yml b/appveyor.yml
@@ -29,6 +29,8 @@ init:
   - "ECHO %PYTHON_VERSION% %MINICONDA%"
 
 install:
+  # setup python 3.8
+  - set PATH=C:\Python38\bin;%PATH%
   # setup hdf5
   #- set PATH=%PATH%;%HDF5_BINDIR%
   #- ps: Invoke-WebRequest "https://support.hdfgroup.org/ftp/HDF5/current18/bin/windows/extra/hdf5-1.8.18-win64-vs2015-shared.zip" -OutFile hdf5.zip
@@ -39,7 +41,8 @@ install:
   # check path
   - "echo %PATH%"
   - "echo %PYTHONPATH%"
-  - pip install numpy
+  - python -c "import sys; print(sys.version); print(sys.executable)"
+  - python -m pip install numpy
   # try to import numpy to as conda gut-check
   - python -c "import numpy as n; print(n.__version__); print(n.get_include());"
   # try to import numpy to as conda gut-check
@@ -52,7 +55,7 @@ install:
   - msmpisdk.msi /passive
   - set PATH=C:\Program Files (x86)\Microsoft SDKs\MPI;%PATH%
   # install mpi4py
-  - pip install mpi4py
+  - python -m pip install mpi4py
 
   # Install CMake 3.24.3
   ############################################################################

diff --git a/scripts/uberenv_configs/packages/conduit/package.py b/scripts/uberenv_configs/packages/conduit/package.py
@@ -128,6 +128,7 @@ class Conduit(CMakePackage):
     #######################
     depends_on("python", when="+python")
     extends("python", when="+python")
+    depends_on("py-pip", when="+python", type=("build", "run"))
     depends_on("py-numpy", when="+python", type=("build", "run"))
     depends_on("py-mpi4py", when="+python+mpi", type=("build", "run"))
 

diff --git a/setup.py b/setup.py
@@ -42,7 +42,7 @@
 from setuptools.command.build_ext import build_ext
 from distutils.version import LooseVersion
 
-CONDUIT_VERSION = '0.8.7'
+CONDUIT_VERSION = '0.8.8'
 
 class CMakeExtension(Extension):
     def __init__(self, name, sourcedir=''):

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -45,6 +45,16 @@ option(ENABLE_OPENMP      "Build OpenMP Support"        OFF)
 # conduit is brought in as a submodule
 option(CONDUIT_ENABLE_TESTS "Build conduit tests" ON)
 
+######################
+# CMake Policy Selections
+######################
+
+# allow FindPythonInterp and FindPythonLibs
+# https://cmake.org/cmake/help/latest/policy/CMP0148.html
+if(POLICY CMP0148)
+    cmake_policy(SET CMP0148 OLD)
+endif()
+
 ################################
 # Invoke CMake Fortran setup
 # if ENABLE_FORTRAN == ON

diff --git a/src/cmake/thirdparty/SetupPython.cmake b/src/cmake/thirdparty/SetupPython.cmake
@@ -18,9 +18,11 @@ endif()
 
 find_package(PythonInterp REQUIRED)
 if(PYTHONINTERP_FOUND)
-
         MESSAGE(STATUS "PYTHON_EXECUTABLE ${PYTHON_EXECUTABLE}")
 
+        # clear extra python module dirs
+        set(EXTRA_PYTHON_MODULE_DIRS "")
+
         execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c" 
                         "import sys;from distutils.sysconfig import get_config_var; sys.stdout.write(get_config_var('VERSION'))"
                         OUTPUT_VARIABLE PYTHON_CONFIG_VERSION
@@ -47,7 +49,9 @@ if(PYTHONINTERP_FOUND)
             MESSAGE(FATAL_ERROR "Reported PYTHON_SITE_PACKAGES_DIR ${PYTHON_SITE_PACKAGES_DIR} does not exist!")
         endif()
 
-
+        # for embedded python, we need to know where the site packages dir is
+        list(APPEND EXTRA_PYTHON_MODULE_DIRS ${PYTHON_SITE_PACKAGES_DIR})
+
         # check if we need "-undefined dynamic_lookup" by inspecting LDSHARED flags
         execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c"
                                 "import sys;import sysconfig;sys.stdout.write(sysconfig.get_config_var('LDSHARED'))"
@@ -182,7 +186,7 @@ find_package_handle_standard_args(Python  DEFAULT_MSG
 ##############################################################################
 # Macro to use a pure python distutils setup script
 ##############################################################################
-FUNCTION(PYTHON_ADD_DISTUTILS_SETUP)
+FUNCTION(PYTHON_ADD_PIP_SETUP)
     set(singleValuedArgs NAME DEST_DIR PY_MODULE_DIR PY_SETUP_FILE FOLDER)
     set(multiValuedArgs  PY_SOURCES)
 
@@ -193,52 +197,52 @@ FUNCTION(PYTHON_ADD_DISTUTILS_SETUP)
     # check req'd args
     if(NOT DEFINED args_NAME)
        message(FATAL_ERROR
-               "PYTHON_ADD_DISTUTILS_SETUP: Missing required argument NAME")
+               "PYTHON_ADD_PIP_SETUP: Missing required argument NAME")
     endif()
 
     if(NOT DEFINED args_DEST_DIR)
        message(FATAL_ERROR
-               "PYTHON_ADD_DISTUTILS_SETUP: Missing required argument DEST_DIR")
+               "PYTHON_ADD_PIP_SETUP: Missing required argument DEST_DIR")
     endif()
 
     if(NOT DEFINED args_PY_MODULE_DIR)
        message(FATAL_ERROR
-       "PYTHON_ADD_DISTUTILS_SETUP: Missing required argument PY_MODULE_DIR")
+       "PYTHON_ADD_PIP_SETUP: Missing required argument PY_MODULE_DIR")
     endif()
 
     if(NOT DEFINED args_PY_SETUP_FILE)
        message(FATAL_ERROR
-       "PYTHON_ADD_DISTUTILS_SETUP: Missing required argument PY_SETUP_FILE")
+       "PYTHON_ADD_PIP_SETUP: Missing required argument PY_SETUP_FILE")
     endif()
 
     if(NOT DEFINED args_PY_SOURCES)
        message(FATAL_ERROR
-       "PYTHON_ADD_DISTUTILS_SETUP: Missing required argument PY_SOURCES")
+       "PYTHON_ADD_PIP_SETUP: Missing required argument PY_SOURCES")
     endif()
 
-    MESSAGE(STATUS "Configuring python distutils setup: ${args_NAME}")
+    MESSAGE(STATUS "Configuring python pip setup: ${args_NAME}")
 
     # dest for build dir
     set(abs_dest_path ${CMAKE_BINARY_DIR}/${args_DEST_DIR})
     if(WIN32)
-        # on windows, distutils seems to need standard "\" style paths
+        # on windows, python seems to need standard "\" style paths
         string(REGEX REPLACE "/" "\\\\" abs_dest_path  ${abs_dest_path})
     endif()
 
+    # NOTE: With pip, you can't directly control build dir with an arg
+    # like we were able to do with distutils, you have to use TMPDIR
+    # TODO: we might want to  explore this in the future
     add_custom_command(OUTPUT  ${CMAKE_CURRENT_BINARY_DIR}/${args_NAME}_build
-            COMMAND ${CMAKE_COMMAND} -E env SETUPTOOLS_USE_DISTUTILS=stdlib
-            ${PYTHON_EXECUTABLE} ${args_PY_SETUP_FILE} -v
-            build
-            --build-base=${CMAKE_CURRENT_BINARY_DIR}/${args_NAME}_build
-            install
-            --install-purelib="${abs_dest_path}"
+            COMMAND ${PYTHON_EXECUTABLE} -m pip install . -V --upgrade
+            --disable-pip-version-check --no-warn-script-location
+            --target "${abs_dest_path}"
             DEPENDS  ${args_PY_SETUP_FILE} ${args_PY_SOURCES}
             WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
 
     add_custom_target(${args_NAME} ALL DEPENDS
                       ${CMAKE_CURRENT_BINARY_DIR}/${args_NAME}_build)
 
-    # also use distutils for the install ...
+    # also use pip for the install ...
     # if PYTHON_MODULE_INSTALL_PREFIX is set, install there
     if(PYTHON_MODULE_INSTALL_PREFIX)
         set(py_mod_inst_prefix ${PYTHON_MODULE_INSTALL_PREFIX})
@@ -249,10 +253,9 @@ FUNCTION(PYTHON_ADD_DISTUTILS_SETUP)
         INSTALL(CODE
             "
             EXECUTE_PROCESS(WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-                COMMAND ${CMAKE_COMMAND} -E env SETUPTOOLS_USE_DISTUTILS=stdlib
-                    ${PYTHON_EXECUTABLE} ${args_PY_SETUP_FILE} -v
-                    build   --build-base=${CMAKE_CURRENT_BINARY_DIR}/${args_NAME}_build_install
-                    install --install-purelib=${py_mod_inst_prefix}
+                COMMAND ${PYTHON_EXECUTABLE} -m pip install . -V --upgrade
+                --disable-pip-version-check --no-warn-script-location
+                --target ${py_mod_inst_prefix}
                 OUTPUT_VARIABLE PY_DIST_UTILS_INSTALL_OUT)
             MESSAGE(STATUS \"\${PY_DIST_UTILS_INSTALL_OUT}\")
             ")
@@ -261,10 +264,9 @@ FUNCTION(PYTHON_ADD_DISTUTILS_SETUP)
         INSTALL(CODE
             "
             EXECUTE_PROCESS(WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-                COMMAND ${CMAKE_COMMAND} -E env SETUPTOOLS_USE_DISTUTILS=stdlib
-                    ${PYTHON_EXECUTABLE} ${args_PY_SETUP_FILE} -v
-                    build   --build-base=${CMAKE_CURRENT_BINARY_DIR}/${args_NAME}_build_install
-                    install --install-purelib=\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/${args_DEST_DIR}
+                COMMAND ${PYTHON_EXECUTABLE} -m pip install . -V --upgrade
+                --disable-pip-version-check --no-warn-script-location
+                --target \$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/${args_DEST_DIR}
                 OUTPUT_VARIABLE PY_DIST_UTILS_INSTALL_OUT)
             MESSAGE(STATUS \"\${PY_DIST_UTILS_INSTALL_OUT}\")
             ")
@@ -275,7 +277,7 @@ FUNCTION(PYTHON_ADD_DISTUTILS_SETUP)
         blt_set_target_folder(TARGET ${args_NAME} FOLDER ${args_FOLDER})
     endif()
 
-ENDFUNCTION(PYTHON_ADD_DISTUTILS_SETUP)
+ENDFUNCTION(PYTHON_ADD_PIP_SETUP)
 
 ##############################################################################
 # Macro to create a compiled python module 
@@ -368,7 +370,7 @@ FUNCTION(PYTHON_ADD_COMPILED_MODULE)
 ENDFUNCTION(PYTHON_ADD_COMPILED_MODULE)
 
 ##############################################################################
-# Macro to create a compiled distutils and compiled python module
+# Macro to create a pip script and compiled python module
 ##############################################################################
 FUNCTION(PYTHON_ADD_HYBRID_MODULE)
     set(singleValuedArgs NAME DEST_DIR PY_MODULE_DIR PY_SETUP_FILE FOLDER)
@@ -411,19 +413,23 @@ FUNCTION(PYTHON_ADD_HYBRID_MODULE)
 
     MESSAGE(STATUS "Configuring hybrid python module: ${args_NAME}")
 
-    PYTHON_ADD_DISTUTILS_SETUP(NAME          "${args_NAME}_py_setup"
-                               DEST_DIR      ${args_DEST_DIR}
-                               PY_MODULE_DIR ${args_PY_MODULE_DIR}
-                               PY_SETUP_FILE ${args_PY_SETUP_FILE}
-                               PY_SOURCES    ${args_PY_SOURCES}
-                               FOLDER        ${args_FOLDER})
+    PYTHON_ADD_PIP_SETUP(NAME          "${args_NAME}_py_setup"
+                         DEST_DIR      ${args_DEST_DIR}
+                         PY_MODULE_DIR ${args_PY_MODULE_DIR}
+                         PY_SETUP_FILE ${args_PY_SETUP_FILE}
+                         PY_SOURCES    ${args_PY_SOURCES}
+                         FOLDER        ${args_FOLDER})
 
     PYTHON_ADD_COMPILED_MODULE(NAME          ${args_NAME}
                                DEST_DIR      ${args_DEST_DIR}
                                PY_MODULE_DIR ${args_PY_MODULE_DIR}
                                SOURCES       ${args_SOURCES}
                                FOLDER        ${args_FOLDER})
 
+    # args_NAME depends on "${args_NAME}_py_setup"
+    add_dependencies( ${args_NAME} "${args_NAME}_py_setup")
+
 ENDFUNCTION(PYTHON_ADD_HYBRID_MODULE)
 
 
+