Merge branch 'branch-24.04' of github.com:rapidsai/devcontainers into…

… branch-24.06

features/src/rapids-build-utils/devcontainer-feature.json

@@ -1,7 +1,7 @@
 {
   "name": "NVIDIA RAPIDS devcontainer build utilities",
   "id": "rapids-build-utils",
-  "version": "24.6.2",
+  "version": "24.6.3",
   "description": "A feature to install the RAPIDS devcontainer build utilities",
   "containerEnv": {
     "BASH_ENV": "/etc/bash.bash_env"

features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh

@@ -9,6 +9,8 @@
 #  -h,--help             Print this text.
 #
 # Options that require values:
+#  -e,--exclude <file>     Path(s) to requirement files of packages to exclude.
+#  -i,--include <file>     Path(s) to requirement files of packages to include.
 #  -k,--key <key>        Only include the key(s)
 # @_include_value_options rapids-list-repos -h | tail -n+2 | head -n-3;
 #  --repo <repo>         Only include dependencies for repo(s).
@@ -36,8 +38,20 @@ make_conda_dependencies() {
     # shellcheck disable=SC1091
     . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'make-conda-env make-conda-dependencies';
 
+    test ${#exclude[@]} -eq 0 && exclude=();
+    test ${#include[@]} -eq 0 && include=();
     test ${#key[@]} -eq 0 && key=(all);
 
+    local -a _exclude=();
+    local exc; for exc in "${exclude[@]}"; do
+        _exclude+=(-f "${exc}");
+    done
+
+    local -a _include=();
+    local inc; for inc in "${include[@]}"; do
+        _include+=(-f "${inc}");
+    done
+
     local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR:-12}.${CUDA_VERSION_MINOR:-0}}";
     cuda_version="$(grep -o '^[0-9]*.[0-9]' <<< "${cuda_version}")";
 
@@ -114,11 +128,14 @@ make_conda_dependencies() {
 
         # shellcheck disable=SC2207
         local conda_noinstall=($(rapids-python-pkg-names) $(rapids-python-conda-pkg-names));
+
         # Generate a combined conda env yaml file.
         conda-merge "${conda_env_yamls[@]}"                                                                                   \
           | (grep -v '^name:'                                                                             || [ "$?" == "1" ]) \
           | (grep -v -P '^[ ]*?\- (\.git\@[^(main|master)])(.*?)$'                                        || [ "$?" == "1" ]) \
           | (grep -v -P "^[ ]*?\- ($(tr -d '[:blank:]' <<< "${conda_noinstall[@]/%/ |}"))(=.*|>.*|<.*)?$" || [ "$?" == "1" ]) \
+          | ( if test ${#_exclude[@]} -gt 0; then grep -E -v "${_exclude[@]}" || [ "$?" == "1" ]; else cat -; fi            ) \
+          | ( if test ${#_include[@]} -gt 0; then grep -E    "${_include[@]}" || [ "$?" == "1" ]; else cat -; fi            ) \
           ;
 
         rm -f "${conda_env_yamls[@]}";

features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh

@@ -18,7 +18,7 @@ make_conda_env() {
     local -;
     set -euo pipefail;
 
-    eval "$(_parse_args --skip '-k,--key -m,--manifest -o,--omit --repo' "${@:2}" <&0)";
+    eval "$(_parse_args --take '-f,--force' "${@:2}" <&0)";
 
     # shellcheck disable=SC1091
     . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'make-conda-env';

features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh

@@ -10,6 +10,8 @@
 #  --no-dedupe             Don't sort and dedupe the combined requirements.txt.
 #
 # Options that require values:
+#  -e,--exclude <file>     Path(s) to requirement files of packages to exclude.
+#  -i,--include <file>     Path(s) to requirement files of packages to include.
 #  -k,--key <key>          Only include the key(s)
 # @_include_value_options rapids-list-repos -h | tail -n+2 | head -n-3;
 #  --repo <repo>           Only include dependencies for repo(s).
@@ -36,7 +38,20 @@ make_pip_dependencies() {
     # shellcheck disable=SC1091
     . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'make-pip-env make-pip-dependencies';
 
+    test ${#exclude[@]} -eq 0 && exclude=();
+    test ${#include[@]} -eq 0 && include=();
     test ${#key[@]} -eq 0 && key=(py_build py_run py_test all);
+    test ${#requirement[@]} -eq 0 && requirement=();
+
+    local -a _exclude=();
+    local exc; for exc in "${exclude[@]}"; do
+        _exclude+=(-f "${exc}");
+    done
+
+    local -a _include=();
+    local inc; for inc in "${include[@]}"; do
+        _include+=(-f "${inc}");
+    done
 
     local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR:-12}.${CUDA_VERSION_MINOR:-0}}";
     cuda_version="$(grep -o '^[0-9]*.[0-9]*' <<< "${cuda_version}")";
@@ -107,7 +122,7 @@ make_pip_dependencies() {
         fi
     done
 
-    if test ${#pip_reqs_txts[@]} -gt 0; then
+    if test ${#requirement[@]} -gt 0 || test ${#pip_reqs_txts[@]} -gt 0; then
 
         for ((i=0; i < ${#pip_reqs_txts[@]}; i+=1)); do
             while ! test -f "${pip_reqs_txts[$i]}"; do
@@ -129,12 +144,13 @@ make_pip_dependencies() {
         done
 
         # Generate a combined requirements.txt file
-        # shellcheck disable=SC2154
         cat "${requirement[@]}" "${pip_reqs_txts[@]}"                                                           \
       | (grep -v '^#' || [ "$?" == "1" ])                                                                       \
       | (grep -v -E '^$' || [ "$?" == "1" ])                                                                    \
       | ( if test -n "${no_dedupe-}"; then cat -; else tr -s "[:blank:]" | LC_ALL=C sort -u; fi )               \
       | (grep -v -P "^($(tr -d '[:blank:]' <<< "${pip_noinstall[@]/%/|}"))(=.*|>.*|<.*)?$" || [ "$?" == "1" ])  \
+      | ( if test ${#_exclude[@]} -gt 0; then grep -E -v "${_exclude[@]}" || [ "$?" == "1" ]; else cat -; fi )  \
+      | ( if test ${#_include[@]} -gt 0; then grep -E    "${_include[@]}" || [ "$?" == "1" ]; else cat -; fi )  \
       | sed -E "s/-cu([0-9]+)/-cu${cuda_version_major}/g"                                                       \
       | sed -E "s/^cupy-cuda[0-9]+x/cupy-cuda${cuda_version_major}x/g"                                          \
       | sed -E "s/^cuda-python.*/cuda-python>=${cuda_version}.0,<$((cuda_version_major+1)).0a0/g"               \

features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh

@@ -8,6 +8,10 @@
 # Boolean options:
 #  -h,--help               Print this text.
 #  -f,--force              Delete the existing pip venv and recreate it from scratch.
+#  --pre                   Include pre-release and development versions. By default, pip only finds
+#                          stable versions.
+#  --no-pre                Don't install pre-release and development versions.
+#  --system-site-packages  Give the virtual environment access to the system site-packages dir.
 # @_include_bool_options rapids-make-pip-dependencies -h | tail -n+2 | head -n-3;
 #
 # @_include_value_options rapids-make-pip-dependencies -h;
@@ -19,18 +23,18 @@ make_pip_env() {
     local -;
     set -euo pipefail;
 
-    eval "$(_parse_args --skip '
-        --no-dedupe
-        -k,--key
-        -m,--manifest
-        -o,--omit
-        --repo
-        -r,--requirement
-    ' "${@:2}" <&0)";
+    eval "$(_parse_args --take '-f,--force --pre --no-pre --system-site-packages' "${@:2}" <&0)";
 
     # shellcheck disable=SC1091
     . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'make-pip-env';
 
+    test ${#pre[@]} -eq 0 && pre=(--pre);
+    test ${#system_site_packages[@]} -eq 0 && system_site_packages=();
+
+    if test -n "${no_pre-}"; then
+        pre=();
+    fi
+
     local env_name="${1}"; shift;
     local env_file_name="${env_name}.requirements.txt";
 
@@ -54,11 +58,11 @@ make_pip_env() {
             cat "${new_env_path}";
             echo "";
 
-            python -m venv --system-site-packages "${HOME}/.local/share/venvs/${env_name}";
+            python -m venv "${system_site_packages[@]}" "${HOME}/.local/share/venvs/${env_name}";
             # shellcheck disable=SC1090
             . "${HOME}/.local/share/venvs/${env_name}/bin/activate";
             python -m pip install -U pip;
-            python -m pip install --pre -I -r "${new_env_path}";
+            python -m pip install "${pre[@]}" -I -r "${new_env_path}";
         # If the venv does exist but it's different from the generated one,
         # print the diff between the envs and update it
         elif ! diff -BNqw "${old_env_path}" "${new_env_path}" >/dev/null 2>&1; then
@@ -76,7 +80,7 @@ make_pip_env() {
             # shellcheck disable=SC1090
             . "${HOME}/.local/share/venvs/${env_name}/bin/activate";
             python -m pip install -U pip;
-            python -m pip install --pre -U -r "${new_env_path}";
+            python -m pip install "${pre[@]}" -U -r "${new_env_path}";
         fi
 
         cp -a "${new_env_path}" "${old_env_path}";