From 5e6e2f19612d5aee1e6a56edab8a64e9402b73e9 Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Mon, 18 Mar 2024 10:53:13 -0700 Subject: [PATCH 1/2] update to nvhpc 24.3 (#248) --- matrix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/matrix.yml b/matrix.yml index 03d6eb6d..3cb5d80d 100644 --- a/matrix.yml +++ b/matrix.yml @@ -29,8 +29,8 @@ x-llvm-prev: &llvm_17 { name: "llvm", version: "17" } x-llvm-curr: &llvm_18 { name: "llvm", version: "18" } x-llvm-env: &llvm_env {CC: "clang", CXX: "clang++", CUDAHOSTCXX: "clang++"} -x-nvhpc-prev: &nvhpc_prev { name: "nvhpc", version: "23.11" } -x-nvhpc-curr: &nvhpc_curr { name: "nvhpc", version: "24.1" } +x-nvhpc-prev: &nvhpc_prev { name: "nvhpc", version: "24.1" } +x-nvhpc-curr: &nvhpc_curr { name: "nvhpc", version: "24.3" } x-nvhpc-env: &nvhpc_env {CC: "nvc", CXX: "nvc++"} From b898f7309bfb0a3bd77883f0ea9560c810df8344 Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Mon, 18 Mar 2024 10:59:51 -0700 Subject: [PATCH 2/2] Generate file keys for C++ lib names (#249) * generate file keys for C++ lib names * add lib prefix * always make conda envs from scratch * bump feature version --- .../devcontainer-feature.json | 2 +- .../bin/make-conda-dependencies.sh | 23 +++++++++++++++++++ .../rapids-build-utils/bin/make-conda-env.sh | 14 +++++++---- .../bin/make-pip-dependencies.sh | 23 +++++++++++++++++++ 4 files changed, 57 insertions(+), 5 deletions(-) diff --git a/features/src/rapids-build-utils/devcontainer-feature.json b/features/src/rapids-build-utils/devcontainer-feature.json index 3e34190e..d8f579fd 100644 --- a/features/src/rapids-build-utils/devcontainer-feature.json +++ b/features/src/rapids-build-utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "NVIDIA RAPIDS devcontainer build utilities", "id": "rapids-build-utils", - "version": "24.4.3", + "version": "24.4.4", "description": "A feature to install the RAPIDS devcontainer build utilities", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh index db526cb8..934c9fe8 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh @@ -56,6 +56,29 @@ make_conda_dependencies() { if [ -f ~/"${!repo_path}/dependencies.yaml" ]; then + local cpp_length="${repo}_cpp_length"; + + for ((j=0; j < ${!cpp_length:-0}; j+=1)); do + local cpp_name="${repo}_cpp_${j}_name"; + + echo "Generating lib${!cpp_name}'s conda env yml" 1>&2; + + local repo_keys=("${key[@]}" "${key[@]/%/_lib${!cpp_name//"-"/"_"}}"); + local keyi; + + for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do + local file="/tmp/${!repo_name}.lib${!cpp_name}.${repo_keys[$keyi]}.env.yaml"; + conda_env_yamls+=("${file}"); + generate_env_yaml \ + "${file}" \ + --file_key "${repo_keys[$keyi]}" \ + --output conda \ + --config ~/"${!repo_path}/dependencies.yaml" \ + --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ + ; + done + done + local py_length="${repo}_python_length"; for ((j=0; j < ${!py_length:-0}; j+=1)); do diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh index a84273a3..959d8722 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh @@ -50,7 +50,7 @@ make_conda_env() { # If the conda env does exist but it's different from the generated one, # print the diff between the envs and update it elif ! diff -BNqw "${old_env_path}" "${new_env_path}" >/dev/null 2>&1; then - echo -e "Updating '${env_name}' conda environment\n" 1>&2; + echo -e "Creating '${env_name}' conda environment\n" 1>&2; echo -e "Environment (${env_file_name}):\n" 1>&2; # Print the diff to the console for debugging @@ -60,12 +60,18 @@ make_conda_env() { || true \ && echo ""; - # Update the current conda env + prune libs that were removed - # Use conda instead of mamba due to https://github.com/mamba-org/mamba/issues/3059 - conda env update -n "${env_name}" -f "${new_env_path}" --prune --solver=libmamba; + # If the conda env exists, recreate it from scratch. + # Most conda issues are due to updating existing envs with new packages. + # We mount in the package cache, so this should still be fast in most cases. + rm -rf "${HOME}/.conda/envs/${env_name}"; + + conda env create -n "${env_name}" -f "${new_env_path}" --solver=libmamba; fi cp -a "${new_env_path}" "${old_env_path}"; + else + rm -f "${new_env_path}" "${old_env_path}"; + echo -e "Not creating '${env_name}' conda environment because '${env_file_name}' is empty." 1>&2; fi } diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index 1371967e..efb9d5f5 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -59,6 +59,29 @@ make_pip_dependencies() { if [ -f ~/"${!repo_path}/dependencies.yaml" ]; then + local cpp_length="${repo}_cpp_length"; + + for ((j=0; j < ${!cpp_length:-0}; j+=1)); do + local cpp_name="${repo}_cpp_${j}_name"; + + echo "Generating lib${!cpp_name}'s requirements.txt" 1>&2; + + local repo_keys=("${key[@]}" "${key[@]/%/_lib${!cpp_name//"-"/"_"}}"); + local keyi; + + for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do + local file="/tmp/${!repo_name}.lib${!cpp_name}.${repo_keys[$keyi]}.requirements.txt"; + pip_reqs_txts+=("${file}"); + generate_requirements \ + "${file}" \ + --file_key "${repo_keys[$keyi]}" \ + --output requirements \ + --config ~/"${!repo_path}/dependencies.yaml" \ + --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ + ; + done + done + local py_length="${repo}_python_length"; for ((j=0; j < ${!py_length:-0}; j+=1)); do