diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8ce60fe6f4..32131bce01 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -85,6 +85,12 @@ repos: hooks: - id: rapids-dependency-file-generator args: ["--clean"] + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.10.0.1 + hooks: + - id: shellcheck + args: ["--severity=warning"] + files: ^ci/ default_language_version: python: python3 diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 05f1f24ee5..984f220cc0 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -1,12 +1,13 @@ #!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. set -euo pipefail rapids-logger "Create test conda environment" . /opt/conda/etc/profile.d/conda.sh RAPIDS_VERSION="$(rapids-version)" -export RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" +RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" +export RAPIDS_VERSION_MAJOR_MINOR rapids-dependency-file-generator \ --output conda \ @@ -29,7 +30,8 @@ rapids-mamba-retry install \ "cuml=${RAPIDS_VERSION}" \ "libcuml=${RAPIDS_VERSION}" -export RAPIDS_DOCS_DIR="$(mktemp -d)" +RAPIDS_DOCS_DIR="$(mktemp -d)" +export RAPIDS_DOCS_DIR rapids-logger "Build CPP docs" pushd cpp diff --git a/ci/checks/black_lists.sh b/ci/checks/black_lists.sh index 6300ab359e..c98eb7a3ee 100755 --- a/ci/checks/black_lists.sh +++ b/ci/checks/black_lists.sh @@ -6,7 +6,7 @@ # PR_TARGET_BRANCH is set by the CI environment -git checkout --quiet $PR_TARGET_BRANCH +git checkout --quiet "$PR_TARGET_BRANCH" # Switch back to tip of PR branch git checkout --quiet current-pr-branch @@ -20,16 +20,16 @@ set +H RETVAL=0 for black_listed in cudaDeviceSynchronize cudaMalloc cudaMallocManaged cudaFree cudaMallocHost cudaHostAlloc cudaFreeHost; do - TMP=`git --no-pager diff --ignore-submodules -w --minimal -U0 -S"$black_listed" $PR_TARGET_BRANCH | grep '^+' | grep -v '^+++' | grep "$black_listed"` + TMP=$(git --no-pager diff --ignore-submodules -w --minimal -U0 -S"$black_listed" "$PR_TARGET_BRANCH" | grep '^+' | grep -v '^+++' | grep "$black_listed") if [ "$TMP" != "" ]; then - for filename in `git --no-pager diff --ignore-submodules -w --minimal --name-only -S"$black_listed" $PR_TARGET_BRANCH`; do + for filename in $(git --no-pager diff --ignore-submodules -w --minimal --name-only -S"$black_listed" "$PR_TARGET_BRANCH"); do basefilename=$(basename -- "$filename") filext="${basefilename##*.}" if [ "$filext" != "md" ] && [ "$filext" != "sh" ]; then - TMP2=`git --no-pager diff --ignore-submodules -w --minimal -U0 -S"$black_listed" $PR_TARGET_BRANCH -- $filename | grep '^+' | grep -v '^+++' | grep "$black_listed" | grep -vE "^\+[[:space:]]*/{2,}.*$black_listed"` + TMP2=$(git --no-pager diff --ignore-submodules -w --minimal -U0 -S"$black_listed" "$PR_TARGET_BRANCH" -- "$filename" | grep '^+' | grep -v '^+++' | grep "$black_listed" | grep -vE "^\+[[:space:]]*/{2,}.*$black_listed") if [ "$TMP2" != "" ]; then echo "=== ERROR: black listed function call $black_listed added to $filename ===" - git --no-pager diff --ignore-submodules -w --minimal -S"$black_listed" $PR_TARGET_BRANCH -- $filename + git --no-pager diff --ignore-submodules -w --minimal -S"$black_listed" "$PR_TARGET_BRANCH" -- "$filename" echo "=== END ERROR ===" RETVAL=1 fi @@ -39,17 +39,16 @@ for black_listed in cudaDeviceSynchronize cudaMalloc cudaMallocManaged cudaFree done for cond_black_listed in cudaMemcpy cudaMemset; do - TMP=`git --no-pager diff --ignore-submodules -w --minimal -U0 -S"$cond_black_listed" $PR_TARGET_BRANCH | grep '^+' | grep -v '^+++' | grep -P "$cond_black_listed(?!Async)"` - + TMP=$(git --no-pager diff --ignore-submodules -w --minimal -U0 -S"$cond_black_listed" "$PR_TARGET_BRANCH" | grep '^+' | grep -v '^+++' | grep -P "$cond_black_listed(?!Async)") if [ "$TMP" != "" ]; then - for filename in `git --no-pager diff --ignore-submodules -w --minimal --name-only -S"$cond_black_listed" $PR_TARGET_BRANCH`; do + for filename in $(git --no-pager diff --ignore-submodules -w --minimal --name-only -S"$cond_black_listed" "$PR_TARGET_BRANCH"); do basefilename=$(basename -- "$filename") filext="${basefilename##*.}" if [ "$filext" != "md" ] && [ "$filext" != "sh" ]; then - TMP2=`git --no-pager diff --ignore-submodules -w --minimal -U0 -S"$cond_black_listed" $PR_TARGET_BRANCH -- $filename | grep '^+' | grep -v '^+++' | grep -P "$cond_black_listed(?!Async)" | grep -vE "^\+[[:space:]]*/{2,}.*$cond_black_listed"` + TMP2=$(git --no-pager diff --ignore-submodules -w --minimal -U0 -S"$cond_black_listed" "$PR_TARGET_BRANCH" -- "$filename" | grep '^+' | grep -v '^+++' | grep -P "$cond_black_listed(?!Async)" | grep -vE "^\+[[:space:]]*/{2,}.*$cond_black_listed") if [ "$TMP2" != "" ]; then echo "=== ERROR: black listed function call $cond_black_listed added to $filename ===" - git --no-pager diff --ignore-submodules -w --minimal -S"$cond_black_listed" $PR_TARGET_BRANCH -- $filename + git --no-pager diff --ignore-submodules -w --minimal -S"$cond_black_listed" "$PR_TARGET_BRANCH" -- "$filename" echo "=== END ERROR ===" RETVAL=1 fi diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index ad7b7fd94d..80daa2bf15 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2019-2024, NVIDIA CORPORATION. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. ######################## # cuML Version Updater # ######################## @@ -13,14 +13,13 @@ NEXT_FULL_TAG=$1 # Get current version CURRENT_TAG=$(git tag --merged HEAD | grep -xE '^v.*' | sort --version-sort | tail -n 1 | tr -d 'v') -CURRENT_MAJOR=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[1]}') -CURRENT_MINOR=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[2]}') -CURRENT_PATCH=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[3]}') +CURRENT_MAJOR=$(echo "$CURRENT_TAG" | awk '{split($0, a, "."); print a[1]}') +CURRENT_MINOR=$(echo "$CURRENT_TAG" | awk '{split($0, a, "."); print a[2]}') CURRENT_SHORT_TAG=${CURRENT_MAJOR}.${CURRENT_MINOR} # Get . for next version -NEXT_MAJOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[1]}') -NEXT_MINOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[2]}') +NEXT_MAJOR=$(echo "$NEXT_FULL_TAG" | awk '{split($0, a, "."); print a[1]}') +NEXT_MINOR=$(echo "$NEXT_FULL_TAG" | awk '{split($0, a, "."); print a[2]}') NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR} # Need to distutils-normalize the original version @@ -30,7 +29,7 @@ echo "Preparing release $CURRENT_TAG => $NEXT_FULL_TAG" # Inplace sed replace; workaround for Linux and Mac function sed_runner() { - sed -i.bak ''"$1"'' $2 && rm -f ${2}.bak + sed -i.bak ''"$1"'' "$2" && rm -f "${2}".bak } diff --git a/ci/run_cuml_dask_pytests.sh b/ci/run_cuml_dask_pytests.sh index 9a28c6bbcc..3063e68414 100755 --- a/ci/run_cuml_dask_pytests.sh +++ b/ci/run_cuml_dask_pytests.sh @@ -1,7 +1,7 @@ #!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. # Support invoking run_cuml_dask_pytests.sh outside the script directory -cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/cuml/tests/dask +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/cuml/tests/dask || exit 1 python -m pytest --cache-clear "$@" . diff --git a/ci/run_cuml_integration_pytests.sh b/ci/run_cuml_integration_pytests.sh index bd74db7895..38572fb061 100755 --- a/ci/run_cuml_integration_pytests.sh +++ b/ci/run_cuml_integration_pytests.sh @@ -1,7 +1,7 @@ #!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. # Support invoking run_cuml_singlegpu_pytests.sh outside the script directory -cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/cuml/tests +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/cuml/tests || exit 1 python -m pytest -p cudf.pandas --cache-clear --ignore=dask -m "not memleak" "$@" --quick_run . diff --git a/ci/run_cuml_singlegpu_accel_pytests.sh b/ci/run_cuml_singlegpu_accel_pytests.sh index 3b13a53ef1..449cc0a098 100755 --- a/ci/run_cuml_singlegpu_accel_pytests.sh +++ b/ci/run_cuml_singlegpu_accel_pytests.sh @@ -1,7 +1,7 @@ #!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. # Support invoking run_cuml_singlegpu_pytests.sh outside the script directory -cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/cuml/tests/experimental/accel +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/cuml/tests/experimental/accel || exit 1 python -m pytest -p cuml.experimental.accel --cache-clear "$@" . diff --git a/ci/run_cuml_singlegpu_memleak_pytests.sh b/ci/run_cuml_singlegpu_memleak_pytests.sh index 89b56e2cb2..dfe2113a89 100755 --- a/ci/run_cuml_singlegpu_memleak_pytests.sh +++ b/ci/run_cuml_singlegpu_memleak_pytests.sh @@ -1,7 +1,7 @@ #!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. # Support invoking run_cuml_singlegpu_pytests.sh outside the script directory -cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/cuml/tests +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/cuml/tests || exit 1 python -m pytest --cache-clear --ignore=dask -m "memleak" "$@" . diff --git a/ci/run_cuml_singlegpu_pytests.sh b/ci/run_cuml_singlegpu_pytests.sh index a395baab1b..336e081d49 100755 --- a/ci/run_cuml_singlegpu_pytests.sh +++ b/ci/run_cuml_singlegpu_pytests.sh @@ -1,7 +1,7 @@ #!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. # Support invoking run_cuml_singlegpu_pytests.sh outside the script directory -cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/cuml/tests +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/cuml/tests || exit 1 python -m pytest --cache-clear --ignore=dask -m "not memleak" "$@" . diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index ea6d1cdc11..2aa92819f6 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. set -euo pipefail @@ -39,4 +39,4 @@ export GTEST_OUTPUT=xml:${RAPIDS_TESTS_DIR}/ ./ci/run_ctests.sh -j9 && EXITCODE=$? || EXITCODE=$?; rapids-logger "Test script exiting with value: $EXITCODE" -exit ${EXITCODE} +exit "${EXITCODE}" diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index d76a754b8b..e010aad666 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2020-2024, NVIDIA CORPORATION. +# Copyright (c) 2020-2025, NVIDIA CORPORATION. set -euo pipefail . /opt/conda/etc/profile.d/conda.sh @@ -40,17 +40,19 @@ SKIPNBS="cuml_benchmarks.ipynb" NBTEST="$(realpath "$(dirname "$0")/utils/nbtest.sh")" cd notebooks + +# shellcheck disable=SC2044 for nb in $(find . -name "*.ipynb"); do nbBasename=$(basename "${nb}") # Skip all NBs that use dask (in the code or even in their name) - if ((echo "${nb}" | grep -qi dask) || \ - (grep -q dask "${nb}")); then + if (echo "${nb}" | grep -qi dask) || \ + (grep -q dask "${nb}"); then echo "--------------------------------------------------------------------------------" echo "SKIPPING: ${nb} (suspected Dask usage, not currently automatable)" echo "--------------------------------------------------------------------------------" elif (echo " ${SKIPNBS} " | grep -q " ${nbBasename} "); then echo "--------------------------------------------------------------------------------" - echo "SKIPPING: "${nb}" (listed in skip list)" + echo "SKIPPING: ${nb} (listed in skip list)" echo "--------------------------------------------------------------------------------" else nvidia-smi diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 8027876005..e9d6f8daad 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -4,7 +4,7 @@ set -euo pipefail mkdir -p ./dist -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" RAPIDS_PY_WHEEL_NAME="cuml_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist RAPIDS_PY_WHEEL_NAME="libcuml_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist diff --git a/ci/utils/nbtest.sh b/ci/utils/nbtest.sh index 28877273bf..9fc37d2569 100755 --- a/ci/utils/nbtest.sh +++ b/ci/utils/nbtest.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2020-2021, NVIDIA CORPORATION. +# Copyright (c) 2020-2025, NVIDIA CORPORATION. MAGIC_OVERRIDE_CODE=" def my_run_line_magic(*args, **kwargs): @@ -24,22 +24,22 @@ get_ipython().run_cell_magic=my_run_cell_magic NO_COLORS=--colors=NoColor EXITCODE=0 NBTMPDIR="$WORKSPACE/tmp" -mkdir -p ${NBTMPDIR} +mkdir -p "${NBTMPDIR}" -for nb in $*; do - NBFILENAME=$1 +for nb in "$@"; do + NBFILENAME=$nb NBNAME=${NBFILENAME%.*} NBNAME=${NBNAME##*/} NBTESTSCRIPT=${NBTMPDIR}/${NBNAME}-test.py shift echo -------------------------------------------------------------------------------- - echo STARTING: ${NBNAME} + echo STARTING: "${NBNAME}" echo -------------------------------------------------------------------------------- - jupyter nbconvert --to script ${NBFILENAME} --output ${NBTMPDIR}/${NBNAME}-test - echo "${MAGIC_OVERRIDE_CODE}" > ${NBTMPDIR}/tmpfile - cat ${NBTESTSCRIPT} >> ${NBTMPDIR}/tmpfile - mv ${NBTMPDIR}/tmpfile ${NBTESTSCRIPT} + jupyter nbconvert --to script "${NBFILENAME}" --output "${NBTMPDIR}"/"${NBNAME}"-test + echo "${MAGIC_OVERRIDE_CODE}" > "${NBTMPDIR}"/tmpfile + cat "${NBTESTSCRIPT}" >> "${NBTMPDIR}"/tmpfile + mv "${NBTMPDIR}"/tmpfile "${NBTESTSCRIPT}" echo "Running \"ipython ${NO_COLORS} ${NBTESTSCRIPT}\" on $(date)" echo @@ -47,7 +47,7 @@ for nb in $*; do NBEXITCODE=$? echo EXIT CODE: ${NBEXITCODE} echo - EXITCODE=$((EXITCODE | ${NBEXITCODE})) + EXITCODE=$((EXITCODE | "${NBEXITCODE}")) done exit ${EXITCODE} diff --git a/ci/validate_wheel.sh b/ci/validate_wheel.sh index c4bd01faab..40bb27feeb 100755 --- a/ci/validate_wheel.sh +++ b/ci/validate_wheel.sh @@ -12,10 +12,10 @@ rapids-logger "validate packages with 'pydistcheck'" pydistcheck \ --inspect \ - "$(echo ${wheel_dir_relative_path}/*.whl)" + "$(echo "${wheel_dir_relative_path}"/*.whl)" rapids-logger "validate packages with 'twine'" twine check \ --strict \ - "$(echo ${wheel_dir_relative_path}/*.whl)" + "$(echo "${wheel_dir_relative_path}"/*.whl)"