From c03e5dbba34a741fae77e70eeac97d17a3861f05 Mon Sep 17 00:00:00 2001 From: Mark Harris Date: Thu, 9 Nov 2023 10:46:54 +0000 Subject: [PATCH] Depend on both RMM 1095 and the dependent cuDF build --- ci/build_cpp.sh | 20 ++++++++++++++++---- ci/build_docs.sh | 24 ++++++++++++++++++++++++ ci/build_python.sh | 32 ++++++++++++++++++++++++-------- ci/test_cpp.sh | 28 +++++++++++++++++++++------- ci/test_notebooks.sh | 23 ++++++++++++++++++----- ci/test_python.sh | 24 +++++++++++++++++++----- 6 files changed, 122 insertions(+), 29 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 7369c0c7e..62305c96f 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -7,11 +7,22 @@ source rapids-env-update export CMAKE_GENERATOR=Ninja -REPO="rmm" -PR_NUMBER="1095" -COMMIT=$(git ls-remote https://github.com/rapidsai/${REPO}.git refs/heads/pull-request/${PR_NUMBER} | cut -c1-7) RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" -LIBRMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) + +rapids_repo_pr_artifact_channel () { + local repo=$1 + local pr=$2 + local commit=$(git ls-remote https://github.com/rapidsai/${repo}.git refs/heads/pull-request/${pr} | cut -c1-7) + + if [[ $3 == "cpp" ]] then + echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) + else + echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) + fi +} + +LIBRMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 cpp) +LIBCUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 cpp) rapids-print-env @@ -21,6 +32,7 @@ rapids-logger "Begin cpp build" RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild \ --channel "${LIBRMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ conda/recipes/libcuspatial rapids-upload-conda-to-s3 cpp diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 427fb23bc..61c34aae5 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -20,9 +20,33 @@ rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) +RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" +PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g') + +rapids_repo_pr_artifact_channel () { + local repo=$1 + local pr=$2 + local commit=$(git ls-remote https://github.com/rapidsai/${repo}.git refs/heads/pull-request/${pr} | cut -c1-7) + + if [[ $3 == "cpp" ]] then + echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) + else + echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) + fi +} + +LIBRMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 cpp) +RMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 python) +LIBCUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 cpp) +CUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 python) + rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ libcuspatial \ cuspatial \ cuproj diff --git a/ci/build_python.sh b/ci/build_python.sh index c333f572a..6e20b2d10 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -15,19 +15,31 @@ version=$(rapids-generate-version) commit=$(git rev-parse HEAD) echo "${version}" > VERSION -for package_name in cuspatial cuproj; do +for package_name in cuspatial cuproj; do sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/${package_name}/_version.py" done -CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) - -REPO="rmm" -PR_NUMBER="1095" -COMMIT=$(git ls-remote https://github.com/rapidsai/${REPO}.git refs/heads/pull-request/${PR_NUMBER} | cut -c1-7) RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g') -LIBRMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) -RMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) + +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) + +rapids_repo_pr_artifact_channel () { + local repo=$1 + local pr=$2 + local commit=$(git ls-remote https://github.com/rapidsai/${repo}.git refs/heads/pull-request/${pr} | cut -c1-7) + + if [[ $3 == "cpp" ]] then + echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) + else + echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) + fi +} + +LIBRMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 cpp) +RMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 python) +LIBCUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 cpp) +CUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 python) rapids-logger "Begin py build cuSpatial" @@ -38,6 +50,8 @@ RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild \ --channel "${CPP_CHANNEL}" \ --channel "${RMM_CHANNEL}" \ --channel "${LIBRMM_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ conda/recipes/cuspatial rapids-logger "Begin py build cuProj" @@ -49,6 +63,8 @@ RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild \ --channel "${CPP_CHANNEL}" \ --channel "${RMM_CHANNEL}" \ --channel "${LIBRMM_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ conda/recipes/cuproj rapids-upload-conda-to-s3 python diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 7f48f4778..b7b6e3e5e 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -18,15 +18,27 @@ set +u conda activate test set -u -CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) - -REPO="rmm" -PR_NUMBER="1095" -COMMIT=$(git ls-remote https://github.com/rapidsai/${REPO}.git refs/heads/pull-request/${PR_NUMBER} | cut -c1-7) RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g') -LIBRMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) -RMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) + +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) + +rapids_repo_pr_artifact_channel () { + local repo=$1 + local pr=$2 + local commit=$(git ls-remote https://github.com/rapidsai/${repo}.git refs/heads/pull-request/${pr} | cut -c1-7) + + if [[ $3 == "cpp" ]] then + echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) + else + echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) + fi +} + +LIBRMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 cpp) +RMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 python) +LIBCUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 cpp) +CUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 python) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ mkdir -p "${RAPIDS_TESTS_DIR}" @@ -40,6 +52,8 @@ rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${LIBRMM_CHANNEL}" \ --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ libcuspatial libcuspatial-tests rapids-logger "Check GPU usage" diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index f7edcebb6..0f154f0bb 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -24,20 +24,33 @@ rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) -REPO="rmm" -PR_NUMBER="1095" -COMMIT=$(git ls-remote https://github.com/rapidsai/${REPO}.git refs/heads/pull-request/${PR_NUMBER} | cut -c1-7) RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g') -LIBRMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) -RMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) +rapids_repo_pr_artifact_channel () { + local repo=$1 + local pr=$2 + local commit=$(git ls-remote https://github.com/rapidsai/${repo}.git refs/heads/pull-request/${pr} | cut -c1-7) + + if [[ $3 == "cpp" ]] then + echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) + else + echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) + fi +} + +LIBRMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 cpp) +RMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 python) +LIBCUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 cpp) +CUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 python) rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ --channel "${LIBRMM_CHANNEL}" \ --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ cuspatial libcuspatial cuproj NBTEST="$(realpath "$(dirname "$0")/utils/nbtest.sh")" diff --git a/ci/test_python.sh b/ci/test_python.sh index 7a0e9c38d..fbd443907 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -22,13 +22,25 @@ rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) -REPO="rmm" -PR_NUMBER="1095" -COMMIT=$(git ls-remote https://github.com/rapidsai/${REPO}.git refs/heads/pull-request/${PR_NUMBER} | cut -c1-7) RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g') -LIBRMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) -RMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) + +rapids_repo_pr_artifact_channel () { + local repo=$1 + local pr=$2 + local commit=$(git ls-remote https://github.com/rapidsai/${repo}.git refs/heads/pull-request/${pr} | cut -c1-7) + + if [[ $3 == "cpp" ]] then + echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) + else + echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) + fi +} + +LIBRMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 cpp) +RMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 python) +LIBCUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 cpp) +CUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 python) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"} @@ -44,6 +56,8 @@ rapids-mamba-retry install \ --channel "${PYTHON_CHANNEL}" \ --channel "${LIBRMM_CHANNEL}" \ --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ libcuspatial cuspatial cuproj rapids-logger "Check GPU usage"