Depend on both RMM 1095 and the dependent cuDF build

rapidsai · Nov 9, 2023 · c03e5db · c03e5db
1 parent a6e6610
commit c03e5db
Show file tree

Hide file tree

Showing 6 changed files with 122 additions and 29 deletions.
diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh
@@ -7,11 +7,22 @@ source rapids-env-update
 
 export CMAKE_GENERATOR=Ninja
 
-REPO="rmm"
-PR_NUMBER="1095"
-COMMIT=$(git ls-remote https://github.com/rapidsai/${REPO}.git refs/heads/pull-request/${PR_NUMBER} | cut -c1-7)
 RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}"
-LIBRMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz)
+
+rapids_repo_pr_artifact_channel () {
+    local repo=$1
+    local pr=$2
+    local commit=$(git ls-remote https://github.com/rapidsai/${repo}.git refs/heads/pull-request/${pr} | cut -c1-7)
+
+    if [[ $3 == "cpp" ]] then
+        echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz)
+    else
+        echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz)
+    fi
+}
+
+LIBRMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 cpp)
+LIBCUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 cpp)
 
 rapids-print-env
 
@@ -21,6 +32,7 @@ rapids-logger "Begin cpp build"
 
 RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild \
     --channel "${LIBRMM_CHANNEL}" \
+    --channel "${LIBCUDF_CHANNEL}" \
     conda/recipes/libcuspatial
 
 rapids-upload-conda-to-s3 cpp
diff --git a/ci/build_docs.sh b/ci/build_docs.sh
@@ -20,9 +20,33 @@ rapids-logger "Downloading artifacts from previous jobs"
 CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
 
+RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}"
+PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g')
+
+rapids_repo_pr_artifact_channel () {
+    local repo=$1
+    local pr=$2
+    local commit=$(git ls-remote https://github.com/rapidsai/${repo}.git refs/heads/pull-request/${pr} | cut -c1-7)
+
+    if [[ $3 == "cpp" ]] then
+        echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz)
+    else
+        echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz)
+    fi
+}
+
+LIBRMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 cpp)
+RMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 python)
+LIBCUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 cpp)
+CUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 python)
+
 rapids-mamba-retry install \
   --channel "${CPP_CHANNEL}" \
   --channel "${PYTHON_CHANNEL}" \
+  --channel "${LIBRMM_CHANNEL}" \
+  --channel "${RMM_CHANNEL}" \
+  --channel "${LIBCUDF_CHANNEL}" \
+  --channel "${CUDF_CHANNEL}" \
   libcuspatial \
   cuspatial \
   cuproj

diff --git a/ci/build_python.sh b/ci/build_python.sh
@@ -15,19 +15,31 @@ version=$(rapids-generate-version)
 commit=$(git rev-parse HEAD)
 
 echo "${version}" > VERSION
-for package_name in cuspatial cuproj; do 
+for package_name in cuspatial cuproj; do
     sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/${package_name}/_version.py"
 done
 
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
-
-REPO="rmm"
-PR_NUMBER="1095"
-COMMIT=$(git ls-remote https://github.com/rapidsai/${REPO}.git refs/heads/pull-request/${PR_NUMBER} | cut -c1-7)
 RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}"
 PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g')
-LIBRMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz)
-RMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz)
+
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
+
+rapids_repo_pr_artifact_channel () {
+    local repo=$1
+    local pr=$2
+    local commit=$(git ls-remote https://github.com/rapidsai/${repo}.git refs/heads/pull-request/${pr} | cut -c1-7)
+
+    if [[ $3 == "cpp" ]] then
+        echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz)
+    else
+        echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz)
+    fi
+}
+
+LIBRMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 cpp)
+RMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 python)
+LIBCUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 cpp)
+CUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 python)
 
 rapids-logger "Begin py build cuSpatial"
 
@@ -38,6 +50,8 @@ RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild \
   --channel "${CPP_CHANNEL}" \
   --channel "${RMM_CHANNEL}" \
   --channel "${LIBRMM_CHANNEL}" \
+  --channel "${CUDF_CHANNEL}" \
+  --channel "${LIBCUDF_CHANNEL}" \
   conda/recipes/cuspatial
 
 rapids-logger "Begin py build cuProj"
@@ -49,6 +63,8 @@ RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild \
   --channel "${CPP_CHANNEL}" \
   --channel "${RMM_CHANNEL}" \
   --channel "${LIBRMM_CHANNEL}" \
+  --channel "${CUDF_CHANNEL}" \
+  --channel "${LIBCUDF_CHANNEL}" \
   conda/recipes/cuproj
 
 rapids-upload-conda-to-s3 python
diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh
@@ -18,15 +18,27 @@ set +u
 conda activate test
 set -u
 
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
-
-REPO="rmm"
-PR_NUMBER="1095"
-COMMIT=$(git ls-remote https://github.com/rapidsai/${REPO}.git refs/heads/pull-request/${PR_NUMBER} | cut -c1-7)
 RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}"
 PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g')
-LIBRMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz)
-RMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz)
+
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
+
+rapids_repo_pr_artifact_channel () {
+    local repo=$1
+    local pr=$2
+    local commit=$(git ls-remote https://github.com/rapidsai/${repo}.git refs/heads/pull-request/${pr} | cut -c1-7)
+
+    if [[ $3 == "cpp" ]] then
+        echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz)
+    else
+        echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz)
+    fi
+}
+
+LIBRMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 cpp)
+RMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 python)
+LIBCUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 cpp)
+CUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 python)
 
 RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/
 mkdir -p "${RAPIDS_TESTS_DIR}"
@@ -40,6 +52,8 @@ rapids-mamba-retry install \
   --channel "${CPP_CHANNEL}" \
   --channel "${LIBRMM_CHANNEL}" \
   --channel "${RMM_CHANNEL}" \
+  --channel "${LIBCUDF_CHANNEL}" \
+  --channel "${CUDF_CHANNEL}" \
   libcuspatial libcuspatial-tests
 
 rapids-logger "Check GPU usage"

diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh
@@ -24,20 +24,33 @@ rapids-logger "Downloading artifacts from previous jobs"
 CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
 
-REPO="rmm"
-PR_NUMBER="1095"
-COMMIT=$(git ls-remote https://github.com/rapidsai/${REPO}.git refs/heads/pull-request/${PR_NUMBER} | cut -c1-7)
 RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}"
 PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g')
-LIBRMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz)
-RMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz)
 
+rapids_repo_pr_artifact_channel () {
+    local repo=$1
+    local pr=$2
+    local commit=$(git ls-remote https://github.com/rapidsai/${repo}.git refs/heads/pull-request/${pr} | cut -c1-7)
+
+    if [[ $3 == "cpp" ]] then
+        echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz)
+    else
+        echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz)
+    fi
+}
+
+LIBRMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 cpp)
+RMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 python)
+LIBCUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 cpp)
+CUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 python)
 
 rapids-mamba-retry install \
   --channel "${CPP_CHANNEL}" \
   --channel "${PYTHON_CHANNEL}" \
   --channel "${LIBRMM_CHANNEL}" \
   --channel "${RMM_CHANNEL}" \
+  --channel "${LIBCUDF_CHANNEL}" \
+  --channel "${CUDF_CHANNEL}" \
   cuspatial libcuspatial cuproj
 
 NBTEST="$(realpath "$(dirname "$0")/utils/nbtest.sh")"

diff --git a/ci/test_python.sh b/ci/test_python.sh
@@ -22,13 +22,25 @@ rapids-logger "Downloading artifacts from previous jobs"
 CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
 
-REPO="rmm"
-PR_NUMBER="1095"
-COMMIT=$(git ls-remote https://github.com/rapidsai/${REPO}.git refs/heads/pull-request/${PR_NUMBER} | cut -c1-7)
 RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}"
 PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g')
-LIBRMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz)
-RMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz)
+
+rapids_repo_pr_artifact_channel () {
+    local repo=$1
+    local pr=$2
+    local commit=$(git ls-remote https://github.com/rapidsai/${repo}.git refs/heads/pull-request/${pr} | cut -c1-7)
+
+    if [[ $3 == "cpp" ]] then
+        echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz)
+    else
+        echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz)
+    fi
+}
+
+LIBRMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 cpp)
+RMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 python)
+LIBCUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 cpp)
+CUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 python)
 
 RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}
 RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"}
@@ -44,6 +56,8 @@ rapids-mamba-retry install \
   --channel "${PYTHON_CHANNEL}" \
   --channel "${LIBRMM_CHANNEL}" \
   --channel "${RMM_CHANNEL}" \
+  --channel "${LIBCUDF_CHANNEL}" \
+  --channel "${CUDF_CHANNEL}" \
   libcuspatial cuspatial cuproj
 
 rapids-logger "Check GPU usage"