Merged pull request #517 from rapidsai/branch-23.08

rapidsai · Aug 9, 2023 · 927a95d · 927a95d
2 parents 862c7d1 + 1eabf32
commit 927a95d
Show file tree

Hide file tree

Showing 191 changed files with 7,831 additions and 11,545 deletions.
diff --git a/.flake8 b/.flake8
@@ -0,0 +1,10 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.
+
+[flake8]
+filename = *.py,
+exclude = __init__.py, *.egg, build, docs, .git
+ignore =
+    # line break before binary operator
+    W503,
+    # whitespace before :
+    E203
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -28,7 +28,7 @@ concurrency:
 jobs:
   python-build:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.08
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -37,7 +37,7 @@ jobs:
   upload-conda:
     needs: [python-build]
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@branch-23.08
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -47,7 +47,7 @@ jobs:
     if: github.ref_type == 'branch'
     needs: python-build
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.08
     with:
       arch: "amd64"
       branch: ${{ inputs.branch }}
@@ -57,3 +57,23 @@ jobs:
       node_type: "gpu-v100-latest-1"
       run_script: "ci/build_docs.sh"
       sha: ${{ inputs.sha }}
+  wheel-build:
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      script: ci/build_wheel.sh
+      matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1")))
+  wheel-publish:
+    needs: wheel-build
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      package-name: cuxfilter
diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -17,28 +17,30 @@ jobs:
       - conda-python-tests
       - conda-notebook-tests
       - docs-build
+      - wheel-build
+      - wheel-tests
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.08
   checks:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@branch-23.08
   conda-python-build:
     needs: checks
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.08
     with:
       build_type: pull-request
   conda-python-tests:
     needs: conda-python-build
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.08
     with:
       build_type: pull-request
       run_codecov: false
   conda-notebook-tests:
     needs: conda-python-build
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.08
     with:
       build_type: pull-request
       node_type: "gpu-v100-latest-1"
@@ -48,10 +50,26 @@ jobs:
   docs-build:
     needs: conda-python-build
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.08
     with:
       build_type: pull-request
       node_type: "gpu-v100-latest-1"
       arch: "amd64"
       container_image: "rapidsai/ci:latest"
       run_script: "ci/build_docs.sh"
+  wheel-build:
+    needs: checks
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      script: ci/build_wheel.sh
+      matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1")))
+  wheel-tests:
+    needs: wheel-build
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      script: ci/test_wheel.sh
+      matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1")))
diff --git a/.github/workflows/test-external.yaml b/.github/workflows/test-external.yaml
@@ -17,13 +17,13 @@ on:
           Will be ignored if external-project=all
           "
         required: false
-    schedule:
-      - cron: "0 0 * * 1" # Run every Sunday evening Pacific Time (Monday 0:00 UTC)
+  schedule:
+    - cron: "0 0 * * 1" # Run every Sunday evening Pacific Time (Monday 0:00 UTC)
 
 jobs:
   test-external:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.08
     with:
       build_type: branch
       node_type: "gpu-v100-latest-1"

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -16,9 +16,19 @@ on:
 jobs:
   conda-python-tests:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.08
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
+  wheel-tests:
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/[email protected]
+    with:
+      build_type: nightly
+      branch: ${{ inputs.branch }}
+      date: ${{ inputs.date }}
+      sha: ${{ inputs.sha }}
+      script: ci/test_wheel.sh
+      matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1")))
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -2,7 +2,7 @@
 
 repos:
       - repo: https://github.com/psf/black
-        rev: 22.10.0
+        rev: 23.7.0
         hooks:
               - id: black
                 files: python/cuxfilter/.*
@@ -12,7 +12,12 @@ repos:
         rev: 6.0.0
         hooks:
               - id: flake8
-                args: ["--config=python/setup.cfg"]
+                args: ["--config=.flake8"]
                 files: python/.*$
+      - repo: https://github.com/rapidsai/dependency-file-generator
+        rev: v1.5.2
+        hooks:
+            - id: rapids-dependency-file-generator
+              args: ["--clean"]
 default_language_version:
       python: python3
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,34 @@
+# cuXfilter 23.08.00 (9 Aug 2023)
+
+## 🐛 Bug Fixes
+
+- Fix dependencies - pyproj ([#514](https://github.com/rapidsai/cuxfilter/pull/514)) [@AjayThorve](https://github.com/AjayThorve)
+- Fix nightly wheel testing workflow. ([#507](https://github.com/rapidsai/cuxfilter/pull/507)) [@bdice](https://github.com/bdice)
+- Fix broken symlink ([#502](https://github.com/rapidsai/cuxfilter/pull/502)) [@raydouglass](https://github.com/raydouglass)
+- Fix GHA: external dependencies tests ([#498](https://github.com/rapidsai/cuxfilter/pull/498)) [@AjayThorve](https://github.com/AjayThorve)
+- Fix scheduled GHA workflow issue ([#493](https://github.com/rapidsai/cuxfilter/pull/493)) [@AjayThorve](https://github.com/AjayThorve)
+- fix incorrect xy-coordinate issue in graphs ([#487](https://github.com/rapidsai/cuxfilter/pull/487)) [@AjayThorve](https://github.com/AjayThorve)
+
+## 📖 Documentation
+
+- Doc updates ([#516](https://github.com/rapidsai/cuxfilter/pull/516)) [@AjayThorve](https://github.com/AjayThorve)
+- Switch Docs to PyData Theme ([#500](https://github.com/rapidsai/cuxfilter/pull/500)) [@exactlyallan](https://github.com/exactlyallan)
+- Added visualization guide notebook ([#496](https://github.com/rapidsai/cuxfilter/pull/496)) [@exactlyallan](https://github.com/exactlyallan)
+- fix Auto-merge: Branch 23.08 merge 23.06 ([#482](https://github.com/rapidsai/cuxfilter/pull/482)) [@AjayThorve](https://github.com/AjayThorve)
+
+## 🛠️ Improvements
+
+- Switch to new wheels pipeline ([#506](https://github.com/rapidsai/cuxfilter/pull/506)) [@divyegala](https://github.com/divyegala)
+- Followup: Revert CUDA 12.0 CI workflows to branch-23.08 ([#504](https://github.com/rapidsai/cuxfilter/pull/504)) [@AjayThorve](https://github.com/AjayThorve)
+- Revert CUDA 12.0 CI workflows to branch-23.08. ([#503](https://github.com/rapidsai/cuxfilter/pull/503)) [@bdice](https://github.com/bdice)
+- cuxfilter: Build CUDA 12 packages ([#499](https://github.com/rapidsai/cuxfilter/pull/499)) [@AjayThorve](https://github.com/AjayThorve)
+- Add wheel builds to cuxfilter ([#497](https://github.com/rapidsai/cuxfilter/pull/497)) [@AjayThorve](https://github.com/AjayThorve)
+- Refactor to use holoviews powered bar charts ([#494](https://github.com/rapidsai/cuxfilter/pull/494)) [@AjayThorve](https://github.com/AjayThorve)
+- Improvement/add panel 1.0+, holoviews 1.16+, bokeh 3.1+ support ([#492](https://github.com/rapidsai/cuxfilter/pull/492)) [@AjayThorve](https://github.com/AjayThorve)
+- use rapids-upload-docs script ([#489](https://github.com/rapidsai/cuxfilter/pull/489)) [@AyodeAwe](https://github.com/AyodeAwe)
+- [Review] Remove Datatiles support ([#488](https://github.com/rapidsai/cuxfilter/pull/488)) [@AjayThorve](https://github.com/AjayThorve)
+- Remove documentation build scripts for Jenkins ([#483](https://github.com/rapidsai/cuxfilter/pull/483)) [@ajschmidt8](https://github.com/ajschmidt8)
+
 # cuXfilter 23.06.00 (7 Jun 2023)
 
 ## 🚨 Breaking Changes

diff --git a/README.md b/README.md
@@ -18,10 +18,6 @@ The current version of cuxfilter leverages jupyter notebook and bokeh server to
 
 ![layout architecture](./docs/_images/RAPIDS_cuxfilter.png)
 
-### What is cuDataTiles?
-
-cuxfilter implements cuDataTiles, a GPU accelerated version of data tiles based on the work of [Falcon](https://github.com/uwdata/falcon). When starting to interact with specific charts in a cuxfilter dashboard, values for the other charts are precomputed to allow for fast slider scrubbing without having to recalculate values.
-
 ### Open Source Projects
 
 cuxfilter wouldn’t be possible without using these great open source projects:
@@ -56,14 +52,15 @@ cux_df = cuxfilter.DataFrame.from_arrow(DATA_DIR+'/auto_accidents.arrow')
 cux_df.data['ST_CASE'] = cux_df.data['ST_CASE'].astype('float64')
 
 label_map = {1: 'Sunday',    2: 'Monday',    3: 'Tuesday',    4: 'Wednesday',   5: 'Thursday',    6: 'Friday',    7: 'Saturday',    9: 'Unknown'}
+cux_df.data['DAY_WEEK_STR'] = cux_df.data.DAY_WEEK.map(label_map)
 gtc_demo_red_blue_palette = [ "#3182bd", "#6baed6", "#7b8ed8", "#e26798", "#ff0068" , "#323232" ]
 
 #declare charts
 chart1 = cuxfilter.charts.scatter(x='dropoff_x', y='dropoff_y', aggregate_col='DAY_WEEK', aggregate_fn='mean',
                                 color_palette=gtc_demo_red_blue_palette, tile_provider='CartoLight', unselected_alpha=0.2,
                                 pixel_shade_type='linear')
 chart2 = cuxfilter.charts.multi_select('YEAR')
-chart3 = cuxfilter.charts.bar('DAY_WEEK', x_label_map=label_map)
+chart3 = cuxfilter.charts.bar('DAY_WEEK_STR')
 chart4 = cuxfilter.charts.bar('MONTH')
 
 #declare dashboard
@@ -96,26 +93,26 @@ datasets_check('mortgage', base_dir=DATA_DIR)
 
 cux_df = cuxfilter.DataFrame.from_arrow(DATA_DIR + '/146M_predictions_v2.arrow')
 
-MAPBOX_API_KEY= "<mapbox-api-key>"
 geoJSONSource='https://raw.githubusercontent.com/rapidsai/cuxfilter/GTC-2018-mortgage-visualization/javascript/demos/GTC%20demo/src/data/zip3-ms-rhs-lessprops.json'
 
 chart0 = cuxfilter.charts.choropleth( x='zip', color_column='delinquency_12_prediction', color_aggregate_fn='mean',
             elevation_column='current_actual_upb', elevation_factor=0.00001, elevation_aggregate_fn='sum',
-            geoJSONSource=geoJSONSource, mapbox_api_key=MAPBOX_API_KEY, data_points=1000
+            geoJSONSource=geoJSONSource, data_points=1000
 )
 chart2 = cuxfilter.charts.bar('delinquency_12_prediction',data_points=50)
 chart3 = cuxfilter.charts.range_slider('borrower_credit_score',data_points=50)
 chart1 = cuxfilter.charts.drop_down('dti')
 
 #declare dashboard
-d = cux_df.dashboard([chart0, chart2],sidebar=[chart3, chart1], layout=cuxfilter.layouts.feature_and_double_base,theme = cuxfilter.themes.light, title='Mortgage Dashboard')
+d = cux_df.dashboard([chart0, chart2],sidebar=[chart3, chart1], layout=cuxfilter.layouts.feature_and_double_base,theme = cuxfilter.themes.dark, title='Mortgage Dashboard')
 
 # run the dashboard within the notebook cell
 # Bokeh and Datashader based charts also have a `save` tool on the side toolbar, which can download and save the individual chart when interacting with the dashboard.
 # d.app()
 
 #run the dashboard as a webapp:
-d.show('jupyter-notebook/lab-url')
+# if running on a port other than localhost:8888, run d.show(jupyter-notebook-lab-url:port)
+d.show()
 
 ```
 
@@ -156,12 +153,12 @@ Please see the [Demo Docker Repository](https://hub.docker.com/r/rapidsai/rapids
 
 cuxfilter can be installed with conda ([miniconda](https://conda.io/miniconda.html), or the full [Anaconda distribution](https://www.anaconda.com/download)) from the `rapidsai` channel:
 
-For `cuxfilter version == 23.06` :
+For `cuxfilter version == 23.08` :
 
 ```bash
 # for CUDA 11.8
 conda install -c rapidsai -c numba -c conda-forge -c nvidia \
-    cuxfilter=23.06 python=3.10 cudatoolkit=11.8
+    cuxfilter=23.08 python=3.10 cudatoolkit=11.8
 ```
 
 For the nightly version of `cuxfilter` :
@@ -182,15 +179,6 @@ See [build instructions](CONTRIBUTING.md#setting-up-your-build-environment).
 
 ## Troubleshooting
 
-**libxcomposite.so.1 not found error**
-
-If the `await d.preview()` throws a libxcomposite.so.1 not found error, execute the following commands:
-
-```bash
-apt-get update
-apt-get install libxcomposite1 libxcursor1 libxdamage1 libxfixes3 libxi6 libxrandr2 libxtst6 libcups2 libxss1 libasound2 libpangocairo-1.0-0 libpango-1.0-0 libatk1.0-0 libgtk-3-0 libgdk-pixbuf2.0-0
-```
-
 **bokeh server in jupyter lab**
 
 To run the bokeh server in a jupyter lab, install jupyterlab dependencies
@@ -233,11 +221,11 @@ Currently supported layout templates and example code can be found on the [layou
 
 | Library       | Chart type                                                                                       |
 | ------------- | ------------------------------------------------------------------------------------------------ |
-| bokeh         | bar, line                                                                                        |
+| bokeh         | bar                                                                                              |
 | datashader    | scatter, scatter_geo, line, stacked_lines, heatmap, graph                                        |
 | panel_widgets | range_slider, date_range_slider, float_slider, int_slider, drop_down, multi_select, card, number |
 | custom        | view_dataframe                                                                                   |
-| pydeck        | choropleth(3d and 2d)                                                                            |
+| deckgl        | choropleth(3d and 2d)                                                                            |
 
 ## Contributing Developers Guide
 

diff --git a/ci/build_docs.sh b/ci/build_docs.sh
@@ -18,21 +18,21 @@ rapids-print-env
 rapids-logger "Downloading artifacts from previous jobs"
 
 PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
-VERSION_NUMBER="23.06"
 
 rapids-mamba-retry install \
     --channel "${PYTHON_CHANNEL}" \
     cuxfilter
 
-# Build Python docs
+export RAPIDS_VERSION_NUMBER="23.08"
+export RAPIDS_DOCS_DIR="$(mktemp -d)"
+
 rapids-logger "Build Python docs"
 pushd docs
 sphinx-build -b dirhtml ./source _html
 sphinx-build -b text ./source _text
+mkdir -p "${RAPIDS_DOCS_DIR}/cuxfilter/"{html,txt}
+mv _html/* "${RAPIDS_DOCS_DIR}/cuxfilter/html"
+mv _text/* "${RAPIDS_DOCS_DIR}/cuxfilter/txt"
 popd
 
-if [[ "${RAPIDS_BUILD_TYPE}" != "pull-request" ]]; then
-  rapids-logger "Upload Docs to S3"
-  aws s3 sync --no-progress --delete docs/_html "s3://rapidsai-docs/cuxfilter/${VERSION_NUMBER}/html"
-  aws s3 sync --no-progress --delete docs/_text "s3://rapidsai-docs/cuxfilter/${VERSION_NUMBER}/txt"
-fi
+rapids-upload-docs
diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+# Copyright (c) 2023, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+source rapids-configure-sccache
+source rapids-date-string
+
+# Use gha-tools rapids-pip-wheel-version to generate wheel version then
+# update the necessary files
+version_override="$(rapids-pip-wheel-version ${RAPIDS_DATE_STRING})"
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+
+ci/release/apply_wheel_modifications.sh ${version_override} "-${RAPIDS_PY_CUDA_SUFFIX}"
+echo "The package name and/or version was modified in the package source. The git diff is:"
+git diff
+
+cd python
+
+python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check
+
+RAPIDS_PY_WHEEL_NAME="cuxfilter_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 dist