From a7730b15ad0a95800fd49c47118be6514c03c714 Mon Sep 17 00:00:00 2001 From: Ray Douglass Date: Fri, 19 Jul 2024 17:26:24 -0400 Subject: [PATCH 01/79] DOC v24.10 Updates [skip ci] --- .../cuda11.8-conda/devcontainer.json | 6 ++-- .devcontainer/cuda11.8-pip/devcontainer.json | 10 +++---- .../cuda12.5-conda/devcontainer.json | 6 ++-- .devcontainer/cuda12.5-pip/devcontainer.json | 10 +++---- .github/workflows/build.yaml | 16 +++++----- .github/workflows/pr.yaml | 26 ++++++++-------- .github/workflows/test.yaml | 10 +++---- README.md | 2 +- VERSION | 2 +- .../all_cuda-118_arch-aarch64.yaml | 14 ++++----- .../all_cuda-118_arch-x86_64.yaml | 14 ++++----- .../all_cuda-125_arch-aarch64.yaml | 14 ++++----- .../all_cuda-125_arch-x86_64.yaml | 14 ++++----- .../bench_ann_cuda-118_arch-aarch64.yaml | 4 +-- .../bench_ann_cuda-118_arch-x86_64.yaml | 4 +-- .../bench_ann_cuda-120_arch-aarch64.yaml | 4 +-- .../bench_ann_cuda-120_arch-x86_64.yaml | 4 +-- .../recipes/raft-dask/conda_build_config.yaml | 4 +-- .../cmake/thirdparty/fetch_rapids.cmake | 2 +- dependencies.yaml | 30 +++++++++---------- docs/source/build.md | 2 +- docs/source/developer_guide.md | 6 ++-- docs/source/raft_ann_benchmarks.md | 12 ++++---- python/pylibraft/pyproject.toml | 4 +-- .../raft-dask/cmake/thirdparty/get_ucxx.cmake | 4 +-- python/raft-dask/pyproject.toml | 10 +++---- 26 files changed, 117 insertions(+), 117 deletions(-) diff --git a/.devcontainer/cuda11.8-conda/devcontainer.json b/.devcontainer/cuda11.8-conda/devcontainer.json index 27f619f391..a2b12764c5 100644 --- a/.devcontainer/cuda11.8-conda/devcontainer.json +++ b/.devcontainer/cuda11.8-conda/devcontainer.json @@ -5,17 +5,17 @@ "args": { "CUDA": "11.8", "PYTHON_PACKAGE_MANAGER": "conda", - "BASE": "rapidsai/devcontainers:24.08-cpp-cuda11.8-mambaforge-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.10-cpp-cuda11.8-mambaforge-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.08-cuda11.8-conda" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda11.8-conda" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.8": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.10": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json index db2a5dbcc6..0fdbac9d91 100644 --- a/.devcontainer/cuda11.8-pip/devcontainer.json +++ b/.devcontainer/cuda11.8-pip/devcontainer.json @@ -5,27 +5,27 @@ "args": { "CUDA": "11.8", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:24.08-cpp-cuda11.8-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.10-cpp-cuda11.8-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.08-cuda11.8-pip" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda11.8-pip" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/ucx:24.8": { + "ghcr.io/rapidsai/devcontainers/features/ucx:24.10": { "version": "1.15.0" }, - "ghcr.io/rapidsai/devcontainers/features/cuda:24.8": { + "ghcr.io/rapidsai/devcontainers/features/cuda:24.10": { "version": "11.8", "installcuBLAS": true, "installcuSOLVER": true, "installcuRAND": true, "installcuSPARSE": true }, - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.8": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.10": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/ucx", diff --git a/.devcontainer/cuda12.5-conda/devcontainer.json b/.devcontainer/cuda12.5-conda/devcontainer.json index 836a5feacd..adc79408a3 100644 --- a/.devcontainer/cuda12.5-conda/devcontainer.json +++ b/.devcontainer/cuda12.5-conda/devcontainer.json @@ -5,17 +5,17 @@ "args": { "CUDA": "12.5", "PYTHON_PACKAGE_MANAGER": "conda", - "BASE": "rapidsai/devcontainers:24.08-cpp-mambaforge-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.10-cpp-mambaforge-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.08-cuda12.5-conda" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda12.5-conda" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.8": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.10": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" diff --git a/.devcontainer/cuda12.5-pip/devcontainer.json b/.devcontainer/cuda12.5-pip/devcontainer.json index 28798cbbf5..39d1c0c969 100644 --- a/.devcontainer/cuda12.5-pip/devcontainer.json +++ b/.devcontainer/cuda12.5-pip/devcontainer.json @@ -5,27 +5,27 @@ "args": { "CUDA": "12.5", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:24.08-cpp-cuda12.5-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.10-cpp-cuda12.5-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.08-cuda12.5-pip" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda12.5-pip" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/ucx:24.8": { + "ghcr.io/rapidsai/devcontainers/features/ucx:24.10": { "version": "1.15.0" }, - "ghcr.io/rapidsai/devcontainers/features/cuda:24.8": { + "ghcr.io/rapidsai/devcontainers/features/cuda:24.10": { "version": "12.5", "installcuBLAS": true, "installcuSOLVER": true, "installcuRAND": true, "installcuSPARSE": true }, - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.8": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.10": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/ucx", diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e6f7043f82..2b0ae5099c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -37,7 +37,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -46,7 +46,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -57,7 +57,7 @@ jobs: if: github.ref_type == 'branch' needs: python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-pylibraft: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -79,7 +79,7 @@ jobs: wheel-publish-pylibraft: needs: wheel-build-pylibraft secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -88,7 +88,7 @@ jobs: package-name: pylibraft wheel-build-raft-dask: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -98,7 +98,7 @@ jobs: wheel-publish-raft-dask: needs: wheel-build-raft-dask secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e6c9604221..98cb724623 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -25,29 +25,29 @@ jobs: - wheel-tests-raft-dask - devcontainer secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 with: enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 with: build_type: pull-request node_type: cpu16 conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 with: build_type: pull-request conda-cpp-checks: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 with: build_type: pull-request enable_check_symbols: true @@ -55,19 +55,19 @@ jobs: conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 with: build_type: pull-request conda-python-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 with: build_type: pull-request docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -77,34 +77,34 @@ jobs: wheel-build-pylibraft: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: pull-request script: ci/build_wheel_pylibraft.sh wheel-tests-pylibraft: needs: wheel-build-pylibraft secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: pull-request script: ci/test_wheel_pylibraft.sh wheel-build-raft-dask: needs: wheel-tests-pylibraft secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: pull-request script: "ci/build_wheel_raft_dask.sh" wheel-tests-raft-dask: needs: wheel-build-raft-dask secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: pull-request script: ci/test_wheel_raft_dask.sh devcontainer: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10 with: arch: '["amd64"]' cuda: '["12.5"]' diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0eba0f27d1..f6ca417dcf 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -26,7 +26,7 @@ jobs: symbol_exclusions: _ZN\d+raft_cutlass conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -34,7 +34,7 @@ jobs: sha: ${{ inputs.sha }} conda-python-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -42,7 +42,7 @@ jobs: sha: ${{ inputs.sha }} wheel-tests-pylibraft: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -51,7 +51,7 @@ jobs: script: ci/test_wheel_pylibraft.sh wheel-tests-raft-dask: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} diff --git a/README.md b/README.md index d6f3ef9320..25ce059630 100755 --- a/README.md +++ b/README.md @@ -293,7 +293,7 @@ You can also install the conda packages individually using the `mamba` command a mamba install -c rapidsai -c conda-forge -c nvidia libraft libraft-headers cuda-version=12.5 ``` -If installing the C++ APIs please see [using libraft](https://docs.rapids.ai/api/raft/nightly/using_libraft/) for more information on using the pre-compiled shared library. You can also refer to the [example C++ template project](https://github.com/rapidsai/raft/tree/branch-24.08/cpp/template) for a ready-to-go CMake configuration that you can drop into your project and build against installed RAFT development artifacts above. +If installing the C++ APIs please see [using libraft](https://docs.rapids.ai/api/raft/nightly/using_libraft/) for more information on using the pre-compiled shared library. You can also refer to the [example C++ template project](https://github.com/rapidsai/raft/tree/branch-24.10/cpp/template) for a ready-to-go CMake configuration that you can drop into your project and build against installed RAFT development artifacts above. ### Installing Python through Pip diff --git a/VERSION b/VERSION index ec8489fda9..7c7ba04436 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -24.08.00 +24.10.00 diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index f2ae41822c..1a28edb974 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -20,8 +20,8 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-cuda==24.8.*,>=0.0.0a0 -- distributed-ucxx==0.39.*,>=0.0.0a0 +- dask-cuda==24.10.*,>=0.0.0a0 +- distributed-ucxx==0.40.*,>=0.0.0a0 - doxygen>=1.8.20 - gcc_linux-aarch64=11.* - graphviz @@ -35,7 +35,7 @@ dependencies: - libcusolver=11.4.1.48 - libcusparse-dev=11.7.5.86 - libcusparse=11.7.5.86 -- libucxx==0.39.*,>=0.0.0a0 +- libucxx==0.40.*,>=0.0.0a0 - nccl>=2.9.9 - ninja - numba>=0.57 @@ -44,18 +44,18 @@ dependencies: - nvcc_linux-aarch64=11.8 - pre-commit - pydata-sphinx-theme -- pylibraft==24.8.*,>=0.0.0a0 +- pylibraft==24.10.*,>=0.0.0a0 - pytest-cov - pytest==7.* - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rapids-dask-dependency==24.8.*,>=0.0.0a0 +- rapids-dask-dependency==24.10.*,>=0.0.0a0 - recommonmark -- rmm==24.8.*,>=0.0.0a0 +- rmm==24.10.*,>=0.0.0a0 - scikit-build-core>=0.7.0 - scikit-learn - scipy - sphinx-copybutton - sphinx-markdown-tables - sysroot_linux-aarch64==2.17 -- ucx-py==0.39.*,>=0.0.0a0 +- ucx-py==0.40.*,>=0.0.0a0 name: all_cuda-118_arch-aarch64 diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 8f5f4d3088..71a7fc4b02 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -20,8 +20,8 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-cuda==24.8.*,>=0.0.0a0 -- distributed-ucxx==0.39.*,>=0.0.0a0 +- dask-cuda==24.10.*,>=0.0.0a0 +- distributed-ucxx==0.40.*,>=0.0.0a0 - doxygen>=1.8.20 - gcc_linux-64=11.* - graphviz @@ -35,7 +35,7 @@ dependencies: - libcusolver=11.4.1.48 - libcusparse-dev=11.7.5.86 - libcusparse=11.7.5.86 -- libucxx==0.39.*,>=0.0.0a0 +- libucxx==0.40.*,>=0.0.0a0 - nccl>=2.9.9 - ninja - numba>=0.57 @@ -44,18 +44,18 @@ dependencies: - nvcc_linux-64=11.8 - pre-commit - pydata-sphinx-theme -- pylibraft==24.8.*,>=0.0.0a0 +- pylibraft==24.10.*,>=0.0.0a0 - pytest-cov - pytest==7.* - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rapids-dask-dependency==24.8.*,>=0.0.0a0 +- rapids-dask-dependency==24.10.*,>=0.0.0a0 - recommonmark -- rmm==24.8.*,>=0.0.0a0 +- rmm==24.10.*,>=0.0.0a0 - scikit-build-core>=0.7.0 - scikit-learn - scipy - sphinx-copybutton - sphinx-markdown-tables - sysroot_linux-64==2.17 -- ucx-py==0.39.*,>=0.0.0a0 +- ucx-py==0.40.*,>=0.0.0a0 name: all_cuda-118_arch-x86_64 diff --git a/conda/environments/all_cuda-125_arch-aarch64.yaml b/conda/environments/all_cuda-125_arch-aarch64.yaml index 2042156224..7bb4a40c05 100644 --- a/conda/environments/all_cuda-125_arch-aarch64.yaml +++ b/conda/environments/all_cuda-125_arch-aarch64.yaml @@ -21,8 +21,8 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-cuda==24.8.*,>=0.0.0a0 -- distributed-ucxx==0.39.*,>=0.0.0a0 +- dask-cuda==24.10.*,>=0.0.0a0 +- distributed-ucxx==0.40.*,>=0.0.0a0 - doxygen>=1.8.20 - gcc_linux-aarch64=11.* - graphviz @@ -32,7 +32,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libucxx==0.39.*,>=0.0.0a0 +- libucxx==0.40.*,>=0.0.0a0 - nccl>=2.9.9 - ninja - numba>=0.57 @@ -40,18 +40,18 @@ dependencies: - numpydoc - pre-commit - pydata-sphinx-theme -- pylibraft==24.8.*,>=0.0.0a0 +- pylibraft==24.10.*,>=0.0.0a0 - pytest-cov - pytest==7.* - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rapids-dask-dependency==24.8.*,>=0.0.0a0 +- rapids-dask-dependency==24.10.*,>=0.0.0a0 - recommonmark -- rmm==24.8.*,>=0.0.0a0 +- rmm==24.10.*,>=0.0.0a0 - scikit-build-core>=0.7.0 - scikit-learn - scipy - sphinx-copybutton - sphinx-markdown-tables - sysroot_linux-aarch64==2.17 -- ucx-py==0.39.*,>=0.0.0a0 +- ucx-py==0.40.*,>=0.0.0a0 name: all_cuda-125_arch-aarch64 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index a2586cc211..58153398f7 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -21,8 +21,8 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-cuda==24.8.*,>=0.0.0a0 -- distributed-ucxx==0.39.*,>=0.0.0a0 +- dask-cuda==24.10.*,>=0.0.0a0 +- distributed-ucxx==0.40.*,>=0.0.0a0 - doxygen>=1.8.20 - gcc_linux-64=11.* - graphviz @@ -32,7 +32,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libucxx==0.39.*,>=0.0.0a0 +- libucxx==0.40.*,>=0.0.0a0 - nccl>=2.9.9 - ninja - numba>=0.57 @@ -40,18 +40,18 @@ dependencies: - numpydoc - pre-commit - pydata-sphinx-theme -- pylibraft==24.8.*,>=0.0.0a0 +- pylibraft==24.10.*,>=0.0.0a0 - pytest-cov - pytest==7.* - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rapids-dask-dependency==24.8.*,>=0.0.0a0 +- rapids-dask-dependency==24.10.*,>=0.0.0a0 - recommonmark -- rmm==24.8.*,>=0.0.0a0 +- rmm==24.10.*,>=0.0.0a0 - scikit-build-core>=0.7.0 - scikit-learn - scipy - sphinx-copybutton - sphinx-markdown-tables - sysroot_linux-64==2.17 -- ucx-py==0.39.*,>=0.0.0a0 +- ucx-py==0.40.*,>=0.0.0a0 name: all_cuda-125_arch-x86_64 diff --git a/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml index 000a8f4a1c..7a961d1ca9 100644 --- a/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml +++ b/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml @@ -30,7 +30,7 @@ dependencies: - libcusolver=11.4.1.48 - libcusparse-dev=11.7.5.86 - libcusparse=11.7.5.86 -- libucxx==0.39.*,>=0.0.0a0 +- libucxx==0.40.*,>=0.0.0a0 - matplotlib - nccl>=2.9.9 - ninja @@ -40,7 +40,7 @@ dependencies: - pandas - pyyaml - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rmm==24.8.*,>=0.0.0a0 +- rmm==24.10.*,>=0.0.0a0 - scikit-build-core>=0.7.0 - sysroot_linux-aarch64==2.17 name: bench_ann_cuda-118_arch-aarch64 diff --git a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml index 52b3a8dc69..3e3f2511d8 100644 --- a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml +++ b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml @@ -30,7 +30,7 @@ dependencies: - libcusolver=11.4.1.48 - libcusparse-dev=11.7.5.86 - libcusparse=11.7.5.86 -- libucxx==0.39.*,>=0.0.0a0 +- libucxx==0.40.*,>=0.0.0a0 - matplotlib - nccl>=2.9.9 - ninja @@ -40,7 +40,7 @@ dependencies: - pandas - pyyaml - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rmm==24.8.*,>=0.0.0a0 +- rmm==24.10.*,>=0.0.0a0 - scikit-build-core>=0.7.0 - sysroot_linux-64==2.17 name: bench_ann_cuda-118_arch-x86_64 diff --git a/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml index 27baeda4b8..972cce6f88 100644 --- a/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml +++ b/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml @@ -27,7 +27,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libucxx==0.39.*,>=0.0.0a0 +- libucxx==0.40.*,>=0.0.0a0 - matplotlib - nccl>=2.9.9 - ninja @@ -36,7 +36,7 @@ dependencies: - pandas - pyyaml - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rmm==24.8.*,>=0.0.0a0 +- rmm==24.10.*,>=0.0.0a0 - scikit-build-core>=0.7.0 - sysroot_linux-aarch64==2.17 name: bench_ann_cuda-120_arch-aarch64 diff --git a/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml index 5274d56bf6..1ecae4658b 100644 --- a/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml +++ b/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml @@ -27,7 +27,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libucxx==0.39.*,>=0.0.0a0 +- libucxx==0.40.*,>=0.0.0a0 - matplotlib - nccl>=2.9.9 - ninja @@ -36,7 +36,7 @@ dependencies: - pandas - pyyaml - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rmm==24.8.*,>=0.0.0a0 +- rmm==24.10.*,>=0.0.0a0 - scikit-build-core>=0.7.0 - sysroot_linux-64==2.17 name: bench_ann_cuda-120_arch-x86_64 diff --git a/conda/recipes/raft-dask/conda_build_config.yaml b/conda/recipes/raft-dask/conda_build_config.yaml index decd1fad18..e6afed2890 100644 --- a/conda/recipes/raft-dask/conda_build_config.yaml +++ b/conda/recipes/raft-dask/conda_build_config.yaml @@ -17,10 +17,10 @@ c_stdlib_version: - "2.17" ucx_py_version: - - "0.39.*" + - "0.40.*" ucxx_version: - - "0.39.*" + - "0.40.*" cmake_version: - ">=3.26.4,!=3.30.0" diff --git a/cpp/template/cmake/thirdparty/fetch_rapids.cmake b/cpp/template/cmake/thirdparty/fetch_rapids.cmake index 0f1d5ff020..f64a924cf5 100644 --- a/cpp/template/cmake/thirdparty/fetch_rapids.cmake +++ b/cpp/template/cmake/thirdparty/fetch_rapids.cmake @@ -12,7 +12,7 @@ # the License. # Use this variable to update RAPIDS and RAFT versions -set(RAPIDS_VERSION "24.08") +set(RAPIDS_VERSION "24.10") if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/RAFT_RAPIDS.cmake) file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-${RAPIDS_VERSION}/RAPIDS.cmake diff --git a/dependencies.yaml b/dependencies.yaml index b16d37f4bb..c04d585e3c 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -170,7 +170,7 @@ dependencies: - c-compiler - cxx-compiler - nccl>=2.9.9 - - libucxx==0.39.*,>=0.0.0a0 + - libucxx==0.40.*,>=0.0.0a0 specific: - output_types: conda matrices: @@ -209,7 +209,7 @@ dependencies: common: - output_types: [conda] packages: - - &rmm_conda rmm==24.8.*,>=0.0.0a0 + - &rmm_conda rmm==24.10.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -234,10 +234,10 @@ dependencies: matrices: - matrix: {cuda: "12.*"} packages: - - &rmm_cu12 rmm-cu12==24.8.*,>=0.0.0a0 + - &rmm_cu12 rmm-cu12==24.10.*,>=0.0.0a0 - matrix: {cuda: "11.*"} packages: - - &rmm_cu11 rmm-cu11==24.8.*,>=0.0.0a0 + - &rmm_cu11 rmm-cu11==24.10.*,>=0.0.0a0 - {matrix: null, packages: [*rmm_conda] } checks: common: @@ -475,15 +475,15 @@ dependencies: common: - output_types: [conda, pyproject] packages: - - dask-cuda==24.8.*,>=0.0.0a0 + - dask-cuda==24.10.*,>=0.0.0a0 - joblib>=0.11 - numba>=0.57 - *numpy - - rapids-dask-dependency==24.8.*,>=0.0.0a0 + - rapids-dask-dependency==24.10.*,>=0.0.0a0 - output_types: conda packages: - - &pylibraft_conda pylibraft==24.8.*,>=0.0.0a0 - - &ucx_py_conda ucx-py==0.39.*,>=0.0.0a0 + - &pylibraft_conda pylibraft==24.10.*,>=0.0.0a0 + - &ucx_py_conda ucx-py==0.40.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -495,12 +495,12 @@ dependencies: matrices: - matrix: {cuda: "12.*"} packages: - - &pylibraft_cu12 pylibraft-cu12==24.8.*,>=0.0.0a0 - - &ucx_py_cu12 ucx-py-cu12==0.39.*,>=0.0.0a0 + - &pylibraft_cu12 pylibraft-cu12==24.10.*,>=0.0.0a0 + - &ucx_py_cu12 ucx-py-cu12==0.40.*,>=0.0.0a0 - matrix: {cuda: "11.*"} packages: - - &pylibraft_cu11 pylibraft-cu11==24.8.*,>=0.0.0a0 - - &ucx_py_cu11 ucx-py-cu11==0.39.*,>=0.0.0a0 + - &pylibraft_cu11 pylibraft-cu11==24.10.*,>=0.0.0a0 + - &ucx_py_cu11 ucx-py-cu11==0.40.*,>=0.0.0a0 - {matrix: null, packages: [*pylibraft_conda, *ucx_py_conda]} test_python_common: common: @@ -520,7 +520,7 @@ dependencies: packages: # UCXX is not currently a hard-dependency thus only installed during tests, # this will change in the future. - - &distributed_ucxx_conda distributed-ucxx==0.39.*,>=0.0.0a0 + - &distributed_ucxx_conda distributed-ucxx==0.40.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -531,10 +531,10 @@ dependencies: matrices: - matrix: {cuda: "12.*"} packages: - - distributed-ucxx-cu12==0.39.*,>=0.0.0a0 + - distributed-ucxx-cu12==0.40.*,>=0.0.0a0 - matrix: {cuda: "11.*"} packages: - - distributed-ucxx-cu11==0.39.*,>=0.0.0a0 + - distributed-ucxx-cu11==0.40.*,>=0.0.0a0 - {matrix: null, packages: [*distributed_ucxx_conda]} depends_on_ucx_build: common: diff --git a/docs/source/build.md b/docs/source/build.md index 64f3bd01a2..4ba087e68d 100644 --- a/docs/source/build.md +++ b/docs/source/build.md @@ -56,7 +56,7 @@ You can also install the conda packages individually using the `mamba` command a mamba install -c rapidsai -c conda-forge -c nvidia libraft libraft-headers cuda-version=12.0 ``` -If installing the C++ APIs Please see [using libraft](https://docs.rapids.ai/api/raft/nightly/using_libraft/) for more information on using the pre-compiled shared library. You can also refer to the [example C++ template project](https://github.com/rapidsai/raft/tree/branch-24.08/cpp/template) for a ready-to-go CMake configuration that you can drop into your project and build against installed RAFT development artifacts above. +If installing the C++ APIs Please see [using libraft](https://docs.rapids.ai/api/raft/nightly/using_libraft/) for more information on using the pre-compiled shared library. You can also refer to the [example C++ template project](https://github.com/rapidsai/raft/tree/branch-24.10/cpp/template) for a ready-to-go CMake configuration that you can drop into your project and build against installed RAFT development artifacts above. ## Installing Python through Pip diff --git a/docs/source/developer_guide.md b/docs/source/developer_guide.md index 36588f3450..516819b1c1 100644 --- a/docs/source/developer_guide.md +++ b/docs/source/developer_guide.md @@ -187,7 +187,7 @@ RAFT relies on `clang-format` to enforce code style across all C++ and CUDA sour 1. Do not split empty functions/records/namespaces. 2. Two-space indentation everywhere, including the line continuations. 3. Disable reflowing of comments. - The reasons behind these deviations from the Google style guide are given in comments [here](https://github.com/rapidsai/raft/blob/branch-24.08/cpp/.clang-format). + The reasons behind these deviations from the Google style guide are given in comments [here](https://github.com/rapidsai/raft/blob/branch-24.10/cpp/.clang-format). [`doxygen`](https://doxygen.nl/) is used as documentation generator and also as a documentation linter. In order to run doxygen as a linter on C++/CUDA code, run @@ -205,7 +205,7 @@ you can run `codespell -i 3 -w .` from the repository root directory. This will bring up an interactive prompt to select which spelling fixes to apply. ### #include style -[include_checker.py](https://github.com/rapidsai/raft/blob/branch-24.08/cpp/scripts/include_checker.py) is used to enforce the include style as follows: +[include_checker.py](https://github.com/rapidsai/raft/blob/branch-24.10/cpp/scripts/include_checker.py) is used to enforce the include style as follows: 1. `#include "..."` should be used for referencing local files only. It is acceptable to be used for referencing files in a sub-folder/parent-folder of the same algorithm, but should never be used to include files in other algorithms or between algorithms and the primitives or other dependencies. 2. `#include <...>` should be used for referencing everything else @@ -230,7 +230,7 @@ Call CUDA APIs via the provided helper macros `RAFT_CUDA_TRY`, `RAFT_CUBLAS_TRY` ## Logging ### Introduction -Anything and everything about logging is defined inside [logger.hpp](https://github.com/rapidsai/raft/blob/branch-24.08/cpp/include/raft/core/logger.hpp). It uses [spdlog](https://github.com/gabime/spdlog) underneath, but this information is transparent to all. +Anything and everything about logging is defined inside [logger.hpp](https://github.com/rapidsai/raft/blob/branch-24.10/cpp/include/raft/core/logger.hpp). It uses [spdlog](https://github.com/gabime/spdlog) underneath, but this information is transparent to all. ### Usage ```cpp diff --git a/docs/source/raft_ann_benchmarks.md b/docs/source/raft_ann_benchmarks.md index 21a8404212..fc11a56ac8 100644 --- a/docs/source/raft_ann_benchmarks.md +++ b/docs/source/raft_ann_benchmarks.md @@ -62,7 +62,7 @@ Nightly images are located in [dockerhub](https://hub.docker.com/r/rapidsai/raft - The following command pulls the nightly container for python version 10, cuda version 12, and RAFT version 23.10: ```bash -docker pull rapidsai/raft-ann-bench:24.08a-cuda12.0-py3.10 #substitute raft-ann-bench for the exact desired container. +docker pull rapidsai/raft-ann-bench:24.10a-cuda12.0-py3.10 #substitute raft-ann-bench for the exact desired container. ``` The CUDA and python versions can be changed for the supported values: @@ -83,7 +83,7 @@ You can see the exact versions as well in the dockerhub site: [//]: # () [//]: # (```bash) -[//]: # (docker pull nvcr.io/nvidia/rapidsai/raft-ann-bench:24.08-cuda11.8-py3.10 #substitute raft-ann-bench for the exact desired container.) +[//]: # (docker pull nvcr.io/nvidia/rapidsai/raft-ann-bench:24.10-cuda11.8-py3.10 #substitute raft-ann-bench for the exact desired container.) [//]: # (```) @@ -344,7 +344,7 @@ For GPU-enabled systems, the `DATA_FOLDER` variable should be a local folder whe export DATA_FOLDER=path/to/store/datasets/and/results docker run --gpus all --rm -it -u $(id -u) \ -v $DATA_FOLDER:/data/benchmarks \ - rapidsai/raft-ann-bench:24.08a-cuda11.8-py3.10 \ + rapidsai/raft-ann-bench:24.10a-cuda11.8-py3.10 \ "--dataset deep-image-96-angular" \ "--normalize" \ "--algorithms raft_cagra,raft_ivf_pq --batch-size 10 -k 10" \ @@ -355,7 +355,7 @@ Usage of the above command is as follows: | Argument | Description | |-----------------------------------------------------------|----------------------------------------------------------------------------------------------------| -| `rapidsai/raft-ann-bench:24.08a-cuda11.8-py3.10` | Image to use. Can be either `raft-ann-bench` or `raft-ann-bench-datasets` | +| `rapidsai/raft-ann-bench:24.10a-cuda11.8-py3.10` | Image to use. Can be either `raft-ann-bench` or `raft-ann-bench-datasets` | | `"--dataset deep-image-96-angular"` | Dataset name | | `"--normalize"` | Whether to normalize the dataset | | `"--algorithms raft_cagra,hnswlib --batch-size 10 -k 10"` | Arguments passed to the `run` script, such as the algorithms to benchmark, the batch size, and `k` | @@ -372,7 +372,7 @@ The container arguments in the above section also be used for the CPU-only conta export DATA_FOLDER=path/to/store/datasets/and/results docker run --rm -it -u $(id -u) \ -v $DATA_FOLDER:/data/benchmarks \ - rapidsai/raft-ann-bench-cpu:24.08a-py3.10 \ + rapidsai/raft-ann-bench-cpu:24.10a-py3.10 \ "--dataset deep-image-96-angular" \ "--normalize" \ "--algorithms hnswlib --batch-size 10 -k 10" \ @@ -389,7 +389,7 @@ docker run --gpus all --rm -it -u $(id -u) \ --entrypoint /bin/bash \ --workdir /data/benchmarks \ -v $DATA_FOLDER:/data/benchmarks \ - rapidsai/raft-ann-bench:24.08a-cuda11.8-py3.10 + rapidsai/raft-ann-bench:24.10a-cuda11.8-py3.10 ``` This will drop you into a command line in the container, with the `raft-ann-bench` python package ready to use, as described in the [Running the benchmarks](#running-the-benchmarks) section above: diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index 1329a3990b..436fdf9500 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -33,7 +33,7 @@ requires-python = ">=3.9" dependencies = [ "cuda-python", "numpy>=1.23,<2.0a0", - "rmm==24.8.*,>=0.0.0a0", + "rmm==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", @@ -120,7 +120,7 @@ requires = [ "cuda-python", "cython>=3.0.0", "ninja", - "rmm==24.8.*,>=0.0.0a0", + "rmm==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. dependencies-file = "../../dependencies.yaml" diff --git a/python/raft-dask/cmake/thirdparty/get_ucxx.cmake b/python/raft-dask/cmake/thirdparty/get_ucxx.cmake index de6a4b109c..db6039393a 100644 --- a/python/raft-dask/cmake/thirdparty/get_ucxx.cmake +++ b/python/raft-dask/cmake/thirdparty/get_ucxx.cmake @@ -47,9 +47,9 @@ endfunction() # Change pinned tag here to test a commit in CI # To use a different RAFT locally, set the CMake variable # CPM_raft_SOURCE=/path/to/local/raft -find_and_configure_ucxx(VERSION 0.39 +find_and_configure_ucxx(VERSION 0.40 FORK rapidsai - PINNED_TAG branch-0.39 + PINNED_TAG branch-0.40 EXCLUDE_FROM_ALL YES UCXX_STATIC ${RAFT_DASK_UCXX_STATIC} ) diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index 24ee27ddea..4b0ae06316 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -31,14 +31,14 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ - "dask-cuda==24.8.*,>=0.0.0a0", - "distributed-ucxx==0.39.*,>=0.0.0a0", + "dask-cuda==24.10.*,>=0.0.0a0", + "distributed-ucxx==0.40.*,>=0.0.0a0", "joblib>=0.11", "numba>=0.57", "numpy>=1.23,<2.0a0", - "pylibraft==24.8.*,>=0.0.0a0", - "rapids-dask-dependency==24.8.*,>=0.0.0a0", - "ucx-py==0.39.*,>=0.0.0a0", + "pylibraft==24.10.*,>=0.0.0a0", + "rapids-dask-dependency==24.10.*,>=0.0.0a0", + "ucx-py==0.40.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", From 09f97c3ac60fddd2f9729f9e624d9bc92fee3c8f Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 24 Jul 2024 20:00:30 -0500 Subject: [PATCH 02/79] split up CUDA-suffixed dependencies in dependencies.yaml (#2388) Contributes to https://github.com/rapidsai/build-planning/issues/31 In short, RAPIDS DLFW builds want to produce wheels with unsuffixed dependencies, e.g. `cudf` depending on `rmm`, not `rmm-cu12`. This PR is part of a series across all of RAPIDS to try to support that type of build by setting up CUDA-suffixed and CUDA-unsuffixed dependency lists in `dependencies.yaml`. For more details, see: * https://github.com/rapidsai/build-planning/issues/31#issuecomment-2245815818 * https://github.com/rapidsai/cudf/pull/16183 ## Notes for Reviewers ### Why target 24.08? This is targeting 24.08 because: 1. it should be very low-risk 2. getting these changes into 24.08 prevents the need to carry around patches for every library in DLFW builds using RAPIDS 24.08 Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/raft/pull/2388 --- dependencies.yaml | 64 ++++++++++++++++++---------- python/pylibraft/pyproject.toml | 1 + python/raft-ann-bench/pyproject.toml | 1 + python/raft-dask/pyproject.toml | 1 + 4 files changed, 45 insertions(+), 22 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index b16d37f4bb..e1cc919d83 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -209,7 +209,7 @@ dependencies: common: - output_types: [conda] packages: - - &rmm_conda rmm==24.8.*,>=0.0.0a0 + - &rmm_unsuffixed rmm==24.8.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -232,13 +232,17 @@ dependencies: - &cuda_python cuda-python - output_types: [requirements, pyproject] matrices: - - matrix: {cuda: "12.*"} + - matrix: + cuda: "12.*" + cuda_suffixed: "true" packages: - &rmm_cu12 rmm-cu12==24.8.*,>=0.0.0a0 - - matrix: {cuda: "11.*"} + - matrix: + cuda: "11.*" + cuda_suffixed: "true" packages: - &rmm_cu11 rmm-cu11==24.8.*,>=0.0.0a0 - - {matrix: null, packages: [*rmm_conda] } + - {matrix: null, packages: [*rmm_unsuffixed] } checks: common: - output_types: [conda, requirements] @@ -260,7 +264,7 @@ dependencies: - h5py>=3.8.0 - benchmark>=1.8.2 - openblas - - *rmm_conda + - *rmm_unsuffixed nn_bench_python: common: - output_types: [conda] @@ -441,7 +445,7 @@ dependencies: - &numpy numpy>=1.23,<2.0a0 - output_types: [conda] packages: - - *rmm_conda + - *rmm_unsuffixed - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -470,7 +474,7 @@ dependencies: - matrix: {cuda: "11.*"} packages: - *rmm_cu11 - - {matrix: null, packages: [*rmm_conda]} + - {matrix: null, packages: [*rmm_unsuffixed]} run_raft_dask: common: - output_types: [conda, pyproject] @@ -482,8 +486,8 @@ dependencies: - rapids-dask-dependency==24.8.*,>=0.0.0a0 - output_types: conda packages: - - &pylibraft_conda pylibraft==24.8.*,>=0.0.0a0 - - &ucx_py_conda ucx-py==0.39.*,>=0.0.0a0 + - &pylibraft_unsuffixed pylibraft==24.8.*,>=0.0.0a0 + - &ucx_py_unsuffixed ucx-py==0.39.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -493,15 +497,19 @@ dependencies: specific: - output_types: [requirements, pyproject] matrices: - - matrix: {cuda: "12.*"} + - matrix: + cuda: "12.*" + cuda_suffixed: "true" packages: - &pylibraft_cu12 pylibraft-cu12==24.8.*,>=0.0.0a0 - &ucx_py_cu12 ucx-py-cu12==0.39.*,>=0.0.0a0 - - matrix: {cuda: "11.*"} + - matrix: + cuda: "11.*" + cuda_suffixed: "true" packages: - &pylibraft_cu11 pylibraft-cu11==24.8.*,>=0.0.0a0 - &ucx_py_cu11 ucx-py-cu11==0.39.*,>=0.0.0a0 - - {matrix: null, packages: [*pylibraft_conda, *ucx_py_conda]} + - {matrix: null, packages: [*pylibraft_unsuffixed, *ucx_py_unsuffixed]} test_python_common: common: - output_types: [conda, requirements, pyproject] @@ -520,7 +528,7 @@ dependencies: packages: # UCXX is not currently a hard-dependency thus only installed during tests, # this will change in the future. - - &distributed_ucxx_conda distributed-ucxx==0.39.*,>=0.0.0a0 + - &distributed_ucxx_unsuffixed distributed-ucxx==0.39.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -529,18 +537,22 @@ dependencies: specific: - output_types: [requirements, pyproject] matrices: - - matrix: {cuda: "12.*"} + - matrix: + cuda: "12.*" + cuda_suffixed: "true" packages: - distributed-ucxx-cu12==0.39.*,>=0.0.0a0 - - matrix: {cuda: "11.*"} + - matrix: + cuda: "11.*" + cuda_suffixed: "true" packages: - distributed-ucxx-cu11==0.39.*,>=0.0.0a0 - - {matrix: null, packages: [*distributed_ucxx_conda]} + - {matrix: null, packages: [*distributed_ucxx_unsuffixed]} depends_on_ucx_build: common: - output_types: conda packages: - - &ucx_conda_build ucx==1.15.0 + - ucx==1.15.0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -549,10 +561,14 @@ dependencies: specific: - output_types: [requirements, pyproject] matrices: - - matrix: {cuda: "12.*"} + - matrix: + cuda: "12.*" + cuda_suffixed: "true" packages: - libucx-cu12==1.15.0 - - matrix: {cuda: "11.*"} + - matrix: + cuda: "11.*" + cuda_suffixed: "true" packages: - libucx-cu11==1.15.0 - matrix: null @@ -562,7 +578,7 @@ dependencies: common: - output_types: conda packages: - - &ucx_conda_run ucx>=1.15.0 + - ucx>=1.15.0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -571,10 +587,14 @@ dependencies: specific: - output_types: [requirements, pyproject] matrices: - - matrix: {cuda: "12.*"} + - matrix: + cuda: "12.*" + cuda_suffixed: "true" packages: - libucx-cu12>=1.15.0 - - matrix: {cuda: "11.*"} + - matrix: + cuda: "11.*" + cuda_suffixed: "true" packages: - libucx-cu11>=1.15.0 - matrix: null diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index 1329a3990b..e32cf5f902 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -123,6 +123,7 @@ requires = [ "rmm==24.8.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true" [tool.pytest.ini_options] filterwarnings = [ diff --git a/python/raft-ann-bench/pyproject.toml b/python/raft-ann-bench/pyproject.toml index 226dc41e40..d22dd567fe 100644 --- a/python/raft-ann-bench/pyproject.toml +++ b/python/raft-ann-bench/pyproject.toml @@ -68,3 +68,4 @@ build-backend = "setuptools.build_meta" requires = [] dependencies-file = "../../dependencies.yaml" commit-files = ["src/raft_ann_bench/GIT_COMMIT"] +matrix-entry = "cuda_suffixed=true" diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index 24ee27ddea..26a1e9ffe2 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -125,3 +125,4 @@ requires = [ "ninja", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true" From 7004a8e2ea5800339b5f68717777dc70fc7a5782 Mon Sep 17 00:00:00 2001 From: Micka Date: Thu, 25 Jul 2024 16:21:56 +0200 Subject: [PATCH 03/79] Add workaround for syevd in CUDA 12.0 (#2332) Bug 4580093 of cusolver is causing an issue with `cusolverDnXsyevd`. This bug has been seen in https://github.com/rapidsai/cuml/issues/5555 and impact PCA and Linear Regression with CUDA 12.0+. Setting the stream to a different value than `cudaStreamPerThread` seems to solve it as a workaround. Authors: - Micka (https://github.com/lowener) Approvers: - Tamas Bela Feher (https://github.com/tfeher) - Artem M. Chirkin (https://github.com/achirkin) - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/raft/pull/2332 --- cpp/include/raft/linalg/detail/eig.cuh | 33 +++++++++++++++++++------- cpp/test/linalg/eig.cu | 18 ++++++++++++++ 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/cpp/include/raft/linalg/detail/eig.cuh b/cpp/include/raft/linalg/detail/eig.cuh index 2a4cfd52ec..ba7ed3dcdf 100644 --- a/cpp/include/raft/linalg/detail/eig.cuh +++ b/cpp/include/raft/linalg/detail/eig.cuh @@ -19,10 +19,12 @@ #include "cusolver_wrappers.hpp" #include +#include #include #include #include +#include #include #include @@ -90,7 +92,19 @@ void eigDC(raft::resources const& handle, { #if CUDART_VERSION < 11010 eigDC_legacy(handle, in, n_rows, n_cols, eig_vectors, eig_vals, stream); + return; +#endif + +#if CUDART_VERSION <= 12040 + // Use a new stream instead of `cudaStreamPerThread` to avoid cusolver bug # 4580093. + rmm::cuda_stream stream_new_wrapper; + cudaStream_t stream_new = stream_new_wrapper.value(); + cudaEvent_t sync_event = resource::detail::get_cuda_stream_sync_event(handle); + RAFT_CUDA_TRY(cudaEventRecord(sync_event, stream)); + RAFT_CUDA_TRY(cudaStreamWaitEvent(stream_new, sync_event)); #else + cudaStream_t stream_new = stream; +#endif cusolverDnHandle_t cusolverH = resource::get_cusolver_dn_handle(handle); cusolverDnParams_t dn_params = nullptr; @@ -108,15 +122,13 @@ void eigDC(raft::resources const& handle, eig_vals, &workspaceDevice, &workspaceHost, - stream)); + stream_new)); - rmm::device_uvector d_work(workspaceDevice / sizeof(math_t), stream); - rmm::device_scalar d_dev_info(stream); + rmm::device_uvector d_work(workspaceDevice / sizeof(math_t), stream_new); + rmm::device_scalar d_dev_info(stream_new); std::vector h_work(workspaceHost / sizeof(math_t)); - raft::matrix::copy(handle, - make_device_matrix_view(in, n_rows, n_cols), - make_device_matrix_view(eig_vectors, n_rows, n_cols)); + raft::copy(eig_vectors, in, n_rows * n_cols, stream_new); RAFT_CUSOLVER_TRY(cusolverDnxsyevd(cusolverH, dn_params, @@ -131,14 +143,19 @@ void eigDC(raft::resources const& handle, h_work.data(), workspaceHost, d_dev_info.data(), - stream)); + stream_new)); RAFT_CUDA_TRY(cudaGetLastError()); RAFT_CUSOLVER_TRY(cusolverDnDestroyParams(dn_params)); - int dev_info = d_dev_info.value(stream); + int dev_info = d_dev_info.value(stream_new); ASSERT(dev_info == 0, "eig.cuh: eigensolver couldn't converge to a solution. " "This usually occurs when some of the features do not vary enough."); + +#if CUDART_VERSION <= 12040 + // Synchronize the created stream with the original stream before return + RAFT_CUDA_TRY(cudaEventRecord(sync_event, stream_new)); + RAFT_CUDA_TRY(cudaStreamWaitEvent(stream, sync_event)); #endif } diff --git a/cpp/test/linalg/eig.cu b/cpp/test/linalg/eig.cu index 460b99aaa0..3ff117cf08 100644 --- a/cpp/test/linalg/eig.cu +++ b/cpp/test/linalg/eig.cu @@ -156,6 +156,24 @@ class EigTest : public ::testing::TestWithParam> { eig_vals_large, eig_vals_jacobi_large; }; +TEST(Raft, EigStream) +{ + // Separate test to check eig_dc stream workaround for CUDA 12+ + raft::resources handle; + auto n_rows = 5000; + auto cov_matrix_stream = + raft::make_device_matrix(handle, n_rows, n_rows); + auto eig_vectors_stream = + raft::make_device_matrix(handle, n_rows, n_rows); + auto eig_vals_stream = raft::make_device_vector(handle, n_rows); + + raft::linalg::eig_dc(handle, + raft::make_const_mdspan(cov_matrix_stream.view()), + eig_vectors_stream.view(), + eig_vals_stream.view()); + raft::resource::sync_stream(handle, raft::resource::get_cuda_stream(handle)); +} + const std::vector> inputsf2 = {{0.001f, 4 * 4, 4, 4, 1234ULL, 256}}; const std::vector> inputsd2 = {{0.001, 4 * 4, 4, 4, 1234ULL, 256}}; From e75bac63906f3d056a65c9bbb3abd67eb256d029 Mon Sep 17 00:00:00 2001 From: rhdong Date: Thu, 25 Jul 2024 08:13:08 -0700 Subject: [PATCH 04/79] [Refactor] move `popc` to under util (#2394) Authors: - rhdong (https://github.com/rhdong) Approvers: - Corey J. Nolet (https://github.com/cjnolet) - Divye Gala (https://github.com/divyegala) URL: https://github.com/rapidsai/raft/pull/2394 --- cpp/bench/prims/CMakeLists.txt | 9 ++++++++- cpp/bench/prims/{core => util}/popc.cu | 2 +- cpp/include/raft/core/bitset.cuh | 2 +- cpp/include/raft/sparse/linalg/detail/masked_matmul.cuh | 1 - cpp/include/raft/{core => util}/detail/popc.cuh | 0 cpp/include/raft/{core/popc.hpp => util/popc.cuh} | 2 +- cpp/test/CMakeLists.txt | 2 +- cpp/test/{core => util}/popc.cu | 2 +- 8 files changed, 13 insertions(+), 7 deletions(-) rename cpp/bench/prims/{core => util}/popc.cu (99%) rename cpp/include/raft/{core => util}/detail/popc.cuh (100%) rename cpp/include/raft/{core/popc.hpp => util/popc.cuh} (97%) rename cpp/test/{core => util}/popc.cu (99%) diff --git a/cpp/bench/prims/CMakeLists.txt b/cpp/bench/prims/CMakeLists.txt index c8c68f19bf..9d80cbaac2 100644 --- a/cpp/bench/prims/CMakeLists.txt +++ b/cpp/bench/prims/CMakeLists.txt @@ -80,7 +80,14 @@ if(BUILD_PRIMS_BENCH) PATH core/bitset.cu core/copy.cu - core/popc.cu + main.cpp + ) + + ConfigureBench( + NAME + UTIL_BENCH + PATH + util/popc.cu main.cpp ) diff --git a/cpp/bench/prims/core/popc.cu b/cpp/bench/prims/util/popc.cu similarity index 99% rename from cpp/bench/prims/core/popc.cu rename to cpp/bench/prims/util/popc.cu index dfa4335140..249dc13d1e 100644 --- a/cpp/bench/prims/core/popc.cu +++ b/cpp/bench/prims/util/popc.cu @@ -16,7 +16,7 @@ #include -#include +#include namespace raft::bench::core { diff --git a/cpp/include/raft/core/bitset.cuh b/cpp/include/raft/core/bitset.cuh index 3b67e56eea..0cdb4c1fb6 100644 --- a/cpp/include/raft/core/bitset.cuh +++ b/cpp/include/raft/core/bitset.cuh @@ -19,12 +19,12 @@ #include #include #include -#include #include #include #include #include #include +#include #include diff --git a/cpp/include/raft/sparse/linalg/detail/masked_matmul.cuh b/cpp/include/raft/sparse/linalg/detail/masked_matmul.cuh index 208328f2f3..ef74316d04 100644 --- a/cpp/include/raft/sparse/linalg/detail/masked_matmul.cuh +++ b/cpp/include/raft/sparse/linalg/detail/masked_matmul.cuh @@ -16,7 +16,6 @@ #pragma once #include -#include #include #include #include diff --git a/cpp/include/raft/core/detail/popc.cuh b/cpp/include/raft/util/detail/popc.cuh similarity index 100% rename from cpp/include/raft/core/detail/popc.cuh rename to cpp/include/raft/util/detail/popc.cuh diff --git a/cpp/include/raft/core/popc.hpp b/cpp/include/raft/util/popc.cuh similarity index 97% rename from cpp/include/raft/core/popc.hpp rename to cpp/include/raft/util/popc.cuh index fc6b6bd177..153694e45e 100644 --- a/cpp/include/raft/core/popc.hpp +++ b/cpp/include/raft/util/popc.cuh @@ -15,7 +15,7 @@ */ #pragma once -#include +#include namespace raft { /** diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index cb96ce2264..e3af6ebb78 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -122,7 +122,6 @@ if(BUILD_TESTS) core/math_host.cpp core/operators_device.cu core/operators_host.cpp - core/popc.cu core/handle.cpp core/interruptible.cu core/nvtx.cpp @@ -509,6 +508,7 @@ if(BUILD_TESTS) util/integer_utils.cpp util/integer_utils.cu util/memory_type_dispatcher.cu + util/popc.cu util/pow2_utils.cu util/reduction.cu ) diff --git a/cpp/test/core/popc.cu b/cpp/test/util/popc.cu similarity index 99% rename from cpp/test/core/popc.cu rename to cpp/test/util/popc.cu index 83dda79b6e..c08faacb07 100644 --- a/cpp/test/core/popc.cu +++ b/cpp/test/util/popc.cu @@ -19,10 +19,10 @@ #include #include #include -#include #include #include #include +#include #include From 7bffdaccbc3bee42dba5d96009debeb099535140 Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Thu, 25 Jul 2024 22:54:29 +0200 Subject: [PATCH 05/79] Add timeout to UCXX generic operations (#2398) https://github.com/rapidsai/ucxx/pull/238 introduced a new timeout argument for `registerGeneric{Pre,Post}` that can be used to prevent blocking indefinitely should there be no UCX worker progress wakeup events. This should also result in new RAFT packages with updated symbols. Authors: - Peter Andreas Entschev (https://github.com/pentschev) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/raft/pull/2398 --- cpp/include/raft/comms/detail/std_comms.hpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cpp/include/raft/comms/detail/std_comms.hpp b/cpp/include/raft/comms/detail/std_comms.hpp index cb1accc95e..c5d64f6a29 100644 --- a/cpp/include/raft/comms/detail/std_comms.hpp +++ b/cpp/include/raft/comms/detail/std_comms.hpp @@ -307,13 +307,16 @@ class std_comms : public comms_iface { bool restart = false; // resets the timeout when any progress was made if (worker->isProgressThreadRunning()) { - // Wait for a UCXX progress thread roundtrip + // Wait for a UCXX progress thread roundtrip, prevent waiting for longer + // than 10ms for each operation, will retry in next iteration. ucxx::utils::CallbackNotifier callbackNotifierPre{}; - worker->registerGenericPre([&callbackNotifierPre]() { callbackNotifierPre.set(); }); + worker->registerGenericPre([&callbackNotifierPre]() { callbackNotifierPre.set(); }, + 10000000 /* 10ms */); callbackNotifierPre.wait(); ucxx::utils::CallbackNotifier callbackNotifierPost{}; - worker->registerGenericPost([&callbackNotifierPost]() { callbackNotifierPost.set(); }); + worker->registerGenericPost([&callbackNotifierPost]() { callbackNotifierPost.set(); }, + 10000000 /* 10ms */); callbackNotifierPost.wait(); } else { // Causes UCXX to progress through the send/recv message queue From f5202353460f259d7a8e7ca6323468d037db6ce1 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 30 Jul 2024 15:45:13 -0500 Subject: [PATCH 06/79] Update pip devcontainers to UCX v1.17.0 (#2401) Contributes to https://github.com/rapidsai/build-planning/issues/77. Follow-up to https://github.com/rapidsai/devcontainers/pull/338 Proposes updating the pip devcontainers to use v1.17.0, as part of an effort to use that version across RAPIDS. Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Ray Douglass (https://github.com/raydouglass) - Peter Andreas Entschev (https://github.com/pentschev) URL: https://github.com/rapidsai/raft/pull/2401 --- .devcontainer/cuda11.8-pip/devcontainer.json | 5 +---- .devcontainer/cuda12.5-pip/devcontainer.json | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json index 0fdbac9d91..f319536b18 100644 --- a/.devcontainer/cuda11.8-pip/devcontainer.json +++ b/.devcontainer/cuda11.8-pip/devcontainer.json @@ -5,7 +5,7 @@ "args": { "CUDA": "11.8", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:24.10-cpp-cuda11.8-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.10-cpp-cuda11.8-ucx1.17.0-openmpi-ubuntu22.04" } }, "runArgs": [ @@ -15,9 +15,6 @@ ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/ucx:24.10": { - "version": "1.15.0" - }, "ghcr.io/rapidsai/devcontainers/features/cuda:24.10": { "version": "11.8", "installcuBLAS": true, diff --git a/.devcontainer/cuda12.5-pip/devcontainer.json b/.devcontainer/cuda12.5-pip/devcontainer.json index 39d1c0c969..26b9a56e48 100644 --- a/.devcontainer/cuda12.5-pip/devcontainer.json +++ b/.devcontainer/cuda12.5-pip/devcontainer.json @@ -5,7 +5,7 @@ "args": { "CUDA": "12.5", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:24.10-cpp-cuda12.5-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.10-cpp-cuda12.5-ucx1.17.0-openmpi-ubuntu22.04" } }, "runArgs": [ @@ -15,9 +15,6 @@ ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/ucx:24.10": { - "version": "1.15.0" - }, "ghcr.io/rapidsai/devcontainers/features/cuda:24.10": { "version": "12.5", "installcuBLAS": true, From 38fadb30c3d85dbf891a65d87c2fa426ccbce6c8 Mon Sep 17 00:00:00 2001 From: Ray Douglass Date: Wed, 7 Aug 2024 10:42:54 -0400 Subject: [PATCH 07/79] Update Changelog [skip ci] --- CHANGELOG.md | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0599dae8a..0685145dca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,53 @@ +# raft 24.08.00 (7 Aug 2024) + +## 🚨 Breaking Changes + +- [Refactor] move `popc` to under util ([#2394](https://github.com/rapidsai/raft/pull/2394)) [@rhdong](https://github.com/rhdong) +- [Opt] Expose the `detail::popc` as public API ([#2346](https://github.com/rapidsai/raft/pull/2346)) [@rhdong](https://github.com/rhdong) + +## 🐛 Bug Fixes + +- Add timeout to UCXX generic operations ([#2398](https://github.com/rapidsai/raft/pull/2398)) [@pentschev](https://github.com/pentschev) +- [Fix] bitmap set/test issue ([#2371](https://github.com/rapidsai/raft/pull/2371)) [@rhdong](https://github.com/rhdong) +- Fix 0 recall issue in `raft_cagra_hnswlib` ANN benchmark ([#2369](https://github.com/rapidsai/raft/pull/2369)) [@divyegala](https://github.com/divyegala) +- Fix `ef` setting in HNSW wrapper ([#2367](https://github.com/rapidsai/raft/pull/2367)) [@divyegala](https://github.com/divyegala) +- Fix cagra graph opt bug ([#2365](https://github.com/rapidsai/raft/pull/2365)) [@enp1s0](https://github.com/enp1s0) +- Fix a bug where the wrong API is used to free the memory ([#2361](https://github.com/rapidsai/raft/pull/2361)) [@PointKernel](https://github.com/PointKernel) +- Allow anonymous user in devcontainer name ([#2355](https://github.com/rapidsai/raft/pull/2355)) [@bdice](https://github.com/bdice) +- Fix compilation error when _CLK_BREAKDOWN is defined in cagra. ([#2350](https://github.com/rapidsai/raft/pull/2350)) [@jiangyinzuo](https://github.com/jiangyinzuo) +- ensure raft-dask wheel tests install pylibraft wheel from the same CI run, fix wheel dependencies ([#2349](https://github.com/rapidsai/raft/pull/2349)) [@jameslamb](https://github.com/jameslamb) +- Change --config-setting to --config-settings ([#2342](https://github.com/rapidsai/raft/pull/2342)) [@KyleFromNVIDIA](https://github.com/KyleFromNVIDIA) +- Add workaround for syevd in CUDA 12.0 ([#2332](https://github.com/rapidsai/raft/pull/2332)) [@lowener](https://github.com/lowener) + +## 🚀 New Features + +- [FEA] add the support of `masked_matmul` ([#2362](https://github.com/rapidsai/raft/pull/2362)) [@rhdong](https://github.com/rhdong) +- [FEA] Dice Distance for Dense Inputs ([#2359](https://github.com/rapidsai/raft/pull/2359)) [@aamijar](https://github.com/aamijar) +- [Opt] Expose the `detail::popc` as public API ([#2346](https://github.com/rapidsai/raft/pull/2346)) [@rhdong](https://github.com/rhdong) +- Enable distance return for NN Descent ([#2345](https://github.com/rapidsai/raft/pull/2345)) [@jinsolp](https://github.com/jinsolp) + +## 🛠️ Improvements + +- [Refactor] move `popc` to under util ([#2394](https://github.com/rapidsai/raft/pull/2394)) [@rhdong](https://github.com/rhdong) +- split up CUDA-suffixed dependencies in dependencies.yaml ([#2388](https://github.com/rapidsai/raft/pull/2388)) [@jameslamb](https://github.com/jameslamb) +- Use workflow branch 24.08 again ([#2385](https://github.com/rapidsai/raft/pull/2385)) [@KyleFromNVIDIA](https://github.com/KyleFromNVIDIA) +- Add cusparseSpMV_preprocess to cusparse wrapper ([#2384](https://github.com/rapidsai/raft/pull/2384)) [@Kh4ster](https://github.com/Kh4ster) +- Consolidate SUM reductions ([#2381](https://github.com/rapidsai/raft/pull/2381)) [@mfoerste4](https://github.com/mfoerste4) +- Use slicing kernel to copy distances inside NN Descent ([#2380](https://github.com/rapidsai/raft/pull/2380)) [@jinsolp](https://github.com/jinsolp) +- Build and test with CUDA 12.5.1 ([#2378](https://github.com/rapidsai/raft/pull/2378)) [@KyleFromNVIDIA](https://github.com/KyleFromNVIDIA) +- Add CUDA_STATIC_MATH_LIBRARIES ([#2376](https://github.com/rapidsai/raft/pull/2376)) [@KyleFromNVIDIA](https://github.com/KyleFromNVIDIA) +- skip CMake 3.30.0 ([#2375](https://github.com/rapidsai/raft/pull/2375)) [@jameslamb](https://github.com/jameslamb) +- Use verify-alpha-spec hook ([#2373](https://github.com/rapidsai/raft/pull/2373)) [@KyleFromNVIDIA](https://github.com/KyleFromNVIDIA) +- Binarize Dice Distance for Dense Inputs ([#2370](https://github.com/rapidsai/raft/pull/2370)) [@aamijar](https://github.com/aamijar) +- [FEA] Add distance epilogue for NN Descent ([#2364](https://github.com/rapidsai/raft/pull/2364)) [@jinsolp](https://github.com/jinsolp) +- resolve dependency-file-generator warning, other rapids-build-backend followup ([#2360](https://github.com/rapidsai/raft/pull/2360)) [@jameslamb](https://github.com/jameslamb) +- Remove text builds of documentation ([#2354](https://github.com/rapidsai/raft/pull/2354)) [@vyasr](https://github.com/vyasr) +- Use default init in reduction ([#2351](https://github.com/rapidsai/raft/pull/2351)) [@akifcorduk](https://github.com/akifcorduk) +- ensure update-version.sh preserves alpha spec, add tests on version constants ([#2344](https://github.com/rapidsai/raft/pull/2344)) [@jameslamb](https://github.com/jameslamb) +- remove unnecessary 'setuptools' dependencies ([#2343](https://github.com/rapidsai/raft/pull/2343)) [@jameslamb](https://github.com/jameslamb) +- Use rapids-build-backend ([#2331](https://github.com/rapidsai/raft/pull/2331)) [@KyleFromNVIDIA](https://github.com/KyleFromNVIDIA) +- Add FAISS with RAFT enabled Benchmarking to raft-ann-bench ([#2026](https://github.com/rapidsai/raft/pull/2026)) [@tarang-jain](https://github.com/tarang-jain) + # raft 24.06.00 (5 Jun 2024) ## 🚨 Breaking Changes From dbdde82a00ac39d0a34279441f5efd7c73a4d7d8 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 8 Aug 2024 10:21:23 -0500 Subject: [PATCH 08/79] Improve update-version.sh (#2408) This PR improves `update-version.sh` by clarifying suffix handling and ucxx/ucx-py version handling. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/raft/pull/2408 --- ci/release/update-version.sh | 43 +++++++++++++++++------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 194ad9a07b..075eb896f2 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -18,16 +18,15 @@ CURRENT_MINOR=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[2]}') CURRENT_PATCH=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[3]}') CURRENT_SHORT_TAG=${CURRENT_MAJOR}.${CURRENT_MINOR} -#Get . for next version +# Get . for next version NEXT_MAJOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[1]}') NEXT_MINOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[2]}') NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR} -NEXT_UCX_PY_SHORT_TAG="$(curl -sL https://version.gpuci.io/rapids/${NEXT_SHORT_TAG})" -NEXT_UCX_PY_VERSION="${NEXT_UCX_PY_SHORT_TAG}.*" +NEXT_UCXX_SHORT_TAG="$(curl -sL https://version.gpuci.io/rapids/${NEXT_SHORT_TAG})" # Need to distutils-normalize the original version NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_SHORT_TAG}'))") -NEXT_UCX_PY_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_UCX_PY_SHORT_TAG}'))") +NEXT_UCXX_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_UCXX_SHORT_TAG}'))") echo "Preparing release $CURRENT_TAG => $NEXT_FULL_TAG" @@ -37,8 +36,8 @@ function sed_runner() { } sed_runner "s/set(RAPIDS_VERSION .*)/set(RAPIDS_VERSION \"${NEXT_SHORT_TAG}\")/g" cpp/template/cmake/thirdparty/fetch_rapids.cmake -sed_runner 's/'"find_and_configure_ucxx(VERSION .*"'/'"find_and_configure_ucxx(VERSION ${NEXT_UCX_PY_SHORT_TAG_PEP440}"'/g' python/raft-dask/cmake/thirdparty/get_ucxx.cmake -sed_runner 's/'"branch-.*"'/'"branch-${NEXT_UCX_PY_SHORT_TAG_PEP440}"'/g' python/raft-dask/cmake/thirdparty/get_ucxx.cmake +sed_runner 's/'"find_and_configure_ucxx(VERSION .*"'/'"find_and_configure_ucxx(VERSION ${NEXT_UCXX_SHORT_TAG_PEP440}"'/g' python/raft-dask/cmake/thirdparty/get_ucxx.cmake +sed_runner 's/'"branch-.*"'/'"branch-${NEXT_UCXX_SHORT_TAG_PEP440}"'/g' python/raft-dask/cmake/thirdparty/get_ucxx.cmake # Centralized version file update echo "${NEXT_FULL_TAG}" > VERSION @@ -46,35 +45,33 @@ echo "${NEXT_FULL_TAG}" > VERSION DEPENDENCIES=( dask-cuda pylibraft - pylibraft-cu11 - pylibraft-cu12 rmm - rmm-cu11 - rmm-cu12 rapids-dask-dependency - # ucx-py and ucxx are handled separately below +) +UCXX_DEPENDENCIES=( + ucx-py + libucxx + distributed-ucxx ) for FILE in dependencies.yaml conda/environments/*.yaml; do for DEP in "${DEPENDENCIES[@]}"; do - sed_runner "/-.* ${DEP}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}\.*,>=0.0.0a0/g" ${FILE}; + sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0/g" "${FILE}" + done + for DEP in "${UCXX_DEPENDENCIES[@]}"; do + sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_UCXX_SHORT_TAG_PEP440}.*/,>=0.0.0a0/g" "${FILE}" done - sed_runner "/-.* ucx-py==/ s/==.*/==${NEXT_UCX_PY_SHORT_TAG_PEP440}\.*,>=0.0.0a0/g" ${FILE}; - sed_runner "/-.* ucx-py-cu11==/ s/==.*/==${NEXT_UCX_PY_SHORT_TAG_PEP440}\.*,>=0.0.0a0/g" ${FILE}; - sed_runner "/-.* ucx-py-cu12==/ s/==.*/==${NEXT_UCX_PY_SHORT_TAG_PEP440}\.*,>=0.0.0a0/g" ${FILE}; - sed_runner "/-.* libucxx==/ s/==.*/==${NEXT_UCX_PY_SHORT_TAG_PEP440}\.*,>=0.0.0a0/g" ${FILE}; - sed_runner "/-.* distributed-ucxx==/ s/==.*/==${NEXT_UCX_PY_SHORT_TAG_PEP440}\.*,>=0.0.0a0/g" ${FILE}; - sed_runner "/-.* distributed-ucxx-cu11==/ s/==.*/==${NEXT_UCX_PY_SHORT_TAG_PEP440}\.*,>=0.0.0a0/g" ${FILE}; - sed_runner "/-.* distributed-ucxx-cu12==/ s/==.*/==${NEXT_UCX_PY_SHORT_TAG_PEP440}\.*,>=0.0.0a0/g" ${FILE}; done for FILE in python/*/pyproject.toml; do for DEP in "${DEPENDENCIES[@]}"; do - sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\"/g" ${FILE} + sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\"/g" "${FILE}" + done + for DEP in "${UCXX_DEPENDENCIES[@]}"; do + sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_UCXX_SHORT_TAG_PEP440}.*,>=0.0.0a0\"/g" "${FILE}" done - sed_runner "/\"ucx-py==/ s/==.*\"/==${NEXT_UCX_PY_SHORT_TAG_PEP440}.*,>=0.0.0a0\"/g" ${FILE} done -sed_runner "/^ucx_py_version:$/ {n;s/.*/ - \"${NEXT_UCX_PY_VERSION}\"/}" conda/recipes/raft-dask/conda_build_config.yaml -sed_runner "/^ucxx_version:$/ {n;s/.*/ - \"${NEXT_UCX_PY_VERSION}\"/}" conda/recipes/raft-dask/conda_build_config.yaml +sed_runner "/^ucx_py_version:$/ {n;s/.*/ - \"${NEXT_UCXX_SHORT_TAG_PEP440}.*\"/}" conda/recipes/raft-dask/conda_build_config.yaml +sed_runner "/^ucxx_version:$/ {n;s/.*/ - \"${NEXT_UCXX_SHORT_TAG_PEP440}.*\"/}" conda/recipes/raft-dask/conda_build_config.yaml for FILE in .github/workflows/*.yaml; do sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}" From a1f4844df016cb7dfd9ceaed92c4a6700baab21b Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 8 Aug 2024 12:51:37 -0500 Subject: [PATCH 09/79] Use tool.scikit-build.cmake.version, set scikit-build-core minimum-version (#2406) Contributes to https://github.com/rapidsai/build-planning/issues/58. `scikit-build-core==0.10.0` was released today (https://github.com/scikit-build/scikit-build-core/releases/tag/v0.10.0), and wheel-building configurations across RAPIDS are incompatible with it. This proposes upgrading to that version and fixing configuration here in a way that: * is compatible with that new `scikit-build-core` version * takes advantage of the forward-compatibility mechanism (`minimum-version`) that `scikit-build-core` provides, to reduce the risk of needing to do this again in the future Authors: - James Lamb (https://github.com/jameslamb) Approvers: - https://github.com/jakirkham URL: https://github.com/rapidsai/raft/pull/2406 --- conda/environments/all_cuda-118_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-125_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-125_arch-x86_64.yaml | 2 +- conda/environments/bench_ann_cuda-118_arch-aarch64.yaml | 2 +- conda/environments/bench_ann_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/bench_ann_cuda-120_arch-aarch64.yaml | 2 +- conda/environments/bench_ann_cuda-120_arch-x86_64.yaml | 2 +- conda/recipes/pylibraft/meta.yaml | 2 +- conda/recipes/raft-dask/meta.yaml | 2 +- dependencies.yaml | 4 ++-- python/pylibraft/pyproject.toml | 5 +++-- python/raft-dask/pyproject.toml | 5 +++-- 13 files changed, 18 insertions(+), 16 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index 1a28edb974..ef8524dce3 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -51,7 +51,7 @@ dependencies: - rapids-dask-dependency==24.10.*,>=0.0.0a0 - recommonmark - rmm==24.10.*,>=0.0.0a0 -- scikit-build-core>=0.7.0 +- scikit-build-core>=0.10.0 - scikit-learn - scipy - sphinx-copybutton diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 71a7fc4b02..6ffb27bb29 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -51,7 +51,7 @@ dependencies: - rapids-dask-dependency==24.10.*,>=0.0.0a0 - recommonmark - rmm==24.10.*,>=0.0.0a0 -- scikit-build-core>=0.7.0 +- scikit-build-core>=0.10.0 - scikit-learn - scipy - sphinx-copybutton diff --git a/conda/environments/all_cuda-125_arch-aarch64.yaml b/conda/environments/all_cuda-125_arch-aarch64.yaml index 7bb4a40c05..fd0e380a1c 100644 --- a/conda/environments/all_cuda-125_arch-aarch64.yaml +++ b/conda/environments/all_cuda-125_arch-aarch64.yaml @@ -47,7 +47,7 @@ dependencies: - rapids-dask-dependency==24.10.*,>=0.0.0a0 - recommonmark - rmm==24.10.*,>=0.0.0a0 -- scikit-build-core>=0.7.0 +- scikit-build-core>=0.10.0 - scikit-learn - scipy - sphinx-copybutton diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 58153398f7..ad4ecb7ff2 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -47,7 +47,7 @@ dependencies: - rapids-dask-dependency==24.10.*,>=0.0.0a0 - recommonmark - rmm==24.10.*,>=0.0.0a0 -- scikit-build-core>=0.7.0 +- scikit-build-core>=0.10.0 - scikit-learn - scipy - sphinx-copybutton diff --git a/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml index 7a961d1ca9..eff1c56840 100644 --- a/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml +++ b/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml @@ -41,6 +41,6 @@ dependencies: - pyyaml - rapids-build-backend>=0.3.0,<0.4.0.dev0 - rmm==24.10.*,>=0.0.0a0 -- scikit-build-core>=0.7.0 +- scikit-build-core>=0.10.0 - sysroot_linux-aarch64==2.17 name: bench_ann_cuda-118_arch-aarch64 diff --git a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml index 3e3f2511d8..87b19d2952 100644 --- a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml +++ b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml @@ -41,6 +41,6 @@ dependencies: - pyyaml - rapids-build-backend>=0.3.0,<0.4.0.dev0 - rmm==24.10.*,>=0.0.0a0 -- scikit-build-core>=0.7.0 +- scikit-build-core>=0.10.0 - sysroot_linux-64==2.17 name: bench_ann_cuda-118_arch-x86_64 diff --git a/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml index 972cce6f88..ff3451c15c 100644 --- a/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml +++ b/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml @@ -37,6 +37,6 @@ dependencies: - pyyaml - rapids-build-backend>=0.3.0,<0.4.0.dev0 - rmm==24.10.*,>=0.0.0a0 -- scikit-build-core>=0.7.0 +- scikit-build-core>=0.10.0 - sysroot_linux-aarch64==2.17 name: bench_ann_cuda-120_arch-aarch64 diff --git a/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml index 1ecae4658b..085e099ae8 100644 --- a/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml +++ b/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml @@ -37,6 +37,6 @@ dependencies: - pyyaml - rapids-build-backend>=0.3.0,<0.4.0.dev0 - rmm==24.10.*,>=0.0.0a0 -- scikit-build-core>=0.7.0 +- scikit-build-core>=0.10.0 - sysroot_linux-64==2.17 name: bench_ann_cuda-120_arch-x86_64 diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 31086e30aa..4ef85fc0e5 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -54,7 +54,7 @@ requirements: - libraft-headers {{ version }} - python x.x - rmm ={{ minor_version }} - - scikit-build-core >=0.7.0 + - scikit-build-core >=0.10.0 - rapids-build-backend>=0.3.0,<0.4.0.dev0 run: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index 641a11a241..74b26b5935 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -54,7 +54,7 @@ requirements: - pylibraft {{ version }} - python x.x - rmm ={{ minor_version }} - - scikit-build-core >=0.7.0 + - scikit-build-core >=0.10.0 - ucx-py {{ ucx_py_version }} - ucxx {{ ucxx_version }} - rapids-build-backend>=0.3.0,<0.4.0.dev0 diff --git a/dependencies.yaml b/dependencies.yaml index 234cafa035..630b5a50c9 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -154,10 +154,10 @@ dependencies: - &rapids_build_backend rapids-build-backend>=0.3.0,<0.4.0.dev0 - output_types: [conda] packages: - - scikit-build-core>=0.7.0 + - scikit-build-core>=0.10.0 - output_types: [requirements, pyproject] packages: - - scikit-build-core[pyproject]>=0.7.0 + - scikit-build-core[pyproject]>=0.10.0 rapids_build: common: - output_types: [conda, requirements, pyproject] diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index b09615cc00..4c1d8e8fa1 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -16,7 +16,7 @@ requires = [ "rapids-build-backend>=0.3.0,<0.4.0.dev0", - "scikit-build-core[pyproject]>=0.7.0", + "scikit-build-core[pyproject]>=0.10.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. build-backend = "rapids_build_backend.build" @@ -102,7 +102,8 @@ skip = [ [tool.scikit-build] build-dir = "build/{wheel_tag}" cmake.build-type = "Release" -cmake.minimum-version = "3.26.4" +cmake.version = "CMakeLists.txt" +minimum-version = "build-system.requires" ninja.make-fallback = true sdist.exclude = ["*tests*"] sdist.reproducible = true diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index 1bb26e98e8..4fadfa5c9f 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -17,7 +17,7 @@ build-backend = "rapids_build_backend.build" requires = [ "rapids-build-backend>=0.3.0,<0.4.0.dev0", - "scikit-build-core[pyproject]>=0.7.0", + "scikit-build-core[pyproject]>=0.10.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project] @@ -105,7 +105,8 @@ skip = [ [tool.scikit-build] build-dir = "build/{wheel_tag}" cmake.build-type = "Release" -cmake.minimum-version = "3.26.4" +cmake.version = "CMakeLists.txt" +minimum-version = "build-system.requires" ninja.make-fallback = true sdist.exclude = ["*tests*"] sdist.reproducible = true From 32f3703b0d28e5e8934fc463cf3ec6bd6cc82a79 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 8 Aug 2024 13:56:14 -0400 Subject: [PATCH 10/79] Update pre-commit hooks (#2409) This PR updates pre-commit hooks to the latest versions that are supported without causing style check errors. Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/raft/pull/2409 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a0a4b4be87..9aed6bf387 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -99,7 +99,7 @@ repos: hooks: - id: check-json - repo: https://github.com/rapidsai/pre-commit-hooks - rev: v0.2.0 + rev: v0.3.1 hooks: - id: verify-copyright files: | From db079987260d8462eaccfd5173346f6180bc70bd Mon Sep 17 00:00:00 2001 From: rhdong Date: Thu, 22 Aug 2024 06:13:44 -0700 Subject: [PATCH 11/79] [FEA] Support for half-float mixed precise in brute-force (#2382) - distance supports half-float - SDDMM support half-float - gemm supports multi-type compose - transpose & copy support half - random supports half Authors: - rhdong (https://github.com/rhdong) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/raft/pull/2382 --- cpp/include/raft/core/detail/copy.hpp | 76 +++--- cpp/include/raft/core/math.hpp | 14 +- cpp/include/raft/core/operators.hpp | 18 +- .../distance/detail/masked_distance_base.cuh | 4 +- .../detail/pairwise_distance_base.cuh | 4 +- cpp/include/raft/linalg/contractions.cuh | 14 +- .../raft/linalg/detail/contractions.cuh | 14 +- cpp/include/raft/linalg/detail/gemm.hpp | 111 ++++----- cpp/include/raft/linalg/detail/norm.cuh | 154 ++++++------ cpp/include/raft/linalg/detail/transpose.cuh | 138 +++++++++-- cpp/include/raft/linalg/gemm.cuh | 65 +++--- cpp/include/raft/linalg/norm.cuh | 22 +- cpp/include/raft/random/detail/rng_device.cuh | 21 ++ cpp/include/raft/random/detail/rng_impl.cuh | 3 +- .../sparse/convert/detail/bitmap_to_csr.cuh | 22 +- .../raft/sparse/detail/cusparse_wrappers.h | 68 +++++- .../raft/sparse/distance/detail/utils.cuh | 44 ++-- .../sparse/linalg/detail/masked_matmul.cuh | 20 +- .../raft/sparse/linalg/detail/sddmm.hpp | 16 +- .../raft/sparse/linalg/masked_matmul.hpp | 11 +- cpp/include/raft/sparse/linalg/sddmm.hpp | 17 +- .../knn/detail/epsilon_neighborhood.cuh | 2 +- cpp/include/raft/util/cuda_dev_essentials.cuh | 19 +- cpp/include/raft/util/cudart_utils.hpp | 6 +- cpp/test/core/mdspan_copy.cu | 53 +++++ cpp/test/linalg/norm.cu | 93 +++++--- cpp/test/linalg/transpose.cu | 43 ++++ cpp/test/sparse/masked_matmul.cu | 221 +++++++++++++----- cpp/test/sparse/sddmm.cu | 198 +++++++++++----- 29 files changed, 1050 insertions(+), 441 deletions(-) diff --git a/cpp/include/raft/core/detail/copy.hpp b/cpp/include/raft/core/detail/copy.hpp index 04e74c4e58..4faded5041 100644 --- a/cpp/include/raft/core/detail/copy.hpp +++ b/cpp/include/raft/core/detail/copy.hpp @@ -32,6 +32,7 @@ #include #include #ifdef __CUDACC__ +#include #include #endif #endif @@ -449,38 +450,51 @@ mdspan_copyable_t copy(resources const& res, DstType&& dst, Sr #endif } else if constexpr (config::can_use_cublas) { #ifndef RAFT_DISABLE_CUDA - auto constexpr const alpha = typename std::remove_reference_t::value_type{1}; - auto constexpr const beta = typename std::remove_reference_t::value_type{0}; - if constexpr (std::is_same_v) { - CUBLAS_TRY(linalg::detail::cublasgeam(resource::get_cublas_handle(res), - CUBLAS_OP_T, - CUBLAS_OP_N, - dst.extent(1), - dst.extent(0), - &alpha, - src.data_handle(), - src.extent(0), - &beta, - dst.data_handle(), - dst.extent(1), - dst.data_handle(), - dst.extent(1), - resource::get_cuda_stream(res))); + if constexpr (!((std::is_same_v::value_type, half>)&&( + std::is_same_v::value_type, half>))) { + auto constexpr const alpha = typename std::remove_reference_t::value_type{1}; + auto constexpr const beta = typename std::remove_reference_t::value_type{0}; + if constexpr (std::is_same_v) { + CUBLAS_TRY(linalg::detail::cublasgeam(resource::get_cublas_handle(res), + CUBLAS_OP_T, + CUBLAS_OP_N, + dst.extent(1), + dst.extent(0), + &alpha, + src.data_handle(), + src.extent(0), + &beta, + dst.data_handle(), + dst.extent(1), + dst.data_handle(), + dst.extent(1), + resource::get_cuda_stream(res))); + } else { + CUBLAS_TRY(linalg::detail::cublasgeam(resource::get_cublas_handle(res), + CUBLAS_OP_T, + CUBLAS_OP_N, + dst.extent(0), + dst.extent(1), + &alpha, + src.data_handle(), + src.extent(1), + &beta, + dst.data_handle(), + dst.extent(0), + dst.data_handle(), + dst.extent(0), + resource::get_cuda_stream(res))); + } } else { - CUBLAS_TRY(linalg::detail::cublasgeam(resource::get_cublas_handle(res), - CUBLAS_OP_T, - CUBLAS_OP_N, - dst.extent(0), - dst.extent(1), - &alpha, - src.data_handle(), - src.extent(1), - &beta, - dst.data_handle(), - dst.extent(0), - dst.data_handle(), - dst.extent(0), - resource::get_cuda_stream(res))); +#ifdef __CUDACC__ + raft::linalg::transpose(res, dst, src); +#else + // Should never actually reach this because of enable_ifs. Included for + // safety. + RAFT_FAIL( + "raft::copy called in a way that requires custom kernel. Please use " + "raft/core/copy.cuh and include the header in a .cu file"); +#endif } #else // Not possible to reach this due to enable_ifs. Included for safety. diff --git a/cpp/include/raft/core/math.hpp b/cpp/include/raft/core/math.hpp index e082aaf41a..c5de345082 100644 --- a/cpp/include/raft/core/math.hpp +++ b/cpp/include/raft/core/math.hpp @@ -106,7 +106,13 @@ template RAFT_INLINE_FUNCTION auto asin(T x) { #ifdef __CUDA_ARCH__ - return ::asin(x); + if constexpr (std::is_same::value) { + float x_float = __half2float(x); + float result_float = ::asin(x_float); + return __float2half(result_float); + } else { + return ::asin(x); + } #else return std::asin(x); #endif @@ -337,6 +343,12 @@ RAFT_INLINE_FUNCTION auto max(const T1& x, const T2& y) ((std::is_same_v || std::is_same_v)&&( std::is_same_v || std::is_same_v))) { return ::max(x, y); + } else if constexpr (std::is_same_v && std::is_same_v) { + const float f_y = __half2float(y); + return (x < f_y) ? f_y : x; + } else if constexpr (std::is_same_v && std::is_same_v) { + const float f_x = __half2float(x); + return (f_x < y) ? y : f_x; } // Else, check that the types are the same and provide a generic implementation else { diff --git a/cpp/include/raft/core/operators.hpp b/cpp/include/raft/core/operators.hpp index e42801fe32..6b10baa332 100644 --- a/cpp/include/raft/core/operators.hpp +++ b/cpp/include/raft/core/operators.hpp @@ -19,6 +19,8 @@ #include #include +#include + #include #include #include @@ -104,13 +106,27 @@ struct sq_op { { return in * in; } + + template + constexpr RAFT_INLINE_FUNCTION auto operator()(const half& in, UnusedArgs...) const + { + return __half2float(in) * __half2float(in); + } }; struct add_op { template constexpr RAFT_INLINE_FUNCTION auto operator()(const T1& a, const T2& b) const { - return a + b; + if constexpr (std::is_same_v && std::is_same_v) { + return __half2float(a) + __half2float(b); + } else if constexpr (std::is_same_v) { + return __half2float(a) + b; + } else if constexpr (std::is_same_v) { + return a + __half2float(b); + } else { + return a + b; + } } }; diff --git a/cpp/include/raft/distance/detail/masked_distance_base.cuh b/cpp/include/raft/distance/detail/masked_distance_base.cuh index 55da634145..96b778f11f 100644 --- a/cpp/include/raft/distance/detail/masked_distance_base.cuh +++ b/cpp/include/raft/distance/detail/masked_distance_base.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -266,7 +266,7 @@ struct MaskedDistances : public BaseClass { for (int i = 0; i < P::AccRowsPerTh; ++i) { #pragma unroll for (int j = 0; j < P::AccColsPerTh; ++j) { - acc[i][j] = BaseClass::Zero; + acc[i][j] = BaseClass::Zero(); } } } diff --git a/cpp/include/raft/distance/detail/pairwise_distance_base.cuh b/cpp/include/raft/distance/detail/pairwise_distance_base.cuh index c6b09be31e..a8a541bf53 100644 --- a/cpp/include/raft/distance/detail/pairwise_distance_base.cuh +++ b/cpp/include/raft/distance/detail/pairwise_distance_base.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -200,7 +200,7 @@ struct PairwiseDistances : public BaseClass { for (int i = 0; i < P::AccRowsPerTh; ++i) { #pragma unroll for (int j = 0; j < P::AccColsPerTh; ++j) { - acc[i][j] = BaseClass::Zero; + acc[i][j] = BaseClass::Zero(); } } } diff --git a/cpp/include/raft/linalg/contractions.cuh b/cpp/include/raft/linalg/contractions.cuh index cb6488bedf..b284bb3370 100644 --- a/cpp/include/raft/linalg/contractions.cuh +++ b/cpp/include/raft/linalg/contractions.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -164,6 +164,12 @@ struct Policy4x4 { typedef KernelPolicy Policy; typedef ColKernelPolicy ColPolicy; }; + +template +struct Policy4x4 { + typedef KernelPolicy Policy; + typedef ColKernelPolicy ColPolicy; +}; /** @} */ /** @@ -204,6 +210,12 @@ struct Policy2x8 { // this is not used just for keeping compiler happy. typedef KernelPolicy Policy; }; + +template +struct Policy2x8 { + typedef KernelPolicy Policy; +}; + /** @} */ /** diff --git a/cpp/include/raft/linalg/detail/contractions.cuh b/cpp/include/raft/linalg/detail/contractions.cuh index b15cb222b4..3bdcc22c1f 100644 --- a/cpp/include/raft/linalg/detail/contractions.cuh +++ b/cpp/include/raft/linalg/detail/contractions.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -72,7 +72,9 @@ struct Contractions_NT { /** block of Y data loaded from global mem after `ldgXY()` */ DataT ldgDataY[P::LdgPerThY][P::Veclen]; - static constexpr DataT Zero = (DataT)0; + // static constexpr DataT Zero = DataT{0}; + + static constexpr DataT Zero() { return DataT{0}; } public: /** @@ -197,7 +199,7 @@ struct Contractions_NT { } else { #pragma unroll for (int j = 0; j < P::Veclen; ++j) { - ldgDataX[i][j] = Zero; + ldgDataX[i][j] = Zero(); } } } @@ -211,7 +213,7 @@ struct Contractions_NT { } else { #pragma unroll for (int j = 0; j < P::Veclen; ++j) { - ldgDataX[i][j] = Zero; + ldgDataX[i][j] = Zero(); } } } @@ -235,7 +237,7 @@ struct Contractions_NT { } else { #pragma unroll for (int j = 0; j < P::Veclen; ++j) { - ldgDataY[i][j] = Zero; + ldgDataY[i][j] = Zero(); } } } @@ -249,7 +251,7 @@ struct Contractions_NT { } else { #pragma unroll for (int j = 0; j < P::Veclen; ++j) { - ldgDataY[i][j] = Zero; + ldgDataY[i][j] = Zero(); } } } diff --git a/cpp/include/raft/linalg/detail/gemm.hpp b/cpp/include/raft/linalg/detail/gemm.hpp index 236c840040..af6a78638c 100644 --- a/cpp/include/raft/linalg/detail/gemm.hpp +++ b/cpp/include/raft/linalg/detail/gemm.hpp @@ -27,82 +27,83 @@ namespace raft::linalg::detail { -template +template void legacy_gemm(raft::resources const& res, const bool trans_a, const bool trans_b, const int m, const int n, const int k, - const T* alpha, - const T* A, + const S_T* alpha, + const A_T* A, const int lda, - const T* B, + const B_T* B, const int ldb, - const T* beta, - T* C, + const S_T* beta, + C_T* C, const int ldc, cudaStream_t stream) { - return legacy_matmul(res, - trans_a, - trans_b, - static_cast(m), - static_cast(n), - static_cast(k), - alpha, - A, - static_cast(lda), - B, - static_cast(ldb), - beta, - C, - static_cast(ldc), - stream); + return legacy_matmul(res, + trans_a, + trans_b, + static_cast(m), + static_cast(n), + static_cast(k), + alpha, + A, + static_cast(lda), + B, + static_cast(ldb), + beta, + C, + static_cast(ldc), + stream); } -template +template void legacy_gemm(raft::resources const& res, - const T* a, + const A_T* a, int n_rows_a, int n_cols_a, - const T* b, - T* c, + const B_T* b, + C_T* c, int n_rows_c, int n_cols_c, cublasOperation_t trans_a, cublasOperation_t trans_b, - T alpha, - T beta, + S_T alpha, + S_T beta, cudaStream_t stream) { int m = n_rows_c; int n = n_cols_c; auto k = trans_a == CUBLAS_OP_T ? n_rows_a : n_cols_a; - return legacy_matmul(res, - trans_a == CUBLAS_OP_T, - trans_b == CUBLAS_OP_T, - static_cast(n_rows_c), - static_cast(n_cols_c), - static_cast(k), - &alpha, - a, - static_cast(trans_a == CUBLAS_OP_T ? k : m), - b, - static_cast(trans_b == CUBLAS_OP_T ? n : k), - &beta, - c, - static_cast(m), - stream); + return legacy_matmul( + res, + trans_a == CUBLAS_OP_T, + trans_b == CUBLAS_OP_T, + static_cast(n_rows_c), + static_cast(n_cols_c), + static_cast(k), + &alpha, + a, + static_cast(trans_a == CUBLAS_OP_T ? k : m), + b, + static_cast(trans_b == CUBLAS_OP_T ? n : k), + &beta, + c, + static_cast(m), + stream); } -template +template void legacy_gemm(raft::resources const& res, - const T* a, + const A_T* a, int n_rows_a, int n_cols_a, - const T* b, - T* c, + const B_T* b, + C_T* c, int n_rows_c, int n_cols_c, cublasOperation_t trans_a, @@ -110,14 +111,14 @@ void legacy_gemm(raft::resources const& res, cudaStream_t stream) { return legacy_gemm( - res, a, n_rows_a, n_cols_a, b, c, n_rows_c, n_cols_c, trans_a, trans_b, T{1}, T{0}, stream); + res, a, n_rows_a, n_cols_a, b, c, n_rows_c, n_cols_c, trans_a, trans_b, C_T{1}, C_T{0}, stream); } -template +template void legacy_gemm(raft::resources const& res, - T* z, - T* x, - T* y, + z_T* z, + x_T* x, + y_T* y, int _M, int _N, int _K, @@ -125,11 +126,11 @@ void legacy_gemm(raft::resources const& res, bool isXColMajor, bool isYColMajor, cudaStream_t stream, - const T* alpha, - const T* beta) + const s_T* alpha, + const s_T* beta) { if (isZColMajor) { - return legacy_matmul( + return legacy_matmul( res, !isXColMajor, !isYColMajor, @@ -146,7 +147,7 @@ void legacy_gemm(raft::resources const& res, static_cast(_M), stream); } else { - return legacy_gemm( + return legacy_gemm( res, z, y, x, _N, _M, _K, true, !isYColMajor, !isXColMajor, stream, alpha, beta); } } diff --git a/cpp/include/raft/linalg/detail/norm.cuh b/cpp/include/raft/linalg/detail/norm.cuh index ed7e360848..24da634575 100644 --- a/cpp/include/raft/linalg/detail/norm.cuh +++ b/cpp/include/raft/linalg/detail/norm.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,8 +24,8 @@ namespace raft { namespace linalg { namespace detail { -template -void rowNormCaller(Type* dots, +template +void rowNormCaller(OutType* dots, const Type* data, IdxType D, IdxType N, @@ -36,53 +36,53 @@ void rowNormCaller(Type* dots, { switch (type) { case L1Norm: - raft::linalg::reduce(dots, - data, - D, - N, - (Type)0, - rowMajor, - true, - stream, - false, - raft::abs_op(), - raft::add_op(), - fin_op); + raft::linalg::reduce(dots, + data, + D, + N, + (OutType)0, + rowMajor, + true, + stream, + false, + raft::abs_op(), + raft::add_op(), + fin_op); break; case L2Norm: - raft::linalg::reduce(dots, - data, - D, - N, - (Type)0, - rowMajor, - true, - stream, - false, - raft::sq_op(), - raft::add_op(), - fin_op); + raft::linalg::reduce(dots, + data, + D, + N, + (OutType)0, + rowMajor, + true, + stream, + false, + raft::sq_op(), + raft::add_op(), + fin_op); break; case LinfNorm: - raft::linalg::reduce(dots, - data, - D, - N, - (Type)0, - rowMajor, - true, - stream, - false, - raft::abs_op(), - raft::max_op(), - fin_op); + raft::linalg::reduce(dots, + data, + D, + N, + (OutType)0, + rowMajor, + true, + stream, + false, + raft::abs_op(), + raft::max_op(), + fin_op); break; default: THROW("Unsupported norm type: %d", type); }; } -template -void colNormCaller(Type* dots, +template +void colNormCaller(OutType* dots, const Type* data, IdxType D, IdxType N, @@ -93,46 +93,46 @@ void colNormCaller(Type* dots, { switch (type) { case L1Norm: - raft::linalg::reduce(dots, - data, - D, - N, - (Type)0, - rowMajor, - false, - stream, - false, - raft::abs_op(), - raft::add_op(), - fin_op); + raft::linalg::reduce(dots, + data, + D, + N, + (OutType)0, + rowMajor, + false, + stream, + false, + raft::abs_op(), + raft::add_op(), + fin_op); break; case L2Norm: - raft::linalg::reduce(dots, - data, - D, - N, - (Type)0, - rowMajor, - false, - stream, - false, - raft::sq_op(), - raft::add_op(), - fin_op); + raft::linalg::reduce(dots, + data, + D, + N, + (OutType)0, + rowMajor, + false, + stream, + false, + raft::sq_op(), + raft::add_op(), + fin_op); break; case LinfNorm: - raft::linalg::reduce(dots, - data, - D, - N, - (Type)0, - rowMajor, - false, - stream, - false, - raft::abs_op(), - raft::max_op(), - fin_op); + raft::linalg::reduce(dots, + data, + D, + N, + (OutType)0, + rowMajor, + false, + stream, + false, + raft::abs_op(), + raft::max_op(), + fin_op); break; default: THROW("Unsupported norm type: %d", type); }; diff --git a/cpp/include/raft/linalg/detail/transpose.cuh b/cpp/include/raft/linalg/detail/transpose.cuh index 999e7f1974..ec60aacc9c 100644 --- a/cpp/include/raft/linalg/detail/transpose.cuh +++ b/cpp/include/raft/linalg/detail/transpose.cuh @@ -28,10 +28,84 @@ #include #include +#include + namespace raft { namespace linalg { namespace detail { +template +RAFT_KERNEL transpose_half_kernel(IndexType n_rows, + IndexType n_cols, + const half* __restrict__ in, + half* __restrict__ out) +{ + __shared__ half tile[TILE_DIM][TILE_DIM + 1]; + + for (int block_offset_y = 0; block_offset_y < n_rows; block_offset_y += gridDim.y * TILE_DIM) { + for (int block_offset_x = 0; block_offset_x < n_cols; block_offset_x += gridDim.x * TILE_DIM) { + auto x = block_offset_x + blockIdx.x * TILE_DIM + threadIdx.x; + auto y = block_offset_y + blockIdx.y * TILE_DIM + threadIdx.y; + + for (int j = 0; j < TILE_DIM; j += BLOCK_ROWS) { + if (x < n_cols && (y + j) < n_rows) { + tile[threadIdx.y + j][threadIdx.x] = __ldg(&in[(y + j) * n_cols + x]); + } + } + __syncthreads(); + + x = block_offset_y + blockIdx.y * TILE_DIM + threadIdx.x; + y = block_offset_x + blockIdx.x * TILE_DIM + threadIdx.y; + + for (int j = 0; j < TILE_DIM; j += BLOCK_ROWS) { + if (x < n_rows && (y + j) < n_cols) { + out[(y + j) * n_rows + x] = tile[threadIdx.x][threadIdx.y + j]; + } + } + __syncthreads(); + } + } +} + +template +void transpose_half( + raft::resources const& handle, IndexType n_rows, IndexType n_cols, const half* in, half* out) +{ + if (n_cols == 0 || n_rows == 0) return; + auto stream = resource::get_cuda_stream(handle); + + int dev_id, sm_count; + + cudaGetDevice(&dev_id); + cudaDeviceGetAttribute(&sm_count, cudaDevAttrMultiProcessorCount, dev_id); + + constexpr int tpb = 256; + constexpr int block_dim_x = 128 / sizeof(half); + constexpr int block_dim_y = tpb / block_dim_x; + + dim3 blocks(block_dim_x, block_dim_y); + + int max_active_blocks = 0; + RAFT_CUDA_TRY(cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &max_active_blocks, transpose_half_kernel, tpb, 0)); + int num_blocks = max_active_blocks * sm_count; + + int grid_x = (n_cols + block_dim_x - 1) / block_dim_x; + int grid_y = (n_rows + block_dim_x - 1) / block_dim_x; + + float ratio = static_cast(grid_y) / static_cast(grid_x); + int adjusted_grid_y = + std::max(std::min(grid_y, static_cast(std::sqrt(num_blocks * ratio))), 1); + int adjusted_grid_x = std::max(std::min(grid_x, num_blocks / adjusted_grid_y), 1); + + dim3 grids(adjusted_grid_x, adjusted_grid_y); + + transpose_half_kernel + <<>>(n_rows, n_cols, in, out); + + RAFT_CUDA_TRY(cudaPeekAtLastError()); +} + template void transpose(raft::resources const& handle, math_t* in, @@ -40,28 +114,31 @@ void transpose(raft::resources const& handle, int n_cols, cudaStream_t stream) { - cublasHandle_t cublas_h = resource::get_cublas_handle(handle); - RAFT_CUBLAS_TRY(cublasSetStream(cublas_h, stream)); - int out_n_rows = n_cols; int out_n_cols = n_rows; - const math_t alpha = 1.0; - const math_t beta = 0.0; - RAFT_CUBLAS_TRY(cublasgeam(cublas_h, - CUBLAS_OP_T, - CUBLAS_OP_N, - out_n_rows, - out_n_cols, - &alpha, - in, - n_rows, - &beta, - out, - out_n_rows, - out, - out_n_rows, - stream)); + if constexpr (std::is_same_v) { + transpose_half(handle, out_n_rows, out_n_cols, in, out); + } else { + cublasHandle_t cublas_h = resource::get_cublas_handle(handle); + RAFT_CUBLAS_TRY(cublasSetStream(cublas_h, stream)); + const math_t alpha = 1.0; + const math_t beta = 0.0; + RAFT_CUBLAS_TRY(cublasgeam(cublas_h, + CUBLAS_OP_T, + CUBLAS_OP_N, + out_n_rows, + out_n_cols, + &alpha, + in, + n_rows, + &beta, + out, + out_n_rows, + out, + out_n_rows, + stream)); + } } template @@ -112,6 +189,17 @@ void transpose_row_major_impl( resource::get_cuda_stream(handle))); } +template +void transpose_row_major_impl( + raft::resources const& handle, + raft::mdspan, LayoutPolicy, AccessorPolicy> in, + raft::mdspan, LayoutPolicy, AccessorPolicy> out) +{ + auto out_n_rows = in.extent(1); + auto out_n_cols = in.extent(0); + transpose_half(handle, out_n_cols, out_n_rows, in.data_handle(), out.data_handle()); +} + template void transpose_col_major_impl( raft::resources const& handle, @@ -138,6 +226,18 @@ void transpose_col_major_impl( out.stride(1), resource::get_cuda_stream(handle))); } + +template +void transpose_col_major_impl( + raft::resources const& handle, + raft::mdspan, LayoutPolicy, AccessorPolicy> in, + raft::mdspan, LayoutPolicy, AccessorPolicy> out) +{ + auto out_n_rows = in.extent(1); + auto out_n_cols = in.extent(0); + transpose_half(handle, out_n_rows, out_n_cols, in.data_handle(), out.data_handle()); +} + }; // end namespace detail }; // end namespace linalg }; // end namespace raft diff --git a/cpp/include/raft/linalg/gemm.cuh b/cpp/include/raft/linalg/gemm.cuh index 7b8d35706b..5444d0c861 100644 --- a/cpp/include/raft/linalg/gemm.cuh +++ b/cpp/include/raft/linalg/gemm.cuh @@ -41,7 +41,10 @@ namespace raft::linalg { * @brief the wrapper of cublas gemm function * It computes the following equation: C = alpha .* opA(A) * opB(B) + beta .* C * - * @tparam math_t the element type + * @tparam A_t the element type of A + * @tparam B_t the element type of B + * @tparam C_t the element type of C + * @tparam S_t the element type of alpha and beta * @tparam DevicePointerMode whether pointers alpha, beta point to device memory * @param [in] handle raft handle * @param [in] trans_a cublas transpose op for A @@ -59,20 +62,20 @@ namespace raft::linalg { * @param [in] ldc leading dimension of C * @param [in] stream */ -template +template void gemm(raft::resources const& handle, const bool trans_a, const bool trans_b, const int m, const int n, const int k, - const math_t* alpha, - const math_t* A, + const S_t* alpha, + const A_t* A, const int lda, - const math_t* B, + const B_t* B, const int ldb, - const math_t* beta, - math_t* C, + const S_t* beta, + C_t* C, const int ldc, cudaStream_t stream) { @@ -83,7 +86,10 @@ void gemm(raft::resources const& handle, /** * @brief the wrapper of cublas gemm function * It computes the following equation: D = alpha . opA(A) * opB(B) + beta . C - * @tparam math_t the type of input/output matrices + * @tparam A_t the element type of A + * @tparam B_t the element type of B + * @tparam C_t the element type of C + * @tparam S_t the element type of alpha and beta * @param handle raft handle * @param a input matrix * @param n_rows_a number of rows of A @@ -98,19 +104,19 @@ void gemm(raft::resources const& handle, * @param beta scalar * @param stream cuda stream */ -template +template void gemm(raft::resources const& handle, - const math_t* a, + const A_t* a, int n_rows_a, int n_cols_a, - const math_t* b, - math_t* c, + const B_t* b, + C_t* c, int n_rows_c, int n_cols_c, cublasOperation_t trans_a, cublasOperation_t trans_b, - math_t alpha, - math_t beta, + S_t alpha, + S_t beta, cudaStream_t stream) { detail::legacy_gemm( @@ -120,7 +126,9 @@ void gemm(raft::resources const& handle, /** * @brief the wrapper of cublas gemm function * It computes the following equation: D = alpha . opA(A) * opB(B) + beta . C - * @tparam math_t the type of input/output matrices + * @tparam A_t the element type of A + * @tparam B_t the element type of B + * @tparam C_t the element type of C * @param handle raft handle * @param a input matrix * @param n_rows_a number of rows of A @@ -133,13 +141,13 @@ void gemm(raft::resources const& handle, * @param trans_b cublas transpose op for B * @param stream cuda stream */ -template +template void gemm(raft::resources const& handle, - const math_t* a, + const A_t* a, int n_rows_a, int n_cols_a, - const math_t* b, - math_t* c, + const B_t* b, + C_t* c, int n_rows_c, int n_cols_c, cublasOperation_t trans_a, @@ -154,7 +162,10 @@ void gemm(raft::resources const& handle, * @brief A wrapper for CUBLS GEMM function designed for handling all possible * combinations of operand layouts. * It computes the following equation: Z = alpha . X * Y + beta . Z - * @tparam T Data type of input/output matrices (float/double) + * @tparam z_T the element type of z + * @tparam x_T the element type of x + * @tparam y_T the element type of y + * @tparam s_T the element type of alpha and beta, equal to z_T by default * @param handle raft handle * @param z output matrix of size M rows x N columns * @param x input matrix of size M rows x K columns @@ -169,11 +180,11 @@ void gemm(raft::resources const& handle, * @param alpha scalar * @param beta scalar */ -template +template void gemm(raft::resources const& handle, - T* z, - T* x, - T* y, + z_T* z, + x_T* x, + y_T* y, int _M, int _N, int _K, @@ -181,10 +192,10 @@ void gemm(raft::resources const& handle, bool isXColMajor, bool isYColMajor, cudaStream_t stream, - T alpha = T(1.0), - T beta = T(0.0)) + s_T alpha = s_T(1.0), + s_T beta = s_T(0.0)) { - return detail::legacy_gemm( + return detail::legacy_gemm( handle, z, x, y, _M, _N, _K, isZColMajor, isXColMajor, isYColMajor, stream, &alpha, &beta); } diff --git a/cpp/include/raft/linalg/norm.cuh b/cpp/include/raft/linalg/norm.cuh index 97a5d6135d..4270149793 100644 --- a/cpp/include/raft/linalg/norm.cuh +++ b/cpp/include/raft/linalg/norm.cuh @@ -41,6 +41,7 @@ namespace linalg { * @tparam Type the data type * @tparam Lambda device final lambda * @tparam IdxType Integer type used to for addressing + * @tparam OutType output type, equal to Type by default * @param dots the output vector of row-wise dot products * @param data the input matrix * @param D number of columns of data @@ -50,8 +51,11 @@ namespace linalg { * @param stream cuda stream where to launch work * @param fin_op the final lambda op */ -template -void rowNorm(Type* dots, +template +void rowNorm(OutType* dots, const Type* data, IdxType D, IdxType N, @@ -68,6 +72,7 @@ void rowNorm(Type* dots, * @tparam Type the data type * @tparam Lambda device final lambda * @tparam IdxType Integer type used to for addressing + * @tparam OutType output type, equal to Type by default * @param dots the output vector of column-wise dot products * @param data the input matrix * @param D number of columns of data @@ -77,8 +82,11 @@ void rowNorm(Type* dots, * @param stream cuda stream where to launch work * @param fin_op the final lambda op */ -template -void colNorm(Type* dots, +template +void colNorm(OutType* dots, const Type* data, IdxType D, IdxType N, @@ -97,7 +105,8 @@ void colNorm(Type* dots, /** * @brief Compute norm of the input matrix and perform fin_op - * @tparam ElementType Input/Output data type + * @tparam ElementType Input data type + * @tparam OutType output data type * @tparam LayoutPolicy the layout of input (raft::row_major or raft::col_major) * @tparam IdxType Integer type used to for addressing * @tparam Lambda device final lambda @@ -110,12 +119,13 @@ void colNorm(Type* dots, * @param[in] fin_op the final lambda op */ template void norm(raft::resources const& handle, raft::device_matrix_view in, - raft::device_vector_view out, + raft::device_vector_view out, NormType type, Apply apply, Lambda fin_op = raft::identity_op()) diff --git a/cpp/include/raft/random/detail/rng_device.cuh b/cpp/include/raft/random/detail/rng_device.cuh index 12c67679ba..ffbb87bd0c 100644 --- a/cpp/include/raft/random/detail/rng_device.cuh +++ b/cpp/include/raft/random/detail/rng_device.cuh @@ -22,6 +22,8 @@ #include +#include + #include #include @@ -504,6 +506,12 @@ struct PhiloxGenerator { return ret; } + DI half next_half() + { + float ret = next_float(); + return __float2half(ret); + } + DI void next(float& ret) { // ret = curand_uniform(&(this->philox_state)); @@ -516,6 +524,12 @@ struct PhiloxGenerator { ret = next_double(); } + DI void next(half& ret) + { + // ret = curand_uniform_double(&(this->philox_state)); + ret = next_half(); + } + DI void next(uint32_t& ret) { ret = next_u32(); } DI void next(uint64_t& ret) { ret = next_u64(); } DI void next(int32_t& ret) { ret = next_i32(); } @@ -636,6 +650,12 @@ struct PCGenerator { return ret; } + HDI half next_half() + { + float ret = next_float(); + return __float2half(ret); + } + HDI void next(uint32_t& ret) { ret = next_u32(); } HDI void next(uint64_t& ret) { ret = next_u64(); } HDI void next(int32_t& ret) { ret = next_i32(); } @@ -643,6 +663,7 @@ struct PCGenerator { HDI void next(float& ret) { ret = next_float(); } HDI void next(double& ret) { ret = next_double(); } + HDI void next(half& ret) { ret = next_half(); } /** @} */ diff --git a/cpp/include/raft/random/detail/rng_impl.cuh b/cpp/include/raft/random/detail/rng_impl.cuh index 61a944e9b6..88654dbe5d 100644 --- a/cpp/include/raft/random/detail/rng_impl.cuh +++ b/cpp/include/raft/random/detail/rng_impl.cuh @@ -30,6 +30,7 @@ #include #include +#include namespace raft { namespace random { @@ -85,7 +86,7 @@ template void uniform( RngState& rng_state, OutType* ptr, LenType len, OutType start, OutType end, cudaStream_t stream) { - static_assert(std::is_floating_point::value, + static_assert(std::is_floating_point::value || std::is_same_v, "Type for 'uniform' can only be floating point!"); UniformDistParams params; params.start = start; diff --git a/cpp/include/raft/sparse/convert/detail/bitmap_to_csr.cuh b/cpp/include/raft/sparse/convert/detail/bitmap_to_csr.cuh index b1b0291a85..769d5de9be 100644 --- a/cpp/include/raft/sparse/convert/detail/bitmap_to_csr.cuh +++ b/cpp/include/raft/sparse/convert/detail/bitmap_to_csr.cuh @@ -66,24 +66,30 @@ RAFT_KERNEL __launch_bounds__(calc_nnz_by_rows_tpb) calc_nnz_by_rows_kernel(cons index_t e_bit = s_bit + num_cols; index_t l_sum = 0; + int s_gap = 0; + int e_gap = 0; + while (offset < num_cols) { index_t bitmap_idx = lane_id + (s_bit + offset) / BITS_PER_BITMAP; std::remove_const_t l_bitmap = 0; if (bitmap_idx * BITS_PER_BITMAP < e_bit) { l_bitmap = bitmap[bitmap_idx]; } - if (s_bit > bitmap_idx * BITS_PER_BITMAP) { - l_bitmap >>= (s_bit - bitmap_idx * BITS_PER_BITMAP); - l_bitmap <<= (s_bit - bitmap_idx * BITS_PER_BITMAP); - } + offset += BITS_PER_BITMAP * warpSize; - if ((bitmap_idx + 1) * BITS_PER_BITMAP > e_bit) { - l_bitmap <<= ((bitmap_idx + 1) * BITS_PER_BITMAP - e_bit); - l_bitmap >>= ((bitmap_idx + 1) * BITS_PER_BITMAP - e_bit); + s_gap = s_bit - bitmap_idx * BITS_PER_BITMAP; + if (s_gap > 0) { + l_bitmap >>= s_gap; + l_bitmap <<= s_gap; + offset -= s_gap; } + e_gap = (bitmap_idx + 1) * BITS_PER_BITMAP - e_bit; + if (e_gap > 0) { + l_bitmap <<= e_gap; + l_bitmap >>= e_gap; + } l_sum += static_cast(raft::detail::popc(l_bitmap)); - offset += BITS_PER_BITMAP * warpSize; } l_sum = cg::reduce(tile, l_sum, cg::plus()); diff --git a/cpp/include/raft/sparse/detail/cusparse_wrappers.h b/cpp/include/raft/sparse/detail/cusparse_wrappers.h index ae552cc687..53a78a8f56 100644 --- a/cpp/include/raft/sparse/detail/cusparse_wrappers.h +++ b/cpp/include/raft/sparse/detail/cusparse_wrappers.h @@ -207,6 +207,27 @@ inline cusparseStatus_t cusparsecreatecsr(cusparseSpMatDescr_t* spMatDescr, CUDA_R_64F); } template <> +inline cusparseStatus_t cusparsecreatecsr(cusparseSpMatDescr_t* spMatDescr, + int64_t rows, + int64_t cols, + int64_t nnz, + int32_t* csrRowOffsets, + int32_t* csrColInd, + half* csrValues) +{ + return cusparseCreateCsr(spMatDescr, + rows, + cols, + nnz, + csrRowOffsets, + csrColInd, + csrValues, + CUSPARSE_INDEX_32I, + CUSPARSE_INDEX_32I, + CUSPARSE_INDEX_BASE_ZERO, + CUDA_R_16F); +} +template <> inline cusparseStatus_t cusparsecreatecsr(cusparseSpMatDescr_t* spMatDescr, int64_t rows, int64_t cols, @@ -302,6 +323,16 @@ inline cusparseStatus_t cusparsecreatednmat(cusparseDnMatDescr_t* dnMatDescr, { return cusparseCreateDnMat(dnMatDescr, rows, cols, ld, values, CUDA_R_64F, order); } +template <> +inline cusparseStatus_t cusparsecreatednmat(cusparseDnMatDescr_t* dnMatDescr, + int64_t rows, + int64_t cols, + int64_t ld, + half* values, + cusparseOrder_t order) +{ + return cusparseCreateDnMat(dnMatDescr, rows, cols, ld, values, CUDA_R_16F, order); +} /** @} */ /** @@ -658,7 +689,7 @@ inline cusparseStatus_t cusparsesddmm(cusparseHandle_t handle, const T* beta, cusparseSpMatDescr_t matC, cusparseSDDMMAlg_t alg, - T* externalBuffer, + void* externalBuffer, cudaStream_t stream); template <> inline cusparseStatus_t cusparsesddmm(cusparseHandle_t handle, @@ -670,7 +701,7 @@ inline cusparseStatus_t cusparsesddmm(cusparseHandle_t handle, const float* beta, cusparseSpMatDescr_t matC, cusparseSDDMMAlg_t alg, - float* externalBuffer, + void* externalBuffer, cudaStream_t stream) { CUSPARSE_CHECK(cusparseSetStream(handle, stream)); @@ -684,7 +715,7 @@ inline cusparseStatus_t cusparsesddmm(cusparseHandle_t handle, matC, CUDA_R_32F, alg, - static_cast(externalBuffer)); + externalBuffer); } template <> inline cusparseStatus_t cusparsesddmm(cusparseHandle_t handle, @@ -696,7 +727,7 @@ inline cusparseStatus_t cusparsesddmm(cusparseHandle_t handle, const double* beta, cusparseSpMatDescr_t matC, cusparseSDDMMAlg_t alg, - double* externalBuffer, + void* externalBuffer, cudaStream_t stream) { CUSPARSE_CHECK(cusparseSetStream(handle, stream)); @@ -710,7 +741,34 @@ inline cusparseStatus_t cusparsesddmm(cusparseHandle_t handle, matC, CUDA_R_64F, alg, - static_cast(externalBuffer)); + externalBuffer); +} + +template <> +inline cusparseStatus_t cusparsesddmm(cusparseHandle_t handle, + cusparseOperation_t opA, + cusparseOperation_t opB, + const half* alpha, + const cusparseDnMatDescr_t matA, + const cusparseDnMatDescr_t matB, + const half* beta, + cusparseSpMatDescr_t matC, + cusparseSDDMMAlg_t alg, + void* externalBuffer, + cudaStream_t stream) +{ + CUSPARSE_CHECK(cusparseSetStream(handle, stream)); + return cusparseSDDMM(handle, + opA, + opB, + static_cast(alpha), + matA, + matB, + static_cast(beta), + matC, + CUDA_R_16F, + alg, + externalBuffer); } /** @} */ diff --git a/cpp/include/raft/sparse/distance/detail/utils.cuh b/cpp/include/raft/sparse/distance/detail/utils.cuh index 42b545180b..864d61ba2f 100644 --- a/cpp/include/raft/sparse/distance/detail/utils.cuh +++ b/cpp/include/raft/sparse/distance/detail/utils.cuh @@ -20,6 +20,7 @@ #include #include +#include #include namespace raft { @@ -41,8 +42,8 @@ inline int max_cols_per_block() sizeof(value_t); } -template -RAFT_KERNEL faster_dot_on_csr_kernel(value_t* __restrict__ dot, +template +RAFT_KERNEL faster_dot_on_csr_kernel(dot_t* __restrict__ dot, const value_idx* __restrict__ indptr, const value_idx* __restrict__ cols, const value_t* __restrict__ A, @@ -74,25 +75,28 @@ RAFT_KERNEL faster_dot_on_csr_kernel(value_t* __restrict__ dot, cur_row = row; } - value_t l_dot_ = 0.0; + dot_t l_dot_ = 0.0; for (value_idx k = vec_id; k < dim; k += blockDim.x) { asm("prefetch.global.L2 [%0];" ::"l"(B_col + k + blockDim.x)); - l_dot_ += s_A[k] * __ldcg(B_col + k); + if constexpr ((std::is_same_v && std::is_same_v)) { + l_dot_ += __half2float(s_A[k]) * __half2float(__ldcg(B_col + k)); + } else { + l_dot_ += s_A[k] * __ldcg(B_col + k); + } } - l_dot_ += __shfl_down_sync(0xffffffff, l_dot_, 16); - l_dot_ += __shfl_down_sync(0xffff, l_dot_, 8); - l_dot_ += __shfl_down_sync(0xff, l_dot_, 4); - l_dot_ += __shfl_down_sync(0xf, l_dot_, 2); - l_dot_ += __shfl_down_sync(0x3, l_dot_, 1); - if (lane_id == 0) { atomicAdd_block(dot + dot_id, l_dot_); } + typedef cub::WarpReduce WarpReduce; + __shared__ typename WarpReduce::TempStorage temp_storage; + dot_t warp_sum = WarpReduce(temp_storage).Sum(l_dot_); + + if (lane_id == 0) { atomicAdd_block(dot + dot_id, warp_sum); } } } } -template +template void faster_dot_on_csr(raft::resources const& handle, - value_t* dot, + dot_t* dot, const value_idx nnz, const value_idx* indptr, const value_idx* cols, @@ -115,47 +119,47 @@ void faster_dot_on_csr(raft::resources const& handle, if (dim < 128) { constexpr int tpb = 64; cudaOccupancyMaxActiveBlocksPerMultiprocessor( - &blocks_per_sm, faster_dot_on_csr_kernel, tpb, smem_size); + &blocks_per_sm, faster_dot_on_csr_kernel, tpb, smem_size); auto block_x = std::min(n_rows, MAX_ROW_PER_ITER); auto block_y = (std::min(value_idx(blocks_per_sm * sm_count * 16), nnz) + block_x - 1) / block_x; dim3 blocks(block_x, block_y, 1); - faster_dot_on_csr_kernel + faster_dot_on_csr_kernel <<>>(dot, indptr, cols, A, B, nnz, n_rows, dim); } else if (dim < 256) { constexpr int tpb = 128; cudaOccupancyMaxActiveBlocksPerMultiprocessor( - &blocks_per_sm, faster_dot_on_csr_kernel, tpb, smem_size); + &blocks_per_sm, faster_dot_on_csr_kernel, tpb, smem_size); auto block_x = std::min(n_rows, MAX_ROW_PER_ITER); auto block_y = (std::min(value_idx(blocks_per_sm * sm_count * 16), nnz) + block_x - 1) / block_x; dim3 blocks(block_x, block_y, 1); - faster_dot_on_csr_kernel + faster_dot_on_csr_kernel <<>>(dot, indptr, cols, A, B, nnz, n_rows, dim); } else if (dim < 512) { constexpr int tpb = 256; cudaOccupancyMaxActiveBlocksPerMultiprocessor( - &blocks_per_sm, faster_dot_on_csr_kernel, tpb, smem_size); + &blocks_per_sm, faster_dot_on_csr_kernel, tpb, smem_size); auto block_x = std::min(n_rows, MAX_ROW_PER_ITER); auto block_y = (std::min(value_idx(blocks_per_sm * sm_count * 16), nnz) + block_x - 1) / block_x; dim3 blocks(block_x, block_y, 1); - faster_dot_on_csr_kernel + faster_dot_on_csr_kernel <<>>(dot, indptr, cols, A, B, nnz, n_rows, dim); } else { constexpr int tpb = 512; cudaOccupancyMaxActiveBlocksPerMultiprocessor( - &blocks_per_sm, faster_dot_on_csr_kernel, tpb, smem_size); + &blocks_per_sm, faster_dot_on_csr_kernel, tpb, smem_size); auto block_x = std::min(n_rows, MAX_ROW_PER_ITER); auto block_y = (std::min(value_idx(blocks_per_sm * sm_count * 16), nnz) + block_x - 1) / block_x; dim3 blocks(block_x, block_y, 1); - faster_dot_on_csr_kernel + faster_dot_on_csr_kernel <<>>(dot, indptr, cols, A, B, nnz, n_rows, dim); } diff --git a/cpp/include/raft/sparse/linalg/detail/masked_matmul.cuh b/cpp/include/raft/sparse/linalg/detail/masked_matmul.cuh index ef74316d04..276960628d 100644 --- a/cpp/include/raft/sparse/linalg/detail/masked_matmul.cuh +++ b/cpp/include/raft/sparse/linalg/detail/masked_matmul.cuh @@ -37,14 +37,14 @@ namespace sparse { namespace linalg { namespace detail { -template +template void masked_matmul(raft::resources const& handle, raft::device_matrix_view& A, raft::device_matrix_view& B, raft::core::bitmap_view& mask, - raft::device_csr_matrix_view& C, - std::optional> alpha, - std::optional> beta) + raft::device_csr_matrix_view& C, + std::optional> alpha, + std::optional> beta) { index_t m = A.extent(0); index_t n = B.extent(0); @@ -60,24 +60,24 @@ void masked_matmul(raft::resources const& handle, auto stream = raft::resource::get_cuda_stream(handle); - auto C_matrix = raft::make_device_csr_matrix(handle, compressed_C_view); + auto C_matrix = raft::make_device_csr_matrix(handle, compressed_C_view); // fill C raft::sparse::convert::bitmap_to_csr(handle, mask, C_matrix); if (m > 10 || alpha.has_value() || beta.has_value()) { - auto C_view = raft::make_device_csr_matrix_view( + auto C_view = raft::make_device_csr_matrix_view( C.get_elements().data(), compressed_C_view); // create B col_major view auto B_col_major = raft::make_device_matrix_view( B.data_handle(), dim, n); - value_t default_alpha = static_cast(1.0f); - value_t default_beta = static_cast(0.0f); + output_t default_alpha = static_cast(1.0f); + output_t default_beta = static_cast(0.0f); - if (!alpha.has_value()) { alpha = raft::make_host_scalar_view(&default_alpha); } - if (!beta.has_value()) { beta = raft::make_host_scalar_view(&default_beta); } + if (!alpha.has_value()) { alpha = raft::make_host_scalar_view(&default_alpha); } + if (!beta.has_value()) { beta = raft::make_host_scalar_view(&default_beta); } raft::sparse::linalg::sddmm(handle, A, diff --git a/cpp/include/raft/sparse/linalg/detail/sddmm.hpp b/cpp/include/raft/sparse/linalg/detail/sddmm.hpp index 5088a20f46..f2e4aba644 100644 --- a/cpp/include/raft/sparse/linalg/detail/sddmm.hpp +++ b/cpp/include/raft/sparse/linalg/detail/sddmm.hpp @@ -35,11 +35,7 @@ namespace detail { * It computes the following equation: C = alpha · (op_a(A) * op_b(B) ∘ spy(C)) + beta · C * where A,B are device matrix views and C is a CSR device matrix view * - * @tparam ValueType Data type of input/output matrices (float/double) - * @tparam IndexType Type of C - * @tparam LayoutPolicyA layout of A - * @tparam LayoutPolicyB layout of B - * @tparam NZType Type of C + * @tparam OutputType Data type of input/output matrices (float/double) * * @param[in] handle raft resource handle * @param[in] descr_a input dense descriptor @@ -50,15 +46,15 @@ namespace detail { * @param[in] alpha scalar pointer * @param[in] beta scalar pointer */ -template +template void sddmm(raft::resources const& handle, cusparseDnMatDescr_t& descr_a, cusparseDnMatDescr_t& descr_b, cusparseSpMatDescr_t& descr_c, cusparseOperation_t op_a, cusparseOperation_t op_b, - const ValueType* alpha, - const ValueType* beta) + const OutputType* alpha, + const OutputType* beta) { auto alg = CUSPARSE_SDDMM_ALG_DEFAULT; size_t bufferSize; @@ -78,7 +74,7 @@ void sddmm(raft::resources const& handle, resource::sync_stream(handle); - rmm::device_uvector tmp(bufferSize, resource::get_cuda_stream(handle)); + rmm::device_uvector tmp(bufferSize, resource::get_cuda_stream(handle)); RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesddmm(resource::get_cusparse_handle(handle), op_a, @@ -89,7 +85,7 @@ void sddmm(raft::resources const& handle, beta, descr_c, alg, - tmp.data(), + reinterpret_cast(tmp.data()), resource::get_cuda_stream(handle))); } diff --git a/cpp/include/raft/sparse/linalg/masked_matmul.hpp b/cpp/include/raft/sparse/linalg/masked_matmul.hpp index 560cd3f715..6cf6e834b9 100644 --- a/cpp/include/raft/sparse/linalg/masked_matmul.hpp +++ b/cpp/include/raft/sparse/linalg/masked_matmul.hpp @@ -35,7 +35,8 @@ namespace linalg { * multiplication using the sparsity pattern provided by the mask. The result is scaled by alpha * and added to beta times the original matrix C. * - * @tparam value_t Data type of elements in the input/output matrices (e.g., float, double) + * @tparam value_t Data type of elements in the input matrices (e.g., half, float, double) + * @tparam output_t Data type of elements in the output matrices (e.g., float, double) * @tparam index_t Type used for matrix indices * @tparam nnz_t Type used for the number of non-zero entries in CSR format * @tparam bitmap_t Type of the bitmap used for the mask @@ -52,14 +53,14 @@ namespace linalg { * std::nullopt) * @param[in] beta Optional scalar multiplier for the original matrix C (default: 0 if std::nullopt) */ -template +template void masked_matmul(raft::resources const& handle, raft::device_matrix_view A, raft::device_matrix_view B, raft::core::bitmap_view mask, - raft::device_csr_matrix_view C, - std::optional> alpha = std::nullopt, - std::optional> beta = std::nullopt) + raft::device_csr_matrix_view C, + std::optional> alpha = std::nullopt, + std::optional> beta = std::nullopt) { detail::masked_matmul(handle, A, B, mask, C, alpha, beta); } diff --git a/cpp/include/raft/sparse/linalg/sddmm.hpp b/cpp/include/raft/sparse/linalg/sddmm.hpp index c19f1d9081..96387e6c8b 100644 --- a/cpp/include/raft/sparse/linalg/sddmm.hpp +++ b/cpp/include/raft/sparse/linalg/sddmm.hpp @@ -29,11 +29,12 @@ namespace linalg { * followed by an element-wise multiplication with the sparsity pattern of C. * It computes the following equation: C = alpha · (opA(A) * opB(B) ∘ spy(C)) + beta · C * where A,B are device matrix views and C is a CSR device matrix view - * @tparam ValueType Data type of input/output matrices (float/double) + * @tparam ValueType Data type of input/output matrices (float/double/half) * @tparam IndexType Type of C * @tparam NZType Type of C * @tparam LayoutPolicyA layout of A * @tparam LayoutPolicyB layout of B + * @tparam OutputType output type, equal to ValueType by default * @param[in] handle raft handle * @param[in] A input raft::device_matrix_view * @param[in] B input raft::device_matrix_view @@ -47,21 +48,23 @@ template + typename LayoutPolicyB, + typename OutputType> void sddmm(raft::resources const& handle, raft::device_matrix_view A, raft::device_matrix_view B, - raft::device_csr_matrix_view C, + raft::device_csr_matrix_view C, const raft::linalg::Operation opA, const raft::linalg::Operation opB, - raft::host_scalar_view alpha, - raft::host_scalar_view beta) + raft::host_scalar_view alpha, + raft::host_scalar_view beta) { RAFT_EXPECTS(raft::is_row_or_column_major(A), "A is not contiguous"); RAFT_EXPECTS(raft::is_row_or_column_major(B), "B is not contiguous"); - static_assert(std::is_same_v || std::is_same_v, - "The `ValueType` of sddmm only supports float/double."); + static_assert(std::is_same_v || std::is_same_v || + std::is_same_v, + "The `ValueType` of sddmm only supports float/double/half."); auto descrA = detail::create_descriptor(A); auto descrB = detail::create_descriptor(B); diff --git a/cpp/include/raft/spatial/knn/detail/epsilon_neighborhood.cuh b/cpp/include/raft/spatial/knn/detail/epsilon_neighborhood.cuh index 7a5a217959..0cc3b864fa 100644 --- a/cpp/include/raft/spatial/knn/detail/epsilon_neighborhood.cuh +++ b/cpp/include/raft/spatial/knn/detail/epsilon_neighborhood.cuh @@ -69,7 +69,7 @@ struct EpsUnexpL2SqNeighborhood : public BaseClass { for (int i = 0; i < P::AccRowsPerTh; ++i) { #pragma unroll for (int j = 0; j < P::AccColsPerTh; ++j) { - acc[i][j] = BaseClass::Zero; + acc[i][j] = BaseClass::Zero(); } } this->stsXY(); diff --git a/cpp/include/raft/util/cuda_dev_essentials.cuh b/cpp/include/raft/util/cuda_dev_essentials.cuh index bb9ebbba59..26f48af68b 100644 --- a/cpp/include/raft/util/cuda_dev_essentials.cuh +++ b/cpp/include/raft/util/cuda_dev_essentials.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,8 @@ #pragma once +#include + // This file provides a few essential functions for use in __device__ code. The // scope is necessarily limited to ensure that compilation times are minimized. // Please make sure not to include large / expensive files from here. @@ -114,4 +116,19 @@ HDI void swapVals(T& a, T& b) b = tmp; } +/** + * @brief Convert half to float + * @tparam T the datatype of the value + * @param a need to convert + */ +template +HDI auto to_float(T& a) +{ + if constexpr (std::is_same_v::type, half>) { + return __half2float(a); + } else { + return a; + } +} + } // namespace raft diff --git a/cpp/include/raft/util/cudart_utils.hpp b/cpp/include/raft/util/cudart_utils.hpp index 2b334d1bbf..f9e7f521be 100644 --- a/cpp/include/raft/util/cudart_utils.hpp +++ b/cpp/include/raft/util/cudart_utils.hpp @@ -189,7 +189,11 @@ void print_host_vector(const char* variable_name, out << variable_name << "=["; for (size_t i = 0; i < componentsCount; ++i) { if (i != 0) out << ","; - out << host_mem[i]; + if constexpr (std::is_same_v) { + out << __half2float(host_mem[i]); + } else { + out << host_mem[i]; + } } out << "];" << std::endl; } diff --git a/cpp/test/core/mdspan_copy.cu b/cpp/test/core/mdspan_copy.cu index b68ba38914..419c1e0859 100644 --- a/cpp/test/core/mdspan_copy.cu +++ b/cpp/test/core/mdspan_copy.cu @@ -161,6 +161,59 @@ TEST(MDSpanCopy, Mdspan2DDeviceDeviceCuda) } } } + +TEST(MDSpanCopy, Mdspan2DDeviceDeviceCudaHalfWithTranspose) +{ + auto res = device_resources{}; + auto constexpr rows = std::uint32_t{30}; + auto constexpr cols = std::uint32_t{20}; + auto in_left = make_device_mdarray( + res, extents{}); + auto in_right = make_device_mdarray( + res, extents{}); + auto gen_unique_entry = [](auto&& x, auto&& y) { return x * 7 + y * 11; }; + + for (auto i = std::uint32_t{}; i < rows; ++i) { + for (auto j = std::uint32_t{}; j < cols; ++j) { + in_left(i, j) = gen_unique_entry(i, j); + in_right(i, j) = gen_unique_entry(i, j); + } + } + + auto out_left = make_device_mdarray( + res, extents{}); + auto out_right = make_device_mdarray( + res, extents{}); + + res.sync_stream(); + + // Test dtype conversion with transpose + static_assert( + detail::mdspan_copyable_with_kernel_v, + "Current implementation should use kernel for this copy"); + copy(res, out_right.view(), in_left.view()); + res.sync_stream(); + for (auto i = std::uint32_t{}; i < rows; ++i) { + for (auto j = std::uint32_t{}; j < cols; ++j) { + ASSERT_TRUE(match(__half2float(out_right(i, j)), + __half2float(gen_unique_entry(i, j)), + CompareApprox{0.0001})); + } + } + static_assert( + detail::mdspan_copyable_with_kernel_v, + "Current implementation should use kernel for this copy"); + copy(res, out_left.view(), in_right.view()); + res.sync_stream(); + for (auto i = std::uint32_t{}; i < rows; ++i) { + for (auto j = std::uint32_t{}; j < cols; ++j) { + ASSERT_TRUE(match(__half2float(out_left(i, j)), + __half2float(gen_unique_entry(i, j)), + CompareApprox{0.0001})); + } + } +} + TEST(MDSpanCopy, Mdspan3DDeviceHostCuda) { auto res = device_resources{}; diff --git a/cpp/test/linalg/norm.cu b/cpp/test/linalg/norm.cu index e4f064949c..f91350f222 100644 --- a/cpp/test/linalg/norm.cu +++ b/cpp/test/linalg/norm.cu @@ -24,6 +24,8 @@ #include #include +#include + #include namespace raft { @@ -48,39 +50,48 @@ template } ///// Row-wise norm test definitions -template +template RAFT_KERNEL naiveRowNormKernel( - Type* dots, const Type* data, IdxT D, IdxT N, NormType type, bool do_sqrt) + OutType* dots, const Type* data, IdxT D, IdxT N, NormType type, bool do_sqrt) { - Type acc = (Type)0; + OutType acc = (OutType)0; IdxT rowStart = threadIdx.x + static_cast(blockIdx.x) * blockDim.x; if (rowStart < N) { for (IdxT i = 0; i < D; ++i) { - if (type == L2Norm) { - acc += data[rowStart * D + i] * data[rowStart * D + i]; + if constexpr (std::is_same_v) { + if (type == L2Norm) { + acc += __half2float(data[rowStart * D + i]) * __half2float(data[rowStart * D + i]); + } else { + acc += raft::abs(__half2float(data[rowStart * D + i])); + } } else { - acc += raft::abs(data[rowStart * D + i]); + if (type == L2Norm) { + acc += data[rowStart * D + i] * data[rowStart * D + i]; + } else { + acc += raft::abs(data[rowStart * D + i]); + } } } dots[rowStart] = do_sqrt ? raft::sqrt(acc) : acc; } } -template +template void naiveRowNorm( - Type* dots, const Type* data, IdxT D, IdxT N, NormType type, bool do_sqrt, cudaStream_t stream) + OutType* dots, const Type* data, IdxT D, IdxT N, NormType type, bool do_sqrt, cudaStream_t stream) { static const IdxT TPB = 64; IdxT nblks = raft::ceildiv(N, TPB); - naiveRowNormKernel<<>>(dots, data, D, N, type, do_sqrt); + naiveRowNormKernel + <<>>(dots, data, D, N, type, do_sqrt); RAFT_CUDA_TRY(cudaPeekAtLastError()); } -template -class RowNormTest : public ::testing::TestWithParam> { +template +class RowNormTest : public ::testing::TestWithParam> { public: RowNormTest() - : params(::testing::TestWithParam>::GetParam()), + : params(::testing::TestWithParam>::GetParam()), stream(resource::get_cuda_stream(handle)), data(params.rows * params.cols, stream), dots_exp(params.rows, stream), @@ -94,7 +105,7 @@ class RowNormTest : public ::testing::TestWithParam> { IdxT rows = params.rows, cols = params.cols, len = rows * cols; uniform(handle, r, data.data(), len, T(-1.0), T(1.0)); naiveRowNorm(dots_exp.data(), data.data(), cols, rows, params.type, params.do_sqrt, stream); - auto output_view = raft::make_device_vector_view(dots_act.data(), params.rows); + auto output_view = raft::make_device_vector_view(dots_act.data(), params.rows); auto input_row_major = raft::make_device_matrix_view( data.data(), params.rows, params.cols); auto input_col_major = raft::make_device_matrix_view( @@ -119,42 +130,44 @@ class RowNormTest : public ::testing::TestWithParam> { raft::resources handle; cudaStream_t stream; - NormInputs params; - rmm::device_uvector data, dots_exp, dots_act; + NormInputs params; + rmm::device_uvector data; + rmm::device_uvector dots_exp, dots_act; }; ///// Column-wise norm test definitisons -template +template RAFT_KERNEL naiveColNormKernel( - Type* dots, const Type* data, IdxT D, IdxT N, NormType type, bool do_sqrt) + OutType* dots, const Type* data, IdxT D, IdxT N, NormType type, bool do_sqrt) { IdxT colID = threadIdx.x + static_cast(blockIdx.x) * blockDim.x; if (colID >= D) return; // avoid out-of-bounds thread - Type acc = 0; + OutType acc = 0; for (IdxT i = 0; i < N; i++) { - Type v = data[colID + i * D]; + OutType v = data[colID + i * D]; acc += type == L2Norm ? v * v : raft::abs(v); } dots[colID] = do_sqrt ? raft::sqrt(acc) : acc; } -template +template void naiveColNorm( - Type* dots, const Type* data, IdxT D, IdxT N, NormType type, bool do_sqrt, cudaStream_t stream) + OutType* dots, const Type* data, IdxT D, IdxT N, NormType type, bool do_sqrt, cudaStream_t stream) { static const IdxT TPB = 64; IdxT nblks = raft::ceildiv(D, TPB); - naiveColNormKernel<<>>(dots, data, D, N, type, do_sqrt); + naiveColNormKernel + <<>>(dots, data, D, N, type, do_sqrt); RAFT_CUDA_TRY(cudaPeekAtLastError()); } -template -class ColNormTest : public ::testing::TestWithParam> { +template +class ColNormTest : public ::testing::TestWithParam> { public: ColNormTest() - : params(::testing::TestWithParam>::GetParam()), + : params(::testing::TestWithParam>::GetParam()), stream(resource::get_cuda_stream(handle)), data(params.rows * params.cols, stream), dots_exp(params.cols, stream), @@ -169,7 +182,7 @@ class ColNormTest : public ::testing::TestWithParam> { uniform(handle, r, data.data(), len, T(-1.0), T(1.0)); naiveColNorm(dots_exp.data(), data.data(), cols, rows, params.type, params.do_sqrt, stream); - auto output_view = raft::make_device_vector_view(dots_act.data(), params.cols); + auto output_view = raft::make_device_vector_view(dots_act.data(), params.cols); auto input_row_major = raft::make_device_matrix_view( data.data(), params.rows, params.cols); auto input_col_major = raft::make_device_matrix_view( @@ -196,8 +209,9 @@ class ColNormTest : public ::testing::TestWithParam> { raft::resources handle; cudaStream_t stream; - NormInputs params; - rmm::device_uvector data, dots_exp, dots_act; + NormInputs params; + rmm::device_uvector data; + rmm::device_uvector dots_exp, dots_act; }; ///// Row- and column-wise tests @@ -246,6 +260,19 @@ const std::vector> inputscd_i64 = {true}, {1234ULL}); +const std::vector> inputsh_i32 = + raft::util::itertools::product>( + {0.00001f}, {11, 1234}, {7, 33, 128, 500}, {L1Norm, L2Norm}, {false, true}, {true}, {1234ULL}); +const std::vector> inputsh_i64 = + raft::util::itertools::product>( + {0.00001f}, {11, 1234}, {7, 33, 128, 500}, {L1Norm, L2Norm}, {false, true}, {true}, {1234ULL}); +const std::vector> inputsch_i32 = + raft::util::itertools::product>( + {0.00001f}, {7, 33, 128, 500}, {11, 1234}, {L1Norm, L2Norm}, {false, true}, {true}, {1234ULL}); +const std::vector> inputsch_i64 = + raft::util::itertools::product>( + {0.00001f}, {7, 33, 128, 500}, {11, 1234}, {L1Norm, L2Norm}, {false, true}, {true}, {1234ULL}); + typedef RowNormTest RowNormTestF_i32; typedef RowNormTest RowNormTestD_i32; typedef RowNormTest RowNormTestF_i64; @@ -255,6 +282,11 @@ typedef ColNormTest ColNormTestD_i32; typedef ColNormTest ColNormTestF_i64; typedef ColNormTest ColNormTestD_i64; +typedef RowNormTest RowNormTestH_i32; +typedef RowNormTest RowNormTestH_i64; +typedef ColNormTest ColNormTestH_i32; +typedef ColNormTest ColNormTestH_i64; + #define ROWNORM_TEST(test_type, test_inputs) \ TEST_P(test_type, Result) \ { \ @@ -272,5 +304,10 @@ ROWNORM_TEST(ColNormTestD_i32, inputscd_i32); ROWNORM_TEST(ColNormTestF_i64, inputscf_i64); ROWNORM_TEST(ColNormTestD_i64, inputscd_i64); +ROWNORM_TEST(RowNormTestH_i32, inputsh_i32); +ROWNORM_TEST(RowNormTestH_i64, inputsh_i64); +ROWNORM_TEST(ColNormTestH_i32, inputsch_i32); +ROWNORM_TEST(ColNormTestH_i64, inputsch_i64); + } // end namespace linalg } // end namespace raft diff --git a/cpp/test/linalg/transpose.cu b/cpp/test/linalg/transpose.cu index f6857d3ffa..cbe869a9a5 100644 --- a/cpp/test/linalg/transpose.cu +++ b/cpp/test/linalg/transpose.cu @@ -25,6 +25,8 @@ #include +#include + #include namespace raft { @@ -84,6 +86,8 @@ const std::vector> inputsf2 = {{0.1f, 3 * 3, 3, 3, 1234ULL const std::vector> inputsd2 = {{0.1, 3 * 3, 3, 3, 1234ULL}}; +const std::vector> inputsh2 = {{0.1, 3 * 3, 3, 3, 1234ULL}}; + typedef TransposeTest TransposeTestValF; TEST_P(TransposeTestValF, Result) { @@ -112,10 +116,49 @@ TEST_P(TransposeTestValD, Result) raft::CompareApproxAbs(params.tolerance))); } +bool validate_half(const half* h_ref, const half* h_result, half tolerance, int len) +{ + bool success = true; + for (int i = 0; i < len; ++i) { + if (raft::abs(__half2float(h_result[i]) - __half2float(h_ref[i])) >= __half2float(tolerance)) { + success = false; + break; + } + if (!success) break; + } + return success; +} + +typedef TransposeTest TransposeTestValH; +TEST_P(TransposeTestValH, Result) +{ + half data_trans_ref_h[params.len]; + half data_trans_h[params.len]; + half data_h[params.len]; + + RAFT_CUDA_TRY(cudaMemcpyAsync(data_trans_ref_h, + data_trans_ref.data(), + params.len * sizeof(half), + cudaMemcpyDeviceToHost, + stream)); + + RAFT_CUDA_TRY(cudaMemcpyAsync( + data_trans_h, data_trans.data(), params.len * sizeof(half), cudaMemcpyDeviceToHost, stream)); + RAFT_CUDA_TRY(cudaMemcpyAsync( + data_h, data.data(), params.len * sizeof(half), cudaMemcpyDeviceToHost, stream)); + + resource::sync_stream(handle, stream); + + ASSERT_TRUE(validate_half(data_trans_ref_h, data_trans_h, params.tolerance, params.len)); + ASSERT_TRUE(validate_half(data_trans_ref_h, data_h, params.tolerance, params.len)); +} + INSTANTIATE_TEST_SUITE_P(TransposeTests, TransposeTestValF, ::testing::ValuesIn(inputsf2)); INSTANTIATE_TEST_SUITE_P(TransposeTests, TransposeTestValD, ::testing::ValuesIn(inputsd2)); +INSTANTIATE_TEST_SUITE_P(TransposeTests, TransposeTestValH, ::testing::ValuesIn(inputsh2)); + namespace { /** * We hide these functions in tests for now until we have a heterogeneous mdarray diff --git a/cpp/test/sparse/masked_matmul.cu b/cpp/test/sparse/masked_matmul.cu index 0ece716a1b..f883beae32 100644 --- a/cpp/test/sparse/masked_matmul.cu +++ b/cpp/test/sparse/masked_matmul.cu @@ -24,6 +24,7 @@ #include +#include #include #include @@ -32,15 +33,15 @@ namespace raft { namespace sparse { -template +template struct MaskedMatmulInputs { - value_t tolerance; + output_t tolerance; index_t m; index_t k; index_t n; - value_t sparsity; + float sparsity; unsigned long long int seed; }; @@ -53,8 +54,13 @@ struct sum_abs_op { } }; -template -::std::ostream& operator<<(::std::ostream& os, const MaskedMatmulInputs& params) +struct float_to_half { + __host__ __device__ __half operator()(const float x) const { return __float2half(x); } +}; + +template +::std::ostream& operator<<(::std::ostream& os, + const MaskedMatmulInputs& params) { os << " m: " << params.m << "\tk: " << params.k << "\tn: " << params.n << "\tsparsity: " << params.sparsity; @@ -62,15 +68,33 @@ template return os; } +bool isCuSparseVersionGreaterThan_12_0_1() +{ + int version; + cusparseHandle_t handle; + cusparseCreate(&handle); + cusparseGetVersion(handle, &version); + + int major = version / 1000; + int minor = (version % 1000) / 100; + int patch = version % 100; + + cusparseDestroy(handle); + + return (major > 12) || (major == 12 && minor > 0) || (major == 12 && minor == 0 && patch >= 2); +} + template -class MaskedMatmulTest : public ::testing::TestWithParam> { +class MaskedMatmulTest + : public ::testing::TestWithParam> { public: MaskedMatmulTest() - : params(::testing::TestWithParam>::GetParam()), + : params(::testing::TestWithParam>::GetParam()), stream(resource::get_cuda_stream(handle)), a_data_d(0, resource::get_cuda_stream(handle)), b_data_d(0, resource::get_cuda_stream(handle)), @@ -142,7 +166,7 @@ class MaskedMatmulTest : public ::testing::TestWithParam& A, const std::vector& B, - std::vector& vals, + std::vector& vals, const std::vector& cols, const std::vector& row_ptrs, bool is_row_major_A, @@ -156,11 +180,15 @@ class MaskedMatmulTest : public ::testing::TestWithParam && std::is_same_v)) { + sum += __half2float(A[a_index]) * __half2float(B[b_index]); + } else { + sum += A[a_index] * B[b_index]; + } } vals[j] = sum; } @@ -183,29 +211,54 @@ class MaskedMatmulTest : public ::testing::TestWithParam(handle, 1, a_size + b_size); + auto blobs_a_b = raft::make_device_matrix(handle, 1, a_size + b_size); auto labels = raft::make_device_vector(handle, 1); - raft::random::make_blobs(blobs_a_b.data_handle(), - labels.data_handle(), - 1, - a_size + b_size, - 1, - stream, - false, - nullptr, - nullptr, - value_t(1.0), - false, - value_t(-1.0f), - value_t(1.0f), - uint64_t(2024)); - - raft::copy(a_data_h.data(), blobs_a_b.data_handle(), a_size, stream); - raft::copy(b_data_h.data(), blobs_a_b.data_handle() + a_size, b_size, stream); - - raft::copy(a_data_d.data(), blobs_a_b.data_handle(), a_size, stream); - raft::copy(b_data_d.data(), blobs_a_b.data_handle() + a_size, b_size, stream); + raft::random::make_blobs(blobs_a_b.data_handle(), + labels.data_handle(), + 1, + a_size + b_size, + 1, + stream, + false, + nullptr, + nullptr, + output_t(1.0), + false, + output_t(-1.0f), + output_t(1.0f), + uint64_t(2024)); + + if constexpr ((std::is_same_v && std::is_same_v)) { + { + thrust::device_ptr d_output_ptr = + thrust::device_pointer_cast(blobs_a_b.data_handle()); + thrust::device_ptr d_value_ptr = thrust::device_pointer_cast(a_data_d.data()); + thrust::transform(thrust::cuda::par.on(stream), + d_output_ptr, + d_output_ptr + a_size, + d_value_ptr, + float_to_half()); + } + { + thrust::device_ptr d_output_ptr = + thrust::device_pointer_cast(blobs_a_b.data_handle() + a_size); + thrust::device_ptr d_value_ptr = thrust::device_pointer_cast(b_data_d.data()); + thrust::transform(thrust::cuda::par.on(stream), + d_output_ptr, + d_output_ptr + b_size, + d_value_ptr, + float_to_half()); + } + raft::copy(a_data_h.data(), a_data_d.data(), a_size, stream); + raft::copy(b_data_h.data(), b_data_d.data(), b_size, stream); + } else { + raft::copy(a_data_h.data(), blobs_a_b.data_handle(), a_size, stream); + raft::copy(b_data_h.data(), blobs_a_b.data_handle() + a_size, b_size, stream); + + raft::copy(a_data_d.data(), blobs_a_b.data_handle(), a_size, stream); + raft::copy(b_data_d.data(), blobs_a_b.data_handle() + a_size, b_size, stream); + } resource::sync_stream(handle); @@ -213,7 +266,7 @@ class MaskedMatmulTest : public ::testing::TestWithParam c_indptr_h(params.m + 1); std::vector c_indices_h(c_true_nnz); - std::vector c_data_h(c_true_nnz); + std::vector c_data_h(c_true_nnz); cpu_convert_to_csr(bitmap_h, params.m, params.n, c_indices_h, c_indptr_h); @@ -236,7 +289,13 @@ class MaskedMatmulTest : public ::testing::TestWithParam && !isCuSparseVersionGreaterThan_12_0_1()) { + GTEST_SKIP() << "Skipping all tests for half-float as cuSparse doesn't support it."; + } + make_data(); + } void Run() { @@ -255,33 +314,33 @@ class MaskedMatmulTest : public ::testing::TestWithParam(c_indices_d.size())); - auto C = raft::make_device_csr_matrix_view(c_data_d.data(), c_structure); + auto C = raft::make_device_csr_matrix_view(c_data_d.data(), c_structure); raft::sparse::linalg::masked_matmul(handle, A, B, mask, C); resource::sync_stream(handle); - ASSERT_TRUE(raft::devArrMatch(c_expected_data_d.data(), - C.get_elements().data(), - c_expected_data_d.size(), - raft::CompareApprox(params.tolerance), - stream)); + ASSERT_TRUE(raft::devArrMatch(c_expected_data_d.data(), + C.get_elements().data(), + c_expected_data_d.size(), + raft::CompareApprox(params.tolerance), + stream)); - thrust::device_ptr expected_data_ptr = + thrust::device_ptr expected_data_ptr = thrust::device_pointer_cast(c_expected_data_d.data()); - value_t sum_abs = thrust::reduce(thrust::cuda::par.on(stream), - expected_data_ptr, - expected_data_ptr + c_expected_data_d.size(), - value_t(0.0f), - sum_abs_op()); - value_t avg = sum_abs / (1.0f * c_expected_data_d.size()); - - ASSERT_GE(avg, (params.tolerance * static_cast(0.001f))); + output_t sum_abs = thrust::reduce(thrust::cuda::par.on(stream), + expected_data_ptr, + expected_data_ptr + c_expected_data_d.size(), + output_t(0.0f), + sum_abs_op()); + output_t avg = sum_abs / (1.0f * c_expected_data_d.size()); + + ASSERT_GE(avg, (params.tolerance * static_cast(0.001f))); } raft::resources handle; cudaStream_t stream; - MaskedMatmulInputs params; + MaskedMatmulInputs params; rmm::device_uvector a_data_d; rmm::device_uvector b_data_d; @@ -289,40 +348,82 @@ class MaskedMatmulTest : public ::testing::TestWithParam c_indptr_d; rmm::device_uvector c_indices_d; - rmm::device_uvector c_data_d; + rmm::device_uvector c_data_d; - rmm::device_uvector c_expected_data_d; + rmm::device_uvector c_expected_data_d; }; -using MaskedMatmulTestF = MaskedMatmulTest; +using MaskedMatmulTestF = MaskedMatmulTest; TEST_P(MaskedMatmulTestF, Result) { Run(); } -using MaskedMatmulTestD = MaskedMatmulTest; +using MaskedMatmulTestD = MaskedMatmulTest; TEST_P(MaskedMatmulTestD, Result) { Run(); } -const std::vector> sddmm_inputs_f = { +using MaskedMatmulTestH = MaskedMatmulTest; +TEST_P(MaskedMatmulTestH, Result) { Run(); } + +const std::vector> sddmm_inputs_f = { + {0.001f, 2, 255, 1023, 0.19, 1234ULL}, + {0.001f, 2, 255, 1023 * 2, 0.19, 1234ULL}, + {0.001f, 2, 255, 1023 * 3, 0.38, 1234ULL}, + {0.0001f, 10, 255, 13000, 0.01, 1234ULL}, {0.0001f, 10, 5, 32, 0.1, 1234ULL}, - {0.0001f, 1024, 32, 1024, 0.1, 1234ULL}, + {0.001f, 11, 255, 1023, 0.19, 1234ULL}, + {0.001f, 11, 255, 1023 * 2, 0.19, 1234ULL}, + {0.001f, 11, 255, 1023 * 3, 0.38, 1234ULL}, {0.0003f, 32, 1024, 1024, 0.2, 1234ULL}, + {0.0001f, 1024, 32, 1024, 0.1, 1234ULL}, {0.001f, 1024, 1024, 1024, 0.19, 1234ULL}, + {0.001f, 1023, 1023, 1023 * 3, 0.38, 1234ULL}, + {0.001f, 1025, 1025, 1025 * 3, 0.31, 1234ULL}, {0.0001f, 1024, 1024, 32, 0.3, 1234ULL}, {0.0001f, 1024, 32, 1024, 0.4, 1234ULL}, - {0.0003f, 32, 1024, 1024, 0.19, 1234ULL}, + {0.0003f, 31, 1025, 1025, 0.19, 1234ULL}, {0.001f, 1024, 1024, 1024, 0.1, 1234ULL}}; -const std::vector> sddmm_inputs_d = { - {0.0001f, 10, 5, 32, 0.01, 1234ULL}, - {0.0001f, 1024, 32, 1024, 0.1, 1234ULL}, +const std::vector> sddmm_inputs_d = { + {0.0001f, 2, 255, 1023, 0.19, 1234ULL}, + {0.0001f, 2, 255, 1023 * 2, 0.19, 1234ULL}, + {0.0001f, 2, 255, 1023 * 3, 0.38, 1234ULL}, + {0.0001f, 10, 255, 13000, 0.01, 1234ULL}, + {0.0001f, 10, 5, 32, 0.1, 1234ULL}, + {0.0001f, 11, 255, 1023, 0.19, 1234ULL}, + {0.0001f, 11, 255, 1023 * 2, 0.19, 1234ULL}, + {0.0001f, 11, 255, 1023 * 3, 0.38, 1234ULL}, {0.0001f, 32, 1024, 1024, 0.2, 1234ULL}, + {0.0001f, 1024, 32, 1024, 0.1, 1234ULL}, {0.0001f, 1024, 1024, 1024, 0.19, 1234ULL}, + {0.0001f, 1023, 1023, 1023 * 3, 0.38, 1234ULL}, + {0.0001f, 1025, 1025, 1025 * 3, 0.31, 1234ULL}, {0.0001f, 1024, 1024, 32, 0.3, 1234ULL}, {0.0001f, 1024, 32, 1024, 0.4, 1234ULL}, - {0.0001f, 32, 1024, 1024, 0.19, 1234ULL}, + {0.0001f, 31, 1025, 1025, 0.19, 1234ULL}, {0.0001f, 1024, 1024, 1024, 0.1, 1234ULL}}; +const std::vector> sddmm_inputs_h = { + {0.001f, 2, 255, 1023, 0.19, 1234ULL}, + {0.001f, 2, 255, 1023 * 2, 0.19, 1234ULL}, + {0.001f, 2, 255, 1023 * 3, 0.38, 1234ULL}, + {0.0001f, 10, 255, 13000, 0.01, 1234ULL}, + {0.0001f, 10, 5, 32, 0.1, 1234ULL}, + {0.001f, 11, 255, 1023, 0.19, 1234ULL}, + {0.001f, 11, 255, 1023 * 2, 0.19, 1234ULL}, + {0.001f, 11, 255, 1023 * 3, 0.38, 1234ULL}, + {0.0003f, 32, 1024, 1024, 0.2, 1234ULL}, + {0.0001f, 1024, 32, 1024, 0.1, 1234ULL}, + {0.001f, 1024, 1024, 1024, 0.19, 1234ULL}, + {0.001f, 1023, 1023, 1023 * 3, 0.38, 1234ULL}, + {0.001f, 1025, 1025, 1025 * 3, 0.31, 1234ULL}, + {0.0001f, 1024, 1024, 32, 0.3, 1234ULL}, + {0.0001f, 1024, 32, 1024, 0.4, 1234ULL}, + {0.0003f, 31, 1025, 1025, 0.19, 1234ULL}, + {0.001f, 1024, 1024, 1024, 0.1, 1234ULL}}; + INSTANTIATE_TEST_CASE_P(MaskedMatmulTest, MaskedMatmulTestF, ::testing::ValuesIn(sddmm_inputs_f)); INSTANTIATE_TEST_CASE_P(MaskedMatmulTest, MaskedMatmulTestD, ::testing::ValuesIn(sddmm_inputs_d)); +INSTANTIATE_TEST_CASE_P(MaskedMatmulTest, MaskedMatmulTestH, ::testing::ValuesIn(sddmm_inputs_h)); + } // namespace sparse } // namespace raft diff --git a/cpp/test/sparse/sddmm.cu b/cpp/test/sparse/sddmm.cu index 8ff20581c9..26c2c519dd 100644 --- a/cpp/test/sparse/sddmm.cu +++ b/cpp/test/sparse/sddmm.cu @@ -22,8 +22,12 @@ #include #include +#include +#include #include +#include +#include #include #include @@ -32,16 +36,16 @@ namespace raft { namespace sparse { -template +template struct SDDMMInputs { - ValueType tolerance; + OutputType tolerance; IndexType m; IndexType k; IndexType n; - ValueType alpha; - ValueType beta; + OutputType alpha; + OutputType beta; bool transpose_a; bool transpose_b; @@ -59,6 +63,10 @@ struct sum_abs_op { } }; +struct float_to_half { + __host__ __device__ __half operator()(const float x) const { return __float2half(x); } +}; + template ::std::ostream& operator<<(::std::ostream& os, const SDDMMInputs& params) { @@ -72,11 +80,12 @@ template template -class SDDMMTest : public ::testing::TestWithParam> { + typename LayoutPolicyB = raft::layout_c_contiguous, + typename OutputType = ValueType> +class SDDMMTest : public ::testing::TestWithParam> { public: SDDMMTest() - : params(::testing::TestWithParam>::GetParam()), + : params(::testing::TestWithParam>::GetParam()), stream(resource::get_cuda_stream(handle)), a_data_d(0, resource::get_cuda_stream(handle)), b_data_d(0, resource::get_cuda_stream(handle)), @@ -88,9 +97,25 @@ class SDDMMTest : public ::testing::TestWithParam 12) || (major == 12 && minor > 0) || (major == 12 && minor == 0 && patch >= 2); + } + IndexType create_sparse_matrix(IndexType m, IndexType n, - ValueType sparsity, + OutputType sparsity, std::vector& matrix) { IndexType total_elements = static_cast(m * n); @@ -119,7 +144,7 @@ class SDDMMTest : public ::testing::TestWithParam& matrix, IndexType rows, IndexType cols, - std::vector& values, + std::vector& values, std::vector& indices, std::vector& indptr) { @@ -130,7 +155,7 @@ class SDDMMTest : public ::testing::TestWithParam(1.0); + values[offset_values] = static_cast(1.0); indices[offset_values] = static_cast(j); offset_values++; } @@ -141,7 +166,7 @@ class SDDMMTest : public ::testing::TestWithParam& A, const std::vector& B, - std::vector& vals, + std::vector& vals, const std::vector& cols, const std::vector& row_ptrs, bool is_row_major_A, @@ -158,11 +183,15 @@ class SDDMMTest : public ::testing::TestWithParam && std::is_same_v)) { + sum += __half2float(A[a_index]) * __half2float(B[b_index]); + } else { + sum += A[a_index] * B[b_index]; + } } vals[j] = params.alpha * sum + params.beta * vals[j]; } @@ -181,29 +210,53 @@ class SDDMMTest : public ::testing::TestWithParam(handle, 1, a_size + b_size); + auto blobs_a_b = raft::make_device_matrix(handle, 1, a_size + b_size); auto labels = raft::make_device_vector(handle, 1); - raft::random::make_blobs(blobs_a_b.data_handle(), - labels.data_handle(), - 1, - a_size + b_size, - 1, - stream, - false, - nullptr, - nullptr, - ValueType(1.0), - false, - ValueType(-1.0f), - ValueType(1.0f), - uint64_t(2024)); - - raft::copy(a_data_h.data(), blobs_a_b.data_handle(), a_size, stream); - raft::copy(b_data_h.data(), blobs_a_b.data_handle() + a_size, b_size, stream); - - raft::copy(a_data_d.data(), blobs_a_b.data_handle(), a_size, stream); - raft::copy(b_data_d.data(), blobs_a_b.data_handle() + a_size, b_size, stream); + raft::random::make_blobs(blobs_a_b.data_handle(), + labels.data_handle(), + 1, + a_size + b_size, + 1, + stream, + false, + nullptr, + nullptr, + OutputType(1.0), + false, + OutputType(-1.0f), + OutputType(1.0f), + uint64_t(2024)); + if constexpr ((std::is_same_v && std::is_same_v)) { + { + thrust::device_ptr d_output_ptr = + thrust::device_pointer_cast(blobs_a_b.data_handle()); + thrust::device_ptr d_value_ptr = thrust::device_pointer_cast(a_data_d.data()); + thrust::transform(thrust::cuda::par.on(stream), + d_output_ptr, + d_output_ptr + a_size, + d_value_ptr, + float_to_half()); + } + { + thrust::device_ptr d_output_ptr = + thrust::device_pointer_cast(blobs_a_b.data_handle() + a_size); + thrust::device_ptr d_value_ptr = thrust::device_pointer_cast(b_data_d.data()); + thrust::transform(thrust::cuda::par.on(stream), + d_output_ptr, + d_output_ptr + b_size, + d_value_ptr, + float_to_half()); + } + raft::copy(a_data_h.data(), a_data_d.data(), a_size, stream); + raft::copy(b_data_h.data(), b_data_d.data(), b_size, stream); + } else { + raft::copy(a_data_h.data(), blobs_a_b.data_handle(), a_size, stream); + raft::copy(b_data_h.data(), blobs_a_b.data_handle() + a_size, b_size, stream); + + raft::copy(a_data_d.data(), blobs_a_b.data_handle(), a_size, stream); + raft::copy(b_data_d.data(), blobs_a_b.data_handle() + a_size, b_size, stream); + } resource::sync_stream(handle); @@ -213,7 +266,7 @@ class SDDMMTest : public ::testing::TestWithParam c_indptr_h(params.m + 1); std::vector c_indices_h(c_true_nnz); - std::vector c_data_h(c_true_nnz); + std::vector c_data_h(c_true_nnz); convert_to_csr(c_dense_data_h, params.m, params.n, c_data_h, c_indices_h, c_indptr_h); @@ -238,7 +291,13 @@ class SDDMMTest : public ::testing::TestWithParam && !isCuSparseVersionGreaterThan_12_0_1()) { + GTEST_SKIP() << "Skipping all tests for half-float as cuSparse doesn't support it."; + } + make_data(); + } void Run() { @@ -258,7 +317,7 @@ class SDDMMTest : public ::testing::TestWithParam(c_indices_d.size())); - auto c = raft::make_device_csr_matrix_view(c_data_d.data(), c_structure); + auto c = raft::make_device_csr_matrix_view(c_data_d.data(), c_structure); auto op_a = params.transpose_a ? raft::linalg::Operation::TRANSPOSE : raft::linalg::Operation::NON_TRANSPOSE; @@ -271,41 +330,41 @@ class SDDMMTest : public ::testing::TestWithParam(¶ms.alpha), - raft::make_host_scalar_view(¶ms.beta)); + raft::make_host_scalar_view(¶ms.alpha), + raft::make_host_scalar_view(¶ms.beta)); resource::sync_stream(handle); - ASSERT_TRUE(raft::devArrMatch(c_expected_data_d.data(), - c.get_elements().data(), - c_expected_data_d.size(), - raft::CompareApprox(params.tolerance), - stream)); + ASSERT_TRUE(raft::devArrMatch(c_expected_data_d.data(), + c.get_elements().data(), + c_expected_data_d.size(), + raft::CompareApprox(params.tolerance), + stream)); - thrust::device_ptr expected_data_ptr = + thrust::device_ptr expected_data_ptr = thrust::device_pointer_cast(c_expected_data_d.data()); - ValueType sum_abs = thrust::reduce(thrust::cuda::par.on(stream), - expected_data_ptr, - expected_data_ptr + c_expected_data_d.size(), - ValueType(0.0f), - sum_abs_op()); - ValueType avg = sum_abs / (1.0f * c_expected_data_d.size()); - - ASSERT_GE(avg, (params.tolerance * static_cast(0.001f))); + OutputType sum_abs = thrust::reduce(thrust::cuda::par.on(stream), + expected_data_ptr, + expected_data_ptr + c_expected_data_d.size(), + OutputType(0.0f), + sum_abs_op()); + OutputType avg = sum_abs / (1.0f * c_expected_data_d.size()); + + ASSERT_GE(avg, (params.tolerance * static_cast(0.001f))); } raft::resources handle; cudaStream_t stream; - SDDMMInputs params; + SDDMMInputs params; rmm::device_uvector a_data_d; rmm::device_uvector b_data_d; rmm::device_uvector c_indptr_d; rmm::device_uvector c_indices_d; - rmm::device_uvector c_data_d; + rmm::device_uvector c_data_d; - rmm::device_uvector c_expected_data_d; + rmm::device_uvector c_expected_data_d; }; using SDDMMTestF_Row_Col = SDDMMTest; @@ -332,6 +391,18 @@ TEST_P(SDDMMTestD_Row_Row, Result) { Run(); } using SDDMMTestD_Col_Col = SDDMMTest; TEST_P(SDDMMTestD_Col_Col, Result) { Run(); } +using SDDMMTestHF_Row_Col = SDDMMTest; +TEST_P(SDDMMTestHF_Row_Col, Result) { Run(); } + +using SDDMMTestHF_Col_Row = SDDMMTest; +TEST_P(SDDMMTestHF_Col_Row, Result) { Run(); } + +using SDDMMTestHF_Row_Row = SDDMMTest; +TEST_P(SDDMMTestHF_Row_Row, Result) { Run(); } + +using SDDMMTestHF_Col_Col = SDDMMTest; +TEST_P(SDDMMTestHF_Col_Col, Result) { Run(); } + const std::vector> sddmm_inputs_f = { {0.0001f, 10, 5, 32, 1.0, 0.0, false, false, 0.01, 1234ULL}, {0.0001f, 1024, 32, 1024, 0.3, 0.0, true, false, 0.1, 1234ULL}, @@ -352,6 +423,16 @@ const std::vector> sddmm_inputs_d = { {0.0001f, 32, 1024, 1024, 2.0, 0.2, false, true, 0.19, 1234ULL}, {0.0001f, 1024, 1024, 1024, 0.0, 1.2, true, true, 0.1, 1234ULL}}; +const std::vector> sddmm_inputs_h_f = { + {0.0001f, 10, 5, 32, 1.0, 0.0, false, false, 0.01, 1234ULL}, + {0.0001f, 1024, 32, 1024, 0.3, 0.0, true, false, 0.1, 1234ULL}, + {0.0003f, 32, 1024, 1024, 1.0, 0.3, false, true, 0.2, 1234ULL}, + {0.001f, 1024, 1024, 1024, 0.2, 0.2, true, true, 0.19, 1234ULL}, + {0.0001f, 1024, 1024, 32, 0.1, 0.2, false, false, 0.3, 1234ULL}, + {0.0001f, 1024, 32, 1024, 1.0, 0.3, true, false, 0.4, 1234ULL}, + {0.0003f, 32, 1024, 1024, 2.0, 0.2, false, true, 0.19, 1234ULL}, + {0.001f, 1024, 1024, 1024, 0.0, 1.2, true, true, 0.1, 1234ULL}}; + INSTANTIATE_TEST_CASE_P(SDDMMTest, SDDMMTestF_Row_Col, ::testing::ValuesIn(sddmm_inputs_f)); INSTANTIATE_TEST_CASE_P(SDDMMTest, SDDMMTestF_Col_Row, ::testing::ValuesIn(sddmm_inputs_f)); INSTANTIATE_TEST_CASE_P(SDDMMTest, SDDMMTestF_Row_Row, ::testing::ValuesIn(sddmm_inputs_f)); @@ -362,5 +443,10 @@ INSTANTIATE_TEST_CASE_P(SDDMMTest, SDDMMTestD_Col_Row, ::testing::ValuesIn(sddmm INSTANTIATE_TEST_CASE_P(SDDMMTest, SDDMMTestD_Row_Row, ::testing::ValuesIn(sddmm_inputs_d)); INSTANTIATE_TEST_CASE_P(SDDMMTest, SDDMMTestD_Col_Col, ::testing::ValuesIn(sddmm_inputs_d)); +INSTANTIATE_TEST_CASE_P(SDDMMTest, SDDMMTestHF_Row_Col, ::testing::ValuesIn(sddmm_inputs_h_f)); +INSTANTIATE_TEST_CASE_P(SDDMMTest, SDDMMTestHF_Col_Row, ::testing::ValuesIn(sddmm_inputs_h_f)); +INSTANTIATE_TEST_CASE_P(SDDMMTest, SDDMMTestHF_Row_Row, ::testing::ValuesIn(sddmm_inputs_h_f)); +INSTANTIATE_TEST_CASE_P(SDDMMTest, SDDMMTestHF_Col_Col, ::testing::ValuesIn(sddmm_inputs_h_f)); + } // namespace sparse } // namespace raft From f1cc0fb79d2f40f6cc266c60928ed70ea64c843a Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 11:51:31 -0400 Subject: [PATCH 12/79] Use CUDA math wheels (#2415) Use CUDA math wheels to reduce wheel size by not statically linking CUDA math libraries. Contributes to https://github.com/rapidsai/build-planning/issues/35 Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) Approvers: - Robert Maynard (https://github.com/robertmaynard) - Bradley Dice (https://github.com/bdice) - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/raft/pull/2415 --- ci/build_wheel.sh | 21 ++++++++++++++++++++- ci/build_wheel_pylibraft.sh | 11 ++++++++++- dependencies.yaml | 31 +++++++++++++++++++++++++++++++ python/pylibraft/CMakeLists.txt | 18 ++++++++++++++++++ python/pylibraft/pyproject.toml | 6 +++++- 5 files changed, 84 insertions(+), 3 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 62d93a668e..e7ae52f33a 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -20,10 +20,29 @@ rapids-generate-version > VERSION cd "${package_dir}" +case "${RAPIDS_CUDA_VERSION}" in + 12.*) + EXCLUDE_ARGS=( + --exclude "libcublas.so.12" + --exclude "libcublasLt.so.12" + --exclude "libcurand.so.10" + --exclude "libcusolver.so.11" + --exclude "libcusparse.so.12" + --exclude "libnvJitLink.so.12" + --exclude "libucp.so.0" + ) + ;; + 11.*) + EXCLUDE_ARGS=( + --exclude "libucp.so.0" + ) + ;; +esac + # Hardcode the output dir python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check mkdir -p final_dist -python -m auditwheel repair -w final_dist --exclude "libucp.so.0" dist/* +python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/* RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist diff --git a/ci/build_wheel_pylibraft.sh b/ci/build_wheel_pylibraft.sh index 895c311f46..ce9f0ed172 100755 --- a/ci/build_wheel_pylibraft.sh +++ b/ci/build_wheel_pylibraft.sh @@ -3,7 +3,16 @@ set -euo pipefail +case "${RAPIDS_CUDA_VERSION}" in + 12.*) + EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=ON" + ;; + 11.*) + EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=OFF" + ;; +esac + # Set up skbuild options. Enable sccache in skbuild config options -export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF" +export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF${EXTRA_CMAKE_ARGS}" ci/build_wheel.sh pylibraft python/pylibraft diff --git a/dependencies.yaml b/dependencies.yaml index 630b5a50c9..92c6d98414 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -83,6 +83,7 @@ files: extras: table: project includes: + - cuda_wheels - run_pylibraft py_test_pylibraft: output: pyproject @@ -377,6 +378,36 @@ dependencies: - *libcusolver114 - *libcusparse_dev114 - *libcusparse114 + cuda_wheels: + specific: + - output_types: pyproject + matrices: + - matrix: + cuda: "12.*" + use_cuda_wheels: "true" + packages: + - nvidia-cublas-cu12 + - nvidia-curand-cu12 + - nvidia-cusolver-cu12 + - nvidia-cusparse-cu12 + # CUDA 11 does not provide wheels, so use the system libraries instead + - matrix: + cuda: "11.*" + use_cuda_wheels: "true" + packages: + # if use_cuda_wheels=false is provided, do not add dependencies on any CUDA wheels + # (e.g. for DLFW and pip devcontainers) + - matrix: + use_cuda_wheels: "false" + packages: + # if no matching matrix selectors passed, list the unsuffixed packages + # (just as a source of documentation, as this populates pyproject.toml in source control) + - matrix: + packages: + - nvidia-cublas + - nvidia-curand + - nvidia-cusolver + - nvidia-cusparse depends_on_cupy: common: diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index 6cbe8e4cbf..c286d3debf 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -30,6 +30,7 @@ project( option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" ON ) +option(USE_CUDA_MATH_WHEELS "Use the CUDA math wheels instead of the system libraries" OFF) # If the user requested it we attempt to find RAFT. if(FIND_RAFT_CPP) @@ -48,15 +49,32 @@ endif() include(rapids-cython-core) if(NOT raft_FOUND) + find_package(CUDAToolkit REQUIRED) + set(BUILD_TESTS OFF) set(BUILD_PRIMS_BENCH OFF) set(BUILD_ANN_BENCH OFF) set(RAFT_COMPILE_LIBRARY ON) set(CUDA_STATIC_RUNTIME ON) set(CUDA_STATIC_MATH_LIBRARIES ON) + if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12.0) + set(CUDA_STATIC_MATH_LIBRARIES OFF) + elseif(USE_CUDA_MATH_WHEELS) + message(FATAL_ERROR "Cannot use CUDA math wheels with CUDA < 12.0") + endif() add_subdirectory(../../cpp raft-cpp EXCLUDE_FROM_ALL) + if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) + set_property(TARGET raft_lib PROPERTY INSTALL_RPATH + "$ORIGIN/../nvidia/cublas/lib" + "$ORIGIN/../nvidia/curand/lib" + "$ORIGIN/../nvidia/cusolver/lib" + "$ORIGIN/../nvidia/cusparse/lib" + "$ORIGIN/../nvidia/nvjitlink/lib" + ) + endif() + # When building the C++ libraries from source we must copy libraft.so alongside the # pairwise_distance and random Cython libraries TODO: when we have a single 'compiled' raft # library, we shouldn't need this diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index 4c1d8e8fa1..9a826e53c6 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -33,6 +33,10 @@ requires-python = ">=3.9" dependencies = [ "cuda-python", "numpy>=1.23,<2.0a0", + "nvidia-cublas", + "nvidia-curand", + "nvidia-cusolver", + "nvidia-cusparse", "rmm==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ @@ -124,7 +128,7 @@ requires = [ "rmm==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. dependencies-file = "../../dependencies.yaml" -matrix-entry = "cuda_suffixed=true" +matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" [tool.pytest.ini_options] filterwarnings = [ From 2f587b13b62d2dd8b15bf6d2902c65d67e7afc7b Mon Sep 17 00:00:00 2001 From: Jinsol Park Date: Thu, 22 Aug 2024 17:17:30 -0700 Subject: [PATCH 13/79] [FEA] Batching NN Descent (#2403) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR implements batching NN Descent. It will be helpful for reducing device memory usages for large datasets (specifically if the dataset is kept on host). `index_params` now has... - `n_clusters`: number of clusters to make. Larger clusters reduce device memory usage. Default is 1, in which case it doesn't do the batched NND. ### Notes - The batching approach may have duplicate indices in the knn graph (in rare cases) because sometimes distances calculated for the same pair may be slightly different. This results in putting the same index far apart after sorting by distances, making it difficult to get unique indices (which is done by looking at 2 indices before the current one). - handled by adding a `max_duplicates` for `check_unique_indices` in tests ### Benchmarks - Dataset for NND (no batch and batch) is on host - Dataset for brute force knn is on device (but still won't be able to run with large datasets even if the data is put on the host because it brings the entire dataset to device anyway) - The dataset is just a slice of the wiki-all dataset (88M, 768) to test for different sizes Screenshot 2024-08-02 at 8 35 58 AM Authors: - Jinsol Park (https://github.com/jinsolp) Approvers: - Divye Gala (https://github.com/divyegala) - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/raft/pull/2403 --- .../raft/neighbors/detail/nn_descent.cuh | 48 +- .../neighbors/detail/nn_descent_batch.cuh | 701 ++++++++++++++++++ cpp/include/raft/neighbors/nn_descent.cuh | 30 +- .../raft/neighbors/nn_descent_types.hpp | 23 + cpp/test/CMakeLists.txt | 36 +- cpp/test/neighbors/ann_nn_descent.cuh | 155 +++- .../test_batch_float_uint32_t.cu | 30 + cpp/test/neighbors/ann_utils.cuh | 18 +- 8 files changed, 981 insertions(+), 60 deletions(-) create mode 100644 cpp/include/raft/neighbors/detail/nn_descent_batch.cuh create mode 100644 cpp/test/neighbors/ann_nn_descent/test_batch_float_uint32_t.cu diff --git a/cpp/include/raft/neighbors/detail/nn_descent.cuh b/cpp/include/raft/neighbors/detail/nn_descent.cuh index 9c37ee146d..02610f9afb 100644 --- a/cpp/include/raft/neighbors/detail/nn_descent.cuh +++ b/cpp/include/raft/neighbors/detail/nn_descent.cuh @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -344,7 +345,9 @@ struct GnndGraph { ~GnndGraph(); }; -template +template > class GNND { public: GNND(raft::resources const& res, const BuildConfig& build_config); @@ -356,9 +359,10 @@ class GNND { Index_t* output_graph, bool return_distances, DistData_t* output_distances, - epilogue_op distance_epilogue = raft::identity_op()); + epilogue_op distance_epilogue = DistEpilogue()); ~GNND() = default; using ID_t = InternalID_t; + void reset(raft::resources const& res); private: void add_reverse_edges(Index_t* graph_ptr, @@ -366,7 +370,8 @@ class GNND { Index_t* d_rev_graph_ptr, int2* list_sizes, cudaStream_t stream = 0); - void local_join(cudaStream_t stream = 0, epilogue_op distance_epilogue = raft::identity_op()); + void local_join(cudaStream_t stream = 0, + epilogue_op distance_epilogue = DistEpilogue()); raft::resources const& res; @@ -701,7 +706,7 @@ __device__ __forceinline__ void remove_duplicates( // is 1024 and 1536 respectively, which means the bounds don't work anymore template , - typename epilogue_op = raft::identity_op> + typename epilogue_op = DistEpilogue> RAFT_KERNEL #ifdef __CUDA_ARCH__ #if (__CUDA_ARCH__) == 750 || ((__CUDA_ARCH__) >= 860 && (__CUDA_ARCH__) <= 890) @@ -1183,18 +1188,23 @@ GNND::GNND(raft::resources const& res, d_list_sizes_old_{raft::make_device_vector(res, nrow_)} { static_assert(NUM_SAMPLES <= 32); - - thrust::fill(thrust::device, - dists_buffer_.data_handle(), - dists_buffer_.data_handle() + dists_buffer_.size(), - std::numeric_limits::max()); - thrust::fill(thrust::device, - reinterpret_cast(graph_buffer_.data_handle()), - reinterpret_cast(graph_buffer_.data_handle()) + graph_buffer_.size(), - std::numeric_limits::max()); - thrust::fill(thrust::device, d_locks_.data_handle(), d_locks_.data_handle() + d_locks_.size(), 0); + raft::matrix::fill(res, dists_buffer_.view(), std::numeric_limits::max()); + auto graph_buffer_view = raft::make_device_matrix_view( + reinterpret_cast(graph_buffer_.data_handle()), nrow_, DEGREE_ON_DEVICE); + raft::matrix::fill(res, graph_buffer_view, std::numeric_limits::max()); + raft::matrix::fill(res, d_locks_.view(), 0); }; +template +void GNND::reset(raft::resources const& res) +{ + raft::matrix::fill(res, dists_buffer_.view(), std::numeric_limits::max()); + auto graph_buffer_view = raft::make_device_matrix_view( + reinterpret_cast(graph_buffer_.data_handle()), nrow_, DEGREE_ON_DEVICE); + raft::matrix::fill(res, graph_buffer_view, std::numeric_limits::max()); + raft::matrix::fill(res, d_locks_.view(), 0); +} + template void GNND::add_reverse_edges(Index_t* graph_ptr, Index_t* h_rev_graph_ptr, @@ -1246,6 +1256,7 @@ void GNND::build(Data_t* data, cudaStream_t stream = raft::resource::get_cuda_stream(res); nrow_ = nrow; + graph_.nrow = nrow; graph_.h_graph = (InternalID_t*)output_graph; cudaPointerAttributes data_ptr_attr; @@ -1384,6 +1395,7 @@ void GNND::build(Data_t* data, static_cast(build_config_.output_graph_degree)}; raft::matrix::slice( res, raft::make_const_mdspan(graph_d_dists.view()), output_dist_view, coords); + raft::resource::sync_stream(res); } Index_t* graph_shrink_buffer = (Index_t*)graph_.h_dists.data_handle(); @@ -1414,14 +1426,14 @@ void GNND::build(Data_t* data, template , typename Accessor = host_device_accessor, memory_type::host>> void build(raft::resources const& res, const index_params& params, mdspan, row_major, Accessor> dataset, index& idx, - epilogue_op distance_epilogue = raft::identity_op()) + epilogue_op distance_epilogue = DistEpilogue()) { RAFT_EXPECTS(dataset.extent(0) < std::numeric_limits::max() - 1, "The dataset size for GNND should be less than %d", @@ -1491,13 +1503,13 @@ void build(raft::resources const& res, template , typename Accessor = host_device_accessor, memory_type::host>> index build(raft::resources const& res, const index_params& params, mdspan, row_major, Accessor> dataset, - epilogue_op distance_epilogue = raft::identity_op()) + epilogue_op distance_epilogue = DistEpilogue()) { size_t intermediate_degree = params.intermediate_graph_degree; size_t graph_degree = params.graph_degree; diff --git a/cpp/include/raft/neighbors/detail/nn_descent_batch.cuh b/cpp/include/raft/neighbors/detail/nn_descent_batch.cuh new file mode 100644 index 0000000000..78467c9741 --- /dev/null +++ b/cpp/include/raft/neighbors/detail/nn_descent_batch.cuh @@ -0,0 +1,701 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#undef RAFT_EXPLICIT_INSTANTIATE_ONLY + +#include "../nn_descent_types.hpp" +#include "nn_descent.cuh" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include + +namespace raft::neighbors::experimental::nn_descent::detail { + +// +// Run balanced kmeans on a subsample of the dataset to get centroids +// +template , memory_type::host>> +void get_balanced_kmeans_centroids( + raft::resources const& res, + raft::distance::DistanceType metric, + mdspan, row_major, Accessor> dataset, + raft::device_matrix_view centroids) +{ + size_t num_rows = static_cast(dataset.extent(0)); + size_t num_cols = static_cast(dataset.extent(1)); + size_t n_clusters = centroids.extent(0); + size_t num_subsamples = + std::min(static_cast(num_rows / n_clusters), static_cast(num_rows * 0.1)); + + auto d_subsample_dataset = + raft::make_device_matrix(res, num_subsamples, num_cols); + raft::matrix::sample_rows( + res, raft::random::RngState{0}, dataset, d_subsample_dataset.view()); + + raft::cluster::kmeans_balanced_params kmeans_params; + kmeans_params.metric = metric; + + auto d_subsample_dataset_const_view = + raft::make_device_matrix_view( + d_subsample_dataset.data_handle(), num_subsamples, num_cols); + raft::cluster::kmeans_balanced::fit( + res, kmeans_params, d_subsample_dataset_const_view, centroids); +} + +// +// Get the top k closest centroid indices for each data point +// Loads the data in batches onto device if data is on host for memory efficiency +// +template +void get_global_nearest_k( + raft::resources const& res, + size_t k, + size_t num_rows, + size_t n_clusters, + const T* dataset, + raft::host_matrix_view global_nearest_cluster, + raft::device_matrix_view centroids, + raft::distance::DistanceType metric) +{ + size_t num_cols = centroids.extent(1); + + cudaPointerAttributes attr; + RAFT_CUDA_TRY(cudaPointerGetAttributes(&attr, dataset)); + float* ptr = reinterpret_cast(attr.devicePointer); + + if (ptr == nullptr) { // data on host + size_t num_batches = n_clusters; + size_t batch_size = (num_rows + n_clusters) / n_clusters; + + auto d_dataset_batch = + raft::make_device_matrix(res, batch_size, num_cols); + + auto nearest_clusters_idx = + raft::make_device_matrix(res, batch_size, k); + auto nearest_clusters_dist = + raft::make_device_matrix(res, batch_size, k); + + for (size_t i = 0; i < num_batches; i++) { + size_t batch_size_ = batch_size; + + if (i == num_batches - 1) { batch_size_ = num_rows - batch_size * i; } + raft::copy(d_dataset_batch.data_handle(), + dataset + i * batch_size * num_cols, + batch_size_ * num_cols, + resource::get_cuda_stream(res)); + + raft::neighbors::brute_force::fused_l2_knn( + res, + raft::make_const_mdspan(centroids), + raft::make_const_mdspan(d_dataset_batch.view()), + nearest_clusters_idx.view(), + nearest_clusters_dist.view(), + metric); + raft::copy(global_nearest_cluster.data_handle() + i * batch_size * k, + nearest_clusters_idx.data_handle(), + batch_size_ * k, + resource::get_cuda_stream(res)); + } + } else { // data on device + auto nearest_clusters_idx = + raft::make_device_matrix(res, num_rows, k); + auto nearest_clusters_dist = + raft::make_device_matrix(res, num_rows, k); + + raft::neighbors::brute_force::fused_l2_knn( + res, + raft::make_const_mdspan(centroids), + raft::make_device_matrix_view(dataset, num_rows, num_cols), + nearest_clusters_idx.view(), + nearest_clusters_dist.view(), + metric); + + raft::copy(global_nearest_cluster.data_handle(), + nearest_clusters_idx.data_handle(), + num_rows * k, + resource::get_cuda_stream(res)); + } +} + +// +// global_nearest_cluster [num_rows X k=2] : top 2 closest clusters for each data point +// inverted_indices [num_rows x k vector] : sparse vector for data indices for each cluster +// cluster_size [n_cluster] : cluster size for each cluster +// offset [n_cluster] : offset in inverted_indices for each cluster +// Loads the data in batches onto device if data is on host for memory efficiency +// +template +void get_inverted_indices(raft::resources const& res, + size_t n_clusters, + size_t& max_cluster_size, + size_t& min_cluster_size, + raft::host_matrix_view global_nearest_cluster, + raft::host_vector_view inverted_indices, + raft::host_vector_view cluster_size, + raft::host_vector_view offset) +{ + // build sparse inverted indices and get number of data points for each cluster + size_t num_rows = global_nearest_cluster.extent(0); + size_t k = global_nearest_cluster.extent(1); + + auto local_offset = raft::make_host_vector(n_clusters); + + max_cluster_size = 0; + min_cluster_size = std::numeric_limits::max(); + + thrust::fill( + thrust::host, cluster_size.data_handle(), cluster_size.data_handle() + n_clusters, 0); + thrust::fill( + thrust::host, local_offset.data_handle(), local_offset.data_handle() + n_clusters, 0); + + // TODO: this part isn't really a bottleneck but maybe worth trying omp parallel + // for with atomic add + for (size_t i = 0; i < num_rows; i++) { + for (size_t j = 0; j < k; j++) { + IdxT cluster_id = global_nearest_cluster(i, j); + cluster_size(cluster_id) += 1; + } + } + + offset(0) = 0; + for (size_t i = 1; i < n_clusters; i++) { + offset(i) = offset(i - 1) + cluster_size(i - 1); + } + for (size_t i = 0; i < num_rows; i++) { + for (size_t j = 0; j < k; j++) { + IdxT cluster_id = global_nearest_cluster(i, j); + inverted_indices(offset(cluster_id) + local_offset(cluster_id)) = i; + local_offset(cluster_id) += 1; + } + } + + max_cluster_size = static_cast( + *std::max_element(cluster_size.data_handle(), cluster_size.data_handle() + n_clusters)); + min_cluster_size = static_cast( + *std::min_element(cluster_size.data_handle(), cluster_size.data_handle() + n_clusters)); +} + +template +struct KeyValuePair { + KeyType key; + ValueType value; +}; + +template +struct CustomKeyComparator { + __device__ bool operator()(const KeyValuePair& a, + const KeyValuePair& b) const + { + if (a.key == b.key) { return a.value < b.value; } + return a.key < b.key; + } +}; + +template +RAFT_KERNEL merge_subgraphs(IdxT* cluster_data_indices, + size_t graph_degree, + size_t num_cluster_in_batch, + float* global_distances, + float* batch_distances, + IdxT* global_indices, + IdxT* batch_indices) +{ + size_t batch_row = blockIdx.x; + typedef cub::BlockMergeSort, BLOCK_SIZE, ITEMS_PER_THREAD> + BlockMergeSortType; + __shared__ typename cub::BlockMergeSort, BLOCK_SIZE, ITEMS_PER_THREAD>:: + TempStorage tmpSmem; + + extern __shared__ char sharedMem[]; + float* blockKeys = reinterpret_cast(sharedMem); + IdxT* blockValues = reinterpret_cast(&sharedMem[graph_degree * 2 * sizeof(float)]); + int16_t* uniqueMask = + reinterpret_cast(&sharedMem[graph_degree * 2 * (sizeof(float) + sizeof(IdxT))]); + + if (batch_row < num_cluster_in_batch) { + // load batch or global depending on threadIdx + size_t global_row = cluster_data_indices[batch_row]; + + KeyValuePair threadKeyValuePair[ITEMS_PER_THREAD]; + + size_t halfway = BLOCK_SIZE / 2; + size_t do_global = threadIdx.x < halfway; + + float* distances; + IdxT* indices; + + if (do_global) { + distances = global_distances; + indices = global_indices; + } else { + distances = batch_distances; + indices = batch_indices; + } + + size_t idxBase = (threadIdx.x * do_global + (threadIdx.x - halfway) * (1lu - do_global)) * + static_cast(ITEMS_PER_THREAD); + size_t arrIdxBase = (global_row * do_global + batch_row * (1lu - do_global)) * graph_degree; + for (int i = 0; i < ITEMS_PER_THREAD; i++) { + size_t colId = idxBase + i; + if (colId < graph_degree) { + threadKeyValuePair[i].key = distances[arrIdxBase + colId]; + threadKeyValuePair[i].value = indices[arrIdxBase + colId]; + } else { + threadKeyValuePair[i].key = std::numeric_limits::max(); + threadKeyValuePair[i].value = std::numeric_limits::max(); + } + } + + __syncthreads(); + + BlockMergeSortType(tmpSmem).Sort(threadKeyValuePair, CustomKeyComparator{}); + + // load sorted result into shared memory to get unique values + idxBase = threadIdx.x * ITEMS_PER_THREAD; + for (int i = 0; i < ITEMS_PER_THREAD; i++) { + size_t colId = idxBase + i; + if (colId < 2 * graph_degree) { + blockKeys[colId] = threadKeyValuePair[i].key; + blockValues[colId] = threadKeyValuePair[i].value; + } + } + + __syncthreads(); + + // get unique mask + if (threadIdx.x == 0) { uniqueMask[0] = 1; } + for (int i = 0; i < ITEMS_PER_THREAD; i++) { + size_t colId = idxBase + i; + if (colId > 0 && colId < 2 * graph_degree) { + uniqueMask[colId] = static_cast(blockValues[colId] != blockValues[colId - 1]); + } + } + + __syncthreads(); + + // prefix sum + if (threadIdx.x == 0) { + for (int i = 1; i < 2 * graph_degree; i++) { + uniqueMask[i] += uniqueMask[i - 1]; + } + } + + __syncthreads(); + // load unique values to global memory + if (threadIdx.x == 0) { + global_distances[global_row * graph_degree] = blockKeys[0]; + global_indices[global_row * graph_degree] = blockValues[0]; + } + + for (int i = 0; i < ITEMS_PER_THREAD; i++) { + size_t colId = idxBase + i; + if (colId > 0 && colId < 2 * graph_degree) { + bool is_unique = uniqueMask[colId] != uniqueMask[colId - 1]; + int16_t global_colId = uniqueMask[colId] - 1; + if (is_unique && static_cast(global_colId) < graph_degree) { + global_distances[global_row * graph_degree + global_colId] = blockKeys[colId]; + global_indices[global_row * graph_degree + global_colId] = blockValues[colId]; + } + } + } + } +} + +// +// builds knn graph using NN Descent and merge with global graph +// +template , + typename Accessor = + host_device_accessor, memory_type::host>> +void build_and_merge(raft::resources const& res, + const index_params& params, + size_t num_data_in_cluster, + size_t graph_degree, + size_t int_graph_node_degree, + T* cluster_data, + IdxT* cluster_data_indices, + int* int_graph, + IdxT* inverted_indices, + IdxT* global_indices_d, + float* global_distances_d, + IdxT* batch_indices_h, + IdxT* batch_indices_d, + float* batch_distances_d, + GNND& nnd, + epilogue_op distance_epilogue) +{ + nnd.build( + cluster_data, num_data_in_cluster, int_graph, true, batch_distances_d, distance_epilogue); + + // remap indices +#pragma omp parallel for + for (size_t i = 0; i < num_data_in_cluster; i++) { + for (size_t j = 0; j < graph_degree; j++) { + size_t local_idx = int_graph[i * int_graph_node_degree + j]; + batch_indices_h[i * graph_degree + j] = inverted_indices[local_idx]; + } + } + + raft::copy(batch_indices_d, + batch_indices_h, + num_data_in_cluster * graph_degree, + raft::resource::get_cuda_stream(res)); + + size_t num_elems = graph_degree * 2; + size_t sharedMemSize = num_elems * (sizeof(float) + sizeof(IdxT) + sizeof(int16_t)); + + if (num_elems <= 128) { + merge_subgraphs + <<>>( + cluster_data_indices, + graph_degree, + num_data_in_cluster, + global_distances_d, + batch_distances_d, + global_indices_d, + batch_indices_d); + } else if (num_elems <= 512) { + merge_subgraphs + <<>>( + cluster_data_indices, + graph_degree, + num_data_in_cluster, + global_distances_d, + batch_distances_d, + global_indices_d, + batch_indices_d); + } else if (num_elems <= 1024) { + merge_subgraphs + <<>>( + cluster_data_indices, + graph_degree, + num_data_in_cluster, + global_distances_d, + batch_distances_d, + global_indices_d, + batch_indices_d); + } else if (num_elems <= 2048) { + merge_subgraphs + <<>>( + cluster_data_indices, + graph_degree, + num_data_in_cluster, + global_distances_d, + batch_distances_d, + global_indices_d, + batch_indices_d); + } else { + // this is as far as we can get due to the shared mem usage of cub::BlockMergeSort + RAFT_FAIL("The degree of knn is too large (%lu). It must be smaller than 1024", graph_degree); + } + raft::resource::sync_stream(res); +} + +// +// For each cluster, gather the data samples that belong to that cluster, and +// call build_and_merge +// +template > +void cluster_nnd(raft::resources const& res, + const index_params& params, + size_t graph_degree, + size_t extended_graph_degree, + size_t max_cluster_size, + raft::host_matrix_view dataset, + IdxT* offsets, + IdxT* cluster_size, + IdxT* cluster_data_indices, + int* int_graph, + IdxT* inverted_indices, + IdxT* global_indices_h, + float* global_distances_h, + IdxT* batch_indices_h, + IdxT* batch_indices_d, + float* batch_distances_d, + const BuildConfig& build_config, + epilogue_op distance_epilogue) +{ + size_t num_rows = dataset.extent(0); + size_t num_cols = dataset.extent(1); + + GNND nnd(res, build_config); + + auto cluster_data_matrix = + raft::make_host_matrix(max_cluster_size, num_cols); + + for (size_t cluster_id = 0; cluster_id < params.n_clusters; cluster_id++) { + RAFT_LOG_DEBUG( + "# Data on host. Running clusters: %lu / %lu", cluster_id + 1, params.n_clusters); + size_t num_data_in_cluster = cluster_size[cluster_id]; + size_t offset = offsets[cluster_id]; + +#pragma omp parallel for + for (size_t i = 0; i < num_data_in_cluster; i++) { + for (size_t j = 0; j < num_cols; j++) { + size_t global_row = (inverted_indices + offset)[i]; + cluster_data_matrix(i, j) = dataset(global_row, j); + } + } + + distance_epilogue.preprocess_for_batch(cluster_data_indices + offset, num_data_in_cluster); + + build_and_merge(res, + params, + num_data_in_cluster, + graph_degree, + extended_graph_degree, + cluster_data_matrix.data_handle(), + cluster_data_indices + offset, + int_graph, + inverted_indices + offset, + global_indices_h, + global_distances_h, + batch_indices_h, + batch_indices_d, + batch_distances_d, + nnd, + distance_epilogue); + nnd.reset(res); + } +} + +template > +void cluster_nnd(raft::resources const& res, + const index_params& params, + size_t graph_degree, + size_t extended_graph_degree, + size_t max_cluster_size, + raft::device_matrix_view dataset, + IdxT* offsets, + IdxT* cluster_size, + IdxT* cluster_data_indices, + int* int_graph, + IdxT* inverted_indices, + IdxT* global_indices_h, + float* global_distances_h, + IdxT* batch_indices_h, + IdxT* batch_indices_d, + float* batch_distances_d, + const BuildConfig& build_config, + epilogue_op distance_epilogue) +{ + size_t num_rows = dataset.extent(0); + size_t num_cols = dataset.extent(1); + + GNND nnd(res, build_config); + + auto cluster_data_matrix = + raft::make_device_matrix(res, max_cluster_size, num_cols); + + for (size_t cluster_id = 0; cluster_id < params.n_clusters; cluster_id++) { + RAFT_LOG_DEBUG( + "# Data on device. Running clusters: %lu / %lu", cluster_id + 1, params.n_clusters); + size_t num_data_in_cluster = cluster_size[cluster_id]; + size_t offset = offsets[cluster_id]; + + auto cluster_data_view = raft::make_device_matrix_view( + cluster_data_matrix.data_handle(), num_data_in_cluster, num_cols); + auto cluster_data_indices_view = raft::make_device_vector_view( + cluster_data_indices + offset, num_data_in_cluster); + distance_epilogue.preprocess_for_batch(cluster_data_indices + offset, num_data_in_cluster); + + auto dataset_IdxT = + raft::make_device_matrix_view(dataset.data_handle(), num_rows, num_cols); + raft::matrix::gather(res, dataset_IdxT, cluster_data_indices_view, cluster_data_view); + + build_and_merge(res, + params, + num_data_in_cluster, + graph_degree, + extended_graph_degree, + cluster_data_view.data_handle(), + cluster_data_indices + offset, + int_graph, + inverted_indices + offset, + global_indices_h, + global_distances_h, + batch_indices_h, + batch_indices_d, + batch_distances_d, + nnd, + distance_epilogue); + nnd.reset(res); + } +} + +template , + typename Accessor = + host_device_accessor, memory_type::host>> +index batch_build(raft::resources const& res, + const index_params& params, + mdspan, row_major, Accessor> dataset, + epilogue_op distance_epilogue = DistEpilogue()) +{ + size_t graph_degree = params.graph_degree; + size_t intermediate_degree = params.intermediate_graph_degree; + + size_t num_rows = static_cast(dataset.extent(0)); + size_t num_cols = static_cast(dataset.extent(1)); + + auto centroids = + raft::make_device_matrix(res, params.n_clusters, num_cols); + get_balanced_kmeans_centroids(res, params.metric, dataset, centroids.view()); + + size_t k = 2; + auto global_nearest_cluster = raft::make_host_matrix(num_rows, k); + get_global_nearest_k(res, + k, + num_rows, + params.n_clusters, + dataset.data_handle(), + global_nearest_cluster.view(), + centroids.view(), + params.metric); + + auto inverted_indices = raft::make_host_vector(num_rows * k); + auto cluster_size = raft::make_host_vector(params.n_clusters); + auto offset = raft::make_host_vector(params.n_clusters); + + size_t max_cluster_size, min_cluster_size; + get_inverted_indices(res, + params.n_clusters, + max_cluster_size, + min_cluster_size, + global_nearest_cluster.view(), + inverted_indices.view(), + cluster_size.view(), + offset.view()); + + if (intermediate_degree >= min_cluster_size) { + RAFT_LOG_WARN( + "Intermediate graph degree cannot be larger than minimum cluster size, reducing it to %lu", + dataset.extent(0)); + intermediate_degree = min_cluster_size - 1; + } + if (intermediate_degree < graph_degree) { + RAFT_LOG_WARN( + "Graph degree (%lu) cannot be larger than intermediate graph degree (%lu), reducing " + "graph_degree.", + graph_degree, + intermediate_degree); + graph_degree = intermediate_degree; + } + + size_t extended_graph_degree = + align32::roundUp(static_cast(graph_degree * (graph_degree <= 32 ? 1.0 : 1.3))); + size_t extended_intermediate_degree = align32::roundUp( + static_cast(intermediate_degree * (intermediate_degree <= 32 ? 1.0 : 1.3))); + + auto int_graph = raft::make_host_matrix( + max_cluster_size, static_cast(extended_graph_degree)); + + BuildConfig build_config{.max_dataset_size = max_cluster_size, + .dataset_dim = num_cols, + .node_degree = extended_graph_degree, + .internal_node_degree = extended_intermediate_degree, + .max_iterations = params.max_iterations, + .termination_threshold = params.termination_threshold, + .output_graph_degree = graph_degree}; + + auto global_indices_h = raft::make_managed_matrix(res, num_rows, graph_degree); + auto global_distances_h = raft::make_managed_matrix(res, num_rows, graph_degree); + + thrust::fill(thrust::host, + global_indices_h.data_handle(), + global_indices_h.data_handle() + num_rows * graph_degree, + std::numeric_limits::max()); + thrust::fill(thrust::host, + global_distances_h.data_handle(), + global_distances_h.data_handle() + num_rows * graph_degree, + std::numeric_limits::max()); + + auto batch_indices_h = + raft::make_host_matrix(max_cluster_size, graph_degree); + auto batch_indices_d = + raft::make_device_matrix(res, max_cluster_size, graph_degree); + auto batch_distances_d = + raft::make_device_matrix(res, max_cluster_size, graph_degree); + + auto cluster_data_indices = raft::make_device_vector(res, num_rows * k); + raft::copy(cluster_data_indices.data_handle(), + inverted_indices.data_handle(), + num_rows * k, + resource::get_cuda_stream(res)); + + cluster_nnd(res, + params, + graph_degree, + extended_graph_degree, + max_cluster_size, + dataset, + offset.data_handle(), + cluster_size.data_handle(), + cluster_data_indices.data_handle(), + int_graph.data_handle(), + inverted_indices.data_handle(), + global_indices_h.data_handle(), + global_distances_h.data_handle(), + batch_indices_h.data_handle(), + batch_indices_d.data_handle(), + batch_distances_d.data_handle(), + build_config, + distance_epilogue); + + index global_idx{ + res, dataset.extent(0), static_cast(graph_degree), params.return_distances}; + + raft::copy(global_idx.graph().data_handle(), + global_indices_h.data_handle(), + num_rows * graph_degree, + raft::resource::get_cuda_stream(res)); + if (params.return_distances && global_idx.distances().has_value()) { + raft::copy(global_idx.distances().value().data_handle(), + global_distances_h.data_handle(), + num_rows * graph_degree, + raft::resource::get_cuda_stream(res)); + } + return global_idx; +} + +} // namespace raft::neighbors::experimental::nn_descent::detail diff --git a/cpp/include/raft/neighbors/nn_descent.cuh b/cpp/include/raft/neighbors/nn_descent.cuh index a46a2006d6..6c08546d3f 100644 --- a/cpp/include/raft/neighbors/nn_descent.cuh +++ b/cpp/include/raft/neighbors/nn_descent.cuh @@ -17,9 +17,11 @@ #pragma once #include "detail/nn_descent.cuh" +#include "detail/nn_descent_batch.cuh" #include #include +#include namespace raft::neighbors::experimental::nn_descent { @@ -57,13 +59,17 @@ namespace raft::neighbors::experimental::nn_descent { * @param[in] distance_epilogue epilogue operation for distances * @return index index containing all-neighbors knn graph in host memory */ -template +template > index build(raft::resources const& res, index_params const& params, raft::device_matrix_view dataset, - epilogue_op distance_epilogue = raft::identity_op()) + epilogue_op distance_epilogue = DistEpilogue()) { - return detail::build(res, params, dataset, distance_epilogue); + if (params.n_clusters > 1) { + return detail::batch_build(res, params, dataset, distance_epilogue); + } else { + return detail::build(res, params, dataset, distance_epilogue); + } } /** @@ -98,12 +104,12 @@ index build(raft::resources const& res, * in host memory * @param[in] distance_epilogue epilogue operation for distances */ -template +template > void build(raft::resources const& res, index_params const& params, raft::device_matrix_view dataset, index& idx, - epilogue_op distance_epilogue = raft::identity_op()) + epilogue_op distance_epilogue = DistEpilogue()) { detail::build(res, params, dataset, idx, distance_epilogue); } @@ -137,13 +143,17 @@ void build(raft::resources const& res, * @param[in] distance_epilogue epilogue operation for distances * @return index index containing all-neighbors knn graph in host memory */ -template +template > index build(raft::resources const& res, index_params const& params, raft::host_matrix_view dataset, - epilogue_op distance_epilogue = raft::identity_op()) + epilogue_op distance_epilogue = DistEpilogue()) { - return detail::build(res, params, dataset, distance_epilogue); + if (params.n_clusters > 1) { + return detail::batch_build(res, params, dataset, distance_epilogue); + } else { + return detail::build(res, params, dataset, distance_epilogue); + } } /** @@ -178,12 +188,12 @@ index build(raft::resources const& res, * in host memory * @param[in] distance_epilogue epilogue operation for distances */ -template +template > void build(raft::resources const& res, index_params const& params, raft::host_matrix_view dataset, index& idx, - epilogue_op distance_epilogue = raft::identity_op()) + epilogue_op distance_epilogue = DistEpilogue()) { detail::build(res, params, dataset, idx, distance_epilogue); } diff --git a/cpp/include/raft/neighbors/nn_descent_types.hpp b/cpp/include/raft/neighbors/nn_descent_types.hpp index 5d23ff2c2e..eb01a423be 100644 --- a/cpp/include/raft/neighbors/nn_descent_types.hpp +++ b/cpp/include/raft/neighbors/nn_descent_types.hpp @@ -48,6 +48,20 @@ namespace raft::neighbors::experimental::nn_descent { * `max_iterations`: The number of iterations that nn-descent will refine * the graph for. More iterations produce a better quality graph at cost of performance * `termination_threshold`: The delta at which nn-descent will terminate its iterations + * `return_distances`: boolean whether to return distances + * `n_clusters`: NN Descent offers batching a dataset to save GPU memory usage. + * Increase `n_clusters` to save GPU memory and run NN Descent with large datasets. + * Most effective when data is put on CPU memory. + * Setting this number too big may results in too much overhead of doing multiple + * iterations of graph building. Recommend starting at 4 and continue to increase + * depending on desired GPU memory usages. + * (Specifically, with n_clusters > 1, the NN Descent build algorithm will first + * find n_clusters number of cluster centroids of the dataset, then consider data + * points that belong to each cluster as a batch. + * Then we build knn subgraphs on each batch of the entire data. This is especially + * useful when the dataset is put on host, since only a subset of the data will + * be on GPU at once, enabling running NN Descent with large datasets that do not + * fit on the GPU as a whole.) * */ struct index_params : ann::index_params { @@ -56,6 +70,7 @@ struct index_params : ann::index_params { size_t max_iterations = 20; // Number of nn-descent iterations. float termination_threshold = 0.0001; // Termination threshold of nn-descent. bool return_distances = false; // return distances if true + size_t n_clusters = 1; // defaults to not using any batching }; /** @@ -178,6 +193,14 @@ struct index : ann::index { bool return_distances_; }; +template +struct DistEpilogue : raft::identity_op { + __host__ void preprocess_for_batch(value_idx* cluster_indices, size_t num_data_in_cluster) + { + return; + } +}; + /** @} */ } // namespace raft::neighbors::experimental::nn_descent diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index e3af6ebb78..a497e6d3ba 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -96,17 +96,8 @@ endfunction() if(BUILD_TESTS) ConfigureTest( - NAME - CLUSTER_TEST - PATH - cluster/kmeans.cu - cluster/kmeans_balanced.cu - cluster/kmeans_find_k.cu - cluster/cluster_solvers.cu - cluster/linkage.cu - cluster/spectral.cu - LIB - EXPLICIT_INSTANTIATE_ONLY + NAME CLUSTER_TEST PATH cluster/kmeans.cu cluster/kmeans_balanced.cu cluster/kmeans_find_k.cu + cluster/cluster_solvers.cu cluster/linkage.cu cluster/spectral.cu LIB EXPLICIT_INSTANTIATE_ONLY ) ConfigureTest( @@ -144,8 +135,8 @@ if(BUILD_TESTS) ) ConfigureTest( - NAME CORE_TEST PATH core/stream_view.cpp core/mdspan_copy.cpp LIB - EXPLICIT_INSTANTIATE_ONLY NOCUDA + NAME CORE_TEST PATH core/stream_view.cpp core/mdspan_copy.cpp LIB EXPLICIT_INSTANTIATE_ONLY + NOCUDA ) ConfigureTest( @@ -301,8 +292,8 @@ if(BUILD_TESTS) ) ConfigureTest( - NAME SOLVERS_TEST PATH cluster/cluster_solvers_deprecated.cu linalg/eigen_solvers.cu - lap/lap.cu sparse/mst.cu LIB EXPLICIT_INSTANTIATE_ONLY + NAME SOLVERS_TEST PATH cluster/cluster_solvers_deprecated.cu linalg/eigen_solvers.cu lap/lap.cu + sparse/mst.cu LIB EXPLICIT_INSTANTIATE_ONLY ) ConfigureTest( @@ -331,19 +322,13 @@ if(BUILD_TESTS) ) ConfigureTest( - NAME SPARSE_DIST_TEST PATH sparse/dist_coo_spmv.cu sparse/distance.cu - sparse/gram.cu LIB EXPLICIT_INSTANTIATE_ONLY + NAME SPARSE_DIST_TEST PATH sparse/dist_coo_spmv.cu sparse/distance.cu sparse/gram.cu LIB + EXPLICIT_INSTANTIATE_ONLY ) ConfigureTest( - NAME - SPARSE_NEIGHBORS_TEST - PATH - sparse/neighbors/cross_component_nn.cu - sparse/neighbors/brute_force.cu - sparse/neighbors/knn_graph.cu - LIB - EXPLICIT_INSTANTIATE_ONLY + NAME SPARSE_NEIGHBORS_TEST PATH sparse/neighbors/cross_component_nn.cu + sparse/neighbors/brute_force.cu sparse/neighbors/knn_graph.cu LIB EXPLICIT_INSTANTIATE_ONLY ) ConfigureTest( @@ -455,6 +440,7 @@ if(BUILD_TESTS) neighbors/ann_nn_descent/test_float_uint32_t.cu neighbors/ann_nn_descent/test_int8_t_uint32_t.cu neighbors/ann_nn_descent/test_uint8_t_uint32_t.cu + neighbors/ann_nn_descent/test_batch_float_uint32_t.cu LIB EXPLICIT_INSTANTIATE_ONLY GPUS diff --git a/cpp/test/neighbors/ann_nn_descent.cuh b/cpp/test/neighbors/ann_nn_descent.cuh index f74cadb415..2f9d4e252b 100644 --- a/cpp/test/neighbors/ann_nn_descent.cuh +++ b/cpp/test/neighbors/ann_nn_descent.cuh @@ -42,6 +42,15 @@ struct AnnNNDescentInputs { double min_recall; }; +struct AnnNNDescentBatchInputs { + std::pair recall_cluster; + int n_rows; + int dim; + int graph_degree; + raft::distance::DistanceType metric; + bool host_dataset; +}; + inline ::std::ostream& operator<<(::std::ostream& os, const AnnNNDescentInputs& p) { os << "dataset shape=" << p.n_rows << "x" << p.dim << ", graph_degree=" << p.graph_degree @@ -50,6 +59,14 @@ inline ::std::ostream& operator<<(::std::ostream& os, const AnnNNDescentInputs& return os; } +inline ::std::ostream& operator<<(::std::ostream& os, const AnnNNDescentBatchInputs& p) +{ + os << "dataset shape=" << p.n_rows << "x" << p.dim << ", graph_degree=" << p.graph_degree + << ", metric=" << static_cast(p.metric) << (p.host_dataset ? ", host" : ", device") + << ", clusters=" << p.recall_cluster.second << std::endl; + return os; +} + template class AnnNNDescentTest : public ::testing::TestWithParam { public: @@ -105,7 +122,9 @@ class AnnNNDescentTest : public ::testing::TestWithParam { raft::copy(database_host.data_handle(), database.data(), database.size(), stream_); auto database_host_view = raft::make_host_matrix_view( (const DataT*)database_host.data_handle(), ps.n_rows, ps.dim); - auto index = nn_descent::build(handle_, index_params, database_host_view); + index index{handle_, ps.n_rows, static_cast(ps.graph_degree), true}; + nn_descent::build( + handle_, index_params, database_host_view, index, DistEpilogue()); raft::copy( indices_NNDescent.data(), index.graph().data_handle(), queries_size, stream_); if (index.distances().has_value()) { @@ -116,7 +135,9 @@ class AnnNNDescentTest : public ::testing::TestWithParam { } } else { - auto index = nn_descent::build(handle_, index_params, database_view); + index index{handle_, ps.n_rows, static_cast(ps.graph_degree), true}; + nn_descent::build( + handle_, index_params, database_view, index, DistEpilogue()); raft::copy( indices_NNDescent.data(), index.graph().data_handle(), queries_size, stream_); if (index.distances().has_value()) { @@ -168,6 +189,127 @@ class AnnNNDescentTest : public ::testing::TestWithParam { rmm::device_uvector database; }; +template +class AnnNNDescentBatchTest : public ::testing::TestWithParam { + public: + AnnNNDescentBatchTest() + : stream_(resource::get_cuda_stream(handle_)), + ps(::testing::TestWithParam::GetParam()), + database(0, stream_) + { + } + + void testNNDescentBatch() + { + size_t queries_size = ps.n_rows * ps.graph_degree; + std::vector indices_NNDescent(queries_size); + std::vector distances_NNDescent(queries_size); + std::vector indices_naive(queries_size); + std::vector distances_naive(queries_size); + + { + rmm::device_uvector distances_naive_dev(queries_size, stream_); + rmm::device_uvector indices_naive_dev(queries_size, stream_); + naive_knn(handle_, + distances_naive_dev.data(), + indices_naive_dev.data(), + database.data(), + database.data(), + ps.n_rows, + ps.n_rows, + ps.dim, + ps.graph_degree, + ps.metric); + update_host(indices_naive.data(), indices_naive_dev.data(), queries_size, stream_); + update_host(distances_naive.data(), distances_naive_dev.data(), queries_size, stream_); + resource::sync_stream(handle_); + } + + { + { + nn_descent::index_params index_params; + index_params.metric = ps.metric; + index_params.graph_degree = ps.graph_degree; + index_params.intermediate_graph_degree = 2 * ps.graph_degree; + index_params.max_iterations = 10; + index_params.return_distances = true; + index_params.n_clusters = ps.recall_cluster.second; + + auto database_view = raft::make_device_matrix_view( + (const DataT*)database.data(), ps.n_rows, ps.dim); + + { + if (ps.host_dataset) { + auto database_host = raft::make_host_matrix(ps.n_rows, ps.dim); + raft::copy(database_host.data_handle(), database.data(), database.size(), stream_); + auto database_host_view = raft::make_host_matrix_view( + (const DataT*)database_host.data_handle(), ps.n_rows, ps.dim); + auto index = nn_descent::build( + handle_, index_params, database_host_view, DistEpilogue()); + raft::copy( + indices_NNDescent.data(), index.graph().data_handle(), queries_size, stream_); + if (index.distances().has_value()) { + raft::copy(distances_NNDescent.data(), + index.distances().value().data_handle(), + queries_size, + stream_); + } + + } else { + auto index = nn_descent::build( + handle_, index_params, database_view, DistEpilogue()); + raft::copy( + indices_NNDescent.data(), index.graph().data_handle(), queries_size, stream_); + if (index.distances().has_value()) { + raft::copy(distances_NNDescent.data(), + index.distances().value().data_handle(), + queries_size, + stream_); + } + }; + } + resource::sync_stream(handle_); + } + double min_recall = ps.recall_cluster.first; + EXPECT_TRUE(eval_neighbours(indices_naive, + indices_NNDescent, + distances_naive, + distances_NNDescent, + ps.n_rows, + ps.graph_degree, + 0.01, + min_recall, + true, + static_cast(ps.graph_degree * 0.1))); + } + } + + void SetUp() override + { + database.resize(((size_t)ps.n_rows) * ps.dim, stream_); + raft::random::RngState r(1234ULL); + if constexpr (std::is_same{}) { + raft::random::normal(handle_, r, database.data(), ps.n_rows * ps.dim, DataT(0.1), DataT(2.0)); + } else { + raft::random::uniformInt( + handle_, r, database.data(), ps.n_rows * ps.dim, DataT(1), DataT(20)); + } + resource::sync_stream(handle_); + } + + void TearDown() override + { + resource::sync_stream(handle_); + database.resize(0, stream_); + } + + private: + raft::resources handle_; + rmm::cuda_stream_view stream_; + AnnNNDescentBatchInputs ps; + rmm::device_uvector database; +}; + const std::vector inputs = raft::util::itertools::product( {1000, 2000}, // n_rows {3, 5, 7, 8, 17, 64, 128, 137, 192, 256, 512, 619, 1024}, // dim @@ -176,4 +318,13 @@ const std::vector inputs = raft::util::itertools::product inputsBatch = + raft::util::itertools::product( + {std::make_pair(0.9, 3lu), std::make_pair(0.9, 2lu)}, // min_recall, n_clusters + {4000, 5000}, // n_rows + {192, 512}, // dim + {32, 64}, // graph_degree + {raft::distance::DistanceType::L2Expanded}, + {false, true}); + } // namespace raft::neighbors::experimental::nn_descent diff --git a/cpp/test/neighbors/ann_nn_descent/test_batch_float_uint32_t.cu b/cpp/test/neighbors/ann_nn_descent/test_batch_float_uint32_t.cu new file mode 100644 index 0000000000..c6f56e8c39 --- /dev/null +++ b/cpp/test/neighbors/ann_nn_descent/test_batch_float_uint32_t.cu @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../ann_nn_descent.cuh" + +#include + +namespace raft::neighbors::experimental::nn_descent { + +typedef AnnNNDescentBatchTest AnnNNDescentBatchTestF_U32; +TEST_P(AnnNNDescentBatchTestF_U32, AnnNNDescentBatch) { this->testNNDescentBatch(); } + +INSTANTIATE_TEST_CASE_P(AnnNNDescentBatchTest, + AnnNNDescentBatchTestF_U32, + ::testing::ValuesIn(inputsBatch)); + +} // namespace raft::neighbors::experimental::nn_descent diff --git a/cpp/test/neighbors/ann_utils.cuh b/cpp/test/neighbors/ann_utils.cuh index 2139e97428..82e3ace9da 100644 --- a/cpp/test/neighbors/ann_utils.cuh +++ b/cpp/test/neighbors/ann_utils.cuh @@ -153,9 +153,13 @@ auto calc_recall(const std::vector& expected_idx, /** check uniqueness of indices */ template -auto check_unique_indices(const std::vector& actual_idx, size_t rows, size_t cols) +auto check_unique_indices(const std::vector& actual_idx, + size_t rows, + size_t cols, + size_t max_duplicates) { size_t max_count; + size_t dup_count = 0lu; std::set unique_indices; for (size_t i = 0; i < rows; ++i) { unique_indices.clear(); @@ -168,8 +172,11 @@ auto check_unique_indices(const std::vector& actual_idx, size_t rows, size_t } else if (unique_indices.find(act_idx) == unique_indices.end()) { unique_indices.insert(act_idx); } else { - return testing::AssertionFailure() - << "Duplicated index " << act_idx << " at k " << k << " for query " << i << "! "; + dup_count++; + if (dup_count > max_duplicates) { + return testing::AssertionFailure() + << "Duplicated index " << act_idx << " at k " << k << " for query " << i << "! "; + } } } } @@ -252,7 +259,8 @@ auto eval_neighbours(const std::vector& expected_idx, size_t cols, double eps, double min_recall, - bool test_unique = true) -> testing::AssertionResult + bool test_unique = true, + size_t max_duplicates = 0) -> testing::AssertionResult { auto [actual_recall, match_count, total_count] = calc_recall(expected_idx, actual_idx, expected_dist, actual_dist, rows, cols, eps); @@ -270,7 +278,7 @@ auto eval_neighbours(const std::vector& expected_idx, << min_recall << "); eps = " << eps << ". "; } if (test_unique) - return check_unique_indices(actual_idx, rows, cols); + return check_unique_indices(actual_idx, rows, cols, max_duplicates); else return testing::AssertionSuccess(); } From 2dafe79af0216ad7189415ca3e2864277194c3e9 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 23 Aug 2024 07:49:30 +0200 Subject: [PATCH 14/79] Remove NumPy <2 pin (#2414) This PR removes the NumPy<2 pin which is expected to work for RAPIDS projects once CuPy 13.3.0 is released (CuPy 13.2.0 had some issues preventing the use with NumPy 2). Authors: - Sebastian Berg (https://github.com/seberg) - https://github.com/jakirkham Approvers: - James Lamb (https://github.com/jameslamb) - https://github.com/jakirkham URL: https://github.com/rapidsai/raft/pull/2414 --- conda/environments/all_cuda-118_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-125_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-125_arch-x86_64.yaml | 2 +- conda/recipes/pylibraft/meta.yaml | 2 +- dependencies.yaml | 3 +-- python/pylibraft/pyproject.toml | 2 +- python/raft-dask/pyproject.toml | 1 - 8 files changed, 7 insertions(+), 9 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index ef8524dce3..462874a7e7 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -39,7 +39,7 @@ dependencies: - nccl>=2.9.9 - ninja - numba>=0.57 -- numpy>=1.23,<2.0a0 +- numpy>=1.23,<3.0a0 - numpydoc - nvcc_linux-aarch64=11.8 - pre-commit diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 6ffb27bb29..cfd974a6a8 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -39,7 +39,7 @@ dependencies: - nccl>=2.9.9 - ninja - numba>=0.57 -- numpy>=1.23,<2.0a0 +- numpy>=1.23,<3.0a0 - numpydoc - nvcc_linux-64=11.8 - pre-commit diff --git a/conda/environments/all_cuda-125_arch-aarch64.yaml b/conda/environments/all_cuda-125_arch-aarch64.yaml index fd0e380a1c..82e391e9ae 100644 --- a/conda/environments/all_cuda-125_arch-aarch64.yaml +++ b/conda/environments/all_cuda-125_arch-aarch64.yaml @@ -36,7 +36,7 @@ dependencies: - nccl>=2.9.9 - ninja - numba>=0.57 -- numpy>=1.23,<2.0a0 +- numpy>=1.23,<3.0a0 - numpydoc - pre-commit - pydata-sphinx-theme diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index ad4ecb7ff2..0389427d13 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -36,7 +36,7 @@ dependencies: - nccl>=2.9.9 - ninja - numba>=0.57 -- numpy>=1.23,<2.0a0 +- numpy>=1.23,<3.0a0 - numpydoc - pre-commit - pydata-sphinx-theme diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 4ef85fc0e5..9d91af712e 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -65,7 +65,7 @@ requirements: {% endif %} - libraft {{ version }} - libraft-headers {{ version }} - - numpy >=1.23,<2.0a0 + - numpy >=1.23,<3.0a0 - python x.x - rmm ={{ minor_version }} diff --git a/dependencies.yaml b/dependencies.yaml index 92c6d98414..a5b818e0f3 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -473,7 +473,7 @@ dependencies: common: - output_types: [conda, pyproject] packages: - - &numpy numpy>=1.23,<2.0a0 + - numpy>=1.23,<3.0a0 - output_types: [conda] packages: - *rmm_unsuffixed @@ -513,7 +513,6 @@ dependencies: - dask-cuda==24.10.*,>=0.0.0a0 - joblib>=0.11 - numba>=0.57 - - *numpy - rapids-dask-dependency==24.10.*,>=0.0.0a0 - output_types: conda packages: diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index 9a826e53c6..9a6e454e7b 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -32,7 +32,7 @@ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ "cuda-python", - "numpy>=1.23,<2.0a0", + "numpy>=1.23,<3.0a0", "nvidia-cublas", "nvidia-curand", "nvidia-cusolver", diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index 4fadfa5c9f..4e2e45206f 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -35,7 +35,6 @@ dependencies = [ "distributed-ucxx==0.40.*,>=0.0.0a0", "joblib>=0.11", "numba>=0.57", - "numpy>=1.23,<2.0a0", "pylibraft==24.10.*,>=0.0.0a0", "rapids-dask-dependency==24.10.*,>=0.0.0a0", "ucx-py==0.40.*,>=0.0.0a0", From 73dc36c5304b338ee8d12d775522dbf4d75a014a Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 23 Aug 2024 00:51:31 -0500 Subject: [PATCH 15/79] Drop Python 3.9 support (#2417) Contributes to https://github.com/rapidsai/build-planning/issues/88 Finishes the work of dropping Python 3.9 support. This project stopped building / testing against Python 3.9 as of https://github.com/rapidsai/shared-workflows/pull/235. This PR updates configuration and docs to reflect that. ## Notes for Reviewers ### How I tested this Checked that there were no remaining uses like this: ```shell git grep -E '3\.9' git grep '39' git grep 'py39' ``` And similar for variations on Python 3.8 (to catch things that were missed the last time this was done). Authors: - James Lamb (https://github.com/jameslamb) Approvers: - https://github.com/jakirkham URL: https://github.com/rapidsai/raft/pull/2417 --- dependencies.yaml | 6 +----- pyproject.toml | 2 +- python/pylibraft/pyproject.toml | 3 +-- python/raft-ann-bench/pyproject.toml | 3 +-- python/raft-dask/pyproject.toml | 3 +-- 5 files changed, 5 insertions(+), 12 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index a5b818e0f3..e4e361548f 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -454,10 +454,6 @@ dependencies: specific: - output_types: conda matrices: - - matrix: - py: "3.9" - packages: - - python=3.9 - matrix: py: "3.10" packages: @@ -468,7 +464,7 @@ dependencies: - python=3.11 - matrix: packages: - - python>=3.9,<3.12 + - python>=3.10,<3.12 run_pylibraft: common: - output_types: [conda, pyproject] diff --git a/pyproject.toml b/pyproject.toml index 1e4ba0b369..5042113388 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.black] line-length = 79 -target-version = ["py39"] +target-version = ["py310"] include = '\.py?$' force-exclude = ''' /( diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index 9a6e454e7b..14f2ba7d2f 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -29,7 +29,7 @@ authors = [ { name = "NVIDIA Corporation" }, ] license = { text = "Apache 2.0" } -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ "cuda-python", "numpy>=1.23,<3.0a0", @@ -42,7 +42,6 @@ dependencies = [ classifiers = [ "Intended Audience :: Developers", "Programming Language :: Python", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", ] diff --git a/python/raft-ann-bench/pyproject.toml b/python/raft-ann-bench/pyproject.toml index d22dd567fe..fa5781893b 100644 --- a/python/raft-ann-bench/pyproject.toml +++ b/python/raft-ann-bench/pyproject.toml @@ -16,7 +16,7 @@ authors = [ { name = "NVIDIA Corporation" }, ] license = { text = "Apache 2.0" } -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ @@ -25,7 +25,6 @@ classifiers = [ "Topic :: Scientific/Engineering", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", ] diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index 4e2e45206f..44012b5f10 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -29,7 +29,7 @@ authors = [ { name = "NVIDIA Corporation" }, ] license = { text = "Apache 2.0" } -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ "dask-cuda==24.10.*,>=0.0.0a0", "distributed-ucxx==0.40.*,>=0.0.0a0", @@ -42,7 +42,6 @@ dependencies = [ classifiers = [ "Intended Audience :: Developers", "Programming Language :: Python", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", ] From e17dcdfdbe6985d9a9dfb4bd76d047dc08481630 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 27 Aug 2024 15:50:10 -0400 Subject: [PATCH 16/79] Update rapidsai/pre-commit-hooks (#2420) This PR updates rapidsai/pre-commit-hooks to the version 0.4.0. Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/raft/pull/2420 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9aed6bf387..88be628e55 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -99,7 +99,7 @@ repos: hooks: - id: check-json - repo: https://github.com/rapidsai/pre-commit-hooks - rev: v0.3.1 + rev: v0.4.0 hooks: - id: verify-copyright files: | From e04a640e489da027986f189608098308ae040ed5 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Thu, 29 Aug 2024 13:50:19 -0400 Subject: [PATCH 17/79] Exclude any kernel symbol that uses cutlass (#2425) Authors: - Robert Maynard (https://github.com/robertmaynard) Approvers: - Kyle Edwards (https://github.com/KyleFromNVIDIA) URL: https://github.com/rapidsai/raft/pull/2425 --- .github/workflows/pr.yaml | 2 +- .github/workflows/test.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 98cb724623..381ca6b378 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -51,7 +51,7 @@ jobs: with: build_type: pull-request enable_check_symbols: true - symbol_exclusions: _ZN\d+raft_cutlass + symbol_exclusions: raft_cutlass conda-python-build: needs: conda-cpp-build secrets: inherit diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f6ca417dcf..ad0456d526 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -23,7 +23,7 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} enable_check_symbols: true - symbol_exclusions: _ZN\d+raft_cutlass + symbol_exclusions: raft_cutlass conda-cpp-tests: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 From 8a71b9831cc2d5701f5dbbe0229a5c677b032e6f Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 9 Sep 2024 12:15:18 -0700 Subject: [PATCH 18/79] Add support for Python 3.12 (#2428) Contributes to https://github.com/rapidsai/build-planning/issues/40 This PR adds support for Python 3.12. ## Notes for Reviewers This is part of ongoing work to add Python 3.12 support across RAPIDS. It temporarily introduces a build/test matrix including Python 3.12, from https://github.com/rapidsai/shared-workflows/pull/213. A follow-up PR will revert back to pointing at the `branch-24.10` branch of `shared-workflows` once all RAPIDS repos have added Python 3.12 support. ### This will fail until all dependencies have been updates to Python 3.12 CI here is expected to fail until all of this project's upstream dependencies support Python 3.12. This can be merged whenever all CI jobs are passing. Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/raft/pull/2428 --- .github/workflows/build.yaml | 16 ++++++++-------- .github/workflows/pr.yaml | 26 +++++++++++++------------- .github/workflows/test.yaml | 10 +++++----- dependencies.yaml | 6 +++++- python/pylibraft/pyproject.toml | 1 + python/raft-ann-bench/pyproject.toml | 1 + python/raft-dask/pyproject.toml | 1 + 7 files changed, 34 insertions(+), 27 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2b0ae5099c..037d7cc05f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -37,7 +37,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -46,7 +46,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -57,7 +57,7 @@ jobs: if: github.ref_type == 'branch' needs: python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-pylibraft: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -79,7 +79,7 @@ jobs: wheel-publish-pylibraft: needs: wheel-build-pylibraft secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -88,7 +88,7 @@ jobs: package-name: pylibraft wheel-build-raft-dask: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -98,7 +98,7 @@ jobs: wheel-publish-raft-dask: needs: wheel-build-raft-dask secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 381ca6b378..26c6928f77 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -25,29 +25,29 @@ jobs: - wheel-tests-raft-dask - devcontainer secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@python-3.12 checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@python-3.12 with: enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@python-3.12 with: build_type: pull-request node_type: cpu16 conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@python-3.12 with: build_type: pull-request conda-cpp-checks: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@python-3.12 with: build_type: pull-request enable_check_symbols: true @@ -55,19 +55,19 @@ jobs: conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@python-3.12 with: build_type: pull-request conda-python-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 with: build_type: pull-request docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -77,34 +77,34 @@ jobs: wheel-build-pylibraft: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: pull-request script: ci/build_wheel_pylibraft.sh wheel-tests-pylibraft: needs: wheel-build-pylibraft secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 with: build_type: pull-request script: ci/test_wheel_pylibraft.sh wheel-build-raft-dask: needs: wheel-tests-pylibraft secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: pull-request script: "ci/build_wheel_raft_dask.sh" wheel-tests-raft-dask: needs: wheel-build-raft-dask secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 with: build_type: pull-request script: ci/test_wheel_raft_dask.sh devcontainer: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@python-3.12 with: arch: '["amd64"]' cuda: '["12.5"]' diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ad0456d526..90518c1132 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@python-3.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -26,7 +26,7 @@ jobs: symbol_exclusions: raft_cutlass conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@python-3.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -34,7 +34,7 @@ jobs: sha: ${{ inputs.sha }} conda-python-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -42,7 +42,7 @@ jobs: sha: ${{ inputs.sha }} wheel-tests-pylibraft: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -51,7 +51,7 @@ jobs: script: ci/test_wheel_pylibraft.sh wheel-tests-raft-dask: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 with: build_type: nightly branch: ${{ inputs.branch }} diff --git a/dependencies.yaml b/dependencies.yaml index e4e361548f..8f5c69245f 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -463,8 +463,12 @@ dependencies: packages: - python=3.11 - matrix: + py: "3.12" packages: - - python>=3.10,<3.12 + - python=3.12 + - matrix: + packages: + - python>=3.10,<3.13 run_pylibraft: common: - output_types: [conda, pyproject] diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index 14f2ba7d2f..a540915585 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -44,6 +44,7 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] [project.optional-dependencies] diff --git a/python/raft-ann-bench/pyproject.toml b/python/raft-ann-bench/pyproject.toml index fa5781893b..0e4fda1f00 100644 --- a/python/raft-ann-bench/pyproject.toml +++ b/python/raft-ann-bench/pyproject.toml @@ -27,6 +27,7 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] [project.urls] diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index 44012b5f10..d1f577120f 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -44,6 +44,7 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] [project.optional-dependencies] From ac53a0fdc35fe36a11a3ac4debd4cc9a4076fe7f Mon Sep 17 00:00:00 2001 From: Micka Date: Tue, 10 Sep 2024 17:20:52 +0200 Subject: [PATCH 19/79] [BUG] Fix bitset function visibility (#2429) `raft::ceildiv` is also being replaced with `raft::div_rounding_up_safe` to avoid including CUDA headers when not needed. Authors: - Micka (https://github.com/lowener) Approvers: - rhdong (https://github.com/rhdong) - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/raft/pull/2429 --- cpp/include/raft/core/bitmap.cuh | 6 +++--- cpp/include/raft/core/bitset.cuh | 24 ++++++------------------ cpp/include/raft/core/bitset.hpp | 11 +++++++++-- 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/cpp/include/raft/core/bitmap.cuh b/cpp/include/raft/core/bitmap.cuh index cafd1977ab..024b1244a6 100644 --- a/cpp/include/raft/core/bitmap.cuh +++ b/cpp/include/raft/core/bitmap.cuh @@ -35,9 +35,9 @@ _RAFT_HOST_DEVICE inline bool bitmap_view::test(const index_t } template -_RAFT_HOST_DEVICE void bitmap_view::set(const index_t row, - const index_t col, - bool new_value) const +_RAFT_DEVICE void bitmap_view::set(const index_t row, + const index_t col, + bool new_value) const { set(row * cols_ + col, new_value); } diff --git a/cpp/include/raft/core/bitset.cuh b/cpp/include/raft/core/bitset.cuh index 0cdb4c1fb6..b6e6128eca 100644 --- a/cpp/include/raft/core/bitset.cuh +++ b/cpp/include/raft/core/bitset.cuh @@ -46,8 +46,8 @@ _RAFT_HOST_DEVICE bool bitset_view::operator[](const index_t } template -_RAFT_HOST_DEVICE void bitset_view::set(const index_t sample_index, - bool set_value) const +_RAFT_DEVICE void bitset_view::set(const index_t sample_index, + bool set_value) const { const index_t bit_element = sample_index / bitset_element_size; const index_t bit_index = sample_index % bitset_element_size; @@ -60,18 +60,12 @@ _RAFT_HOST_DEVICE void bitset_view::set(const index_t sample_ } } -template -_RAFT_HOST_DEVICE inline index_t bitset_view::n_elements() const -{ - return raft::ceildiv(bitset_len_, bitset_element_size); -} - template bitset::bitset(const raft::resources& res, raft::device_vector_view mask_index, index_t bitset_len, bool default_value) - : bitset_{std::size_t(raft::ceildiv(bitset_len, bitset_element_size)), + : bitset_{std::size_t(raft::div_rounding_up_safe(bitset_len, bitset_element_size)), raft::resource::get_cuda_stream(res)}, bitset_len_{bitset_len} { @@ -83,26 +77,20 @@ template bitset::bitset(const raft::resources& res, index_t bitset_len, bool default_value) - : bitset_{std::size_t(raft::ceildiv(bitset_len, bitset_element_size)), + : bitset_{std::size_t(raft::div_rounding_up_safe(bitset_len, bitset_element_size)), raft::resource::get_cuda_stream(res)}, bitset_len_{bitset_len} { reset(res, default_value); } -template -index_t bitset::n_elements() const -{ - return raft::ceildiv(bitset_len_, bitset_element_size); -} - template void bitset::resize(const raft::resources& res, index_t new_bitset_len, bool default_value) { - auto old_size = raft::ceildiv(bitset_len_, bitset_element_size); - auto new_size = raft::ceildiv(new_bitset_len, bitset_element_size); + auto old_size = raft::div_rounding_up_safe(bitset_len_, bitset_element_size); + auto new_size = raft::div_rounding_up_safe(new_bitset_len, bitset_element_size); bitset_.resize(new_size); bitset_len_ = new_bitset_len; if (old_size < new_size) { diff --git a/cpp/include/raft/core/bitset.hpp b/cpp/include/raft/core/bitset.hpp index 0df12f25e6..3608ee43fa 100644 --- a/cpp/include/raft/core/bitset.hpp +++ b/cpp/include/raft/core/bitset.hpp @@ -20,6 +20,7 @@ #include #include #include +#include namespace raft::core { /** @@ -89,7 +90,10 @@ struct bitset_view { /** * @brief Get the number of elements used by the bitset representation. */ - inline _RAFT_HOST_DEVICE auto n_elements() const -> index_t; + inline _RAFT_HOST_DEVICE auto n_elements() const -> index_t + { + return raft::div_rounding_up_safe(bitset_len_, bitset_element_size); + } inline auto to_mdspan() -> raft::device_vector_view { @@ -173,7 +177,10 @@ struct bitset { /** * @brief Get the number of elements used by the bitset representation. */ - inline auto n_elements() const -> index_t; + inline auto n_elements() const -> index_t + { + return raft::div_rounding_up_safe(bitset_len_, bitset_element_size); + } /** @brief Get an mdspan view of the current bitset */ inline auto to_mdspan() -> raft::device_vector_view From 20718412dfa2026b94de2de2e822b73fe26a3bbc Mon Sep 17 00:00:00 2001 From: Ben Frederickson Date: Tue, 17 Sep 2024 11:56:06 -0700 Subject: [PATCH 20/79] Allow coo_sort to work on int64_t indices (#2432) Authors: - Ben Frederickson (https://github.com/benfred) Approvers: - Divye Gala (https://github.com/divyegala) URL: https://github.com/rapidsai/raft/pull/2432 --- cpp/include/raft/sparse/op/detail/sort.h | 10 +++++----- cpp/include/raft/sparse/op/sort.cuh | 14 +++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cpp/include/raft/sparse/op/detail/sort.h b/cpp/include/raft/sparse/op/detail/sort.h index 85ae825035..02287c2367 100644 --- a/cpp/include/raft/sparse/op/detail/sort.h +++ b/cpp/include/raft/sparse/op/detail/sort.h @@ -68,8 +68,8 @@ struct TupleComp { * @param vals vals array from coo matrix * @param stream: cuda stream to use */ -template -void coo_sort(int m, int n, int nnz, int* rows, int* cols, T* vals, cudaStream_t stream) +template +void coo_sort(IdxT m, IdxT n, IdxT nnz, IdxT* rows, IdxT* cols, T* vals, cudaStream_t stream) { auto coo_indices = thrust::make_zip_iterator(thrust::make_tuple(rows, cols)); @@ -83,10 +83,10 @@ void coo_sort(int m, int n, int nnz, int* rows, int* cols, T* vals, cudaStream_t * @param in: COO to sort by row * @param stream: the cuda stream to use */ -template -void coo_sort(COO* const in, cudaStream_t stream) +template +void coo_sort(COO* const in, cudaStream_t stream) { - coo_sort(in->n_rows, in->n_cols, in->nnz, in->rows(), in->cols(), in->vals(), stream); + coo_sort(in->n_rows, in->n_cols, in->nnz, in->rows(), in->cols(), in->vals(), stream); } /** diff --git a/cpp/include/raft/sparse/op/sort.cuh b/cpp/include/raft/sparse/op/sort.cuh index c6c3c2e220..5b8a792429 100644 --- a/cpp/include/raft/sparse/op/sort.cuh +++ b/cpp/include/raft/sparse/op/sort.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,8 +37,8 @@ namespace op { * @param vals vals array from coo matrix * @param stream: cuda stream to use */ -template -void coo_sort(int m, int n, int nnz, int* rows, int* cols, T* vals, cudaStream_t stream) +template +void coo_sort(IdxT m, IdxT n, IdxT nnz, IdxT* rows, IdxT* cols, T* vals, cudaStream_t stream) { detail::coo_sort(m, n, nnz, rows, cols, vals, stream); } @@ -49,10 +49,10 @@ void coo_sort(int m, int n, int nnz, int* rows, int* cols, T* vals, cudaStream_t * @param in: COO to sort by row * @param stream: the cuda stream to use */ -template -void coo_sort(COO* const in, cudaStream_t stream) +template +void coo_sort(COO* const in, cudaStream_t stream) { - coo_sort(in->n_rows, in->n_cols, in->nnz, in->rows(), in->cols(), in->vals(), stream); + coo_sort(in->n_rows, in->n_cols, in->nnz, in->rows(), in->cols(), in->vals(), stream); } /** @@ -75,4 +75,4 @@ void coo_sort_by_weight( }; // end NAMESPACE sparse }; // end NAMESPACE raft -#endif \ No newline at end of file +#endif From e60cd7d063c3ab7830959d5aef465461f2daf99f Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 18 Sep 2024 03:05:44 -0500 Subject: [PATCH 21/79] Update to flake8 7.1.1. (#2435) We need to update flake8 to fix a false-positive that appears with older flake8 versions on Python 3.12. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/raft/pull/2435 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 88be628e55..458d8b1b51 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: # Explicitly specify the pyproject.toml at the repo root, not per-project. args: ["--config", "pyproject.toml"] - repo: https://github.com/PyCQA/flake8 - rev: 5.0.4 + rev: 7.1.1 hooks: - id: flake8 args: ["--config=.flake8"] From c5dcba0333baa4daf41292a8901044442c8d79c8 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 18 Sep 2024 14:43:38 -0500 Subject: [PATCH 22/79] Use CI workflow branch 'branch-24.10' again [skip ci] (#2437) --- .github/workflows/build.yaml | 16 ++++++++-------- .github/workflows/pr.yaml | 26 +++++++++++++------------- .github/workflows/test.yaml | 10 +++++----- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 037d7cc05f..2b0ae5099c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -37,7 +37,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -46,7 +46,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -57,7 +57,7 @@ jobs: if: github.ref_type == 'branch' needs: python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-pylibraft: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -79,7 +79,7 @@ jobs: wheel-publish-pylibraft: needs: wheel-build-pylibraft secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -88,7 +88,7 @@ jobs: package-name: pylibraft wheel-build-raft-dask: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -98,7 +98,7 @@ jobs: wheel-publish-raft-dask: needs: wheel-build-raft-dask secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 26c6928f77..381ca6b378 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -25,29 +25,29 @@ jobs: - wheel-tests-raft-dask - devcontainer secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 with: enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 with: build_type: pull-request node_type: cpu16 conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 with: build_type: pull-request conda-cpp-checks: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 with: build_type: pull-request enable_check_symbols: true @@ -55,19 +55,19 @@ jobs: conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 with: build_type: pull-request conda-python-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 with: build_type: pull-request docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -77,34 +77,34 @@ jobs: wheel-build-pylibraft: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: pull-request script: ci/build_wheel_pylibraft.sh wheel-tests-pylibraft: needs: wheel-build-pylibraft secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: pull-request script: ci/test_wheel_pylibraft.sh wheel-build-raft-dask: needs: wheel-tests-pylibraft secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: pull-request script: "ci/build_wheel_raft_dask.sh" wheel-tests-raft-dask: needs: wheel-build-raft-dask secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: pull-request script: ci/test_wheel_raft_dask.sh devcontainer: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10 with: arch: '["amd64"]' cuda: '["12.5"]' diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 90518c1132..ad0456d526 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -26,7 +26,7 @@ jobs: symbol_exclusions: raft_cutlass conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -34,7 +34,7 @@ jobs: sha: ${{ inputs.sha }} conda-python-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -42,7 +42,7 @@ jobs: sha: ${{ inputs.sha }} wheel-tests-pylibraft: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -51,7 +51,7 @@ jobs: script: ci/test_wheel_pylibraft.sh wheel-tests-raft-dask: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} From 12537c57aadc7436962fbf3109444cb7d50b4103 Mon Sep 17 00:00:00 2001 From: Ray Douglass Date: Thu, 19 Sep 2024 12:04:39 -0400 Subject: [PATCH 23/79] DOC v24.12 Updates [skip ci] --- .../cuda11.8-conda/devcontainer.json | 6 ++-- .devcontainer/cuda11.8-pip/devcontainer.json | 8 ++--- .../cuda12.5-conda/devcontainer.json | 6 ++-- .devcontainer/cuda12.5-pip/devcontainer.json | 8 ++--- .github/workflows/build.yaml | 16 +++++----- .github/workflows/pr.yaml | 26 ++++++++-------- .github/workflows/test.yaml | 10 +++---- README.md | 2 +- VERSION | 2 +- .../all_cuda-118_arch-aarch64.yaml | 14 ++++----- .../all_cuda-118_arch-x86_64.yaml | 14 ++++----- .../all_cuda-125_arch-aarch64.yaml | 14 ++++----- .../all_cuda-125_arch-x86_64.yaml | 14 ++++----- .../bench_ann_cuda-118_arch-aarch64.yaml | 4 +-- .../bench_ann_cuda-118_arch-x86_64.yaml | 4 +-- .../bench_ann_cuda-120_arch-aarch64.yaml | 4 +-- .../bench_ann_cuda-120_arch-x86_64.yaml | 4 +-- .../recipes/raft-dask/conda_build_config.yaml | 4 +-- .../cmake/thirdparty/fetch_rapids.cmake | 2 +- dependencies.yaml | 30 +++++++++---------- docs/source/build.md | 2 +- docs/source/developer_guide.md | 6 ++-- docs/source/raft_ann_benchmarks.md | 12 ++++---- python/pylibraft/pyproject.toml | 4 +-- .../raft-dask/cmake/thirdparty/get_ucxx.cmake | 4 +-- python/raft-dask/pyproject.toml | 10 +++---- 26 files changed, 115 insertions(+), 115 deletions(-) diff --git a/.devcontainer/cuda11.8-conda/devcontainer.json b/.devcontainer/cuda11.8-conda/devcontainer.json index a2b12764c5..008bf8730a 100644 --- a/.devcontainer/cuda11.8-conda/devcontainer.json +++ b/.devcontainer/cuda11.8-conda/devcontainer.json @@ -5,17 +5,17 @@ "args": { "CUDA": "11.8", "PYTHON_PACKAGE_MANAGER": "conda", - "BASE": "rapidsai/devcontainers:24.10-cpp-cuda11.8-mambaforge-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.12-cpp-cuda11.8-mambaforge-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda11.8-conda" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.12-cuda11.8-conda" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.10": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.12": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json index f319536b18..75aed80f9f 100644 --- a/.devcontainer/cuda11.8-pip/devcontainer.json +++ b/.devcontainer/cuda11.8-pip/devcontainer.json @@ -5,24 +5,24 @@ "args": { "CUDA": "11.8", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:24.10-cpp-cuda11.8-ucx1.17.0-openmpi-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.12-cpp-cuda11.8-ucx1.17.0-openmpi-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda11.8-pip" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.12-cuda11.8-pip" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/cuda:24.10": { + "ghcr.io/rapidsai/devcontainers/features/cuda:24.12": { "version": "11.8", "installcuBLAS": true, "installcuSOLVER": true, "installcuRAND": true, "installcuSPARSE": true }, - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.10": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.12": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/ucx", diff --git a/.devcontainer/cuda12.5-conda/devcontainer.json b/.devcontainer/cuda12.5-conda/devcontainer.json index adc79408a3..240ba02131 100644 --- a/.devcontainer/cuda12.5-conda/devcontainer.json +++ b/.devcontainer/cuda12.5-conda/devcontainer.json @@ -5,17 +5,17 @@ "args": { "CUDA": "12.5", "PYTHON_PACKAGE_MANAGER": "conda", - "BASE": "rapidsai/devcontainers:24.10-cpp-mambaforge-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.12-cpp-mambaforge-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda12.5-conda" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.12-cuda12.5-conda" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.10": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.12": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" diff --git a/.devcontainer/cuda12.5-pip/devcontainer.json b/.devcontainer/cuda12.5-pip/devcontainer.json index 26b9a56e48..c23c79017a 100644 --- a/.devcontainer/cuda12.5-pip/devcontainer.json +++ b/.devcontainer/cuda12.5-pip/devcontainer.json @@ -5,24 +5,24 @@ "args": { "CUDA": "12.5", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:24.10-cpp-cuda12.5-ucx1.17.0-openmpi-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.12-cpp-cuda12.5-ucx1.17.0-openmpi-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda12.5-pip" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.12-cuda12.5-pip" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/cuda:24.10": { + "ghcr.io/rapidsai/devcontainers/features/cuda:24.12": { "version": "12.5", "installcuBLAS": true, "installcuSOLVER": true, "installcuRAND": true, "installcuSPARSE": true }, - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.10": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.12": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/ucx", diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2b0ae5099c..db379c9d47 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -37,7 +37,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -46,7 +46,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -57,7 +57,7 @@ jobs: if: github.ref_type == 'branch' needs: python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-pylibraft: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -79,7 +79,7 @@ jobs: wheel-publish-pylibraft: needs: wheel-build-pylibraft secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -88,7 +88,7 @@ jobs: package-name: pylibraft wheel-build-raft-dask: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -98,7 +98,7 @@ jobs: wheel-publish-raft-dask: needs: wheel-build-raft-dask secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 381ca6b378..844f9f9441 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -25,29 +25,29 @@ jobs: - wheel-tests-raft-dask - devcontainer secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.12 checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.12 with: enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.12 with: build_type: pull-request node_type: cpu16 conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.12 with: build_type: pull-request conda-cpp-checks: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 with: build_type: pull-request enable_check_symbols: true @@ -55,19 +55,19 @@ jobs: conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12 with: build_type: pull-request conda-python-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 with: build_type: pull-request docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -77,34 +77,34 @@ jobs: wheel-build-pylibraft: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: ci/build_wheel_pylibraft.sh wheel-tests-pylibraft: needs: wheel-build-pylibraft secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 with: build_type: pull-request script: ci/test_wheel_pylibraft.sh wheel-build-raft-dask: needs: wheel-tests-pylibraft secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: "ci/build_wheel_raft_dask.sh" wheel-tests-raft-dask: needs: wheel-build-raft-dask secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 with: build_type: pull-request script: ci/test_wheel_raft_dask.sh devcontainer: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.12 with: arch: '["amd64"]' cuda: '["12.5"]' diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ad0456d526..2bee8a3d1d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -26,7 +26,7 @@ jobs: symbol_exclusions: raft_cutlass conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -34,7 +34,7 @@ jobs: sha: ${{ inputs.sha }} conda-python-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -42,7 +42,7 @@ jobs: sha: ${{ inputs.sha }} wheel-tests-pylibraft: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -51,7 +51,7 @@ jobs: script: ci/test_wheel_pylibraft.sh wheel-tests-raft-dask: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 with: build_type: nightly branch: ${{ inputs.branch }} diff --git a/README.md b/README.md index 25ce059630..2db45cf749 100755 --- a/README.md +++ b/README.md @@ -293,7 +293,7 @@ You can also install the conda packages individually using the `mamba` command a mamba install -c rapidsai -c conda-forge -c nvidia libraft libraft-headers cuda-version=12.5 ``` -If installing the C++ APIs please see [using libraft](https://docs.rapids.ai/api/raft/nightly/using_libraft/) for more information on using the pre-compiled shared library. You can also refer to the [example C++ template project](https://github.com/rapidsai/raft/tree/branch-24.10/cpp/template) for a ready-to-go CMake configuration that you can drop into your project and build against installed RAFT development artifacts above. +If installing the C++ APIs please see [using libraft](https://docs.rapids.ai/api/raft/nightly/using_libraft/) for more information on using the pre-compiled shared library. You can also refer to the [example C++ template project](https://github.com/rapidsai/raft/tree/branch-24.12/cpp/template) for a ready-to-go CMake configuration that you can drop into your project and build against installed RAFT development artifacts above. ### Installing Python through Pip diff --git a/VERSION b/VERSION index 7c7ba04436..af28c42b52 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -24.10.00 +24.12.00 diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index 462874a7e7..12ce2afcb6 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -20,8 +20,8 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-cuda==24.10.*,>=0.0.0a0 -- distributed-ucxx==0.40.*,>=0.0.0a0 +- dask-cuda==24.12.*,>=0.0.0a0 +- distributed-ucxx==0.41.*,>=0.0.0a0 - doxygen>=1.8.20 - gcc_linux-aarch64=11.* - graphviz @@ -35,7 +35,7 @@ dependencies: - libcusolver=11.4.1.48 - libcusparse-dev=11.7.5.86 - libcusparse=11.7.5.86 -- libucxx==0.40.*,>=0.0.0a0 +- libucxx==0.41.*,>=0.0.0a0 - nccl>=2.9.9 - ninja - numba>=0.57 @@ -44,18 +44,18 @@ dependencies: - nvcc_linux-aarch64=11.8 - pre-commit - pydata-sphinx-theme -- pylibraft==24.10.*,>=0.0.0a0 +- pylibraft==24.12.*,>=0.0.0a0 - pytest-cov - pytest==7.* - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rapids-dask-dependency==24.10.*,>=0.0.0a0 +- rapids-dask-dependency==24.12.*,>=0.0.0a0 - recommonmark -- rmm==24.10.*,>=0.0.0a0 +- rmm==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - scikit-learn - scipy - sphinx-copybutton - sphinx-markdown-tables - sysroot_linux-aarch64==2.17 -- ucx-py==0.40.*,>=0.0.0a0 +- ucx-py==0.41.*,>=0.0.0a0 name: all_cuda-118_arch-aarch64 diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index cfd974a6a8..0f178a648d 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -20,8 +20,8 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-cuda==24.10.*,>=0.0.0a0 -- distributed-ucxx==0.40.*,>=0.0.0a0 +- dask-cuda==24.12.*,>=0.0.0a0 +- distributed-ucxx==0.41.*,>=0.0.0a0 - doxygen>=1.8.20 - gcc_linux-64=11.* - graphviz @@ -35,7 +35,7 @@ dependencies: - libcusolver=11.4.1.48 - libcusparse-dev=11.7.5.86 - libcusparse=11.7.5.86 -- libucxx==0.40.*,>=0.0.0a0 +- libucxx==0.41.*,>=0.0.0a0 - nccl>=2.9.9 - ninja - numba>=0.57 @@ -44,18 +44,18 @@ dependencies: - nvcc_linux-64=11.8 - pre-commit - pydata-sphinx-theme -- pylibraft==24.10.*,>=0.0.0a0 +- pylibraft==24.12.*,>=0.0.0a0 - pytest-cov - pytest==7.* - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rapids-dask-dependency==24.10.*,>=0.0.0a0 +- rapids-dask-dependency==24.12.*,>=0.0.0a0 - recommonmark -- rmm==24.10.*,>=0.0.0a0 +- rmm==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - scikit-learn - scipy - sphinx-copybutton - sphinx-markdown-tables - sysroot_linux-64==2.17 -- ucx-py==0.40.*,>=0.0.0a0 +- ucx-py==0.41.*,>=0.0.0a0 name: all_cuda-118_arch-x86_64 diff --git a/conda/environments/all_cuda-125_arch-aarch64.yaml b/conda/environments/all_cuda-125_arch-aarch64.yaml index 82e391e9ae..8e985df433 100644 --- a/conda/environments/all_cuda-125_arch-aarch64.yaml +++ b/conda/environments/all_cuda-125_arch-aarch64.yaml @@ -21,8 +21,8 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-cuda==24.10.*,>=0.0.0a0 -- distributed-ucxx==0.40.*,>=0.0.0a0 +- dask-cuda==24.12.*,>=0.0.0a0 +- distributed-ucxx==0.41.*,>=0.0.0a0 - doxygen>=1.8.20 - gcc_linux-aarch64=11.* - graphviz @@ -32,7 +32,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libucxx==0.40.*,>=0.0.0a0 +- libucxx==0.41.*,>=0.0.0a0 - nccl>=2.9.9 - ninja - numba>=0.57 @@ -40,18 +40,18 @@ dependencies: - numpydoc - pre-commit - pydata-sphinx-theme -- pylibraft==24.10.*,>=0.0.0a0 +- pylibraft==24.12.*,>=0.0.0a0 - pytest-cov - pytest==7.* - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rapids-dask-dependency==24.10.*,>=0.0.0a0 +- rapids-dask-dependency==24.12.*,>=0.0.0a0 - recommonmark -- rmm==24.10.*,>=0.0.0a0 +- rmm==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - scikit-learn - scipy - sphinx-copybutton - sphinx-markdown-tables - sysroot_linux-aarch64==2.17 -- ucx-py==0.40.*,>=0.0.0a0 +- ucx-py==0.41.*,>=0.0.0a0 name: all_cuda-125_arch-aarch64 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 0389427d13..9bf6d83f1f 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -21,8 +21,8 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-cuda==24.10.*,>=0.0.0a0 -- distributed-ucxx==0.40.*,>=0.0.0a0 +- dask-cuda==24.12.*,>=0.0.0a0 +- distributed-ucxx==0.41.*,>=0.0.0a0 - doxygen>=1.8.20 - gcc_linux-64=11.* - graphviz @@ -32,7 +32,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libucxx==0.40.*,>=0.0.0a0 +- libucxx==0.41.*,>=0.0.0a0 - nccl>=2.9.9 - ninja - numba>=0.57 @@ -40,18 +40,18 @@ dependencies: - numpydoc - pre-commit - pydata-sphinx-theme -- pylibraft==24.10.*,>=0.0.0a0 +- pylibraft==24.12.*,>=0.0.0a0 - pytest-cov - pytest==7.* - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rapids-dask-dependency==24.10.*,>=0.0.0a0 +- rapids-dask-dependency==24.12.*,>=0.0.0a0 - recommonmark -- rmm==24.10.*,>=0.0.0a0 +- rmm==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - scikit-learn - scipy - sphinx-copybutton - sphinx-markdown-tables - sysroot_linux-64==2.17 -- ucx-py==0.40.*,>=0.0.0a0 +- ucx-py==0.41.*,>=0.0.0a0 name: all_cuda-125_arch-x86_64 diff --git a/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml index eff1c56840..b182d5ae1a 100644 --- a/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml +++ b/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml @@ -30,7 +30,7 @@ dependencies: - libcusolver=11.4.1.48 - libcusparse-dev=11.7.5.86 - libcusparse=11.7.5.86 -- libucxx==0.40.*,>=0.0.0a0 +- libucxx==0.41.*,>=0.0.0a0 - matplotlib - nccl>=2.9.9 - ninja @@ -40,7 +40,7 @@ dependencies: - pandas - pyyaml - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rmm==24.10.*,>=0.0.0a0 +- rmm==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - sysroot_linux-aarch64==2.17 name: bench_ann_cuda-118_arch-aarch64 diff --git a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml index 87b19d2952..92f61150b7 100644 --- a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml +++ b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml @@ -30,7 +30,7 @@ dependencies: - libcusolver=11.4.1.48 - libcusparse-dev=11.7.5.86 - libcusparse=11.7.5.86 -- libucxx==0.40.*,>=0.0.0a0 +- libucxx==0.41.*,>=0.0.0a0 - matplotlib - nccl>=2.9.9 - ninja @@ -40,7 +40,7 @@ dependencies: - pandas - pyyaml - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rmm==24.10.*,>=0.0.0a0 +- rmm==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - sysroot_linux-64==2.17 name: bench_ann_cuda-118_arch-x86_64 diff --git a/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml index ff3451c15c..207680b788 100644 --- a/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml +++ b/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml @@ -27,7 +27,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libucxx==0.40.*,>=0.0.0a0 +- libucxx==0.41.*,>=0.0.0a0 - matplotlib - nccl>=2.9.9 - ninja @@ -36,7 +36,7 @@ dependencies: - pandas - pyyaml - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rmm==24.10.*,>=0.0.0a0 +- rmm==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - sysroot_linux-aarch64==2.17 name: bench_ann_cuda-120_arch-aarch64 diff --git a/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml index 085e099ae8..510d82eb60 100644 --- a/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml +++ b/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml @@ -27,7 +27,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libucxx==0.40.*,>=0.0.0a0 +- libucxx==0.41.*,>=0.0.0a0 - matplotlib - nccl>=2.9.9 - ninja @@ -36,7 +36,7 @@ dependencies: - pandas - pyyaml - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rmm==24.10.*,>=0.0.0a0 +- rmm==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - sysroot_linux-64==2.17 name: bench_ann_cuda-120_arch-x86_64 diff --git a/conda/recipes/raft-dask/conda_build_config.yaml b/conda/recipes/raft-dask/conda_build_config.yaml index e6afed2890..ffff76e378 100644 --- a/conda/recipes/raft-dask/conda_build_config.yaml +++ b/conda/recipes/raft-dask/conda_build_config.yaml @@ -17,10 +17,10 @@ c_stdlib_version: - "2.17" ucx_py_version: - - "0.40.*" + - "0.41.*" ucxx_version: - - "0.40.*" + - "0.41.*" cmake_version: - ">=3.26.4,!=3.30.0" diff --git a/cpp/template/cmake/thirdparty/fetch_rapids.cmake b/cpp/template/cmake/thirdparty/fetch_rapids.cmake index f64a924cf5..6f4c627ed4 100644 --- a/cpp/template/cmake/thirdparty/fetch_rapids.cmake +++ b/cpp/template/cmake/thirdparty/fetch_rapids.cmake @@ -12,7 +12,7 @@ # the License. # Use this variable to update RAPIDS and RAFT versions -set(RAPIDS_VERSION "24.10") +set(RAPIDS_VERSION "24.12") if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/RAFT_RAPIDS.cmake) file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-${RAPIDS_VERSION}/RAPIDS.cmake diff --git a/dependencies.yaml b/dependencies.yaml index 8f5c69245f..857532237e 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -171,7 +171,7 @@ dependencies: - c-compiler - cxx-compiler - nccl>=2.9.9 - - libucxx==0.40.*,>=0.0.0a0 + - libucxx==0.41.*,>=0.0.0a0 specific: - output_types: conda matrices: @@ -210,7 +210,7 @@ dependencies: common: - output_types: [conda] packages: - - &rmm_unsuffixed rmm==24.10.*,>=0.0.0a0 + - &rmm_unsuffixed rmm==24.12.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -237,12 +237,12 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - &rmm_cu12 rmm-cu12==24.10.*,>=0.0.0a0 + - &rmm_cu12 rmm-cu12==24.12.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - &rmm_cu11 rmm-cu11==24.10.*,>=0.0.0a0 + - &rmm_cu11 rmm-cu11==24.12.*,>=0.0.0a0 - {matrix: null, packages: [*rmm_unsuffixed] } checks: common: @@ -510,14 +510,14 @@ dependencies: common: - output_types: [conda, pyproject] packages: - - dask-cuda==24.10.*,>=0.0.0a0 + - dask-cuda==24.12.*,>=0.0.0a0 - joblib>=0.11 - numba>=0.57 - - rapids-dask-dependency==24.10.*,>=0.0.0a0 + - rapids-dask-dependency==24.12.*,>=0.0.0a0 - output_types: conda packages: - - &pylibraft_unsuffixed pylibraft==24.10.*,>=0.0.0a0 - - &ucx_py_unsuffixed ucx-py==0.40.*,>=0.0.0a0 + - &pylibraft_unsuffixed pylibraft==24.12.*,>=0.0.0a0 + - &ucx_py_unsuffixed ucx-py==0.41.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -531,14 +531,14 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - &pylibraft_cu12 pylibraft-cu12==24.10.*,>=0.0.0a0 - - &ucx_py_cu12 ucx-py-cu12==0.40.*,>=0.0.0a0 + - &pylibraft_cu12 pylibraft-cu12==24.12.*,>=0.0.0a0 + - &ucx_py_cu12 ucx-py-cu12==0.41.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - &pylibraft_cu11 pylibraft-cu11==24.10.*,>=0.0.0a0 - - &ucx_py_cu11 ucx-py-cu11==0.40.*,>=0.0.0a0 + - &pylibraft_cu11 pylibraft-cu11==24.12.*,>=0.0.0a0 + - &ucx_py_cu11 ucx-py-cu11==0.41.*,>=0.0.0a0 - {matrix: null, packages: [*pylibraft_unsuffixed, *ucx_py_unsuffixed]} test_python_common: common: @@ -558,7 +558,7 @@ dependencies: packages: # UCXX is not currently a hard-dependency thus only installed during tests, # this will change in the future. - - &distributed_ucxx_unsuffixed distributed-ucxx==0.40.*,>=0.0.0a0 + - &distributed_ucxx_unsuffixed distributed-ucxx==0.41.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -571,12 +571,12 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - distributed-ucxx-cu12==0.40.*,>=0.0.0a0 + - distributed-ucxx-cu12==0.41.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - distributed-ucxx-cu11==0.40.*,>=0.0.0a0 + - distributed-ucxx-cu11==0.41.*,>=0.0.0a0 - {matrix: null, packages: [*distributed_ucxx_unsuffixed]} depends_on_ucx_build: common: diff --git a/docs/source/build.md b/docs/source/build.md index 4ba087e68d..b9a1832b02 100644 --- a/docs/source/build.md +++ b/docs/source/build.md @@ -56,7 +56,7 @@ You can also install the conda packages individually using the `mamba` command a mamba install -c rapidsai -c conda-forge -c nvidia libraft libraft-headers cuda-version=12.0 ``` -If installing the C++ APIs Please see [using libraft](https://docs.rapids.ai/api/raft/nightly/using_libraft/) for more information on using the pre-compiled shared library. You can also refer to the [example C++ template project](https://github.com/rapidsai/raft/tree/branch-24.10/cpp/template) for a ready-to-go CMake configuration that you can drop into your project and build against installed RAFT development artifacts above. +If installing the C++ APIs Please see [using libraft](https://docs.rapids.ai/api/raft/nightly/using_libraft/) for more information on using the pre-compiled shared library. You can also refer to the [example C++ template project](https://github.com/rapidsai/raft/tree/branch-24.12/cpp/template) for a ready-to-go CMake configuration that you can drop into your project and build against installed RAFT development artifacts above. ## Installing Python through Pip diff --git a/docs/source/developer_guide.md b/docs/source/developer_guide.md index 516819b1c1..c4a099fabb 100644 --- a/docs/source/developer_guide.md +++ b/docs/source/developer_guide.md @@ -187,7 +187,7 @@ RAFT relies on `clang-format` to enforce code style across all C++ and CUDA sour 1. Do not split empty functions/records/namespaces. 2. Two-space indentation everywhere, including the line continuations. 3. Disable reflowing of comments. - The reasons behind these deviations from the Google style guide are given in comments [here](https://github.com/rapidsai/raft/blob/branch-24.10/cpp/.clang-format). + The reasons behind these deviations from the Google style guide are given in comments [here](https://github.com/rapidsai/raft/blob/branch-24.12/cpp/.clang-format). [`doxygen`](https://doxygen.nl/) is used as documentation generator and also as a documentation linter. In order to run doxygen as a linter on C++/CUDA code, run @@ -205,7 +205,7 @@ you can run `codespell -i 3 -w .` from the repository root directory. This will bring up an interactive prompt to select which spelling fixes to apply. ### #include style -[include_checker.py](https://github.com/rapidsai/raft/blob/branch-24.10/cpp/scripts/include_checker.py) is used to enforce the include style as follows: +[include_checker.py](https://github.com/rapidsai/raft/blob/branch-24.12/cpp/scripts/include_checker.py) is used to enforce the include style as follows: 1. `#include "..."` should be used for referencing local files only. It is acceptable to be used for referencing files in a sub-folder/parent-folder of the same algorithm, but should never be used to include files in other algorithms or between algorithms and the primitives or other dependencies. 2. `#include <...>` should be used for referencing everything else @@ -230,7 +230,7 @@ Call CUDA APIs via the provided helper macros `RAFT_CUDA_TRY`, `RAFT_CUBLAS_TRY` ## Logging ### Introduction -Anything and everything about logging is defined inside [logger.hpp](https://github.com/rapidsai/raft/blob/branch-24.10/cpp/include/raft/core/logger.hpp). It uses [spdlog](https://github.com/gabime/spdlog) underneath, but this information is transparent to all. +Anything and everything about logging is defined inside [logger.hpp](https://github.com/rapidsai/raft/blob/branch-24.12/cpp/include/raft/core/logger.hpp). It uses [spdlog](https://github.com/gabime/spdlog) underneath, but this information is transparent to all. ### Usage ```cpp diff --git a/docs/source/raft_ann_benchmarks.md b/docs/source/raft_ann_benchmarks.md index fc11a56ac8..faea9189c6 100644 --- a/docs/source/raft_ann_benchmarks.md +++ b/docs/source/raft_ann_benchmarks.md @@ -62,7 +62,7 @@ Nightly images are located in [dockerhub](https://hub.docker.com/r/rapidsai/raft - The following command pulls the nightly container for python version 10, cuda version 12, and RAFT version 23.10: ```bash -docker pull rapidsai/raft-ann-bench:24.10a-cuda12.0-py3.10 #substitute raft-ann-bench for the exact desired container. +docker pull rapidsai/raft-ann-bench:24.12a-cuda12.0-py3.10 #substitute raft-ann-bench for the exact desired container. ``` The CUDA and python versions can be changed for the supported values: @@ -83,7 +83,7 @@ You can see the exact versions as well in the dockerhub site: [//]: # () [//]: # (```bash) -[//]: # (docker pull nvcr.io/nvidia/rapidsai/raft-ann-bench:24.10-cuda11.8-py3.10 #substitute raft-ann-bench for the exact desired container.) +[//]: # (docker pull nvcr.io/nvidia/rapidsai/raft-ann-bench:24.12-cuda11.8-py3.10 #substitute raft-ann-bench for the exact desired container.) [//]: # (```) @@ -344,7 +344,7 @@ For GPU-enabled systems, the `DATA_FOLDER` variable should be a local folder whe export DATA_FOLDER=path/to/store/datasets/and/results docker run --gpus all --rm -it -u $(id -u) \ -v $DATA_FOLDER:/data/benchmarks \ - rapidsai/raft-ann-bench:24.10a-cuda11.8-py3.10 \ + rapidsai/raft-ann-bench:24.12a-cuda11.8-py3.10 \ "--dataset deep-image-96-angular" \ "--normalize" \ "--algorithms raft_cagra,raft_ivf_pq --batch-size 10 -k 10" \ @@ -355,7 +355,7 @@ Usage of the above command is as follows: | Argument | Description | |-----------------------------------------------------------|----------------------------------------------------------------------------------------------------| -| `rapidsai/raft-ann-bench:24.10a-cuda11.8-py3.10` | Image to use. Can be either `raft-ann-bench` or `raft-ann-bench-datasets` | +| `rapidsai/raft-ann-bench:24.12a-cuda11.8-py3.10` | Image to use. Can be either `raft-ann-bench` or `raft-ann-bench-datasets` | | `"--dataset deep-image-96-angular"` | Dataset name | | `"--normalize"` | Whether to normalize the dataset | | `"--algorithms raft_cagra,hnswlib --batch-size 10 -k 10"` | Arguments passed to the `run` script, such as the algorithms to benchmark, the batch size, and `k` | @@ -372,7 +372,7 @@ The container arguments in the above section also be used for the CPU-only conta export DATA_FOLDER=path/to/store/datasets/and/results docker run --rm -it -u $(id -u) \ -v $DATA_FOLDER:/data/benchmarks \ - rapidsai/raft-ann-bench-cpu:24.10a-py3.10 \ + rapidsai/raft-ann-bench-cpu:24.12a-py3.10 \ "--dataset deep-image-96-angular" \ "--normalize" \ "--algorithms hnswlib --batch-size 10 -k 10" \ @@ -389,7 +389,7 @@ docker run --gpus all --rm -it -u $(id -u) \ --entrypoint /bin/bash \ --workdir /data/benchmarks \ -v $DATA_FOLDER:/data/benchmarks \ - rapidsai/raft-ann-bench:24.10a-cuda11.8-py3.10 + rapidsai/raft-ann-bench:24.12a-cuda11.8-py3.10 ``` This will drop you into a command line in the container, with the `raft-ann-bench` python package ready to use, as described in the [Running the benchmarks](#running-the-benchmarks) section above: diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index a540915585..f0f3849c6d 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "nvidia-curand", "nvidia-cusolver", "nvidia-cusparse", - "rmm==24.10.*,>=0.0.0a0", + "rmm==24.12.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", @@ -125,7 +125,7 @@ requires = [ "cuda-python", "cython>=3.0.0", "ninja", - "rmm==24.10.*,>=0.0.0a0", + "rmm==24.12.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. dependencies-file = "../../dependencies.yaml" matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" diff --git a/python/raft-dask/cmake/thirdparty/get_ucxx.cmake b/python/raft-dask/cmake/thirdparty/get_ucxx.cmake index db6039393a..db9b5c6b4d 100644 --- a/python/raft-dask/cmake/thirdparty/get_ucxx.cmake +++ b/python/raft-dask/cmake/thirdparty/get_ucxx.cmake @@ -47,9 +47,9 @@ endfunction() # Change pinned tag here to test a commit in CI # To use a different RAFT locally, set the CMake variable # CPM_raft_SOURCE=/path/to/local/raft -find_and_configure_ucxx(VERSION 0.40 +find_and_configure_ucxx(VERSION 0.41 FORK rapidsai - PINNED_TAG branch-0.40 + PINNED_TAG branch-0.41 EXCLUDE_FROM_ALL YES UCXX_STATIC ${RAFT_DASK_UCXX_STATIC} ) diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index d1f577120f..d71f89085b 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -31,13 +31,13 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.10" dependencies = [ - "dask-cuda==24.10.*,>=0.0.0a0", - "distributed-ucxx==0.40.*,>=0.0.0a0", + "dask-cuda==24.12.*,>=0.0.0a0", + "distributed-ucxx==0.41.*,>=0.0.0a0", "joblib>=0.11", "numba>=0.57", - "pylibraft==24.10.*,>=0.0.0a0", - "rapids-dask-dependency==24.10.*,>=0.0.0a0", - "ucx-py==0.40.*,>=0.0.0a0", + "pylibraft==24.12.*,>=0.0.0a0", + "rapids-dask-dependency==24.12.*,>=0.0.0a0", + "ucx-py==0.41.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", From cd02a8f2b8dc22ec2115e80e559a4ab98d402c6e Mon Sep 17 00:00:00 2001 From: Ray Douglass <3107146+raydouglass@users.noreply.github.com> Date: Fri, 20 Sep 2024 13:05:44 -0400 Subject: [PATCH 24/79] Fix sed syntax [skip-ci] (#2441) --- ci/release/update-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 075eb896f2..5bb98511cf 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -58,7 +58,7 @@ for FILE in dependencies.yaml conda/environments/*.yaml; do sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0/g" "${FILE}" done for DEP in "${UCXX_DEPENDENCIES[@]}"; do - sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_UCXX_SHORT_TAG_PEP440}.*/,>=0.0.0a0/g" "${FILE}" + sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_UCXX_SHORT_TAG_PEP440}.*,>=0.0.0a0/g" "${FILE}" done done for FILE in python/*/pyproject.toml; do From f49567e1fb9f722f8864bc84880745b9b776eb61 Mon Sep 17 00:00:00 2001 From: Micka Date: Sat, 21 Sep 2024 01:09:00 +0200 Subject: [PATCH 25/79] Use runtime check of cudart version for eig (#2430) Authors: - Micka (https://github.com/lowener) Approvers: - Bradley Dice (https://github.com/bdice) - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/raft/pull/2430 --- cpp/include/raft/linalg/detail/eig.cuh | 31 ++++++++++++++------------ 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/cpp/include/raft/linalg/detail/eig.cuh b/cpp/include/raft/linalg/detail/eig.cuh index ba7ed3dcdf..561187178c 100644 --- a/cpp/include/raft/linalg/detail/eig.cuh +++ b/cpp/include/raft/linalg/detail/eig.cuh @@ -95,16 +95,19 @@ void eigDC(raft::resources const& handle, return; #endif -#if CUDART_VERSION <= 12040 - // Use a new stream instead of `cudaStreamPerThread` to avoid cusolver bug # 4580093. + int cudart_version = 0; + RAFT_CUDA_TRY(cudaRuntimeGetVersion(&cudart_version)); + cudaStream_t stream_new; + cudaEvent_t sync_event = resource::detail::get_cuda_stream_sync_event(handle); rmm::cuda_stream stream_new_wrapper; - cudaStream_t stream_new = stream_new_wrapper.value(); - cudaEvent_t sync_event = resource::detail::get_cuda_stream_sync_event(handle); - RAFT_CUDA_TRY(cudaEventRecord(sync_event, stream)); - RAFT_CUDA_TRY(cudaStreamWaitEvent(stream_new, sync_event)); -#else - cudaStream_t stream_new = stream; -#endif + if (cudart_version < 12050) { + // Use a new stream instead of `cudaStreamPerThread` to avoid cusolver bug # 4580093. + stream_new = stream_new_wrapper.value(); + RAFT_CUDA_TRY(cudaEventRecord(sync_event, stream)); + RAFT_CUDA_TRY(cudaStreamWaitEvent(stream_new, sync_event)); + } else { + stream_new = stream; + } cusolverDnHandle_t cusolverH = resource::get_cusolver_dn_handle(handle); cusolverDnParams_t dn_params = nullptr; @@ -152,11 +155,11 @@ void eigDC(raft::resources const& handle, "eig.cuh: eigensolver couldn't converge to a solution. " "This usually occurs when some of the features do not vary enough."); -#if CUDART_VERSION <= 12040 - // Synchronize the created stream with the original stream before return - RAFT_CUDA_TRY(cudaEventRecord(sync_event, stream_new)); - RAFT_CUDA_TRY(cudaStreamWaitEvent(stream, sync_event)); -#endif + if (cudart_version < 12050) { + // Synchronize the created stream with the original stream before return + RAFT_CUDA_TRY(cudaEventRecord(sync_event, stream_new)); + RAFT_CUDA_TRY(cudaStreamWaitEvent(stream, sync_event)); + } } enum EigVecMemUsage { OVERWRITE_INPUT, COPY_INPUT }; From b79f15d2f229849bc02425b2e4ffd7bd3db89d4c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 23 Sep 2024 16:09:01 -0500 Subject: [PATCH 26/79] Update fmt (to 11.0.2) and spdlog (to 1.14.1). (#2433) * Update fmt (to 11.0.2) and spdlog (to 1.14.1). * use rmm and ucxx CI artifacts * try using librmm wheels * try again to use rmm wheels * rapids-get-pr-wheel-artifact was missing RAPIDS_PY_WHEEL_NAME * ok you do not need to provide the Python slug yourself for rapids-get-pr-wheel-artifact * constraints need 'file://' protocol * try suppressing unreachable-code diagnostics from nvcc (this should be narrowed down / upstreamed before merging) * fix checks * Revert "try suppressing unreachable-code diagnostics from nvcc (this should be narrowed down / upstreamed before merging)" This reverts commit 3ba2201d678cc8befdfcaa0e89630a95a00a78b2. * copyright * move rapids-cmake overrides [skip ci] * kick off a build * fix dependency graph * devcontainer * run all CI * remove testing-only changes [skip ci] --- conda/recipes/libraft/conda_build_config.yaml | 4 ++-- .../raft-ann-bench-cpu/conda_build_config.yaml | 4 ++-- cpp/cmake/thirdparty/get_spdlog.cmake | 11 +---------- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index 00b133c821..5c0047fb9c 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -73,7 +73,7 @@ cuda11_cuda_profiler_api_run_version: - ">=11.4.240,<12" spdlog_version: - - ">=1.12.0,<1.13" + - ">=1.14.1,<1.15" fmt_version: - - ">=10.1.1,<11" + - ">=11.0.2,<12" diff --git a/conda/recipes/raft-ann-bench-cpu/conda_build_config.yaml b/conda/recipes/raft-ann-bench-cpu/conda_build_config.yaml index 70d1f0490e..ed6f708e14 100644 --- a/conda/recipes/raft-ann-bench-cpu/conda_build_config.yaml +++ b/conda/recipes/raft-ann-bench-cpu/conda_build_config.yaml @@ -23,7 +23,7 @@ nlohmann_json_version: - ">=3.11.2" spdlog_version: - - ">=1.12.0,<1.13" + - ">=1.14.1,<1.15" fmt_version: - - ">=10.1.1,<11" + - ">=11.0.2,<12" diff --git a/cpp/cmake/thirdparty/get_spdlog.cmake b/cpp/cmake/thirdparty/get_spdlog.cmake index 7be7804c7e..57e38c2638 100644 --- a/cpp/cmake/thirdparty/get_spdlog.cmake +++ b/cpp/cmake/thirdparty/get_spdlog.cmake @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2021-2023, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -19,15 +19,6 @@ function(find_and_configure_spdlog) rapids_cpm_spdlog(FMT_OPTION "EXTERNAL_FMT_HO" INSTALL_EXPORT_SET rmm-exports) rapids_export_package(BUILD spdlog rmm-exports) - if(spdlog_ADDED) - rapids_export( - BUILD spdlog - EXPORT_SET spdlog - GLOBAL_TARGETS spdlog spdlog_header_only - NAMESPACE spdlog::) - include("${rapids-cmake-dir}/export/find_package_root.cmake") - rapids_export_find_package_root(BUILD spdlog [=[${CMAKE_CURRENT_LIST_DIR}]=] EXPORT_SET rmm-exports) - endif() endfunction() find_and_configure_spdlog() \ No newline at end of file From 878cefc923f25df07fc362126d1c269bdff30c0c Mon Sep 17 00:00:00 2001 From: Jake Awe <50372925+AyodeAwe@users.noreply.github.com> Date: Tue, 24 Sep 2024 14:10:57 -0500 Subject: [PATCH 27/79] update update-version.sh to use packaging lib (#2447) --- ci/release/update-version.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 5bb98511cf..032b88b4aa 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -25,8 +25,8 @@ NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR} NEXT_UCXX_SHORT_TAG="$(curl -sL https://version.gpuci.io/rapids/${NEXT_SHORT_TAG})" # Need to distutils-normalize the original version -NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_SHORT_TAG}'))") -NEXT_UCXX_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_UCXX_SHORT_TAG}'))") +NEXT_SHORT_TAG_PEP440=$(python -c "from packaging.version import Version; print(Version('${NEXT_SHORT_TAG}'))") +NEXT_UCXX_SHORT_TAG_PEP440=$(python -c "from packaging.version import Version; print(Version('${NEXT_UCXX_SHORT_TAG}'))") echo "Preparing release $CURRENT_TAG => $NEXT_FULL_TAG" From 17757b84c38dfa671abe5e8794fd2bd9bc636772 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 25 Sep 2024 13:49:36 -0500 Subject: [PATCH 28/79] Switch traceback to `native` (#2446) In cudf we have observed a ~10% speed up of pytest suite execution by switching pytest traceback to `--native`: ``` currently: 102474 passed, 2117 skipped, 902 xfailed in 892.16s (0:14:52) --tb=short: 102474 passed, 2117 skipped, 902 xfailed in 898.99s (0:14:58) --tb=no: 102474 passed, 2117 skipped, 902 xfailed in 815.98s (0:13:35) --tb=native: 102474 passed, 2117 skipped, 902 xfailed in 820.92s (0:13:40) ``` This PR makes a similar change to `raft` repo. xref: https://github.com/rapidsai/cudf/pull/16851 Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/raft/pull/2446 --- python/pylibraft/pylibraft/test/pytest.ini | 5 +++++ python/raft-dask/pytest.ini | 1 + python/raft-dask/raft_dask/test/pytest.ini | 5 +++++ 3 files changed, 11 insertions(+) create mode 100644 python/pylibraft/pylibraft/test/pytest.ini create mode 100644 python/raft-dask/raft_dask/test/pytest.ini diff --git a/python/pylibraft/pylibraft/test/pytest.ini b/python/pylibraft/pylibraft/test/pytest.ini new file mode 100644 index 0000000000..bf70c06f84 --- /dev/null +++ b/python/pylibraft/pylibraft/test/pytest.ini @@ -0,0 +1,5 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +[pytest] +addopts = --tb=native + diff --git a/python/raft-dask/pytest.ini b/python/raft-dask/pytest.ini index 2467e2089a..e09c2b173d 100644 --- a/python/raft-dask/pytest.ini +++ b/python/raft-dask/pytest.ini @@ -10,3 +10,4 @@ markers = nccl: marks a test as using NCCL ucx: marks a test as using UCX-Py ucxx: marks a test as using UCXX +addopts = --tb=native diff --git a/python/raft-dask/raft_dask/test/pytest.ini b/python/raft-dask/raft_dask/test/pytest.ini new file mode 100644 index 0000000000..bf70c06f84 --- /dev/null +++ b/python/raft-dask/raft_dask/test/pytest.ini @@ -0,0 +1,5 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +[pytest] +addopts = --tb=native + From 0f8b09720054e5cf7a65dddfa02a2f06e7ba8adb Mon Sep 17 00:00:00 2001 From: Divye Gala Date: Wed, 25 Sep 2024 20:53:54 -0400 Subject: [PATCH 29/79] Disable NN Descent Batch tests temporarily (#2453) Linked issue https://github.com/rapidsai/raft/issues/2450 Authors: - Divye Gala (https://github.com/divyegala) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/raft/pull/2453 --- cpp/test/CMakeLists.txt | 4 +++- cpp/test/neighbors/ann_nn_descent.cuh | 18 ++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index a497e6d3ba..5d504d2100 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -440,7 +440,9 @@ if(BUILD_TESTS) neighbors/ann_nn_descent/test_float_uint32_t.cu neighbors/ann_nn_descent/test_int8_t_uint32_t.cu neighbors/ann_nn_descent/test_uint8_t_uint32_t.cu - neighbors/ann_nn_descent/test_batch_float_uint32_t.cu + # TODO: Investigate why this test is failing Reference issue + # https://github.com/rapidsai/raft/issues/2450 + # neighbors/ann_nn_descent/test_batch_float_uint32_t.cu LIB EXPLICIT_INSTANTIATE_ONLY GPUS diff --git a/cpp/test/neighbors/ann_nn_descent.cuh b/cpp/test/neighbors/ann_nn_descent.cuh index 2f9d4e252b..5070d83b15 100644 --- a/cpp/test/neighbors/ann_nn_descent.cuh +++ b/cpp/test/neighbors/ann_nn_descent.cuh @@ -318,13 +318,15 @@ const std::vector inputs = raft::util::itertools::product inputsBatch = - raft::util::itertools::product( - {std::make_pair(0.9, 3lu), std::make_pair(0.9, 2lu)}, // min_recall, n_clusters - {4000, 5000}, // n_rows - {192, 512}, // dim - {32, 64}, // graph_degree - {raft::distance::DistanceType::L2Expanded}, - {false, true}); +// TODO: Investigate why this test is failing +// Reference issue https://github.com/rapidsai/raft/issues/2450 +// const std::vector inputsBatch = +// raft::util::itertools::product( +// {std::make_pair(0.9, 3lu), std::make_pair(0.9, 2lu)}, // min_recall, n_clusters +// {4000, 5000}, // n_rows +// {192, 512}, // dim +// {32, 64}, // graph_degree +// {raft::distance::DistanceType::L2Expanded}, +// {false, true}); } // namespace raft::neighbors::experimental::nn_descent From 2e705fbf5bfa9af148871ba509ac55593da9a098 Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Wed, 25 Sep 2024 23:01:05 -0700 Subject: [PATCH 30/79] Add missing `cuda_suffixed: true` (#2440) This change ensures `rmm-cuXX` is not in the package dependencies when building the pylibraft wheel with `-C "rapidsai.matrix-entry=cuda_suffixed=false`. Authors: - Paul Taylor (https://github.com/trxcllnt) - James Lamb (https://github.com/jameslamb) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/raft/pull/2440 --- dependencies.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 8f5c69245f..91fb176b56 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -499,10 +499,14 @@ dependencies: - *cuda_python - output_types: [requirements, pyproject] matrices: - - matrix: {cuda: "12.*"} + - matrix: + cuda: "12.*" + cuda_suffixed: "true" packages: - *rmm_cu12 - - matrix: {cuda: "11.*"} + - matrix: + cuda: "11.*" + cuda_suffixed: "true" packages: - *rmm_cu11 - {matrix: null, packages: [*rmm_unsuffixed]} From 0284b424367f07220de133715043a963b341485a Mon Sep 17 00:00:00 2001 From: rhdong Date: Thu, 26 Sep 2024 07:42:29 -0700 Subject: [PATCH 31/79] [Feat] add `repeat`, `sparsity`, `eval_n_elements` APIs to `bitset` (#2439) - This PR is a part of the feature that applies the prefilter brute-force in Cagra. Authors: - rhdong (https://github.com/rhdong) - Corey J. Nolet (https://github.com/cjnolet) Approvers: - Micka (https://github.com/lowener) URL: https://github.com/rapidsai/raft/pull/2439 --- cpp/bench/prims/util/popc.cu | 7 +- cpp/include/raft/core/bitset.cuh | 107 +++++++++++++++++++++++- cpp/include/raft/core/bitset.hpp | 76 +++++++++++++++++ cpp/include/raft/util/detail/popc.cuh | 6 +- cpp/include/raft/util/popc.cuh | 4 +- cpp/test/core/bitset.cu | 114 +++++++++++++++++++++++--- cpp/test/util/popc.cu | 22 +++-- 7 files changed, 311 insertions(+), 25 deletions(-) diff --git a/cpp/bench/prims/util/popc.cu b/cpp/bench/prims/util/popc.cu index 249dc13d1e..c6249fb2bd 100644 --- a/cpp/bench/prims/util/popc.cu +++ b/cpp/bench/prims/util/popc.cu @@ -89,10 +89,9 @@ struct popc_bench : public fixture { auto bits_view = raft::make_device_vector_view(bits_d.data_handle(), bits_d.size()); - index_t max_len = params.n_rows * params.n_cols; - auto max_len_view = raft::make_host_scalar_view(&max_len); - auto nnz_actual_view = - nnz_actual_d.view(); // raft::make_device_scalar_view(nnz_actual_d.data_handle()); + index_t max_len = params.n_rows * params.n_cols; + auto max_len_view = raft::make_host_scalar_view(&max_len); + auto nnz_actual_view = nnz_actual_d.view(); raft::popc(this->handle, bits_view, max_len_view, nnz_actual_view); }); } diff --git a/cpp/include/raft/core/bitset.cuh b/cpp/include/raft/core/bitset.cuh index b6e6128eca..d1bffdb81e 100644 --- a/cpp/include/raft/core/bitset.cuh +++ b/cpp/include/raft/core/bitset.cuh @@ -26,6 +26,8 @@ #include #include +#include + #include namespace raft::core { @@ -60,6 +62,109 @@ _RAFT_DEVICE void bitset_view::set(const index_t sample_index } } +template +void bitset_view::count(const raft::resources& res, + raft::device_scalar_view count_gpu_scalar) const +{ + auto max_len = raft::make_host_scalar_view(&bitset_len_); + auto values = raft::make_device_vector_view(bitset_ptr_, n_elements()); + raft::popc(res, values, max_len, count_gpu_scalar); +} + +template +RAFT_KERNEL bitset_repeat_kernel(const bitset_t* src, + bitset_t* output, + index_t src_bit_len, + index_t repeat_times) +{ + constexpr index_t bits_per_element = sizeof(bitset_t) * 8; + int output_idx = blockIdx.x * blockDim.x + threadIdx.x; + + index_t total_bits = src_bit_len * repeat_times; + index_t output_size = (total_bits + bits_per_element - 1) / bits_per_element; + index_t src_size = (src_bit_len + bits_per_element - 1) / bits_per_element; + + if (output_idx < output_size) { + bitset_t result = 0; + index_t bit_written = 0; + + index_t start_bit = output_idx * bits_per_element; + + while (bit_written < bits_per_element && start_bit + bit_written < total_bits) { + index_t bit_idx = (start_bit + bit_written) % src_bit_len; + index_t src_word_idx = bit_idx / bits_per_element; + index_t src_offset = bit_idx % bits_per_element; + + index_t remaining_bits = min(bits_per_element - bit_written, src_bit_len - bit_idx); + + bitset_t src_value = (src[src_word_idx] >> src_offset); + + if (src_offset + remaining_bits > bits_per_element) { + bitset_t next_value = src[(src_word_idx + 1) % src_size]; + src_value |= (next_value << (bits_per_element - src_offset)); + } + src_value &= ((bitset_t{1} << remaining_bits) - 1); + result |= (src_value << bit_written); + bit_written += remaining_bits; + } + output[output_idx] = result; + } +} + +template +void bitset_repeat(raft::resources const& handle, + const bitset_t* d_src, + bitset_t* d_output, + index_t src_bit_len, + index_t repeat_times) +{ + if (src_bit_len == 0 || repeat_times == 0) return; + auto stream = resource::get_cuda_stream(handle); + + constexpr index_t bits_per_element = sizeof(bitset_t) * 8; + const index_t total_bits = src_bit_len * repeat_times; + const index_t output_size = (total_bits + bits_per_element - 1) / bits_per_element; + + int threadsPerBlock = 128; + int blocksPerGrid = (output_size + threadsPerBlock - 1) / threadsPerBlock; + bitset_repeat_kernel<<>>( + d_src, d_output, src_bit_len, repeat_times); + + return; +} + +template +void bitset_view::repeat(const raft::resources& res, + index_t times, + bitset_t* output_device_ptr) const +{ + auto thrust_policy = raft::resource::get_thrust_policy(res); + constexpr index_t bits_per_element = sizeof(bitset_t) * 8; + + if (bitset_len_ % bits_per_element == 0) { + index_t num_elements_to_copy = bitset_len_ / bits_per_element; + + for (index_t i = 0; i < times; ++i) { + raft::copy(output_device_ptr + i * num_elements_to_copy, + bitset_ptr_, + num_elements_to_copy, + raft::resource::get_cuda_stream(res)); + } + } else { + bitset_repeat(res, bitset_ptr_, output_device_ptr, bitset_len_, times); + } +} + +template +double bitset_view::sparsity(const raft::resources& res) const +{ + index_t size_h = this->size(); + if (0 == size_h) { return static_cast(1.0); } + index_t count_h = this->count(res); + + return static_cast((1.0 * (size_h - count_h)) / (1.0 * size_h)); +} + template bitset::bitset(const raft::resources& res, raft::device_vector_view mask_index, @@ -155,7 +260,7 @@ template void bitset::count(const raft::resources& res, raft::device_scalar_view count_gpu_scalar) { - auto max_len = raft::make_host_scalar_view(&bitset_len_); + auto max_len = raft::make_host_scalar_view(&bitset_len_); auto values = raft::make_device_vector_view(bitset_.data(), n_elements()); raft::popc(res, values, max_len, count_gpu_scalar); diff --git a/cpp/include/raft/core/bitset.hpp b/cpp/include/raft/core/bitset.hpp index 3608ee43fa..be828def87 100644 --- a/cpp/include/raft/core/bitset.hpp +++ b/cpp/include/raft/core/bitset.hpp @@ -22,6 +22,8 @@ #include #include +#include + namespace raft::core { /** * @defgroup bitset Bitset @@ -103,6 +105,80 @@ struct bitset_view { { return raft::make_device_vector_view(bitset_ptr_, n_elements()); } + /** + * @brief Returns the number of bits set to true in count_gpu_scalar. + * + * @param[in] res RAFT resources + * @param[out] count_gpu_scalar Device scalar to store the count + */ + void count(const raft::resources& res, raft::device_scalar_view count_gpu_scalar) const; + /** + * @brief Returns the number of bits set to true. + * + * @param res RAFT resources + * @return index_t Number of bits set to true + */ + auto count(const raft::resources& res) const -> index_t + { + auto count_gpu_scalar = raft::make_device_scalar(res, 0.0); + count(res, count_gpu_scalar.view()); + index_t count_cpu = 0; + raft::update_host( + &count_cpu, count_gpu_scalar.data_handle(), 1, resource::get_cuda_stream(res)); + resource::sync_stream(res); + return count_cpu; + } + + /** + * @brief Repeats the bitset data and copies it to the output device pointer. + * + * This function takes the original bitset data stored in the device memory + * and repeats it a specified number of times into a new location in the device memory. + * The bits are copied bit-by-bit to ensure that even if the number of bits (bitset_len_) + * is not a multiple of the bitset element size (e.g., 32 for uint32_t), the bits are + * tightly packed without any gaps between rows. + * + * @param res RAFT resources for managing CUDA streams and execution policies. + * @param times Number of times the bitset data should be repeated in the output. + * @param output_device_ptr Device pointer where the repeated bitset data will be stored. + * + * The caller must ensure that the output device pointer has enough memory allocated + * to hold `times * bitset_len` bits, where `bitset_len` is the number of bits in the original + * bitset. This function uses Thrust parallel algorithms to efficiently perform the operation on + * the GPU. + */ + void repeat(const raft::resources& res, index_t times, bitset_t* output_device_ptr) const; + + /** + * @brief Calculate the sparsity (fraction of 0s) of the bitset. + * + * This function computes the sparsity of the bitset, defined as the ratio of unset bits (0s) + * to the total number of bits in the set. If the total number of bits is zero, the function + * returns 1.0, indicating the set is fully sparse. + * + * @param res RAFT resources for managing CUDA streams and execution policies. + * @return double The sparsity of the bitset, i.e., the fraction of unset bits. + * + * This API will synchronize on the stream of `res`. + */ + double sparsity(const raft::resources& res) const; + + /** + * @brief Calculates the number of `bitset_t` elements required to store a bitset. + * + * This function computes the number of `bitset_t` elements needed to store a bitset, ensuring + * that all bits are accounted for. If the bitset length is not a multiple of the `bitset_t` size + * (in bits), the calculation rounds up to include the remaining bits in an additional `bitset_t` + * element. + * + * @param bitset_len The total length of the bitset in bits. + * @return size_t The number of `bitset_t` elements required to store the bitset. + */ + static inline size_t eval_n_elements(size_t bitset_len) + { + const size_t bits_per_element = sizeof(bitset_t) * 8; + return (bitset_len + bits_per_element - 1) / bits_per_element; + } private: bitset_t* bitset_ptr_; diff --git a/cpp/include/raft/util/detail/popc.cuh b/cpp/include/raft/util/detail/popc.cuh index 20b4814216..f335be6fd0 100644 --- a/cpp/include/raft/util/detail/popc.cuh +++ b/cpp/include/raft/util/detail/popc.cuh @@ -36,12 +36,12 @@ namespace raft::detail { */ template void popc(const raft::resources& res, - device_vector_view values, - raft::host_scalar_view max_len, + device_vector_view values, + raft::host_scalar_view max_len, raft::device_scalar_view counter) { auto values_size = values.size(); - auto values_matrix = raft::make_device_matrix_view( + auto values_matrix = raft::make_device_matrix_view( values.data_handle(), values_size, 1); auto counter_vector = raft::make_device_vector_view(counter.data_handle(), 1); diff --git a/cpp/include/raft/util/popc.cuh b/cpp/include/raft/util/popc.cuh index 153694e45e..d4bc01e274 100644 --- a/cpp/include/raft/util/popc.cuh +++ b/cpp/include/raft/util/popc.cuh @@ -31,8 +31,8 @@ namespace raft { */ template void popc(const raft::resources& res, - device_vector_view values, - raft::host_scalar_view max_len, + device_vector_view values, + raft::host_scalar_view max_len, raft::device_scalar_view counter) { detail::popc(res, values, max_len, counter); diff --git a/cpp/test/core/bitset.cu b/cpp/test/core/bitset.cu index b799297e8c..ac601274c1 100644 --- a/cpp/test/core/bitset.cu +++ b/cpp/test/core/bitset.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,12 +32,13 @@ struct test_spec_bitset { uint64_t bitset_len; uint64_t mask_len; uint64_t query_len; + uint64_t repeat_times; }; auto operator<<(std::ostream& os, const test_spec_bitset& ss) -> std::ostream& { os << "bitset{bitset_len: " << ss.bitset_len << ", mask_len: " << ss.mask_len - << ", query_len: " << ss.query_len << "}"; + << ", query_len: " << ss.query_len << ", repeat_times: " << ss.repeat_times << "}"; return os; } @@ -80,6 +81,48 @@ void flip_cpu_bitset(std::vector& bitset) } } +template +void repeat_cpu_bitset(std::vector& input, + size_t input_bits, + size_t repeat, + std::vector& output) +{ + const size_t output_bits = input_bits * repeat; + const size_t output_units = (output_bits + sizeof(bitset_t) * 8 - 1) / (sizeof(bitset_t) * 8); + + std::memset(output.data(), 0, output_units * sizeof(bitset_t)); + + size_t output_bit_index = 0; + + for (size_t r = 0; r < repeat; ++r) { + for (size_t i = 0; i < input_bits; ++i) { + size_t input_unit_index = i / (sizeof(bitset_t) * 8); + size_t input_bit_offset = i % (sizeof(bitset_t) * 8); + bool bit = (input[input_unit_index] >> input_bit_offset) & 1; + + size_t output_unit_index = output_bit_index / (sizeof(bitset_t) * 8); + size_t output_bit_offset = output_bit_index % (sizeof(bitset_t) * 8); + + output[output_unit_index] |= (static_cast(bit) << output_bit_offset); + + ++output_bit_index; + } + } +} + +template +double sparsity_cpu_bitset(std::vector& data, size_t total_bits) +{ + size_t one_count = 0; + for (size_t i = 0; i < total_bits; ++i) { + size_t unit_index = i / (sizeof(bitset_t) * 8); + size_t bit_offset = i % (sizeof(bitset_t) * 8); + bool bit = (data[unit_index] >> bit_offset) & 1; + if (bit == 1) { ++one_count; } + } + return static_cast((total_bits - one_count) / (1.0 * total_bits)); +} + template class BitsetTest : public testing::TestWithParam { protected: @@ -87,13 +130,19 @@ class BitsetTest : public testing::TestWithParam { const test_spec_bitset spec; std::vector bitset_result; std::vector bitset_ref; + std::vector bitset_repeat_ref; + std::vector bitset_repeat_result; raft::resources res; public: explicit BitsetTest() : spec(testing::TestWithParam::GetParam()), bitset_result(raft::ceildiv(spec.bitset_len, uint64_t(bitset_element_size))), - bitset_ref(raft::ceildiv(spec.bitset_len, uint64_t(bitset_element_size))) + bitset_ref(raft::ceildiv(spec.bitset_len, uint64_t(bitset_element_size))), + bitset_repeat_ref( + raft::ceildiv(spec.bitset_len * spec.repeat_times, uint64_t(bitset_element_size))), + bitset_repeat_result( + raft::ceildiv(spec.bitset_len * spec.repeat_times, uint64_t(bitset_element_size))) { } @@ -145,6 +194,50 @@ class BitsetTest : public testing::TestWithParam { resource::sync_stream(res, stream); ASSERT_TRUE(hostVecMatch(bitset_ref, bitset_result, raft::Compare())); + // test sparsity, repeat and eval_n_elements + { + auto my_bitset_view = my_bitset.view(); + auto sparsity_result = my_bitset_view.sparsity(res); + auto sparsity_ref = sparsity_cpu_bitset(bitset_ref, size_t(spec.bitset_len)); + ASSERT_EQ(sparsity_result, sparsity_ref); + + auto eval_n_elements = + bitset_view::eval_n_elements(spec.bitset_len * spec.repeat_times); + ASSERT_EQ(bitset_repeat_ref.size(), eval_n_elements); + + auto repeat_device = raft::make_device_vector(res, eval_n_elements); + RAFT_CUDA_TRY(cudaMemsetAsync( + repeat_device.data_handle(), 0, eval_n_elements * sizeof(bitset_t), stream)); + repeat_cpu_bitset( + bitset_ref, size_t(spec.bitset_len), size_t(spec.repeat_times), bitset_repeat_ref); + + my_bitset_view.repeat(res, index_t(spec.repeat_times), repeat_device.data_handle()); + + ASSERT_EQ(bitset_repeat_ref.size(), repeat_device.size()); + update_host( + bitset_repeat_result.data(), repeat_device.data_handle(), repeat_device.size(), stream); + ASSERT_EQ(bitset_repeat_ref.size(), bitset_repeat_result.size()); + + index_t errors = 0; + static constexpr index_t len_per_item = sizeof(bitset_t) * 8; + bitset_t tail_len = (index_t(spec.bitset_len * spec.repeat_times) % len_per_item); + bitset_t tail_mask = + tail_len ? (bitset_t)((bitset_t{1} << tail_len) - bitset_t{1}) : ~bitset_t{0}; + for (index_t i = 0; i < bitset_repeat_ref.size(); i++) { + if (i == bitset_repeat_ref.size() - 1) { + errors += (bitset_repeat_ref[i] & tail_mask) != (bitset_repeat_result[i] & tail_mask); + } else { + errors += (bitset_repeat_ref[i] != bitset_repeat_result[i]); + } + } + ASSERT_EQ(errors, 0); + + // recheck the sparsity after repeat + sparsity_result = + sparsity_cpu_bitset(bitset_repeat_result, size_t(spec.bitset_len * spec.repeat_times)); + ASSERT_EQ(sparsity_result, sparsity_ref); + } + // Flip the bitset and re-test auto bitset_count = my_bitset.count(res); my_bitset.flip(res); @@ -167,13 +260,14 @@ class BitsetTest : public testing::TestWithParam { } }; -auto inputs_bitset = ::testing::Values(test_spec_bitset{32, 5, 10}, - test_spec_bitset{100, 30, 10}, - test_spec_bitset{1024, 55, 100}, - test_spec_bitset{10000, 1000, 1000}, - test_spec_bitset{1 << 15, 1 << 3, 1 << 12}, - test_spec_bitset{1 << 15, 1 << 24, 1 << 13}, - test_spec_bitset{1 << 25, 1 << 23, 1 << 14}); +auto inputs_bitset = ::testing::Values(test_spec_bitset{32, 5, 10, 101}, + test_spec_bitset{100, 30, 10, 13}, + test_spec_bitset{1024, 55, 100, 1}, + test_spec_bitset{10000, 1000, 1000, 100}, + test_spec_bitset{1 << 15, 1 << 3, 1 << 12, 5}, + test_spec_bitset{1 << 15, 1 << 24, 1 << 13, 3}, + test_spec_bitset{1 << 25, 1 << 23, 1 << 14, 3}, + test_spec_bitset{1 << 25, 1 << 23, 1 << 14, 21}); using Uint16_32 = BitsetTest; TEST_P(Uint16_32, Run) { run(); } diff --git a/cpp/test/util/popc.cu b/cpp/test/util/popc.cu index c08faacb07..28eaad2fcb 100644 --- a/cpp/test/util/popc.cu +++ b/cpp/test/util/popc.cu @@ -76,7 +76,7 @@ class PopcTest : public ::testing::TestWithParam> { index_t bit_position = index % (8 * sizeof(bits_t)); if (((element >> bit_position) & 1) == 0) { - element |= (static_cast(1) << bit_position); + element |= (static_cast(1) << bit_position); num_ones--; } } @@ -101,7 +101,7 @@ class PopcTest : public ::testing::TestWithParam> { raft::make_device_vector_view(bits_d.data(), bits_d.size()); index_t max_len = params.n_rows * params.n_cols; - auto max_len_view = raft::make_host_scalar_view(&max_len); + auto max_len_view = raft::make_host_scalar_view(&max_len); index_t nnz_actual_h = 0; rmm::device_scalar nnz_actual_d(0, stream); @@ -123,8 +123,17 @@ class PopcTest : public ::testing::TestWithParam> { index_t nnz_expected; }; -using PopcTestI32 = PopcTest; -TEST_P(PopcTestI32, Result) { Run(); } +using PopcTestI32_U32 = PopcTest; +TEST_P(PopcTestI32_U32, Result) { Run(); } + +using PopcTestI32_U64 = PopcTest; +TEST_P(PopcTestI32_U64, Result) { Run(); } + +using PopcTestI32_U16 = PopcTest; +TEST_P(PopcTestI32_U16, Result) { Run(); } + +using PopcTestI32_U8 = PopcTest; +TEST_P(PopcTestI32_U8, Result) { Run(); } template const std::vector> popc_inputs = { @@ -154,6 +163,9 @@ const std::vector> popc_inputs = { {2, 33, 0.2}, }; -INSTANTIATE_TEST_CASE_P(PopcTest, PopcTestI32, ::testing::ValuesIn(popc_inputs)); +INSTANTIATE_TEST_CASE_P(PopcTest, PopcTestI32_U32, ::testing::ValuesIn(popc_inputs)); +INSTANTIATE_TEST_CASE_P(PopcTest, PopcTestI32_U64, ::testing::ValuesIn(popc_inputs)); +INSTANTIATE_TEST_CASE_P(PopcTest, PopcTestI32_U16, ::testing::ValuesIn(popc_inputs)); +INSTANTIATE_TEST_CASE_P(PopcTest, PopcTestI32_U8, ::testing::ValuesIn(popc_inputs)); } // namespace raft From f37c41c54fc64a4e3689e5a61851ba3821800fee Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 26 Sep 2024 09:45:11 -0500 Subject: [PATCH 32/79] bump NCCL floor to 2.18.1.1 (#2443) Contributes to https://github.com/rapidsai/build-planning/issues/102 Some RAPIDS libraries are using `ncclCommSplit()`, which was introduced in `nccl==2.18.1.1`. This is part of a series of PRs across RAPIDS updating libraries' pins to `nccl>=2.18.1.1` to ensure they get a new-enough version that supports that. Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/raft/pull/2443 --- conda/environments/all_cuda-118_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-125_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-125_arch-x86_64.yaml | 2 +- conda/environments/bench_ann_cuda-118_arch-aarch64.yaml | 2 +- conda/environments/bench_ann_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/bench_ann_cuda-120_arch-aarch64.yaml | 2 +- conda/environments/bench_ann_cuda-120_arch-x86_64.yaml | 2 +- conda/recipes/libraft/conda_build_config.yaml | 2 +- conda/recipes/raft-ann-bench/conda_build_config.yaml | 2 +- conda/recipes/raft-dask/conda_build_config.yaml | 3 +++ conda/recipes/raft-dask/meta.yaml | 4 ++-- dependencies.yaml | 2 +- 13 files changed, 16 insertions(+), 13 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index 462874a7e7..de4eb7e690 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -36,7 +36,7 @@ dependencies: - libcusparse-dev=11.7.5.86 - libcusparse=11.7.5.86 - libucxx==0.40.*,>=0.0.0a0 -- nccl>=2.9.9 +- nccl>=2.18.1.1 - ninja - numba>=0.57 - numpy>=1.23,<3.0a0 diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index cfd974a6a8..26f4c1efaa 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -36,7 +36,7 @@ dependencies: - libcusparse-dev=11.7.5.86 - libcusparse=11.7.5.86 - libucxx==0.40.*,>=0.0.0a0 -- nccl>=2.9.9 +- nccl>=2.18.1.1 - ninja - numba>=0.57 - numpy>=1.23,<3.0a0 diff --git a/conda/environments/all_cuda-125_arch-aarch64.yaml b/conda/environments/all_cuda-125_arch-aarch64.yaml index 82e391e9ae..692956502b 100644 --- a/conda/environments/all_cuda-125_arch-aarch64.yaml +++ b/conda/environments/all_cuda-125_arch-aarch64.yaml @@ -33,7 +33,7 @@ dependencies: - libcusolver-dev - libcusparse-dev - libucxx==0.40.*,>=0.0.0a0 -- nccl>=2.9.9 +- nccl>=2.18.1.1 - ninja - numba>=0.57 - numpy>=1.23,<3.0a0 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 0389427d13..133d42bfee 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -33,7 +33,7 @@ dependencies: - libcusolver-dev - libcusparse-dev - libucxx==0.40.*,>=0.0.0a0 -- nccl>=2.9.9 +- nccl>=2.18.1.1 - ninja - numba>=0.57 - numpy>=1.23,<3.0a0 diff --git a/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml index eff1c56840..f99cedd627 100644 --- a/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml +++ b/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml @@ -32,7 +32,7 @@ dependencies: - libcusparse=11.7.5.86 - libucxx==0.40.*,>=0.0.0a0 - matplotlib -- nccl>=2.9.9 +- nccl>=2.18.1.1 - ninja - nlohmann_json>=3.11.2 - nvcc_linux-aarch64=11.8 diff --git a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml index 87b19d2952..08aea32ab1 100644 --- a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml +++ b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml @@ -32,7 +32,7 @@ dependencies: - libcusparse=11.7.5.86 - libucxx==0.40.*,>=0.0.0a0 - matplotlib -- nccl>=2.9.9 +- nccl>=2.18.1.1 - ninja - nlohmann_json>=3.11.2 - nvcc_linux-64=11.8 diff --git a/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml index ff3451c15c..572ad85ab5 100644 --- a/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml +++ b/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml @@ -29,7 +29,7 @@ dependencies: - libcusparse-dev - libucxx==0.40.*,>=0.0.0a0 - matplotlib -- nccl>=2.9.9 +- nccl>=2.18.1.1 - ninja - nlohmann_json>=3.11.2 - openblas diff --git a/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml index 085e099ae8..9fd23edb03 100644 --- a/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml +++ b/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml @@ -29,7 +29,7 @@ dependencies: - libcusparse-dev - libucxx==0.40.*,>=0.0.0a0 - matplotlib -- nccl>=2.9.9 +- nccl>=2.18.1.1 - ninja - nlohmann_json>=3.11.2 - openblas diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index 5c0047fb9c..6dc06648bd 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -20,7 +20,7 @@ cmake_version: - ">=3.26.4,!=3.30.0" nccl_version: - - ">=2.9.9" + - ">=2.18.1.1" glog_version: - ">=0.6.0" diff --git a/conda/recipes/raft-ann-bench/conda_build_config.yaml b/conda/recipes/raft-ann-bench/conda_build_config.yaml index db0083b583..bdb4e883ea 100644 --- a/conda/recipes/raft-ann-bench/conda_build_config.yaml +++ b/conda/recipes/raft-ann-bench/conda_build_config.yaml @@ -20,7 +20,7 @@ cmake_version: - ">=3.26.4,!=3.30.0" nccl_version: - - ">=2.9.9" + - ">=2.18.1.1" glog_version: - ">=0.6.0" diff --git a/conda/recipes/raft-dask/conda_build_config.yaml b/conda/recipes/raft-dask/conda_build_config.yaml index e6afed2890..58e8ec3c9e 100644 --- a/conda/recipes/raft-dask/conda_build_config.yaml +++ b/conda/recipes/raft-dask/conda_build_config.yaml @@ -24,3 +24,6 @@ ucxx_version: cmake_version: - ">=3.26.4,!=3.30.0" + +nccl_version: + - ">=2.18.1.1" diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index 74b26b5935..bc13d352b7 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -50,7 +50,7 @@ requirements: {% endif %} - cuda-version ={{ cuda_version }} - cython >=3.0.0 - - nccl >=2.9.9 + - nccl {{ nccl_version }} - pylibraft {{ version }} - python x.x - rmm ={{ minor_version }} @@ -68,7 +68,7 @@ requirements: - dask-cuda ={{ minor_version }} - rapids-dask-dependency ={{ minor_version }} - joblib >=0.11 - - nccl >=2.9.9 + - nccl {{ nccl_version }} - pylibraft {{ version }} - python x.x - rmm ={{ minor_version }} diff --git a/dependencies.yaml b/dependencies.yaml index 91fb176b56..e833e8519a 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -170,7 +170,7 @@ dependencies: packages: - c-compiler - cxx-compiler - - nccl>=2.9.9 + - nccl>=2.18.1.1 - libucxx==0.40.*,>=0.0.0a0 specific: - output_types: conda From 704feb1a0249e73bb0cab96968b892e8d5823b46 Mon Sep 17 00:00:00 2001 From: rhdong Date: Thu, 26 Sep 2024 09:32:41 -0700 Subject: [PATCH 33/79] [Opt] Enforce the UT Coverity and add benchmark for `transpose` (#2438) Authors: - rhdong (https://github.com/rhdong) Approvers: - Artem M. Chirkin (https://github.com/achirkin) - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/raft/pull/2438 --- cpp/bench/prims/CMakeLists.txt | 1 + cpp/bench/prims/linalg/transpose.cu | 85 ++++ cpp/include/raft/linalg/detail/transpose.cuh | 67 ++- cpp/test/linalg/transpose.cu | 445 ++++++++++++++----- 4 files changed, 465 insertions(+), 133 deletions(-) create mode 100644 cpp/bench/prims/linalg/transpose.cu diff --git a/cpp/bench/prims/CMakeLists.txt b/cpp/bench/prims/CMakeLists.txt index 9d80cbaac2..52c63ad73b 100644 --- a/cpp/bench/prims/CMakeLists.txt +++ b/cpp/bench/prims/CMakeLists.txt @@ -132,6 +132,7 @@ if(BUILD_PRIMS_BENCH) linalg/reduce_rows_by_key.cu linalg/reduce.cu linalg/sddmm.cu + linalg/transpose.cu main.cpp ) diff --git a/cpp/bench/prims/linalg/transpose.cu b/cpp/bench/prims/linalg/transpose.cu new file mode 100644 index 0000000000..e60e50c125 --- /dev/null +++ b/cpp/bench/prims/linalg/transpose.cu @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include +#include + +#include + +namespace raft::bench::linalg { + +template +struct transpose_input { + IdxT rows, cols; +}; + +template +inline auto operator<<(std::ostream& os, const transpose_input& p) -> std::ostream& +{ + os << p.rows << "#" << p.cols; + return os; +} + +template +struct TransposeBench : public fixture { + TransposeBench(const transpose_input& p) + : params(p), in(p.rows * p.cols, stream), out(p.rows * p.cols, stream) + { + raft::random::RngState rng{1234}; + raft::random::uniform(handle, rng, in.data(), p.rows * p.cols, (T)-10.0, (T)10.0); + } + + void run_benchmark(::benchmark::State& state) override + { + std::ostringstream label_stream; + label_stream << params; + state.SetLabel(label_stream.str()); + + loop_on_state(state, [this]() { + auto input_view = + raft::make_device_matrix_view(in.data(), params.rows, params.cols); + auto output_view = raft::make_device_vector_view(out.data(), params.rows); + raft::linalg::transpose(handle, + input_view.data_handle(), + output_view.data_handle(), + params.rows, + params.cols, + handle.get_stream()); + }); + } + + private: + transpose_input params; + rmm::device_uvector in, out; +}; // struct TransposeBench + +const std::vector> transpose_inputs_i32 = + raft::util::itertools::product>({10, 128, 256, 512, 1024}, + {10000, 100000, 1000000}); + +RAFT_BENCH_REGISTER((TransposeBench), "", transpose_inputs_i32); +RAFT_BENCH_REGISTER((TransposeBench), "", transpose_inputs_i32); + +RAFT_BENCH_REGISTER((TransposeBench), "", transpose_inputs_i32); +RAFT_BENCH_REGISTER((TransposeBench), "", transpose_inputs_i32); + +} // namespace raft::bench::linalg diff --git a/cpp/include/raft/linalg/detail/transpose.cuh b/cpp/include/raft/linalg/detail/transpose.cuh index ec60aacc9c..c5f0544b5c 100644 --- a/cpp/include/raft/linalg/detail/transpose.cuh +++ b/cpp/include/raft/linalg/detail/transpose.cuh @@ -38,7 +38,9 @@ template RAFT_KERNEL transpose_half_kernel(IndexType n_rows, IndexType n_cols, const half* __restrict__ in, - half* __restrict__ out) + half* __restrict__ out, + const IndexType stride_in, + const IndexType stride_out) { __shared__ half tile[TILE_DIM][TILE_DIM + 1]; @@ -49,7 +51,7 @@ RAFT_KERNEL transpose_half_kernel(IndexType n_rows, for (int j = 0; j < TILE_DIM; j += BLOCK_ROWS) { if (x < n_cols && (y + j) < n_rows) { - tile[threadIdx.y + j][threadIdx.x] = __ldg(&in[(y + j) * n_cols + x]); + tile[threadIdx.y + j][threadIdx.x] = __ldg(&in[(y + j) * stride_in + x]); } } __syncthreads(); @@ -59,7 +61,7 @@ RAFT_KERNEL transpose_half_kernel(IndexType n_rows, for (int j = 0; j < TILE_DIM; j += BLOCK_ROWS) { if (x < n_rows && (y + j) < n_cols) { - out[(y + j) * n_rows + x] = tile[threadIdx.x][threadIdx.y + j]; + out[(y + j) * stride_out + x] = tile[threadIdx.x][threadIdx.y + j]; } } __syncthreads(); @@ -67,9 +69,33 @@ RAFT_KERNEL transpose_half_kernel(IndexType n_rows, } } +/** + * @brief Transposes a matrix stored in row-major order. + * + * This function transposes a matrix of half-precision floating-point numbers (`half`). + * Both the input (`in`) and output (`out`) matrices are assumed to be stored in row-major order. + * + * @tparam IndexType The type used for indexing the matrix dimensions (e.g., int). + * @param handle The RAFT resource handle which contains resources. + * @param n_rows The number of rows in the input matrix. + * @param n_cols The number of columns in the input matrix. + * @param in Pointer to the input matrix in row-major order. + * @param out Pointer to the output matrix in row-major order, where the transposed matrix will be + * stored. + * @param stride_in The stride (number of elements between consecutive rows) for the input matrix. + * Default is 1, which means the input matrix is contiguous in memory. + * @param stride_out The stride (number of elements between consecutive rows) for the output matrix. + * Default is 1, which means the output matrix is contiguous in memory. + */ + template -void transpose_half( - raft::resources const& handle, IndexType n_rows, IndexType n_cols, const half* in, half* out) +void transpose_half(raft::resources const& handle, + IndexType n_rows, + IndexType n_cols, + const half* in, + half* out, + const IndexType stride_in = 1, + const IndexType stride_out = 1) { if (n_cols == 0 || n_rows == 0) return; auto stream = resource::get_cuda_stream(handle); @@ -100,8 +126,13 @@ void transpose_half( dim3 grids(adjusted_grid_x, adjusted_grid_y); - transpose_half_kernel - <<>>(n_rows, n_cols, in, out); + if (stride_in > 1 || stride_out > 1) { + transpose_half_kernel + <<>>(n_rows, n_cols, in, out, stride_in, stride_out); + } else { + transpose_half_kernel + <<>>(n_rows, n_cols, in, out, n_cols, n_rows); + } RAFT_CUDA_TRY(cudaPeekAtLastError()); } @@ -118,7 +149,7 @@ void transpose(raft::resources const& handle, int out_n_cols = n_rows; if constexpr (std::is_same_v) { - transpose_half(handle, out_n_rows, out_n_cols, in, out); + transpose_half(handle, n_cols, n_rows, in, out); } else { cublasHandle_t cublas_h = resource::get_cublas_handle(handle); RAFT_CUBLAS_TRY(cublasSetStream(cublas_h, stream)); @@ -195,9 +226,13 @@ void transpose_row_major_impl( raft::mdspan, LayoutPolicy, AccessorPolicy> in, raft::mdspan, LayoutPolicy, AccessorPolicy> out) { - auto out_n_rows = in.extent(1); - auto out_n_cols = in.extent(0); - transpose_half(handle, out_n_cols, out_n_rows, in.data_handle(), out.data_handle()); + transpose_half(handle, + in.extent(0), + in.extent(1), + in.data_handle(), + out.data_handle(), + in.stride(0), + out.stride(0)); } template @@ -233,9 +268,13 @@ void transpose_col_major_impl( raft::mdspan, LayoutPolicy, AccessorPolicy> in, raft::mdspan, LayoutPolicy, AccessorPolicy> out) { - auto out_n_rows = in.extent(1); - auto out_n_cols = in.extent(0); - transpose_half(handle, out_n_rows, out_n_cols, in.data_handle(), out.data_handle()); + transpose_half(handle, + in.extent(1), + in.extent(0), + in.data_handle(), + out.data_handle(), + in.stride(1), + out.stride(1)); } }; // end namespace detail diff --git a/cpp/test/linalg/transpose.cu b/cpp/test/linalg/transpose.cu index cbe869a9a5..22fc1c1d60 100644 --- a/cpp/test/linalg/transpose.cu +++ b/cpp/test/linalg/transpose.cu @@ -29,48 +29,104 @@ #include +#include + +namespace std { +template <> +struct is_floating_point : std::true_type {}; +} // namespace std + namespace raft { namespace linalg { template -struct TranposeInputs { +void initialize_array(T* data_h, size_t size) +{ + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_real_distribution<> dis(0.0, 1.0); + + for (size_t i = 0; i < size; ++i) { + if constexpr (std::is_same_v) { + data_h[i] = __float2half(static_cast(dis(gen))); + } else { + data_h[i] = static_cast(dis(gen)); + } + } +} + +template +void cpu_transpose_row_major( + const T* input, T* output, int rows, int cols, int stride_in = -1, int stride_out = -1) +{ + stride_in = stride_in == -1 ? cols : stride_in; + stride_out = stride_out == -1 ? rows : stride_out; + if (stride_in) + for (int i = 0; i < rows; ++i) { + for (int j = 0; j < cols; ++j) { + output[j * stride_out + i] = input[i * stride_in + j]; + } + } +} + +template +void cpu_transpose_col_major( + const T* input, T* output, int rows, int cols, int stride_in = -1, int stride_out = -1) +{ + cpu_transpose_row_major(input, output, cols, rows, stride_in, stride_out); +} + +bool validate_half(const half* h_ref, const half* h_result, half tolerance, int len) +{ + bool success = true; + for (int i = 0; i < len; ++i) { + if (raft::abs(__half2float(h_result[i]) - __half2float(h_ref[i])) >= __half2float(tolerance)) { + success = false; + break; + } + if (!success) break; + } + return success; +} + +namespace transpose_regular_test { + +template +struct TransposeInputs { T tolerance; - int len; int n_row; int n_col; unsigned long long int seed; }; template -::std::ostream& operator<<(::std::ostream& os, const TranposeInputs& dims) -{ - return os; -} - -template -class TransposeTest : public ::testing::TestWithParam> { +class TransposeTest : public ::testing::TestWithParam> { public: TransposeTest() - : params(::testing::TestWithParam>::GetParam()), + : params(::testing::TestWithParam>::GetParam()), stream(resource::get_cuda_stream(handle)), - data(params.len, stream), - data_trans_ref(params.len, stream), - data_trans(params.len, stream) + data(params.n_row * params.n_col, stream), + data_trans_ref(params.n_row * params.n_col, stream), + data_trans(params.n_row * params.n_col, stream) { } protected: void SetUp() override { - int len = params.len; - ASSERT(params.len == 9, "This test works only with len=9!"); - T data_h[] = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0}; - raft::update_device(data.data(), data_h, len, stream); - T data_ref_h[] = {1.0, 4.0, 7.0, 2.0, 5.0, 8.0, 3.0, 6.0, 9.0}; - raft::update_device(data_trans_ref.data(), data_ref_h, len, stream); + int len = params.n_row * params.n_col; + std::vector data_h(len); + std::vector data_ref_h(len); + + initialize_array(data_h.data(), len); + + cpu_transpose_col_major(data_h.data(), data_ref_h.data(), params.n_row, params.n_col); + + raft::update_device(data.data(), data_h.data(), len, stream); + raft::update_device(data_trans_ref.data(), data_ref_h.data(), len, stream); transpose(handle, data.data(), data_trans.data(), params.n_row, params.n_col, stream); - transpose(data.data(), params.n_row, stream); + if (params.n_row == params.n_col) { transpose(data.data(), params.n_col, stream); } resource::sync_stream(handle, stream); } @@ -78,28 +134,45 @@ class TransposeTest : public ::testing::TestWithParam> { raft::resources handle; cudaStream_t stream; - TranposeInputs params; + TransposeInputs params; rmm::device_uvector data, data_trans, data_trans_ref; }; -const std::vector> inputsf2 = {{0.1f, 3 * 3, 3, 3, 1234ULL}}; - -const std::vector> inputsd2 = {{0.1, 3 * 3, 3, 3, 1234ULL}}; - -const std::vector> inputsh2 = {{0.1, 3 * 3, 3, 3, 1234ULL}}; +const std::vector> inputsf2 = {{0.1f, 3, 3, 1234ULL}, + {0.1f, 3, 4, 1234ULL}, + {0.1f, 300, 300, 1234ULL}, + {0.1f, 300, 4100, 1234ULL}, + {0.1f, 1, 13000, 1234ULL}, + {0.1f, 3, 130001, 1234ULL}}; + +const std::vector> inputsd2 = {{0.1f, 3, 3, 1234ULL}, + {0.1f, 3, 4, 1234ULL}, + {0.1f, 300, 300, 1234ULL}, + {0.1f, 300, 4100, 1234ULL}, + {0.1f, 1, 13000, 1234ULL}, + {0.1f, 3, 130001, 1234ULL}}; + +const std::vector> inputsh2 = {{0.1f, 3, 3, 1234ULL}, + {0.1f, 3, 4, 1234ULL}, + {0.1f, 300, 300, 1234ULL}, + {0.1f, 300, 4100, 1234ULL}, + {0.1f, 1, 13000, 1234ULL}, + {0.1f, 3, 130001, 1234ULL}}; typedef TransposeTest TransposeTestValF; TEST_P(TransposeTestValF, Result) { ASSERT_TRUE(raft::devArrMatch(data_trans_ref.data(), data_trans.data(), - params.len, + params.n_row * params.n_col, raft::CompareApproxAbs(params.tolerance))); - ASSERT_TRUE(raft::devArrMatch(data_trans_ref.data(), - data.data(), - params.len, - raft::CompareApproxAbs(params.tolerance))); + if (params.n_row == params.n_col) { + ASSERT_TRUE(raft::devArrMatch(data_trans_ref.data(), + data.data(), + params.n_row * params.n_col, + raft::CompareApproxAbs(params.tolerance))); + } } typedef TransposeTest TransposeTestValD; @@ -107,59 +180,47 @@ TEST_P(TransposeTestValD, Result) { ASSERT_TRUE(raft::devArrMatch(data_trans_ref.data(), data_trans.data(), - params.len, - raft::CompareApproxAbs(params.tolerance))); - - ASSERT_TRUE(raft::devArrMatch(data_trans_ref.data(), - data.data(), - params.len, + params.n_row * params.n_col, raft::CompareApproxAbs(params.tolerance))); -} - -bool validate_half(const half* h_ref, const half* h_result, half tolerance, int len) -{ - bool success = true; - for (int i = 0; i < len; ++i) { - if (raft::abs(__half2float(h_result[i]) - __half2float(h_ref[i])) >= __half2float(tolerance)) { - success = false; - break; - } - if (!success) break; + if (params.n_row == params.n_col) { + ASSERT_TRUE(raft::devArrMatch(data_trans_ref.data(), + data.data(), + params.n_row * params.n_col, + raft::CompareApproxAbs(params.tolerance))); } - return success; } typedef TransposeTest TransposeTestValH; TEST_P(TransposeTestValH, Result) { - half data_trans_ref_h[params.len]; - half data_trans_h[params.len]; - half data_h[params.len]; + auto len = params.n_row * params.n_col; - RAFT_CUDA_TRY(cudaMemcpyAsync(data_trans_ref_h, - data_trans_ref.data(), - params.len * sizeof(half), - cudaMemcpyDeviceToHost, - stream)); - - RAFT_CUDA_TRY(cudaMemcpyAsync( - data_trans_h, data_trans.data(), params.len * sizeof(half), cudaMemcpyDeviceToHost, stream)); - RAFT_CUDA_TRY(cudaMemcpyAsync( - data_h, data.data(), params.len * sizeof(half), cudaMemcpyDeviceToHost, stream)); + std::vector data_trans_ref_h(len); + std::vector data_trans_h(len); + std::vector data_h(len); + raft::copy( + data_trans_ref_h.data(), data_trans_ref.data(), len, resource::get_cuda_stream(handle)); + raft::copy(data_trans_h.data(), data_trans.data(), len, resource::get_cuda_stream(handle)); + raft::copy(data_h.data(), data.data(), len, resource::get_cuda_stream(handle)); resource::sync_stream(handle, stream); - ASSERT_TRUE(validate_half(data_trans_ref_h, data_trans_h, params.tolerance, params.len)); - ASSERT_TRUE(validate_half(data_trans_ref_h, data_h, params.tolerance, params.len)); + ASSERT_TRUE(validate_half( + data_trans_ref_h.data(), data_trans_h.data(), params.tolerance, params.n_row * params.n_col)); + + if (params.n_row == params.n_col) { + ASSERT_TRUE(validate_half( + data_trans_ref_h.data(), data_h.data(), params.tolerance, params.n_row * params.n_col)); + } } INSTANTIATE_TEST_SUITE_P(TransposeTests, TransposeTestValF, ::testing::ValuesIn(inputsf2)); - INSTANTIATE_TEST_SUITE_P(TransposeTests, TransposeTestValD, ::testing::ValuesIn(inputsd2)); - INSTANTIATE_TEST_SUITE_P(TransposeTests, TransposeTestValH, ::testing::ValuesIn(inputsh2)); +} // namespace transpose_regular_test + +namespace transpose_extra_test { -namespace { /** * We hide these functions in tests for now until we have a heterogeneous mdarray * implementation. @@ -230,79 +291,225 @@ template } } +template +struct TransposeMdspanInputs { + int n_row; + int n_col; + T tolerance = T{0.01}; +}; + template -void test_transpose_with_mdspan() +void test_transpose_with_mdspan(const TransposeMdspanInputs& param) { + auto len = param.n_row * param.n_col; + std::vector in_h(len); + std::vector out_ref_h(len); + + initialize_array(in_h.data(), len); + raft::resources handle; - auto v = make_device_matrix(handle, 32, 3); - T k{0}; - for (size_t i = 0; i < v.extent(0); ++i) { - for (size_t j = 0; j < v.extent(1); ++j) { - v(i, j) = k++; - } + auto stream = resource::get_cuda_stream(handle); + auto in = make_device_matrix(handle, param.n_row, param.n_col); + auto out_ref = make_device_matrix(handle, param.n_row, param.n_col); + resource::sync_stream(handle, stream); + if constexpr (std::is_same_v) { + cpu_transpose_row_major(in_h.data(), out_ref_h.data(), param.n_row, param.n_col); + } else { + cpu_transpose_col_major(in_h.data(), out_ref_h.data(), param.n_row, param.n_col); } - auto out = transpose(handle, v.view()); - static_assert(std::is_same_v); - ASSERT_EQ(out.extent(0), v.extent(1)); - ASSERT_EQ(out.extent(1), v.extent(0)); + raft::copy(in.data_handle(), in_h.data(), len, resource::get_cuda_stream(handle)); + raft::copy(out_ref.data_handle(), out_ref_h.data(), len, resource::get_cuda_stream(handle)); - k = 0; - for (size_t i = 0; i < out.extent(1); ++i) { - for (size_t j = 0; j < out.extent(0); ++j) { - ASSERT_EQ(out(j, i), k++); - } + auto out = transpose(handle, in.view()); + static_assert(std::is_same_v); + ASSERT_EQ(out.extent(0), in.extent(1)); + ASSERT_EQ(out.extent(1), in.extent(0)); + if constexpr (std::is_same_v) { + std::vector out_h(len); + raft::copy(out_h.data(), out.data_handle(), len, resource::get_cuda_stream(handle)); + ASSERT_TRUE(validate_half(out_ref_h.data(), out_h.data(), param.tolerance, len)); + } else { + ASSERT_TRUE(raft::devArrMatch( + out_ref.data_handle(), out.data_handle(), len, raft::CompareApproxAbs(param.tolerance))); } } -} // namespace -TEST(TransposeTest, MDSpan) +const std::vector> inputs_mdspan_f = {{3, 3}, + {3, 4}, + {300, 300}, + {300, 4100}, + {1, 13000}, + {3, 130001}, + {4100, 300}, + {13000, 1}, + {130001, 3}}; +const std::vector> inputs_mdspan_d = {{3, 3}, + {3, 4}, + {300, 300}, + {300, 4100}, + {1, 13000}, + {3, 130001}, + {4100, 300}, + {13000, 1}, + {130001, 3}}; +const std::vector> inputs_mdspan_h = {{3, 3}, + {3, 4}, + {300, 300}, + {300, 4100}, + {1, 13000}, + {3, 130001}, + {4100, 300}, + {13000, 1}, + {130001, 3}}; + +TEST(TransposeTest, MDSpanFloat) { - test_transpose_with_mdspan(); - test_transpose_with_mdspan(); - - test_transpose_with_mdspan(); - test_transpose_with_mdspan(); + for (const auto& p : inputs_mdspan_f) { + test_transpose_with_mdspan(p); + test_transpose_with_mdspan(p); + } +} +TEST(TransposeTest, MDSpanDouble) +{ + for (const auto& p : inputs_mdspan_d) { + test_transpose_with_mdspan(p); + test_transpose_with_mdspan(p); + } +} +TEST(TransposeTest, MDSpanHalf) +{ + for (const auto& p : inputs_mdspan_h) { + test_transpose_with_mdspan(p); + test_transpose_with_mdspan(p); + } } -namespace { +template +struct TransposeSubmatrixInputs { + int n_row; + int n_col; + int row_beg; + int row_end; + int col_beg; + int col_end; + T tolerance = T{0.01}; +}; + template -void test_transpose_submatrix() +void test_transpose_submatrix(const TransposeSubmatrixInputs& param) { + auto len = param.n_row * param.n_col; + auto sub_len = (param.row_end - param.row_beg) * (param.col_end - param.col_beg); + + std::vector in_h(len); + std::vector out_ref_h(sub_len); + + initialize_array(in_h.data(), len); + raft::resources handle; - auto v = make_device_matrix(handle, 32, 33); - T k{0}; - size_t row_beg{3}, row_end{13}, col_beg{2}, col_end{11}; - for (size_t i = row_beg; i < row_end; ++i) { - for (size_t j = col_beg; j < col_end; ++j) { - v(i, j) = k++; - } + auto stream = resource::get_cuda_stream(handle); + + auto in = make_device_matrix(handle, param.n_row, param.n_col); + auto out_ref = make_device_matrix( + handle, (param.row_end - param.row_beg), (param.col_end - param.col_beg)); + + if constexpr (std::is_same_v) { + auto offset = param.row_beg * param.n_col + param.col_beg; + cpu_transpose_row_major(in_h.data() + offset, + out_ref_h.data(), + (param.row_end - param.row_beg), + (param.col_end - param.col_beg), + in.extent(1), + (param.row_end - param.row_beg)); + } else { + auto offset = param.col_beg * param.n_row + param.row_beg; + cpu_transpose_col_major(in_h.data() + offset, + out_ref_h.data(), + (param.row_end - param.row_beg), + (param.col_end - param.col_beg), + in.extent(0), + (param.col_end - param.col_beg)); } - auto vv = v.view(); - auto submat = std::experimental::submdspan( - vv, std::make_tuple(row_beg, row_end), std::make_tuple(col_beg, col_end)); - static_assert(std::is_same_v); + raft::copy(in.data_handle(), in_h.data(), len, resource::get_cuda_stream(handle)); + raft::copy(out_ref.data_handle(), out_ref_h.data(), sub_len, resource::get_cuda_stream(handle)); + resource::sync_stream(handle, stream); - auto out = transpose(handle, submat); - ASSERT_EQ(out.extent(0), submat.extent(1)); - ASSERT_EQ(out.extent(1), submat.extent(0)); + auto in_submat = std::experimental::submdspan(in.view(), + std::make_tuple(param.row_beg, param.row_end), + std::make_tuple(param.col_beg, param.col_end)); - k = 0; - for (size_t i = 0; i < out.extent(1); ++i) { - for (size_t j = 0; j < out.extent(0); ++j) { - ASSERT_EQ(out(j, i), k++); - } + static_assert(std::is_same_v); + auto out = transpose(handle, in_submat); + + ASSERT_EQ(out.extent(0), in_submat.extent(1)); + ASSERT_EQ(out.extent(1), in_submat.extent(0)); + + if constexpr (std::is_same_v) { + std::vector out_h(sub_len); + + raft::copy(out_h.data(), out.data_handle(), sub_len, resource::get_cuda_stream(handle)); + ASSERT_TRUE(validate_half(out_ref_h.data(), out_h.data(), param.tolerance, sub_len)); + } else { + ASSERT_TRUE(raft::devArrMatch(out_ref.data_handle(), + out.data_handle(), + sub_len, + raft::CompareApproxAbs(param.tolerance))); } } -} // namespace - -TEST(TransposeTest, SubMatrix) +const std::vector> inputs_submatrix_f = { + {3, 3, 1, 2, 0, 2}, + {3, 4, 1, 3, 2, 3}, + {300, 300, 1, 299, 2, 239}, + {300, 4100, 3, 299, 101, 4001}, + {2, 13000, 0, 1, 3, 13000}, + {3, 130001, 0, 3, 3999, 129999}, + {4100, 300, 159, 4001, 125, 300}, + {13000, 5, 0, 11111, 0, 3}, + {130001, 3, 19, 130000, 2, 3}}; +const std::vector> inputs_submatrix_d = { + {3, 3, 1, 2, 0, 2}, + {3, 4, 1, 3, 2, 3}, + {300, 300, 1, 299, 2, 239}, + {300, 4100, 3, 299, 101, 4001}, + {2, 13000, 0, 1, 3, 13000}, + {3, 130001, 0, 3, 3999, 129999}, + {4100, 300, 159, 4001, 125, 300}, + {13000, 5, 0, 11111, 0, 3}, + {130001, 3, 19, 130000, 2, 3}}; +const std::vector> inputs_submatrix_h = { + {3, 3, 1, 2, 0, 2}, + {3, 4, 1, 3, 2, 3}, + {300, 300, 1, 299, 2, 239}, + {300, 4100, 3, 299, 101, 4001}, + {2, 13000, 0, 1, 3, 13000}, + {3, 130001, 0, 3, 3999, 129999}, + {4100, 300, 159, 4001, 125, 300}, + {13000, 5, 0, 11111, 0, 3}, + {130001, 3, 19, 130000, 2, 3}}; + +TEST(TransposeTest, SubMatrixFloat) { - test_transpose_submatrix(); - test_transpose_submatrix(); - - test_transpose_submatrix(); - test_transpose_submatrix(); + for (const auto& p : inputs_submatrix_f) { + test_transpose_submatrix(p); + test_transpose_submatrix(p); + } } +TEST(TransposeTest, SubMatrixDouble) +{ + for (const auto& p : inputs_submatrix_d) { + test_transpose_submatrix(p); + test_transpose_submatrix(p); + } +} +TEST(TransposeTest, SubMatrixHalf) +{ + for (const auto& p : inputs_submatrix_h) { + test_transpose_submatrix(p); + test_transpose_submatrix(p); + } +} + +} // namespace transpose_extra_test } // end namespace linalg } // end namespace raft From b78a07d9d2a117c12f396f330526a8e8ed23a9ba Mon Sep 17 00:00:00 2001 From: Victor Lafargue Date: Fri, 27 Sep 2024 00:36:44 +0200 Subject: [PATCH 34/79] Adding NCCL clique to the RAFT handle (#2431) Authors: - Victor Lafargue (https://github.com/viclafargue) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/raft/pull/2431 --- cpp/include/raft/comms/detail/std_comms.hpp | 8 +- cpp/include/raft/comms/nccl_clique.hpp | 156 ++++++++++++++++++ .../raft/core/resource/nccl_clique.hpp | 66 ++++++++ .../raft/core/resource/resource_types.hpp | 1 + 4 files changed, 227 insertions(+), 4 deletions(-) create mode 100644 cpp/include/raft/comms/nccl_clique.hpp create mode 100644 cpp/include/raft/core/resource/nccl_clique.hpp diff --git a/cpp/include/raft/comms/detail/std_comms.hpp b/cpp/include/raft/comms/detail/std_comms.hpp index c5d64f6a29..ed869e6cae 100644 --- a/cpp/include/raft/comms/detail/std_comms.hpp +++ b/cpp/include/raft/comms/detail/std_comms.hpp @@ -310,13 +310,13 @@ class std_comms : public comms_iface { // Wait for a UCXX progress thread roundtrip, prevent waiting for longer // than 10ms for each operation, will retry in next iteration. ucxx::utils::CallbackNotifier callbackNotifierPre{}; - worker->registerGenericPre([&callbackNotifierPre]() { callbackNotifierPre.set(); }, - 10000000 /* 10ms */); + (void)worker->registerGenericPre( + [&callbackNotifierPre]() { callbackNotifierPre.set(); }, 10000000 /* 10ms */); callbackNotifierPre.wait(); ucxx::utils::CallbackNotifier callbackNotifierPost{}; - worker->registerGenericPost([&callbackNotifierPost]() { callbackNotifierPost.set(); }, - 10000000 /* 10ms */); + (void)worker->registerGenericPost( + [&callbackNotifierPost]() { callbackNotifierPost.set(); }, 10000000 /* 10ms */); callbackNotifierPost.wait(); } else { // Causes UCXX to progress through the send/recv message queue diff --git a/cpp/include/raft/comms/nccl_clique.hpp b/cpp/include/raft/comms/nccl_clique.hpp new file mode 100644 index 0000000000..c6520af753 --- /dev/null +++ b/cpp/include/raft/comms/nccl_clique.hpp @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include + +#include + +/** + * @brief Error checking macro for NCCL runtime API functions. + * + * Invokes a NCCL runtime API function call, if the call does not return ncclSuccess, throws an + * exception detailing the NCCL error that occurred + */ +#define RAFT_NCCL_TRY(call) \ + do { \ + ncclResult_t const status = (call); \ + if (ncclSuccess != status) { \ + std::string msg{}; \ + SET_ERROR_MSG(msg, \ + "NCCL error encountered at: ", \ + "call='%s', Reason=%d:%s", \ + #call, \ + status, \ + ncclGetErrorString(status)); \ + throw raft::logic_error(msg); \ + } \ + } while (0); + +namespace raft::comms { +void build_comms_nccl_only(raft::resources* handle, ncclComm_t nccl_comm, int num_ranks, int rank); +} + +namespace raft::comms { + +struct nccl_clique { + using pool_mr = rmm::mr::pool_memory_resource; + + /** + * Instantiates a NCCL clique with all available GPUs + * + * @param[in] percent_of_free_memory percentage of device memory to pre-allocate as memory pool + * + */ + nccl_clique(int percent_of_free_memory = 80) + : root_rank_(0), + percent_of_free_memory_(percent_of_free_memory), + per_device_pools_(0), + device_resources_(0) + { + cudaGetDeviceCount(&num_ranks_); + device_ids_.resize(num_ranks_); + std::iota(device_ids_.begin(), device_ids_.end(), 0); + nccl_comms_.resize(num_ranks_); + nccl_clique_init(); + } + + /** + * Instantiates a NCCL clique + * + * Usage example: + * @code{.cpp} + * int n_devices; + * cudaGetDeviceCount(&n_devices); + * std::vector device_ids(n_devices); + * std::iota(device_ids.begin(), device_ids.end(), 0); + * cuvs::neighbors::mg::nccl_clique& clique(device_ids); // first device is the root rank + * @endcode + * + * @param[in] device_ids list of device IDs to be used to initiate the clique + * @param[in] percent_of_free_memory percentage of device memory to pre-allocate as memory pool + * + */ + nccl_clique(const std::vector& device_ids, int percent_of_free_memory = 80) + : root_rank_(0), + num_ranks_(device_ids.size()), + percent_of_free_memory_(percent_of_free_memory), + device_ids_(device_ids), + nccl_comms_(device_ids.size()), + per_device_pools_(0), + device_resources_(0) + { + nccl_clique_init(); + } + + void nccl_clique_init() + { + RAFT_NCCL_TRY(ncclCommInitAll(nccl_comms_.data(), num_ranks_, device_ids_.data())); + + for (int rank = 0; rank < num_ranks_; rank++) { + RAFT_CUDA_TRY(cudaSetDevice(device_ids_[rank])); + + // create a pool memory resource for each device + auto old_mr = rmm::mr::get_current_device_resource(); + per_device_pools_.push_back(std::make_unique( + old_mr, rmm::percent_of_free_device_memory(percent_of_free_memory_))); + rmm::cuda_device_id id(device_ids_[rank]); + rmm::mr::set_per_device_resource(id, per_device_pools_.back().get()); + + // create a device resource handle for each device + device_resources_.emplace_back(); + + // add NCCL communications to the device resource handle + raft::comms::build_comms_nccl_only( + &device_resources_[rank], nccl_comms_[rank], num_ranks_, rank); + } + + for (int rank = 0; rank < num_ranks_; rank++) { + RAFT_CUDA_TRY(cudaSetDevice(device_ids_[rank])); + raft::resource::sync_stream(device_resources_[rank]); + } + } + + const raft::device_resources& set_current_device_to_root_rank() const + { + int root_device_id = device_ids_[root_rank_]; + RAFT_CUDA_TRY(cudaSetDevice(root_device_id)); + return device_resources_[root_rank_]; + } + + ~nccl_clique() + { +#pragma omp parallel for // necessary to avoid hangs + for (int rank = 0; rank < num_ranks_; rank++) { + cudaSetDevice(device_ids_[rank]); + ncclCommDestroy(nccl_comms_[rank]); + rmm::cuda_device_id id(device_ids_[rank]); + rmm::mr::set_per_device_resource(id, nullptr); + } + } + + int root_rank_; + int num_ranks_; + int percent_of_free_memory_; + std::vector device_ids_; + std::vector nccl_comms_; + std::vector> per_device_pools_; + std::vector device_resources_; +}; + +} // namespace raft::comms diff --git a/cpp/include/raft/core/resource/nccl_clique.hpp b/cpp/include/raft/core/resource/nccl_clique.hpp new file mode 100644 index 0000000000..edda5043ae --- /dev/null +++ b/cpp/include/raft/core/resource/nccl_clique.hpp @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include + +#include + +namespace raft::resource { + +class nccl_clique_resource : public resource { + public: + nccl_clique_resource() : clique_(std::make_unique()) {} + ~nccl_clique_resource() override {} + void* get_resource() override { return clique_.get(); } + + private: + std::unique_ptr clique_; +}; + +/** Factory that knows how to construct a specific raft::resource to populate the res_t. */ +class nccl_clique_resource_factory : public resource_factory { + public: + resource_type get_resource_type() override { return resource_type::NCCL_CLIQUE; } + resource* make_resource() override { return new nccl_clique_resource(); } +}; + +/** + * @defgroup nccl_clique_resource resource functions + * @{ + */ + +/** + * Retrieves a NCCL clique from raft res if it exists, otherwise initializes it and return it. + * + * @param[in] res the raft resources object + * @return NCCL clique + */ +inline const raft::comms::nccl_clique& get_nccl_clique(resources const& res) +{ + if (!res.has_resource_factory(resource_type::NCCL_CLIQUE)) { + res.add_resource_factory(std::make_shared()); + } + return *res.get_resource(resource_type::NCCL_CLIQUE); +}; + +/** + * @} + */ + +} // namespace raft::resource diff --git a/cpp/include/raft/core/resource/resource_types.hpp b/cpp/include/raft/core/resource/resource_types.hpp index d9126251c9..4fa84c3bdb 100644 --- a/cpp/include/raft/core/resource/resource_types.hpp +++ b/cpp/include/raft/core/resource/resource_types.hpp @@ -46,6 +46,7 @@ enum resource_type { CUBLASLT_HANDLE, // cublasLt handle CUSTOM, // runtime-shared default-constructible resource LARGE_WORKSPACE_RESOURCE, // rmm device memory resource for somewhat large temporary allocations + NCCL_CLIQUE, // nccl clique LAST_KEY // reserved for the last key }; From 5ee0e79a7aa033d1f312e805fca1194cdf0483b3 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 26 Sep 2024 18:22:10 -0500 Subject: [PATCH 35/79] bump NCCL floor to 2.19 (#2458) Follow-up to #2443 As part of the work to support NumPy 2 across RAPIDS, we found reason to upgrade some libraries like `cugraph` to slightly newer NCCL (`>=2.19`). Context: https://github.com/rapidsai/build-planning/issues/102#issuecomment-2375595743 This applies that same bump here, to keep the range of NCCL versions consistent across RAPIDS. Authors: - James Lamb (https://github.com/jameslamb) Approvers: - https://github.com/jakirkham - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/raft/pull/2458 --- conda/environments/all_cuda-118_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-125_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-125_arch-x86_64.yaml | 2 +- conda/environments/bench_ann_cuda-118_arch-aarch64.yaml | 2 +- conda/environments/bench_ann_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/bench_ann_cuda-120_arch-aarch64.yaml | 2 +- conda/environments/bench_ann_cuda-120_arch-x86_64.yaml | 2 +- conda/recipes/libraft/conda_build_config.yaml | 2 +- conda/recipes/raft-ann-bench/conda_build_config.yaml | 2 +- conda/recipes/raft-dask/conda_build_config.yaml | 2 +- dependencies.yaml | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index de4eb7e690..0b84772fad 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -36,7 +36,7 @@ dependencies: - libcusparse-dev=11.7.5.86 - libcusparse=11.7.5.86 - libucxx==0.40.*,>=0.0.0a0 -- nccl>=2.18.1.1 +- nccl>=2.19 - ninja - numba>=0.57 - numpy>=1.23,<3.0a0 diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 26f4c1efaa..d1c01f1b16 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -36,7 +36,7 @@ dependencies: - libcusparse-dev=11.7.5.86 - libcusparse=11.7.5.86 - libucxx==0.40.*,>=0.0.0a0 -- nccl>=2.18.1.1 +- nccl>=2.19 - ninja - numba>=0.57 - numpy>=1.23,<3.0a0 diff --git a/conda/environments/all_cuda-125_arch-aarch64.yaml b/conda/environments/all_cuda-125_arch-aarch64.yaml index 692956502b..4c506f5297 100644 --- a/conda/environments/all_cuda-125_arch-aarch64.yaml +++ b/conda/environments/all_cuda-125_arch-aarch64.yaml @@ -33,7 +33,7 @@ dependencies: - libcusolver-dev - libcusparse-dev - libucxx==0.40.*,>=0.0.0a0 -- nccl>=2.18.1.1 +- nccl>=2.19 - ninja - numba>=0.57 - numpy>=1.23,<3.0a0 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 133d42bfee..a123950e3a 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -33,7 +33,7 @@ dependencies: - libcusolver-dev - libcusparse-dev - libucxx==0.40.*,>=0.0.0a0 -- nccl>=2.18.1.1 +- nccl>=2.19 - ninja - numba>=0.57 - numpy>=1.23,<3.0a0 diff --git a/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml index f99cedd627..864eb2130b 100644 --- a/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml +++ b/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml @@ -32,7 +32,7 @@ dependencies: - libcusparse=11.7.5.86 - libucxx==0.40.*,>=0.0.0a0 - matplotlib -- nccl>=2.18.1.1 +- nccl>=2.19 - ninja - nlohmann_json>=3.11.2 - nvcc_linux-aarch64=11.8 diff --git a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml index 08aea32ab1..5da6eaf17e 100644 --- a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml +++ b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml @@ -32,7 +32,7 @@ dependencies: - libcusparse=11.7.5.86 - libucxx==0.40.*,>=0.0.0a0 - matplotlib -- nccl>=2.18.1.1 +- nccl>=2.19 - ninja - nlohmann_json>=3.11.2 - nvcc_linux-64=11.8 diff --git a/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml index 572ad85ab5..65de97c170 100644 --- a/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml +++ b/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml @@ -29,7 +29,7 @@ dependencies: - libcusparse-dev - libucxx==0.40.*,>=0.0.0a0 - matplotlib -- nccl>=2.18.1.1 +- nccl>=2.19 - ninja - nlohmann_json>=3.11.2 - openblas diff --git a/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml index 9fd23edb03..7e1adbc483 100644 --- a/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml +++ b/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml @@ -29,7 +29,7 @@ dependencies: - libcusparse-dev - libucxx==0.40.*,>=0.0.0a0 - matplotlib -- nccl>=2.18.1.1 +- nccl>=2.19 - ninja - nlohmann_json>=3.11.2 - openblas diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index 6dc06648bd..bc0ff1fae7 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -20,7 +20,7 @@ cmake_version: - ">=3.26.4,!=3.30.0" nccl_version: - - ">=2.18.1.1" + - ">=2.19" glog_version: - ">=0.6.0" diff --git a/conda/recipes/raft-ann-bench/conda_build_config.yaml b/conda/recipes/raft-ann-bench/conda_build_config.yaml index bdb4e883ea..47bd730daf 100644 --- a/conda/recipes/raft-ann-bench/conda_build_config.yaml +++ b/conda/recipes/raft-ann-bench/conda_build_config.yaml @@ -20,7 +20,7 @@ cmake_version: - ">=3.26.4,!=3.30.0" nccl_version: - - ">=2.18.1.1" + - ">=2.19" glog_version: - ">=0.6.0" diff --git a/conda/recipes/raft-dask/conda_build_config.yaml b/conda/recipes/raft-dask/conda_build_config.yaml index 58e8ec3c9e..65c589fc0c 100644 --- a/conda/recipes/raft-dask/conda_build_config.yaml +++ b/conda/recipes/raft-dask/conda_build_config.yaml @@ -26,4 +26,4 @@ cmake_version: - ">=3.26.4,!=3.30.0" nccl_version: - - ">=2.18.1.1" + - ">=2.19" diff --git a/dependencies.yaml b/dependencies.yaml index e833e8519a..d0991f4d04 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -170,7 +170,7 @@ dependencies: packages: - c-compiler - cxx-compiler - - nccl>=2.18.1.1 + - nccl>=2.19 - libucxx==0.40.*,>=0.0.0a0 specific: - output_types: conda From 6c4fdfb3bfa32c3f08d03adad0d461c167f277ce Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Thu, 26 Sep 2024 21:25:22 -0400 Subject: [PATCH 36/79] Deprecating vector search APIs and updating README accordingly (#2448) I opted to deprecate just the necessary pieces, such as the `index` classes instead of deprecating every single function. Authors: - Corey J. Nolet (https://github.com/cjnolet) Approvers: - Ben Frederickson (https://github.com/benfred) URL: https://github.com/rapidsai/raft/pull/2448 --- README.md | 111 +-------------- cpp/include/raft/cluster/kmeans.cuh | 63 +++++---- cpp/include/raft/cluster/kmeans_balanced.cuh | 68 ++++----- cpp/include/raft/cluster/single_linkage.cuh | 33 ++--- cpp/include/raft/distance/distance-ext.cuh | 133 +++++++++--------- cpp/include/raft/neighbors/ball_cover.cuh | 1 + .../raft/neighbors/brute_force_types.hpp | 32 +++-- cpp/include/raft/neighbors/cagra_types.hpp | 14 +- cpp/include/raft/neighbors/hnsw_types.hpp | 6 +- cpp/include/raft/neighbors/ivf_flat_types.hpp | 16 ++- cpp/include/raft/neighbors/ivf_pq_types.hpp | 20 +-- .../raft/neighbors/nn_descent_types.hpp | 16 ++- cpp/include/raft/neighbors/refine-ext.cuh | 32 ++--- docs/source/conf.py | 2 +- docs/source/raft_ann_benchmarks.md | 4 + 15 files changed, 239 insertions(+), 312 deletions(-) diff --git a/README.md b/README.md index 25ce059630..af2219fdd1 100755 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ #
 RAFT: Reusable Accelerated Functions and Tools for Vector Search and More
> [!IMPORTANT] -> The vector search and clustering algorithms in RAFT are being migrated to a new library dedicated to vector search called [cuVS](https://github.com/rapidsai/cuvs). We will continue to support the vector search algorithms in RAFT during this move, but will no longer update them after the RAPIDS 24.06 (June) release. We plan to complete the migration by RAPIDS 24.08 (August) release. +> The vector search and clustering algorithms in RAFT are being migrated to a new library dedicated to vector search called [cuVS](https://github.com/rapidsai/cuvs). We will continue to support the vector search algorithms in RAFT during this move, but will no longer update them after the RAPIDS 24.06 (June) release. We plan to complete the migration by RAPIDS 24.10 (October) release and will be removing them altogether in the 24.12 (December) release. ![RAFT tech stack](img/raft-tech-stack-vss.png) @@ -36,7 +36,7 @@ ## What is RAFT? -RAFT contains fundamental widely-used algorithms and primitives for machine learning and information retrieval. The algorithms are CUDA-accelerated and form building blocks for more easily writing high performance applications. +RAFT contains fundamental widely-used algorithms and primitives for machine learning and data mining. The algorithms are CUDA-accelerated and form building blocks for more easily writing high performance applications. By taking a primitives-based approach to algorithm development, RAFT - accelerates algorithm construction time @@ -47,12 +47,10 @@ While not exhaustive, the following general categories help summarize the accele ##### | Category | Accelerated Functions in RAFT | |-----------------------|-----------------------------------------------------------------------------------------------------------------------------------| -| **Nearest Neighbors** | vector search, neighborhood graph construction, epsilon neighborhoods, pairwise distances | -| **Basic Clustering** | spectral clustering, hierarchical clustering, k-means | -| **Solvers** | combinatorial optimization, iterative solvers | | **Data Formats** | sparse & dense, conversions, data generation | | **Dense Operations** | linear algebra, matrix and vector operations, reductions, slicing, norms, factorization, least squares, svd & eigenvalue problems | | **Sparse Operations** | linear algebra, eigenvalue problems, slicing, norms, reductions, factorization, symmetrization, components & labeling | +| **Solvers** | combinatorial optimization, iterative solvers | | **Statistics** | sampling, moments and summary statistics, metrics, model evaluation | | **Tools & Utilities** | common tools and utilities for developing CUDA applications, multi-node multi-gpu infrastructure | @@ -67,42 +65,6 @@ In addition being a C++ library, RAFT also provides 2 Python libraries: ![RAFT is a C++ header-only template library with optional shared library and lightweight Python wrappers](img/arch.png) -## Use cases - -### Vector Similarity Search - -RAFT contains state-of-the-art implementations of approximate nearest neighbors search (ANNS) algorithms on the GPU, such as: - -* [Brute force](https://docs.rapids.ai/api/raft/nightly/pylibraft_api/neighbors/#brute-force). Performs a brute force nearest neighbors search without an index. -* [IVF-Flat](https://docs.rapids.ai/api/raft/nightly/pylibraft_api/neighbors/#ivf-flat) and [IVF-PQ](https://docs.rapids.ai/api/raft/nightly/pylibraft_api/neighbors/#ivf-pq). Use an inverted file index structure to map contents to their locations. IVF-PQ additionally uses product quantization to reduce the memory usage of vectors. These methods were originally popularized by the [FAISS](https://github.com/facebookresearch/faiss) library. -* [CAGRA](https://docs.rapids.ai/api/raft/nightly/pylibraft_api/neighbors/#cagra) (Cuda Anns GRAph-based). Uses a fast ANNS graph construction and search implementation optimized for the GPU. CAGRA outperforms state-of-the art CPU methods (i.e. HNSW) for large batch queries, single queries, and graph construction time. - -Projects that use the RAFT ANNS algorithms for accelerating vector search include: [Milvus](https://milvus.io/), [Redis](https://redis.io/), and [Faiss](https://github.com/facebookresearch/faiss). - -Please see the example [Jupyter notebook](https://github.com/rapidsai/raft/blob/HEAD/notebooks/VectorSearch_QuestionRetrieval.ipynb) to get started RAFT for vector search in Python. - - - -### Information Retrieval - -RAFT contains a catalog of reusable primitives for composing algorithms that require fast neighborhood computations, such as - -1. Computing distances between vectors and computing kernel gramm matrices -2. Performing ball radius queries for constructing epsilon neighborhoods -3. Clustering points to partition a space for smaller and faster searches -4. Constructing neighborhood "connectivities" graphs from dense vectors - -### Machine Learning - -RAFT's primitives are used in several RAPIDS libraries, including [cuML](https://github.com/rapidsai/cuml), [cuGraph](https://github.com/rapidsai/cugraph), and [cuOpt](https://github.com/rapidsai/cuopt) to build many end-to-end machine learning algorithms that span a large spectrum of different applications, including -- data generation -- model evaluation -- classification and regression -- clustering -- manifold learning -- dimensionality reduction. - -RAFT is also used by the popular collaborative filtering library [implicit](https://github.com/benfred/implicit) for recommender systems. ## Is RAFT right for me? @@ -327,70 +289,3 @@ When citing RAFT generally, please consider referencing this Github project. year={2022} } ``` -If citing the sparse pairwise distances API, please consider using the following bibtex: -```bibtex -@article{nolet2021semiring, - title={Semiring primitives for sparse neighborhood methods on the gpu}, - author={Nolet, Corey J and Gala, Divye and Raff, Edward and Eaton, Joe and Rees, Brad and Zedlewski, John and Oates, Tim}, - journal={arXiv preprint arXiv:2104.06357}, - year={2021} -} -``` - -If citing the single-linkage agglomerative clustering APIs, please consider the following bibtex: -```bibtex -@misc{nolet2023cuslink, - title={cuSLINK: Single-linkage Agglomerative Clustering on the GPU}, - author={Corey J. Nolet and Divye Gala and Alex Fender and Mahesh Doijade and Joe Eaton and Edward Raff and John Zedlewski and Brad Rees and Tim Oates}, - year={2023}, - eprint={2306.16354}, - archivePrefix={arXiv}, - primaryClass={cs.LG} -} -``` - -If citing CAGRA, please consider the following bibtex: -```bibtex -@misc{ootomo2023cagra, - title={CAGRA: Highly Parallel Graph Construction and Approximate Nearest Neighbor Search for GPUs}, - author={Hiroyuki Ootomo and Akira Naruse and Corey Nolet and Ray Wang and Tamas Feher and Yong Wang}, - year={2024}, - series = {ICDE '24} -} -``` - -If citing the k-selection routines, please consider the following bibtex: - -```bibtex -@proceedings{10.1145/3581784, - title = {Parallel Top-K Algorithms on GPU: A Comprehensive Study and New Methods}, - author={Jingrong Zhang, Akira Naruse, Xipeng Li, and Yong Wang}, - year = {2023}, - isbn = {9798400701092}, - publisher = {Association for Computing Machinery}, - address = {New York, NY, USA}, - location = {Denver, CO, USA}, - series = {SC '23} -} -``` - -If citing the nearest neighbors descent API, please consider the following bibtex: -```bibtex -@inproceedings{10.1145/3459637.3482344, - author = {Wang, Hui and Zhao, Wan-Lei and Zeng, Xiangxiang and Yang, Jianye}, - title = {Fast K-NN Graph Construction by GPU Based NN-Descent}, - year = {2021}, - isbn = {9781450384469}, - publisher = {Association for Computing Machinery}, - address = {New York, NY, USA}, - url = {https://doi.org/10.1145/3459637.3482344}, - doi = {10.1145/3459637.3482344}, - abstract = {NN-Descent is a classic k-NN graph construction approach. It is still widely employed in machine learning, computer vision, and information retrieval tasks due to its efficiency and genericness. However, the current design only works well on CPU. In this paper, NN-Descent has been redesigned to adapt to the GPU architecture. A new graph update strategy called selective update is proposed. It reduces the data exchange between GPU cores and GPU global memory significantly, which is the processing bottleneck under GPU computation architecture. This redesign leads to full exploitation of the parallelism of the GPU hardware. In the meantime, the genericness, as well as the simplicity of NN-Descent, are well-preserved. Moreover, a procedure that allows to k-NN graph to be merged efficiently on GPU is proposed. It makes the construction of high-quality k-NN graphs for out-of-GPU-memory datasets tractable. Our approach is 100-250\texttimes{} faster than the single-thread NN-Descent and is 2.5-5\texttimes{} faster than the existing GPU-based approaches as we tested on million as well as billion scale datasets.}, - booktitle = {Proceedings of the 30th ACM International Conference on Information \& Knowledge Management}, - pages = {1929–1938}, - numpages = {10}, - keywords = {high-dimensional, nn-descent, gpu, k-nearest neighbor graph}, - location = {Virtual Event, Queensland, Australia}, - series = {CIKM '21} -} -``` diff --git a/cpp/include/raft/cluster/kmeans.cuh b/cpp/include/raft/cluster/kmeans.cuh index eb28cc1626..38318e8ec8 100644 --- a/cpp/include/raft/cluster/kmeans.cuh +++ b/cpp/include/raft/cluster/kmeans.cuh @@ -86,13 +86,14 @@ using KeyValueIndexOp = detail::KeyValueIndexOp; * @param[out] n_iter Number of iterations run. */ template -void fit(raft::resources const& handle, - const KMeansParams& params, - raft::device_matrix_view X, - std::optional> sample_weight, - raft::device_matrix_view centroids, - raft::host_scalar_view inertia, - raft::host_scalar_view n_iter) +[[deprecated("Use cuVS instead")]] void fit( + raft::resources const& handle, + const KMeansParams& params, + raft::device_matrix_view X, + std::optional> sample_weight, + raft::device_matrix_view centroids, + raft::host_scalar_view inertia, + raft::host_scalar_view n_iter) { detail::kmeans_fit(handle, params, X, sample_weight, centroids, inertia, n_iter); } @@ -150,14 +151,15 @@ void fit(raft::resources const& handle, * their closest cluster center. */ template -void predict(raft::resources const& handle, - const KMeansParams& params, - raft::device_matrix_view X, - std::optional> sample_weight, - raft::device_matrix_view centroids, - raft::device_vector_view labels, - bool normalize_weight, - raft::host_scalar_view inertia) +[[deprecated("Use cuVS instead")]] void predict( + raft::resources const& handle, + const KMeansParams& params, + raft::device_matrix_view X, + std::optional> sample_weight, + raft::device_matrix_view centroids, + raft::device_vector_view labels, + bool normalize_weight, + raft::host_scalar_view inertia) { detail::kmeans_predict( handle, params, X, sample_weight, centroids, labels, normalize_weight, inertia); @@ -213,14 +215,15 @@ void predict(raft::resources const& handle, * @param[out] n_iter Number of iterations run. */ template -void fit_predict(raft::resources const& handle, - const KMeansParams& params, - raft::device_matrix_view X, - std::optional> sample_weight, - std::optional> centroids, - raft::device_vector_view labels, - raft::host_scalar_view inertia, - raft::host_scalar_view n_iter) +[[deprecated("Use cuVS instead")]] void fit_predict( + raft::resources const& handle, + const KMeansParams& params, + raft::device_matrix_view X, + std::optional> sample_weight, + std::optional> centroids, + raft::device_vector_view labels, + raft::host_scalar_view inertia, + raft::host_scalar_view n_iter) { detail::kmeans_fit_predict( handle, params, X, sample_weight, centroids, labels, inertia, n_iter); @@ -252,13 +255,13 @@ void transform(raft::resources const& handle, } template -void transform(raft::resources const& handle, - const KMeansParams& params, - const DataT* X, - const DataT* centroids, - IndexT n_samples, - IndexT n_features, - DataT* X_new) +[[deprecated("Use cuVS instead")]] void transform(raft::resources const& handle, + const KMeansParams& params, + const DataT* X, + const DataT* centroids, + IndexT n_samples, + IndexT n_features, + DataT* X_new) { detail::kmeans_transform( handle, params, X, centroids, n_samples, n_features, X_new); diff --git a/cpp/include/raft/cluster/kmeans_balanced.cuh b/cpp/include/raft/cluster/kmeans_balanced.cuh index a1a182608b..7479047fce 100644 --- a/cpp/include/raft/cluster/kmeans_balanced.cuh +++ b/cpp/include/raft/cluster/kmeans_balanced.cuh @@ -73,11 +73,11 @@ namespace raft::cluster::kmeans_balanced { * datatype. If DataT == MathT, this must be the identity. */ template -void fit(const raft::resources& handle, - kmeans_balanced_params const& params, - raft::device_matrix_view X, - raft::device_matrix_view centroids, - MappingOpT mapping_op = raft::identity_op()) +[[deprecated("Use cuVS instead")]] void fit(const raft::resources& handle, + kmeans_balanced_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + MappingOpT mapping_op = raft::identity_op()) { RAFT_EXPECTS(X.extent(1) == centroids.extent(1), "Number of features in dataset and centroids are different"); @@ -131,12 +131,13 @@ template -void predict(const raft::resources& handle, - kmeans_balanced_params const& params, - raft::device_matrix_view X, - raft::device_matrix_view centroids, - raft::device_vector_view labels, - MappingOpT mapping_op = raft::identity_op()) +[[deprecated("Use cuVS instead")]] void predict( + const raft::resources& handle, + kmeans_balanced_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels, + MappingOpT mapping_op = raft::identity_op()) { RAFT_EXPECTS(X.extent(0) == labels.extent(0), "Number of rows in dataset and labels are different"); @@ -196,12 +197,13 @@ template -void fit_predict(const raft::resources& handle, - kmeans_balanced_params const& params, - raft::device_matrix_view X, - raft::device_matrix_view centroids, - raft::device_vector_view labels, - MappingOpT mapping_op = raft::identity_op()) +[[deprecated("Use cuVS instead")]] void fit_predict( + const raft::resources& handle, + kmeans_balanced_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels, + MappingOpT mapping_op = raft::identity_op()) { auto centroids_const = raft::make_device_matrix_view( centroids.data_handle(), centroids.extent(0), centroids.extent(1)); @@ -255,14 +257,15 @@ template -void build_clusters(const raft::resources& handle, - const kmeans_balanced_params& params, - raft::device_matrix_view X, - raft::device_matrix_view centroids, - raft::device_vector_view labels, - raft::device_vector_view cluster_sizes, - MappingOpT mapping_op = raft::identity_op(), - std::optional> X_norm = std::nullopt) +[[deprecated("Use cuVS instead")]] void build_clusters( + const raft::resources& handle, + const kmeans_balanced_params& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels, + raft::device_vector_view cluster_sizes, + MappingOpT mapping_op = raft::identity_op(), + std::optional> X_norm = std::nullopt) { RAFT_EXPECTS(X.extent(0) == labels.extent(0), "Number of rows in dataset and labels are different"); @@ -334,13 +337,14 @@ template -void calc_centers_and_sizes(const raft::resources& handle, - raft::device_matrix_view X, - raft::device_vector_view labels, - raft::device_matrix_view centroids, - raft::device_vector_view cluster_sizes, - bool reset_counters = true, - MappingOpT mapping_op = raft::identity_op()) +[[deprecated("Use cuVS instead")]] void calc_centers_and_sizes( + const raft::resources& handle, + raft::device_matrix_view X, + raft::device_vector_view labels, + raft::device_matrix_view centroids, + raft::device_vector_view cluster_sizes, + bool reset_counters = true, + MappingOpT mapping_op = raft::identity_op()) { RAFT_EXPECTS(X.extent(0) == labels.extent(0), "Number of rows in dataset and labels are different"); diff --git a/cpp/include/raft/cluster/single_linkage.cuh b/cpp/include/raft/cluster/single_linkage.cuh index d9eba6edc5..067445c542 100644 --- a/cpp/include/raft/cluster/single_linkage.cuh +++ b/cpp/include/raft/cluster/single_linkage.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -50,14 +50,14 @@ namespace raft::cluster { template -void single_linkage(raft::resources const& handle, - const value_t* X, - size_t m, - size_t n, - raft::distance::DistanceType metric, - linkage_output* out, - int c, - size_t n_clusters) +[[deprecated("Use cuVS instead")]] void single_linkage(raft::resources const& handle, + const value_t* X, + size_t m, + size_t n, + raft::distance::DistanceType metric, + linkage_output* out, + int c, + size_t n_clusters) { detail::single_linkage( handle, X, m, n, metric, out, c, n_clusters); @@ -87,13 +87,14 @@ constexpr int DEFAULT_CONST_C = 15; control of k. The algorithm will set `k = log(n) + c` */ template -void single_linkage(raft::resources const& handle, - raft::device_matrix_view X, - raft::device_matrix_view dendrogram, - raft::device_vector_view labels, - raft::distance::DistanceType metric, - size_t n_clusters, - std::optional c = std::make_optional(DEFAULT_CONST_C)) +[[deprecated("Use cuVS instead")]] void single_linkage( + raft::resources const& handle, + raft::device_matrix_view X, + raft::device_matrix_view dendrogram, + raft::device_vector_view labels, + raft::distance::DistanceType metric, + size_t n_clusters, + std::optional c = std::make_optional(DEFAULT_CONST_C)) { linkage_output out_arrs; out_arrs.children = dendrogram.data_handle(); diff --git a/cpp/include/raft/distance/distance-ext.cuh b/cpp/include/raft/distance/distance-ext.cuh index 2d41e029fe..dcbfbfdbc3 100644 --- a/cpp/include/raft/distance/distance-ext.cuh +++ b/cpp/include/raft/distance/distance-ext.cuh @@ -35,42 +35,43 @@ template -void distance(raft::resources const& handle, - const DataT* x, - const DataT* y, - OutT* dist, - IdxT m, - IdxT n, - IdxT k, - void* workspace, - size_t worksize, - FinalLambda fin_op, - bool isRowMajor = true, - DataT metric_arg = 2.0f) RAFT_EXPLICIT; +[[deprecated("Use cuVS instead")]] void distance(raft::resources const& handle, + const DataT* x, + const DataT* y, + OutT* dist, + IdxT m, + IdxT n, + IdxT k, + void* workspace, + size_t worksize, + FinalLambda fin_op, + bool isRowMajor = true, + DataT metric_arg = 2.0f) RAFT_EXPLICIT; template -void distance(raft::resources const& handle, - const DataT* x, - const DataT* y, - OutT* dist, - IdxT m, - IdxT n, - IdxT k, - void* workspace, - size_t worksize, - bool isRowMajor = true, - DataT metric_arg = 2.0f) RAFT_EXPLICIT; +[[deprecated("Use cuVS instead")]] void distance(raft::resources const& handle, + const DataT* x, + const DataT* y, + OutT* dist, + IdxT m, + IdxT n, + IdxT k, + void* workspace, + size_t worksize, + bool isRowMajor = true, + DataT metric_arg = 2.0f) RAFT_EXPLICIT; template -size_t getWorkspaceSize(const DataT* x, const DataT* y, IdxT m, IdxT n, IdxT k) RAFT_EXPLICIT; +[[deprecated("Use cuVS instead")]] size_t getWorkspaceSize( + const DataT* x, const DataT* y, IdxT m, IdxT n, IdxT k) RAFT_EXPLICIT; template -void distance(raft::resources const& handle, - const DataT* x, - const DataT* y, - OutT* dist, - IdxT m, - IdxT n, - IdxT k, - bool isRowMajor = true, - DataT metric_arg = 2.0f) RAFT_EXPLICIT; +[[deprecated("Use cuVS instead")]] void distance(raft::resources const& handle, + const DataT* x, + const DataT* y, + OutT* dist, + IdxT m, + IdxT n, + IdxT k, + bool isRowMajor = true, + DataT metric_arg = 2.0f) RAFT_EXPLICIT; template -void pairwise_distance(raft::resources const& handle, - const Type* x, - const Type* y, - Type* dist, - IdxT m, - IdxT n, - IdxT k, - rmm::device_uvector& workspace, - raft::distance::DistanceType metric, - bool isRowMajor = true, - Type metric_arg = 2.0f) RAFT_EXPLICIT; +[[deprecated("Use cuVS instead")]] void pairwise_distance(raft::resources const& handle, + const Type* x, + const Type* y, + Type* dist, + IdxT m, + IdxT n, + IdxT k, + rmm::device_uvector& workspace, + raft::distance::DistanceType metric, + bool isRowMajor = true, + Type metric_arg = 2.0f) RAFT_EXPLICIT; template -void pairwise_distance(raft::resources const& handle, - const Type* x, - const Type* y, - Type* dist, - IdxT m, - IdxT n, - IdxT k, - raft::distance::DistanceType metric, - bool isRowMajor = true, - Type metric_arg = 2.0f) RAFT_EXPLICIT; +[[deprecated("Use cuVS instead")]] void pairwise_distance(raft::resources const& handle, + const Type* x, + const Type* y, + Type* dist, + IdxT m, + IdxT n, + IdxT k, + raft::distance::DistanceType metric, + bool isRowMajor = true, + Type metric_arg = 2.0f) RAFT_EXPLICIT; template -void distance(raft::resources const& handle, - raft::device_matrix_view const x, - raft::device_matrix_view const y, - raft::device_matrix_view dist, - DataT metric_arg = 2.0f) RAFT_EXPLICIT; +[[deprecated("Use cuVS instead")]] void distance( + raft::resources const& handle, + raft::device_matrix_view const x, + raft::device_matrix_view const y, + raft::device_matrix_view dist, + DataT metric_arg = 2.0f) RAFT_EXPLICIT; template -void pairwise_distance(raft::resources const& handle, - device_matrix_view const x, - device_matrix_view const y, - device_matrix_view dist, - raft::distance::DistanceType metric, - Type metric_arg = 2.0f) RAFT_EXPLICIT; +[[deprecated("Use cuVS instead")]] void pairwise_distance( + raft::resources const& handle, + device_matrix_view const x, + device_matrix_view const y, + device_matrix_view dist, + raft::distance::DistanceType metric, + Type metric_arg = 2.0f) RAFT_EXPLICIT; }; // namespace distance }; // namespace raft diff --git a/cpp/include/raft/neighbors/ball_cover.cuh b/cpp/include/raft/neighbors/ball_cover.cuh index 20c88f3318..09938020b9 100644 --- a/cpp/include/raft/neighbors/ball_cover.cuh +++ b/cpp/include/raft/neighbors/ball_cover.cuh @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + #pragma once #ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "ball_cover-inl.cuh" diff --git a/cpp/include/raft/neighbors/brute_force_types.hpp b/cpp/include/raft/neighbors/brute_force_types.hpp index a8f073edc6..4511f8d8ba 100644 --- a/cpp/include/raft/neighbors/brute_force_types.hpp +++ b/cpp/include/raft/neighbors/brute_force_types.hpp @@ -94,12 +94,14 @@ struct index : ann::index { * the dataset. If the dataset is in host memory, it will be copied to the device and the * index will own the device memory. */ + template - index(raft::resources const& res, - mdspan, row_major, data_accessor> dataset, - std::optional>&& norms, - raft::distance::DistanceType metric, - T metric_arg = 0.0) + [[deprecated("Use cuVS instead")]] index( + raft::resources const& res, + mdspan, row_major, data_accessor> dataset, + std::optional>&& norms, + raft::distance::DistanceType metric, + T metric_arg = 0.0) : ann::index(), metric_(metric), dataset_(make_device_matrix(res, 0, 0)), @@ -116,11 +118,12 @@ struct index : ann::index { * This class stores a non-owning reference to the dataset and norms here. * Having precomputed norms gives us a performance advantage at query time. */ - index(raft::resources const& res, - raft::device_matrix_view dataset_view, - std::optional> norms_view, - raft::distance::DistanceType metric, - T metric_arg = 0.0) + [[deprecated("Use cuVS instead")]] index( + raft::resources const& res, + raft::device_matrix_view dataset_view, + std::optional> norms_view, + raft::distance::DistanceType metric, + T metric_arg = 0.0) : ann::index(), metric_(metric), dataset_(make_device_matrix(res, 0, 0)), @@ -131,10 +134,11 @@ struct index : ann::index { } template - index(raft::resources const& res, - index_params const& params, - mdspan, row_major, data_accessor> dataset, - std::optional>&& norms = std::nullopt) + [[deprecated("Use cuVS instead")]] index( + raft::resources const& res, + index_params const& params, + mdspan, row_major, data_accessor> dataset, + std::optional>&& norms = std::nullopt) : ann::index(), metric_(params.metric), dataset_(make_device_matrix(res, 0, 0)), diff --git a/cpp/include/raft/neighbors/cagra_types.hpp b/cpp/include/raft/neighbors/cagra_types.hpp index 97c9c0d098..bc7c380db1 100644 --- a/cpp/include/raft/neighbors/cagra_types.hpp +++ b/cpp/include/raft/neighbors/cagra_types.hpp @@ -201,8 +201,9 @@ struct index : ann::index { ~index() = default; /** Construct an empty index. */ - index(raft::resources const& res, - raft::distance::DistanceType metric = raft::distance::DistanceType::L2Expanded) + [[deprecated("Use cuVS instead")]] index( + raft::resources const& res, + raft::distance::DistanceType metric = raft::distance::DistanceType::L2Expanded) : ann::index(), metric_(metric), graph_(make_device_matrix(res, 0, 0)), @@ -265,10 +266,11 @@ struct index : ann::index { * */ template - index(raft::resources const& res, - raft::distance::DistanceType metric, - mdspan, row_major, data_accessor> dataset, - mdspan, row_major, graph_accessor> knn_graph) + [[deprecated("Use cuVS instead")]] index( + raft::resources const& res, + raft::distance::DistanceType metric, + mdspan, row_major, data_accessor> dataset, + mdspan, row_major, graph_accessor> knn_graph) : ann::index(), metric_(metric), graph_(make_device_matrix(res, 0, 0)), diff --git a/cpp/include/raft/neighbors/hnsw_types.hpp b/cpp/include/raft/neighbors/hnsw_types.hpp index f90de6f01b..f78571f491 100644 --- a/cpp/include/raft/neighbors/hnsw_types.hpp +++ b/cpp/include/raft/neighbors/hnsw_types.hpp @@ -38,7 +38,6 @@ struct search_params : ann::search_params { int num_threads = 0; // number of host threads to use for concurrent searches. Value of 0 // automatically maximizes parallelism }; - template struct index : ann::index { public: @@ -51,7 +50,10 @@ struct index : ann::index { * @param[in] dim dimensions of the training dataset * @param[in] metric distance metric to search. Supported metrics ("L2Expanded", "InnerProduct") */ - index(int dim, raft::distance::DistanceType metric) : dim_{dim}, metric_{metric} {} + [[deprecated("Use cuVS instead")]] index(int dim, raft::distance::DistanceType metric) + : dim_{dim}, metric_{metric} + { + } /** @brief Get underlying index diff --git a/cpp/include/raft/neighbors/ivf_flat_types.hpp b/cpp/include/raft/neighbors/ivf_flat_types.hpp index 7605bd82a3..2cafceb512 100644 --- a/cpp/include/raft/neighbors/ivf_flat_types.hpp +++ b/cpp/include/raft/neighbors/ivf_flat_types.hpp @@ -261,12 +261,12 @@ struct index : ann::index { ~index() = default; /** Construct an empty index. It needs to be trained and then populated. */ - index(raft::resources const& res, - raft::distance::DistanceType metric, - uint32_t n_lists, - bool adaptive_centers, - bool conservative_memory_allocation, - uint32_t dim) + [[deprecated("Use cuVS instead")]] index(raft::resources const& res, + raft::distance::DistanceType metric, + uint32_t n_lists, + bool adaptive_centers, + bool conservative_memory_allocation, + uint32_t dim) : ann::index(), veclen_(calculate_veclen(dim)), metric_(metric), @@ -285,7 +285,9 @@ struct index : ann::index { } /** Construct an empty index. It needs to be trained and then populated. */ - index(raft::resources const& res, const index_params& params, uint32_t dim) + [[deprecated("Use cuVS instead")]] index(raft::resources const& res, + const index_params& params, + uint32_t dim) : index(res, params.metric, params.n_lists, diff --git a/cpp/include/raft/neighbors/ivf_pq_types.hpp b/cpp/include/raft/neighbors/ivf_pq_types.hpp index 3ee350c6fb..d5906d621c 100644 --- a/cpp/include/raft/neighbors/ivf_pq_types.hpp +++ b/cpp/include/raft/neighbors/ivf_pq_types.hpp @@ -361,14 +361,14 @@ struct index : ann::index { ~index() = default; /** Construct an empty index. It needs to be trained and then populated. */ - index(raft::resources const& handle, - raft::distance::DistanceType metric, - codebook_gen codebook_kind, - uint32_t n_lists, - uint32_t dim, - uint32_t pq_bits = 8, - uint32_t pq_dim = 0, - bool conservative_memory_allocation = false) + [[deprecated("Use cuVS instead")]] index(raft::resources const& handle, + raft::distance::DistanceType metric, + codebook_gen codebook_kind, + uint32_t n_lists, + uint32_t dim, + uint32_t pq_bits = 8, + uint32_t pq_dim = 0, + bool conservative_memory_allocation = false) : ann::index(), metric_(metric), codebook_kind_(codebook_kind), @@ -391,7 +391,9 @@ struct index : ann::index { } /** Construct an empty index. It needs to be trained and then populated. */ - index(raft::resources const& handle, const index_params& params, uint32_t dim) + [[deprecated("Use cuVS instead")]] index(raft::resources const& handle, + const index_params& params, + uint32_t dim) : index(handle, params.metric, params.codebook_kind, diff --git a/cpp/include/raft/neighbors/nn_descent_types.hpp b/cpp/include/raft/neighbors/nn_descent_types.hpp index eb01a423be..9decf47f39 100644 --- a/cpp/include/raft/neighbors/nn_descent_types.hpp +++ b/cpp/include/raft/neighbors/nn_descent_types.hpp @@ -101,7 +101,10 @@ struct index : ann::index { * @param n_cols number of cols in knn-graph * @param return_distances whether to allocate and get distances information */ - index(raft::resources const& res, int64_t n_rows, int64_t n_cols, bool return_distances = false) + [[deprecated("Use cuVS instead")]] index(raft::resources const& res, + int64_t n_rows, + int64_t n_cols, + bool return_distances = false) : ann::index(), res_{res}, metric_{raft::distance::DistanceType::L2Expanded}, @@ -128,11 +131,12 @@ struct index : ann::index { * storing knn-graph distances * @param return_distances whether to allocate and get distances information */ - index(raft::resources const& res, - raft::host_matrix_view graph_view, - std::optional> distances_view = - std::nullopt, - bool return_distances = false) + [[deprecated("Use cuVS instead")]] index( + raft::resources const& res, + raft::host_matrix_view graph_view, + std::optional> distances_view = + std::nullopt, + bool return_distances = false) : ann::index(), res_{res}, metric_{raft::distance::DistanceType::L2Expanded}, diff --git a/cpp/include/raft/neighbors/refine-ext.cuh b/cpp/include/raft/neighbors/refine-ext.cuh index 7948a0e4f2..216e1b9ab5 100644 --- a/cpp/include/raft/neighbors/refine-ext.cuh +++ b/cpp/include/raft/neighbors/refine-ext.cuh @@ -29,24 +29,24 @@ namespace raft::neighbors { template -void refine(raft::resources const& handle, - raft::device_matrix_view dataset, - raft::device_matrix_view queries, - raft::device_matrix_view neighbor_candidates, - raft::device_matrix_view indices, - raft::device_matrix_view distances, - raft::distance::DistanceType metric = distance::DistanceType::L2Unexpanded) - RAFT_EXPLICIT; +[[deprecated("Use cuVS instead")]] void refine( + raft::resources const& handle, + raft::device_matrix_view dataset, + raft::device_matrix_view queries, + raft::device_matrix_view neighbor_candidates, + raft::device_matrix_view indices, + raft::device_matrix_view distances, + raft::distance::DistanceType metric = distance::DistanceType::L2Unexpanded) RAFT_EXPLICIT; template -void refine(raft::resources const& handle, - raft::host_matrix_view dataset, - raft::host_matrix_view queries, - raft::host_matrix_view neighbor_candidates, - raft::host_matrix_view indices, - raft::host_matrix_view distances, - raft::distance::DistanceType metric = distance::DistanceType::L2Unexpanded) - RAFT_EXPLICIT; +[[deprecated("Use cuVS instead")]] void refine( + raft::resources const& handle, + raft::host_matrix_view dataset, + raft::host_matrix_view queries, + raft::host_matrix_view neighbor_candidates, + raft::host_matrix_view indices, + raft::host_matrix_view distances, + raft::distance::DistanceType metric = distance::DistanceType::L2Unexpanded) RAFT_EXPLICIT; } // namespace raft::neighbors diff --git a/docs/source/conf.py b/docs/source/conf.py index 8b2040baa2..7a287b689f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -71,7 +71,7 @@ .. attention:: - The vector search and clustering algorithms in RAFT are being migrated to a new library dedicated to vector search called `cuVS `_. We will continue to support the vector search algorithms in RAFT during this move, but will no longer update them after the RAPIDS 24.06 (June) release. We plan to complete the migration by RAPIDS 24.08 (August) release. + The vector search and clustering algorithms in RAFT are being migrated to a new library dedicated to vector search called `cuVS `_. We will continue to support the vector search algorithms in RAFT during this move, but will no longer update them after the RAPIDS 24.06 (June) release. We plan to complete the migration by RAPIDS 24.10 (October) release and they will be removed from RAFT altogether in the 24.12 (December) release. """ diff --git a/docs/source/raft_ann_benchmarks.md b/docs/source/raft_ann_benchmarks.md index fc11a56ac8..7bac2047fc 100644 --- a/docs/source/raft_ann_benchmarks.md +++ b/docs/source/raft_ann_benchmarks.md @@ -2,6 +2,10 @@ This project provides a benchmark program for various ANN search implementations. It's especially suitable for comparing GPU implementations as well as comparing GPU against CPU. +> [!IMPORTANT] +> The vector search and clustering algorithms in RAFT are being migrated to a new library dedicated to vector search called [cuVS](https://github.com/rapidsai/cuvs). As a result, `raft-ann-bench` is being migrated to `cuvs-bench` and will be removed from RAFT altogether in the 24.12 (December) release. + + ## Table of Contents - [Installing the benchmarks](#installing-the-benchmarks) From 94cc2c23bba3ca38514a3a5c8b736a968e5aa48e Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 4 Oct 2024 10:23:58 -0400 Subject: [PATCH 37/79] Prune workflows based on changed files (#2466) Contributes to https://github.com/rapidsai/build-planning/issues/94 Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/raft/pull/2466 --- .github/workflows/pr.yaml | 49 +++++++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 844f9f9441..fe8e730921 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,6 +12,7 @@ concurrency: jobs: pr-builder: needs: + - changed-files - checks - conda-cpp-build - conda-cpp-tests @@ -26,6 +27,42 @@ jobs: - devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.12 + if: always() + with: + needs: ${{ toJSON(needs) }} + changed-files: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-24.12 + with: + files_yaml: | + test_cpp: + - '**' + - '!.devcontainer/**' + - '!.pre-commit-config.yaml' + - '!CONTRIBUTING.md' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!notebooks/**' + - '!python/**' + - '!thirdparty/LICENSES/**' + test_notebooks: + - '**' + - '!.devcontainer/**' + - '!.pre-commit-config.yaml' + - '!CONTRIBUTING.md' + - '!README.md' + - '!thirdparty/LICENSES/**' + test_python: + - '**' + - '!.devcontainer/**' + - '!.pre-commit-config.yaml' + - '!CONTRIBUTING.md' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!notebooks/**' + - '!thirdparty/LICENSES/**' checks: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.12 @@ -39,9 +76,10 @@ jobs: build_type: pull-request node_type: cpu16 conda-cpp-tests: - needs: conda-cpp-build + needs: [conda-cpp-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp with: build_type: pull-request conda-cpp-checks: @@ -59,9 +97,10 @@ jobs: with: build_type: pull-request conda-python-tests: - needs: conda-python-build + needs: [conda-python-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request docs-build: @@ -82,9 +121,10 @@ jobs: build_type: pull-request script: ci/build_wheel_pylibraft.sh wheel-tests-pylibraft: - needs: wheel-build-pylibraft + needs: [wheel-build-pylibraft, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request script: ci/test_wheel_pylibraft.sh @@ -96,9 +136,10 @@ jobs: build_type: pull-request script: "ci/build_wheel_raft_dask.sh" wheel-tests-raft-dask: - needs: wheel-build-raft-dask + needs: [wheel-build-raft-dask, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request script: ci/test_wheel_raft_dask.sh From b7d0e986d843b28e7e8be4b4b99fca02dfa62f91 Mon Sep 17 00:00:00 2001 From: Matthew Murray <41342305+Matt711@users.noreply.github.com> Date: Mon, 7 Oct 2024 22:04:26 -0400 Subject: [PATCH 38/79] Update all rmm imports to use pylibrmm/librmm (#2451) This PR updates all the RMM imports to use pylibrmm/librmm now that `rmm._lib` is deprecated . It should be merged after [rmm/1676](https://github.com/rapidsai/rmm/pull/1676). Authors: - Matthew Murray (https://github.com/Matt711) Approvers: - Ben Frederickson (https://github.com/benfred) URL: https://github.com/rapidsai/raft/pull/2451 --- python/pylibraft/pylibraft/common/handle.pxd | 6 +++--- python/pylibraft/pylibraft/common/handle.pyx | 7 +++++-- python/pylibraft/pylibraft/common/interruptible.pxd | 4 ++-- python/pylibraft/pylibraft/common/interruptible.pyx | 4 ++-- python/pylibraft/pylibraft/neighbors/cagra/cagra.pyx | 6 ++---- .../pylibraft/pylibraft/neighbors/cagra/cpp/c_cagra.pxd | 2 +- python/pylibraft/pylibraft/neighbors/cpp/brute_force.pxd | 2 +- python/pylibraft/pylibraft/neighbors/cpp/rbc.pxd | 2 +- .../pylibraft/neighbors/ivf_flat/cpp/c_ivf_flat.pxd | 4 ++-- .../pylibraft/pylibraft/neighbors/ivf_flat/ivf_flat.pyx | 8 +++----- .../pylibraft/pylibraft/neighbors/ivf_pq/cpp/c_ivf_pq.pxd | 2 +- python/pylibraft/pylibraft/neighbors/ivf_pq/ivf_pq.pyx | 6 ++---- 12 files changed, 25 insertions(+), 28 deletions(-) diff --git a/python/pylibraft/pylibraft/common/handle.pxd b/python/pylibraft/pylibraft/common/handle.pxd index c090663547..78a07ac8bf 100644 --- a/python/pylibraft/pylibraft/common/handle.pxd +++ b/python/pylibraft/pylibraft/common/handle.pxd @@ -1,5 +1,5 @@ # -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,8 +22,8 @@ from libcpp.memory cimport shared_ptr, unique_ptr -from rmm._lib.cuda_stream_pool cimport cuda_stream_pool -from rmm._lib.cuda_stream_view cimport cuda_stream_view +from rmm.librmm.cuda_stream_pool cimport cuda_stream_pool +from rmm.librmm.cuda_stream_view cimport cuda_stream_view # Keeping `handle_t` around for backwards compatibility at the diff --git a/python/pylibraft/pylibraft/common/handle.pyx b/python/pylibraft/pylibraft/common/handle.pyx index 7e3dc289e0..d256e671bf 100644 --- a/python/pylibraft/pylibraft/common/handle.pyx +++ b/python/pylibraft/pylibraft/common/handle.pyx @@ -1,5 +1,5 @@ # -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,7 +24,10 @@ import functools from cuda.ccudart cimport cudaStream_t from libc.stdint cimport uintptr_t -from rmm._lib.cuda_stream_view cimport cuda_stream_per_thread, cuda_stream_view +from rmm.librmm.cuda_stream_view cimport ( + cuda_stream_per_thread, + cuda_stream_view, +) from .cuda cimport Stream diff --git a/python/pylibraft/pylibraft/common/interruptible.pxd b/python/pylibraft/pylibraft/common/interruptible.pxd index aaccf8aeab..27259571ea 100644 --- a/python/pylibraft/pylibraft/common/interruptible.pxd +++ b/python/pylibraft/pylibraft/common/interruptible.pxd @@ -1,5 +1,5 @@ # -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ from libcpp.memory cimport shared_ptr -from rmm._lib.cuda_stream_view cimport cuda_stream_view +from rmm.librmm.cuda_stream_view cimport cuda_stream_view cdef extern from "raft/core/interruptible.hpp" namespace "raft" nogil: diff --git a/python/pylibraft/pylibraft/common/interruptible.pyx b/python/pylibraft/pylibraft/common/interruptible.pyx index bb5415428f..c489f2ee20 100644 --- a/python/pylibraft/pylibraft/common/interruptible.pyx +++ b/python/pylibraft/pylibraft/common/interruptible.pyx @@ -1,5 +1,5 @@ # -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ import signal from cuda.ccudart cimport cudaStream_t from cython.operator cimport dereference -from rmm._lib.cuda_stream_view cimport cuda_stream_view +from rmm.librmm.cuda_stream_view cimport cuda_stream_view from .cuda cimport Stream diff --git a/python/pylibraft/pylibraft/neighbors/cagra/cagra.pyx b/python/pylibraft/pylibraft/neighbors/cagra/cagra.pyx index 0e488a51ca..9b376f5f0a 100644 --- a/python/pylibraft/pylibraft/neighbors/cagra/cagra.pyx +++ b/python/pylibraft/pylibraft/neighbors/cagra/cagra.pyx @@ -52,10 +52,8 @@ from pylibraft.common.handle cimport device_resources from pylibraft.common.handle import auto_sync_handle from pylibraft.common.input_validation import is_c_contiguous -from rmm._lib.memory_resource cimport ( - DeviceMemoryResource, - device_memory_resource, -) +from rmm.librmm.memory_resource cimport device_memory_resource +from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource cimport pylibraft.neighbors.cagra.cpp.c_cagra as c_cagra from pylibraft.common.optional cimport make_optional, optional diff --git a/python/pylibraft/pylibraft/neighbors/cagra/cpp/c_cagra.pxd b/python/pylibraft/pylibraft/neighbors/cagra/cpp/c_cagra.pxd index 1dffd40186..75ace7f1a8 100644 --- a/python/pylibraft/pylibraft/neighbors/cagra/cpp/c_cagra.pxd +++ b/python/pylibraft/pylibraft/neighbors/cagra/cpp/c_cagra.pxd @@ -27,7 +27,7 @@ from libc.stdint cimport int8_t, int64_t, uint8_t, uint32_t, uint64_t from libcpp cimport bool, nullptr from libcpp.string cimport string -from rmm._lib.memory_resource cimport device_memory_resource +from rmm.librmm.memory_resource cimport device_memory_resource from pylibraft.common.cpp.mdspan cimport ( device_matrix_view, diff --git a/python/pylibraft/pylibraft/neighbors/cpp/brute_force.pxd b/python/pylibraft/pylibraft/neighbors/cpp/brute_force.pxd index 5f6a83a9dc..f513517868 100644 --- a/python/pylibraft/pylibraft/neighbors/cpp/brute_force.pxd +++ b/python/pylibraft/pylibraft/neighbors/cpp/brute_force.pxd @@ -28,7 +28,7 @@ from libcpp cimport bool, nullptr from libcpp.string cimport string from libcpp.vector cimport vector -from rmm._lib.memory_resource cimport device_memory_resource +from rmm.librmm.memory_resource cimport device_memory_resource from pylibraft.common.cpp.mdspan cimport ( device_matrix_view, diff --git a/python/pylibraft/pylibraft/neighbors/cpp/rbc.pxd b/python/pylibraft/pylibraft/neighbors/cpp/rbc.pxd index 531c0dc2c1..d544797119 100644 --- a/python/pylibraft/pylibraft/neighbors/cpp/rbc.pxd +++ b/python/pylibraft/pylibraft/neighbors/cpp/rbc.pxd @@ -28,7 +28,7 @@ from libcpp cimport bool, nullptr from libcpp.string cimport string from libcpp.vector cimport vector -from rmm._lib.memory_resource cimport device_memory_resource +from rmm.librmm.memory_resource cimport device_memory_resource from pylibraft.common.cpp.mdspan cimport ( device_matrix_view, diff --git a/python/pylibraft/pylibraft/neighbors/ivf_flat/cpp/c_ivf_flat.pxd b/python/pylibraft/pylibraft/neighbors/ivf_flat/cpp/c_ivf_flat.pxd index a281d33310..22b08b3f19 100644 --- a/python/pylibraft/pylibraft/neighbors/ivf_flat/cpp/c_ivf_flat.pxd +++ b/python/pylibraft/pylibraft/neighbors/ivf_flat/cpp/c_ivf_flat.pxd @@ -1,5 +1,5 @@ # -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -27,7 +27,7 @@ from libc.stdint cimport int8_t, int64_t, uint8_t, uint32_t, uintptr_t from libcpp cimport bool, nullptr from libcpp.string cimport string -from rmm._lib.memory_resource cimport device_memory_resource +from rmm.librmm.memory_resource cimport device_memory_resource from pylibraft.common.cpp.mdspan cimport ( device_matrix_view, diff --git a/python/pylibraft/pylibraft/neighbors/ivf_flat/ivf_flat.pyx b/python/pylibraft/pylibraft/neighbors/ivf_flat/ivf_flat.pyx index d8fbdc74da..6826b2bc59 100644 --- a/python/pylibraft/pylibraft/neighbors/ivf_flat/ivf_flat.pyx +++ b/python/pylibraft/pylibraft/neighbors/ivf_flat/ivf_flat.pyx @@ -1,5 +1,5 @@ # -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -51,10 +51,8 @@ from pylibraft.common.handle cimport device_resources from pylibraft.common.handle import auto_sync_handle from pylibraft.common.input_validation import is_c_contiguous -from rmm._lib.memory_resource cimport ( - DeviceMemoryResource, - device_memory_resource, -) +from rmm.librmm.memory_resource cimport device_memory_resource +from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource cimport pylibraft.neighbors.ivf_flat.cpp.c_ivf_flat as c_ivf_flat from pylibraft.common.cpp.optional cimport optional diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/cpp/c_ivf_pq.pxd b/python/pylibraft/pylibraft/neighbors/ivf_pq/cpp/c_ivf_pq.pxd index 895abbadca..18319bf452 100644 --- a/python/pylibraft/pylibraft/neighbors/ivf_pq/cpp/c_ivf_pq.pxd +++ b/python/pylibraft/pylibraft/neighbors/ivf_pq/cpp/c_ivf_pq.pxd @@ -27,7 +27,7 @@ from libc.stdint cimport int8_t, int64_t, uint8_t, uint32_t, uintptr_t from libcpp cimport bool, nullptr from libcpp.string cimport string -from rmm._lib.memory_resource cimport device_memory_resource +from rmm.librmm.memory_resource cimport device_memory_resource from pylibraft.common.cpp.mdspan cimport ( device_matrix_view, diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/ivf_pq.pyx b/python/pylibraft/pylibraft/neighbors/ivf_pq/ivf_pq.pyx index 5b89f0d9a5..f467957fd6 100644 --- a/python/pylibraft/pylibraft/neighbors/ivf_pq/ivf_pq.pyx +++ b/python/pylibraft/pylibraft/neighbors/ivf_pq/ivf_pq.pyx @@ -44,10 +44,8 @@ from pylibraft.common.handle cimport device_resources from pylibraft.common.handle import auto_sync_handle from pylibraft.common.input_validation import is_c_contiguous -from rmm._lib.memory_resource cimport ( - DeviceMemoryResource, - device_memory_resource, -) +from rmm.librmm.memory_resource cimport device_memory_resource +from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource cimport pylibraft.neighbors.ivf_flat.cpp.c_ivf_flat as c_ivf_flat cimport pylibraft.neighbors.ivf_pq.cpp.c_ivf_pq as c_ivf_pq From ec12aba62032e2075ba10bdd40cf863937eca7fd Mon Sep 17 00:00:00 2001 From: Ray Douglass Date: Wed, 9 Oct 2024 09:39:25 -0400 Subject: [PATCH 39/79] Update Changelog [skip ci] --- CHANGELOG.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0685145dca..9caa5ef571 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,48 @@ +# raft 24.10.00 (9 Oct 2024) + +## 🚨 Breaking Changes + +- [Feat] add `repeat`, `sparsity`, `eval_n_elements` APIs to `bitset` ([#2439](https://github.com/rapidsai/raft/pull/2439)) [@rhdong](https://github.com/rhdong) + +## 🐛 Bug Fixes + +- Disable NN Descent Batch tests temporarily ([#2453](https://github.com/rapidsai/raft/pull/2453)) [@divyegala](https://github.com/divyegala) +- Fix sed syntax in `update-version.sh` ([#2441](https://github.com/rapidsai/raft/pull/2441)) [@raydouglass](https://github.com/raydouglass) +- Use runtime check of cudart version for eig ([#2430](https://github.com/rapidsai/raft/pull/2430)) [@lowener](https://github.com/lowener) +- [BUG] Fix bitset function visibility ([#2429](https://github.com/rapidsai/raft/pull/2429)) [@lowener](https://github.com/lowener) +- Exclude any kernel symbol that uses cutlass ([#2425](https://github.com/rapidsai/raft/pull/2425)) [@robertmaynard](https://github.com/robertmaynard) + +## 🚀 New Features + +- [Feat] add `repeat`, `sparsity`, `eval_n_elements` APIs to `bitset` ([#2439](https://github.com/rapidsai/raft/pull/2439)) [@rhdong](https://github.com/rhdong) +- [Opt] Enforce the UT Coverity and add benchmark for `transpose` ([#2438](https://github.com/rapidsai/raft/pull/2438)) [@rhdong](https://github.com/rhdong) +- [FEA] Support for half-float mixed precise in brute-force ([#2382](https://github.com/rapidsai/raft/pull/2382)) [@rhdong](https://github.com/rhdong) + +## 🛠️ Improvements + +- bump NCCL floor to 2.19 ([#2458](https://github.com/rapidsai/raft/pull/2458)) [@jameslamb](https://github.com/jameslamb) +- Deprecating vector search APIs and updating README accordingly ([#2448](https://github.com/rapidsai/raft/pull/2448)) [@cjnolet](https://github.com/cjnolet) +- Update update-version.sh to use packaging lib ([#2447](https://github.com/rapidsai/raft/pull/2447)) [@AyodeAwe](https://github.com/AyodeAwe) +- Switch traceback to `native` ([#2446](https://github.com/rapidsai/raft/pull/2446)) [@galipremsagar](https://github.com/galipremsagar) +- bump NCCL floor to 2.18.1.1 ([#2443](https://github.com/rapidsai/raft/pull/2443)) [@jameslamb](https://github.com/jameslamb) +- Add missing `cuda_suffixed: true` ([#2440](https://github.com/rapidsai/raft/pull/2440)) [@trxcllnt](https://github.com/trxcllnt) +- Use CI workflow branch 'branch-24.10' again ([#2437](https://github.com/rapidsai/raft/pull/2437)) [@jameslamb](https://github.com/jameslamb) +- Update to flake8 7.1.1. ([#2435](https://github.com/rapidsai/raft/pull/2435)) [@bdice](https://github.com/bdice) +- Update fmt (to 11.0.2) and spdlog (to 1.14.1). ([#2433](https://github.com/rapidsai/raft/pull/2433)) [@jameslamb](https://github.com/jameslamb) +- Allow coo_sort to work on int64_t indices ([#2432](https://github.com/rapidsai/raft/pull/2432)) [@benfred](https://github.com/benfred) +- Adding NCCL clique to the RAFT handle ([#2431](https://github.com/rapidsai/raft/pull/2431)) [@viclafargue](https://github.com/viclafargue) +- Add support for Python 3.12 ([#2428](https://github.com/rapidsai/raft/pull/2428)) [@jameslamb](https://github.com/jameslamb) +- Update rapidsai/pre-commit-hooks ([#2420](https://github.com/rapidsai/raft/pull/2420)) [@KyleFromNVIDIA](https://github.com/KyleFromNVIDIA) +- Drop Python 3.9 support ([#2417](https://github.com/rapidsai/raft/pull/2417)) [@jameslamb](https://github.com/jameslamb) +- Use CUDA math wheels ([#2415](https://github.com/rapidsai/raft/pull/2415)) [@KyleFromNVIDIA](https://github.com/KyleFromNVIDIA) +- Remove NumPy <2 pin ([#2414](https://github.com/rapidsai/raft/pull/2414)) [@seberg](https://github.com/seberg) +- Update pre-commit hooks ([#2409](https://github.com/rapidsai/raft/pull/2409)) [@KyleFromNVIDIA](https://github.com/KyleFromNVIDIA) +- Improve update-version.sh ([#2408](https://github.com/rapidsai/raft/pull/2408)) [@bdice](https://github.com/bdice) +- Use tool.scikit-build.cmake.version, set scikit-build-core minimum-version ([#2406](https://github.com/rapidsai/raft/pull/2406)) [@jameslamb](https://github.com/jameslamb) +- [FEA] Batching NN Descent ([#2403](https://github.com/rapidsai/raft/pull/2403)) [@jinsolp](https://github.com/jinsolp) +- Update pip devcontainers to UCX v1.17.0 ([#2401](https://github.com/rapidsai/raft/pull/2401)) [@jameslamb](https://github.com/jameslamb) +- Merge branch-24.08 into branch-24.10 ([#2397](https://github.com/rapidsai/raft/pull/2397)) [@jameslamb](https://github.com/jameslamb) + # raft 24.08.00 (7 Aug 2024) ## 🚨 Breaking Changes From 907edb8261a94a06f8857d0a10f2ec9abcd1ae42 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 11 Oct 2024 08:53:52 -0500 Subject: [PATCH 40/79] make package installations in CI stricter (#2467) Contributes to https://github.com/rapidsai/build-planning/issues/106 Proposes specifying the RAPIDS version in `conda install` calls in CI that install CI artifacts, to reduce the risk of CI jobs picking up artifacts from other releases. Also proposes combining together successive `pip install` calls. `pip install AB` is safer than `pip install A; pip install B` because `pip` doesn't take the current set of installed packages into consideration when it installs new packages. Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Kyle Edwards (https://github.com/KyleFromNVIDIA) URL: https://github.com/rapidsai/raft/pull/2467 --- ci/build_docs.sh | 16 ++++++++-------- ci/test_cpp.sh | 6 +++++- ci/test_python.sh | 7 ++++++- ci/test_wheel_raft_dask.sh | 5 +++-- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index a2447f5f06..aff7674892 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -6,6 +6,9 @@ set -euo pipefail rapids-logger "Create test conda environment" . /opt/conda/etc/profile.d/conda.sh +RAPIDS_VERSION="$(rapids-version)" +export RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" + rapids-dependency-file-generator \ --output conda \ --file-key docs \ @@ -23,14 +26,11 @@ PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ - libraft \ - libraft-headers \ - pylibraft \ - raft-dask + "libraft=${RAPIDS_VERSION}" \ + "libraft-headers=${RAPIDS_VERSION}" \ + "pylibraft=${RAPIDS_VERSION}" \ + "raft-dask=${RAPIDS_VERSION}" -export RAPIDS_VERSION="$(rapids-version)" -export RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" -export RAPIDS_VERSION_NUMBER="$RAPIDS_VERSION_MAJOR_MINOR" export RAPIDS_DOCS_DIR="$(mktemp -d)" rapids-logger "Build CPP docs" @@ -45,4 +45,4 @@ mkdir -p "${RAPIDS_DOCS_DIR}/raft/"html mv _html/* "${RAPIDS_DOCS_DIR}/raft/html" popd -rapids-upload-docs +RAPIDS_VERSION_NUMBER="${RAPIDS_VERSION_MAJOR_MINOR}" rapids-upload-docs diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 05323e4f5d..9d0edc6b21 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -8,6 +8,8 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ . /opt/conda/etc/profile.d/conda.sh +RAPIDS_VERSION="$(rapids-version)" + rapids-logger "Generate C++ testing dependencies" rapids-dependency-file-generator \ --output conda \ @@ -29,7 +31,9 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ - libraft-headers libraft libraft-tests + "libraft-headers=${RAPIDS_VERSION}" \ + "libraft=${RAPIDS_VERSION}" \ + "libraft-tests=${RAPIDS_VERSION}" rapids-logger "Check GPU usage" nvidia-smi diff --git a/ci/test_python.sh b/ci/test_python.sh index 01e5ac9456..af93d2e04b 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -8,6 +8,8 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ . /opt/conda/etc/profile.d/conda.sh +RAPIDS_VERSION="$(rapids-version)" + rapids-logger "Generate Python testing dependencies" rapids-dependency-file-generator \ --output conda \ @@ -34,7 +36,10 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ - libraft libraft-headers pylibraft raft-dask + "libraft=${RAPIDS_VERSION}" \ + "libraft-headers=${RAPIDS_VERSION}" \ + "pylibraft=${RAPIDS_VERSION}" \ + "raft-dask=${RAPIDS_VERSION}" rapids-logger "Check GPU usage" nvidia-smi diff --git a/ci/test_wheel_raft_dask.sh b/ci/test_wheel_raft_dask.sh index 9b1187592d..a778a3ec51 100755 --- a/ci/test_wheel_raft_dask.sh +++ b/ci/test_wheel_raft_dask.sh @@ -9,10 +9,11 @@ RAPIDS_PY_WHEEL_NAME="raft_dask_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels # Download the pylibraft built in the previous step RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibraft-dep -python -m pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl # echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install -v "$(echo ./dist/raft_dask_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" +python -m pip install -v \ + ./local-pylibraft-dep/pylibraft*.whl \ + "$(echo ./dist/raft_dask_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" test_dir="python/raft-dask/raft_dask/test" From 714e07b3124e41180a88041402a341ccab519f10 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 21 Oct 2024 14:24:25 -0500 Subject: [PATCH 41/79] Use Python for computation. (#2474) --- build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index a77dd188f4..bd60914517 100755 --- a/build.sh +++ b/build.sh @@ -463,14 +463,14 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft || hasArg docs || hasArg tests || has if [[ ${CACHE_TOOL} == "sccache" && -x "$(command -v sccache)" ]]; then COMPILE_REQUESTS=$(sccache -s | grep "Compile requests \+ [0-9]\+$" | awk '{ print $NF }') CACHE_HITS=$(sccache -s | grep "Cache hits \+ [0-9]\+$" | awk '{ print $NF }') - HIT_RATE=$(echo - | awk "{printf \"%.2f\n\", $CACHE_HITS / $COMPILE_REQUESTS * 100}") + HIT_RATE=$(python3 -c "print(f'{${CACHE_HITS} / ${COMPILE_REQUESTS}:.2f}' if ${COMPILE_REQUESTS} else 'nan')") MSG="${MSG}
cache hit rate ${HIT_RATE} %" elif [[ ${CACHE_TOOL} == "ccache" && -x "$(command -v ccache)" ]]; then CACHE_STATS_LINE=$(ccache -s | grep "Hits: \+ [0-9]\+ / [0-9]\+" | tail -n1) if [[ ! -z "$CACHE_STATS_LINE" ]]; then CACHE_HITS=$(echo "$CACHE_STATS_LINE" - | awk '{ print $2 }') COMPILE_REQUESTS=$(echo "$CACHE_STATS_LINE" - | awk '{ print $4 }') - HIT_RATE=$(echo - | awk "{printf \"%.2f\n\", $CACHE_HITS / $COMPILE_REQUESTS * 100}") + HIT_RATE=$(python3 -c "print(f'{${CACHE_HITS} / ${COMPILE_REQUESTS}:.2f}' if ${COMPILE_REQUESTS} else 'nan')") MSG="${MSG}
cache hit rate ${HIT_RATE} %" fi fi From 8dc8245be55a757c4ef04565562c134c6455063d Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 22 Oct 2024 11:02:27 -0500 Subject: [PATCH 42/79] Use environment variables in cache hit rate computation. (#2475) Follow-up PR to address feedback: https://github.com/rapidsai/raft/pull/2474#discussion_r1809398110 Authors: - Bradley Dice (https://github.com/bdice) Approvers: - Kyle Edwards (https://github.com/KyleFromNVIDIA) URL: https://github.com/rapidsai/raft/pull/2475 --- build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index bd60914517..feb2d7256e 100755 --- a/build.sh +++ b/build.sh @@ -463,14 +463,14 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft || hasArg docs || hasArg tests || has if [[ ${CACHE_TOOL} == "sccache" && -x "$(command -v sccache)" ]]; then COMPILE_REQUESTS=$(sccache -s | grep "Compile requests \+ [0-9]\+$" | awk '{ print $NF }') CACHE_HITS=$(sccache -s | grep "Cache hits \+ [0-9]\+$" | awk '{ print $NF }') - HIT_RATE=$(python3 -c "print(f'{${CACHE_HITS} / ${COMPILE_REQUESTS}:.2f}' if ${COMPILE_REQUESTS} else 'nan')") + HIT_RATE=$(COMPILE_REQUESTS="${COMPILE_REQUESTS}" CACHE_HITS="${CACHE_HITS}" python3 -c "import os; print(f'{int(os.getenv(\"CACHE_HITS\")) / int(os.getenv(\"COMPILE_REQUESTS\")):.2f}' if int(os.getenv(\"COMPILE_REQUESTS\")) else 'nan')") MSG="${MSG}
cache hit rate ${HIT_RATE} %" elif [[ ${CACHE_TOOL} == "ccache" && -x "$(command -v ccache)" ]]; then CACHE_STATS_LINE=$(ccache -s | grep "Hits: \+ [0-9]\+ / [0-9]\+" | tail -n1) if [[ ! -z "$CACHE_STATS_LINE" ]]; then CACHE_HITS=$(echo "$CACHE_STATS_LINE" - | awk '{ print $2 }') COMPILE_REQUESTS=$(echo "$CACHE_STATS_LINE" - | awk '{ print $4 }') - HIT_RATE=$(python3 -c "print(f'{${CACHE_HITS} / ${COMPILE_REQUESTS}:.2f}' if ${COMPILE_REQUESTS} else 'nan')") + HIT_RATE=$(COMPILE_REQUESTS="${COMPILE_REQUESTS}" CACHE_HITS="${CACHE_HITS}" python3 -c "import os; print(f'{int(os.getenv(\"CACHE_HITS\")) / int(os.getenv(\"COMPILE_REQUESTS\")):.2f}' if int(os.getenv(\"COMPILE_REQUESTS\")) else 'nan')") MSG="${MSG}
cache hit rate ${HIT_RATE} %" fi fi From 673a5bbd5c968d7768e1677ac8e7f4dc40dfaf8b Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Thu, 24 Oct 2024 17:25:25 -0400 Subject: [PATCH 43/79] Allow compilation with CUDA 12.6.1 (#2469) The 12.6.1 CUDA compiler has issues with enable_if inside the template arguments of some kernels. We can simplify kernel logic and remove the usage of enable_if. Authors: - Robert Maynard (https://github.com/robertmaynard) - Paul Taylor (https://github.com/trxcllnt) Approvers: - Dante Gama Dessavre (https://github.com/dantegd) URL: https://github.com/rapidsai/raft/pull/2469 --- .../raft/matrix/detail/columnWiseSort.cuh | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/cpp/include/raft/matrix/detail/columnWiseSort.cuh b/cpp/include/raft/matrix/detail/columnWiseSort.cuh index a02621d054..9e94724d3d 100644 --- a/cpp/include/raft/matrix/detail/columnWiseSort.cuh +++ b/cpp/include/raft/matrix/detail/columnWiseSort.cuh @@ -72,12 +72,11 @@ RAFT_KERNEL devOffsetKernel(T* in, T value, int n_times) } // block level radix sort - can only sort as much data we can fit within shared memory -template < - typename InType, - typename OutType, - int BLOCK_SIZE, - int ITEMS_PER_THREAD, - typename std::enable_if::IsValid, InType>::type* = nullptr> +template ::IsValid, bool> = true> RAFT_KERNEL __launch_bounds__(1024, 1) devKeyValSortColumnPerRow(const InType* inputKeys, InType* outputKeys, OutType* inputVals, @@ -120,12 +119,11 @@ RAFT_KERNEL __launch_bounds__(1024, 1) devKeyValSortColumnPerRow(const InType* i } } -template < - typename InType, - typename OutType, - int BLOCK_SIZE, - int ITEMS_PER_THREAD, - typename std::enable_if::IsValid), InType>::type* = nullptr> +template ::IsValid, bool> = true> RAFT_KERNEL devKeyValSortColumnPerRow(const InType* inputKeys, InType* outputKeys, OutType* inputVals, From c877dc7e570fe39f26b82230c51fc72379424f26 Mon Sep 17 00:00:00 2001 From: Jordan Jacobelli Date: Fri, 25 Oct 2024 19:19:53 +0200 Subject: [PATCH 44/79] devcontainer: replace `VAULT_HOST` with `AWS_ROLE_ARN` (#2472) This PR is replacing the `VAULT_HOST` variable with `AWS_ROLE_ARN`. This is required to use the new token service to get AWS credentials. Authors: - Jordan Jacobelli (https://github.com/jjacobelli) Approvers: - Paul Taylor (https://github.com/trxcllnt) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/raft/pull/2472 --- .devcontainer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 594ba8c3c4..dc12ab2ade 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -33,5 +33,5 @@ ENV PYTHONDONTWRITEBYTECODE="1" ENV SCCACHE_REGION="us-east-2" ENV SCCACHE_BUCKET="rapids-sccache-devs" -ENV VAULT_HOST="https://vault.ops.k8s.rapids.ai" +ENV AWS_ROLE_ARN="arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs" ENV HISTFILE="/home/coder/.cache/._bash_history" From 41ddbfefa5058d206cc0aeed3f9667782869a94e Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 31 Oct 2024 17:14:51 -0500 Subject: [PATCH 45/79] print sccache stats in builds (#2470) Contributes to https://github.com/rapidsai/build-planning/issues/111 Proposes some small packaging/CI changes, matching similar changes being made across RAPIDS. * printing `sccache` stats to CI logs * reducing `pip`'s verbosity in wheel building scripts * updating to the latest `rapids-dependency-file-generator` (v1.16.0) * always explicitly specifying `cpp` / `python` in calls to `rapids-upload-wheels-to-s3` ## Notes for Reviewers This originally also ran wheel builds with `--no-build-isolation`, but I reverted that based on https://github.com/rapidsai/build-planning/issues/108#issuecomment-2436764212. Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/raft/pull/2470 --- .pre-commit-config.yaml | 2 +- ci/build_cpp.sh | 4 ++++ ci/build_python.sh | 14 ++++++++++++++ ci/build_wheel.sh | 18 ++++++++++++++---- 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 458d8b1b51..5a5342a74e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -117,7 +117,7 @@ repos: cpp/cmake/modules/FindAVX[.]cmake - id: verify-alpha-spec - repo: https://github.com/rapidsai/dependency-file-generator - rev: v1.13.11 + rev: v1.16.0 hooks: - id: rapids-dependency-file-generator args: ["--clean"] diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index c456bcae80..92586c7c0a 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -15,6 +15,10 @@ rapids-print-env rapids-logger "Begin cpp build" +sccache --zero-stats + RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild conda/recipes/libraft +sccache --show-adv-stats + rapids-upload-conda-to-s3 cpp diff --git a/ci/build_python.sh b/ci/build_python.sh index 80d37b5ae3..dc303de4f5 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -22,6 +22,8 @@ git_commit=$(git rev-parse HEAD) export RAPIDS_PACKAGE_VERSION=${version} echo "${version}" > VERSION +sccache --zero-stats + # TODO: Remove `--no-test` flags once importing on a CPU # node works correctly rapids-conda-retry mambabuild \ @@ -29,12 +31,18 @@ rapids-conda-retry mambabuild \ --channel "${CPP_CHANNEL}" \ conda/recipes/pylibraft +sccache --show-adv-stats +sccache --zero-stats + rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ conda/recipes/raft-dask +sccache --show-adv-stats +sccache --zero-stats + # Build ann-bench for each cuda and python version rapids-conda-retry mambabuild \ --no-test \ @@ -42,15 +50,21 @@ rapids-conda-retry mambabuild \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ conda/recipes/raft-ann-bench +sccache --show-adv-stats + # Build ann-bench-cpu only in CUDA 11 jobs since it only depends on python # version RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then + sccache --zero-stats + rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ conda/recipes/raft-ann-bench-cpu + + sccache --show-adv-stats fi rapids-upload-conda-to-s3 python diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index e7ae52f33a..326ee9a4c7 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -16,7 +16,7 @@ source rapids-date-string RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -rapids-generate-version > VERSION +rapids-generate-version > ./VERSION cd "${package_dir}" @@ -39,10 +39,20 @@ case "${RAPIDS_CUDA_VERSION}" in ;; esac -# Hardcode the output dir -python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check +sccache --zero-stats + +rapids-logger "Building '${package_name}' wheel" + +python -m pip wheel \ + -w dist \ + -v \ + --no-deps \ + --disable-pip-version-check \ + . + +sccache --show-adv-stats mkdir -p final_dist python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/* -RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist +RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist From 45f24ab8989a994b63e76c9b3af3cd9fd86d5885 Mon Sep 17 00:00:00 2001 From: "Artem M. Chirkin" <9253178+achirkin@users.noreply.github.com> Date: Wed, 6 Nov 2024 07:43:40 +0100 Subject: [PATCH 46/79] Do not initialize the pinned mdarray at construction time (#2478) `thrust::host_vector` initializes its elements at creation and requires the element type be default-constructible. This translates to `raft::pinned_mdarray` and makes the mdarray unusable for non-default-constructible objects, like `cuda::atomic<>` (and many user-defined types). This is against all other mdarray types in raft, which are based on `rmm::device_uvector` and are not initialized at construction time. The PR changes the underlying container to a plain pointer + cudaMallocHost/cudaFreeHost. **Breaking change**: if anyone relies on the `pinned_mdarray` to initialize itself, the code will break (but mdarrays should not initialize at construction in raft anyway). The affected classes have different private members now, so the ABI changes as well. Authors: - Artem M. Chirkin (https://github.com/achirkin) Approvers: - William Hicks (https://github.com/wphicks) URL: https://github.com/rapidsai/raft/pull/2478 --- .../raft/core/pinned_container_policy.hpp | 63 +++++++------------ 1 file changed, 24 insertions(+), 39 deletions(-) diff --git a/cpp/include/raft/core/pinned_container_policy.hpp b/cpp/include/raft/core/pinned_container_policy.hpp index 4870e2c5dc..b661fa8860 100644 --- a/cpp/include/raft/core/pinned_container_policy.hpp +++ b/cpp/include/raft/core/pinned_container_policy.hpp @@ -19,9 +19,9 @@ #include #ifndef RAFT_DISABLE_CUDA -#include -#include -#include +#include + +#include #else #include #endif @@ -30,20 +30,16 @@ namespace raft { #ifndef RAFT_DISABLE_CUDA /** - * @brief A thin wrapper over thrust::host_vector for implementing the pinned mdarray container - * policy. + * @brief A thin wrapper over cudaMallocHost/cudaFreeHost for implementing the pinned mdarray + * container policy. * */ template struct pinned_container { - using value_type = T; - using allocator_type = - thrust::mr::stateless_resource_allocator; + using value_type = std::remove_cv_t; private: - using underlying_container_type = thrust::host_vector; - underlying_container_type data_; + value_type* data_ = nullptr; public: using size_type = std::size_t; @@ -57,21 +53,24 @@ struct pinned_container { using iterator = pointer; using const_iterator = const_pointer; - ~pinned_container() = default; - pinned_container(pinned_container&&) noexcept = default; - pinned_container(pinned_container const& that) : data_{that.data_} {} + explicit pinned_container(std::size_t size) + { + RAFT_CUDA_TRY(cudaMallocHost(&data_, size * sizeof(value_type))); + } + ~pinned_container() noexcept + { + if (data_ != nullptr) { RAFT_CUDA_TRY_NO_THROW(cudaFreeHost(data_)); } + } - auto operator=(pinned_container const& that) -> pinned_container& + pinned_container(pinned_container&& other) { std::swap(this->data_, other.data_); } + pinned_container& operator=(pinned_container&& other) { - data_ = underlying_container_type{that.data_}; + std::swap(this->data_, other.data_); return *this; } - auto operator=(pinned_container&& that) noexcept -> pinned_container& = default; + pinned_container(pinned_container const&) = delete; // Copying disallowed: one array one owner + pinned_container& operator=(pinned_container const&) = delete; - /** - * @brief Ctor that accepts a size. - */ - explicit pinned_container(std::size_t size, allocator_type const& alloc) : data_{size, alloc} {} /** * @brief Index operator that returns a reference to the actual data. */ @@ -84,15 +83,13 @@ struct pinned_container { * @brief Index operator that returns a reference to the actual data. */ template - auto operator[](Index i) const noexcept + auto operator[](Index i) const noexcept -> const_reference { return data_[i]; } - void resize(size_type size) { data_.resize(size, data_.stream()); } - - [[nodiscard]] auto data() noexcept -> pointer { return data_.data().get(); } - [[nodiscard]] auto data() const noexcept -> const_pointer { return data_.data().get(); } + [[nodiscard]] auto data() noexcept -> pointer { return data_; } + [[nodiscard]] auto data() const noexcept -> const_pointer { return data_; } }; /** @@ -102,7 +99,6 @@ template struct pinned_vector_policy { using element_type = ElementType; using container_type = pinned_container; - using allocator_type = typename container_type::allocator_type; using pointer = typename container_type::pointer; using const_pointer = typename container_type::const_pointer; using reference = typename container_type::reference; @@ -110,15 +106,7 @@ struct pinned_vector_policy { using accessor_policy = std::experimental::default_accessor; using const_accessor_policy = std::experimental::default_accessor; - auto create(raft::resources const&, size_t n) -> container_type - { - return container_type(n, allocator_); - } - - constexpr pinned_vector_policy() noexcept(std::is_nothrow_default_constructible_v) - : allocator_{} - { - } + auto create(raft::resources const&, size_t n) -> container_type { return container_type(n); } [[nodiscard]] constexpr auto access(container_type& c, size_t n) const noexcept -> reference { @@ -132,9 +120,6 @@ struct pinned_vector_policy { [[nodiscard]] auto make_accessor_policy() noexcept { return accessor_policy{}; } [[nodiscard]] auto make_accessor_policy() const noexcept { return const_accessor_policy{}; } - - private: - allocator_type allocator_; }; #else template From 53b9caab7bda762a81f76470a4cf936d3951b600 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Wed, 6 Nov 2024 10:31:56 -0500 Subject: [PATCH 47/79] Don't presume pointers location infers usability. (#2480) Here is the results of looking at the cudaPointerGetAttributes of different allocation types on Grace + Hopper. Allocations of `malloc` are still usable on the GPU. ``` ccudaPointerGetAttributes attributes malloc ptr is_dev_ptr -> 1 is_host_ptr -> 1 memory loc -> unregistered cudaPointerGetAttributes attributes cudaMalloc ptr is_dev_ptr -> 1 is_host_ptr -> 0 memory loc -> device cudaPointerGetAttributes attributes cudaMallocManaged cudaMemAttachGlobal ptr is_dev_ptr -> 1 is_host_ptr -> 1 memory loc -> managed ``` Authors: - Robert Maynard (https://github.com/robertmaynard) Approvers: - Micka (https://github.com/lowener) URL: https://github.com/rapidsai/raft/pull/2480 --- cpp/include/raft/spatial/knn/detail/ann_utils.cuh | 11 +++-------- cpp/include/raft/util/cudart_utils.hpp | 5 +++-- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/cpp/include/raft/spatial/knn/detail/ann_utils.cuh b/cpp/include/raft/spatial/knn/detail/ann_utils.cuh index 920249172f..e795a02d46 100644 --- a/cpp/include/raft/spatial/knn/detail/ann_utils.cuh +++ b/cpp/include/raft/spatial/knn/detail/ann_utils.cuh @@ -61,14 +61,9 @@ struct pointer_residency_count { auto [on_device, on_host] = pointer_residency_count::run(ptrs...); cudaPointerAttributes attr; RAFT_CUDA_TRY(cudaPointerGetAttributes(&attr, ptr)); - switch (attr.type) { - case cudaMemoryTypeUnregistered: return std::make_tuple(on_device, on_host + 1); - case cudaMemoryTypeHost: - return std::make_tuple(on_device + int(attr.devicePointer == ptr), on_host + 1); - case cudaMemoryTypeDevice: return std::make_tuple(on_device + 1, on_host); - case cudaMemoryTypeManaged: return std::make_tuple(on_device + 1, on_host + 1); - default: return std::make_tuple(on_device, on_host); - } + if (attr.devicePointer || attr.type == cudaMemoryTypeDevice) { ++on_device; } + if (attr.hostPointer || attr.type == cudaMemoryTypeUnregistered) { ++on_host; } + return std::make_tuple(on_device, on_host); } }; diff --git a/cpp/include/raft/util/cudart_utils.hpp b/cpp/include/raft/util/cudart_utils.hpp index f9e7f521be..48b8525cad 100644 --- a/cpp/include/raft/util/cudart_utils.hpp +++ b/cpp/include/raft/util/cudart_utils.hpp @@ -223,12 +223,13 @@ void print_vector(const char* variable_name, const T* ptr, size_t componentsCoun { cudaPointerAttributes attr; RAFT_CUDA_TRY(cudaPointerGetAttributes(&attr, ptr)); - if (attr.hostPointer != nullptr) { + if (attr.hostPointer) { print_host_vector(variable_name, reinterpret_cast(attr.hostPointer), componentsCount, out); } else if (attr.type == cudaMemoryTypeUnregistered) { print_host_vector(variable_name, ptr, componentsCount, out); } else { - print_device_vector(variable_name, ptr, componentsCount, out); + print_device_vector( + variable_name, reinterpret_cast(attr.devicePointer), componentsCount, out); } } /** @} */ From 615447f8f28423ea6dbed2bf808d4d3ac05b7903 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 7 Nov 2024 01:53:02 -0500 Subject: [PATCH 48/79] Put a ceiling on cuda-python (#2486) This project is incompatible with newer versions of `cuda-python`. This puts ceilings of `<=11.8.3` (CUDA 11) and `<=12.6.0` (CUDA 12) on that library. Those ceilings should be removed and replaced with `!=` constraints once new releases of `cuda-python` are up that this project is compatible with. See https://github.com/rapidsai/build-planning/issues/116 for more information. Authors: - Bradley Dice (https://github.com/bdice) - James Lamb (https://github.com/jameslamb) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/raft/pull/2486 --- conda/environments/all_cuda-118_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-125_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-125_arch-x86_64.yaml | 2 +- conda/recipes/pylibraft/meta.yaml | 7 +++++-- conda/recipes/raft-dask/meta.yaml | 7 +++++-- dependencies.yaml | 4 ++-- python/pylibraft/pyproject.toml | 1 + 8 files changed, 17 insertions(+), 10 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index c5ff93ebb9..a21f30dd94 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -14,7 +14,7 @@ dependencies: - cmake>=3.26.4,!=3.30.0 - cuda-nvtx=11.8 - cuda-profiler-api=11.8.86 -- cuda-python>=11.7.1,<12.0a0 +- cuda-python>=11.7.1,<12.0a0,<=11.8.3 - cuda-version=11.8 - cudatoolkit - cupy>=12.0.0 diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 069896c137..3eb7cdea67 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -14,7 +14,7 @@ dependencies: - cmake>=3.26.4,!=3.30.0 - cuda-nvtx=11.8 - cuda-profiler-api=11.8.86 -- cuda-python>=11.7.1,<12.0a0 +- cuda-python>=11.7.1,<12.0a0,<=11.8.3 - cuda-version=11.8 - cudatoolkit - cupy>=12.0.0 diff --git a/conda/environments/all_cuda-125_arch-aarch64.yaml b/conda/environments/all_cuda-125_arch-aarch64.yaml index 932934fb18..c13b4a4bfa 100644 --- a/conda/environments/all_cuda-125_arch-aarch64.yaml +++ b/conda/environments/all_cuda-125_arch-aarch64.yaml @@ -16,7 +16,7 @@ dependencies: - cuda-nvcc - cuda-nvtx-dev - cuda-profiler-api -- cuda-python>=12.0,<13.0a0 +- cuda-python>=12.0,<13.0a0,<=12.6.0 - cuda-version=12.5 - cupy>=12.0.0 - cxx-compiler diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 5f0cfdec68..5820017b5d 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -16,7 +16,7 @@ dependencies: - cuda-nvcc - cuda-nvtx-dev - cuda-profiler-api -- cuda-python>=12.0,<13.0a0 +- cuda-python>=12.0,<13.0a0,<=12.6.0 - cuda-version=12.5 - cupy>=12.0.0 - cxx-compiler diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 9d91af712e..ceed46a2d7 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -26,6 +26,7 @@ build: - {{ compiler('cuda') }} - cuda-cudart-dev {% endif %} + - cuda-python requirements: build: @@ -42,10 +43,10 @@ requirements: - {{ stdlib("c") }} host: {% if cuda_major == "11" %} - - cuda-python >=11.7.1,<12.0a0 + - cuda-python >=11.7.1,<12.0a0,<=11.8.3 - cudatoolkit {% else %} - - cuda-python >=12.0,<13.0a0 + - cuda-python >=12.0,<13.0a0,<=12.6.0 - cuda-cudart-dev {% endif %} - cuda-version ={{ cuda_version }} @@ -60,8 +61,10 @@ requirements: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} - cudatoolkit + - cuda-python >=11.7.1,<12.0a0,<=11.8.3 {% else %} - cuda-cudart + - cuda-python >=12.0,<13.0a0,<=12.6.0 {% endif %} - libraft {{ version }} - libraft-headers {{ version }} diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index bc13d352b7..b2e468f7ea 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -26,6 +26,7 @@ build: - {{ compiler('cuda') }} - cuda-cudart-dev {% endif %} + - cuda-python requirements: build: @@ -42,10 +43,10 @@ requirements: - {{ stdlib("c") }} host: {% if cuda_major == "11" %} - - cuda-python >=11.7.1,<12.0a0 + - cuda-python >=11.7.1,<12.0a0,<=11.8.3 - cudatoolkit {% else %} - - cuda-python >=12.0,<13.0a0 + - cuda-python >=12.0,<13.0a0,<=12.6.0 - cuda-cudart-dev {% endif %} - cuda-version ={{ cuda_version }} @@ -61,8 +62,10 @@ requirements: run: {% if cuda_major == "11" %} - cudatoolkit + - cuda-python >=11.7.1,<12.0a0,<=11.8.3 {% else %} - cuda-cudart + - cuda-python >=12.0,<13.0a0,<=12.6.0 {% endif %} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - dask-cuda ={{ minor_version }} diff --git a/dependencies.yaml b/dependencies.yaml index 6c33ba92b5..f31464aa91 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -223,11 +223,11 @@ dependencies: - matrix: cuda: "12.*" packages: - - &cuda_python12 cuda-python>=12.0,<13.0a0 + - &cuda_python12 cuda-python>=12.0,<13.0a0,<=12.6.0 - matrix: cuda: "11.*" packages: - - &cuda_python11 cuda-python>=11.7.1,<12.0a0 + - &cuda_python11 cuda-python>=11.7.1,<12.0a0,<=11.8.3 - matrix: packages: - &cuda_python cuda-python diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index f0f3849c6d..599b3dd54e 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -133,4 +133,5 @@ matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" [tool.pytest.ini_options] filterwarnings = [ "error", + "ignore:.*cuda..* module is deprecated.*:DeprecationWarning", ] From a6712ba10e6ba60547bed6968fe26c6397a7a5b0 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 14 Nov 2024 08:40:36 -0600 Subject: [PATCH 49/79] enforce wheel size limits and README formatting in CI, put a ceiling on Cython dependency (#2490) Contributes to https://github.com/rapidsai/build-planning/issues/110 Proposes adding 2 types of validation on wheels in CI, to ensure we continue to produce wheels that are suitable for PyPI. * checks on wheel size (compressed), - *to be sure they're under PyPI limits* - *and to prompt discussion on PRs that significantly increase wheel sizes* * checks on README formatting - *to ensure they'll render properly as the PyPI project homepages* - *e.g. like how https://github.com/scikit-learn/scikit-learn/blob/main/README.rst becomes https://pypi.org/project/scikit-learn/* Also puts a ceiling on Cython to its latest stable release (`<=3.0.11`), to fix https://github.com/rapidsai/raft/pull/2490#issuecomment-2474375209. Work to relax that is tracked in (https://github.com/rapidsai/raft/issues/2491). Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/raft/pull/2490 --- ci/build_wheel_pylibraft.sh | 5 ++++- ci/build_wheel_raft_dask.sh | 5 ++++- ci/validate_wheel.sh | 21 +++++++++++++++++++ .../all_cuda-118_arch-aarch64.yaml | 2 +- .../all_cuda-118_arch-x86_64.yaml | 2 +- .../all_cuda-125_arch-aarch64.yaml | 2 +- .../all_cuda-125_arch-x86_64.yaml | 2 +- .../bench_ann_cuda-118_arch-aarch64.yaml | 2 +- .../bench_ann_cuda-118_arch-x86_64.yaml | 2 +- .../bench_ann_cuda-120_arch-aarch64.yaml | 2 +- .../bench_ann_cuda-120_arch-x86_64.yaml | 2 +- conda/recipes/pylibraft/meta.yaml | 2 +- conda/recipes/raft-dask/meta.yaml | 2 +- dependencies.yaml | 2 +- python/pylibraft/pyproject.toml | 10 ++++++++- python/raft-dask/pyproject.toml | 10 ++++++++- 16 files changed, 58 insertions(+), 15 deletions(-) create mode 100755 ci/validate_wheel.sh diff --git a/ci/build_wheel_pylibraft.sh b/ci/build_wheel_pylibraft.sh index ce9f0ed172..dacaa1190e 100755 --- a/ci/build_wheel_pylibraft.sh +++ b/ci/build_wheel_pylibraft.sh @@ -3,6 +3,8 @@ set -euo pipefail +package_dir="python/pylibraft" + case "${RAPIDS_CUDA_VERSION}" in 12.*) EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=ON" @@ -15,4 +17,5 @@ esac # Set up skbuild options. Enable sccache in skbuild config options export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF${EXTRA_CMAKE_ARGS}" -ci/build_wheel.sh pylibraft python/pylibraft +ci/build_wheel.sh pylibraft ${package_dir} +ci/validate_wheel.sh ${package_dir} final_dist diff --git a/ci/build_wheel_raft_dask.sh b/ci/build_wheel_raft_dask.sh index feba2d7a5b..e4f3f0a833 100755 --- a/ci/build_wheel_raft_dask.sh +++ b/ci/build_wheel_raft_dask.sh @@ -3,7 +3,10 @@ set -euo pipefail +package_dir="python/raft-dask" + # Set up skbuild options. Enable sccache in skbuild config options export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF" -ci/build_wheel.sh raft-dask python/raft-dask +ci/build_wheel.sh raft-dask ${package_dir} +ci/validate_wheel.sh ${package_dir} final_dist diff --git a/ci/validate_wheel.sh b/ci/validate_wheel.sh new file mode 100755 index 0000000000..5910a5c59f --- /dev/null +++ b/ci/validate_wheel.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +package_dir=$1 +wheel_dir_relative_path=$2 + +cd "${package_dir}" + +rapids-logger "validate packages with 'pydistcheck'" + +pydistcheck \ + --inspect \ + "$(echo ${wheel_dir_relative_path}/*.whl)" + +rapids-logger "validate packages with 'twine'" + +twine check \ + --strict \ + "$(echo ${wheel_dir_relative_path}/*.whl)" diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index a21f30dd94..6098cd12bf 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -19,7 +19,7 @@ dependencies: - cudatoolkit - cupy>=12.0.0 - cxx-compiler -- cython>=3.0.0 +- cython>=3.0.0,<3.1.0a0 - dask-cuda==24.12.*,>=0.0.0a0 - distributed-ucxx==0.41.*,>=0.0.0a0 - doxygen>=1.8.20 diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 3eb7cdea67..0fe8fbab39 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -19,7 +19,7 @@ dependencies: - cudatoolkit - cupy>=12.0.0 - cxx-compiler -- cython>=3.0.0 +- cython>=3.0.0,<3.1.0a0 - dask-cuda==24.12.*,>=0.0.0a0 - distributed-ucxx==0.41.*,>=0.0.0a0 - doxygen>=1.8.20 diff --git a/conda/environments/all_cuda-125_arch-aarch64.yaml b/conda/environments/all_cuda-125_arch-aarch64.yaml index c13b4a4bfa..dfb9ac0b97 100644 --- a/conda/environments/all_cuda-125_arch-aarch64.yaml +++ b/conda/environments/all_cuda-125_arch-aarch64.yaml @@ -20,7 +20,7 @@ dependencies: - cuda-version=12.5 - cupy>=12.0.0 - cxx-compiler -- cython>=3.0.0 +- cython>=3.0.0,<3.1.0a0 - dask-cuda==24.12.*,>=0.0.0a0 - distributed-ucxx==0.41.*,>=0.0.0a0 - doxygen>=1.8.20 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 5820017b5d..bf6f5d6462 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -20,7 +20,7 @@ dependencies: - cuda-version=12.5 - cupy>=12.0.0 - cxx-compiler -- cython>=3.0.0 +- cython>=3.0.0,<3.1.0a0 - dask-cuda==24.12.*,>=0.0.0a0 - distributed-ucxx==0.41.*,>=0.0.0a0 - doxygen>=1.8.20 diff --git a/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml index 4c9d308ecd..39bdf2671d 100644 --- a/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml +++ b/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml @@ -17,7 +17,7 @@ dependencies: - cuda-version=11.8 - cudatoolkit - cxx-compiler -- cython>=3.0.0 +- cython>=3.0.0,<3.1.0a0 - gcc_linux-aarch64=11.* - glog>=0.6.0 - h5py>=3.8.0 diff --git a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml index 1b62c492cf..56004fa818 100644 --- a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml +++ b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml @@ -17,7 +17,7 @@ dependencies: - cuda-version=11.8 - cudatoolkit - cxx-compiler -- cython>=3.0.0 +- cython>=3.0.0,<3.1.0a0 - gcc_linux-64=11.* - glog>=0.6.0 - h5py>=3.8.0 diff --git a/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml index 54d67f462a..5f0599d9ae 100644 --- a/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml +++ b/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml @@ -18,7 +18,7 @@ dependencies: - cuda-profiler-api - cuda-version=12.0 - cxx-compiler -- cython>=3.0.0 +- cython>=3.0.0,<3.1.0a0 - gcc_linux-aarch64=11.* - glog>=0.6.0 - h5py>=3.8.0 diff --git a/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml index 4f39378047..849e6c1412 100644 --- a/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml +++ b/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml @@ -18,7 +18,7 @@ dependencies: - cuda-profiler-api - cuda-version=12.0 - cxx-compiler -- cython>=3.0.0 +- cython>=3.0.0,<3.1.0a0 - gcc_linux-64=11.* - glog>=0.6.0 - h5py>=3.8.0 diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index ceed46a2d7..01a9d61f0f 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -50,7 +50,7 @@ requirements: - cuda-cudart-dev {% endif %} - cuda-version ={{ cuda_version }} - - cython >=3.0.0 + - cython >=3.0.0,<3.1.0a0 - libraft {{ version }} - libraft-headers {{ version }} - python x.x diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index b2e468f7ea..02a8957b06 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -50,7 +50,7 @@ requirements: - cuda-cudart-dev {% endif %} - cuda-version ={{ cuda_version }} - - cython >=3.0.0 + - cython >=3.0.0,<3.1.0a0 - nccl {{ nccl_version }} - pylibraft {{ version }} - python x.x diff --git a/dependencies.yaml b/dependencies.yaml index f31464aa91..7766481c99 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -164,7 +164,7 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - &cmake_ver cmake>=3.26.4,!=3.30.0 - - cython>=3.0.0 + - cython>=3.0.0,<3.1.0a0 - ninja - output_types: [conda] packages: diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index 599b3dd54e..bb01602b33 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -123,13 +123,21 @@ build-backend = "scikit_build_core.build" requires = [ "cmake>=3.26.4,!=3.30.0", "cuda-python", - "cython>=3.0.0", + "cython>=3.0.0,<3.1.0a0", "ninja", "rmm==24.12.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. dependencies-file = "../../dependencies.yaml" matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" +[tool.pydistcheck] +select = [ + "distro-too-large-compressed", +] + +# detect when package size grows significantly +max_allowed_size_compressed = '825M' + [tool.pytest.ini_options] filterwarnings = [ "error", diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index d71f89085b..a9f4de5dc3 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -120,9 +120,17 @@ regex = "(?P.*)" build-backend = "scikit_build_core.build" requires = [ "cmake>=3.26.4,!=3.30.0", - "cython>=3.0.0", + "cython>=3.0.0,<3.1.0a0", "libucx==1.15.0", "ninja", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. dependencies-file = "../../dependencies.yaml" matrix-entry = "cuda_suffixed=true" + +[tool.pydistcheck] +select = [ + "distro-too-large-compressed", +] + +# detect when package size grows significantly +max_allowed_size_compressed = '300M' From 8a9bf5c1ee685547e904119482da34cdd252f9dd Mon Sep 17 00:00:00 2001 From: Tarang Jain <40517122+tarang-jain@users.noreply.github.com> Date: Fri, 15 Nov 2024 09:14:11 +0530 Subject: [PATCH 50/79] Pin FAISS Version for raft-ann-bench (#2496) A simple PR for pinning the FAISS version to fetch the compatible tag for raft-ann-bench Authors: - Tarang Jain (https://github.com/tarang-jain) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/raft/pull/2496 --- cpp/cmake/patches/faiss_override.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/cmake/patches/faiss_override.json b/cpp/cmake/patches/faiss_override.json index 19dad362b9..5d18c77fec 100644 --- a/cpp/cmake/patches/faiss_override.json +++ b/cpp/cmake/patches/faiss_override.json @@ -1,9 +1,9 @@ { "packages" : { "faiss" : { - "version": "1.7.4", + "version": "1.9.0", "git_url": "https://github.com/facebookresearch/faiss.git", - "git_tag": "main" + "git_tag": "v1.9.0" } } } From 31d6c5dbd57b85c169564c4b582e578fd6915dba Mon Sep 17 00:00:00 2001 From: Ray Douglass Date: Fri, 15 Nov 2024 09:31:27 -0500 Subject: [PATCH 51/79] DOC v25.02 Updates [skip ci] --- .../cuda11.8-conda/devcontainer.json | 6 ++-- .devcontainer/cuda11.8-pip/devcontainer.json | 8 ++--- .../cuda12.5-conda/devcontainer.json | 6 ++-- .devcontainer/cuda12.5-pip/devcontainer.json | 8 ++--- .github/workflows/build.yaml | 16 +++++----- .github/workflows/pr.yaml | 28 ++++++++--------- .github/workflows/test.yaml | 10 +++---- README.md | 2 +- VERSION | 2 +- .../all_cuda-118_arch-aarch64.yaml | 14 ++++----- .../all_cuda-118_arch-x86_64.yaml | 14 ++++----- .../all_cuda-125_arch-aarch64.yaml | 14 ++++----- .../all_cuda-125_arch-x86_64.yaml | 14 ++++----- .../bench_ann_cuda-118_arch-aarch64.yaml | 4 +-- .../bench_ann_cuda-118_arch-x86_64.yaml | 4 +-- .../bench_ann_cuda-120_arch-aarch64.yaml | 4 +-- .../bench_ann_cuda-120_arch-x86_64.yaml | 4 +-- .../recipes/raft-dask/conda_build_config.yaml | 4 +-- .../cmake/thirdparty/fetch_rapids.cmake | 2 +- dependencies.yaml | 30 +++++++++---------- docs/source/build.md | 2 +- docs/source/developer_guide.md | 6 ++-- docs/source/raft_ann_benchmarks.md | 12 ++++---- python/pylibraft/pyproject.toml | 4 +-- .../raft-dask/cmake/thirdparty/get_ucxx.cmake | 4 +-- python/raft-dask/pyproject.toml | 10 +++---- 26 files changed, 116 insertions(+), 116 deletions(-) diff --git a/.devcontainer/cuda11.8-conda/devcontainer.json b/.devcontainer/cuda11.8-conda/devcontainer.json index 008bf8730a..8c857961c2 100644 --- a/.devcontainer/cuda11.8-conda/devcontainer.json +++ b/.devcontainer/cuda11.8-conda/devcontainer.json @@ -5,17 +5,17 @@ "args": { "CUDA": "11.8", "PYTHON_PACKAGE_MANAGER": "conda", - "BASE": "rapidsai/devcontainers:24.12-cpp-cuda11.8-mambaforge-ubuntu22.04" + "BASE": "rapidsai/devcontainers:25.02-cpp-cuda11.8-mambaforge-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.12-cuda11.8-conda" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.02-cuda11.8-conda" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.12": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:25.2": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json index 75aed80f9f..c691ed6007 100644 --- a/.devcontainer/cuda11.8-pip/devcontainer.json +++ b/.devcontainer/cuda11.8-pip/devcontainer.json @@ -5,24 +5,24 @@ "args": { "CUDA": "11.8", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:24.12-cpp-cuda11.8-ucx1.17.0-openmpi-ubuntu22.04" + "BASE": "rapidsai/devcontainers:25.02-cpp-cuda11.8-ucx1.17.0-openmpi-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.12-cuda11.8-pip" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.02-cuda11.8-pip" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/cuda:24.12": { + "ghcr.io/rapidsai/devcontainers/features/cuda:25.2": { "version": "11.8", "installcuBLAS": true, "installcuSOLVER": true, "installcuRAND": true, "installcuSPARSE": true }, - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.12": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:25.2": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/ucx", diff --git a/.devcontainer/cuda12.5-conda/devcontainer.json b/.devcontainer/cuda12.5-conda/devcontainer.json index 240ba02131..dc4fcd02fd 100644 --- a/.devcontainer/cuda12.5-conda/devcontainer.json +++ b/.devcontainer/cuda12.5-conda/devcontainer.json @@ -5,17 +5,17 @@ "args": { "CUDA": "12.5", "PYTHON_PACKAGE_MANAGER": "conda", - "BASE": "rapidsai/devcontainers:24.12-cpp-mambaforge-ubuntu22.04" + "BASE": "rapidsai/devcontainers:25.02-cpp-mambaforge-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.12-cuda12.5-conda" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.02-cuda12.5-conda" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.12": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:25.2": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" diff --git a/.devcontainer/cuda12.5-pip/devcontainer.json b/.devcontainer/cuda12.5-pip/devcontainer.json index c23c79017a..bc43900ef3 100644 --- a/.devcontainer/cuda12.5-pip/devcontainer.json +++ b/.devcontainer/cuda12.5-pip/devcontainer.json @@ -5,24 +5,24 @@ "args": { "CUDA": "12.5", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:24.12-cpp-cuda12.5-ucx1.17.0-openmpi-ubuntu22.04" + "BASE": "rapidsai/devcontainers:25.02-cpp-cuda12.5-ucx1.17.0-openmpi-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.12-cuda12.5-pip" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.02-cuda12.5-pip" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/cuda:24.12": { + "ghcr.io/rapidsai/devcontainers/features/cuda:25.2": { "version": "12.5", "installcuBLAS": true, "installcuSOLVER": true, "installcuRAND": true, "installcuSPARSE": true }, - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.12": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:25.2": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/ucx", diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index db379c9d47..7879f22879 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-25.02 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -37,7 +37,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-25.02 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -46,7 +46,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-25.02 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -57,7 +57,7 @@ jobs: if: github.ref_type == 'branch' needs: python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.02 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-pylibraft: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -79,7 +79,7 @@ jobs: wheel-publish-pylibraft: needs: wheel-build-pylibraft secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-25.02 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -88,7 +88,7 @@ jobs: package-name: pylibraft wheel-build-raft-dask: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -98,7 +98,7 @@ jobs: wheel-publish-raft-dask: needs: wheel-build-raft-dask secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-25.02 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index fe8e730921..e349b25ce6 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -26,13 +26,13 @@ jobs: - wheel-tests-raft-dask - devcontainer secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-25.02 if: always() with: needs: ${{ toJSON(needs) }} changed-files: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-25.02 with: files_yaml: | test_cpp: @@ -65,27 +65,27 @@ jobs: - '!thirdparty/LICENSES/**' checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-25.02 with: enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-25.02 with: build_type: pull-request node_type: cpu16 conda-cpp-tests: needs: [conda-cpp-build, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-25.02 if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp with: build_type: pull-request conda-cpp-checks: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-25.02 with: build_type: pull-request enable_check_symbols: true @@ -93,20 +93,20 @@ jobs: conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-25.02 with: build_type: pull-request conda-python-tests: needs: [conda-python-build, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-25.02 if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.02 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -116,14 +116,14 @@ jobs: wheel-build-pylibraft: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 with: build_type: pull-request script: ci/build_wheel_pylibraft.sh wheel-tests-pylibraft: needs: [wheel-build-pylibraft, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.02 if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request @@ -131,21 +131,21 @@ jobs: wheel-build-raft-dask: needs: wheel-tests-pylibraft secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 with: build_type: pull-request script: "ci/build_wheel_raft_dask.sh" wheel-tests-raft-dask: needs: [wheel-build-raft-dask, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.02 if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request script: ci/test_wheel_raft_dask.sh devcontainer: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.02 with: arch: '["amd64"]' cuda: '["12.5"]' diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2bee8a3d1d..1ae093bc56 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-25.02 with: build_type: nightly branch: ${{ inputs.branch }} @@ -26,7 +26,7 @@ jobs: symbol_exclusions: raft_cutlass conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-25.02 with: build_type: nightly branch: ${{ inputs.branch }} @@ -34,7 +34,7 @@ jobs: sha: ${{ inputs.sha }} conda-python-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-25.02 with: build_type: nightly branch: ${{ inputs.branch }} @@ -42,7 +42,7 @@ jobs: sha: ${{ inputs.sha }} wheel-tests-pylibraft: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.02 with: build_type: nightly branch: ${{ inputs.branch }} @@ -51,7 +51,7 @@ jobs: script: ci/test_wheel_pylibraft.sh wheel-tests-raft-dask: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.02 with: build_type: nightly branch: ${{ inputs.branch }} diff --git a/README.md b/README.md index 8870e9385e..8d16fc5842 100755 --- a/README.md +++ b/README.md @@ -255,7 +255,7 @@ You can also install the conda packages individually using the `mamba` command a mamba install -c rapidsai -c conda-forge -c nvidia libraft libraft-headers cuda-version=12.5 ``` -If installing the C++ APIs please see [using libraft](https://docs.rapids.ai/api/raft/nightly/using_libraft/) for more information on using the pre-compiled shared library. You can also refer to the [example C++ template project](https://github.com/rapidsai/raft/tree/branch-24.12/cpp/template) for a ready-to-go CMake configuration that you can drop into your project and build against installed RAFT development artifacts above. +If installing the C++ APIs please see [using libraft](https://docs.rapids.ai/api/raft/nightly/using_libraft/) for more information on using the pre-compiled shared library. You can also refer to the [example C++ template project](https://github.com/rapidsai/raft/tree/branch-25.02/cpp/template) for a ready-to-go CMake configuration that you can drop into your project and build against installed RAFT development artifacts above. ### Installing Python through Pip diff --git a/VERSION b/VERSION index af28c42b52..72eefaf7c7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -24.12.00 +25.02.00 diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index 6098cd12bf..269af03e9f 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -20,8 +20,8 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0,<3.1.0a0 -- dask-cuda==24.12.*,>=0.0.0a0 -- distributed-ucxx==0.41.*,>=0.0.0a0 +- dask-cuda==25.2.*,>=0.0.0a0 +- distributed-ucxx==0.42.*,>=0.0.0a0 - doxygen>=1.8.20 - gcc_linux-aarch64=11.* - graphviz @@ -35,7 +35,7 @@ dependencies: - libcusolver=11.4.1.48 - libcusparse-dev=11.7.5.86 - libcusparse=11.7.5.86 -- libucxx==0.41.*,>=0.0.0a0 +- libucxx==0.42.*,>=0.0.0a0 - nccl>=2.19 - ninja - numba>=0.57 @@ -44,18 +44,18 @@ dependencies: - nvcc_linux-aarch64=11.8 - pre-commit - pydata-sphinx-theme -- pylibraft==24.12.*,>=0.0.0a0 +- pylibraft==25.2.*,>=0.0.0a0 - pytest-cov - pytest==7.* - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rapids-dask-dependency==24.12.*,>=0.0.0a0 +- rapids-dask-dependency==25.2.*,>=0.0.0a0 - recommonmark -- rmm==24.12.*,>=0.0.0a0 +- rmm==25.2.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - scikit-learn - scipy - sphinx-copybutton - sphinx-markdown-tables - sysroot_linux-aarch64==2.17 -- ucx-py==0.41.*,>=0.0.0a0 +- ucx-py==0.42.*,>=0.0.0a0 name: all_cuda-118_arch-aarch64 diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 0fe8fbab39..4c7150264b 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -20,8 +20,8 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0,<3.1.0a0 -- dask-cuda==24.12.*,>=0.0.0a0 -- distributed-ucxx==0.41.*,>=0.0.0a0 +- dask-cuda==25.2.*,>=0.0.0a0 +- distributed-ucxx==0.42.*,>=0.0.0a0 - doxygen>=1.8.20 - gcc_linux-64=11.* - graphviz @@ -35,7 +35,7 @@ dependencies: - libcusolver=11.4.1.48 - libcusparse-dev=11.7.5.86 - libcusparse=11.7.5.86 -- libucxx==0.41.*,>=0.0.0a0 +- libucxx==0.42.*,>=0.0.0a0 - nccl>=2.19 - ninja - numba>=0.57 @@ -44,18 +44,18 @@ dependencies: - nvcc_linux-64=11.8 - pre-commit - pydata-sphinx-theme -- pylibraft==24.12.*,>=0.0.0a0 +- pylibraft==25.2.*,>=0.0.0a0 - pytest-cov - pytest==7.* - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rapids-dask-dependency==24.12.*,>=0.0.0a0 +- rapids-dask-dependency==25.2.*,>=0.0.0a0 - recommonmark -- rmm==24.12.*,>=0.0.0a0 +- rmm==25.2.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - scikit-learn - scipy - sphinx-copybutton - sphinx-markdown-tables - sysroot_linux-64==2.17 -- ucx-py==0.41.*,>=0.0.0a0 +- ucx-py==0.42.*,>=0.0.0a0 name: all_cuda-118_arch-x86_64 diff --git a/conda/environments/all_cuda-125_arch-aarch64.yaml b/conda/environments/all_cuda-125_arch-aarch64.yaml index dfb9ac0b97..648a5a00f0 100644 --- a/conda/environments/all_cuda-125_arch-aarch64.yaml +++ b/conda/environments/all_cuda-125_arch-aarch64.yaml @@ -21,8 +21,8 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0,<3.1.0a0 -- dask-cuda==24.12.*,>=0.0.0a0 -- distributed-ucxx==0.41.*,>=0.0.0a0 +- dask-cuda==25.2.*,>=0.0.0a0 +- distributed-ucxx==0.42.*,>=0.0.0a0 - doxygen>=1.8.20 - gcc_linux-aarch64=11.* - graphviz @@ -32,7 +32,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libucxx==0.41.*,>=0.0.0a0 +- libucxx==0.42.*,>=0.0.0a0 - nccl>=2.19 - ninja - numba>=0.57 @@ -40,18 +40,18 @@ dependencies: - numpydoc - pre-commit - pydata-sphinx-theme -- pylibraft==24.12.*,>=0.0.0a0 +- pylibraft==25.2.*,>=0.0.0a0 - pytest-cov - pytest==7.* - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rapids-dask-dependency==24.12.*,>=0.0.0a0 +- rapids-dask-dependency==25.2.*,>=0.0.0a0 - recommonmark -- rmm==24.12.*,>=0.0.0a0 +- rmm==25.2.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - scikit-learn - scipy - sphinx-copybutton - sphinx-markdown-tables - sysroot_linux-aarch64==2.17 -- ucx-py==0.41.*,>=0.0.0a0 +- ucx-py==0.42.*,>=0.0.0a0 name: all_cuda-125_arch-aarch64 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index bf6f5d6462..7d7b9c4454 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -21,8 +21,8 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0,<3.1.0a0 -- dask-cuda==24.12.*,>=0.0.0a0 -- distributed-ucxx==0.41.*,>=0.0.0a0 +- dask-cuda==25.2.*,>=0.0.0a0 +- distributed-ucxx==0.42.*,>=0.0.0a0 - doxygen>=1.8.20 - gcc_linux-64=11.* - graphviz @@ -32,7 +32,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libucxx==0.41.*,>=0.0.0a0 +- libucxx==0.42.*,>=0.0.0a0 - nccl>=2.19 - ninja - numba>=0.57 @@ -40,18 +40,18 @@ dependencies: - numpydoc - pre-commit - pydata-sphinx-theme -- pylibraft==24.12.*,>=0.0.0a0 +- pylibraft==25.2.*,>=0.0.0a0 - pytest-cov - pytest==7.* - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rapids-dask-dependency==24.12.*,>=0.0.0a0 +- rapids-dask-dependency==25.2.*,>=0.0.0a0 - recommonmark -- rmm==24.12.*,>=0.0.0a0 +- rmm==25.2.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - scikit-learn - scipy - sphinx-copybutton - sphinx-markdown-tables - sysroot_linux-64==2.17 -- ucx-py==0.41.*,>=0.0.0a0 +- ucx-py==0.42.*,>=0.0.0a0 name: all_cuda-125_arch-x86_64 diff --git a/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml index 39bdf2671d..777d2ddb7f 100644 --- a/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml +++ b/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml @@ -30,7 +30,7 @@ dependencies: - libcusolver=11.4.1.48 - libcusparse-dev=11.7.5.86 - libcusparse=11.7.5.86 -- libucxx==0.41.*,>=0.0.0a0 +- libucxx==0.42.*,>=0.0.0a0 - matplotlib - nccl>=2.19 - ninja @@ -40,7 +40,7 @@ dependencies: - pandas - pyyaml - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rmm==24.12.*,>=0.0.0a0 +- rmm==25.2.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - sysroot_linux-aarch64==2.17 name: bench_ann_cuda-118_arch-aarch64 diff --git a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml index 56004fa818..7fa432c8d6 100644 --- a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml +++ b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml @@ -30,7 +30,7 @@ dependencies: - libcusolver=11.4.1.48 - libcusparse-dev=11.7.5.86 - libcusparse=11.7.5.86 -- libucxx==0.41.*,>=0.0.0a0 +- libucxx==0.42.*,>=0.0.0a0 - matplotlib - nccl>=2.19 - ninja @@ -40,7 +40,7 @@ dependencies: - pandas - pyyaml - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rmm==24.12.*,>=0.0.0a0 +- rmm==25.2.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - sysroot_linux-64==2.17 name: bench_ann_cuda-118_arch-x86_64 diff --git a/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml index 5f0599d9ae..0f59fc6090 100644 --- a/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml +++ b/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml @@ -27,7 +27,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libucxx==0.41.*,>=0.0.0a0 +- libucxx==0.42.*,>=0.0.0a0 - matplotlib - nccl>=2.19 - ninja @@ -36,7 +36,7 @@ dependencies: - pandas - pyyaml - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rmm==24.12.*,>=0.0.0a0 +- rmm==25.2.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - sysroot_linux-aarch64==2.17 name: bench_ann_cuda-120_arch-aarch64 diff --git a/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml index 849e6c1412..273d6a9f9b 100644 --- a/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml +++ b/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml @@ -27,7 +27,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libucxx==0.41.*,>=0.0.0a0 +- libucxx==0.42.*,>=0.0.0a0 - matplotlib - nccl>=2.19 - ninja @@ -36,7 +36,7 @@ dependencies: - pandas - pyyaml - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rmm==24.12.*,>=0.0.0a0 +- rmm==25.2.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - sysroot_linux-64==2.17 name: bench_ann_cuda-120_arch-x86_64 diff --git a/conda/recipes/raft-dask/conda_build_config.yaml b/conda/recipes/raft-dask/conda_build_config.yaml index d7d2f68b42..68140e6bc0 100644 --- a/conda/recipes/raft-dask/conda_build_config.yaml +++ b/conda/recipes/raft-dask/conda_build_config.yaml @@ -17,10 +17,10 @@ c_stdlib_version: - "2.17" ucx_py_version: - - "0.41.*" + - "0.42.*" ucxx_version: - - "0.41.*" + - "0.42.*" cmake_version: - ">=3.26.4,!=3.30.0" diff --git a/cpp/template/cmake/thirdparty/fetch_rapids.cmake b/cpp/template/cmake/thirdparty/fetch_rapids.cmake index 6f4c627ed4..23c8490b40 100644 --- a/cpp/template/cmake/thirdparty/fetch_rapids.cmake +++ b/cpp/template/cmake/thirdparty/fetch_rapids.cmake @@ -12,7 +12,7 @@ # the License. # Use this variable to update RAPIDS and RAFT versions -set(RAPIDS_VERSION "24.12") +set(RAPIDS_VERSION "25.02") if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/RAFT_RAPIDS.cmake) file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-${RAPIDS_VERSION}/RAPIDS.cmake diff --git a/dependencies.yaml b/dependencies.yaml index 7766481c99..fb58e93f71 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -171,7 +171,7 @@ dependencies: - c-compiler - cxx-compiler - nccl>=2.19 - - libucxx==0.41.*,>=0.0.0a0 + - libucxx==0.42.*,>=0.0.0a0 specific: - output_types: conda matrices: @@ -210,7 +210,7 @@ dependencies: common: - output_types: [conda] packages: - - &rmm_unsuffixed rmm==24.12.*,>=0.0.0a0 + - &rmm_unsuffixed rmm==25.2.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -237,12 +237,12 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - &rmm_cu12 rmm-cu12==24.12.*,>=0.0.0a0 + - &rmm_cu12 rmm-cu12==25.2.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - &rmm_cu11 rmm-cu11==24.12.*,>=0.0.0a0 + - &rmm_cu11 rmm-cu11==25.2.*,>=0.0.0a0 - {matrix: null, packages: [*rmm_unsuffixed] } checks: common: @@ -514,14 +514,14 @@ dependencies: common: - output_types: [conda, pyproject] packages: - - dask-cuda==24.12.*,>=0.0.0a0 + - dask-cuda==25.2.*,>=0.0.0a0 - joblib>=0.11 - numba>=0.57 - - rapids-dask-dependency==24.12.*,>=0.0.0a0 + - rapids-dask-dependency==25.2.*,>=0.0.0a0 - output_types: conda packages: - - &pylibraft_unsuffixed pylibraft==24.12.*,>=0.0.0a0 - - &ucx_py_unsuffixed ucx-py==0.41.*,>=0.0.0a0 + - &pylibraft_unsuffixed pylibraft==25.2.*,>=0.0.0a0 + - &ucx_py_unsuffixed ucx-py==0.42.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -535,14 +535,14 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - &pylibraft_cu12 pylibraft-cu12==24.12.*,>=0.0.0a0 - - &ucx_py_cu12 ucx-py-cu12==0.41.*,>=0.0.0a0 + - &pylibraft_cu12 pylibraft-cu12==25.2.*,>=0.0.0a0 + - &ucx_py_cu12 ucx-py-cu12==0.42.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - &pylibraft_cu11 pylibraft-cu11==24.12.*,>=0.0.0a0 - - &ucx_py_cu11 ucx-py-cu11==0.41.*,>=0.0.0a0 + - &pylibraft_cu11 pylibraft-cu11==25.2.*,>=0.0.0a0 + - &ucx_py_cu11 ucx-py-cu11==0.42.*,>=0.0.0a0 - {matrix: null, packages: [*pylibraft_unsuffixed, *ucx_py_unsuffixed]} test_python_common: common: @@ -562,7 +562,7 @@ dependencies: packages: # UCXX is not currently a hard-dependency thus only installed during tests, # this will change in the future. - - &distributed_ucxx_unsuffixed distributed-ucxx==0.41.*,>=0.0.0a0 + - &distributed_ucxx_unsuffixed distributed-ucxx==0.42.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -575,12 +575,12 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - distributed-ucxx-cu12==0.41.*,>=0.0.0a0 + - distributed-ucxx-cu12==0.42.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - distributed-ucxx-cu11==0.41.*,>=0.0.0a0 + - distributed-ucxx-cu11==0.42.*,>=0.0.0a0 - {matrix: null, packages: [*distributed_ucxx_unsuffixed]} depends_on_ucx_build: common: diff --git a/docs/source/build.md b/docs/source/build.md index b9a1832b02..0c4ab17ed0 100644 --- a/docs/source/build.md +++ b/docs/source/build.md @@ -56,7 +56,7 @@ You can also install the conda packages individually using the `mamba` command a mamba install -c rapidsai -c conda-forge -c nvidia libraft libraft-headers cuda-version=12.0 ``` -If installing the C++ APIs Please see [using libraft](https://docs.rapids.ai/api/raft/nightly/using_libraft/) for more information on using the pre-compiled shared library. You can also refer to the [example C++ template project](https://github.com/rapidsai/raft/tree/branch-24.12/cpp/template) for a ready-to-go CMake configuration that you can drop into your project and build against installed RAFT development artifacts above. +If installing the C++ APIs Please see [using libraft](https://docs.rapids.ai/api/raft/nightly/using_libraft/) for more information on using the pre-compiled shared library. You can also refer to the [example C++ template project](https://github.com/rapidsai/raft/tree/branch-25.02/cpp/template) for a ready-to-go CMake configuration that you can drop into your project and build against installed RAFT development artifacts above. ## Installing Python through Pip diff --git a/docs/source/developer_guide.md b/docs/source/developer_guide.md index c4a099fabb..5cc694dc8f 100644 --- a/docs/source/developer_guide.md +++ b/docs/source/developer_guide.md @@ -187,7 +187,7 @@ RAFT relies on `clang-format` to enforce code style across all C++ and CUDA sour 1. Do not split empty functions/records/namespaces. 2. Two-space indentation everywhere, including the line continuations. 3. Disable reflowing of comments. - The reasons behind these deviations from the Google style guide are given in comments [here](https://github.com/rapidsai/raft/blob/branch-24.12/cpp/.clang-format). + The reasons behind these deviations from the Google style guide are given in comments [here](https://github.com/rapidsai/raft/blob/branch-25.02/cpp/.clang-format). [`doxygen`](https://doxygen.nl/) is used as documentation generator and also as a documentation linter. In order to run doxygen as a linter on C++/CUDA code, run @@ -205,7 +205,7 @@ you can run `codespell -i 3 -w .` from the repository root directory. This will bring up an interactive prompt to select which spelling fixes to apply. ### #include style -[include_checker.py](https://github.com/rapidsai/raft/blob/branch-24.12/cpp/scripts/include_checker.py) is used to enforce the include style as follows: +[include_checker.py](https://github.com/rapidsai/raft/blob/branch-25.02/cpp/scripts/include_checker.py) is used to enforce the include style as follows: 1. `#include "..."` should be used for referencing local files only. It is acceptable to be used for referencing files in a sub-folder/parent-folder of the same algorithm, but should never be used to include files in other algorithms or between algorithms and the primitives or other dependencies. 2. `#include <...>` should be used for referencing everything else @@ -230,7 +230,7 @@ Call CUDA APIs via the provided helper macros `RAFT_CUDA_TRY`, `RAFT_CUBLAS_TRY` ## Logging ### Introduction -Anything and everything about logging is defined inside [logger.hpp](https://github.com/rapidsai/raft/blob/branch-24.12/cpp/include/raft/core/logger.hpp). It uses [spdlog](https://github.com/gabime/spdlog) underneath, but this information is transparent to all. +Anything and everything about logging is defined inside [logger.hpp](https://github.com/rapidsai/raft/blob/branch-25.02/cpp/include/raft/core/logger.hpp). It uses [spdlog](https://github.com/gabime/spdlog) underneath, but this information is transparent to all. ### Usage ```cpp diff --git a/docs/source/raft_ann_benchmarks.md b/docs/source/raft_ann_benchmarks.md index 12a94e45ce..b7f7cc81d4 100644 --- a/docs/source/raft_ann_benchmarks.md +++ b/docs/source/raft_ann_benchmarks.md @@ -66,7 +66,7 @@ Nightly images are located in [dockerhub](https://hub.docker.com/r/rapidsai/raft - The following command pulls the nightly container for python version 10, cuda version 12, and RAFT version 23.10: ```bash -docker pull rapidsai/raft-ann-bench:24.12a-cuda12.0-py3.10 #substitute raft-ann-bench for the exact desired container. +docker pull rapidsai/raft-ann-bench:25.02a-cuda12.0-py3.10 #substitute raft-ann-bench for the exact desired container. ``` The CUDA and python versions can be changed for the supported values: @@ -87,7 +87,7 @@ You can see the exact versions as well in the dockerhub site: [//]: # () [//]: # (```bash) -[//]: # (docker pull nvcr.io/nvidia/rapidsai/raft-ann-bench:24.12-cuda11.8-py3.10 #substitute raft-ann-bench for the exact desired container.) +[//]: # (docker pull nvcr.io/nvidia/rapidsai/raft-ann-bench:25.02-cuda11.8-py3.10 #substitute raft-ann-bench for the exact desired container.) [//]: # (```) @@ -348,7 +348,7 @@ For GPU-enabled systems, the `DATA_FOLDER` variable should be a local folder whe export DATA_FOLDER=path/to/store/datasets/and/results docker run --gpus all --rm -it -u $(id -u) \ -v $DATA_FOLDER:/data/benchmarks \ - rapidsai/raft-ann-bench:24.12a-cuda11.8-py3.10 \ + rapidsai/raft-ann-bench:25.02a-cuda11.8-py3.10 \ "--dataset deep-image-96-angular" \ "--normalize" \ "--algorithms raft_cagra,raft_ivf_pq --batch-size 10 -k 10" \ @@ -359,7 +359,7 @@ Usage of the above command is as follows: | Argument | Description | |-----------------------------------------------------------|----------------------------------------------------------------------------------------------------| -| `rapidsai/raft-ann-bench:24.12a-cuda11.8-py3.10` | Image to use. Can be either `raft-ann-bench` or `raft-ann-bench-datasets` | +| `rapidsai/raft-ann-bench:25.02a-cuda11.8-py3.10` | Image to use. Can be either `raft-ann-bench` or `raft-ann-bench-datasets` | | `"--dataset deep-image-96-angular"` | Dataset name | | `"--normalize"` | Whether to normalize the dataset | | `"--algorithms raft_cagra,hnswlib --batch-size 10 -k 10"` | Arguments passed to the `run` script, such as the algorithms to benchmark, the batch size, and `k` | @@ -376,7 +376,7 @@ The container arguments in the above section also be used for the CPU-only conta export DATA_FOLDER=path/to/store/datasets/and/results docker run --rm -it -u $(id -u) \ -v $DATA_FOLDER:/data/benchmarks \ - rapidsai/raft-ann-bench-cpu:24.12a-py3.10 \ + rapidsai/raft-ann-bench-cpu:25.02a-py3.10 \ "--dataset deep-image-96-angular" \ "--normalize" \ "--algorithms hnswlib --batch-size 10 -k 10" \ @@ -393,7 +393,7 @@ docker run --gpus all --rm -it -u $(id -u) \ --entrypoint /bin/bash \ --workdir /data/benchmarks \ -v $DATA_FOLDER:/data/benchmarks \ - rapidsai/raft-ann-bench:24.12a-cuda11.8-py3.10 + rapidsai/raft-ann-bench:25.02a-cuda11.8-py3.10 ``` This will drop you into a command line in the container, with the `raft-ann-bench` python package ready to use, as described in the [Running the benchmarks](#running-the-benchmarks) section above: diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index bb01602b33..3502d82fd4 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "nvidia-curand", "nvidia-cusolver", "nvidia-cusparse", - "rmm==24.12.*,>=0.0.0a0", + "rmm==25.2.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", @@ -125,7 +125,7 @@ requires = [ "cuda-python", "cython>=3.0.0,<3.1.0a0", "ninja", - "rmm==24.12.*,>=0.0.0a0", + "rmm==25.2.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. dependencies-file = "../../dependencies.yaml" matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" diff --git a/python/raft-dask/cmake/thirdparty/get_ucxx.cmake b/python/raft-dask/cmake/thirdparty/get_ucxx.cmake index db9b5c6b4d..f5daf70f92 100644 --- a/python/raft-dask/cmake/thirdparty/get_ucxx.cmake +++ b/python/raft-dask/cmake/thirdparty/get_ucxx.cmake @@ -47,9 +47,9 @@ endfunction() # Change pinned tag here to test a commit in CI # To use a different RAFT locally, set the CMake variable # CPM_raft_SOURCE=/path/to/local/raft -find_and_configure_ucxx(VERSION 0.41 +find_and_configure_ucxx(VERSION 0.42 FORK rapidsai - PINNED_TAG branch-0.41 + PINNED_TAG branch-0.42 EXCLUDE_FROM_ALL YES UCXX_STATIC ${RAFT_DASK_UCXX_STATIC} ) diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index a9f4de5dc3..33643c481e 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -31,13 +31,13 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.10" dependencies = [ - "dask-cuda==24.12.*,>=0.0.0a0", - "distributed-ucxx==0.41.*,>=0.0.0a0", + "dask-cuda==25.2.*,>=0.0.0a0", + "distributed-ucxx==0.42.*,>=0.0.0a0", "joblib>=0.11", "numba>=0.57", - "pylibraft==24.12.*,>=0.0.0a0", - "rapids-dask-dependency==24.12.*,>=0.0.0a0", - "ucx-py==0.41.*,>=0.0.0a0", + "pylibraft==25.2.*,>=0.0.0a0", + "rapids-dask-dependency==25.2.*,>=0.0.0a0", + "ucx-py==0.42.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", From fccf33e367e40f3b459973025aef0dc60ea08df8 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Fri, 15 Nov 2024 21:04:10 -0500 Subject: [PATCH 52/79] Remove raft-ann-bench (#2497) This PR removes raft-ann-bench from the conda packages, build system, and documentation. This removal was previously announced for the 24.12 release in #2448. Authors: - Corey J. Nolet (https://github.com/cjnolet) - Bradley Dice (https://github.com/bdice) Approvers: - Ben Frederickson (https://github.com/benfred) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/raft/pull/2497 --- .github/workflows/pr.yaml | 2 +- .gitignore | 1 - .pre-commit-config.yaml | 3 +- README.md | 4 +- build.sh | 46 +- ci/build_python.sh | 25 - ci/release/update-version.sh | 2 - .../bench_ann_cuda-118_arch-aarch64.yaml | 46 - .../bench_ann_cuda-118_arch-x86_64.yaml | 46 - .../bench_ann_cuda-120_arch-aarch64.yaml | 42 - .../bench_ann_cuda-120_arch-x86_64.yaml | 42 - conda/recipes/libraft/conda_build_config.yaml | 15 - conda/recipes/raft-ann-bench-cpu/build.sh | 5 - .../conda_build_config.yaml | 29 - conda/recipes/raft-ann-bench-cpu/meta.yaml | 69 - conda/recipes/raft-ann-bench/build.sh | 5 - .../raft-ann-bench/conda_build_config.yaml | 70 - conda/recipes/raft-ann-bench/meta.yaml | 110 -- cpp/CMakeLists.txt | 88 +- cpp/bench/ann/CMakeLists.txt | 349 ----- cpp/bench/ann/README.md | 3 - cpp/bench/ann/src/common/ann_types.hpp | 168 -- cpp/bench/ann/src/common/benchmark.cpp | 111 -- cpp/bench/ann/src/common/benchmark.hpp | 736 --------- cpp/bench/ann/src/common/conf.hpp | 161 -- .../src/common/cuda_huge_page_resource.hpp | 104 -- .../ann/src/common/cuda_pinned_resource.hpp | 98 -- cpp/bench/ann/src/common/cuda_stub.hpp | 240 --- cpp/bench/ann/src/common/dataset.hpp | 495 ------ cpp/bench/ann/src/common/thread_pool.hpp | 134 -- cpp/bench/ann/src/common/util.hpp | 557 ------- .../ann/src/faiss/faiss_cpu_benchmark.cpp | 163 -- cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h | 326 ---- .../ann/src/faiss/faiss_gpu_benchmark.cu | 192 --- cpp/bench/ann/src/faiss/faiss_gpu_wrapper.h | 515 ------- cpp/bench/ann/src/ggnn/ggnn_benchmark.cu | 130 -- cpp/bench/ann/src/ggnn/ggnn_wrapper.cuh | 322 ---- .../ann/src/hnswlib/hnswlib_benchmark.cpp | 120 -- cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h | 239 --- .../src/raft/raft_ann_bench_param_parser.h | 275 ---- cpp/bench/ann/src/raft/raft_ann_bench_utils.h | 255 ---- cpp/bench/ann/src/raft/raft_benchmark.cu | 140 -- cpp/bench/ann/src/raft/raft_cagra_float.cu | 20 - cpp/bench/ann/src/raft/raft_cagra_half.cu | 20 - cpp/bench/ann/src/raft/raft_cagra_hnswlib.cu | 100 -- .../ann/src/raft/raft_cagra_hnswlib_wrapper.h | 108 -- cpp/bench/ann/src/raft/raft_cagra_int8_t.cu | 20 - cpp/bench/ann/src/raft/raft_cagra_uint8_t.cu | 20 - cpp/bench/ann/src/raft/raft_cagra_wrapper.h | 339 ----- cpp/bench/ann/src/raft/raft_ivf_flat.cu | 22 - .../ann/src/raft/raft_ivf_flat_wrapper.h | 165 -- cpp/bench/ann/src/raft/raft_ivf_pq.cu | 23 - cpp/bench/ann/src/raft/raft_ivf_pq_wrapper.h | 203 --- cpp/bench/ann/src/raft/raft_wrapper.h | 156 -- cpp/cmake/config.json | 2 +- cpp/cmake/modules/FindAVX.cmake | 110 -- cpp/cmake/patches/faiss_override.json | 9 - cpp/cmake/patches/ggnn.diff | 230 --- cpp/cmake/patches/ggnn_override.json | 16 - cpp/cmake/thirdparty/get_faiss.cmake | 119 -- cpp/cmake/thirdparty/get_fmt.cmake | 22 - cpp/cmake/thirdparty/get_ggnn.cmake | 50 - cpp/cmake/thirdparty/get_glog.cmake | 48 - cpp/cmake/thirdparty/get_nlohmann_json.cmake | 39 - cpp/template/cmake/thirdparty/get_raft.cmake | 1 - dependencies.yaml | 46 - docs/source/ann_benchmarks_build.md | 51 - docs/source/ann_benchmarks_dataset.md | 63 - docs/source/ann_benchmarks_low_level.md | 219 --- docs/source/ann_benchmarks_param_tuning.md | 178 --- docs/source/build.md | 25 +- docs/source/index.rst | 1 - docs/source/raft_ann_benchmarks.md | 597 -------- docs/source/vector_search_tutorial.md | 6 +- docs/source/wiki_all_dataset.md | 47 - python/pylibraft/CMakeLists.txt | 1 - python/raft-ann-bench/LICENSE | 1 - python/raft-ann-bench/pyproject.toml | 71 - .../raft-ann-bench/src/raft_ann_bench/VERSION | 1 - .../src/raft_ann_bench/__init__.py | 16 - .../src/raft_ann_bench/_version.py | 34 - .../raft_ann_bench/constraints/__init__.py | 77 - .../raft_ann_bench/data_export/__main__.py | 257 ---- .../generate_groundtruth/__main__.py | 240 --- .../generate_groundtruth/utils.py | 103 -- .../raft_ann_bench/get_dataset/__main__.py | 115 -- .../get_dataset/fbin_to_f16bin.py | 49 - .../get_dataset/hdf5_to_fbin.py | 90 -- .../src/raft_ann_bench/plot/__main__.py | 623 -------- .../src/raft_ann_bench/run/__main__.py | 614 -------- .../src/raft_ann_bench/run/algos.yaml | 42 - .../run/conf/algos/faiss_cpu_flat.yaml | 5 - .../run/conf/algos/faiss_cpu_ivf_flat.yaml | 10 - .../run/conf/algos/faiss_cpu_ivf_pq.yaml | 18 - .../run/conf/algos/faiss_gpu_flat.yaml | 5 - .../run/conf/algos/faiss_gpu_ivf_flat.yaml | 21 - .../run/conf/algos/faiss_gpu_ivf_pq.yaml | 77 - .../run/conf/algos/hnswlib.yaml | 10 - .../run/conf/algos/raft_brute_force.yaml | 5 - .../run/conf/algos/raft_cagra.yaml | 13 - .../run/conf/algos/raft_cagra_hnswlib.yaml | 11 - .../run/conf/algos/raft_ivf_flat.yaml | 9 - .../run/conf/algos/raft_ivf_pq.yaml | 41 - .../raft_ann_bench/run/conf/bigann-100M.json | 192 --- .../src/raft_ann_bench/run/conf/datasets.yaml | 127 -- .../raft_ann_bench/run/conf/deep-100M.json | 458 ------ .../src/raft_ann_bench/run/conf/deep-1B.json | 34 - .../run/conf/deep-image-96-inner.json | 1013 ------------ .../run/conf/fashion-mnist-784-euclidean.json | 1352 ----------------- .../run/conf/gist-960-euclidean.json | 1351 ---------------- .../run/conf/glove-100-angular.json | 1351 ---------------- .../run/conf/glove-100-inner.json | 1314 ---------------- .../run/conf/glove-50-angular.json | 1351 ---------------- .../run/conf/glove-50-inner.json | 1351 ---------------- .../run/conf/lastfm-65-angular.json | 1351 ---------------- .../run/conf/mnist-784-euclidean.json | 1352 ----------------- .../run/conf/nytimes-256-angular.json | 1352 ----------------- .../run/conf/nytimes-256-inner.json | 1352 ----------------- .../run/conf/sift-128-euclidean.json | 498 ------ .../raft_ann_bench/run/conf/wiki_all_10M.json | 200 --- .../raft_ann_bench/run/conf/wiki_all_1M.json | 216 --- .../raft_ann_bench/run/conf/wiki_all_88M.json | 200 --- .../split_groundtruth/__main__.py | 57 - .../split_groundtruth/split_groundtruth.pl | 45 - python/raft-dask/CMakeLists.txt | 1 - 125 files changed, 46 insertions(+), 28589 deletions(-) delete mode 100644 conda/environments/bench_ann_cuda-118_arch-aarch64.yaml delete mode 100644 conda/environments/bench_ann_cuda-118_arch-x86_64.yaml delete mode 100644 conda/environments/bench_ann_cuda-120_arch-aarch64.yaml delete mode 100644 conda/environments/bench_ann_cuda-120_arch-x86_64.yaml delete mode 100644 conda/recipes/raft-ann-bench-cpu/build.sh delete mode 100644 conda/recipes/raft-ann-bench-cpu/conda_build_config.yaml delete mode 100644 conda/recipes/raft-ann-bench-cpu/meta.yaml delete mode 100644 conda/recipes/raft-ann-bench/build.sh delete mode 100644 conda/recipes/raft-ann-bench/conda_build_config.yaml delete mode 100644 conda/recipes/raft-ann-bench/meta.yaml delete mode 100644 cpp/bench/ann/CMakeLists.txt delete mode 100644 cpp/bench/ann/README.md delete mode 100644 cpp/bench/ann/src/common/ann_types.hpp delete mode 100644 cpp/bench/ann/src/common/benchmark.cpp delete mode 100644 cpp/bench/ann/src/common/benchmark.hpp delete mode 100644 cpp/bench/ann/src/common/conf.hpp delete mode 100644 cpp/bench/ann/src/common/cuda_huge_page_resource.hpp delete mode 100644 cpp/bench/ann/src/common/cuda_pinned_resource.hpp delete mode 100644 cpp/bench/ann/src/common/cuda_stub.hpp delete mode 100644 cpp/bench/ann/src/common/dataset.hpp delete mode 100644 cpp/bench/ann/src/common/thread_pool.hpp delete mode 100644 cpp/bench/ann/src/common/util.hpp delete mode 100644 cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp delete mode 100644 cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h delete mode 100644 cpp/bench/ann/src/faiss/faiss_gpu_benchmark.cu delete mode 100644 cpp/bench/ann/src/faiss/faiss_gpu_wrapper.h delete mode 100644 cpp/bench/ann/src/ggnn/ggnn_benchmark.cu delete mode 100644 cpp/bench/ann/src/ggnn/ggnn_wrapper.cuh delete mode 100644 cpp/bench/ann/src/hnswlib/hnswlib_benchmark.cpp delete mode 100644 cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h delete mode 100644 cpp/bench/ann/src/raft/raft_ann_bench_param_parser.h delete mode 100644 cpp/bench/ann/src/raft/raft_ann_bench_utils.h delete mode 100644 cpp/bench/ann/src/raft/raft_benchmark.cu delete mode 100644 cpp/bench/ann/src/raft/raft_cagra_float.cu delete mode 100644 cpp/bench/ann/src/raft/raft_cagra_half.cu delete mode 100644 cpp/bench/ann/src/raft/raft_cagra_hnswlib.cu delete mode 100644 cpp/bench/ann/src/raft/raft_cagra_hnswlib_wrapper.h delete mode 100644 cpp/bench/ann/src/raft/raft_cagra_int8_t.cu delete mode 100644 cpp/bench/ann/src/raft/raft_cagra_uint8_t.cu delete mode 100644 cpp/bench/ann/src/raft/raft_cagra_wrapper.h delete mode 100644 cpp/bench/ann/src/raft/raft_ivf_flat.cu delete mode 100644 cpp/bench/ann/src/raft/raft_ivf_flat_wrapper.h delete mode 100644 cpp/bench/ann/src/raft/raft_ivf_pq.cu delete mode 100644 cpp/bench/ann/src/raft/raft_ivf_pq_wrapper.h delete mode 100644 cpp/bench/ann/src/raft/raft_wrapper.h delete mode 100644 cpp/cmake/modules/FindAVX.cmake delete mode 100644 cpp/cmake/patches/faiss_override.json delete mode 100644 cpp/cmake/patches/ggnn.diff delete mode 100644 cpp/cmake/patches/ggnn_override.json delete mode 100644 cpp/cmake/thirdparty/get_faiss.cmake delete mode 100644 cpp/cmake/thirdparty/get_fmt.cmake delete mode 100644 cpp/cmake/thirdparty/get_ggnn.cmake delete mode 100644 cpp/cmake/thirdparty/get_glog.cmake delete mode 100644 cpp/cmake/thirdparty/get_nlohmann_json.cmake delete mode 100644 docs/source/ann_benchmarks_build.md delete mode 100644 docs/source/ann_benchmarks_dataset.md delete mode 100644 docs/source/ann_benchmarks_low_level.md delete mode 100644 docs/source/ann_benchmarks_param_tuning.md delete mode 100644 docs/source/raft_ann_benchmarks.md delete mode 100644 docs/source/wiki_all_dataset.md delete mode 120000 python/raft-ann-bench/LICENSE delete mode 100644 python/raft-ann-bench/pyproject.toml delete mode 120000 python/raft-ann-bench/src/raft_ann_bench/VERSION delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/__init__.py delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/_version.py delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/constraints/__init__.py delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/data_export/__main__.py delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/generate_groundtruth/__main__.py delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/generate_groundtruth/utils.py delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/get_dataset/__main__.py delete mode 100755 python/raft-ann-bench/src/raft_ann_bench/get_dataset/fbin_to_f16bin.py delete mode 100755 python/raft-ann-bench/src/raft_ann_bench/get_dataset/hdf5_to_fbin.py delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/plot/__main__.py delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/__main__.py delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/algos.yaml delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_cpu_flat.yaml delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_cpu_ivf_flat.yaml delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_cpu_ivf_pq.yaml delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_gpu_flat.yaml delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_gpu_ivf_flat.yaml delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_gpu_ivf_pq.yaml delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/hnswlib.yaml delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_brute_force.yaml delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_cagra.yaml delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_cagra_hnswlib.yaml delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_ivf_flat.yaml delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_ivf_pq.yaml delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/bigann-100M.json delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/datasets.yaml delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/deep-100M.json delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/deep-1B.json delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/deep-image-96-inner.json delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/fashion-mnist-784-euclidean.json delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/gist-960-euclidean.json delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-100-angular.json delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-100-inner.json delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-50-angular.json delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-50-inner.json delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/lastfm-65-angular.json delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/mnist-784-euclidean.json delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/nytimes-256-angular.json delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/nytimes-256-inner.json delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/sift-128-euclidean.json delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/wiki_all_10M.json delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/wiki_all_1M.json delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/run/conf/wiki_all_88M.json delete mode 100644 python/raft-ann-bench/src/raft_ann_bench/split_groundtruth/__main__.py delete mode 100755 python/raft-ann-bench/src/raft_ann_bench/split_groundtruth/split_groundtruth.pl diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index fe8e730921..82e56cd95d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -151,5 +151,5 @@ jobs: cuda: '["12.5"]' build_command: | sccache -z; - build-all -DBUILD_PRIMS_BENCH=ON -DBUILD_ANN_BENCH=ON --verbose; + build-all -DBUILD_PRIMS_BENCH=ON --verbose; sccache -s; diff --git a/.gitignore b/.gitignore index 11b7bc3eba..3d6c84a83f 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,6 @@ log dask-worker-space/ *.egg-info/ *.bin -bench/ann/data temporary_*.json ## scikit-build diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5a5342a74e..d8ccf92ce5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -62,7 +62,7 @@ repos: entry: ./cpp/scripts/run-cmake-format.sh cmake-format language: python types: [cmake] - exclude: .*/thirdparty/.*|.*FindAVX.cmake.* + exclude: .*/thirdparty/.* # Note that pre-commit autoupdate does not update the versions # of dependencies, so we'll have to update this manually. additional_dependencies: @@ -114,7 +114,6 @@ repos: cpp/include/raft/neighbors/detail/faiss_select/| cpp/include/raft/thirdparty/| docs/source/sphinxext/github_link[.]py| - cpp/cmake/modules/FindAVX[.]cmake - id: verify-alpha-spec - repo: https://github.com/rapidsai/dependency-file-generator rev: v1.16.0 diff --git a/README.md b/README.md index 8870e9385e..7f43eb89dc 100755 --- a/README.md +++ b/README.md @@ -223,7 +223,7 @@ pairwise_distance(in1, in2, out=output, metric="euclidean") ## Installing -RAFT's C++ and Python libraries can both be installed through Conda and the Python libraries through Pip. +RAFT's C++ and Python libraries can both be installed through Conda and the Python libraries through Pip. ### Installing C++ and Python through Conda @@ -233,8 +233,6 @@ The easiest way to install RAFT is through conda and several packages are provid - `libraft` (optional) C++ shared library containing pre-compiled template instantiations and runtime API. - `pylibraft` (optional) Python library - `raft-dask` (optional) Python library for deployment of multi-node multi-GPU algorithms that use the RAFT `raft::comms` abstraction layer in Dask clusters. -- `raft-ann-bench` (optional) Benchmarking tool for easily producing benchmarks that compare RAFT's vector search algorithms against other state-of-the-art implementations. -- `raft-ann-bench-cpu` (optional) Reproducible benchmarking tool similar to above, but doesn't require CUDA to be installed on the machine. Can be used to test in environments with competitive CPUs. Use the following command, depending on your CUDA version, to install all of the RAFT packages with conda (replace `rapidsai` with `rapidsai-nightly` to install more up-to-date but less stable nightly packages). `mamba` is preferred over the `conda` command. ```bash diff --git a/build.sh b/build.sh index feb2d7256e..d54a8895a3 100755 --- a/build.sh +++ b/build.sh @@ -18,8 +18,8 @@ ARGS=$* # scripts, and that this script resides in the repo dir! REPODIR=$(cd $(dirname $0); pwd) -VALIDARGS="clean libraft pylibraft raft-dask docs tests template bench-prims bench-ann clean --uninstall -v -g -n --compile-lib --compile-static-lib --allgpuarch --no-nvtx --cpu-only --show_depr_warn --incl-cache-stats --time -h" -HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool=] [--limit-tests=] [--limit-bench-prims=] [--limit-bench-ann=] [--build-metrics=] +VALIDARGS="clean libraft pylibraft raft-dask docs tests template bench-prims clean --uninstall -v -g -n --compile-lib --compile-static-lib --allgpuarch --no-nvtx --show_depr_warn --incl-cache-stats --time -h" +HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool=] [--limit-tests=] [--limit-bench-prims=] [--build-metrics=] where is: clean - remove all existing build artifacts and configuration (start over) libraft - build the raft C++ code only. Also builds the C-wrapper library @@ -29,7 +29,6 @@ HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool= is: @@ -39,10 +38,8 @@ HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool==1.8.2 -- c-compiler -- clang-tools=16.0.6 -- clang==16.0.6 -- cmake>=3.26.4,!=3.30.0 -- cuda-nvtx=11.8 -- cuda-profiler-api=11.8.86 -- cuda-version=11.8 -- cudatoolkit -- cxx-compiler -- cython>=3.0.0,<3.1.0a0 -- gcc_linux-aarch64=11.* -- glog>=0.6.0 -- h5py>=3.8.0 -- hnswlib=0.7.0 -- libcublas-dev=11.11.3.6 -- libcublas=11.11.3.6 -- libcurand-dev=10.3.0.86 -- libcurand=10.3.0.86 -- libcusolver-dev=11.4.1.48 -- libcusolver=11.4.1.48 -- libcusparse-dev=11.7.5.86 -- libcusparse=11.7.5.86 -- libucxx==0.41.*,>=0.0.0a0 -- matplotlib -- nccl>=2.19 -- ninja -- nlohmann_json>=3.11.2 -- nvcc_linux-aarch64=11.8 -- openblas -- pandas -- pyyaml -- rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rmm==24.12.*,>=0.0.0a0 -- scikit-build-core>=0.10.0 -- sysroot_linux-aarch64==2.17 -name: bench_ann_cuda-118_arch-aarch64 diff --git a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml deleted file mode 100644 index 56004fa818..0000000000 --- a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# This file is generated by `rapids-dependency-file-generator`. -# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -channels: -- rapidsai -- rapidsai-nightly -- dask/label/dev -- conda-forge -- nvidia -dependencies: -- benchmark>=1.8.2 -- c-compiler -- clang-tools=16.0.6 -- clang==16.0.6 -- cmake>=3.26.4,!=3.30.0 -- cuda-nvtx=11.8 -- cuda-profiler-api=11.8.86 -- cuda-version=11.8 -- cudatoolkit -- cxx-compiler -- cython>=3.0.0,<3.1.0a0 -- gcc_linux-64=11.* -- glog>=0.6.0 -- h5py>=3.8.0 -- hnswlib=0.7.0 -- libcublas-dev=11.11.3.6 -- libcublas=11.11.3.6 -- libcurand-dev=10.3.0.86 -- libcurand=10.3.0.86 -- libcusolver-dev=11.4.1.48 -- libcusolver=11.4.1.48 -- libcusparse-dev=11.7.5.86 -- libcusparse=11.7.5.86 -- libucxx==0.41.*,>=0.0.0a0 -- matplotlib -- nccl>=2.19 -- ninja -- nlohmann_json>=3.11.2 -- nvcc_linux-64=11.8 -- openblas -- pandas -- pyyaml -- rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rmm==24.12.*,>=0.0.0a0 -- scikit-build-core>=0.10.0 -- sysroot_linux-64==2.17 -name: bench_ann_cuda-118_arch-x86_64 diff --git a/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml deleted file mode 100644 index 5f0599d9ae..0000000000 --- a/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# This file is generated by `rapids-dependency-file-generator`. -# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -channels: -- rapidsai -- rapidsai-nightly -- dask/label/dev -- conda-forge -- nvidia -dependencies: -- benchmark>=1.8.2 -- c-compiler -- clang-tools=16.0.6 -- clang==16.0.6 -- cmake>=3.26.4,!=3.30.0 -- cuda-cudart-dev -- cuda-nvcc -- cuda-nvtx-dev -- cuda-profiler-api -- cuda-version=12.0 -- cxx-compiler -- cython>=3.0.0,<3.1.0a0 -- gcc_linux-aarch64=11.* -- glog>=0.6.0 -- h5py>=3.8.0 -- hnswlib=0.7.0 -- libcublas-dev -- libcurand-dev -- libcusolver-dev -- libcusparse-dev -- libucxx==0.41.*,>=0.0.0a0 -- matplotlib -- nccl>=2.19 -- ninja -- nlohmann_json>=3.11.2 -- openblas -- pandas -- pyyaml -- rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rmm==24.12.*,>=0.0.0a0 -- scikit-build-core>=0.10.0 -- sysroot_linux-aarch64==2.17 -name: bench_ann_cuda-120_arch-aarch64 diff --git a/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml deleted file mode 100644 index 849e6c1412..0000000000 --- a/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# This file is generated by `rapids-dependency-file-generator`. -# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -channels: -- rapidsai -- rapidsai-nightly -- dask/label/dev -- conda-forge -- nvidia -dependencies: -- benchmark>=1.8.2 -- c-compiler -- clang-tools=16.0.6 -- clang==16.0.6 -- cmake>=3.26.4,!=3.30.0 -- cuda-cudart-dev -- cuda-nvcc -- cuda-nvtx-dev -- cuda-profiler-api -- cuda-version=12.0 -- cxx-compiler -- cython>=3.0.0,<3.1.0a0 -- gcc_linux-64=11.* -- glog>=0.6.0 -- h5py>=3.8.0 -- hnswlib=0.7.0 -- libcublas-dev -- libcurand-dev -- libcusolver-dev -- libcusparse-dev -- libucxx==0.41.*,>=0.0.0a0 -- matplotlib -- nccl>=2.19 -- ninja -- nlohmann_json>=3.11.2 -- openblas -- pandas -- pyyaml -- rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rmm==24.12.*,>=0.0.0a0 -- scikit-build-core>=0.10.0 -- sysroot_linux-64==2.17 -name: bench_ann_cuda-120_arch-x86_64 diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index bc0ff1fae7..4857f12cd1 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -19,21 +19,6 @@ c_stdlib_version: cmake_version: - ">=3.26.4,!=3.30.0" -nccl_version: - - ">=2.19" - -glog_version: - - ">=0.6.0" - -faiss_version: - - ">=1.7.1" - -h5py_version: - - ">=3.8.0" - -nlohmann_json_version: - - ">=3.11.2" - # The CTK libraries below are missing from the conda-forge::cudatoolkit package # for CUDA 11. The "*_host_*" version specifiers correspond to `11.8` packages # and the "*_run_*" version specifiers correspond to `11.x` packages. diff --git a/conda/recipes/raft-ann-bench-cpu/build.sh b/conda/recipes/raft-ann-bench-cpu/build.sh deleted file mode 100644 index 4462d5124b..0000000000 --- a/conda/recipes/raft-ann-bench-cpu/build.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2023, NVIDIA CORPORATION. - -./build.sh bench-ann --cpu-only --no-nvtx --build-metrics=bench_ann_cpu --incl-cache-stats -cmake --install cpp/build --component ann_bench diff --git a/conda/recipes/raft-ann-bench-cpu/conda_build_config.yaml b/conda/recipes/raft-ann-bench-cpu/conda_build_config.yaml deleted file mode 100644 index ed6f708e14..0000000000 --- a/conda/recipes/raft-ann-bench-cpu/conda_build_config.yaml +++ /dev/null @@ -1,29 +0,0 @@ -c_compiler_version: - - 11 - -cxx_compiler_version: - - 11 - -c_stdlib: - - sysroot - -c_stdlib_version: - - "2.17" - -cmake_version: - - ">=3.26.4,!=3.30.0" - -glog_version: - - ">=0.6.0" - -h5py_version: - - ">=3.8.0" - -nlohmann_json_version: - - ">=3.11.2" - -spdlog_version: - - ">=1.14.1,<1.15" - -fmt_version: - - ">=11.0.2,<12" diff --git a/conda/recipes/raft-ann-bench-cpu/meta.yaml b/conda/recipes/raft-ann-bench-cpu/meta.yaml deleted file mode 100644 index 94f7102726..0000000000 --- a/conda/recipes/raft-ann-bench-cpu/meta.yaml +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. - -# Usage: -# conda build . -c conda-forge -c nvidia -c rapidsai -{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') + environ.get('VERSION_SUFFIX', '') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set py_version = environ['CONDA_PY'] %} -{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} -{% set date_string = environ['RAPIDS_DATE_STRING'] %} - -package: - name: raft-ann-bench-cpu - version: {{ version }} - script: build.sh - -source: - path: ../../.. - -build: - script_env: - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN - - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CUDA_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - - CMAKE_GENERATOR - - PARALLEL_LEVEL - - RAPIDS_ARTIFACTS_DIR - - SCCACHE_BUCKET - - SCCACHE_IDLE_TIMEOUT - - SCCACHE_REGION - - SCCACHE_S3_KEY_PREFIX=libraft-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=libraft-linux64 # [linux64] - - SCCACHE_S3_USE_SSL - number: {{ GIT_DESCRIBE_NUMBER }} - string: py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - -requirements: - build: - - {{ compiler('c') }} - - {{ compiler('cxx') }} - - cmake {{ cmake_version }} - - ninja - - {{ stdlib("c") }} - - host: - - glog {{ glog_version }} - - matplotlib - - nlohmann_json {{ nlohmann_json_version }} - - spdlog {{ spdlog_version }} - - fmt {{ fmt_version }} - - python - - pyyaml - - pandas - - rapids-build-backend>=0.3.0,<0.4.0.dev0 - - run: - - glog {{ glog_version }} - - h5py {{ h5py_version }} - - matplotlib - - python - - pyyaml - - pandas - - benchmark -about: - home: https://rapids.ai/ - license: Apache-2.0 - summary: RAFT ANN CPU benchmarks diff --git a/conda/recipes/raft-ann-bench/build.sh b/conda/recipes/raft-ann-bench/build.sh deleted file mode 100644 index 00078792a1..0000000000 --- a/conda/recipes/raft-ann-bench/build.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2023, NVIDIA CORPORATION. - -./build.sh bench-ann --allgpuarch --no-nvtx --build-metrics=bench_ann --incl-cache-stats -cmake --install cpp/build --component ann_bench diff --git a/conda/recipes/raft-ann-bench/conda_build_config.yaml b/conda/recipes/raft-ann-bench/conda_build_config.yaml deleted file mode 100644 index 47bd730daf..0000000000 --- a/conda/recipes/raft-ann-bench/conda_build_config.yaml +++ /dev/null @@ -1,70 +0,0 @@ -c_compiler_version: - - 11 - -cxx_compiler_version: - - 11 - -cuda_compiler: - - cuda-nvcc - -cuda11_compiler: - - nvcc - -c_stdlib: - - sysroot - -c_stdlib_version: - - "2.17" - -cmake_version: - - ">=3.26.4,!=3.30.0" - -nccl_version: - - ">=2.19" - -glog_version: - - ">=0.6.0" - -h5py_version: - - ">=3.8.0" - -nlohmann_json_version: - - ">=3.11.2" - -# The CTK libraries below are missing from the conda-forge::cudatoolkit package -# for CUDA 11. The "*_host_*" version specifiers correspond to `11.8` packages -# and the "*_run_*" version specifiers correspond to `11.x` packages. - -cuda11_libcublas_host_version: - - "=11.11.3.6" - -cuda11_libcublas_run_version: - - ">=11.5.2.43,<12.0.0" - -cuda11_libcurand_host_version: - - "=10.3.0.86" - -cuda11_libcurand_run_version: - - ">=10.2.5.43,<10.3.1" - -cuda11_libcusolver_host_version: - - "=11.4.1.48" - -cuda11_libcusolver_run_version: - - ">=11.2.0.43,<11.4.2" - -cuda11_libcusparse_host_version: - - "=11.7.5.86" - -cuda11_libcusparse_run_version: - - ">=11.6.0.43,<12.0.0" - -# `cuda-profiler-api` only has `11.8.0` and `12.0.0` packages for all -# architectures. The "*_host_*" version specifiers correspond to `11.8` packages and the -# "*_run_*" version specifiers correspond to `11.x` packages. - -cuda11_cuda_profiler_api_host_version: - - "=11.8.86" - -cuda11_cuda_profiler_api_run_version: - - ">=11.4.240,<12" diff --git a/conda/recipes/raft-ann-bench/meta.yaml b/conda/recipes/raft-ann-bench/meta.yaml deleted file mode 100644 index d6aeb5f860..0000000000 --- a/conda/recipes/raft-ann-bench/meta.yaml +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. - -# Usage: -# conda build . -c conda-forge -c nvidia -c rapidsai -{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') + environ.get('VERSION_SUFFIX', '') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set py_version = environ['CONDA_PY'] %} -{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} -{% set cuda_major = cuda_version.split('.')[0] %} -{% set date_string = environ['RAPIDS_DATE_STRING'] %} - -package: - name: raft-ann-bench - version: {{ version }} - script: build.sh - -source: - path: ../../.. - -build: - script_env: - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN - - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CUDA_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - - CMAKE_GENERATOR - - PARALLEL_LEVEL - - RAPIDS_ARTIFACTS_DIR - - SCCACHE_BUCKET - - SCCACHE_IDLE_TIMEOUT - - SCCACHE_REGION - - SCCACHE_S3_KEY_PREFIX=libraft-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=libraft-linux64 # [linux64] - - SCCACHE_S3_USE_SSL - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - ignore_run_exports_from: - {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} - {% else %} - - {{ compiler('cuda') }} - - cuda-cudart-dev - - libcublas-dev - {% endif %} - -requirements: - build: - - {{ compiler('c') }} - - {{ compiler('cxx') }} - {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} ={{ cuda_version }} - {% else %} - - {{ compiler('cuda') }} - {% endif %} - - cuda-version ={{ cuda_version }} - - cmake {{ cmake_version }} - - ninja - - {{ stdlib("c") }} - - host: - - python - - libraft {{ version }} - - cuda-version ={{ cuda_version }} - {% if cuda_major == "11" %} - - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - - libcublas {{ cuda11_libcublas_host_version }} - - libcublas-dev {{ cuda11_libcublas_host_version }} - {% else %} - - cuda-cudart-dev - - cuda-profiler-api - - libcublas-dev - {% endif %} - - glog {{ glog_version }} - - nlohmann_json {{ nlohmann_json_version }} - - h5py {{ h5py_version }} - - benchmark - - matplotlib - - python - - pandas - - pyyaml - # rmm is needed to determine if package is gpu-enabled - - rmm ={{ minor_version }} - - rapids-build-backend>=0.3.0,<0.4.0.dev0 - - run: - - python - - libraft {{ version }} - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - {% if cuda_major == "11" %} - - cudatoolkit - {% else %} - - cuda-cudart - - libcublas - {% endif %} - - glog {{ glog_version }} - - h5py {{ h5py_version }} - - benchmark - - glog {{ glog_version }} - - matplotlib - - python - - pandas - - pyyaml - # rmm is needed to determine if package is gpu-enabled - - rmm ={{ minor_version }} -about: - home: https://rapids.ai/ - license: Apache-2.0 - summary: RAFT ANN GPU and CPU benchmarks diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d7eeb60b27..f4c18d53a8 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -17,17 +17,13 @@ include(rapids-cpm) include(rapids-export) include(rapids-find) -option(BUILD_CPU_ONLY "Build CPU only components. Applies to RAFT ANN benchmarks currently" OFF) - # workaround for rapids_cuda_init_architectures not working for arch detection with # enable_language(CUDA) set(lang_list "CXX") -if(NOT BUILD_CPU_ONLY) - include(rapids-cuda) - rapids_cuda_init_architectures(RAFT) - list(APPEND lang_list "CUDA") -endif() +include(rapids-cuda) +rapids_cuda_init_architectures(RAFT) +list(APPEND lang_list "CUDA") project( RAFT @@ -53,7 +49,6 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) option(BUILD_SHARED_LIBS "Build raft shared libraries" ON) option(BUILD_TESTS "Build raft unit-tests" ON) option(BUILD_PRIMS_BENCH "Build raft C++ benchmark tests" OFF) -option(BUILD_ANN_BENCH "Build raft ann benchmarks" OFF) option(BUILD_CAGRA_HNSWLIB "Build CAGRA+hnswlib interface" ON) option(CUDA_ENABLE_KERNELINFO "Enable kernel resource usage info" OFF) option(CUDA_ENABLE_LINEINFO @@ -68,23 +63,13 @@ option(DISABLE_OPENMP "Disable OpenMP" OFF) option(RAFT_NVTX "Enable nvtx markers" OFF) set(RAFT_COMPILE_LIBRARY_DEFAULT OFF) -if((BUILD_TESTS - OR BUILD_PRIMS_BENCH - OR BUILD_ANN_BENCH - ) - AND NOT BUILD_CPU_ONLY -) +if(BUILD_TESTS OR BUILD_PRIMS_BENCH) set(RAFT_COMPILE_LIBRARY_DEFAULT ON) endif() option(RAFT_COMPILE_LIBRARY "Enable building raft shared library instantiations" ${RAFT_COMPILE_LIBRARY_DEFAULT} ) -if(BUILD_CPU_ONLY) - set(BUILD_SHARED_LIBS OFF) - set(BUILD_TESTS OFF) -endif() - # Needed because GoogleBenchmark changes the state of FindThreads.cmake, causing subsequent runs to # have different values for the `Threads::Threads` target. Setting this flag ensures # `Threads::Threads` is the same value across all builds so that cache hits occur @@ -97,20 +82,14 @@ include(CMakeDependentOption) message(VERBOSE "RAFT: Building optional components: ${raft_FIND_COMPONENTS}") message(VERBOSE "RAFT: Build RAFT unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAFT: Building raft C++ benchmarks: ${BUILD_PRIMS_BENCH}") -message(VERBOSE "RAFT: Building ANN benchmarks: ${BUILD_ANN_BENCH}") -message(VERBOSE "RAFT: Build CPU only components: ${BUILD_CPU_ONLY}") message(VERBOSE "RAFT: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") message(VERBOSE "RAFT: Disable depreaction warnings " ${DISABLE_DEPRECATION_WARNINGS}) message(VERBOSE "RAFT: Disable OpenMP: ${DISABLE_OPENMP}") message(VERBOSE "RAFT: Enable kernel resource usage info: ${CUDA_ENABLE_KERNELINFO}") message(VERBOSE "RAFT: Enable lineinfo in nvcc: ${CUDA_ENABLE_LINEINFO}") message(VERBOSE "RAFT: Enable nvtx markers: ${RAFT_NVTX}") -message(VERBOSE - "RAFT: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}" -) -message(VERBOSE - "RAFT: Statically link the CUDA math libraries: ${CUDA_STATIC_MATH_LIBRARIES}" -) +message(VERBOSE "RAFT: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}") +message(VERBOSE "RAFT: Statically link the CUDA math libraries: ${CUDA_STATIC_MATH_LIBRARIES}") # Set RMM logging level set(RMM_LOGGING_LEVEL @@ -143,21 +122,17 @@ if(CUDA_STATIC_MATH_LIBRARIES) set(_ctk_static_suffix "_static") endif() -if(NOT BUILD_CPU_ONLY) - # CUDA runtime - rapids_cuda_init_runtime(USE_STATIC ${CUDA_STATIC_RUNTIME}) - # * find CUDAToolkit package - # * determine GPU architectures - # * enable the CMake CUDA language - # * set other CUDA compilation flags - rapids_find_package( - CUDAToolkit REQUIRED - BUILD_EXPORT_SET raft-exports - INSTALL_EXPORT_SET raft-exports - ) -else() - add_compile_definitions(BUILD_CPU_ONLY) -endif() +# CUDA runtime +rapids_cuda_init_runtime(USE_STATIC ${CUDA_STATIC_RUNTIME}) +# * find CUDAToolkit package +# * determine GPU architectures +# * enable the CMake CUDA language +# * set other CUDA compilation flags +rapids_find_package( + CUDAToolkit REQUIRED + BUILD_EXPORT_SET raft-exports + INSTALL_EXPORT_SET raft-exports +) if(NOT DISABLE_OPENMP) rapids_find_package( @@ -178,22 +153,20 @@ include(cmake/modules/ConfigureCUDA.cmake) # add third party dependencies using CPM rapids_cpm_init() -if(NOT BUILD_CPU_ONLY) - # CCCL before rmm/cuco so we get the right version of CCCL - include(cmake/thirdparty/get_cccl.cmake) - include(cmake/thirdparty/get_rmm.cmake) - include(cmake/thirdparty/get_cutlass.cmake) +# CCCL before rmm/cuco so we get the right version of CCCL +include(cmake/thirdparty/get_cccl.cmake) +include(cmake/thirdparty/get_rmm.cmake) +include(cmake/thirdparty/get_cutlass.cmake) - include(${rapids-cmake-dir}/cpm/cuco.cmake) - rapids_cpm_cuco(BUILD_EXPORT_SET raft-exports INSTALL_EXPORT_SET raft-exports) -endif() +include(${rapids-cmake-dir}/cpm/cuco.cmake) +rapids_cpm_cuco(BUILD_EXPORT_SET raft-exports INSTALL_EXPORT_SET raft-exports) if(BUILD_TESTS) include(${rapids-cmake-dir}/cpm/gtest.cmake) rapids_cpm_gtest(BUILD_STATIC) endif() -if(BUILD_PRIMS_BENCH OR BUILD_ANN_BENCH) +if(BUILD_PRIMS_BENCH) include(${rapids-cmake-dir}/cpm/gbench.cmake) rapids_cpm_gbench(BUILD_STATIC) endif() @@ -214,10 +187,8 @@ if(BUILD_CAGRA_HNSWLIB) target_link_libraries(raft INTERFACE hnswlib::hnswlib) endif() -if(NOT BUILD_CPU_ONLY) - # Keep RAFT as lightweight as possible. Only CUDA libs and rmm should be used in global target. - target_link_libraries(raft INTERFACE rmm::rmm cuco::cuco nvidia::cutlass::cutlass CCCL::CCCL) -endif() +# Keep RAFT as lightweight as possible. Only CUDA libs and rmm should be used in global target. +target_link_libraries(raft INTERFACE rmm::rmm cuco::cuco nvidia::cutlass::cutlass CCCL::CCCL) target_compile_features(raft INTERFACE cxx_std_17 $) target_compile_options( @@ -847,10 +818,3 @@ endif() if(BUILD_PRIMS_BENCH) add_subdirectory(bench/prims/) endif() - -# ################################################################################################## -# * build ann benchmark executable ----------------------------------------------- - -if(BUILD_ANN_BENCH) - add_subdirectory(bench/ann/) -endif() diff --git a/cpp/bench/ann/CMakeLists.txt b/cpp/bench/ann/CMakeLists.txt deleted file mode 100644 index 35df378438..0000000000 --- a/cpp/bench/ann/CMakeLists.txt +++ /dev/null @@ -1,349 +0,0 @@ -# ============================================================================= -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -list(APPEND CMAKE_MODULE_PATH "${RAFT_SOURCE_DIR}") - -# ################################################################################################## -# * benchmark options ------------------------------------------------------------------------------ - -option(RAFT_ANN_BENCH_USE_FAISS_GPU_FLAT "Include faiss' brute-force knn algorithm in benchmark" ON) -option(RAFT_ANN_BENCH_USE_FAISS_GPU_IVF_FLAT "Include faiss' ivf flat algorithm in benchmark" ON) -option(RAFT_ANN_BENCH_USE_FAISS_GPU_IVF_PQ "Include faiss' ivf pq algorithm in benchmark" ON) -option(RAFT_ANN_BENCH_USE_FAISS_CPU_FLAT "Include faiss' cpu brute-force algorithm in benchmark" ON) - -option(RAFT_ANN_BENCH_USE_FAISS_CPU_IVF_FLAT "Include faiss' cpu ivf flat algorithm in benchmark" - ON -) -option(RAFT_ANN_BENCH_USE_FAISS_CPU_IVF_PQ "Include faiss' cpu ivf pq algorithm in benchmark" ON) -option(RAFT_ANN_BENCH_USE_RAFT_IVF_FLAT "Include raft's ivf flat algorithm in benchmark" ON) -option(RAFT_ANN_BENCH_USE_RAFT_IVF_PQ "Include raft's ivf pq algorithm in benchmark" ON) -option(RAFT_ANN_BENCH_USE_RAFT_CAGRA "Include raft's CAGRA in benchmark" ON) -option(RAFT_ANN_BENCH_USE_RAFT_BRUTE_FORCE "Include raft's brute force knn in benchmark" ON) -option(RAFT_ANN_BENCH_USE_RAFT_CAGRA_HNSWLIB "Include raft's CAGRA in benchmark" ON) -option(RAFT_ANN_BENCH_USE_HNSWLIB "Include hnsw algorithm in benchmark" ON) -option(RAFT_ANN_BENCH_USE_GGNN "Include ggnn algorithm in benchmark" ON) -option(RAFT_ANN_BENCH_SINGLE_EXE - "Make a single executable with benchmark as shared library modules" OFF -) - -# ################################################################################################## -# * Process options ---------------------------------------------------------- - -find_package(Threads REQUIRED) - -set(RAFT_ANN_BENCH_USE_FAISS ON) -set(RAFT_FAISS_ENABLE_GPU ON) -set(RAFT_USE_FAISS_STATIC ON) - -if(BUILD_CPU_ONLY) - - # Include necessary logging dependencies - include(cmake/thirdparty/get_fmt) - include(cmake/thirdparty/get_spdlog) - set(RAFT_FAISS_ENABLE_GPU OFF) - set(RAFT_ANN_BENCH_USE_RAFT_IVF_FLAT OFF) - set(RAFT_ANN_BENCH_USE_RAFT_IVF_PQ OFF) - set(RAFT_ANN_BENCH_USE_RAFT_CAGRA OFF) - set(RAFT_ANN_BENCH_USE_RAFT_BRUTE_FORCE OFF) - set(RAFT_ANN_BENCH_USE_RAFT_CAGRA_HNSWLIB OFF) - set(RAFT_ANN_BENCH_USE_GGNN OFF) -endif() - -set(RAFT_ANN_BENCH_USE_RAFT OFF) -if(RAFT_ANN_BENCH_USE_RAFT_IVF_PQ - OR RAFT_ANN_BENCH_USE_RAFT_BRUTE_FORCE - OR RAFT_ANN_BENCH_USE_RAFT_IVF_FLAT - OR RAFT_ANN_BENCH_USE_RAFT_CAGRA - OR RAFT_ANN_BENCH_USE_RAFT_CAGRA_HNSWLIB -) - set(RAFT_ANN_BENCH_USE_RAFT ON) -endif() - -# ################################################################################################## -# * Fetch requirements ------------------------------------------------------------- - -if(RAFT_ANN_BENCH_USE_HNSWLIB OR RAFT_ANN_BENCH_USE_RAFT_CAGRA_HNSWLIB) - include(cmake/thirdparty/get_hnswlib) -endif() - -include(cmake/thirdparty/get_nlohmann_json) - -if(RAFT_ANN_BENCH_USE_GGNN) - include(cmake/thirdparty/get_ggnn) -endif() - -if(RAFT_ANN_BENCH_USE_FAISS) - include(cmake/thirdparty/get_faiss) -endif() - -# ################################################################################################## -# * Enable NVTX if available - -# Note: ANN_BENCH wrappers have extra NVTX code not related to raft::nvtx.They track gbench -# benchmark cases and iterations. This is to make limited NVTX available to all algos, not just -# raft. -if(TARGET CUDA::nvtx3) - set(_CMAKE_REQUIRED_INCLUDES_ORIG ${CMAKE_REQUIRED_INCLUDES}) - get_target_property(CMAKE_REQUIRED_INCLUDES CUDA::nvtx3 INTERFACE_INCLUDE_DIRECTORIES) - unset(NVTX3_HEADERS_FOUND CACHE) - # Check the headers explicitly to make sure the cpu-only build succeeds - CHECK_INCLUDE_FILE_CXX(nvtx3/nvToolsExt.h NVTX3_HEADERS_FOUND) - set(CMAKE_REQUIRED_INCLUDES ${_CMAKE_REQUIRED_INCLUDES_ORIG}) -endif() - -# ################################################################################################## -# * Configure tests function------------------------------------------------------------- - -function(ConfigureAnnBench) - - set(oneValueArgs NAME) - set(multiValueArgs PATH LINKS CXXFLAGS) - - if(NOT BUILD_CPU_ONLY) - set(GPU_BUILD ON) - endif() - - cmake_parse_arguments( - ConfigureAnnBench "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} - ) - - set(BENCH_NAME ${ConfigureAnnBench_NAME}_ANN_BENCH) - - if(RAFT_ANN_BENCH_SINGLE_EXE) - add_library(${BENCH_NAME} SHARED ${ConfigureAnnBench_PATH}) - string(TOLOWER ${BENCH_NAME} BENCH_LIB_NAME) - set_target_properties(${BENCH_NAME} PROPERTIES OUTPUT_NAME ${BENCH_LIB_NAME}) - add_dependencies(${BENCH_NAME} ANN_BENCH) - else() - add_executable(${BENCH_NAME} ${ConfigureAnnBench_PATH}) - target_compile_definitions( - ${BENCH_NAME} PRIVATE ANN_BENCH_BUILD_MAIN - $<$:ANN_BENCH_NVTX3_HEADERS_FOUND> - ) - target_link_libraries( - ${BENCH_NAME} PRIVATE benchmark::benchmark $<$:CUDA::nvtx3> - ) - endif() - - target_link_libraries( - ${BENCH_NAME} - PRIVATE raft::raft - nlohmann_json::nlohmann_json - ${ConfigureAnnBench_LINKS} - Threads::Threads - $<$:${RAFT_CTK_MATH_DEPENDENCIES}> - $ - $ - $<$:fmt::fmt-header-only> - $<$:spdlog::spdlog_header_only> - ) - - set_target_properties( - ${BENCH_NAME} - PROPERTIES # set target compile options - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON - CUDA_STANDARD 17 - CUDA_STANDARD_REQUIRED ON - POSITION_INDEPENDENT_CODE ON - INTERFACE_POSITION_INDEPENDENT_CODE ON - BUILD_RPATH "\$ORIGIN" - INSTALL_RPATH "\$ORIGIN" - ) - - set(${ConfigureAnnBench_CXXFLAGS} ${RAFT_CXX_FLAGS} ${ConfigureAnnBench_CXXFLAGS}) - - target_compile_options( - ${BENCH_NAME} PRIVATE "$<$:${ConfigureAnnBench_CXXFLAGS}>" - "$<$:${RAFT_CUDA_FLAGS}>" - ) - - if(RAFT_ANN_BENCH_USE_${ConfigureAnnBench_NAME}) - target_compile_definitions( - ${BENCH_NAME} - PUBLIC - RAFT_ANN_BENCH_USE_${ConfigureAnnBench_NAME}=RAFT_ANN_BENCH_USE_${ConfigureAnnBench_NAME} - ) - endif() - - target_include_directories( - ${BENCH_NAME} - PUBLIC "$" - PRIVATE ${ConfigureAnnBench_INCLUDES} - ) - - install( - TARGETS ${BENCH_NAME} - COMPONENT ann_bench - DESTINATION bin/ann - ) -endfunction() - -# ################################################################################################## -# * Configure tests------------------------------------------------------------- - -if(RAFT_ANN_BENCH_USE_HNSWLIB) - ConfigureAnnBench( - NAME HNSWLIB PATH src/hnswlib/hnswlib_benchmark.cpp LINKS hnswlib::hnswlib - ) - -endif() - -if(RAFT_ANN_BENCH_USE_RAFT_IVF_PQ) - ConfigureAnnBench( - NAME - RAFT_IVF_PQ - PATH - src/raft/raft_benchmark.cu - src/raft/raft_ivf_pq.cu - LINKS - raft::compiled - ) -endif() - -if(RAFT_ANN_BENCH_USE_RAFT_IVF_FLAT) - ConfigureAnnBench( - NAME - RAFT_IVF_FLAT - PATH - src/raft/raft_benchmark.cu - src/raft/raft_ivf_flat.cu - LINKS - raft::compiled - ) -endif() - -if(RAFT_ANN_BENCH_USE_RAFT_BRUTE_FORCE) - ConfigureAnnBench( - NAME RAFT_BRUTE_FORCE PATH src/raft/raft_benchmark.cu LINKS raft::compiled - ) -endif() - -if(RAFT_ANN_BENCH_USE_RAFT_CAGRA) - ConfigureAnnBench( - NAME - RAFT_CAGRA - PATH - src/raft/raft_benchmark.cu - src/raft/raft_cagra_float.cu - src/raft/raft_cagra_half.cu - src/raft/raft_cagra_int8_t.cu - src/raft/raft_cagra_uint8_t.cu - LINKS - raft::compiled - ) -endif() - -if(RAFT_ANN_BENCH_USE_RAFT_CAGRA_HNSWLIB) - ConfigureAnnBench( - NAME RAFT_CAGRA_HNSWLIB PATH src/raft/raft_cagra_hnswlib.cu LINKS raft::compiled - hnswlib::hnswlib - ) -endif() - -message("RAFT_FAISS_TARGETS: ${RAFT_FAISS_TARGETS}") -message("CUDAToolkit_LIBRARY_DIR: ${CUDAToolkit_LIBRARY_DIR}") -if(RAFT_ANN_BENCH_USE_FAISS_CPU_FLAT) - ConfigureAnnBench( - NAME FAISS_CPU_FLAT PATH src/faiss/faiss_cpu_benchmark.cpp LINKS - ${RAFT_FAISS_TARGETS} - ) -endif() - -if(RAFT_ANN_BENCH_USE_FAISS_CPU_IVF_FLAT) - ConfigureAnnBench( - NAME FAISS_CPU_IVF_FLAT PATH src/faiss/faiss_cpu_benchmark.cpp LINKS - ${RAFT_FAISS_TARGETS} - ) -endif() - -if(RAFT_ANN_BENCH_USE_FAISS_CPU_IVF_PQ) - ConfigureAnnBench( - NAME FAISS_CPU_IVF_PQ PATH src/faiss/faiss_cpu_benchmark.cpp LINKS - ${RAFT_FAISS_TARGETS} - ) -endif() - -if(RAFT_ANN_BENCH_USE_FAISS_GPU_IVF_FLAT AND RAFT_FAISS_ENABLE_GPU) - ConfigureAnnBench( - NAME FAISS_GPU_IVF_FLAT PATH src/faiss/faiss_gpu_benchmark.cu LINKS - ${RAFT_FAISS_TARGETS} - ) -endif() - -if(RAFT_ANN_BENCH_USE_FAISS_GPU_IVF_PQ AND RAFT_FAISS_ENABLE_GPU) - ConfigureAnnBench( - NAME FAISS_GPU_IVF_PQ PATH src/faiss/faiss_gpu_benchmark.cu LINKS - ${RAFT_FAISS_TARGETS} - ) -endif() - -if(RAFT_ANN_BENCH_USE_FAISS_GPU_FLAT AND RAFT_FAISS_ENABLE_GPU) - ConfigureAnnBench( - NAME FAISS_GPU_FLAT PATH src/faiss/faiss_gpu_benchmark.cu LINKS ${RAFT_FAISS_TARGETS} - ) -endif() - -if(RAFT_ANN_BENCH_USE_GGNN) - include(cmake/thirdparty/get_glog) - ConfigureAnnBench(NAME GGNN PATH src/ggnn/ggnn_benchmark.cu LINKS glog::glog ggnn::ggnn) -endif() - -# ################################################################################################## -# * Dynamically-loading ANN_BENCH executable ------------------------------------------------------- -if(RAFT_ANN_BENCH_SINGLE_EXE) - add_executable(ANN_BENCH src/common/benchmark.cpp) - - target_include_directories(ANN_BENCH PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - - target_link_libraries( - ANN_BENCH - PRIVATE raft::raft - nlohmann_json::nlohmann_json - benchmark::benchmark - dl - -static-libgcc - fmt::fmt-header-only - spdlog::spdlog_header_only - -static-libstdc++ - $<$:CUDA::nvtx3> - ) - set_target_properties( - ANN_BENCH - PROPERTIES # set target compile options - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON - CUDA_STANDARD 17 - CUDA_STANDARD_REQUIRED ON - POSITION_INDEPENDENT_CODE ON - INTERFACE_POSITION_INDEPENDENT_CODE ON - BUILD_RPATH "\$ORIGIN" - INSTALL_RPATH "\$ORIGIN" - ) - target_compile_definitions( - ANN_BENCH - PRIVATE - $<$:ANN_BENCH_LINK_CUDART="libcudart.so.${CUDAToolkit_VERSION_MAJOR}.${CUDAToolkit_VERSION_MINOR}.${CUDAToolkit_VERSION_PATCH}"> - $<$:ANN_BENCH_NVTX3_HEADERS_FOUND> - ) - - target_link_options(ANN_BENCH PRIVATE -export-dynamic) - - install( - TARGETS ANN_BENCH - COMPONENT ann_bench - DESTINATION bin/ann - EXCLUDE_FROM_ALL - ) -endif() diff --git a/cpp/bench/ann/README.md b/cpp/bench/ann/README.md deleted file mode 100644 index 1a8af2e448..0000000000 --- a/cpp/bench/ann/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# RAFT CUDA ANN Benchmarks - -Please see the [ANN Benchmarks](https://docs.rapids.ai/api/raft/stable/cuda_ann_benchmarks.html) section of the RAFT documentation for instructions on building and using the ANN benchmarks. \ No newline at end of file diff --git a/cpp/bench/ann/src/common/ann_types.hpp b/cpp/bench/ann/src/common/ann_types.hpp deleted file mode 100644 index b010063dee..0000000000 --- a/cpp/bench/ann/src/common/ann_types.hpp +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "cuda_stub.hpp" // cudaStream_t - -#include -#include -#include -#include - -namespace raft::bench::ann { - -enum Objective { - THROUGHPUT, // See how many vectors we can push through - LATENCY // See how fast we can push a vector through -}; - -enum class MemoryType { - Host, - HostMmap, - Device, -}; - -enum class Metric { - kInnerProduct, - kEuclidean, -}; - -inline auto parse_metric(const std::string& metric_str) -> Metric -{ - if (metric_str == "inner_product") { - return raft::bench::ann::Metric::kInnerProduct; - } else if (metric_str == "euclidean") { - return raft::bench::ann::Metric::kEuclidean; - } else { - throw std::runtime_error("invalid metric: '" + metric_str + "'"); - } -} - -inline auto parse_memory_type(const std::string& memory_type) -> MemoryType -{ - if (memory_type == "host") { - return MemoryType::Host; - } else if (memory_type == "mmap") { - return MemoryType::HostMmap; - } else if (memory_type == "device") { - return MemoryType::Device; - } else { - throw std::runtime_error("invalid memory type: '" + memory_type + "'"); - } -} - -struct AlgoProperty { - MemoryType dataset_memory_type; - // neighbors/distances should have same memory type as queries - MemoryType query_memory_type; -}; - -class AnnBase { - public: - using index_type = size_t; - - inline AnnBase(Metric metric, int dim) : metric_(metric), dim_(dim) {} - virtual ~AnnBase() noexcept = default; - - protected: - Metric metric_; - int dim_; -}; - -/** - * The GPU-based algorithms, which do not perform CPU synchronization at the end of their build or - * search methods, must implement this interface. - * - * The `cuda_timer` / `cuda_lap` from `util.hpp` uses this stream to record GPU times with events - * and, if necessary, also synchronize (via events) between iterations. - * - * If the algo does not implement this interface, GPU timings are disabled. - */ -class AnnGPU { - public: - /** - * Return the main cuda stream for this algorithm. - * If any work is done in multiple streams, they should synchornize with the main stream at the - * end. - */ - [[nodiscard]] virtual auto get_sync_stream() const noexcept -> cudaStream_t = 0; - /** - * By default a GPU algorithm uses a fixed stream to order GPU operations. - * However, an algorithm may need to synchronize with the host at the end of its execution. - * In that case, also synchronizing with a benchmark event would put it at disadvantage. - * - * We can disable event sync by passing `false` here - * - ONLY IF THE ALGORITHM HAS PRODUCED ITS OUTPUT BY THE TIME IT SYNCHRONIZES WITH CPU. - */ - [[nodiscard]] virtual auto uses_stream() const noexcept -> bool { return true; } - virtual ~AnnGPU() noexcept = default; -}; - -template -class ANN : public AnnBase { - public: - struct AnnSearchParam { - Objective metric_objective = Objective::LATENCY; - virtual ~AnnSearchParam() = default; - [[nodiscard]] virtual auto needs_dataset() const -> bool { return false; }; - }; - - inline ANN(Metric metric, int dim) : AnnBase(metric, dim) {} - virtual ~ANN() noexcept override = default; - - virtual void build(const T* dataset, size_t nrow) = 0; - - virtual void set_search_param(const AnnSearchParam& param) = 0; - // TODO: this assumes that an algorithm can always return k results. - // This is not always possible. - virtual void search(const T* queries, - int batch_size, - int k, - AnnBase::index_type* neighbors, - float* distances) const = 0; - - virtual void save(const std::string& file) const = 0; - virtual void load(const std::string& file) = 0; - - virtual AlgoProperty get_preference() const = 0; - - // Some algorithms don't save the building dataset in their indices. - // So they should be given the access to that dataset during searching. - // The advantage of this way is that index has smaller size - // and many indices can share one dataset. - // - // SearchParam::needs_dataset() of such algorithm should be true, - // and set_search_dataset() should save the passed-in pointer somewhere. - // The client code should call set_search_dataset() before searching, - // and should not release dataset before searching is finished. - virtual void set_search_dataset(const T* /*dataset*/, size_t /*nrow*/){}; - - /** - * Make a shallow copy of the ANN wrapper that shares the resources and ensures thread-safe access - * to them. */ - virtual auto copy() -> std::unique_ptr> = 0; -}; - -} // namespace raft::bench::ann - -#define REGISTER_ALGO_INSTANCE(DataT) \ - template auto raft::bench::ann::create_algo( \ - const std::string&, const std::string&, int, const nlohmann::json&, const std::vector&) \ - ->std::unique_ptr>; \ - template auto raft::bench::ann::create_search_param(const std::string&, \ - const nlohmann::json&) \ - ->std::unique_ptr::AnnSearchParam>; diff --git a/cpp/bench/ann/src/common/benchmark.cpp b/cpp/bench/ann/src/common/benchmark.cpp deleted file mode 100644 index 5510abf42f..0000000000 --- a/cpp/bench/ann/src/common/benchmark.cpp +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// clang-format off -#include "cuda_stub.hpp" // must go first -// clang-format on - -#include "ann_types.hpp" - -#include -#define JSON_DIAGNOSTICS 1 -#include - -#include -#include -#include - -namespace raft::bench::ann { - -struct lib_handle { - void* handle{nullptr}; - explicit lib_handle(const std::string& name) - { - handle = dlopen(name.c_str(), RTLD_LAZY | RTLD_LOCAL); - if (handle == nullptr) { - auto error_msg = "Failed to load " + name; - auto err = dlerror(); - if (err != nullptr && err[0] != '\0') { error_msg += ": " + std::string(err); } - throw std::runtime_error(error_msg); - } - } - ~lib_handle() noexcept - { - if (handle != nullptr) { dlclose(handle); } - } -}; - -auto load_lib(const std::string& algo) -> void* -{ - static std::unordered_map libs{}; - auto found = libs.find(algo); - - if (found != libs.end()) { return found->second.handle; } - auto lib_name = "lib" + algo + "_ann_bench.so"; - return libs.emplace(algo, lib_name).first->second.handle; -} - -auto get_fun_name(void* addr) -> std::string -{ - Dl_info dl_info; - if (dladdr(addr, &dl_info) != 0) { - if (dl_info.dli_sname != nullptr && dl_info.dli_sname[0] != '\0') { - return std::string{dl_info.dli_sname}; - } - } - throw std::logic_error("Failed to find out name of the looked up function"); -} - -template -auto create_algo(const std::string& algo, - const std::string& distance, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) -> std::unique_ptr> -{ - static auto fname = get_fun_name(reinterpret_cast(&create_algo)); - auto handle = load_lib(algo); - auto fun_addr = dlsym(handle, fname.c_str()); - if (fun_addr == nullptr) { - throw std::runtime_error("Couldn't load the create_algo function (" + algo + ")"); - } - auto fun = reinterpret_cast)>(fun_addr); - return fun(algo, distance, dim, conf, dev_list); -} - -template -std::unique_ptr::AnnSearchParam> create_search_param( - const std::string& algo, const nlohmann::json& conf) -{ - static auto fname = get_fun_name(reinterpret_cast(&create_search_param)); - auto handle = load_lib(algo); - auto fun_addr = dlsym(handle, fname.c_str()); - if (fun_addr == nullptr) { - throw std::runtime_error("Couldn't load the create_search_param function (" + algo + ")"); - } - auto fun = reinterpret_cast)>(fun_addr); - return fun(algo, conf); -} - -}; // namespace raft::bench::ann - -REGISTER_ALGO_INSTANCE(float); -REGISTER_ALGO_INSTANCE(std::int8_t); -REGISTER_ALGO_INSTANCE(std::uint8_t); - -#include "benchmark.hpp" - -int main(int argc, char** argv) { return raft::bench::ann::run_main(argc, argv); } diff --git a/cpp/bench/ann/src/common/benchmark.hpp b/cpp/bench/ann/src/common/benchmark.hpp deleted file mode 100644 index 185d54a0a3..0000000000 --- a/cpp/bench/ann/src/common/benchmark.hpp +++ /dev/null @@ -1,736 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "ann_types.hpp" -#include "conf.hpp" -#include "dataset.hpp" -#include "util.hpp" - -#include - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace raft::bench::ann { - -static inline std::unique_ptr current_algo{nullptr}; -static inline std::unique_ptr current_algo_props{nullptr}; - -using kv_series = std::vector>>; - -inline auto apply_overrides(const std::vector& configs, - const kv_series& overrides, - std::size_t override_idx = 0) -> std::vector -{ - std::vector results{}; - if (override_idx >= overrides.size()) { - auto n = configs.size(); - for (size_t i = 0; i < n; i++) { - auto c = configs[i]; - c["override_suffix"] = n > 1 ? "/" + std::to_string(i) : ""; - results.push_back(c); - } - return results; - } - auto rec_configs = apply_overrides(configs, overrides, override_idx + 1); - auto [key, vals] = overrides[override_idx]; - auto n = vals.size(); - for (size_t i = 0; i < n; i++) { - const auto& val = vals[i]; - for (auto rc : rec_configs) { - if (n > 1) { - rc["override_suffix"] = - static_cast(rc["override_suffix"]) + "/" + std::to_string(i); - } - rc[key] = val; - results.push_back(rc); - } - } - return results; -} - -inline auto apply_overrides(const nlohmann::json& config, - const kv_series& overrides, - std::size_t override_idx = 0) -{ - return apply_overrides(std::vector{config}, overrides, 0); -} - -inline void dump_parameters(::benchmark::State& state, nlohmann::json params) -{ - std::string label = ""; - bool label_empty = true; - for (auto& [key, val] : params.items()) { - if (val.is_number()) { - state.counters.insert({{key, val}}); - } else if (val.is_boolean()) { - state.counters.insert({{key, val ? 1.0 : 0.0}}); - } else { - auto kv = key + "=" + val.dump(); - if (label_empty) { - label = kv; - } else { - label += "#" + kv; - } - label_empty = false; - } - } - if (!label_empty) { state.SetLabel(label); } -} - -inline auto parse_algo_property(AlgoProperty prop, const nlohmann::json& conf) -> AlgoProperty -{ - if (conf.contains("dataset_memory_type")) { - prop.dataset_memory_type = parse_memory_type(conf.at("dataset_memory_type")); - } - if (conf.contains("query_memory_type")) { - prop.query_memory_type = parse_memory_type(conf.at("query_memory_type")); - } - return prop; -}; - -template -void bench_build(::benchmark::State& state, - std::shared_ptr> dataset, - Configuration::Index index, - bool force_overwrite) -{ - // NB: these two thread-local vars can be used within algo wrappers - raft::bench::ann::benchmark_thread_id = state.thread_index(); - raft::bench::ann::benchmark_n_threads = state.threads(); - dump_parameters(state, index.build_param); - if (file_exists(index.file)) { - if (force_overwrite) { - log_info("Overwriting file: %s", index.file.c_str()); - } else { - return state.SkipWithMessage( - "Index file already exists (use --force to overwrite the index)."); - } - } - - std::unique_ptr> algo; - try { - algo = ann::create_algo( - index.algo, dataset->distance(), dataset->dim(), index.build_param, index.dev_list); - } catch (const std::exception& e) { - return state.SkipWithError("Failed to create an algo: " + std::string(e.what())); - } - - const auto algo_property = parse_algo_property(algo->get_preference(), index.build_param); - - const T* base_set = dataset->base_set(algo_property.dataset_memory_type); - std::size_t index_size = dataset->base_set_size(); - - cuda_timer gpu_timer{algo}; - { - nvtx_case nvtx{state.name()}; - for (auto _ : state) { - [[maybe_unused]] auto ntx_lap = nvtx.lap(); - [[maybe_unused]] auto gpu_lap = gpu_timer.lap(); - try { - algo->build(base_set, index_size); - } catch (const std::exception& e) { - state.SkipWithError(std::string(e.what())); - } - } - } - if (gpu_timer.active()) { - state.counters.insert({"GPU", {gpu_timer.total_time(), benchmark::Counter::kAvgIterations}}); - } - state.counters.insert({{"index_size", index_size}}); - - if (state.skipped()) { return; } - make_sure_parent_dir_exists(index.file); - algo->save(index.file); -} - -template -void bench_search(::benchmark::State& state, - Configuration::Index index, - std::size_t search_param_ix, - std::shared_ptr> dataset, - Objective metric_objective) -{ - // NB: these two thread-local vars can be used within algo wrappers - raft::bench::ann::benchmark_thread_id = state.thread_index(); - raft::bench::ann::benchmark_n_threads = state.threads(); - std::size_t queries_processed = 0; - - const auto& sp_json = index.search_params[search_param_ix]; - - if (state.thread_index() == 0) { dump_parameters(state, sp_json); } - - // NB: `k` and `n_queries` are guaranteed to be populated in conf.cpp - const std::uint32_t k = sp_json["k"]; - // Amount of data processes in one go - const std::size_t n_queries = sp_json["n_queries"]; - // Round down the query data to a multiple of the batch size to loop over full batches of data - const std::size_t query_set_size = (dataset->query_set_size() / n_queries) * n_queries; - - if (dataset->query_set_size() < n_queries) { - std::stringstream msg; - msg << "Not enough queries in benchmark set. Expected " << n_queries << ", actual " - << dataset->query_set_size(); - state.SkipWithError(msg.str()); - return; - } - - // Each thread start from a different offset, so that the queries that they process do not - // overlap. - std::ptrdiff_t batch_offset = (state.thread_index() * n_queries) % query_set_size; - std::ptrdiff_t queries_stride = state.threads() * n_queries; - // Output is saved into a contiguous buffer (separate buffers for each thread). - std::ptrdiff_t out_offset = 0; - - const T* query_set = nullptr; - - if (!file_exists(index.file)) { - state.SkipWithError("Index file is missing. Run the benchmark in the build mode first."); - return; - } - - /** - * Make sure the first thread loads the algo and dataset - */ - progress_barrier load_barrier{}; - if (load_barrier.arrive(1) == 0) { - // algo is static to cache it between close search runs to save time on index loading - static std::string index_file = ""; - if (index.file != index_file) { - current_algo.reset(); - index_file = index.file; - } - - std::unique_ptr::AnnSearchParam> search_param; - ANN* algo; - try { - if (!current_algo || (algo = dynamic_cast*>(current_algo.get())) == nullptr) { - auto ualgo = ann::create_algo( - index.algo, dataset->distance(), dataset->dim(), index.build_param, index.dev_list); - algo = ualgo.get(); - algo->load(index_file); - current_algo = std::move(ualgo); - } - search_param = ann::create_search_param(index.algo, sp_json); - search_param->metric_objective = metric_objective; - } catch (const std::exception& e) { - state.SkipWithError("Failed to create an algo: " + std::string(e.what())); - return; - } - - current_algo_props = std::make_unique( - std::move(parse_algo_property(algo->get_preference(), sp_json))); - - if (search_param->needs_dataset()) { - try { - algo->set_search_dataset(dataset->base_set(current_algo_props->dataset_memory_type), - dataset->base_set_size()); - } catch (const std::exception& ex) { - state.SkipWithError("The algorithm '" + index.name + - "' requires the base set, but it's not available. " + - "Exception: " + std::string(ex.what())); - return; - } - } - try { - algo->set_search_param(*search_param); - } catch (const std::exception& ex) { - state.SkipWithError("An error occurred setting search parameters: " + std::string(ex.what())); - return; - } - - query_set = dataset->query_set(current_algo_props->query_memory_type); - load_barrier.arrive(state.threads()); - } else { - // All other threads will wait for the first thread to initialize the algo. - load_barrier.wait(state.threads() * 2); - // gbench ensures that all threads are synchronized at the start of the benchmark loop. - // We are accessing shared variables (like current_algo, current_algo_probs) before the - // benchmark loop, therefore the synchronization here is necessary. - } - query_set = dataset->query_set(current_algo_props->query_memory_type); - - /** - * Each thread will manage its own outputs - */ - using index_type = AnnBase::index_type; - constexpr size_t kAlignResultBuf = 64; - size_t result_elem_count = k * query_set_size; - result_elem_count = - ((result_elem_count + kAlignResultBuf - 1) / kAlignResultBuf) * kAlignResultBuf; - auto& result_buf = - get_result_buffer_from_global_pool(result_elem_count * (sizeof(float) + sizeof(index_type))); - auto* neighbors_ptr = - reinterpret_cast(result_buf.data(current_algo_props->query_memory_type)); - auto* distances_ptr = reinterpret_cast(neighbors_ptr + result_elem_count); - - { - nvtx_case nvtx{state.name()}; - - std::unique_ptr> algo{nullptr}; - try { - dynamic_cast*>(current_algo.get())->copy().swap(algo); - } catch (const std::exception& e) { - state.SkipWithError("Algo::copy: " + std::string(e.what())); - return; - } - // Initialize with algo, so that the timer.lap() object can sync with algo::get_sync_stream() - cuda_timer gpu_timer{algo}; - auto start = std::chrono::high_resolution_clock::now(); - for (auto _ : state) { - [[maybe_unused]] auto ntx_lap = nvtx.lap(); - [[maybe_unused]] auto gpu_lap = gpu_timer.lap(); - try { - algo->search(query_set + batch_offset * dataset->dim(), - n_queries, - k, - neighbors_ptr + out_offset * k, - distances_ptr + out_offset * k); - } catch (const std::exception& e) { - state.SkipWithError("Benchmark loop: " + std::string(e.what())); - break; - } - - // advance to the next batch - batch_offset = (batch_offset + queries_stride) % query_set_size; - out_offset = (out_offset + n_queries) % query_set_size; - - queries_processed += n_queries; - } - auto end = std::chrono::high_resolution_clock::now(); - auto duration = std::chrono::duration_cast>(end - start).count(); - if (state.thread_index() == 0) { state.counters.insert({{"end_to_end", duration}}); } - state.counters.insert({"Latency", {duration, benchmark::Counter::kAvgIterations}}); - - if (gpu_timer.active()) { - state.counters.insert({"GPU", {gpu_timer.total_time(), benchmark::Counter::kAvgIterations}}); - } - } - - state.SetItemsProcessed(queries_processed); - - // This will be the total number of queries across all threads - state.counters.insert({{"total_queries", queries_processed}}); - - if (state.skipped()) { return; } - - // Each thread calculates recall on their partition of queries. - // evaluate recall - if (dataset->max_k() >= k) { - const std::int32_t* gt = dataset->gt_set(); - const std::uint32_t max_k = dataset->max_k(); - result_buf.transfer_data(MemoryType::Host, current_algo_props->query_memory_type); - auto* neighbors_host = reinterpret_cast(result_buf.data(MemoryType::Host)); - std::size_t rows = std::min(queries_processed, query_set_size); - std::size_t match_count = 0; - std::size_t total_count = rows * static_cast(k); - - // We go through the groundtruth with same stride as the benchmark loop. - size_t out_offset = 0; - size_t batch_offset = (state.thread_index() * n_queries) % query_set_size; - while (out_offset < rows) { - for (std::size_t i = 0; i < n_queries; i++) { - size_t i_orig_idx = batch_offset + i; - size_t i_out_idx = out_offset + i; - if (i_out_idx < rows) { - for (std::uint32_t j = 0; j < k; j++) { - auto act_idx = std::int32_t(neighbors_host[i_out_idx * k + j]); - for (std::uint32_t l = 0; l < k; l++) { - auto exp_idx = gt[i_orig_idx * max_k + l]; - if (act_idx == exp_idx) { - match_count++; - break; - } - } - } - } - } - out_offset += n_queries; - batch_offset = (batch_offset + queries_stride) % query_set_size; - } - double actual_recall = static_cast(match_count) / static_cast(total_count); - state.counters.insert({"Recall", {actual_recall, benchmark::Counter::kAvgThreads}}); - } -} - -inline void printf_usage() -{ - ::benchmark::PrintDefaultHelp(); - fprintf(stdout, - " [--build|--search] \n" - " [--force]\n" - " [--data_prefix=]\n" - " [--index_prefix=]\n" - " [--override_kv=]\n" - " [--mode=\n" - " [--threads=min[:max]]\n" - " .json\n" - "\n" - "Note the non-standard benchmark parameters:\n" - " --build: build mode, will build index\n" - " --search: search mode, will search using the built index\n" - " one and only one of --build and --search should be specified\n" - " --force: force overwriting existing index files\n" - " --data_prefix=:" - " prepend to dataset file paths specified in the .json (default = " - "'data/').\n" - " --index_prefix=:" - " prepend to index file paths specified in the .json (default = " - "'index/').\n" - " --override_kv=:" - " override a build/search key one or more times multiplying the number of configurations;" - " you can use this parameter multiple times to get the Cartesian product of benchmark" - " configs.\n" - " --mode=" - " run the benchmarks in latency (accumulate times spent in each batch) or " - " throughput (pipeline batches and measure end-to-end) mode\n" - " --threads=min[:max] specify the number threads to use for throughput benchmark." - " Power of 2 values between 'min' and 'max' will be used. If only 'min' is specified," - " then a single test is run with 'min' threads. By default min=1, max=.\n"); -} - -template -void register_build(std::shared_ptr> dataset, - std::vector indices, - bool force_overwrite) -{ - for (auto index : indices) { - auto suf = static_cast(index.build_param["override_suffix"]); - auto file_suf = suf; - index.build_param.erase("override_suffix"); - std::replace(file_suf.begin(), file_suf.end(), '/', '-'); - index.file += file_suf; - auto* b = ::benchmark::RegisterBenchmark( - index.name + suf, bench_build, dataset, index, force_overwrite); - b->Unit(benchmark::kSecond); - b->MeasureProcessCPUTime(); - b->UseRealTime(); - } -} - -template -void register_search(std::shared_ptr> dataset, - std::vector indices, - Objective metric_objective, - const std::vector& threads) -{ - for (auto index : indices) { - for (std::size_t i = 0; i < index.search_params.size(); i++) { - auto suf = static_cast(index.search_params[i]["override_suffix"]); - index.search_params[i].erase("override_suffix"); - - auto* b = ::benchmark::RegisterBenchmark( - index.name + suf, bench_search, index, i, dataset, metric_objective) - ->Unit(benchmark::kMillisecond) - /** - * The following are important for getting accuracy QPS measurements on both CPU - * and GPU These make sure that - * - `end_to_end` ~ (`Time` * `Iterations`) - * - `items_per_second` ~ (`total_queries` / `end_to_end`) - * - Throughput = `items_per_second` - */ - ->MeasureProcessCPUTime() - ->UseRealTime(); - if (metric_objective == Objective::THROUGHPUT) { - if (index.algo.find("faiss_gpu") != std::string::npos) { - log_warn( - "FAISS GPU does not work in throughput mode because the underlying " - "StandardGpuResources object is not thread-safe. This will cause unexpected results"); - } - b->ThreadRange(threads[0], threads[1]); - } - } - } -} - -template -void dispatch_benchmark(const Configuration& conf, - bool force_overwrite, - bool build_mode, - bool search_mode, - std::string data_prefix, - std::string index_prefix, - kv_series override_kv, - Objective metric_objective, - const std::vector& threads) -{ - if (cudart.found()) { - for (auto [key, value] : cuda_info()) { - ::benchmark::AddCustomContext(key, value); - } - } - const auto dataset_conf = conf.get_dataset_conf(); - auto base_file = combine_path(data_prefix, dataset_conf.base_file); - auto query_file = combine_path(data_prefix, dataset_conf.query_file); - auto gt_file = dataset_conf.groundtruth_neighbors_file; - if (gt_file.has_value()) { gt_file.emplace(combine_path(data_prefix, gt_file.value())); } - auto dataset = std::make_shared>(dataset_conf.name, - base_file, - dataset_conf.subset_first_row, - dataset_conf.subset_size, - query_file, - dataset_conf.distance, - gt_file); - ::benchmark::AddCustomContext("dataset", dataset_conf.name); - ::benchmark::AddCustomContext("distance", dataset_conf.distance); - std::vector indices = conf.get_indices(); - if (build_mode) { - if (file_exists(base_file)) { - log_info("Using the dataset file '%s'", base_file.c_str()); - ::benchmark::AddCustomContext("n_records", std::to_string(dataset->base_set_size())); - ::benchmark::AddCustomContext("dim", std::to_string(dataset->dim())); - } else { - log_warn("Dataset file '%s' does not exist; benchmarking index building is impossible.", - base_file.c_str()); - } - std::vector more_indices{}; - for (auto& index : indices) { - for (auto param : apply_overrides(index.build_param, override_kv)) { - auto modified_index = index; - modified_index.build_param = param; - modified_index.file = combine_path(index_prefix, modified_index.file); - more_indices.push_back(modified_index); - } - } - register_build(dataset, more_indices, force_overwrite); - } else if (search_mode) { - if (file_exists(query_file)) { - log_info("Using the query file '%s'", query_file.c_str()); - ::benchmark::AddCustomContext("max_n_queries", std::to_string(dataset->query_set_size())); - ::benchmark::AddCustomContext("dim", std::to_string(dataset->dim())); - if (gt_file.has_value()) { - if (file_exists(*gt_file)) { - log_info("Using the ground truth file '%s'", gt_file->c_str()); - ::benchmark::AddCustomContext("max_k", std::to_string(dataset->max_k())); - } else { - log_warn("Ground truth file '%s' does not exist; the recall won't be reported.", - gt_file->c_str()); - } - } else { - log_warn( - "Ground truth file is not provided; the recall won't be reported. NB: use " - "the 'groundtruth_neighbors_file' alongside the 'query_file' key to specify the " - "path to " - "the ground truth in your conf.json."); - } - } else { - log_warn("Query file '%s' does not exist; benchmarking search is impossible.", - query_file.c_str()); - } - for (auto& index : indices) { - index.search_params = apply_overrides(index.search_params, override_kv); - index.file = combine_path(index_prefix, index.file); - } - register_search(dataset, indices, metric_objective, threads); - } -} - -inline auto parse_bool_flag(const char* arg, const char* pat, bool& result) -> bool -{ - if (strcmp(arg, pat) == 0) { - result = true; - return true; - } - return false; -} - -inline auto parse_string_flag(const char* arg, const char* pat, std::string& result) -> bool -{ - auto n = strlen(pat); - if (strncmp(pat, arg, strlen(pat)) == 0) { - result = arg + n + 1; - return true; - } - return false; -} - -inline auto run_main(int argc, char** argv) -> int -{ - bool force_overwrite = false; - bool build_mode = false; - bool search_mode = false; - std::string data_prefix = "data"; - std::string index_prefix = "index"; - std::string new_override_kv = ""; - std::string mode = "latency"; - std::string threads_arg_txt = ""; - std::vector threads = {1, -1}; // min_thread, max_thread - std::string log_level_str = ""; - int raft_log_level = raft::logger::get(RAFT_NAME).get_level(); - kv_series override_kv{}; - - char arg0_default[] = "benchmark"; // NOLINT - char* args_default = arg0_default; - if (!argv) { - argc = 1; - argv = &args_default; - } - if (argc == 1) { - printf_usage(); - return -1; - } - - char* conf_path = argv[--argc]; - std::ifstream conf_stream(conf_path); - - for (int i = 1; i < argc; i++) { - if (parse_bool_flag(argv[i], "--force", force_overwrite) || - parse_bool_flag(argv[i], "--build", build_mode) || - parse_bool_flag(argv[i], "--search", search_mode) || - parse_string_flag(argv[i], "--data_prefix", data_prefix) || - parse_string_flag(argv[i], "--index_prefix", index_prefix) || - parse_string_flag(argv[i], "--mode", mode) || - parse_string_flag(argv[i], "--override_kv", new_override_kv) || - parse_string_flag(argv[i], "--threads", threads_arg_txt) || - parse_string_flag(argv[i], "--raft_log_level", log_level_str)) { - if (!log_level_str.empty()) { - raft_log_level = std::stoi(log_level_str); - log_level_str = ""; - } - if (!threads_arg_txt.empty()) { - auto threads_arg = split(threads_arg_txt, ':'); - threads[0] = std::stoi(threads_arg[0]); - if (threads_arg.size() > 1) { - threads[1] = std::stoi(threads_arg[1]); - } else { - threads[1] = threads[0]; - } - threads_arg_txt = ""; - } - if (!new_override_kv.empty()) { - auto kvv = split(new_override_kv, ':'); - auto key = kvv[0]; - std::vector vals{}; - for (std::size_t j = 1; j < kvv.size(); j++) { - vals.push_back(nlohmann::json::parse(kvv[j])); - } - override_kv.emplace_back(key, vals); - new_override_kv = ""; - } - for (int j = i; j < argc - 1; j++) { - argv[j] = argv[j + 1]; - } - argc--; - i--; - } - } - - raft::logger::get(RAFT_NAME).set_level(raft_log_level); - - Objective metric_objective = Objective::LATENCY; - if (mode == "throughput") { metric_objective = Objective::THROUGHPUT; } - - int max_threads = - (metric_objective == Objective::THROUGHPUT) ? std::thread::hardware_concurrency() : 1; - if (threads[1] == -1) threads[1] = max_threads; - - if (metric_objective == Objective::LATENCY) { - if (threads[0] != 1 || threads[1] != 1) { - log_warn("Latency mode enabled. Overriding threads arg, running with single thread."); - threads = {1, 1}; - } - } - - if (build_mode == search_mode) { - log_error("One and only one of --build and --search should be specified"); - printf_usage(); - return -1; - } - - if (!conf_stream) { - log_error("Can't open configuration file: %s", conf_path); - return -1; - } - - if (cudart.needed() && !cudart.found()) { - log_warn("cudart library is not found, GPU-based indices won't work."); - } - - Configuration conf(conf_stream); - std::string dtype = conf.get_dataset_conf().dtype; - - if (dtype == "float") { - dispatch_benchmark(conf, - force_overwrite, - build_mode, - search_mode, - data_prefix, - index_prefix, - override_kv, - metric_objective, - threads); - } else if (dtype == "half") { - dispatch_benchmark(conf, - force_overwrite, - build_mode, - search_mode, - data_prefix, - index_prefix, - override_kv, - metric_objective, - threads); - } else if (dtype == "uint8") { - dispatch_benchmark(conf, - force_overwrite, - build_mode, - search_mode, - data_prefix, - index_prefix, - override_kv, - metric_objective, - threads); - } else if (dtype == "int8") { - dispatch_benchmark(conf, - force_overwrite, - build_mode, - search_mode, - data_prefix, - index_prefix, - override_kv, - metric_objective, - threads); - } else { - log_error("datatype '%s' is not supported", dtype.c_str()); - return -1; - } - - ::benchmark::Initialize(&argc, argv, printf_usage); - if (::benchmark::ReportUnrecognizedArguments(argc, argv)) return -1; - ::benchmark::RunSpecifiedBenchmarks(); - ::benchmark::Shutdown(); - // Release a possibly cached ANN object, so that it cannot be alive longer than the handle - // to a shared library it depends on (dynamic benchmark executable). - current_algo.reset(); - current_algo_props.reset(); - reset_global_device_resources(); - return 0; -} -}; // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/common/conf.hpp b/cpp/bench/ann/src/common/conf.hpp deleted file mode 100644 index 92ba86c6cf..0000000000 --- a/cpp/bench/ann/src/common/conf.hpp +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "util.hpp" - -#include -#include -#include -#include -#include - -#define JSON_DIAGNOSTICS 1 -#include - -namespace raft::bench::ann { - -class Configuration { - public: - struct Index { - std::string name; - std::string algo; - nlohmann::json build_param; - std::string file; - std::vector dev_list; - - int batch_size; - int k; - std::vector search_params; - }; - - struct DatasetConf { - std::string name; - std::string base_file; - // use only a subset of base_file, - // the range of rows is [subset_first_row, subset_first_row + subset_size) - // however, subset_size = 0 means using all rows after subset_first_row - // that is, the subset is [subset_first_row, #rows in base_file) - size_t subset_first_row{0}; - size_t subset_size{0}; - std::string query_file; - std::string distance; - std::optional groundtruth_neighbors_file{std::nullopt}; - - // data type of input dataset, possible values ["float", "int8", "uint8"] - std::string dtype; - }; - - explicit inline Configuration(std::istream& conf_stream) - { - // to enable comments in json - auto conf = nlohmann::json::parse(conf_stream, nullptr, true, true); - - parse_dataset_(conf.at("dataset")); - parse_index_(conf.at("index"), conf.at("search_basic_param")); - } - - [[nodiscard]] inline auto get_dataset_conf() const -> DatasetConf { return dataset_conf_; } - [[nodiscard]] inline auto get_indices() const -> std::vector { return indices_; }; - - private: - inline void parse_dataset_(const nlohmann::json& conf) - { - dataset_conf_.name = conf.at("name"); - dataset_conf_.base_file = conf.at("base_file"); - dataset_conf_.query_file = conf.at("query_file"); - dataset_conf_.distance = conf.at("distance"); - - if (conf.contains("groundtruth_neighbors_file")) { - dataset_conf_.groundtruth_neighbors_file = conf.at("groundtruth_neighbors_file"); - } - if (conf.contains("subset_first_row")) { - dataset_conf_.subset_first_row = conf.at("subset_first_row"); - } - if (conf.contains("subset_size")) { dataset_conf_.subset_size = conf.at("subset_size"); } - - if (conf.contains("dtype")) { - dataset_conf_.dtype = conf.at("dtype"); - } else { - auto filename = dataset_conf_.base_file; - if (filename.size() > 6 && filename.compare(filename.size() - 6, 6, "f16bin") == 0) { - dataset_conf_.dtype = "half"; - } else if (filename.size() > 9 && - filename.compare(filename.size() - 9, 9, "fp16.fbin") == 0) { - dataset_conf_.dtype = "half"; - } else if (filename.size() > 4 && filename.compare(filename.size() - 4, 4, "fbin") == 0) { - dataset_conf_.dtype = "float"; - } else if (filename.size() > 5 && filename.compare(filename.size() - 5, 5, "u8bin") == 0) { - dataset_conf_.dtype = "uint8"; - } else if (filename.size() > 5 && filename.compare(filename.size() - 5, 5, "i8bin") == 0) { - dataset_conf_.dtype = "int8"; - } else { - log_error("Could not determine data type of the dataset %s", filename.c_str()); - } - } - } - inline void parse_index_(const nlohmann::json& index_conf, - const nlohmann::json& search_basic_conf) - { - const int batch_size = search_basic_conf.at("batch_size"); - const int k = search_basic_conf.at("k"); - - for (const auto& conf : index_conf) { - Index index; - index.name = conf.at("name"); - index.algo = conf.at("algo"); - index.build_param = conf.at("build_param"); - index.file = conf.at("file"); - index.batch_size = batch_size; - index.k = k; - - if (conf.contains("multigpu")) { - for (auto it : conf.at("multigpu")) { - index.dev_list.push_back(it); - } - if (index.dev_list.empty()) { throw std::runtime_error("dev_list shouln't be empty!"); } - index.dev_list.shrink_to_fit(); - index.build_param["multigpu"] = conf["multigpu"]; - } - - for (auto param : conf.at("search_params")) { - /* ### Special parameters for backward compatibility ### - - - Local values of `k` and `n_queries` take priority. - - The legacy "batch_size" renamed to `n_queries`. - - Basic search params are used otherwise. - */ - if (!param.contains("k")) { param["k"] = k; } - if (!param.contains("n_queries")) { - if (param.contains("batch_size")) { - param["n_queries"] = param["batch_size"]; - param.erase("batch_size"); - } else { - param["n_queries"] = batch_size; - } - } - index.search_params.push_back(param); - } - - indices_.push_back(index); - } - } - - DatasetConf dataset_conf_; - std::vector indices_; -}; - -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/common/cuda_huge_page_resource.hpp b/cpp/bench/ann/src/common/cuda_huge_page_resource.hpp deleted file mode 100644 index 27be26dfe9..0000000000 --- a/cpp/bench/ann/src/common/cuda_huge_page_resource.hpp +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include - -#include -#include - -#include - -#include -#include - -namespace raft::mr { -/** - * @brief `device_memory_resource` derived class that uses mmap to allocate memory. - * This class enables memory allocation using huge pages. - * It is assumed that the allocated memory is directly accessible on device. This currently only - * works on GH systems. - * - * TODO(tfeher): consider improving or removing this helper once we made progress with - * https://github.com/rapidsai/raft/issues/1819 - */ -class cuda_huge_page_resource final : public rmm::mr::device_memory_resource { - public: - cuda_huge_page_resource() = default; - ~cuda_huge_page_resource() override = default; - cuda_huge_page_resource(cuda_huge_page_resource const&) = default; - cuda_huge_page_resource(cuda_huge_page_resource&&) = default; - cuda_huge_page_resource& operator=(cuda_huge_page_resource const&) = default; - cuda_huge_page_resource& operator=(cuda_huge_page_resource&&) = default; - - private: - /** - * @brief Allocates memory of size at least `bytes` using cudaMalloc. - * - * The returned pointer has at least 256B alignment. - * - * @note Stream argument is ignored - * - * @throws `rmm::bad_alloc` if the requested allocation could not be fulfilled - * - * @param bytes The size, in bytes, of the allocation - * @return void* Pointer to the newly allocated memory - */ - void* do_allocate(std::size_t bytes, rmm::cuda_stream_view) override - { - void* _addr{nullptr}; - _addr = mmap(NULL, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (_addr == MAP_FAILED) { RAFT_FAIL("huge_page_resource::MAP FAILED"); } - if (madvise(_addr, bytes, MADV_HUGEPAGE) == -1) { - munmap(_addr, bytes); - RAFT_FAIL("huge_page_resource::madvise MADV_HUGEPAGE"); - } - memset(_addr, 0, bytes); - return _addr; - } - - /** - * @brief Deallocate memory pointed to by \p p. - * - * @note Stream argument is ignored. - * - * @throws Nothing. - * - * @param p Pointer to be deallocated - */ - void do_deallocate(void* ptr, std::size_t size, rmm::cuda_stream_view) override - { - if (munmap(ptr, size) == -1) { RAFT_FAIL("huge_page_resource::munmap"); } - } - - /** - * @brief Compare this resource to another. - * - * Two cuda_huge_page_resources always compare equal, because they can each - * deallocate memory allocated by the other. - * - * @throws Nothing. - * - * @param other The other resource to compare to - * @return true If the two resources are equivalent - * @return false If the two resources are not equal - */ - [[nodiscard]] bool do_is_equal(device_memory_resource const& other) const noexcept override - { - return dynamic_cast(&other) != nullptr; - } -}; -} // namespace raft::mr diff --git a/cpp/bench/ann/src/common/cuda_pinned_resource.hpp b/cpp/bench/ann/src/common/cuda_pinned_resource.hpp deleted file mode 100644 index 3256fc293c..0000000000 --- a/cpp/bench/ann/src/common/cuda_pinned_resource.hpp +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include -#include -#include - -#include - -namespace raft::mr { -/** - * @brief `device_memory_resource` derived class that uses cudaMallocHost/Free for - * allocation/deallocation. - * - * This is almost the same as rmm::mr::host::pinned_memory_resource, but it has - * device_memory_resource as base class. Pinned memory can be accessed from device, - * and using this allocator we can create device_mdarray backed by pinned allocator. - * - * TODO(tfeher): it would be preferred to just rely on the existing allocator from rmm - * (pinned_memory_resource), but that is incompatible with the container_policy class - * for device matrix, because the latter expects a device_memory_resource. We shall - * revise this once we progress with Issue https://github.com/rapidsai/raft/issues/1819 - */ -class cuda_pinned_resource final : public rmm::mr::device_memory_resource { - public: - cuda_pinned_resource() = default; - ~cuda_pinned_resource() override = default; - cuda_pinned_resource(cuda_pinned_resource const&) = default; - cuda_pinned_resource(cuda_pinned_resource&&) = default; - cuda_pinned_resource& operator=(cuda_pinned_resource const&) = default; - cuda_pinned_resource& operator=(cuda_pinned_resource&&) = default; - - private: - /** - * @brief Allocates memory of size at least `bytes` using cudaMalloc. - * - * The returned pointer has at least 256B alignment. - * - * @note Stream argument is ignored - * - * @throws `rmm::bad_alloc` if the requested allocation could not be fulfilled - * - * @param bytes The size, in bytes, of the allocation - * @return void* Pointer to the newly allocated memory - */ - void* do_allocate(std::size_t bytes, rmm::cuda_stream_view) override - { - void* ptr{nullptr}; - RMM_CUDA_TRY_ALLOC(cudaMallocHost(&ptr, bytes)); - return ptr; - } - - /** - * @brief Deallocate memory pointed to by \p p. - * - * @note Stream argument is ignored. - * - * @throws Nothing. - * - * @param p Pointer to be deallocated - */ - void do_deallocate(void* ptr, std::size_t, rmm::cuda_stream_view) override - { - RMM_ASSERT_CUDA_SUCCESS(cudaFreeHost(ptr)); - } - - /** - * @brief Compare this resource to another. - * - * Two cuda_pinned_resources always compare equal, because they can each - * deallocate memory allocated by the other. - * - * @throws Nothing. - * - * @param other The other resource to compare to - * @return true If the two resources are equivalent - * @return false If the two resources are not equal - */ - [[nodiscard]] bool do_is_equal(device_memory_resource const& other) const noexcept override - { - return dynamic_cast(&other) != nullptr; - } -}; -} // namespace raft::mr diff --git a/cpp/bench/ann/src/common/cuda_stub.hpp b/cpp/bench/ann/src/common/cuda_stub.hpp deleted file mode 100644 index 5ed138a86d..0000000000 --- a/cpp/bench/ann/src/common/cuda_stub.hpp +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* -The content of this header is governed by two preprocessor definitions: - - - BUILD_CPU_ONLY - whether none of the CUDA functions are used. - - ANN_BENCH_LINK_CUDART - dynamically link against this string if defined. - -___________________________________________________________________________________ -|BUILD_CPU_ONLY | ANN_BENCH_LINK_CUDART | cudart | cuda_runtime_api.h | -| | | found | needed | included | -|---------------|-----------------------|-----------|---------|--------------------| -| ON | | false | false | NO | -| ON | "cudart.so.xx.xx" | false | false | NO | -| OFF | | true | true | YES | -| OFF | "cudart.so.xx.xx" | | true | YES | ------------------------------------------------------------------------------------- -*/ - -#pragma once - -#ifndef BUILD_CPU_ONLY -#include -#include -#ifdef ANN_BENCH_LINK_CUDART -#include - -#include -#endif -#else -#include - -typedef void* cudaStream_t; -typedef void* cudaEvent_t; -typedef uint16_t half; -#endif - -namespace raft::bench::ann { - -struct cuda_lib_handle { - void* handle{nullptr}; - explicit cuda_lib_handle() - { -#ifdef ANN_BENCH_LINK_CUDART - constexpr int kFlags = RTLD_NOW | RTLD_GLOBAL | RTLD_DEEPBIND | RTLD_NODELETE; - // The full name of the linked cudart library 'cudart.so.MAJOR.MINOR.PATCH' - char libname[] = ANN_BENCH_LINK_CUDART; // NOLINT - handle = dlopen(ANN_BENCH_LINK_CUDART, kFlags); - if (handle != nullptr) { return; } - // try strip the PATCH - auto p = strrchr(libname, '.'); - p[0] = 0; - handle = dlopen(libname, kFlags); - if (handle != nullptr) { return; } - // try set the MINOR version to 0 - p = strrchr(libname, '.'); - p[1] = '0'; - p[2] = 0; - handle = dlopen(libname, kFlags); - if (handle != nullptr) { return; } - // try strip the MINOR - p[0] = 0; - handle = dlopen(libname, kFlags); - if (handle != nullptr) { return; } - // try strip the MAJOR - p = strrchr(libname, '.'); - p[0] = 0; - handle = dlopen(libname, kFlags); -#endif - } - ~cuda_lib_handle() noexcept - { -#ifdef ANN_BENCH_LINK_CUDART - if (handle != nullptr) { dlclose(handle); } -#endif - } - - template - auto sym(const char* name) -> Symbol - { -#ifdef ANN_BENCH_LINK_CUDART - return reinterpret_cast(dlsym(handle, name)); -#else - return nullptr; -#endif - } - - /** Whether this is NOT a cpu-only package. */ - [[nodiscard]] constexpr inline auto needed() const -> bool - { -#if defined(BUILD_CPU_ONLY) - return false; -#else - return true; -#endif - } - - /** CUDA found, either at compile time or at runtime. */ - [[nodiscard]] inline auto found() const -> bool - { -#if defined(BUILD_CPU_ONLY) - return false; -#elif defined(ANN_BENCH_LINK_CUDART) - return handle != nullptr; -#else - return true; -#endif - } -}; - -static inline cuda_lib_handle cudart{}; - -#ifdef ANN_BENCH_LINK_CUDART -namespace stub { - -[[gnu::weak, gnu::noinline]] cudaError_t cudaMemcpy(void* dst, - const void* src, - size_t count, - enum cudaMemcpyKind kind) -{ - return cudaSuccess; -} - -[[gnu::weak, gnu::noinline]] cudaError_t cudaMalloc(void** ptr, size_t size) -{ - *ptr = nullptr; - return cudaSuccess; -} -[[gnu::weak, gnu::noinline]] cudaError_t cudaMemset(void* devPtr, int value, size_t count) -{ - return cudaSuccess; -} -[[gnu::weak, gnu::noinline]] cudaError_t cudaFree(void* devPtr) { return cudaSuccess; } -[[gnu::weak, gnu::noinline]] cudaError_t cudaStreamCreate(cudaStream_t* pStream) -{ - *pStream = 0; - return cudaSuccess; -} -[[gnu::weak, gnu::noinline]] cudaError_t cudaStreamCreateWithFlags(cudaStream_t* pStream, - unsigned int flags) -{ - *pStream = 0; - return cudaSuccess; -} -[[gnu::weak, gnu::noinline]] cudaError_t cudaStreamDestroy(cudaStream_t pStream) -{ - return cudaSuccess; -} -[[gnu::weak, gnu::noinline]] cudaError_t cudaDeviceSynchronize() { return cudaSuccess; } - -[[gnu::weak, gnu::noinline]] cudaError_t cudaStreamSynchronize(cudaStream_t pStream) -{ - return cudaSuccess; -} -[[gnu::weak, gnu::noinline]] cudaError_t cudaEventCreate(cudaEvent_t* event) -{ - *event = 0; - return cudaSuccess; -} -[[gnu::weak, gnu::noinline]] cudaError_t cudaEventRecord(cudaEvent_t event, cudaStream_t stream) -{ - return cudaSuccess; -} -[[gnu::weak, gnu::noinline]] cudaError_t cudaEventSynchronize(cudaEvent_t event) -{ - return cudaSuccess; -} -[[gnu::weak, gnu::noinline]] cudaError_t cudaEventElapsedTime(float* ms, - cudaEvent_t start, - cudaEvent_t end) -{ - *ms = 0; - return cudaSuccess; -} -[[gnu::weak, gnu::noinline]] cudaError_t cudaEventDestroy(cudaEvent_t event) { return cudaSuccess; } -[[gnu::weak, gnu::noinline]] cudaError_t cudaGetDevice(int* device) -{ - *device = 0; - return cudaSuccess; -}; -[[gnu::weak, gnu::noinline]] cudaError_t cudaDriverGetVersion(int* driver) -{ - *driver = 0; - return cudaSuccess; -}; -[[gnu::weak, gnu::noinline]] cudaError_t cudaRuntimeGetVersion(int* runtime) -{ - *runtime = 0; - return cudaSuccess; -}; -[[gnu::weak, gnu::noinline]] cudaError_t cudaGetDeviceProperties(struct cudaDeviceProp* prop, - int device) -{ - *prop = cudaDeviceProp{}; - return cudaSuccess; -} - -} // namespace stub - -#define RAFT_DECLARE_CUDART(fun) \ - static inline decltype(&stub::fun) fun = \ - cudart.found() ? cudart.sym(#fun) : &stub::fun - -RAFT_DECLARE_CUDART(cudaMemcpy); -RAFT_DECLARE_CUDART(cudaMalloc); -RAFT_DECLARE_CUDART(cudaMemset); -RAFT_DECLARE_CUDART(cudaFree); -RAFT_DECLARE_CUDART(cudaStreamCreate); -RAFT_DECLARE_CUDART(cudaStreamCreateWithFlags); -RAFT_DECLARE_CUDART(cudaStreamDestroy); -RAFT_DECLARE_CUDART(cudaDeviceSynchronize); -RAFT_DECLARE_CUDART(cudaStreamSynchronize); -RAFT_DECLARE_CUDART(cudaEventCreate); -RAFT_DECLARE_CUDART(cudaEventRecord); -RAFT_DECLARE_CUDART(cudaEventSynchronize); -RAFT_DECLARE_CUDART(cudaEventElapsedTime); -RAFT_DECLARE_CUDART(cudaEventDestroy); -RAFT_DECLARE_CUDART(cudaGetDevice); -RAFT_DECLARE_CUDART(cudaDriverGetVersion); -RAFT_DECLARE_CUDART(cudaRuntimeGetVersion); -RAFT_DECLARE_CUDART(cudaGetDeviceProperties); - -#undef RAFT_DECLARE_CUDART -#endif - -}; // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/common/dataset.hpp b/cpp/bench/ann/src/common/dataset.hpp deleted file mode 100644 index 8fcff77d3c..0000000000 --- a/cpp/bench/ann/src/common/dataset.hpp +++ /dev/null @@ -1,495 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "util.hpp" - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace raft::bench::ann { - -// http://big-ann-benchmarks.com/index.html: -// binary format that starts with 8 bytes of data consisting of num_points(uint32_t) -// num_dimensions(uint32) followed by num_pts x num_dimensions x sizeof(type) bytes of -// data stored one vector after another. -// Data files will have suffixes .fbin, .u8bin, and .i8bin to represent float32, uint8 -// and int8 type data. -// As extensions for this benchmark, half and int data files will have suffixes .f16bin -// and .ibin, respectively. -template -class BinFile { - public: - BinFile(const std::string& file, - const std::string& mode, - uint32_t subset_first_row = 0, - uint32_t subset_size = 0); - ~BinFile() - { - if (mapped_ptr_ != nullptr) { unmap(); } - if (fp_ != nullptr) { fclose(fp_); } - } - BinFile(const BinFile&) = delete; - BinFile& operator=(const BinFile&) = delete; - - void get_shape(size_t* nrows, int* ndims) const - { - assert(read_mode_); - if (!fp_) { open_file_(); } - *nrows = nrows_; - *ndims = ndims_; - } - - void read(T* data) const - { - assert(read_mode_); - if (!fp_) { open_file_(); } - size_t total = static_cast(nrows_) * ndims_; - if (fread(data, sizeof(T), total, fp_) != total) { - throw std::runtime_error("fread() BinFile " + file_ + " failed"); - } - } - - void write(const T* data, uint32_t nrows, uint32_t ndims) - { - assert(!read_mode_); - if (!fp_) { open_file_(); } - if (fwrite(&nrows, sizeof(uint32_t), 1, fp_) != 1) { - throw std::runtime_error("fwrite() BinFile " + file_ + " failed"); - } - if (fwrite(&ndims, sizeof(uint32_t), 1, fp_) != 1) { - throw std::runtime_error("fwrite() BinFile " + file_ + " failed"); - } - - size_t total = static_cast(nrows) * ndims; - if (fwrite(data, sizeof(T), total, fp_) != total) { - throw std::runtime_error("fwrite() BinFile " + file_ + " failed"); - } - } - - T* map() const - { - assert(read_mode_); - if (!fp_) { open_file_(); } - int fid = fileno(fp_); - mapped_ptr_ = mmap(nullptr, file_size_, PROT_READ, MAP_PRIVATE, fid, 0); - if (mapped_ptr_ == MAP_FAILED) { - mapped_ptr_ = nullptr; - throw std::runtime_error("mmap error: Value of errno " + std::to_string(errno) + ", " + - std::string(strerror(errno))); - } - return reinterpret_cast(reinterpret_cast(mapped_ptr_) + 2 * sizeof(uint32_t) + - subset_first_row_ * ndims_ * sizeof(T)); - } - - void unmap() const - { - if (munmap(mapped_ptr_, file_size_) == -1) { - throw std::runtime_error("munmap error: " + std::string(strerror(errno))); - } - } - - private: - void check_suffix_(); - void open_file_() const; - - std::string file_; - bool read_mode_; - uint32_t subset_first_row_; - uint32_t subset_size_; - - mutable FILE* fp_{nullptr}; - mutable uint32_t nrows_; - mutable uint32_t ndims_; - mutable size_t file_size_; - mutable void* mapped_ptr_{nullptr}; -}; - -template -BinFile::BinFile(const std::string& file, - const std::string& mode, - uint32_t subset_first_row, - uint32_t subset_size) - : file_(file), - read_mode_(mode == "r"), - subset_first_row_(subset_first_row), - subset_size_(subset_size), - fp_(nullptr) -{ - check_suffix_(); - - if (!read_mode_) { - if (mode == "w") { - if (subset_first_row != 0) { - throw std::runtime_error("subset_first_row should be zero for write mode"); - } - if (subset_size != 0) { - throw std::runtime_error("subset_size should be zero for write mode"); - } - } else { - throw std::runtime_error("BinFile's mode must be either 'r' or 'w': " + file_); - } - } -} - -template -void BinFile::open_file_() const -{ - fp_ = fopen(file_.c_str(), read_mode_ ? "r" : "w"); - if (!fp_) { throw std::runtime_error("open BinFile failed: " + file_); } - - if (read_mode_) { - struct stat statbuf; - if (stat(file_.c_str(), &statbuf) != 0) { throw std::runtime_error("stat() failed: " + file_); } - file_size_ = statbuf.st_size; - - uint32_t header[2]; - if (fread(header, sizeof(uint32_t), 2, fp_) != 2) { - throw std::runtime_error("read header of BinFile failed: " + file_); - } - nrows_ = header[0]; - ndims_ = header[1]; - - size_t expected_file_size = - 2 * sizeof(uint32_t) + static_cast(nrows_) * ndims_ * sizeof(T); - if (file_size_ != expected_file_size) { - throw std::runtime_error("expected file size of " + file_ + " is " + - std::to_string(expected_file_size) + ", however, actual size is " + - std::to_string(file_size_)); - } - - if (subset_first_row_ >= nrows_) { - throw std::runtime_error(file_ + ": subset_first_row (" + std::to_string(subset_first_row_) + - ") >= nrows (" + std::to_string(nrows_) + ")"); - } - if (subset_first_row_ + subset_size_ > nrows_) { - throw std::runtime_error(file_ + ": subset_first_row (" + std::to_string(subset_first_row_) + - ") + subset_size (" + std::to_string(subset_size_) + ") > nrows (" + - std::to_string(nrows_) + ")"); - } - - if (subset_first_row_) { - static_assert(sizeof(long) == 8, "fseek() don't support 64-bit offset"); - if (fseek(fp_, sizeof(T) * subset_first_row_ * ndims_, SEEK_CUR) == -1) { - throw std::runtime_error(file_ + ": fseek failed"); - } - nrows_ -= subset_first_row_; - } - if (subset_size_) { nrows_ = subset_size_; } - } -} - -template -void BinFile::check_suffix_() -{ - auto pos = file_.rfind('.'); - if (pos == std::string::npos) { - throw std::runtime_error("name of BinFile doesn't have a suffix: " + file_); - } - std::string suffix = file_.substr(pos + 1); - - if constexpr (std::is_same_v) { - if (suffix != "fbin") { - throw std::runtime_error("BinFile should has .fbin suffix: " + file_); - } - } else if constexpr (std::is_same_v) { - if (suffix != "f16bin" && suffix != "fbin") { - throw std::runtime_error("BinFile should has .f16bin suffix: " + file_); - } - } else if constexpr (std::is_same_v) { - if (suffix != "ibin") { - throw std::runtime_error("BinFile should has .ibin suffix: " + file_); - } - } else if constexpr (std::is_same_v) { - if (suffix != "u8bin") { - throw std::runtime_error("BinFile should has .u8bin suffix: " + file_); - } - } else if constexpr (std::is_same_v) { - if (suffix != "i8bin") { - throw std::runtime_error("BinFile should has .i8bin suffix: " + file_); - } - } else { - throw std::runtime_error( - "T of BinFile should be one of float, half, int, uint8_t, or int8_t"); - } -} - -template -class Dataset { - public: - Dataset(const std::string& name) : name_(name) {} - Dataset(const std::string& name, const std::string& distance) : name_(name), distance_(distance) - { - } - Dataset(const Dataset&) = delete; - Dataset& operator=(const Dataset&) = delete; - virtual ~Dataset(); - - std::string name() const { return name_; } - std::string distance() const { return distance_; } - virtual int dim() const = 0; - virtual uint32_t max_k() const = 0; - virtual size_t base_set_size() const = 0; - virtual size_t query_set_size() const = 0; - - // load data lazily, so don't pay the overhead of reading unneeded set - // e.g. don't load base set when searching - const T* base_set() const - { - if (!base_set_) { load_base_set_(); } - return base_set_; - } - - const T* query_set() const - { - if (!query_set_) { load_query_set_(); } - return query_set_; - } - - const int32_t* gt_set() const - { - if (!gt_set_) { load_gt_set_(); } - return gt_set_; - } - - const T* base_set_on_gpu() const; - const T* query_set_on_gpu() const; - const T* mapped_base_set() const; - - auto query_set(MemoryType memory_type) const -> const T* - { - switch (memory_type) { - case MemoryType::Device: return query_set_on_gpu(); - default: return query_set(); - } - } - - auto base_set(MemoryType memory_type) const -> const T* - { - switch (memory_type) { - case MemoryType::Device: return base_set_on_gpu(); - case MemoryType::Host: return base_set(); - case MemoryType::HostMmap: return mapped_base_set(); - default: return nullptr; - } - } - - protected: - virtual void load_base_set_() const = 0; - virtual void load_gt_set_() const = 0; - virtual void load_query_set_() const = 0; - virtual void map_base_set_() const = 0; - - std::string name_; - std::string distance_; - - mutable T* base_set_ = nullptr; - mutable T* query_set_ = nullptr; - mutable T* d_base_set_ = nullptr; - mutable T* d_query_set_ = nullptr; - mutable T* mapped_base_set_ = nullptr; - mutable int32_t* gt_set_ = nullptr; -}; - -template -Dataset::~Dataset() -{ - delete[] base_set_; - delete[] query_set_; - delete[] gt_set_; -#ifndef BUILD_CPU_ONLY - if (d_base_set_) { cudaFree(d_base_set_); } - if (d_query_set_) { cudaFree(d_query_set_); } -#endif -} - -template -const T* Dataset::base_set_on_gpu() const -{ -#ifndef BUILD_CPU_ONLY - if (!d_base_set_) { - base_set(); - cudaMalloc((void**)&d_base_set_, base_set_size() * dim() * sizeof(T)); - cudaMemcpy(d_base_set_, base_set_, base_set_size() * dim() * sizeof(T), cudaMemcpyHostToDevice); - } -#endif - return d_base_set_; -} - -template -const T* Dataset::query_set_on_gpu() const -{ -#ifndef BUILD_CPU_ONLY - if (!d_query_set_) { - query_set(); - cudaMalloc((void**)&d_query_set_, query_set_size() * dim() * sizeof(T)); - cudaMemcpy( - d_query_set_, query_set_, query_set_size() * dim() * sizeof(T), cudaMemcpyHostToDevice); - } -#endif - return d_query_set_; -} - -template -const T* Dataset::mapped_base_set() const -{ - if (!mapped_base_set_) { map_base_set_(); } - return mapped_base_set_; -} - -template -class BinDataset : public Dataset { - public: - BinDataset(const std::string& name, - const std::string& base_file, - size_t subset_first_row, - size_t subset_size, - const std::string& query_file, - const std::string& distance, - const std::optional& groundtruth_neighbors_file); - - int dim() const override; - uint32_t max_k() const override; - size_t base_set_size() const override; - size_t query_set_size() const override; - - private: - void load_base_set_() const override; - void load_query_set_() const override; - void load_gt_set_() const override; - void map_base_set_() const override; - - mutable int dim_ = 0; - mutable uint32_t max_k_ = 0; - mutable size_t base_set_size_ = 0; - mutable size_t query_set_size_ = 0; - - BinFile base_file_; - BinFile query_file_; - std::optional> gt_file_{std::nullopt}; -}; - -template -BinDataset::BinDataset(const std::string& name, - const std::string& base_file, - size_t subset_first_row, - size_t subset_size, - const std::string& query_file, - const std::string& distance, - const std::optional& groundtruth_neighbors_file) - : Dataset(name, distance), - base_file_(base_file, "r", subset_first_row, subset_size), - query_file_(query_file, "r") -{ - if (groundtruth_neighbors_file.has_value()) { - gt_file_.emplace(groundtruth_neighbors_file.value(), "r"); - } -} - -template -int BinDataset::dim() const -{ - if (dim_ > 0) { return dim_; } - if (base_set_size() > 0) { return dim_; } - if (query_set_size() > 0) { return dim_; } - return dim_; -} - -template -uint32_t BinDataset::max_k() const -{ - if (!this->gt_set_) { load_gt_set_(); } - return max_k_; -} - -template -size_t BinDataset::query_set_size() const -{ - if (query_set_size_ > 0) { return query_set_size_; } - int dim; - query_file_.get_shape(&query_set_size_, &dim); - if (query_set_size_ == 0) { throw std::runtime_error("Zero query set size"); } - if (dim == 0) { throw std::runtime_error("Zero query set dim"); } - if (dim_ == 0) { - dim_ = dim; - } else if (dim_ != dim) { - throw std::runtime_error("base set dim (" + std::to_string(dim_) + ") != query set dim (" + - std::to_string(dim)); - } - return query_set_size_; -} - -template -size_t BinDataset::base_set_size() const -{ - if (base_set_size_ > 0) { return base_set_size_; } - int dim; - base_file_.get_shape(&base_set_size_, &dim); - if (base_set_size_ == 0) { throw std::runtime_error("Zero base set size"); } - if (dim == 0) { throw std::runtime_error("Zero base set dim"); } - if (dim_ == 0) { - dim_ = dim; - } else if (dim_ != dim) { - throw std::runtime_error("base set dim (" + std::to_string(dim) + ") != query set dim (" + - std::to_string(dim_)); - } - return base_set_size_; -} - -template -void BinDataset::load_base_set_() const -{ - this->base_set_ = new T[base_set_size() * dim()]; - base_file_.read(this->base_set_); -} - -template -void BinDataset::load_query_set_() const -{ - this->query_set_ = new T[query_set_size() * dim()]; - query_file_.read(this->query_set_); -} - -template -void BinDataset::load_gt_set_() const -{ - if (gt_file_.has_value()) { - size_t queries; - int k; - gt_file_->get_shape(&queries, &k); - this->gt_set_ = new std::int32_t[queries * k]; - gt_file_->read(this->gt_set_); - max_k_ = k; - } -} - -template -void BinDataset::map_base_set_() const -{ - this->mapped_base_set_ = base_file_.map(); -} - -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/common/thread_pool.hpp b/cpp/bench/ann/src/common/thread_pool.hpp deleted file mode 100644 index 4a5684ecb3..0000000000 --- a/cpp/bench/ann/src/common/thread_pool.hpp +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include - -#include -#include -#include -#include -#include -#include -#include - -class FixedThreadPool { - public: - FixedThreadPool(int num_threads) - { - if (num_threads < 1) { - throw std::runtime_error("num_threads must >= 1"); - } else if (num_threads == 1) { - return; - } - - tasks_ = new Task_[num_threads]; - - threads_.reserve(num_threads); - for (int i = 0; i < num_threads; ++i) { - threads_.emplace_back([&, i] { - auto& task = tasks_[i]; - while (true) { - std::unique_lock lock(task.mtx); - task.cv.wait(lock, - [&] { return task.has_task || finished_.load(std::memory_order_relaxed); }); - if (finished_.load(std::memory_order_relaxed)) { break; } - - task.task(); - task.has_task = false; - } - }); - } - } - - ~FixedThreadPool() - { - if (threads_.empty()) { return; } - - finished_.store(true, std::memory_order_relaxed); - for (unsigned i = 0; i < threads_.size(); ++i) { - auto& task = tasks_[i]; - std::lock_guard(task.mtx); - - task.cv.notify_one(); - threads_[i].join(); - } - - delete[] tasks_; - } - - template - void submit(Func f, IdxT len) - { - // Run functions in main thread if thread pool has no threads - if (threads_.empty()) { - for (IdxT i = 0; i < len; ++i) { - f(i); - } - return; - } - - const int num_threads = threads_.size(); - // one extra part for competition among threads - const IdxT items_per_thread = len / (num_threads + 1); - std::atomic cnt(items_per_thread * num_threads); - - // Wrap function - auto wrapped_f = [&](IdxT start, IdxT end) { - for (IdxT i = start; i < end; ++i) { - f(i); - } - - while (true) { - IdxT i = cnt.fetch_add(1, std::memory_order_relaxed); - if (i >= len) { break; } - f(i); - } - }; - - std::vector> futures; - futures.reserve(num_threads); - for (int i = 0; i < num_threads; ++i) { - IdxT start = i * items_per_thread; - auto& task = tasks_[i]; - { - std::lock_guard lock(task.mtx); - (void)lock; // stop nvcc warning - task.task = std::packaged_task([=] { wrapped_f(start, start + items_per_thread); }); - futures.push_back(task.task.get_future()); - task.has_task = true; - } - task.cv.notify_one(); - } - - for (auto& fut : futures) { - fut.wait(); - } - return; - } - - private: - struct alignas(64) Task_ { - std::mutex mtx; - std::condition_variable cv; - bool has_task = false; - std::packaged_task task; - }; - - Task_* tasks_; - std::vector threads_; - std::atomic finished_{false}; -}; diff --git a/cpp/bench/ann/src/common/util.hpp b/cpp/bench/ann/src/common/util.hpp deleted file mode 100644 index 96185c79eb..0000000000 --- a/cpp/bench/ann/src/common/util.hpp +++ /dev/null @@ -1,557 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "ann_types.hpp" -#include "cuda_stub.hpp" // cuda-related utils - -#ifdef ANN_BENCH_NVTX3_HEADERS_FOUND -#include -#endif - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace raft::bench::ann { - -/** - * Current thread id as given by the benchmark State. - * It's populated on every call of a benchmark case. - * It's relevant in the 'throughput' mode of the search benchmarks, - * where some algorithms might want to coordinate allocation of the resources. - */ -inline thread_local int benchmark_thread_id = 0; -/** - * Total concurrent thread count as given by the benchmark State. - * It's populated on every call of a benchmark case. - * It's relevant in the 'throughput' mode of the search benchmarks, - * where some algorithms might want to coordinate allocation of the resources. - */ -inline thread_local int benchmark_n_threads = 1; - -struct cuda_timer { - private: - std::optional stream_; - cudaEvent_t start_{nullptr}; - cudaEvent_t stop_{nullptr}; - double total_time_{0}; - - template - static inline auto extract_stream(AnnT* algo) -> std::optional - { - auto gpu_ann = dynamic_cast(algo); - if (gpu_ann != nullptr && gpu_ann->uses_stream()) { - return std::make_optional(gpu_ann->get_sync_stream()); - } - return std::nullopt; - } - - public: - struct cuda_lap { - private: - cudaStream_t stream_; - cudaEvent_t start_; - cudaEvent_t stop_; - double& total_time_; - - public: - cuda_lap(cudaStream_t stream, cudaEvent_t start, cudaEvent_t stop, double& total_time) - : start_(start), stop_(stop), stream_(stream), total_time_(total_time) - { -#ifndef BUILD_CPU_ONLY - cudaEventRecord(start_, stream_); -#endif - } - cuda_lap() = delete; - - ~cuda_lap() noexcept - { -#ifndef BUILD_CPU_ONLY - cudaEventRecord(stop_, stream_); - cudaEventSynchronize(stop_); - float milliseconds = 0.0f; - cudaEventElapsedTime(&milliseconds, start_, stop_); - total_time_ += milliseconds / 1000.0; -#endif - } - }; - - explicit cuda_timer(std::optional stream) : stream_{stream} - { -#ifndef BUILD_CPU_ONLY - if (stream_.has_value()) { - cudaEventCreate(&stop_); - cudaEventCreate(&start_); - } -#endif - } - - template - explicit cuda_timer(const std::unique_ptr& algo) : cuda_timer{extract_stream(algo.get())} - { - } - - ~cuda_timer() noexcept - { -#ifndef BUILD_CPU_ONLY - if (stream_.has_value()) { - cudaStreamSynchronize(stream_.value()); - cudaEventDestroy(start_); - cudaEventDestroy(stop_); - } -#endif - } - - cuda_timer() = delete; - cuda_timer(cuda_timer const&) = delete; - cuda_timer(cuda_timer&&) = delete; - auto operator=(cuda_timer const&) -> cuda_timer& = delete; - auto operator=(cuda_timer&&) -> cuda_timer& = delete; - - [[nodiscard]] auto stream() const -> std::optional { return stream_; } - - [[nodiscard]] auto active() const -> bool { return stream_.has_value(); } - - [[nodiscard]] auto total_time() const -> double { return total_time_; } - - [[nodiscard]] auto lap(bool enabled = true) -> std::optional - { - return enabled && stream_.has_value() - ? std::make_optional(stream_.value(), start_, stop_, total_time_) - : std::nullopt; - } -}; - -#ifndef BUILD_CPU_ONLY -// ATM, rmm::stream does not support passing in flags; hence this helper type. -struct non_blocking_stream { - non_blocking_stream() { cudaStreamCreateWithFlags(&stream_, cudaStreamNonBlocking); } - ~non_blocking_stream() noexcept - { - if (stream_ != nullptr) { cudaStreamDestroy(stream_); } - } - non_blocking_stream(non_blocking_stream const&) = delete; - non_blocking_stream(non_blocking_stream&& other) noexcept { std::swap(stream_, other.stream_); } - auto operator=(non_blocking_stream const&) -> non_blocking_stream& = delete; - auto operator=(non_blocking_stream&&) -> non_blocking_stream& = delete; - [[nodiscard]] auto view() const noexcept -> cudaStream_t { return stream_; } - - private: - cudaStream_t stream_{nullptr}; -}; - -namespace detail { -inline std::vector global_stream_pool(0); -inline std::mutex gsp_mutex; -} // namespace detail -#endif - -/** - * Get a stream associated with the current benchmark thread. - * - * Note, the streams are reused between the benchmark cases. - * This makes it easier to profile and analyse multiple benchmark cases in one timeline using tools - * like nsys. - */ -inline auto get_stream_from_global_pool() -> cudaStream_t -{ -#ifndef BUILD_CPU_ONLY - std::lock_guard guard(detail::gsp_mutex); - if (int(detail::global_stream_pool.size()) < benchmark_n_threads) { - detail::global_stream_pool.resize(benchmark_n_threads); - } - return detail::global_stream_pool[benchmark_thread_id].view(); -#else - return nullptr; -#endif -} - -struct result_buffer { - explicit result_buffer(size_t size, cudaStream_t stream) : size_{size}, stream_{stream} - { - if (size_ == 0) { return; } - data_host_ = malloc(size_); -#ifndef BUILD_CPU_ONLY - cudaMallocAsync(&data_device_, size_, stream_); - cudaStreamSynchronize(stream_); -#endif - } - result_buffer() = delete; - result_buffer(result_buffer&&) = delete; - result_buffer& operator=(result_buffer&&) = delete; - result_buffer(const result_buffer&) = delete; - result_buffer& operator=(const result_buffer&) = delete; - ~result_buffer() noexcept - { - if (size_ == 0) { return; } -#ifndef BUILD_CPU_ONLY - cudaFreeAsync(data_device_, stream_); - cudaStreamSynchronize(stream_); -#endif - free(data_host_); - } - - [[nodiscard]] auto size() const noexcept { return size_; } - [[nodiscard]] auto data(ann::MemoryType loc) const noexcept - { - switch (loc) { - case MemoryType::Device: return data_device_; - default: return data_host_; - } - } - - void transfer_data(ann::MemoryType dst, ann::MemoryType src) - { - auto dst_ptr = data(dst); - auto src_ptr = data(src); - if (dst_ptr == src_ptr) { return; } -#ifndef BUILD_CPU_ONLY - cudaMemcpyAsync(dst_ptr, src_ptr, size_, cudaMemcpyDefault, stream_); - cudaStreamSynchronize(stream_); -#endif - } - - private: - size_t size_{0}; - cudaStream_t stream_ = nullptr; - void* data_host_ = nullptr; - void* data_device_ = nullptr; -}; - -namespace detail { -inline std::vector> global_result_buffer_pool(0); -inline std::mutex grp_mutex; -} // namespace detail - -/** - * Get a result buffer associated with the current benchmark thread. - * - * Note, the allocations are reused between the benchmark cases. - * This reduces the setup overhead and number of times the context is being blocked - * (this is relevant if there is a persistent kernel running across multiples benchmark cases). - */ -inline auto get_result_buffer_from_global_pool(size_t size) -> result_buffer& -{ - auto stream = get_stream_from_global_pool(); - auto& rb = [stream, size]() -> result_buffer& { - std::lock_guard guard(detail::grp_mutex); - if (static_cast(detail::global_result_buffer_pool.size()) < benchmark_n_threads) { - detail::global_result_buffer_pool.resize(benchmark_n_threads); - } - auto& rb = detail::global_result_buffer_pool[benchmark_thread_id]; - if (!rb || rb->size() < size) { rb = std::make_unique(size, stream); } - return *rb; - }(); - - memset(rb.data(MemoryType::Host), 0, size); -#ifndef BUILD_CPU_ONLY - cudaMemsetAsync(rb.data(MemoryType::Device), 0, size, stream); - cudaStreamSynchronize(stream); -#endif - return rb; -} - -/** - * Delete all streams and memory allocations in the global pool. - * It's called at the end of the `main` function - before global/static variables and cuda context - * is destroyed - to make sure they are destroyed gracefully and correctly seen by analysis tools - * such as nsys. - */ -inline void reset_global_device_resources() -{ -#ifndef BUILD_CPU_ONLY - std::lock_guard guard(detail::gsp_mutex); - detail::global_result_buffer_pool.resize(0); - detail::global_stream_pool.resize(0); -#endif -} - -inline auto cuda_info() -{ - std::vector> props; -#ifndef BUILD_CPU_ONLY - int dev, driver = 0, runtime = 0; - cudaDriverGetVersion(&driver); - cudaRuntimeGetVersion(&runtime); - - cudaDeviceProp device_prop; - cudaGetDevice(&dev); - cudaGetDeviceProperties(&device_prop, dev); - props.emplace_back("gpu_name", std::string(device_prop.name)); - props.emplace_back("gpu_sm_count", std::to_string(device_prop.multiProcessorCount)); - props.emplace_back("gpu_sm_freq", std::to_string(device_prop.clockRate * 1e3)); - props.emplace_back("gpu_mem_freq", std::to_string(device_prop.memoryClockRate * 1e3)); - props.emplace_back("gpu_mem_bus_width", std::to_string(device_prop.memoryBusWidth)); - props.emplace_back("gpu_mem_global_size", std::to_string(device_prop.totalGlobalMem)); - props.emplace_back("gpu_mem_shared_size", std::to_string(device_prop.sharedMemPerMultiprocessor)); - props.emplace_back("gpu_driver_version", - std::to_string(driver / 1000) + "." + std::to_string((driver % 100) / 10)); - props.emplace_back("gpu_runtime_version", - std::to_string(runtime / 1000) + "." + std::to_string((runtime % 100) / 10)); -#endif - return props; -} - -struct nvtx_case { -#ifdef ANN_BENCH_NVTX3_HEADERS_FOUND - private: - std::string case_name_; - std::array iter_name_{0}; - nvtxDomainHandle_t domain_; - int64_t iteration_ = 0; - nvtxEventAttributes_t case_attrib_{0}; - nvtxEventAttributes_t iter_attrib_{0}; -#endif - - public: - struct nvtx_lap { -#ifdef ANN_BENCH_NVTX3_HEADERS_FOUND - private: - nvtxDomainHandle_t domain_; - - public: - nvtx_lap(nvtxDomainHandle_t domain, nvtxEventAttributes_t* attr) : domain_(domain) - { - nvtxDomainRangePushEx(domain_, attr); - } - nvtx_lap() = delete; - ~nvtx_lap() noexcept { nvtxDomainRangePop(domain_); } -#endif - }; - -#ifdef ANN_BENCH_NVTX3_HEADERS_FOUND - explicit nvtx_case(std::string case_name) - : case_name_(std::move(case_name)), domain_(nvtxDomainCreateA("ANN benchmark")) - { - case_attrib_.version = NVTX_VERSION; - iter_attrib_.version = NVTX_VERSION; - case_attrib_.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE; - iter_attrib_.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE; - case_attrib_.colorType = NVTX_COLOR_ARGB; - iter_attrib_.colorType = NVTX_COLOR_ARGB; - case_attrib_.messageType = NVTX_MESSAGE_TYPE_ASCII; - iter_attrib_.messageType = NVTX_MESSAGE_TYPE_ASCII; - case_attrib_.message.ascii = case_name_.c_str(); - auto c = std::hash{}(case_name_); - case_attrib_.color = c | 0xA0A0A0; - nvtxDomainRangePushEx(domain_, &case_attrib_); - } - - ~nvtx_case() - { - nvtxDomainRangePop(domain_); - nvtxDomainDestroy(domain_); - } -#else - explicit nvtx_case(std::string) {} -#endif - - [[nodiscard]] auto lap() -> nvtx_case::nvtx_lap - { -#ifdef ANN_BENCH_NVTX3_HEADERS_FOUND - auto i = iteration_++; - uint32_t c = (i % 5); - uint32_t r = 150 + c * 20; - uint32_t g = 200 + c * 10; - uint32_t b = 220 + c * 5; - std::snprintf(iter_name_.data(), iter_name_.size(), "Lap %zd", i); - iter_attrib_.message.ascii = iter_name_.data(); - iter_attrib_.color = (r << 16) + (g << 8) + b; - return nvtx_lap{domain_, &iter_attrib_}; -#else - return nvtx_lap{}; -#endif - } -}; - -/** - * A progress tracker that allows syncing threads multiple times and resets the global - * progress once the threads are done. - */ -struct progress_barrier { - progress_barrier() = default; - ~progress_barrier() noexcept - { - { - // Lock makes sure the notified threads see the updates to `done_`. - std::unique_lock lk(mutex_); - done_.store(true, std::memory_order_relaxed); - cv_.notify_all(); - } - // This is the only place where the order of the updates to thread_progress_ and done_ is - // important. They are not guarded by the mutex, and `done_` must not be reset to `true` by - // other threads after the `total_progress_` is zero. - // Hence the default memory order (std::memory_order_seq_cst). - auto rem = total_progress_.fetch_sub(thread_progress_); - if (rem == thread_progress_) { - // the last thread to exit clears the progress state. - done_.store(false); - } - } - - /** - * Advance the progress counter by `n` and return the previous `progress` value. - * - * This can be used to track which thread arrives on the call site first. - * - * @return the previous progress counter value (before incrementing it by `n`). - */ - auto arrive(int n) - { - thread_progress_ += n; - // Lock makes sure the notified threads see the updates to `total_progress_`. - std::unique_lock lk(mutex_); - auto prev = total_progress_.fetch_add(n, std::memory_order_relaxed); - cv_.notify_all(); - return prev; - } - - /** - * Wait till the progress counter reaches `n` or finishes abnormally. - * - * @return the latest observed value of the progress counter. - */ - auto wait(int limit) - { - int cur = total_progress_.load(std::memory_order_relaxed); - if (cur >= limit) { return cur; } - auto done = done_.load(std::memory_order_relaxed); - if (done) { return cur; } - std::unique_lock lk(mutex_); - while (cur < limit && !done) { - using namespace std::chrono_literals; - cv_.wait_for(lk, 10ms); - cur = total_progress_.load(std::memory_order_relaxed); - done = done_.load(std::memory_order_relaxed); - } - return cur; - } - - private: - static inline std::atomic total_progress_; - static inline std::atomic done_; - static inline std::mutex mutex_; - static inline std::condition_variable cv_; - int thread_progress_{0}; -}; - -inline std::vector split(const std::string& s, char delimiter) -{ - std::vector tokens; - std::string token; - std::istringstream iss(s); - while (getline(iss, token, delimiter)) { - if (!token.empty()) { tokens.push_back(token); } - } - return tokens; -} - -inline bool file_exists(const std::string& filename) -{ - struct stat statbuf; - if (stat(filename.c_str(), &statbuf) != 0) { return false; } - return S_ISREG(statbuf.st_mode); -} - -inline bool dir_exists(const std::string& dir) -{ - struct stat statbuf; - if (stat(dir.c_str(), &statbuf) != 0) { return false; } - return S_ISDIR(statbuf.st_mode); -} - -inline bool create_dir(const std::string& dir) -{ - const auto path = split(dir, '/'); - - std::string cwd; - if (!dir.empty() && dir[0] == '/') { cwd += '/'; } - - for (const auto& p : path) { - cwd += p + "/"; - if (!dir_exists(cwd)) { - int ret = mkdir(cwd.c_str(), S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); - if (ret != 0) { return false; } - } - } - return true; -} - -inline void make_sure_parent_dir_exists(const std::string& file_path) -{ - const auto pos = file_path.rfind('/'); - if (pos != std::string::npos) { - auto dir = file_path.substr(0, pos); - if (!dir_exists(dir)) { create_dir(dir); } - } -} - -inline auto combine_path(const std::string& dir, const std::string& path) -{ - std::filesystem::path p_dir(dir); - std::filesystem::path p_suf(path); - return (p_dir / p_suf).string(); -} - -template -void log_(const char* level, const Ts&... vs) -{ - char buf[20]; - std::time_t now = std::time(nullptr); - std::strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", std::localtime(&now)); - printf("%s [%s] ", buf, level); - if constexpr (sizeof...(Ts) == 1) { - printf("%s", vs...); - } else { - printf(vs...); - } - printf("\n"); - fflush(stdout); -} - -template -void log_info(Ts&&... vs) -{ - log_("info", std::forward(vs)...); -} - -template -void log_warn(Ts&&... vs) -{ - log_("warn", std::forward(vs)...); -} - -template -void log_error(Ts&&... vs) -{ - log_("error", std::forward(vs)...); -} - -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp b/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp deleted file mode 100644 index 234b33d80a..0000000000 --- a/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../common/ann_types.hpp" -#include "faiss_cpu_wrapper.h" - -#define JSON_DIAGNOSTICS 1 -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace raft::bench::ann { - -template -void parse_base_build_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissCpu::BuildParam& param) -{ - param.nlist = conf.at("nlist"); - if (conf.contains("ratio")) { param.ratio = conf.at("ratio"); } -} - -template -void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissCpuIVFFlat::BuildParam& param) -{ - parse_base_build_param(conf, param); -} - -template -void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissCpuIVFPQ::BuildParam& param) -{ - parse_base_build_param(conf, param); - param.M = conf.at("M"); - if (conf.contains("use_precomputed_table")) { - param.use_precomputed_table = conf.at("use_precomputed_table"); - } else { - param.use_precomputed_table = false; - } - if (conf.contains("bitsPerCode")) { - param.bitsPerCode = conf.at("bitsPerCode"); - } else { - param.bitsPerCode = 8; - } -} - -template -void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissCpuIVFSQ::BuildParam& param) -{ - parse_base_build_param(conf, param); - param.quantizer_type = conf.at("quantizer_type"); -} - -template -void parse_search_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissCpu::SearchParam& param) -{ - param.nprobe = conf.at("nprobe"); - if (conf.contains("refine_ratio")) { param.refine_ratio = conf.at("refine_ratio"); } - if (conf.contains("numThreads")) { param.num_threads = conf.at("numThreads"); } -} - -template class Algo> -std::unique_ptr> make_algo(raft::bench::ann::Metric metric, - int dim, - const nlohmann::json& conf) -{ - typename Algo::BuildParam param; - parse_build_param(conf, param); - return std::make_unique>(metric, dim, param); -} - -template class Algo> -std::unique_ptr> make_algo(raft::bench::ann::Metric metric, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) -{ - typename Algo::BuildParam param; - parse_build_param(conf, param); - - (void)dev_list; - return std::make_unique>(metric, dim, param); -} - -template -std::unique_ptr> create_algo(const std::string& algo, - const std::string& distance, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) -{ - // stop compiler warning; not all algorithms support multi-GPU so it may not be used - (void)dev_list; - - std::unique_ptr> ann; - - if constexpr (std::is_same_v) { - raft::bench::ann::Metric metric = parse_metric(distance); - if (algo == "faiss_cpu_ivf_flat") { - ann = make_algo(metric, dim, conf, dev_list); - } else if (algo == "faiss_cpu_ivf_pq") { - ann = make_algo(metric, dim, conf); - } else if (algo == "faiss_cpu_ivf_sq") { - ann = make_algo(metric, dim, conf); - } else if (algo == "faiss_cpu_flat") { - ann = std::make_unique>(metric, dim); - } - } - - if constexpr (std::is_same_v) {} - - if (!ann) { throw std::runtime_error("invalid algo: '" + algo + "'"); } - - return ann; -} - -template -std::unique_ptr::AnnSearchParam> create_search_param( - const std::string& algo, const nlohmann::json& conf) -{ - if (algo == "faiss_cpu_ivf_flat" || algo == "faiss_cpu_ivf_pq" || algo == "faiss_cpu_ivf_sq") { - auto param = std::make_unique::SearchParam>(); - parse_search_param(conf, *param); - return param; - } else if (algo == "faiss_cpu_flat") { - auto param = std::make_unique::SearchParam>(); - return param; - } - // else - throw std::runtime_error("invalid algo: '" + algo + "'"); -} - -} // namespace raft::bench::ann - -REGISTER_ALGO_INSTANCE(float); -REGISTER_ALGO_INSTANCE(std::int8_t); -REGISTER_ALGO_INSTANCE(std::uint8_t); - -#ifdef ANN_BENCH_BUILD_MAIN -#include "../common/benchmark.hpp" -int main(int argc, char** argv) { return raft::bench::ann::run_main(argc, argv); } -#endif diff --git a/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h b/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h deleted file mode 100644 index c7ce4595b5..0000000000 --- a/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h +++ /dev/null @@ -1,326 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "../common/ann_types.hpp" -#include "../common/thread_pool.hpp" - -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace { - -faiss::MetricType parse_metric_type(raft::bench::ann::Metric metric) -{ - if (metric == raft::bench::ann::Metric::kInnerProduct) { - return faiss::METRIC_INNER_PRODUCT; - } else if (metric == raft::bench::ann::Metric::kEuclidean) { - return faiss::METRIC_L2; - } else { - throw std::runtime_error("faiss supports only metric type of inner product and L2"); - } -} -} // namespace - -namespace raft::bench::ann { - -template -class FaissCpu : public ANN { - public: - using typename ANN::AnnSearchParam; - struct SearchParam : public AnnSearchParam { - int nprobe; - float refine_ratio = 1.0; - int num_threads = omp_get_num_procs(); - }; - - struct BuildParam { - int nlist = 1; - int ratio = 2; - }; - - FaissCpu(Metric metric, int dim, const BuildParam& param) - : ANN(metric, dim), - metric_type_(parse_metric_type(metric)), - nlist_{param.nlist}, - training_sample_fraction_{1.0 / double(param.ratio)} - { - static_assert(std::is_same_v, "faiss support only float type"); - } - - void build(const T* dataset, size_t nrow) final; - - void set_search_param(const AnnSearchParam& param) override; - - void init_quantizer(int dim) - { - if (this->metric_type_ == faiss::MetricType::METRIC_L2) { - this->quantizer_ = std::make_shared(dim); - } else if (this->metric_type_ == faiss::MetricType::METRIC_INNER_PRODUCT) { - this->quantizer_ = std::make_shared(dim); - } - } - - // TODO: if the number of results is less than k, the remaining elements of 'neighbors' - // will be filled with (size_t)-1 - void search(const T* queries, - int batch_size, - int k, - AnnBase::index_type* neighbors, - float* distances) const final; - - AlgoProperty get_preference() const override - { - AlgoProperty property; - // to enable building big dataset which is larger than memory - property.dataset_memory_type = MemoryType::Host; - property.query_memory_type = MemoryType::Host; - return property; - } - - protected: - template - void save_(const std::string& file) const; - - template - void load_(const std::string& file); - - std::shared_ptr index_; - std::shared_ptr quantizer_; - std::shared_ptr index_refine_; - faiss::MetricType metric_type_; - int nlist_; - double training_sample_fraction_; - - int num_threads_; - std::shared_ptr thread_pool_; -}; - -template -void FaissCpu::build(const T* dataset, size_t nrow) -{ - auto index_ivf = dynamic_cast(index_.get()); - if (index_ivf != nullptr) { - // set the min/max training size for clustering to use the whole provided training set. - double trainset_size = training_sample_fraction_ * static_cast(nrow); - double points_per_centroid = trainset_size / static_cast(nlist_); - int max_ppc = std::ceil(points_per_centroid); - int min_ppc = std::floor(points_per_centroid); - if (min_ppc < index_ivf->cp.min_points_per_centroid) { - RAFT_LOG_WARN( - "The suggested training set size %zu (data size %zu, training sample ratio %f) yields %d " - "points per cluster (n_lists = %d). This is smaller than the FAISS default " - "min_points_per_centroid = %d.", - static_cast(trainset_size), - nrow, - training_sample_fraction_, - min_ppc, - nlist_, - index_ivf->cp.min_points_per_centroid); - } - index_ivf->cp.max_points_per_centroid = max_ppc; - index_ivf->cp.min_points_per_centroid = min_ppc; - } - index_->train(nrow, dataset); // faiss::IndexFlat::train() will do nothing - assert(index_->is_trained); - index_->add(nrow, dataset); - index_refine_ = std::make_shared(this->index_.get(), dataset); -} - -template -void FaissCpu::set_search_param(const AnnSearchParam& param) -{ - auto search_param = dynamic_cast(param); - int nprobe = search_param.nprobe; - assert(nprobe <= nlist_); - dynamic_cast(index_.get())->nprobe = nprobe; - - if (search_param.refine_ratio > 1.0) { - this->index_refine_.get()->k_factor = search_param.refine_ratio; - } - - if (!thread_pool_ || num_threads_ != search_param.num_threads) { - num_threads_ = search_param.num_threads; - thread_pool_ = std::make_shared(num_threads_); - } -} - -template -void FaissCpu::search( - const T* queries, int batch_size, int k, AnnBase::index_type* neighbors, float* distances) const -{ - static_assert(sizeof(size_t) == sizeof(faiss::idx_t), - "sizes of size_t and faiss::idx_t are different"); - - thread_pool_->submit( - [&](int i) { - // Use thread pool for batch size = 1. FAISS multi-threads internally for batch size > 1. - index_->search(batch_size, queries, k, distances, reinterpret_cast(neighbors)); - }, - 1); -} - -template -template -void FaissCpu::save_(const std::string& file) const -{ - faiss::write_index(index_.get(), file.c_str()); -} - -template -template -void FaissCpu::load_(const std::string& file) -{ - index_ = std::shared_ptr(dynamic_cast(faiss::read_index(file.c_str()))); -} - -template -class FaissCpuIVFFlat : public FaissCpu { - public: - using typename FaissCpu::BuildParam; - - FaissCpuIVFFlat(Metric metric, int dim, const BuildParam& param) : FaissCpu(metric, dim, param) - { - this->init_quantizer(dim); - this->index_ = std::make_shared( - this->quantizer_.get(), dim, param.nlist, this->metric_type_); - } - - void save(const std::string& file) const override - { - this->template save_(file); - } - void load(const std::string& file) override { this->template load_(file); } - - std::unique_ptr> copy() - { - return std::make_unique>(*this); // use copy constructor - } -}; - -template -class FaissCpuIVFPQ : public FaissCpu { - public: - struct BuildParam : public FaissCpu::BuildParam { - int M; - int bitsPerCode; - bool use_precomputed_table; - }; - - FaissCpuIVFPQ(Metric metric, int dim, const BuildParam& param) : FaissCpu(metric, dim, param) - { - this->init_quantizer(dim); - this->index_ = std::make_shared( - this->quantizer_.get(), dim, param.nlist, param.M, param.bitsPerCode, this->metric_type_); - } - - void save(const std::string& file) const override - { - this->template save_(file); - } - void load(const std::string& file) override { this->template load_(file); } - - std::unique_ptr> copy() - { - return std::make_unique>(*this); // use copy constructor - } -}; - -// TODO: Enable this in cmake -// ref: https://github.com/rapidsai/raft/issues/1876 -template -class FaissCpuIVFSQ : public FaissCpu { - public: - struct BuildParam : public FaissCpu::BuildParam { - std::string quantizer_type; - }; - - FaissCpuIVFSQ(Metric metric, int dim, const BuildParam& param) : FaissCpu(metric, dim, param) - { - faiss::ScalarQuantizer::QuantizerType qtype; - if (param.quantizer_type == "fp16") { - qtype = faiss::ScalarQuantizer::QT_fp16; - } else if (param.quantizer_type == "int8") { - qtype = faiss::ScalarQuantizer::QT_8bit; - } else { - throw std::runtime_error("FaissCpuIVFSQ supports only fp16 and int8 but got " + - param.quantizer_type); - } - - this->init_quantizer(dim); - this->index_ = std::make_shared( - this->quantizer_.get(), dim, param.nlist, qtype, this->metric_type_, true); - } - - void save(const std::string& file) const override - { - this->template save_(file); - } - void load(const std::string& file) override - { - this->template load_(file); - } - - std::unique_ptr> copy() - { - return std::make_unique>(*this); // use copy constructor - } -}; - -template -class FaissCpuFlat : public FaissCpu { - public: - FaissCpuFlat(Metric metric, int dim) - : FaissCpu(metric, dim, typename FaissCpu::BuildParam{}) - { - this->index_ = std::make_shared(dim, this->metric_type_); - } - - // class FaissCpu is more like a IVF class, so need special treating here - void set_search_param(const typename ANN::AnnSearchParam& param) override - { - auto search_param = dynamic_cast::SearchParam&>(param); - if (!this->thread_pool_ || this->num_threads_ != search_param.num_threads) { - this->num_threads_ = search_param.num_threads; - this->thread_pool_ = std::make_shared(this->num_threads_); - } - }; - - void save(const std::string& file) const override - { - this->template save_(file); - } - void load(const std::string& file) override { this->template load_(file); } - - std::unique_ptr> copy() - { - return std::make_unique>(*this); // use copy constructor - } -}; - -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/faiss/faiss_gpu_benchmark.cu b/cpp/bench/ann/src/faiss/faiss_gpu_benchmark.cu deleted file mode 100644 index b47c497e3d..0000000000 --- a/cpp/bench/ann/src/faiss/faiss_gpu_benchmark.cu +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../common/ann_types.hpp" - -#undef WARP_SIZE -#include "faiss_gpu_wrapper.h" - -#define JSON_DIAGNOSTICS 1 -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace raft::bench::ann { - -template -void parse_base_build_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissGpu::BuildParam& param) -{ - param.nlist = conf.at("nlist"); - if (conf.contains("ratio")) { param.ratio = conf.at("ratio"); } -} - -template -void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissGpuIVFFlat::BuildParam& param) -{ - parse_base_build_param(conf, param); - if (conf.contains("use_raft")) { - param.use_raft = conf.at("use_raft"); - } else { - param.use_raft = false; - } -} - -template -void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissGpuIVFPQ::BuildParam& param) -{ - parse_base_build_param(conf, param); - param.M = conf.at("M"); - if (conf.contains("usePrecomputed")) { - param.usePrecomputed = conf.at("usePrecomputed"); - } else { - param.usePrecomputed = false; - } - if (conf.contains("useFloat16")) { - param.useFloat16 = conf.at("useFloat16"); - } else { - param.useFloat16 = false; - } - if (conf.contains("use_raft")) { - param.use_raft = conf.at("use_raft"); - } else { - param.use_raft = false; - } - if (conf.contains("bitsPerCode")) { - param.bitsPerCode = conf.at("bitsPerCode"); - } else { - param.bitsPerCode = 8; - } -} - -template -void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissGpuIVFSQ::BuildParam& param) -{ - parse_base_build_param(conf, param); - param.quantizer_type = conf.at("quantizer_type"); -} - -template -void parse_search_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissGpu::SearchParam& param) -{ - param.nprobe = conf.at("nprobe"); - if (conf.contains("refine_ratio")) { param.refine_ratio = conf.at("refine_ratio"); } -} - -template class Algo> -std::unique_ptr> make_algo(raft::bench::ann::Metric metric, - int dim, - const nlohmann::json& conf) -{ - typename Algo::BuildParam param; - parse_build_param(conf, param); - return std::make_unique>(metric, dim, param); -} - -template class Algo> -std::unique_ptr> make_algo(raft::bench::ann::Metric metric, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) -{ - typename Algo::BuildParam param; - parse_build_param(conf, param); - - (void)dev_list; - return std::make_unique>(metric, dim, param); -} - -template -std::unique_ptr> create_algo(const std::string& algo, - const std::string& distance, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) -{ - // stop compiler warning; not all algorithms support multi-GPU so it may not be used - (void)dev_list; - - std::unique_ptr> ann; - - if constexpr (std::is_same_v) { - raft::bench::ann::Metric metric = parse_metric(distance); - if (algo == "faiss_gpu_ivf_flat") { - ann = make_algo(metric, dim, conf, dev_list); - } else if (algo == "faiss_gpu_ivf_pq") { - ann = make_algo(metric, dim, conf); - } else if (algo == "faiss_gpu_ivf_sq") { - ann = make_algo(metric, dim, conf); - } else if (algo == "faiss_gpu_flat") { - ann = std::make_unique>(metric, dim); - } - } - - if constexpr (std::is_same_v) {} - - if (!ann) { throw std::runtime_error("invalid algo: '" + algo + "'"); } - - return ann; -} - -template -std::unique_ptr::AnnSearchParam> create_search_param( - const std::string& algo, const nlohmann::json& conf) -{ - if (algo == "faiss_gpu_ivf_flat" || algo == "faiss_gpu_ivf_pq" || algo == "faiss_gpu_ivf_sq") { - auto param = std::make_unique::SearchParam>(); - parse_search_param(conf, *param); - return param; - } else if (algo == "faiss_gpu_flat") { - auto param = std::make_unique::SearchParam>(); - return param; - } - // else - throw std::runtime_error("invalid algo: '" + algo + "'"); -} - -} // namespace raft::bench::ann - -REGISTER_ALGO_INSTANCE(float); -REGISTER_ALGO_INSTANCE(std::int8_t); -REGISTER_ALGO_INSTANCE(std::uint8_t); - -#ifdef ANN_BENCH_BUILD_MAIN -#include "../common/benchmark.hpp" -int main(int argc, char** argv) -{ - rmm::mr::cuda_memory_resource cuda_mr; - // Construct a resource that uses a coalescing best-fit pool allocator - // and is initially sized to half of free device memory. - rmm::mr::pool_memory_resource pool_mr{ - &cuda_mr, rmm::percent_of_free_device_memory(50)}; - // Updates the current device resource pointer to `pool_mr` - auto old_mr = rmm::mr::set_current_device_resource(&pool_mr); - auto ret = raft::bench::ann::run_main(argc, argv); - // Restores the current device resource pointer to its previous value - rmm::mr::set_current_device_resource(old_mr); - return ret; -} -#endif diff --git a/cpp/bench/ann/src/faiss/faiss_gpu_wrapper.h b/cpp/bench/ann/src/faiss/faiss_gpu_wrapper.h deleted file mode 100644 index 6955201c5d..0000000000 --- a/cpp/bench/ann/src/faiss/faiss_gpu_wrapper.h +++ /dev/null @@ -1,515 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef FAISS_WRAPPER_H_ -#define FAISS_WRAPPER_H_ - -#include "../common/ann_types.hpp" -#include "../raft/raft_ann_bench_utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace { - -faiss::MetricType parse_metric_faiss(raft::bench::ann::Metric metric) -{ - if (metric == raft::bench::ann::Metric::kInnerProduct) { - return faiss::METRIC_INNER_PRODUCT; - } else if (metric == raft::bench::ann::Metric::kEuclidean) { - return faiss::METRIC_L2; - } else { - throw std::runtime_error("faiss supports only metric type of inner product and L2"); - } -} - -// note BLAS library can still use multi-threading, and -// setting environment variable like OPENBLAS_NUM_THREADS can control it -class OmpSingleThreadScope { - public: - OmpSingleThreadScope() - { - max_threads_ = omp_get_max_threads(); - omp_set_num_threads(1); - } - ~OmpSingleThreadScope() - { - // the best we can do - omp_set_num_threads(max_threads_); - } - - private: - int max_threads_; -}; - -} // namespace - -namespace raft::bench::ann { - -template -class FaissGpu : public ANN, public AnnGPU { - public: - using typename ANN::AnnSearchParam; - struct SearchParam : public AnnSearchParam { - int nprobe; - float refine_ratio = 1.0; - auto needs_dataset() const -> bool override { return refine_ratio > 1.0f; } - }; - - struct BuildParam { - int nlist = 1; - int ratio = 2; - }; - - FaissGpu(Metric metric, int dim, const BuildParam& param) - : ANN(metric, dim), - gpu_resource_{std::make_shared()}, - metric_type_(parse_metric_faiss(metric)), - nlist_{param.nlist}, - training_sample_fraction_{1.0 / double(param.ratio)} - { - static_assert(std::is_same_v, "faiss support only float type"); - RAFT_CUDA_TRY(cudaGetDevice(&device_)); - } - - void build(const T* dataset, size_t nrow) final; - - virtual void set_search_param(const FaissGpu::AnnSearchParam& param) {} - - void set_search_dataset(const T* dataset, size_t nrow) override { dataset_ = dataset; } - - // TODO: if the number of results is less than k, the remaining elements of 'neighbors' - // will be filled with (size_t)-1 - void search(const T* queries, - int batch_size, - int k, - AnnBase::index_type* neighbors, - float* distances) const final; - - [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override - { - return gpu_resource_->getDefaultStream(device_); - } - - AlgoProperty get_preference() const override - { - AlgoProperty property; - // to enable building big dataset which is larger than GPU memory - property.dataset_memory_type = MemoryType::Host; - property.query_memory_type = MemoryType::Device; - return property; - } - - protected: - template - void save_(const std::string& file) const; - - template - void load_(const std::string& file); - - /** [NOTE Multithreading] - * - * `gpu_resource_` is a shared resource: - * 1. It uses a shared_ptr under the hood, so the copies of it refer to the same - * resource implementation instance - * 2. GpuIndex is probably keeping a reference to it, as it's passed to the constructor - * - * To avoid copying the index (database) in each thread, we make both the index and - * the gpu_resource shared. - * This means faiss GPU streams are possibly shared among the CPU threads; - * the throughput search mode may be inaccurate. - * - * WARNING: we haven't investigated whether faiss::gpu::GpuIndex or - * faiss::gpu::StandardGpuResources are thread-safe. - * - */ - mutable std::shared_ptr gpu_resource_; - std::shared_ptr index_; - std::shared_ptr index_refine_{nullptr}; - faiss::MetricType metric_type_; - int nlist_; - int device_; - double training_sample_fraction_; - std::shared_ptr search_params_; - std::shared_ptr refine_search_params_{nullptr}; - const T* dataset_; - float refine_ratio_ = 1.0; - Objective metric_objective_; -}; - -template -void FaissGpu::build(const T* dataset, size_t nrow) -{ - OmpSingleThreadScope omp_single_thread; - auto index_ivf = dynamic_cast(index_.get()); - if (index_ivf != nullptr) { - // set the min/max training size for clustering to use the whole provided training set. - double trainset_size = training_sample_fraction_ * static_cast(nrow); - double points_per_centroid = trainset_size / static_cast(nlist_); - int max_ppc = std::ceil(points_per_centroid); - int min_ppc = std::floor(points_per_centroid); - if (min_ppc < index_ivf->cp.min_points_per_centroid) { - RAFT_LOG_WARN( - "The suggested training set size %zu (data size %zu, training sample ratio %f) yields %d " - "points per cluster (n_lists = %d). This is smaller than the FAISS default " - "min_points_per_centroid = %d.", - static_cast(trainset_size), - nrow, - training_sample_fraction_, - min_ppc, - nlist_, - index_ivf->cp.min_points_per_centroid); - } - index_ivf->cp.max_points_per_centroid = max_ppc; - index_ivf->cp.min_points_per_centroid = min_ppc; - } - index_->train(nrow, dataset); // faiss::gpu::GpuIndexFlat::train() will do nothing - assert(index_->is_trained); - index_->add(nrow, dataset); -} - -template -void FaissGpu::search( - const T* queries, int batch_size, int k, AnnBase::index_type* neighbors, float* distances) const -{ - ASSERT(Objective::LATENCY, "l2Knn: rowMajorIndex and rowMajorQuery should have same layout"); - using IdxT = faiss::idx_t; - static_assert(sizeof(size_t) == sizeof(faiss::idx_t), - "sizes of size_t and faiss::idx_t are different"); - - if (refine_ratio_ > 1.0) { - if (raft::get_device_for_address(queries) >= 0) { - uint32_t k0 = static_cast(refine_ratio_ * k); - auto distances_tmp = raft::make_device_matrix( - gpu_resource_->getRaftHandle(device_), batch_size, k0); - auto candidates = - raft::make_device_matrix(gpu_resource_->getRaftHandle(device_), batch_size, k0); - index_->search(batch_size, - queries, - k0, - distances_tmp.data_handle(), - candidates.data_handle(), - this->search_params_.get()); - - auto queries_host = raft::make_host_matrix(batch_size, index_->d); - auto candidates_host = raft::make_host_matrix(batch_size, k0); - auto neighbors_host = raft::make_host_matrix(batch_size, k); - auto distances_host = raft::make_host_matrix(batch_size, k); - auto dataset_v = raft::make_host_matrix_view( - this->dataset_, index_->ntotal, index_->d); - - raft::device_resources handle_ = gpu_resource_->getRaftHandle(device_); - - raft::copy(queries_host.data_handle(), queries, queries_host.size(), handle_.get_stream()); - raft::copy(candidates_host.data_handle(), - candidates.data_handle(), - candidates_host.size(), - handle_.get_stream()); - - // wait for the queries to copy to host in 'stream` - handle_.sync_stream(); - - raft::runtime::neighbors::refine(handle_, - dataset_v, - queries_host.view(), - candidates_host.view(), - neighbors_host.view(), - distances_host.view(), - parse_metric_type(this->metric_)); - - raft::copy(neighbors, - (size_t*)neighbors_host.data_handle(), - neighbors_host.size(), - handle_.get_stream()); - raft::copy( - distances, distances_host.data_handle(), distances_host.size(), handle_.get_stream()); - } else { - index_refine_->search(batch_size, - queries, - k, - distances, - reinterpret_cast(neighbors), - this->refine_search_params_.get()); - } - } else { - index_->search(batch_size, - queries, - k, - distances, - reinterpret_cast(neighbors), - this->search_params_.get()); - } -} - -template -template -void FaissGpu::save_(const std::string& file) const -{ - OmpSingleThreadScope omp_single_thread; - - auto cpu_index = std::make_unique(); - dynamic_cast(index_.get())->copyTo(cpu_index.get()); - faiss::write_index(cpu_index.get(), file.c_str()); -} - -template -template -void FaissGpu::load_(const std::string& file) -{ - OmpSingleThreadScope omp_single_thread; - - std::unique_ptr cpu_index(dynamic_cast(faiss::read_index(file.c_str()))); - assert(cpu_index); - - try { - dynamic_cast(index_.get())->copyFrom(cpu_index.get()); - - } catch (const std::exception& e) { - std::cout << "Error loading index file: " << std::string(e.what()) << std::endl; - } -} - -template -class FaissGpuIVFFlat : public FaissGpu { - public: - struct BuildParam : public FaissGpu::BuildParam { - bool use_raft; - }; - - FaissGpuIVFFlat(Metric metric, int dim, const BuildParam& param) : FaissGpu(metric, dim, param) - { - faiss::gpu::GpuIndexIVFFlatConfig config; - config.device = this->device_; - config.use_raft = param.use_raft; - this->index_ = std::make_shared( - this->gpu_resource_.get(), dim, param.nlist, this->metric_type_, config); - } - - void set_search_param(const typename FaissGpu::AnnSearchParam& param) override - { - auto search_param = dynamic_cast::SearchParam&>(param); - int nprobe = search_param.nprobe; - assert(nprobe <= nlist_); - - faiss::IVFSearchParameters faiss_search_params; - faiss_search_params.nprobe = nprobe; - this->search_params_ = std::make_shared(faiss_search_params); - this->refine_ratio_ = search_param.refine_ratio; - } - - void save(const std::string& file) const override - { - this->template save_(file); - } - void load(const std::string& file) override - { - this->template load_(file); - } - std::unique_ptr> copy() override { return std::make_unique>(*this); }; -}; - -template -class FaissGpuIVFPQ : public FaissGpu { - public: - struct BuildParam : public FaissGpu::BuildParam { - int M; - bool useFloat16; - bool usePrecomputed; - bool use_raft; - int bitsPerCode; - }; - - FaissGpuIVFPQ(Metric metric, int dim, const BuildParam& param) : FaissGpu(metric, dim, param) - { - faiss::gpu::GpuIndexIVFPQConfig config; - config.useFloat16LookupTables = param.useFloat16; - config.usePrecomputedTables = param.usePrecomputed; - config.use_raft = param.use_raft; - config.interleavedLayout = param.use_raft; - config.device = this->device_; - - this->index_ = std::make_shared(this->gpu_resource_.get(), - dim, - param.nlist, - param.M, - param.bitsPerCode, - this->metric_type_, - config); - } - - void set_search_param(const typename FaissGpu::AnnSearchParam& param) override - { - auto search_param = dynamic_cast::SearchParam&>(param); - int nprobe = search_param.nprobe; - assert(nprobe <= nlist_); - this->refine_ratio_ = search_param.refine_ratio; - faiss::IVFPQSearchParameters faiss_search_params; - faiss_search_params.nprobe = nprobe; - - this->search_params_ = std::make_shared(faiss_search_params); - - if (search_param.refine_ratio > 1.0) { - this->index_refine_ = - std::make_shared(this->index_.get(), this->dataset_); - this->index_refine_.get()->k_factor = search_param.refine_ratio; - faiss::IndexRefineSearchParameters faiss_refine_search_params; - faiss_refine_search_params.k_factor = this->index_refine_.get()->k_factor; - faiss_refine_search_params.base_index_params = this->search_params_.get(); - this->refine_search_params_ = - std::make_unique(faiss_refine_search_params); - } - } - - void save(const std::string& file) const override - { - this->template save_(file); - } - void load(const std::string& file) override - { - this->template load_(file); - } - std::unique_ptr> copy() override { return std::make_unique>(*this); }; -}; - -// TODO: Enable this in cmake -// ref: https://github.com/rapidsai/raft/issues/1876 -template -class FaissGpuIVFSQ : public FaissGpu { - public: - struct BuildParam : public FaissGpu::BuildParam { - std::string quantizer_type; - }; - - FaissGpuIVFSQ(Metric metric, int dim, const BuildParam& param) : FaissGpu(metric, dim, param) - { - faiss::ScalarQuantizer::QuantizerType qtype; - if (param.quantizer_type == "fp16") { - qtype = faiss::ScalarQuantizer::QT_fp16; - } else if (param.quantizer_type == "int8") { - qtype = faiss::ScalarQuantizer::QT_8bit; - } else { - throw std::runtime_error("FaissGpuIVFSQ supports only fp16 and int8 but got " + - param.quantizer_type); - } - - faiss::gpu::GpuIndexIVFScalarQuantizerConfig config; - config.device = this->device_; - this->index_ = std::make_shared( - this->gpu_resource_.get(), dim, param.nlist, qtype, this->metric_type_, true, config); - } - - void set_search_param(const typename FaissGpu::AnnSearchParam& param) override - { - auto search_param = dynamic_cast::SearchParam&>(param); - int nprobe = search_param.nprobe; - assert(nprobe <= nlist_); - - faiss::IVFSearchParameters faiss_search_params; - faiss_search_params.nprobe = nprobe; - - this->search_params_ = std::make_shared(faiss_search_params); - this->refine_ratio_ = search_param.refine_ratio; - if (search_param.refine_ratio > 1.0) { - this->index_refine_ = - std::make_shared(this->index_.get(), this->dataset_); - this->index_refine_.get()->k_factor = search_param.refine_ratio; - faiss::IndexRefineSearchParameters faiss_refine_search_params; - faiss_refine_search_params.k_factor = this->index_refine_.get()->k_factor; - faiss_refine_search_params.base_index_params = this->search_params_.get(); - this->refine_search_params_ = - std::make_unique(faiss_refine_search_params); - } - } - - void save(const std::string& file) const override - { - this->template save_( - file); - } - void load(const std::string& file) override - { - this->template load_( - file); - } - std::unique_ptr> copy() override { return std::make_unique>(*this); }; -}; - -template -class FaissGpuFlat : public FaissGpu { - public: - FaissGpuFlat(Metric metric, int dim) - : FaissGpu(metric, dim, typename FaissGpu::BuildParam{}) - { - faiss::gpu::GpuIndexFlatConfig config; - config.device = this->device_; - this->index_ = std::make_shared( - this->gpu_resource_.get(), dim, this->metric_type_, config); - } - void set_search_param(const typename FaissGpu::AnnSearchParam& param) override - { - auto search_param = dynamic_cast::SearchParam&>(param); - int nprobe = search_param.nprobe; - assert(nprobe <= nlist_); - - this->search_params_ = std::make_shared(); - } - - void save(const std::string& file) const override - { - this->template save_(file); - } - void load(const std::string& file) override - { - this->template load_(file); - } - std::unique_ptr> copy() override { return std::make_unique>(*this); }; -}; - -} // namespace raft::bench::ann - -#endif diff --git a/cpp/bench/ann/src/ggnn/ggnn_benchmark.cu b/cpp/bench/ann/src/ggnn/ggnn_benchmark.cu deleted file mode 100644 index 48d41388d4..0000000000 --- a/cpp/bench/ann/src/ggnn/ggnn_benchmark.cu +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../common/ann_types.hpp" -#include "ggnn_wrapper.cuh" - -#define JSON_DIAGNOSTICS 1 -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace raft::bench::ann { - -template -void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::Ggnn::BuildParam& param) -{ - param.k = conf.at("k"); - - if (conf.contains("k_build")) { param.k_build = conf.at("k_build"); } - if (conf.contains("segment_size")) { param.segment_size = conf.at("segment_size"); } - if (conf.contains("num_layers")) { param.num_layers = conf.at("num_layers"); } - if (conf.contains("tau")) { param.tau = conf.at("tau"); } - if (conf.contains("refine_iterations")) { - param.refine_iterations = conf.at("refine_iterations"); - } -} - -template -void parse_search_param(const nlohmann::json& conf, - typename raft::bench::ann::Ggnn::SearchParam& param) -{ - param.tau = conf.at("tau"); - - if (conf.contains("block_dim")) { param.block_dim = conf.at("block_dim"); } - if (conf.contains("max_iterations")) { param.max_iterations = conf.at("max_iterations"); } - if (conf.contains("cache_size")) { param.cache_size = conf.at("cache_size"); } - if (conf.contains("sorted_size")) { param.sorted_size = conf.at("sorted_size"); } -} - -template class Algo> -std::unique_ptr> make_algo(raft::bench::ann::Metric metric, - int dim, - const nlohmann::json& conf) -{ - typename Algo::BuildParam param; - parse_build_param(conf, param); - return std::make_unique>(metric, dim, param); -} - -template class Algo> -std::unique_ptr> make_algo(raft::bench::ann::Metric metric, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) -{ - typename Algo::BuildParam param; - parse_build_param(conf, param); - - (void)dev_list; - return std::make_unique>(metric, dim, param); -} - -template -std::unique_ptr> create_algo(const std::string& algo, - const std::string& distance, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) -{ - // stop compiler warning; not all algorithms support multi-GPU so it may not be used - (void)dev_list; - - raft::bench::ann::Metric metric = parse_metric(distance); - std::unique_ptr> ann; - - if constexpr (std::is_same_v || std::is_same_v || - std::is_same_v) { - if (algo == "ggnn") { ann = make_algo(metric, dim, conf); } - } - if (!ann) { throw std::runtime_error("invalid algo: '" + algo + "'"); } - - return ann; -} - -template -std::unique_ptr::AnnSearchParam> create_search_param( - const std::string& algo, const nlohmann::json& conf) -{ - if constexpr (std::is_same_v || std::is_same_v || - std::is_same_v) { - if (algo == "ggnn") { - auto param = std::make_unique::SearchParam>(); - parse_search_param(conf, *param); - return param; - } - } - // else - throw std::runtime_error("invalid algo: '" + algo + "'"); -} - -} // namespace raft::bench::ann - -REGISTER_ALGO_INSTANCE(float); -REGISTER_ALGO_INSTANCE(std::int8_t); -REGISTER_ALGO_INSTANCE(std::uint8_t); - -#ifdef ANN_BENCH_BUILD_MAIN -#include "../common/benchmark.hpp" -int main(int argc, char** argv) { return raft::bench::ann::run_main(argc, argv); } -#endif diff --git a/cpp/bench/ann/src/ggnn/ggnn_wrapper.cuh b/cpp/bench/ann/src/ggnn/ggnn_wrapper.cuh deleted file mode 100644 index 59cf3df806..0000000000 --- a/cpp/bench/ann/src/ggnn/ggnn_wrapper.cuh +++ /dev/null @@ -1,322 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "../common/ann_types.hpp" -#include "../common/util.hpp" - -#include - -#include - -#include -#include - -namespace raft::bench::ann { - -template -class GgnnImpl; - -template -class Ggnn : public ANN, public AnnGPU { - public: - struct BuildParam { - int k_build{24}; // KBuild - int segment_size{32}; // S - int num_layers{4}; // L - float tau{0.5}; - int refine_iterations{2}; - int k; // GGNN requires to know k during building - }; - - using typename ANN::AnnSearchParam; - struct SearchParam : public AnnSearchParam { - float tau; - int block_dim{32}; - int max_iterations{400}; - int cache_size{512}; - int sorted_size{256}; - auto needs_dataset() const -> bool override { return true; } - }; - - Ggnn(Metric metric, int dim, const BuildParam& param); - - void build(const T* dataset, size_t nrow) override { impl_->build(dataset, nrow); } - - void set_search_param(const AnnSearchParam& param) override { impl_->set_search_param(param); } - void search(const T* queries, - int batch_size, - int k, - AnnBase::index_type* neighbors, - float* distances) const override - { - impl_->search(queries, batch_size, k, neighbors, distances); - } - [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override - { - return dynamic_cast(impl_.get())->get_sync_stream(); - } - - void save(const std::string& file) const override { impl_->save(file); } - void load(const std::string& file) override { impl_->load(file); } - std::unique_ptr> copy() override { return std::make_unique>(*this); }; - - AlgoProperty get_preference() const override { return impl_->get_preference(); } - - void set_search_dataset(const T* dataset, size_t nrow) override - { - impl_->set_search_dataset(dataset, nrow); - }; - - private: - std::shared_ptr> impl_; -}; - -template -Ggnn::Ggnn(Metric metric, int dim, const BuildParam& param) : ANN(metric, dim) -{ - // ggnn/src/sift1m.cu - if (metric == Metric::kEuclidean && dim == 128 && param.k_build == 24 && param.k == 10 && - param.segment_size == 32) { - impl_ = std::make_shared>(metric, dim, param); - } - // ggnn/src/deep1b_multi_gpu.cu, and adapt it deep1B - else if (metric == Metric::kEuclidean && dim == 96 && param.k_build == 24 && param.k == 10 && - param.segment_size == 32) { - impl_ = std::make_shared>(metric, dim, param); - } else if (metric == Metric::kInnerProduct && dim == 96 && param.k_build == 24 && param.k == 10 && - param.segment_size == 32) { - impl_ = std::make_shared>(metric, dim, param); - } else if (metric == Metric::kInnerProduct && dim == 96 && param.k_build == 96 && param.k == 10 && - param.segment_size == 64) { - impl_ = std::make_shared>(metric, dim, param); - } - // ggnn/src/glove200.cu, adapt it to glove100 - else if (metric == Metric::kInnerProduct && dim == 100 && param.k_build == 96 && param.k == 10 && - param.segment_size == 64) { - impl_ = std::make_shared>(metric, dim, param); - } else { - throw std::runtime_error( - "ggnn: not supported combination of metric, dim and build param; " - "see Ggnn's constructor in ggnn_wrapper.cuh for available combinations"); - } -} - -template -class GgnnImpl : public ANN, public AnnGPU { - public: - using typename ANN::AnnSearchParam; - - GgnnImpl(Metric metric, int dim, const typename Ggnn::BuildParam& param); - - void build(const T* dataset, size_t nrow) override; - - void set_search_param(const AnnSearchParam& param) override; - void search(const T* queries, - int batch_size, - int k, - AnnBase::index_type* neighbors, - float* distances) const override; - [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override { return stream_; } - - void save(const std::string& file) const override; - void load(const std::string& file) override; - std::unique_ptr> copy() override - { - auto r = std::make_unique>(*this); - // set the thread-local stream to the copied handle. - r->stream_ = raft::bench::ann::get_stream_from_global_pool(); - return r; - }; - - AlgoProperty get_preference() const override - { - AlgoProperty property; - property.dataset_memory_type = MemoryType::Device; - property.query_memory_type = MemoryType::Device; - return property; - } - - void set_search_dataset(const T* dataset, size_t nrow) override; - - private: - using ANN::metric_; - using ANN::dim_; - - using GGNNGPUInstance = GGNNGPUInstance; - std::shared_ptr ggnn_; - typename Ggnn::BuildParam build_param_; - typename Ggnn::SearchParam search_param_; - cudaStream_t stream_; - const T* base_dataset = nullptr; - size_t base_n_rows = 0; - std::optional graph_file = std::nullopt; - - void load_impl() - { - if (base_dataset == nullptr) { return; } - if (base_n_rows == 0) { return; } - int device; - RAFT_CUDA_TRY(cudaGetDevice(&device)); - ggnn_ = std::make_shared( - device, base_n_rows, build_param_.num_layers, true, build_param_.tau); - ggnn_->set_base_data(base_dataset); - ggnn_->set_stream(get_sync_stream()); - if (graph_file.has_value()) { - auto& ggnn_host = ggnn_->ggnn_cpu_buffers.at(0); - auto& ggnn_device = ggnn_->ggnn_shards.at(0); - ggnn_->set_stream(get_sync_stream()); - - ggnn_host.load(graph_file.value()); - ggnn_host.uploadAsync(ggnn_device); - RAFT_CUDA_TRY(cudaStreamSynchronize(ggnn_device.stream)); - } - } -}; - -template -GgnnImpl::GgnnImpl(Metric metric, - int dim, - const typename Ggnn::BuildParam& param) - : ANN(metric, dim), - build_param_(param), - stream_(raft::bench::ann::get_stream_from_global_pool()) -{ - if (metric_ == Metric::kInnerProduct) { - if (measure != Cosine) { throw std::runtime_error("mis-matched metric"); } - } else if (metric_ == Metric::kEuclidean) { - if (measure != Euclidean) { throw std::runtime_error("mis-matched metric"); } - } else { - throw std::runtime_error( - "ggnn supports only metric type of InnerProduct, Cosine and Euclidean"); - } - - if (dim != D) { throw std::runtime_error("mis-matched dim"); } -} - -template -void GgnnImpl::build(const T* dataset, size_t nrow) -{ - base_dataset = dataset; - base_n_rows = nrow; - graph_file = std::nullopt; - load_impl(); - ggnn_->build(0); - for (int i = 0; i < build_param_.refine_iterations; ++i) { - ggnn_->refine(); - } -} - -template -void GgnnImpl::set_search_dataset(const T* dataset, size_t nrow) -{ - if (base_dataset != dataset || base_n_rows != nrow) { - base_dataset = dataset; - base_n_rows = nrow; - load_impl(); - } -} - -template -void GgnnImpl::set_search_param(const AnnSearchParam& param) -{ - search_param_ = dynamic_cast::SearchParam&>(param); -} - -template -void GgnnImpl::search( - const T* queries, int batch_size, int k, AnnBase::index_type* neighbors, float* distances) const -{ - static_assert(sizeof(size_t) == sizeof(int64_t), "sizes of size_t and GGNN's KeyT are different"); - if (k != KQuery) { - throw std::runtime_error( - "k = " + std::to_string(k) + - ", but this GGNN instance only supports k = " + std::to_string(KQuery)); - } - - ggnn_->set_stream(get_sync_stream()); - RAFT_CUDA_TRY(cudaMemcpyToSymbol(c_tau_query, &search_param_.tau, sizeof(float))); - - const int block_dim = search_param_.block_dim; - const int max_iterations = search_param_.max_iterations; - const int cache_size = search_param_.cache_size; - const int sorted_size = search_param_.sorted_size; - // default value - if (block_dim == 32 && max_iterations == 400 && cache_size == 512 && sorted_size == 256) { - ggnn_->template queryLayer<32, 400, 512, 256, false>( - queries, batch_size, reinterpret_cast(neighbors), distances); - } - // ggnn/src/sift1m.cu - else if (block_dim == 32 && max_iterations == 200 && cache_size == 256 && sorted_size == 64) { - ggnn_->template queryLayer<32, 200, 256, 64, false>( - queries, batch_size, reinterpret_cast(neighbors), distances); - } - // ggnn/src/sift1m.cu - else if (block_dim == 32 && max_iterations == 400 && cache_size == 448 && sorted_size == 64) { - ggnn_->template queryLayer<32, 400, 448, 64, false>( - queries, batch_size, reinterpret_cast(neighbors), distances); - } - // ggnn/src/glove200.cu - else if (block_dim == 128 && max_iterations == 2000 && cache_size == 2048 && sorted_size == 32) { - ggnn_->template queryLayer<128, 2000, 2048, 32, false>( - queries, batch_size, reinterpret_cast(neighbors), distances); - } - // for glove100 - else if (block_dim == 64 && max_iterations == 400 && cache_size == 512 && sorted_size == 32) { - ggnn_->template queryLayer<64, 400, 512, 32, false>( - queries, batch_size, reinterpret_cast(neighbors), distances); - } else if (block_dim == 128 && max_iterations == 2000 && cache_size == 1024 && - sorted_size == 32) { - ggnn_->template queryLayer<128, 2000, 1024, 32, false>( - queries, batch_size, reinterpret_cast(neighbors), distances); - } else { - throw std::runtime_error("ggnn: not supported search param"); - } -} - -template -void GgnnImpl::save(const std::string& file) const -{ - auto& ggnn_host = ggnn_->ggnn_cpu_buffers.at(0); - auto& ggnn_device = ggnn_->ggnn_shards.at(0); - ggnn_->set_stream(get_sync_stream()); - - ggnn_host.downloadAsync(ggnn_device); - RAFT_CUDA_TRY(cudaStreamSynchronize(ggnn_device.stream)); - ggnn_host.store(file); -} - -template -void GgnnImpl::load(const std::string& file) -{ - if (!graph_file.has_value() || graph_file.value() != file) { - graph_file = file; - load_impl(); - } -} - -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/hnswlib/hnswlib_benchmark.cpp b/cpp/bench/ann/src/hnswlib/hnswlib_benchmark.cpp deleted file mode 100644 index df82c68830..0000000000 --- a/cpp/bench/ann/src/hnswlib/hnswlib_benchmark.cpp +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../common/ann_types.hpp" -#include "hnswlib_wrapper.h" - -#define JSON_DIAGNOSTICS 1 -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace raft::bench::ann { - -template -void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::HnswLib::BuildParam& param) -{ - param.ef_construction = conf.at("efConstruction"); - param.M = conf.at("M"); - if (conf.contains("numThreads")) { param.num_threads = conf.at("numThreads"); } -} - -template -void parse_search_param(const nlohmann::json& conf, - typename raft::bench::ann::HnswLib::SearchParam& param) -{ - param.ef = conf.at("ef"); - if (conf.contains("numThreads")) { param.num_threads = conf.at("numThreads"); } -} - -template class Algo> -std::unique_ptr> make_algo(raft::bench::ann::Metric metric, - int dim, - const nlohmann::json& conf) -{ - typename Algo::BuildParam param; - parse_build_param(conf, param); - return std::make_unique>(metric, dim, param); -} - -template class Algo> -std::unique_ptr> make_algo(raft::bench::ann::Metric metric, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) -{ - typename Algo::BuildParam param; - parse_build_param(conf, param); - - (void)dev_list; - return std::make_unique>(metric, dim, param); -} - -template -std::unique_ptr> create_algo(const std::string& algo, - const std::string& distance, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) -{ - // stop compiler warning; not all algorithms support multi-GPU so it may not be used - (void)dev_list; - - raft::bench::ann::Metric metric = parse_metric(distance); - std::unique_ptr> ann; - - if constexpr (std::is_same_v) { - if (algo == "hnswlib") { ann = make_algo(metric, dim, conf); } - } - - if constexpr (std::is_same_v) { - if (algo == "hnswlib") { ann = make_algo(metric, dim, conf); } - } - - if (!ann) { throw std::runtime_error("invalid algo: '" + algo + "'"); } - return ann; -} - -template -std::unique_ptr::AnnSearchParam> create_search_param( - const std::string& algo, const nlohmann::json& conf) -{ - if (algo == "hnswlib") { - auto param = std::make_unique::SearchParam>(); - parse_search_param(conf, *param); - return param; - } - // else - throw std::runtime_error("invalid algo: '" + algo + "'"); -} - -}; // namespace raft::bench::ann - -REGISTER_ALGO_INSTANCE(float); -REGISTER_ALGO_INSTANCE(std::int8_t); -REGISTER_ALGO_INSTANCE(std::uint8_t); - -#ifdef ANN_BENCH_BUILD_MAIN -#include "../common/benchmark.hpp" -int main(int argc, char** argv) { return raft::bench::ann::run_main(argc, argv); } -#endif diff --git a/cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h b/cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h deleted file mode 100644 index 5743632bf4..0000000000 --- a/cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "../common/ann_types.hpp" -#include "../common/thread_pool.hpp" - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace raft::bench::ann { - -template -struct hnsw_dist_t { - using type = void; -}; - -template <> -struct hnsw_dist_t { - using type = float; -}; - -template <> -struct hnsw_dist_t { - using type = int; -}; - -template <> -struct hnsw_dist_t { - using type = int; -}; - -template -class HnswLib : public ANN { - public: - // https://github.com/nmslib/hnswlib/blob/master/ALGO_PARAMS.md - struct BuildParam { - int M; - int ef_construction; - int num_threads = omp_get_num_procs(); - }; - - using typename ANN::AnnSearchParam; - struct SearchParam : public AnnSearchParam { - int ef; - int num_threads = 1; - }; - - HnswLib(Metric metric, int dim, const BuildParam& param); - - void build(const T* dataset, size_t nrow) override; - - void set_search_param(const AnnSearchParam& param) override; - void search(const T* query, - int batch_size, - int k, - AnnBase::index_type* indices, - float* distances) const override; - - void save(const std::string& path_to_index) const override; - void load(const std::string& path_to_index) override; - std::unique_ptr> copy() override { return std::make_unique>(*this); }; - - AlgoProperty get_preference() const override - { - AlgoProperty property; - property.dataset_memory_type = MemoryType::Host; - property.query_memory_type = MemoryType::Host; - return property; - } - - void set_base_layer_only() { appr_alg_->base_layer_only = true; } - - private: - void get_search_knn_results_(const T* query, - int k, - AnnBase::index_type* indices, - float* distances) const; - - std::shared_ptr::type>> appr_alg_; - std::shared_ptr::type>> space_; - - using ANN::metric_; - using ANN::dim_; - int ef_construction_; - int m_; - int num_threads_; - std::shared_ptr thread_pool_; - Objective metric_objective_; -}; - -template -HnswLib::HnswLib(Metric metric, int dim, const BuildParam& param) : ANN(metric, dim) -{ - assert(dim_ > 0); - static_assert(std::is_same_v || std::is_same_v); - if constexpr (std::is_same_v) { - if (metric_ != Metric::kEuclidean) { - throw std::runtime_error("hnswlib only supports Euclidean distance"); - } - } - - ef_construction_ = param.ef_construction; - m_ = param.M; - num_threads_ = param.num_threads; -} - -template -void HnswLib::build(const T* dataset, size_t nrow) -{ - if constexpr (std::is_same_v) { - if (metric_ == Metric::kInnerProduct) { - space_ = std::make_shared(dim_); - } else { - space_ = std::make_shared(dim_); - } - } else if constexpr (std::is_same_v) { - space_ = std::make_shared>(dim_); - } - - appr_alg_ = std::make_shared::type>>( - space_.get(), nrow, m_, ef_construction_); - - thread_pool_ = std::make_shared(num_threads_); - const size_t items_per_thread = nrow / (num_threads_ + 1); - - thread_pool_->submit( - [&](size_t i) { - if (i < items_per_thread && i % 10000 == 0) { - char buf[20]; - std::time_t now = std::time(nullptr); - std::strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", std::localtime(&now)); - printf("%s building %zu / %zu\n", buf, i, items_per_thread); - fflush(stdout); - } - - appr_alg_->addPoint(dataset + i * dim_, i); - }, - nrow); -} - -template -void HnswLib::set_search_param(const AnnSearchParam& param_) -{ - auto param = dynamic_cast(param_); - appr_alg_->ef_ = param.ef; - metric_objective_ = param.metric_objective; - num_threads_ = param.num_threads; - - // Create a pool if multiple query threads have been set and the pool hasn't been created already - bool create_pool = (metric_objective_ == Objective::LATENCY && num_threads_ > 1 && !thread_pool_); - if (create_pool) { thread_pool_ = std::make_shared(num_threads_); } -} - -template -void HnswLib::search( - const T* query, int batch_size, int k, AnnBase::index_type* indices, float* distances) const -{ - auto f = [&](int i) { - // hnsw can only handle a single vector at a time. - get_search_knn_results_(query + i * dim_, k, indices + i * k, distances + i * k); - }; - if (metric_objective_ == Objective::LATENCY && num_threads_ > 1) { - thread_pool_->submit(f, batch_size); - } else { - for (int i = 0; i < batch_size; i++) { - f(i); - } - } -} - -template -void HnswLib::save(const std::string& path_to_index) const -{ - appr_alg_->saveIndex(std::string(path_to_index)); -} - -template -void HnswLib::load(const std::string& path_to_index) -{ - if constexpr (std::is_same_v) { - if (metric_ == Metric::kInnerProduct) { - space_ = std::make_shared(dim_); - } else { - space_ = std::make_shared(dim_); - } - } else if constexpr (std::is_same_v) { - space_ = std::make_shared>(dim_); - } - - appr_alg_ = std::make_shared::type>>( - space_.get(), path_to_index); -} - -template -void HnswLib::get_search_knn_results_(const T* query, - int k, - AnnBase::index_type* indices, - float* distances) const -{ - auto result = appr_alg_->searchKnn(query, k); - assert(result.size() >= static_cast(k)); - - for (int i = k - 1; i >= 0; --i) { - indices[i] = result.top().second; - distances[i] = result.top().first; - result.pop(); - } -} - -}; // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/raft/raft_ann_bench_param_parser.h b/cpp/bench/ann/src/raft/raft_ann_bench_param_parser.h deleted file mode 100644 index 48bf1d70d8..0000000000 --- a/cpp/bench/ann/src/raft/raft_ann_bench_param_parser.h +++ /dev/null @@ -1,275 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#define JSON_DIAGNOSTICS 1 -#include - -#undef WARP_SIZE -#ifdef RAFT_ANN_BENCH_USE_RAFT_BRUTE_FORCE -#include "raft_wrapper.h" -#endif -#ifdef RAFT_ANN_BENCH_USE_RAFT_IVF_FLAT -#include "raft_ivf_flat_wrapper.h" -extern template class raft::bench::ann::RaftIvfFlatGpu; -extern template class raft::bench::ann::RaftIvfFlatGpu; -extern template class raft::bench::ann::RaftIvfFlatGpu; -#endif -#if defined(RAFT_ANN_BENCH_USE_RAFT_IVF_PQ) || defined(RAFT_ANN_BENCH_USE_RAFT_CAGRA) || \ - defined(RAFT_ANN_BENCH_USE_RAFT_CAGRA_HNSWLIB) -#include "raft_ivf_pq_wrapper.h" -#endif -#ifdef RAFT_ANN_BENCH_USE_RAFT_IVF_PQ -extern template class raft::bench::ann::RaftIvfPQ; -extern template class raft::bench::ann::RaftIvfPQ; -extern template class raft::bench::ann::RaftIvfPQ; -#endif -#if defined(RAFT_ANN_BENCH_USE_RAFT_CAGRA) || defined(RAFT_ANN_BENCH_USE_RAFT_CAGRA_HNSWLIB) -#include "raft_cagra_wrapper.h" -#endif -#ifdef RAFT_ANN_BENCH_USE_RAFT_CAGRA -extern template class raft::bench::ann::RaftCagra; -extern template class raft::bench::ann::RaftCagra; -extern template class raft::bench::ann::RaftCagra; -extern template class raft::bench::ann::RaftCagra; -#endif - -#ifdef RAFT_ANN_BENCH_USE_RAFT_IVF_FLAT -template -void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::RaftIvfFlatGpu::BuildParam& param) -{ - param.n_lists = conf.at("nlist"); - if (conf.contains("niter")) { param.kmeans_n_iters = conf.at("niter"); } - if (conf.contains("ratio")) { param.kmeans_trainset_fraction = 1.0 / (double)conf.at("ratio"); } -} - -template -void parse_search_param(const nlohmann::json& conf, - typename raft::bench::ann::RaftIvfFlatGpu::SearchParam& param) -{ - param.ivf_flat_params.n_probes = conf.at("nprobe"); -} -#endif - -#if defined(RAFT_ANN_BENCH_USE_RAFT_IVF_PQ) || defined(RAFT_ANN_BENCH_USE_RAFT_CAGRA) || \ - defined(RAFT_ANN_BENCH_USE_RAFT_CAGRA_HNSWLIB) -template -void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::RaftIvfPQ::BuildParam& param) -{ - if (conf.contains("nlist")) { param.n_lists = conf.at("nlist"); } - if (conf.contains("niter")) { param.kmeans_n_iters = conf.at("niter"); } - if (conf.contains("ratio")) { param.kmeans_trainset_fraction = 1.0 / (double)conf.at("ratio"); } - if (conf.contains("pq_bits")) { param.pq_bits = conf.at("pq_bits"); } - if (conf.contains("pq_dim")) { param.pq_dim = conf.at("pq_dim"); } - if (conf.contains("codebook_kind")) { - std::string kind = conf.at("codebook_kind"); - if (kind == "cluster") { - param.codebook_kind = raft::neighbors::ivf_pq::codebook_gen::PER_CLUSTER; - } else if (kind == "subspace") { - param.codebook_kind = raft::neighbors::ivf_pq::codebook_gen::PER_SUBSPACE; - } else { - throw std::runtime_error("codebook_kind: '" + kind + - "', should be either 'cluster' or 'subspace'"); - } - } -} - -template -void parse_search_param(const nlohmann::json& conf, - typename raft::bench::ann::RaftIvfPQ::SearchParam& param) -{ - if (conf.contains("nprobe")) { param.pq_param.n_probes = conf.at("nprobe"); } - if (conf.contains("internalDistanceDtype")) { - std::string type = conf.at("internalDistanceDtype"); - if (type == "float") { - param.pq_param.internal_distance_dtype = CUDA_R_32F; - } else if (type == "half") { - param.pq_param.internal_distance_dtype = CUDA_R_16F; - } else { - throw std::runtime_error("internalDistanceDtype: '" + type + - "', should be either 'float' or 'half'"); - } - } else { - // set half as default type - param.pq_param.internal_distance_dtype = CUDA_R_16F; - } - - if (conf.contains("smemLutDtype")) { - std::string type = conf.at("smemLutDtype"); - if (type == "float") { - param.pq_param.lut_dtype = CUDA_R_32F; - } else if (type == "half") { - param.pq_param.lut_dtype = CUDA_R_16F; - } else if (type == "fp8") { - param.pq_param.lut_dtype = CUDA_R_8U; - } else { - throw std::runtime_error("smemLutDtype: '" + type + - "', should be either 'float', 'half' or 'fp8'"); - } - } else { - // set half as default - param.pq_param.lut_dtype = CUDA_R_16F; - } - if (conf.contains("refine_ratio")) { - param.refine_ratio = conf.at("refine_ratio"); - if (param.refine_ratio < 1.0f) { throw std::runtime_error("refine_ratio should be >= 1.0"); } - } -} -#endif - -#if defined(RAFT_ANN_BENCH_USE_RAFT_CAGRA) || defined(RAFT_ANN_BENCH_USE_RAFT_CAGRA_HNSWLIB) -template -void parse_build_param(const nlohmann::json& conf, - raft::neighbors::experimental::nn_descent::index_params& param) -{ - if (conf.contains("graph_degree")) { param.graph_degree = conf.at("graph_degree"); } - if (conf.contains("intermediate_graph_degree")) { - param.intermediate_graph_degree = conf.at("intermediate_graph_degree"); - } - // we allow niter shorthand for max_iterations - if (conf.contains("niter")) { param.max_iterations = conf.at("niter"); } - if (conf.contains("max_iterations")) { param.max_iterations = conf.at("max_iterations"); } - if (conf.contains("termination_threshold")) { - param.termination_threshold = conf.at("termination_threshold"); - } -} - -inline void parse_build_param(const nlohmann::json& conf, raft::neighbors::vpq_params& param) -{ - if (conf.contains("pq_bits")) { param.pq_bits = conf.at("pq_bits"); } - if (conf.contains("pq_dim")) { param.pq_dim = conf.at("pq_dim"); } - if (conf.contains("vq_n_centers")) { param.vq_n_centers = conf.at("vq_n_centers"); } - if (conf.contains("kmeans_n_iters")) { param.kmeans_n_iters = conf.at("kmeans_n_iters"); } - if (conf.contains("vq_kmeans_trainset_fraction")) { - param.vq_kmeans_trainset_fraction = conf.at("vq_kmeans_trainset_fraction"); - } - if (conf.contains("pq_kmeans_trainset_fraction")) { - param.pq_kmeans_trainset_fraction = conf.at("pq_kmeans_trainset_fraction"); - } -} - -nlohmann::json collect_conf_with_prefix(const nlohmann::json& conf, - const std::string& prefix, - bool remove_prefix = true) -{ - nlohmann::json out; - for (auto& i : conf.items()) { - if (i.key().compare(0, prefix.size(), prefix) == 0) { - auto new_key = remove_prefix ? i.key().substr(prefix.size()) : i.key(); - out[new_key] = i.value(); - } - } - return out; -} - -template -void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::RaftCagra::BuildParam& param) -{ - if (conf.contains("graph_degree")) { - param.cagra_params.graph_degree = conf.at("graph_degree"); - param.cagra_params.intermediate_graph_degree = param.cagra_params.graph_degree * 2; - } - if (conf.contains("intermediate_graph_degree")) { - param.cagra_params.intermediate_graph_degree = conf.at("intermediate_graph_degree"); - } - if (conf.contains("graph_build_algo")) { - if (conf.at("graph_build_algo") == "IVF_PQ") { - param.cagra_params.build_algo = raft::neighbors::cagra::graph_build_algo::IVF_PQ; - } else if (conf.at("graph_build_algo") == "NN_DESCENT") { - param.cagra_params.build_algo = raft::neighbors::cagra::graph_build_algo::NN_DESCENT; - } - } - nlohmann::json ivf_pq_build_conf = collect_conf_with_prefix(conf, "ivf_pq_build_"); - if (!ivf_pq_build_conf.empty()) { - raft::neighbors::ivf_pq::index_params bparam; - parse_build_param(ivf_pq_build_conf, bparam); - param.ivf_pq_build_params = bparam; - } - nlohmann::json ivf_pq_search_conf = collect_conf_with_prefix(conf, "ivf_pq_search_"); - if (!ivf_pq_search_conf.empty()) { - typename raft::bench::ann::RaftIvfPQ::SearchParam sparam; - parse_search_param(ivf_pq_search_conf, sparam); - param.ivf_pq_search_params = sparam.pq_param; - param.ivf_pq_refine_rate = sparam.refine_ratio; - } - nlohmann::json nn_descent_conf = collect_conf_with_prefix(conf, "nn_descent_"); - if (!nn_descent_conf.empty()) { - raft::neighbors::experimental::nn_descent::index_params nn_param; - nn_param.intermediate_graph_degree = 1.5 * param.cagra_params.intermediate_graph_degree; - parse_build_param(nn_descent_conf, nn_param); - if (nn_param.graph_degree != param.cagra_params.intermediate_graph_degree) { - nn_param.graph_degree = param.cagra_params.intermediate_graph_degree; - } - param.nn_descent_params = nn_param; - } - nlohmann::json comp_search_conf = collect_conf_with_prefix(conf, "compression_"); - if (!comp_search_conf.empty()) { - raft::neighbors::vpq_params vpq_pams; - parse_build_param(comp_search_conf, vpq_pams); - param.cagra_params.compression.emplace(vpq_pams); - } -} - -raft::bench::ann::AllocatorType parse_allocator(std::string mem_type) -{ - if (mem_type == "device") { - return raft::bench::ann::AllocatorType::Device; - } else if (mem_type == "host_pinned") { - return raft::bench::ann::AllocatorType::HostPinned; - } else if (mem_type == "host_huge_page") { - return raft::bench::ann::AllocatorType::HostHugePage; - } - THROW( - "Invalid value for memory type %s, must be one of [\"device\", \"host_pinned\", " - "\"host_huge_page\"", - mem_type.c_str()); -} - -template -void parse_search_param(const nlohmann::json& conf, - typename raft::bench::ann::RaftCagra::SearchParam& param) -{ - if (conf.contains("itopk")) { param.p.itopk_size = conf.at("itopk"); } - if (conf.contains("search_width")) { param.p.search_width = conf.at("search_width"); } - if (conf.contains("max_iterations")) { param.p.max_iterations = conf.at("max_iterations"); } - if (conf.contains("algo")) { - if (conf.at("algo") == "single_cta") { - param.p.algo = raft::neighbors::experimental::cagra::search_algo::SINGLE_CTA; - } else if (conf.at("algo") == "multi_cta") { - param.p.algo = raft::neighbors::experimental::cagra::search_algo::MULTI_CTA; - } else if (conf.at("algo") == "multi_kernel") { - param.p.algo = raft::neighbors::experimental::cagra::search_algo::MULTI_KERNEL; - } else if (conf.at("algo") == "auto") { - param.p.algo = raft::neighbors::experimental::cagra::search_algo::AUTO; - } else { - std::string tmp = conf.at("algo"); - THROW("Invalid value for algo: %s", tmp.c_str()); - } - } - if (conf.contains("graph_memory_type")) { - param.graph_mem = parse_allocator(conf.at("graph_memory_type")); - } - if (conf.contains("internal_dataset_memory_type")) { - param.dataset_mem = parse_allocator(conf.at("internal_dataset_memory_type")); - } - // Same ratio as in IVF-PQ - param.refine_ratio = conf.value("refine_ratio", 1.0f); -} -#endif diff --git a/cpp/bench/ann/src/raft/raft_ann_bench_utils.h b/cpp/bench/ann/src/raft/raft_ann_bench_utils.h deleted file mode 100644 index 9b086fdb23..0000000000 --- a/cpp/bench/ann/src/raft/raft_ann_bench_utils.h +++ /dev/null @@ -1,255 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "../common/util.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace raft::bench::ann { - -inline raft::distance::DistanceType parse_metric_type(raft::bench::ann::Metric metric) -{ - if (metric == raft::bench::ann::Metric::kInnerProduct) { - return raft::distance::DistanceType::InnerProduct; - } else if (metric == raft::bench::ann::Metric::kEuclidean) { - // Even for L2 expanded RAFT IVF Flat uses unexpanded formula - return raft::distance::DistanceType::L2Expanded; - } else { - throw std::runtime_error("raft supports only metric type of inner product and L2"); - } -} - -/** Report a more verbose error with a backtrace when OOM occurs on RMM side. */ -inline auto rmm_oom_callback(std::size_t bytes, void*) -> bool -{ - auto cuda_status = cudaGetLastError(); - size_t free = 0; - size_t total = 0; - RAFT_CUDA_TRY_NO_THROW(cudaMemGetInfo(&free, &total)); - RAFT_FAIL( - "Failed to allocate %zu bytes using RMM memory resource. " - "NB: latest cuda status = %s, free memory = %zu, total memory = %zu.", - bytes, - cudaGetErrorName(cuda_status), - free, - total); -} - -/** - * This container keeps the part of raft state that should be shared among multiple copies of raft - * handles (in different CPU threads). - * An example of this is an RMM memory resource: if we had an RMM memory pool per thread, we'd - * quickly run out of memory. - */ -class shared_raft_resources { - public: - using pool_mr_type = rmm::mr::pool_memory_resource; - using mr_type = rmm::mr::failure_callback_resource_adaptor; - using large_mr_type = rmm::mr::managed_memory_resource; - - shared_raft_resources() - try : orig_resource_{rmm::mr::get_current_device_resource()}, - pool_resource_(orig_resource_, 1024 * 1024 * 1024ull), - resource_(&pool_resource_, rmm_oom_callback, nullptr), large_mr_() { - rmm::mr::set_current_device_resource(&resource_); - } catch (const std::exception& e) { - auto cuda_status = cudaGetLastError(); - size_t free = 0; - size_t total = 0; - RAFT_CUDA_TRY_NO_THROW(cudaMemGetInfo(&free, &total)); - RAFT_FAIL( - "Failed to initialize shared raft resources (NB: latest cuda status = %s, free memory = %zu, " - "total memory = %zu): %s", - cudaGetErrorName(cuda_status), - free, - total, - e.what()); - } - - shared_raft_resources(shared_raft_resources&&) = delete; - shared_raft_resources& operator=(shared_raft_resources&&) = delete; - shared_raft_resources(const shared_raft_resources& res) = delete; - shared_raft_resources& operator=(const shared_raft_resources& other) = delete; - - ~shared_raft_resources() noexcept { rmm::mr::set_current_device_resource(orig_resource_); } - - auto get_large_memory_resource() noexcept - { - return static_cast(&large_mr_); - } - - private: - rmm::mr::device_memory_resource* orig_resource_; - pool_mr_type pool_resource_; - mr_type resource_; - large_mr_type large_mr_; -}; - -/** - * This struct is used by multiple raft benchmark wrappers. It serves as a thread-safe keeper of - * shared and private GPU resources (see below). - * - * - Accessing the same `configured_raft_resources` from concurrent threads is not safe. - * - Accessing the copies of `configured_raft_resources` from concurrent threads is safe. - * - There must be at most one "original" `configured_raft_resources` at any time, but as many - * copies of it as needed (modifies the program static state). - */ -class configured_raft_resources { - public: - /** - * This constructor has the shared state passed unmodified but creates the local state anew. - * It's used by the copy constructor. - */ - explicit configured_raft_resources(const std::shared_ptr& shared_res) - : shared_res_{shared_res}, - res_{std::make_unique( - rmm::cuda_stream_view(get_stream_from_global_pool()))} - { - // set the large workspace resource to the raft handle, but without the deleter - // (this resource is managed by the shared_res). - raft::resource::set_large_workspace_resource( - *res_, - std::shared_ptr(shared_res_->get_large_memory_resource(), - raft::void_op{})); - } - - /** Default constructor creates all resources anew. */ - configured_raft_resources() : configured_raft_resources{std::make_shared()} - { - } - - configured_raft_resources(configured_raft_resources&&); - configured_raft_resources& operator=(configured_raft_resources&&); - ~configured_raft_resources() = default; - configured_raft_resources(const configured_raft_resources& res) - : configured_raft_resources{res.shared_res_} - { - } - configured_raft_resources& operator=(const configured_raft_resources& other) - { - this->shared_res_ = other.shared_res_; - return *this; - } - - operator raft::resources&() noexcept { return *res_; } - operator const raft::resources&() const noexcept { return *res_; } - - /** Get the main stream */ - [[nodiscard]] auto get_sync_stream() const noexcept { return resource::get_cuda_stream(*res_); } - - private: - /** The resources shared among multiple raft handles / threads. */ - std::shared_ptr shared_res_; - /** - * Until we make the use of copies of raft::resources thread-safe, each benchmark wrapper must - * have its own copy of it. - */ - std::unique_ptr res_ = std::make_unique(); -}; - -inline configured_raft_resources::configured_raft_resources(configured_raft_resources&&) = default; -inline configured_raft_resources& configured_raft_resources::operator=( - configured_raft_resources&&) = default; - -/** A helper to refine the neighbors when the data is on device or on host. */ -template -void refine_helper(const raft::resources& res, - DatasetT dataset, - QueriesT queries, - CandidatesT candidates, - int k, - AnnBase::index_type* neighbors, - float* distances, - raft::distance::DistanceType metric) -{ - using data_type = typename DatasetT::value_type; - using index_type = AnnBase::index_type; - using extents_type = index_type; // device-side refine requires this - - static_assert(std::is_same_v); - static_assert(std::is_same_v); - static_assert(std::is_same_v); - - extents_type batch_size = queries.extent(0); - extents_type dim = queries.extent(1); - extents_type k0 = candidates.extent(1); - - if (raft::get_device_for_address(dataset.data_handle()) >= 0) { - auto dataset_device = raft::make_device_matrix_view( - dataset.data_handle(), dataset.extent(0), dataset.extent(1)); - auto queries_device = raft::make_device_matrix_view( - queries.data_handle(), batch_size, dim); - auto candidates_device = raft::make_device_matrix_view( - candidates.data_handle(), batch_size, k0); - auto neighbors_device = - raft::make_device_matrix_view(neighbors, batch_size, k); - auto distances_device = - raft::make_device_matrix_view(distances, batch_size, k); - - raft::neighbors::refine(res, - dataset_device, - queries_device, - candidates_device, - neighbors_device, - distances_device, - metric); - } else { - auto dataset_host = raft::make_host_matrix_view( - dataset.data_handle(), dataset.extent(0), dataset.extent(1)); - auto queries_host = raft::make_host_matrix(batch_size, dim); - auto candidates_host = raft::make_host_matrix(batch_size, k0); - auto neighbors_host = raft::make_host_matrix(batch_size, k); - auto distances_host = raft::make_host_matrix(batch_size, k); - - auto stream = resource::get_cuda_stream(res); - raft::copy(queries_host.data_handle(), queries.data_handle(), queries_host.size(), stream); - raft::copy( - candidates_host.data_handle(), candidates.data_handle(), candidates_host.size(), stream); - - raft::resource::sync_stream(res); // wait for the queries and candidates - raft::neighbors::refine(res, - dataset_host, - queries_host.view(), - candidates_host.view(), - neighbors_host.view(), - distances_host.view(), - metric); - - raft::copy(neighbors, neighbors_host.data_handle(), neighbors_host.size(), stream); - raft::copy(distances, distances_host.data_handle(), distances_host.size(), stream); - } -} - -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/raft/raft_benchmark.cu b/cpp/bench/ann/src/raft/raft_benchmark.cu deleted file mode 100644 index 8bb4d9423c..0000000000 --- a/cpp/bench/ann/src/raft/raft_benchmark.cu +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../common/ann_types.hpp" -#include "raft_ann_bench_param_parser.h" - -#include - -#include - -#define JSON_DIAGNOSTICS 1 -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace raft::bench::ann { - -template -std::unique_ptr> create_algo(const std::string& algo, - const std::string& distance, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) -{ - // stop compiler warning; not all algorithms support multi-GPU so it may not be used - (void)dev_list; - - [[maybe_unused]] raft::bench::ann::Metric metric = parse_metric(distance); - std::unique_ptr> ann; - - if constexpr (std::is_same_v) { -#ifdef RAFT_ANN_BENCH_USE_RAFT_BRUTE_FORCE - if (algo == "raft_brute_force") { - ann = std::make_unique>(metric, dim); - } -#endif - } - - if constexpr (std::is_same_v) {} - -#ifdef RAFT_ANN_BENCH_USE_RAFT_IVF_FLAT - if constexpr (std::is_same_v || std::is_same_v || - std::is_same_v) { - if (algo == "raft_ivf_flat") { - typename raft::bench::ann::RaftIvfFlatGpu::BuildParam param; - parse_build_param(conf, param); - ann = std::make_unique>(metric, dim, param); - } - } -#endif -#ifdef RAFT_ANN_BENCH_USE_RAFT_IVF_PQ - if (algo == "raft_ivf_pq") { - typename raft::bench::ann::RaftIvfPQ::BuildParam param; - parse_build_param(conf, param); - ann = std::make_unique>(metric, dim, param); - } -#endif -#ifdef RAFT_ANN_BENCH_USE_RAFT_CAGRA - if (algo == "raft_cagra") { - typename raft::bench::ann::RaftCagra::BuildParam param; - parse_build_param(conf, param); - ann = std::make_unique>(metric, dim, param); - } -#endif - - if (!ann) { throw std::runtime_error("invalid algo: '" + algo + "'"); } - - return ann; -} - -template -std::unique_ptr::AnnSearchParam> create_search_param( - const std::string& algo, const nlohmann::json& conf) -{ -#ifdef RAFT_ANN_BENCH_USE_RAFT_BRUTE_FORCE - if (algo == "raft_brute_force") { - auto param = std::make_unique::AnnSearchParam>(); - return param; - } -#endif -#ifdef RAFT_ANN_BENCH_USE_RAFT_IVF_FLAT - if constexpr (std::is_same_v || std::is_same_v || - std::is_same_v) { - if (algo == "raft_ivf_flat") { - auto param = - std::make_unique::SearchParam>(); - parse_search_param(conf, *param); - return param; - } - } -#endif -#ifdef RAFT_ANN_BENCH_USE_RAFT_IVF_PQ - if (algo == "raft_ivf_pq") { - auto param = std::make_unique::SearchParam>(); - parse_search_param(conf, *param); - return param; - } -#endif -#ifdef RAFT_ANN_BENCH_USE_RAFT_CAGRA - if (algo == "raft_cagra") { - auto param = std::make_unique::SearchParam>(); - parse_search_param(conf, *param); - return param; - } -#endif - - // else - throw std::runtime_error("invalid algo: '" + algo + "'"); -} - -}; // namespace raft::bench::ann - -REGISTER_ALGO_INSTANCE(float); -REGISTER_ALGO_INSTANCE(half); -REGISTER_ALGO_INSTANCE(std::int8_t); -REGISTER_ALGO_INSTANCE(std::uint8_t); - -#ifdef ANN_BENCH_BUILD_MAIN -#include "../common/benchmark.hpp" -int main(int argc, char** argv) { return raft::bench::ann::run_main(argc, argv); } -#endif diff --git a/cpp/bench/ann/src/raft/raft_cagra_float.cu b/cpp/bench/ann/src/raft/raft_cagra_float.cu deleted file mode 100644 index 058f5bf34a..0000000000 --- a/cpp/bench/ann/src/raft/raft_cagra_float.cu +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "raft_cagra_wrapper.h" - -namespace raft::bench::ann { -template class RaftCagra; -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/raft/raft_cagra_half.cu b/cpp/bench/ann/src/raft/raft_cagra_half.cu deleted file mode 100644 index a015819ec5..0000000000 --- a/cpp/bench/ann/src/raft/raft_cagra_half.cu +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "raft_cagra_wrapper.h" - -namespace raft::bench::ann { -template class RaftCagra; -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/raft/raft_cagra_hnswlib.cu b/cpp/bench/ann/src/raft/raft_cagra_hnswlib.cu deleted file mode 100644 index d9ef1d74a3..0000000000 --- a/cpp/bench/ann/src/raft/raft_cagra_hnswlib.cu +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../common/ann_types.hpp" -#include "raft_ann_bench_param_parser.h" -#include "raft_cagra_hnswlib_wrapper.h" - -#include -#include -#include - -#define JSON_DIAGNOSTICS 1 -#include - -namespace raft::bench::ann { - -template -void parse_search_param(const nlohmann::json& conf, - typename raft::bench::ann::RaftCagraHnswlib::SearchParam& param) -{ - param.ef = conf.at("ef"); - if (conf.contains("numThreads")) { param.num_threads = conf.at("numThreads"); } -} - -template -std::unique_ptr> create_algo(const std::string& algo, - const std::string& distance, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) -{ - // stop compiler warning; not all algorithms support multi-GPU so it may not be used - (void)dev_list; - - [[maybe_unused]] raft::bench::ann::Metric metric = parse_metric(distance); - std::unique_ptr> ann; - - if constexpr (std::is_same_v or std::is_same_v) { - if (algo == "raft_cagra_hnswlib") { - typename raft::bench::ann::RaftCagraHnswlib::BuildParam param; - parse_build_param(conf, param); - ann = std::make_unique>(metric, dim, param); - } - } - - if (!ann) { throw std::runtime_error("invalid algo: '" + algo + "'"); } - - return ann; -} - -template -std::unique_ptr::AnnSearchParam> create_search_param( - const std::string& algo, const nlohmann::json& conf) -{ - if (algo == "raft_cagra_hnswlib") { - auto param = - std::make_unique::SearchParam>(); - parse_search_param(conf, *param); - return param; - } - - throw std::runtime_error("invalid algo: '" + algo + "'"); -} - -} // namespace raft::bench::ann - -REGISTER_ALGO_INSTANCE(float); -REGISTER_ALGO_INSTANCE(std::int8_t); -REGISTER_ALGO_INSTANCE(std::uint8_t); - -#ifdef ANN_BENCH_BUILD_MAIN -#include "../common/benchmark.hpp" -int main(int argc, char** argv) -{ - rmm::mr::cuda_memory_resource cuda_mr; - // Construct a resource that uses a coalescing best-fit pool allocator - // and is initially sized to half of free device memory. - rmm::mr::pool_memory_resource pool_mr{ - &cuda_mr, rmm::percent_of_free_device_memory(50)}; - // Updates the current device resource pointer to `pool_mr` - auto old_mr = rmm::mr::set_current_device_resource(&pool_mr); - auto ret = raft::bench::ann::run_main(argc, argv); - // Restores the current device resource pointer to its previous value - rmm::mr::set_current_device_resource(old_mr); - return ret; -} -#endif diff --git a/cpp/bench/ann/src/raft/raft_cagra_hnswlib_wrapper.h b/cpp/bench/ann/src/raft/raft_cagra_hnswlib_wrapper.h deleted file mode 100644 index 1d2a1076ab..0000000000 --- a/cpp/bench/ann/src/raft/raft_cagra_hnswlib_wrapper.h +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "../hnswlib/hnswlib_wrapper.h" -#include "raft_cagra_wrapper.h" - -#include - -namespace raft::bench::ann { - -template -class RaftCagraHnswlib : public ANN, public AnnGPU { - public: - using typename ANN::AnnSearchParam; - using BuildParam = typename RaftCagra::BuildParam; - using SearchParam = typename HnswLib::SearchParam; - - RaftCagraHnswlib(Metric metric, int dim, const BuildParam& param, int concurrent_searches = 1) - : ANN(metric, dim), - cagra_build_{metric, dim, param, concurrent_searches, true}, - // HnswLib param values don't matter since we don't build with HnswLib - hnswlib_search_{metric, dim, typename HnswLib::BuildParam{50, 100}} - { - } - - void build(const T* dataset, size_t nrow) final; - - void set_search_param(const AnnSearchParam& param) override; - - void search(const T* queries, - int batch_size, - int k, - AnnBase::index_type* neighbors, - float* distances) const override; - - [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override - { - return cagra_build_.get_sync_stream(); - } - - // to enable dataset access from GPU memory - AlgoProperty get_preference() const override - { - AlgoProperty property; - property.dataset_memory_type = MemoryType::HostMmap; - property.query_memory_type = MemoryType::Host; - return property; - } - - void save(const std::string& file) const override; - void load(const std::string&) override; - std::unique_ptr> copy() override - { - return std::make_unique>(*this); - } - - private: - RaftCagra cagra_build_; - HnswLib hnswlib_search_; -}; - -template -void RaftCagraHnswlib::build(const T* dataset, size_t nrow) -{ - cagra_build_.build(dataset, nrow); -} - -template -void RaftCagraHnswlib::set_search_param(const AnnSearchParam& param_) -{ - hnswlib_search_.set_search_param(param_); -} - -template -void RaftCagraHnswlib::save(const std::string& file) const -{ - cagra_build_.save_to_hnswlib(file); -} - -template -void RaftCagraHnswlib::load(const std::string& file) -{ - hnswlib_search_.load(file); - hnswlib_search_.set_base_layer_only(); -} - -template -void RaftCagraHnswlib::search( - const T* queries, int batch_size, int k, AnnBase::index_type* neighbors, float* distances) const -{ - hnswlib_search_.search(queries, batch_size, k, neighbors, distances); -} - -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/raft/raft_cagra_int8_t.cu b/cpp/bench/ann/src/raft/raft_cagra_int8_t.cu deleted file mode 100644 index be3b83ee60..0000000000 --- a/cpp/bench/ann/src/raft/raft_cagra_int8_t.cu +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "raft_cagra_wrapper.h" - -namespace raft::bench::ann { -template class RaftCagra; -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/raft/raft_cagra_uint8_t.cu b/cpp/bench/ann/src/raft/raft_cagra_uint8_t.cu deleted file mode 100644 index c9679e404d..0000000000 --- a/cpp/bench/ann/src/raft/raft_cagra_uint8_t.cu +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "raft_cagra_wrapper.h" - -namespace raft::bench::ann { -template class RaftCagra; -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/raft/raft_cagra_wrapper.h b/cpp/bench/ann/src/raft/raft_cagra_wrapper.h deleted file mode 100644 index b03f875a8e..0000000000 --- a/cpp/bench/ann/src/raft/raft_cagra_wrapper.h +++ /dev/null @@ -1,339 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "../common/ann_types.hpp" -#include "../common/cuda_huge_page_resource.hpp" -#include "../common/cuda_pinned_resource.hpp" -#include "raft_ann_bench_utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace raft::bench::ann { - -enum class AllocatorType { HostPinned, HostHugePage, Device }; -template -class RaftCagra : public ANN, public AnnGPU { - public: - using typename ANN::AnnSearchParam; - - struct SearchParam : public AnnSearchParam { - raft::neighbors::experimental::cagra::search_params p; - float refine_ratio; - AllocatorType graph_mem = AllocatorType::Device; - AllocatorType dataset_mem = AllocatorType::Device; - auto needs_dataset() const -> bool override { return true; } - }; - - struct BuildParam { - raft::neighbors::cagra::index_params cagra_params; - std::optional nn_descent_params = - std::nullopt; - std::optional ivf_pq_refine_rate = std::nullopt; - std::optional ivf_pq_build_params = std::nullopt; - std::optional ivf_pq_search_params = std::nullopt; - }; - - RaftCagra(Metric metric, - int dim, - const BuildParam& param, - int concurrent_searches = 1, - bool shall_include_dataset = false) - : ANN(metric, dim), - index_params_(param), - dimension_(dim), - need_dataset_update_(true), - shall_include_dataset_(shall_include_dataset), - dataset_(std::make_shared>( - std::move(make_device_matrix(handle_, 0, 0)))), - graph_(std::make_shared>( - std::move(make_device_matrix(handle_, 0, 0)))), - input_dataset_v_( - std::make_shared>(nullptr, 0, 0)), - graph_mem_(AllocatorType::Device), - dataset_mem_(AllocatorType::Device) - { - index_params_.cagra_params.metric = parse_metric_type(metric); - index_params_.ivf_pq_build_params->metric = parse_metric_type(metric); - } - - void build(const T* dataset, size_t nrow) final; - - void set_search_param(const AnnSearchParam& param) override; - - void set_search_dataset(const T* dataset, size_t nrow) override; - - void search(const T* queries, - int batch_size, - int k, - AnnBase::index_type* neighbors, - float* distances) const override; - void search_base(const T* queries, - int batch_size, - int k, - AnnBase::index_type* neighbors, - float* distances) const; - - [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override - { - return handle_.get_sync_stream(); - } - - // to enable dataset access from GPU memory - AlgoProperty get_preference() const override - { - AlgoProperty property; - property.dataset_memory_type = MemoryType::HostMmap; - property.query_memory_type = MemoryType::Device; - return property; - } - void save(const std::string& file) const override; - void load(const std::string&) override; - void save_to_hnswlib(const std::string& file) const; - std::unique_ptr> copy() override; - - private: - // handle_ must go first to make sure it dies last and all memory allocated in pool - configured_raft_resources handle_{}; - raft::mr::cuda_pinned_resource mr_pinned_; - raft::mr::cuda_huge_page_resource mr_huge_page_; - AllocatorType graph_mem_; - AllocatorType dataset_mem_; - float refine_ratio_; - BuildParam index_params_; - bool need_dataset_update_; - bool shall_include_dataset_; - raft::neighbors::cagra::search_params search_params_; - std::shared_ptr> index_; - int dimension_; - std::shared_ptr> graph_; - std::shared_ptr> dataset_; - std::shared_ptr> input_dataset_v_; - - inline rmm::device_async_resource_ref get_mr(AllocatorType mem_type) - { - switch (mem_type) { - case (AllocatorType::HostPinned): return &mr_pinned_; - case (AllocatorType::HostHugePage): return &mr_huge_page_; - default: return rmm::mr::get_current_device_resource(); - } - } -}; - -template -void RaftCagra::build(const T* dataset, size_t nrow) -{ - auto dataset_view = - raft::make_host_matrix_view(dataset, IdxT(nrow), dimension_); - - auto& params = index_params_.cagra_params; - - // Do include the compressed dataset for the CAGRA-Q - bool include_dataset = params.compression.has_value() || shall_include_dataset_; - - index_ = std::make_shared>( - std::move(raft::neighbors::cagra::detail::build(handle_, - params, - dataset_view, - index_params_.nn_descent_params, - index_params_.ivf_pq_refine_rate, - index_params_.ivf_pq_build_params, - index_params_.ivf_pq_search_params, - include_dataset))); -} - -inline std::string allocator_to_string(AllocatorType mem_type) -{ - if (mem_type == AllocatorType::Device) { - return "device"; - } else if (mem_type == AllocatorType::HostPinned) { - return "host_pinned"; - } else if (mem_type == AllocatorType::HostHugePage) { - return "host_huge_page"; - } - return ""; -} - -template -void RaftCagra::set_search_param(const AnnSearchParam& param) -{ - auto search_param = dynamic_cast(param); - search_params_ = search_param.p; - refine_ratio_ = search_param.refine_ratio; - if (search_param.graph_mem != graph_mem_) { - // Move graph to correct memory space - graph_mem_ = search_param.graph_mem; - RAFT_LOG_DEBUG("moving graph to new memory space: %s", allocator_to_string(graph_mem_).c_str()); - // We create a new graph and copy to it from existing graph - auto mr = get_mr(graph_mem_); - auto new_graph = make_device_mdarray( - handle_, mr, make_extents(index_->graph().extent(0), index_->graph_degree())); - - raft::copy(new_graph.data_handle(), - index_->graph().data_handle(), - index_->graph().size(), - resource::get_cuda_stream(handle_)); - - index_->update_graph(handle_, make_const_mdspan(new_graph.view())); - // update_graph() only stores a view in the index. We need to keep the graph object alive. - *graph_ = std::move(new_graph); - } - - if (search_param.dataset_mem != dataset_mem_ || need_dataset_update_) { - dataset_mem_ = search_param.dataset_mem; - - // First free up existing memory - *dataset_ = make_device_matrix(handle_, 0, 0); - index_->update_dataset(handle_, make_const_mdspan(dataset_->view())); - - // Allocate space using the correct memory resource. - RAFT_LOG_DEBUG("moving dataset to new memory space: %s", - allocator_to_string(dataset_mem_).c_str()); - - auto mr = get_mr(dataset_mem_); - raft::neighbors::cagra::detail::copy_with_padding(handle_, *dataset_, *input_dataset_v_, mr); - - auto dataset_view = raft::make_device_strided_matrix_view( - dataset_->data_handle(), dataset_->extent(0), this->dim_, dataset_->extent(1)); - index_->update_dataset(handle_, dataset_view); - - need_dataset_update_ = false; - } -} - -template -void RaftCagra::set_search_dataset(const T* dataset, size_t nrow) -{ - using ds_idx_type = decltype(index_->data().n_rows()); - bool is_vpq = - dynamic_cast*>(&index_->data()) || - dynamic_cast*>(&index_->data()); - // It can happen that we are re-using a previous algo object which already has - // the dataset set. Check if we need update. - if (static_cast(input_dataset_v_->extent(0)) != nrow || - input_dataset_v_->data_handle() != dataset) { - *input_dataset_v_ = make_device_matrix_view(dataset, nrow, this->dim_); - need_dataset_update_ = !is_vpq; // ignore update if this is a VPQ dataset. - } -} - -template -void RaftCagra::save(const std::string& file) const -{ - raft::neighbors::cagra::serialize(handle_, file, *index_); -} - -template -void RaftCagra::save_to_hnswlib(const std::string& file) const -{ - raft::neighbors::cagra::serialize_to_hnswlib(handle_, file, *index_); -} - -template -void RaftCagra::load(const std::string& file) -{ - index_ = std::make_shared>( - std::move(raft::neighbors::cagra::deserialize(handle_, file))); -} - -template -std::unique_ptr> RaftCagra::copy() -{ - return std::make_unique>(*this); // use copy constructor -} - -template -void RaftCagra::search_base( - const T* queries, int batch_size, int k, AnnBase::index_type* neighbors, float* distances) const -{ - static_assert(std::is_integral_v); - static_assert(std::is_integral_v); - - IdxT* neighbors_IdxT; - std::optional> neighbors_storage{std::nullopt}; - if constexpr (sizeof(IdxT) == sizeof(AnnBase::index_type)) { - neighbors_IdxT = reinterpret_cast(neighbors); - } else { - neighbors_storage.emplace(batch_size * k, resource::get_cuda_stream(handle_)); - neighbors_IdxT = neighbors_storage->data(); - } - - auto queries_view = - raft::make_device_matrix_view(queries, batch_size, dimension_); - auto neighbors_view = raft::make_device_matrix_view(neighbors_IdxT, batch_size, k); - auto distances_view = raft::make_device_matrix_view(distances, batch_size, k); - - raft::neighbors::cagra::search( - handle_, search_params_, *index_, queries_view, neighbors_view, distances_view); - - if constexpr (sizeof(IdxT) != sizeof(AnnBase::index_type)) { - raft::linalg::unaryOp(neighbors, - neighbors_IdxT, - batch_size * k, - raft::cast_op(), - raft::resource::get_cuda_stream(handle_)); - } -} - -template -void RaftCagra::search( - const T* queries, int batch_size, int k, AnnBase::index_type* neighbors, float* distances) const -{ - auto k0 = static_cast(refine_ratio_ * k); - const bool disable_refinement = k0 <= static_cast(k); - const raft::resources& res = handle_; - - if (disable_refinement) { - search_base(queries, batch_size, k, neighbors, distances); - } else { - auto queries_v = - raft::make_device_matrix_view(queries, batch_size, dimension_); - auto candidate_ixs = - raft::make_device_matrix(res, batch_size, k0); - auto candidate_dists = - raft::make_device_matrix(res, batch_size, k0); - search_base( - queries, batch_size, k0, candidate_ixs.data_handle(), candidate_dists.data_handle()); - refine_helper( - res, *input_dataset_v_, queries_v, candidate_ixs, k, neighbors, distances, index_->metric()); - } -} -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/raft/raft_ivf_flat.cu b/cpp/bench/ann/src/raft/raft_ivf_flat.cu deleted file mode 100644 index bcd23723a4..0000000000 --- a/cpp/bench/ann/src/raft/raft_ivf_flat.cu +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "raft_ivf_flat_wrapper.h" - -namespace raft::bench::ann { -template class RaftIvfFlatGpu; -template class RaftIvfFlatGpu; -template class RaftIvfFlatGpu; -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/raft/raft_ivf_flat_wrapper.h b/cpp/bench/ann/src/raft/raft_ivf_flat_wrapper.h deleted file mode 100644 index 83a3a63aba..0000000000 --- a/cpp/bench/ann/src/raft/raft_ivf_flat_wrapper.h +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "../common/ann_types.hpp" -#include "raft_ann_bench_utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace raft::bench::ann { - -template -class RaftIvfFlatGpu : public ANN, public AnnGPU { - public: - using typename ANN::AnnSearchParam; - - struct SearchParam : public AnnSearchParam { - raft::neighbors::ivf_flat::search_params ivf_flat_params; - }; - - using BuildParam = raft::neighbors::ivf_flat::index_params; - - RaftIvfFlatGpu(Metric metric, int dim, const BuildParam& param) - : ANN(metric, dim), index_params_(param), dimension_(dim) - { - index_params_.metric = parse_metric_type(metric); - index_params_.conservative_memory_allocation = true; - RAFT_CUDA_TRY(cudaGetDevice(&device_)); - } - - void build(const T* dataset, size_t nrow) final; - - void set_search_param(const AnnSearchParam& param) override; - - void search(const T* queries, - int batch_size, - int k, - AnnBase::index_type* neighbors, - float* distances) const override; - - [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override - { - return handle_.get_sync_stream(); - } - - // to enable dataset access from GPU memory - AlgoProperty get_preference() const override - { - AlgoProperty property; - property.dataset_memory_type = MemoryType::HostMmap; - property.query_memory_type = MemoryType::Device; - return property; - } - void save(const std::string& file) const override; - void load(const std::string&) override; - std::unique_ptr> copy() override; - - private: - // handle_ must go first to make sure it dies last and all memory allocated in pool - configured_raft_resources handle_{}; - BuildParam index_params_; - raft::neighbors::ivf_flat::search_params search_params_; - std::shared_ptr> index_; - int device_; - int dimension_; -}; - -template -void RaftIvfFlatGpu::build(const T* dataset, size_t nrow) -{ - index_ = std::make_shared>(std::move( - raft::neighbors::ivf_flat::build(handle_, index_params_, dataset, IdxT(nrow), dimension_))); -} - -template -void RaftIvfFlatGpu::set_search_param(const AnnSearchParam& param) -{ - auto search_param = dynamic_cast(param); - search_params_ = search_param.ivf_flat_params; - assert(search_params_.n_probes <= index_params_.n_lists); -} - -template -void RaftIvfFlatGpu::save(const std::string& file) const -{ - raft::neighbors::ivf_flat::serialize(handle_, file, *index_); - return; -} - -template -void RaftIvfFlatGpu::load(const std::string& file) -{ - index_ = std::make_shared>( - std::move(raft::neighbors::ivf_flat::deserialize(handle_, file))); - return; -} - -template -std::unique_ptr> RaftIvfFlatGpu::copy() -{ - return std::make_unique>(*this); // use copy constructor -} - -template -void RaftIvfFlatGpu::search( - const T* queries, int batch_size, int k, AnnBase::index_type* neighbors, float* distances) const -{ - static_assert(std::is_integral_v); - static_assert(std::is_integral_v); - - IdxT* neighbors_IdxT; - std::optional> neighbors_storage{std::nullopt}; - if constexpr (sizeof(IdxT) == sizeof(AnnBase::index_type)) { - neighbors_IdxT = reinterpret_cast(neighbors); - } else { - neighbors_storage.emplace(batch_size * k, resource::get_cuda_stream(handle_)); - neighbors_IdxT = neighbors_storage->data(); - } - raft::neighbors::ivf_flat::search(handle_, - search_params_, - *index_, - queries, - batch_size, - k, - neighbors_IdxT, - distances, - resource::get_workspace_resource(handle_)); - if constexpr (sizeof(IdxT) != sizeof(AnnBase::index_type)) { - raft::linalg::unaryOp(neighbors, - neighbors_IdxT, - batch_size * k, - raft::cast_op(), - raft::resource::get_cuda_stream(handle_)); - } -} -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/raft/raft_ivf_pq.cu b/cpp/bench/ann/src/raft/raft_ivf_pq.cu deleted file mode 100644 index d4f68c1c7d..0000000000 --- a/cpp/bench/ann/src/raft/raft_ivf_pq.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "raft_ivf_pq_wrapper.h" - -namespace raft::bench::ann { -template class RaftIvfPQ; -template class RaftIvfPQ; -template class RaftIvfPQ; -template class RaftIvfPQ; -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/raft/raft_ivf_pq_wrapper.h b/cpp/bench/ann/src/raft/raft_ivf_pq_wrapper.h deleted file mode 100644 index 7201467969..0000000000 --- a/cpp/bench/ann/src/raft/raft_ivf_pq_wrapper.h +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "../common/ann_types.hpp" -#include "raft_ann_bench_utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace raft::bench::ann { - -template -class RaftIvfPQ : public ANN, public AnnGPU { - public: - using typename ANN::AnnSearchParam; - using ANN::dim_; - - struct SearchParam : public AnnSearchParam { - raft::neighbors::ivf_pq::search_params pq_param; - float refine_ratio = 1.0f; - auto needs_dataset() const -> bool override { return refine_ratio > 1.0f; } - }; - - using BuildParam = raft::neighbors::ivf_pq::index_params; - - RaftIvfPQ(Metric metric, int dim, const BuildParam& param) - : ANN(metric, dim), index_params_(param), dimension_(dim) - { - index_params_.metric = parse_metric_type(metric); - } - - void build(const T* dataset, size_t nrow) final; - - void set_search_param(const AnnSearchParam& param) override; - void set_search_dataset(const T* dataset, size_t nrow) override; - - void search(const T* queries, - int batch_size, - int k, - AnnBase::index_type* neighbors, - float* distances) const override; - void search_base(const T* queries, - int batch_size, - int k, - AnnBase::index_type* neighbors, - float* distances) const; - - [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override - { - return handle_.get_sync_stream(); - } - - // to enable dataset access from GPU memory - AlgoProperty get_preference() const override - { - AlgoProperty property; - property.dataset_memory_type = MemoryType::Host; - property.query_memory_type = MemoryType::Device; - return property; - } - void save(const std::string& file) const override; - void load(const std::string&) override; - std::unique_ptr> copy() override; - - private: - // handle_ must go first to make sure it dies last and all memory allocated in pool - configured_raft_resources handle_{}; - BuildParam index_params_; - raft::neighbors::ivf_pq::search_params search_params_; - std::shared_ptr> index_; - int dimension_; - float refine_ratio_ = 1.0; - raft::device_matrix_view dataset_; -}; - -template -void RaftIvfPQ::save(const std::string& file) const -{ - raft::neighbors::ivf_pq::serialize(handle_, file, *index_); -} - -template -void RaftIvfPQ::load(const std::string& file) -{ - index_ = std::make_shared>( - std::move(raft::neighbors::ivf_pq::deserialize(handle_, file))); -} - -template -void RaftIvfPQ::build(const T* dataset, size_t nrow) -{ - auto dataset_v = raft::make_device_matrix_view(dataset, IdxT(nrow), dim_); - std::make_shared>( - std::move(raft::neighbors::ivf_pq::build(handle_, index_params_, dataset_v))) - .swap(index_); -} - -template -std::unique_ptr> RaftIvfPQ::copy() -{ - return std::make_unique>(*this); // use copy constructor -} - -template -void RaftIvfPQ::set_search_param(const AnnSearchParam& param) -{ - auto search_param = dynamic_cast(param); - search_params_ = search_param.pq_param; - refine_ratio_ = search_param.refine_ratio; - assert(search_params_.n_probes <= index_params_.n_lists); -} - -template -void RaftIvfPQ::set_search_dataset(const T* dataset, size_t nrow) -{ - dataset_ = raft::make_device_matrix_view(dataset, nrow, index_->dim()); -} - -template -void RaftIvfPQ::search_base( - const T* queries, int batch_size, int k, AnnBase::index_type* neighbors, float* distances) const -{ - static_assert(std::is_integral_v); - static_assert(std::is_integral_v); - - IdxT* neighbors_IdxT; - std::optional> neighbors_storage{std::nullopt}; - if constexpr (sizeof(IdxT) == sizeof(AnnBase::index_type)) { - neighbors_IdxT = reinterpret_cast(neighbors); - } else { - neighbors_storage.emplace(batch_size * k, resource::get_cuda_stream(handle_)); - neighbors_IdxT = neighbors_storage->data(); - } - - auto queries_view = - raft::make_device_matrix_view(queries, batch_size, dimension_); - auto neighbors_view = - raft::make_device_matrix_view(neighbors_IdxT, batch_size, k); - auto distances_view = raft::make_device_matrix_view(distances, batch_size, k); - - raft::neighbors::ivf_pq::search( - handle_, search_params_, *index_, queries_view, neighbors_view, distances_view); - - if constexpr (sizeof(IdxT) != sizeof(AnnBase::index_type)) { - raft::linalg::unaryOp(neighbors, - neighbors_IdxT, - batch_size * k, - raft::cast_op(), - raft::resource::get_cuda_stream(handle_)); - } -} - -template -void RaftIvfPQ::search( - const T* queries, int batch_size, int k, AnnBase::index_type* neighbors, float* distances) const -{ - auto k0 = static_cast(refine_ratio_ * k); - const bool disable_refinement = k0 <= static_cast(k); - const raft::resources& res = handle_; - - if (disable_refinement) { - search_base(queries, batch_size, k, neighbors, distances); - } else { - auto queries_v = - raft::make_device_matrix_view(queries, batch_size, dimension_); - auto candidate_ixs = - raft::make_device_matrix(res, batch_size, k0); - auto candidate_dists = - raft::make_device_matrix(res, batch_size, k0); - search_base( - queries, batch_size, k0, candidate_ixs.data_handle(), candidate_dists.data_handle()); - refine_helper( - res, dataset_, queries_v, candidate_ixs, k, neighbors, distances, index_->metric()); - } -} -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/raft/raft_wrapper.h b/cpp/bench/ann/src/raft/raft_wrapper.h deleted file mode 100644 index 2c996058b2..0000000000 --- a/cpp/bench/ann/src/raft/raft_wrapper.h +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "../common/ann_types.hpp" -#include "raft_ann_bench_utils.h" - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace raft_temp { - -inline raft::distance::DistanceType parse_metric_type(raft::bench::ann::Metric metric) -{ - switch (metric) { - case raft::bench::ann::Metric::kInnerProduct: return raft::distance::DistanceType::InnerProduct; - case raft::bench::ann::Metric::kEuclidean: return raft::distance::DistanceType::L2Expanded; - default: throw std::runtime_error("raft supports only metric type of inner product and L2"); - } -} -} // namespace raft_temp - -namespace raft::bench::ann { - -// brute force KNN - RAFT -template -class RaftGpu : public ANN, public AnnGPU { - public: - using typename ANN::AnnSearchParam; - - RaftGpu(Metric metric, int dim); - - void build(const T*, size_t) final; - - void set_search_param(const AnnSearchParam& param) override; - - void search(const T* queries, - int batch_size, - int k, - AnnBase::index_type* neighbors, - float* distances) const final; - - // to enable dataset access from GPU memory - AlgoProperty get_preference() const override - { - AlgoProperty property; - property.dataset_memory_type = MemoryType::Device; - property.query_memory_type = MemoryType::Device; - return property; - } - [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override - { - return handle_.get_sync_stream(); - } - void set_search_dataset(const T* dataset, size_t nrow) override; - void save(const std::string& file) const override; - void load(const std::string&) override; - std::unique_ptr> copy() override; - - protected: - // handle_ must go first to make sure it dies last and all memory allocated in pool - configured_raft_resources handle_{}; - std::shared_ptr> index_; - raft::distance::DistanceType metric_type_; - int device_; - const T* dataset_; - size_t nrow_; -}; - -template -RaftGpu::RaftGpu(Metric metric, int dim) - : ANN(metric, dim), metric_type_(raft_temp::parse_metric_type(metric)) -{ - static_assert(std::is_same_v || std::is_same_v, - "raft bfknn only supports float/double"); - RAFT_CUDA_TRY(cudaGetDevice(&device_)); -} - -template -void RaftGpu::build(const T* dataset, size_t nrow) -{ - auto dataset_view = raft::make_host_matrix_view(dataset, nrow, this->dim_); - index_ = std::make_shared>( - std::move(raft::neighbors::brute_force::build(handle_, dataset_view))); -} - -template -void RaftGpu::set_search_param(const AnnSearchParam&) -{ - // Nothing to set here as it is brute force implementation -} - -template -void RaftGpu::set_search_dataset(const T* dataset, size_t nrow) -{ - dataset_ = dataset; - nrow_ = nrow; -} - -template -void RaftGpu::save(const std::string& file) const -{ - raft::neighbors::brute_force::serialize(handle_, file, *index_); -} - -template -void RaftGpu::load(const std::string& file) -{ - index_ = std::make_shared>( - std::move(raft::neighbors::brute_force::deserialize(handle_, file))); -} - -template -void RaftGpu::search( - const T* queries, int batch_size, int k, AnnBase::index_type* neighbors, float* distances) const -{ - auto queries_view = - raft::make_device_matrix_view(queries, batch_size, this->dim_); - - auto neighbors_view = - raft::make_device_matrix_view(neighbors, batch_size, k); - auto distances_view = raft::make_device_matrix_view(distances, batch_size, k); - - raft::neighbors::brute_force::search( - handle_, *index_, queries_view, neighbors_view, distances_view); -} - -template -std::unique_ptr> RaftGpu::copy() -{ - return std::make_unique>(*this); // use copy constructor -} - -} // namespace raft::bench::ann diff --git a/cpp/cmake/config.json b/cpp/cmake/config.json index f7cc50e513..3c568d9766 100644 --- a/cpp/cmake/config.json +++ b/cpp/cmake/config.json @@ -9,7 +9,7 @@ "VERSION": "?", "GIT_SHALLOW": "?", "OPTIONS": "*", - "FIND_PACKAGE_ARGUMENTS": "*" + "FIND_PACKAGE_ARGUMENTS": "*" } }, "ConfigureTest": { diff --git a/cpp/cmake/modules/FindAVX.cmake b/cpp/cmake/modules/FindAVX.cmake deleted file mode 100644 index 7f3b2dfc76..0000000000 --- a/cpp/cmake/modules/FindAVX.cmake +++ /dev/null @@ -1,110 +0,0 @@ -# ============================================================================= -# Copyright (c) 2016- Facebook, Inc (Adam Paszke) -# Copyright (c) 2014- Facebook, Inc (Soumith Chintala) -# Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) -# Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) -# Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) -# Copyright (c) 2011-2013 NYU (Clement Farabet) -# Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) -# Copyright (c) 2006 Idiap Research Institute (Samy Bengio) -# Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) -# -# Note: This file was copied from PyTorch and modified for use in the RAFT library. -# Refer to thirdparty/LICENSES/LICENSE.pytorch for license and additional -# copyright information. -# ============================================================================= - -INCLUDE(CheckCXXSourceRuns) - -SET(AVX_CODE - " - #include - - int main() - { - __m256 a; - a = _mm256_set1_ps(0); - return 0; - } -" -) - -SET(AVX512_CODE - " - #include - - int main() - { - __m512i a = _mm512_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0); - __m512i b = a; - __mmask64 equality_mask = _mm512_cmp_epi8_mask(a, b, _MM_CMPINT_EQ); - return 0; - } -" -) - -SET(AVX2_CODE - " - #include - - int main() - { - __m256i a = {0}; - a = _mm256_abs_epi16(a); - __m256i x; - _mm256_extract_epi64(x, 0); // we rely on this in our AVX2 code - return 0; - } -" -) - -MACRO(CHECK_SSE lang type flags) - SET(__FLAG_I 1) - SET(CMAKE_REQUIRED_FLAGS_SAVE ${CMAKE_REQUIRED_FLAGS}) - FOREACH(__FLAG ${flags}) - IF(NOT ${lang}_${type}_FOUND) - SET(CMAKE_REQUIRED_FLAGS ${__FLAG}) - CHECK_CXX_SOURCE_RUNS("${${type}_CODE}" ${lang}_HAS_${type}_${__FLAG_I}) - IF(${lang}_HAS_${type}_${__FLAG_I}) - SET(${lang}_${type}_FOUND - TRUE - CACHE BOOL "${lang} ${type} support" - ) - SET(${lang}_${type}_FLAGS - "${__FLAG}" - CACHE STRING "${lang} ${type} flags" - ) - ENDIF() - MATH(EXPR __FLAG_I "${__FLAG_I}+1") - ENDIF() - ENDFOREACH() - SET(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_SAVE}) - - IF(NOT ${lang}_${type}_FOUND) - SET(${lang}_${type}_FOUND - FALSE - CACHE BOOL "${lang} ${type} support" - ) - SET(${lang}_${type}_FLAGS - "" - CACHE STRING "${lang} ${type} flags" - ) - ENDIF() - - MARK_AS_ADVANCED(${lang}_${type}_FOUND ${lang}_${type}_FLAGS) - -ENDMACRO() - -# CHECK_SSE(C "AVX" " ;-mavx;/arch:AVX") CHECK_SSE(C "AVX2" " ;-mavx2 -mfma;/arch:AVX2") CHECK_SSE(C -# "AVX512" " ;-mavx512f -mavx512dq -mavx512vl -mavx512bw -mfma;/arch:AVX512") -# -CHECK_SSE(CXX "AVX" " ;-mavx;/arch:AVX") -CHECK_SSE(CXX "AVX2" " ;-mavx2 -mfma;/arch:AVX2") -CHECK_SSE(CXX "AVX512" " ;-mavx512f -mavx512dq -mavx512vl -mavx512bw -mfma;/arch:AVX512") diff --git a/cpp/cmake/patches/faiss_override.json b/cpp/cmake/patches/faiss_override.json deleted file mode 100644 index 5d18c77fec..0000000000 --- a/cpp/cmake/patches/faiss_override.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "packages" : { - "faiss" : { - "version": "1.9.0", - "git_url": "https://github.com/facebookresearch/faiss.git", - "git_tag": "v1.9.0" - } - } -} diff --git a/cpp/cmake/patches/ggnn.diff b/cpp/cmake/patches/ggnn.diff deleted file mode 100644 index fc45298803..0000000000 --- a/cpp/cmake/patches/ggnn.diff +++ /dev/null @@ -1,230 +0,0 @@ ---- a/include/ggnn/cache/cuda_simple_knn_sym_cache.cuh -+++ b/include/ggnn/cache/cuda_simple_knn_sym_cache.cuh -@@ -62,7 +62,7 @@ struct SimpleKNNSymCache { - const ValueT dist_half) - : dist_query(dist_query), dist_half(dist_half) {} - -- __device__ __forceinline__ DistQueryAndHalf() {} -+ DistQueryAndHalf() = default; - }; - - struct DistanceAndNorm { -@@ -98,8 +98,7 @@ struct SimpleKNNSymCache { - KeyT cache; - DistQueryAndHalf dist; - bool flag; -- -- __device__ __forceinline__ SyncTempStorage() {} -+ SyncTempStorage() = default; - }; - - public: -diff --git a/include/ggnn/cuda_knn_ggnn_gpu_instance.cuh b/include/ggnn/cuda_knn_ggnn_gpu_instance.cuh -index 8cbaf0d..6eb72ac 100644 ---- a/include/ggnn/cuda_knn_ggnn_gpu_instance.cuh -+++ b/include/ggnn/cuda_knn_ggnn_gpu_instance.cuh -@@ -41,7 +41,6 @@ limitations under the License. - #include "ggnn/sym/cuda_knn_sym_query_layer.cuh" - #include "ggnn/utils/cuda_knn_utils.cuh" - #include "ggnn/utils/cuda_knn_constants.cuh" --#include "ggnn/utils/cuda_knn_dataset.cuh" - - template - __global__ void divide(ValueT* res, ValueT* input, ValueT N) { -@@ -98,9 +97,7 @@ struct GGNNGPUInstance { - typedef GGNNGraphDevice GGNNGraphDevice; - typedef GGNNGraphHost GGNNGraphHost; - -- const Dataset* dataset; - GGNNGraphBuffer* ggnn_buffer {nullptr}; -- GGNNQuery ggnn_query; - - // Graph Shards resident on the GPU - std::vector ggnn_shards; -@@ -117,13 +114,12 @@ struct GGNNGPUInstance { - // number of shards that need to be processed by this instance - const int num_parts; - -- GGNNGPUInstance(const int gpu_id, const Dataset* dataset, -+ GGNNGPUInstance(const int gpu_id, - const int N_shard, const int L, - const bool enable_construction, const float tau_build, - const int num_parts=1, const int num_cpu_buffers=1) : - N_shard{N_shard}, L{L}, tau_build{tau_build}, -- dataset{dataset}, gpu_id{gpu_id}, -- ggnn_query{dataset->N_query, D, KQuery, num_parts}, -+ gpu_id{gpu_id}, - num_parts{num_parts} - { - CHECK_LE(L, MAX_LAYER); -@@ -135,7 +131,6 @@ struct GGNNGPUInstance { - CHECK_EQ(current_gpu_id, gpu_id) << "cudaSetDevice() needs to be called in advance!"; - } - -- ggnn_query.loadQueriesAsync(dataset->h_query, 0); - - computeGraphParameters(); - -@@ -186,7 +181,7 @@ struct GGNNGPUInstance { - } - - GGNNGPUInstance(const GGNNGPUInstance& other) -- : dataset{nullptr}, ggnn_query{0, D, KQuery}, -+ : - gpu_id{0}, N_shard{0}, num_parts{0} { - // this exists to allow using vector::emplace_back - // when it triggers a reallocation, this code will be called. -@@ -305,6 +300,7 @@ struct GGNNGPUInstance { - - // io - -+ /* - void waitForDiskIO(const int shard_id) { - auto& cpu_buffer = ggnn_cpu_buffers[shard_id%ggnn_cpu_buffers.size()]; - if (cpu_buffer.disk_io_thread.joinable()) -@@ -468,11 +464,12 @@ struct GGNNGPUInstance { - CHECK_CUDA(cudaDeviceSynchronize()); - CHECK_CUDA(cudaPeekAtLastError()); - } -+ */ - - // graph operations - - template -- void queryLayer(const int shard_id = 0) const { -+ void queryLayer(const BaseT* d_query, int batch_size, KeyT* d_query_result_ids, ValueT* d_query_result_dists, const int shard_id = 0) const { - CHECK_CUDA(cudaSetDevice(gpu_id)); - const auto& shard = ggnn_shards.at(shard_id%ggnn_shards.size()); - -@@ -482,21 +479,21 @@ struct GGNNGPUInstance { - - int* m_dist_statistics = nullptr; - if (DIST_STATS) -- cudaMallocManaged(&m_dist_statistics, dataset->N_query * sizeof(int)); -+ cudaMallocManaged(&m_dist_statistics, batch_size * sizeof(int)); - - QueryKernel query_kernel; - query_kernel.d_base = shard.d_base; -- query_kernel.d_query = ggnn_query.d_query; -+ query_kernel.d_query = d_query; - - query_kernel.d_graph = shard.d_graph; -- query_kernel.d_query_results = ggnn_query.d_query_result_ids; -- query_kernel.d_query_results_dists = ggnn_query.d_query_result_dists; -+ query_kernel.d_query_results = d_query_result_ids; -+ query_kernel.d_query_results_dists = d_query_result_dists; - - query_kernel.d_translation = shard.d_translation; - - query_kernel.d_nn1_stats = shard.d_nn1_stats; - -- query_kernel.N = dataset->N_query; -+ query_kernel.N = batch_size; - query_kernel.N_offset = 0; - - query_kernel.d_dist_stats = m_dist_statistics; -@@ -771,6 +768,16 @@ struct GGNNGPUInstance { - sym(layer, shard_id); - } - } -+ -+ void set_stream(cudaStream_t stream) { -+ assert(ggnn_shards.size() == 1); -+ ggnn_shards.at(0).stream = stream; -+ } -+ -+ void set_base_data(const BaseT* dataset) { -+ assert(ggnn_shards.size() == 1); -+ ggnn_shards.at(0).d_base = dataset; -+ } - }; - - #endif // INCLUDE_GGNN_CUDA_KNN_GGNN_GPU_INSTANCE_CUH_ -diff --git a/include/ggnn/graph/cuda_knn_ggnn_graph_device.cuh b/include/ggnn/graph/cuda_knn_ggnn_graph_device.cuh -index c94a8f1..781226d 100644 ---- a/include/ggnn/graph/cuda_knn_ggnn_graph_device.cuh -+++ b/include/ggnn/graph/cuda_knn_ggnn_graph_device.cuh -@@ -50,7 +50,7 @@ struct GGNNGraphDevice { - ValueT* d_nn1_stats; - - /// base data pointer for the shard. -- BaseT* d_base; -+ const BaseT* d_base; - - /// combined memory pool - char* d_memory; -@@ -69,7 +69,9 @@ struct GGNNGraphDevice { - const size_t selection_translation_size = align8(ST_all * sizeof(KeyT)); - const size_t nn1_stats_size = align8(2 * sizeof(ValueT)); - total_graph_size = graph_size + 2 * selection_translation_size + nn1_stats_size; -- base_size = align8(static_cast(N) * D * sizeof(BaseT)); -+ // base_size = align8(static_cast(N) * D * sizeof(BaseT)); -+ (void) N; -+ (void) D; - - const size_t total_size = base_size+total_graph_size; - -@@ -86,8 +88,7 @@ struct GGNNGraphDevice { - CHECK_CUDA(cudaMalloc(&d_memory, total_size)); - - size_t pos = 0; -- d_base = reinterpret_cast(d_memory+pos); -- pos += base_size; -+ d_base = nullptr; - d_graph = reinterpret_cast(d_memory+pos); - pos += graph_size; - d_translation = reinterpret_cast(d_memory+pos); -@@ -99,14 +100,14 @@ struct GGNNGraphDevice { - - CHECK_EQ(pos, total_size); - -- CHECK_CUDA(cudaStreamCreate(&stream)); -+ // CHECK_CUDA(cudaStreamCreate(&stream)); - - CHECK_CUDA(cudaPeekAtLastError()); - CHECK_CUDA(cudaDeviceSynchronize()); - CHECK_CUDA(cudaPeekAtLastError()); - } - -- GGNNGraphDevice(const GGNNGraphDevice& other) { -+ GGNNGraphDevice(const GGNNGraphDevice&) { - // this exists to allow using vector::emplace_back - // when it triggers a reallocation, this code will be called. - // always make sure that enough memory is reserved ahead of time. -@@ -116,7 +117,7 @@ struct GGNNGraphDevice { - ~GGNNGraphDevice() { - cudaFree(d_memory); - -- CHECK_CUDA(cudaStreamDestroy(stream)); -+ // CHECK_CUDA(cudaStreamDestroy(stream)); - } - }; - -diff --git a/include/ggnn/graph/cuda_knn_ggnn_graph_host.cuh b/include/ggnn/graph/cuda_knn_ggnn_graph_host.cuh -index 2055f9e..ef5843a 100644 ---- a/include/ggnn/graph/cuda_knn_ggnn_graph_host.cuh -+++ b/include/ggnn/graph/cuda_knn_ggnn_graph_host.cuh -@@ -92,7 +92,7 @@ struct GGNNGraphHost { - CHECK_CUDA(cudaPeekAtLastError()); - } - -- GGNNGraphHost(const GGNNGraphHost& other) { -+ GGNNGraphHost(const GGNNGraphHost&) { - // this exists to allow using vector::emplace_back - // when it triggers a reallocation, this code will be called. - // always make sure that enough memory is reserved ahead of time. -diff --git a/include/ggnn/select/cuda_knn_wrs_select_layer.cuh b/include/ggnn/select/cuda_knn_wrs_select_layer.cuh -index 49d76a1..eef69e6 100644 ---- a/include/ggnn/select/cuda_knn_wrs_select_layer.cuh -+++ b/include/ggnn/select/cuda_knn_wrs_select_layer.cuh -@@ -22,7 +22,6 @@ limitations under the License. - #include - #include - --#include - #include - - #include "ggnn/utils/cuda_knn_constants.cuh" --- -2.43.0 - diff --git a/cpp/cmake/patches/ggnn_override.json b/cpp/cmake/patches/ggnn_override.json deleted file mode 100644 index 768fae8b0c..0000000000 --- a/cpp/cmake/patches/ggnn_override.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "packages" : { - "ggnn" : { - "version": "0.5", - "git_url": "https://github.com/cgtuebingen/ggnn.git", - "git_tag": "release_${version}", - "patches" : [ - { - "file" : "${current_json_dir}/ggnn.diff", - "issue" : "Correct compilation issues", - "fixed_in" : "" - } - ] - } - } -} diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake deleted file mode 100644 index 706b0c2f11..0000000000 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ /dev/null @@ -1,119 +0,0 @@ -#============================================================================= -# Copyright (c) 2021-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -function(find_and_configure_faiss) - set(oneValueArgs VERSION REPOSITORY PINNED_TAG BUILD_STATIC_LIBS EXCLUDE_FROM_ALL ENABLE_GPU) - cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" - "${multiValueArgs}" ${ARGN} ) - - rapids_find_generate_module(faiss - HEADER_NAMES faiss/IndexFlat.h - LIBRARY_NAMES faiss - ) - - set(patch_dir "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../patches") - rapids_cpm_package_override("${patch_dir}/faiss_override.json") - - include("${rapids-cmake-dir}/cpm/detail/package_details.cmake") - rapids_cpm_package_details(faiss version repository tag shallow exclude) - - include("${rapids-cmake-dir}/cpm/detail/generate_patch_command.cmake") - rapids_cpm_generate_patch_command(faiss ${version} patch_command) - - set(BUILD_SHARED_LIBS ON) - if (PKG_BUILD_STATIC_LIBS) - set(BUILD_SHARED_LIBS OFF) - set(CPM_DOWNLOAD_faiss ON) - endif() - - include(cmake/modules/FindAVX) - # Link against AVX CPU lib if it exists - set(RAFT_FAISS_OPT_LEVEL "generic") - if(CXX_AVX2_FOUND) - set(RAFT_FAISS_OPT_LEVEL "avx2") - endif() - - rapids_cpm_find(faiss ${version} - GLOBAL_TARGETS faiss faiss_avx2 faiss_gpu faiss::faiss faiss::faiss_avx2 - CPM_ARGS - GIT_REPOSITORY ${repository} - GIT_TAG ${tag} - GIT_SHALLOW ${shallow} ${patch_command} - EXCLUDE_FROM_ALL ${exclude} - OPTIONS - "FAISS_ENABLE_GPU ${PKG_ENABLE_GPU}" - "FAISS_ENABLE_RAFT ${PKG_ENABLE_GPU}" - "FAISS_ENABLE_PYTHON OFF" - "FAISS_OPT_LEVEL ${RAFT_FAISS_OPT_LEVEL}" - "FAISS_USE_CUDA_TOOLKIT_STATIC ${CUDA_STATIC_RUNTIME}" - "BUILD_TESTING OFF" - "CMAKE_MESSAGE_LOG_LEVEL VERBOSE" - ) - - include("${rapids-cmake-dir}/cpm/detail/display_patch_status.cmake") - rapids_cpm_display_patch_status(hnswlib) - - if(TARGET faiss AND NOT TARGET faiss::faiss) - add_library(faiss::faiss ALIAS faiss) - # We need to ensure that faiss has all the conda information. So we use this approach so that - # faiss will have the conda includes/link dirs - target_link_libraries(faiss PRIVATE $) - endif() - if(TARGET faiss_avx2 AND NOT TARGET faiss::faiss_avx2) - add_library(faiss::faiss_avx2 ALIAS faiss_avx2) - # We need to ensure that faiss has all the conda information. So we use this approach so that - # faiss will have the conda includes/link dirs - target_link_libraries(faiss_avx2 PRIVATE $) - endif() - if(TARGET faiss_gpu AND NOT TARGET faiss::faiss_gpu) - add_library(faiss::faiss_gpu ALIAS faiss_gpu) - # We need to ensure that faiss has all the conda information. So we use this approach so that - # faiss will have the conda includes/link dirs - target_link_libraries(faiss_gpu PRIVATE $) - endif() - - if(faiss_ADDED) - rapids_export(BUILD faiss - EXPORT_SET faiss-targets - GLOBAL_TARGETS ${RAFT_FAISS_EXPORT_GLOBAL_TARGETS} - NAMESPACE faiss::) - endif() - - # Need to tell CMake to rescan the link group of faiss::faiss_gpu and faiss - # so that we get proper link order when they are static - # - # We don't look at the existence of `faiss_avx2` as it will always exist - # even when CXX_AVX2_FOUND is false. In addition for arm builds the - # faiss_avx2 is marked as `EXCLUDE_FROM_ALL` so we don't want to add - # a dependency to it. Adding a dependency will cause it to compile, - # and fail due to invalid compiler flags. - if(PKG_ENABLE_GPU AND PKG_BUILD_STATIC_LIBS AND CXX_AVX2_FOUND) - set(RAFT_FAISS_TARGETS "$,faiss::faiss_avx2>" PARENT_SCOPE) - elseif(PKG_ENABLE_GPU AND PKG_BUILD_STATIC_LIBS) - set(RAFT_FAISS_TARGETS "$,faiss::faiss>" PARENT_SCOPE) - elseif(CXX_AVX2_FOUND) - set(RAFT_FAISS_TARGETS faiss::faiss_avx2 PARENT_SCOPE) - else() - set(RAFT_FAISS_TARGETS faiss::faiss PARENT_SCOPE) - endif() - -endfunction() - - -find_and_configure_faiss( - BUILD_STATIC_LIBS ${RAFT_USE_FAISS_STATIC} - ENABLE_GPU ${RAFT_FAISS_ENABLE_GPU} -) \ No newline at end of file diff --git a/cpp/cmake/thirdparty/get_fmt.cmake b/cpp/cmake/thirdparty/get_fmt.cmake deleted file mode 100644 index c06f8a78bb..0000000000 --- a/cpp/cmake/thirdparty/get_fmt.cmake +++ /dev/null @@ -1,22 +0,0 @@ -# ============================================================================= -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -# Use CPM to find or clone fmt -function(find_and_configure_fmt) - - include(${rapids-cmake-dir}/cpm/fmt.cmake) - rapids_cpm_fmt(INSTALL_EXPORT_SET rmm-exports BUILD_EXPORT_SET rmm-exports) -endfunction() - -find_and_configure_fmt() \ No newline at end of file diff --git a/cpp/cmake/thirdparty/get_ggnn.cmake b/cpp/cmake/thirdparty/get_ggnn.cmake deleted file mode 100644 index d8af4971a7..0000000000 --- a/cpp/cmake/thirdparty/get_ggnn.cmake +++ /dev/null @@ -1,50 +0,0 @@ -#============================================================================= -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -function(find_and_configure_ggnn) - - include(${rapids-cmake-dir}/cpm/package_override.cmake) - set(patch_dir "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../patches") - rapids_cpm_package_override("${patch_dir}/ggnn_override.json") - - include("${rapids-cmake-dir}/cpm/detail/package_details.cmake") - rapids_cpm_package_details(ggnn version repository tag shallow exclude) - - include("${rapids-cmake-dir}/cpm/detail/generate_patch_command.cmake") - rapids_cpm_generate_patch_command(ggnn ${version} patch_command) - - rapids_cpm_find( - ggnn ${version} - GLOBAL_TARGETS ggnn::ggnn - CPM_ARGS - GIT_REPOSITORY ${repository} - GIT_TAG ${tag} - GIT_SHALLOW ${shallow} ${patch_command} - EXCLUDE_FROM_ALL ${exclude} - DOWNLOAD_ONLY ON - ) - - include("${rapids-cmake-dir}/cpm/detail/display_patch_status.cmake") - rapids_cpm_display_patch_status(ggnn) - - if(NOT TARGET ggnn::ggnn) - add_library(ggnn INTERFACE) - target_include_directories(ggnn INTERFACE "$") - add_library(ggnn::ggnn ALIAS ggnn) - endif() - -endfunction() -find_and_configure_ggnn() diff --git a/cpp/cmake/thirdparty/get_glog.cmake b/cpp/cmake/thirdparty/get_glog.cmake deleted file mode 100644 index 35a9170f99..0000000000 --- a/cpp/cmake/thirdparty/get_glog.cmake +++ /dev/null @@ -1,48 +0,0 @@ -#============================================================================= -# Copyright (c) 2021-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -function(find_and_configure_glog) - set(oneValueArgs VERSION FORK PINNED_TAG EXCLUDE_FROM_ALL) - cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" - "${multiValueArgs}" ${ARGN} ) - - rapids_cpm_find(glog ${PKG_VERSION} - GLOBAL_TARGETS glog::glog - BUILD_EXPORT_SET raft-exports - INSTALL_EXPORT_SET raft-exports - CPM_ARGS - GIT_REPOSITORY https://github.com/${PKG_FORK}/glog.git - GIT_TAG ${PKG_PINNED_TAG} - EXCLUDE_FROM_ALL ${PKG_EXCLUDE_FROM_ALL} - ) - - if(glog_ADDED) - message(VERBOSE "RAFT: Using glog located in ${glog_SOURCE_DIR}") - else() - message(VERBOSE "RAFT: Using glog located in ${glog_DIR}") - endif() - - -endfunction() - -# Change pinned tag here to test a commit in CI -# To use a different RAFT locally, set the CMake variable -# CPM_glog_SOURCE=/path/to/local/glog -find_and_configure_glog(VERSION 0.6.0 - FORK google - PINNED_TAG v0.6.0 - EXCLUDE_FROM_ALL ON - ) diff --git a/cpp/cmake/thirdparty/get_nlohmann_json.cmake b/cpp/cmake/thirdparty/get_nlohmann_json.cmake deleted file mode 100644 index 5de98a47ce..0000000000 --- a/cpp/cmake/thirdparty/get_nlohmann_json.cmake +++ /dev/null @@ -1,39 +0,0 @@ -#============================================================================= -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -function(find_and_configure_nlohmann_json) - set(oneValueArgs VERSION FORK PINNED_TAG EXCLUDE_FROM_ALL) - cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" - "${multiValueArgs}" ${ARGN} ) - - rapids_cpm_find(nlohmann_json ${PKG_VERSION} - GLOBAL_TARGETS nlohmann_json::nlohmann_json - BUILD_EXPORT_SET raft-bench-ann-exports - INSTALL_EXPORT_SET raft-bench-ann-exports - CPM_ARGS - GIT_REPOSITORY https://github.com/${PKG_FORK}/json.git - GIT_TAG ${PKG_PINNED_TAG} - EXCLUDE_FROM_ALL ${PKG_EXCLUDE_FROM_ALL}) - -endfunction() - -# Change pinned tag here to test a commit in CI -# To use a different RAFT locally, set the CMake variable -# CPM_raft_SOURCE=/path/to/local/raft -find_and_configure_nlohmann_json(VERSION 3.11.2 - FORK nlohmann - PINNED_TAG v3.11.2 - EXCLUDE_FROM_ALL YES) diff --git a/cpp/template/cmake/thirdparty/get_raft.cmake b/cpp/template/cmake/thirdparty/get_raft.cmake index 07b0897be0..4474fd2875 100644 --- a/cpp/template/cmake/thirdparty/get_raft.cmake +++ b/cpp/template/cmake/thirdparty/get_raft.cmake @@ -51,7 +51,6 @@ function(find_and_configure_raft) OPTIONS "BUILD_TESTS OFF" "BUILD_PRIMS_BENCH OFF" - "BUILD_ANN_BENCH OFF" "RAFT_COMPILE_LIBRARY ${PKG_COMPILE_LIBRARY}" ) endfunction() diff --git a/dependencies.yaml b/dependencies.yaml index 7766481c99..1772c5d539 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -22,19 +22,6 @@ files: - run_pylibraft - test_python_common - test_pylibraft - bench_ann: - output: conda - matrix: - cuda: ["11.8", "12.0"] - arch: [x86_64, aarch64] - includes: - - rapids_build - - cuda - - cuda_version - - develop - - nn_bench - - nn_bench_python - - rapids_build_skbuild test_cpp: output: none includes: @@ -127,20 +114,6 @@ files: key: test includes: - test_python_common - py_build_raft_ann_bench: - output: pyproject - pyproject_dir: python/raft-ann-bench - extras: - table: build-system - includes: - - rapids_build_setuptools - py_run_raft_ann_bench: - output: pyproject - pyproject_dir: python/raft-ann-bench - extras: - table: project - includes: - - nn_bench_python channels: - rapidsai - rapidsai-nightly @@ -255,25 +228,6 @@ dependencies: packages: - clang==16.0.6 - clang-tools=16.0.6 - nn_bench: - common: - - output_types: [conda, pyproject, requirements] - packages: - - hnswlib=0.7.0 - - nlohmann_json>=3.11.2 - - glog>=0.6.0 - - h5py>=3.8.0 - - benchmark>=1.8.2 - - openblas - - *rmm_unsuffixed - nn_bench_python: - common: - - output_types: [conda] - packages: - - matplotlib - - pandas - - pyyaml - - pandas cuda_version: specific: - output_types: conda diff --git a/docs/source/ann_benchmarks_build.md b/docs/source/ann_benchmarks_build.md deleted file mode 100644 index 56af8e555c..0000000000 --- a/docs/source/ann_benchmarks_build.md +++ /dev/null @@ -1,51 +0,0 @@ -### Dependencies - -CUDA 11 and a GPU with Pascal architecture or later are required to run the benchmarks. - -Please refer to the [installation docs](https://docs.rapids.ai/api/raft/stable/build.html#cuda-gpu-requirements) for the base requirements to build RAFT. - -In addition to the base requirements for building RAFT, additional dependencies needed to build the ANN benchmarks include: -1. FAISS GPU >= 1.7.1 -2. Google Logging (GLog) -3. H5Py -4. HNSWLib -5. nlohmann_json -6. GGNN - -[rapids-cmake](https://github.com/rapidsai/rapids-cmake) is used to build the ANN benchmarks so the code for dependencies not already supplied in the CUDA toolkit will be downloaded and built automatically. - -The easiest (and most reproducible) way to install the dependencies needed to build the ANN benchmarks is to use the conda environment file located in the `conda/environments` directory of the RAFT repository. The following command will use `mamba` (which is preferred over `conda`) to build and activate a new environment for compiling the benchmarks: - -```bash -mamba env create --name raft_ann_benchmarks -f conda/environments/bench_ann_cuda-118_arch-x86_64.yaml -conda activate raft_ann_benchmarks -``` - -The above conda environment will also reduce the compile times as dependencies like FAISS will already be installed and not need to be compiled with `rapids-cmake`. - -### Compiling the Benchmarks - -After the needed dependencies are satisfied, the easiest way to compile ANN benchmarks is through the `build.sh` script in the root of the RAFT source code repository. The following will build the executables for all the support algorithms: -```bash -./build.sh bench-ann -``` - -You can limit the algorithms that are built by providing a semicolon-delimited list of executable names (each algorithm is suffixed with `_ANN_BENCH`): -```bash -./build.sh bench-ann -n --limit-bench-ann=HNSWLIB_ANN_BENCH;RAFT_IVF_PQ_ANN_BENCH -``` - -Available targets to use with `--limit-bench-ann` are: -- FAISS_GPU_IVF_FLAT_ANN_BENCH -- FAISS_GPU_IVF_PQ_ANN_BENCH -- FAISS_CPU_IVF_FLAT_ANN_BENCH -- FAISS_CPU_IVF_PQ_ANN_BENCH -- FAISS_GPU_FLAT_ANN_BENCH -- FAISS_CPU_FLAT_ANN_BENCH -- GGNN_ANN_BENCH -- HNSWLIB_ANN_BENCH -- RAFT_CAGRA_ANN_BENCH -- RAFT_IVF_PQ_ANN_BENCH -- RAFT_IVF_FLAT_ANN_BENCH - -By default, the `*_ANN_BENCH` executables program infer the dataset's datatype from the filename's extension. For example, an extension of `fbin` uses a `float` datatype, `f16bin` uses a `float16` datatype, extension of `i8bin` uses `int8_t` datatype, and `u8bin` uses `uint8_t` type. Currently, only `float`, `float16`, int8_t`, and `unit8_t` are supported. \ No newline at end of file diff --git a/docs/source/ann_benchmarks_dataset.md b/docs/source/ann_benchmarks_dataset.md deleted file mode 100644 index 26c1559504..0000000000 --- a/docs/source/ann_benchmarks_dataset.md +++ /dev/null @@ -1,63 +0,0 @@ -# ANN Benchmarks Datasets - -A dataset usually has 4 binary files containing database vectors, query vectors, ground truth neighbors and their corresponding distances. For example, Glove-100 dataset has files `base.fbin` (database vectors), `query.fbin` (query vectors), `groundtruth.neighbors.ibin` (ground truth neighbors), and `groundtruth.distances.fbin` (ground truth distances). The first two files are for index building and searching, while the other two are associated with a particular distance and are used for evaluation. - -The file suffixes `.fbin`, `.f16bin`, `.ibin`, `.u8bin`, and `.i8bin` denote that the data type of vectors stored in the file are `float32`, `float16`(a.k.a `half`), `int`, `uint8`, and `int8`, respectively. -These binary files are little-endian and the format is: the first 8 bytes are `num_vectors` (`uint32_t`) and `num_dimensions` (`uint32_t`), and the following `num_vectors * num_dimensions * sizeof(type)` bytes are vectors stored in row-major order. - -Some implementation can take `float16` database and query vectors as inputs and will have better performance. Use `script/fbin_to_f16bin.py` to transform dataset from `float32` to `float16` type. - -Commonly used datasets can be downloaded from two websites: -1. Million-scale datasets can be found at the [Data sets](https://github.com/erikbern/ann-benchmarks#data-sets) section of [`ann-benchmarks`](https://github.com/erikbern/ann-benchmarks). - - However, these datasets are in HDF5 format. Use `cpp/bench/ann/scripts/hdf5_to_fbin.py` to transform the format. A few Python packages are required to run it: - ```bash - pip3 install numpy h5py - ``` - The usage of this script is: - ```bash - $ cpp/bench/ann/scripts/hdf5_to_fbin.py - usage: scripts/hdf5_to_fbin.py [-n] .hdf5 - -n: normalize base/query set - outputs: .base.fbin - .query.fbin - .groundtruth.neighbors.ibin - .groundtruth.distances.fbin - ``` - So for an input `.hdf5` file, four output binary files will be produced. See previous section for an example of prepossessing GloVe dataset. - - Most datasets provided by `ann-benchmarks` use `Angular` or `Euclidean` distance. `Angular` denotes cosine distance. However, computing cosine distance reduces to computing inner product by normalizing vectors beforehand. In practice, we can always do the normalization to decrease computation cost, so it's better to measure the performance of inner product rather than cosine distance. The `-n` option of `hdf5_to_fbin.py` can be used to normalize the dataset. - -2. Billion-scale datasets can be found at [`big-ann-benchmarks`](http://big-ann-benchmarks.com). The ground truth file contains both neighbors and distances, thus should be split. A script is provided for this: - ```bash - $ cpp/bench/ann/scripts/split_groundtruth.pl - usage: script/split_groundtruth.pl input output_prefix - ``` - Take Deep-1B dataset as an example: - ```bash - pushd - cd cpp/bench/ann - mkdir -p data/deep-1B && cd data/deep-1B - # download manually "Ground Truth" file of "Yandex DEEP" - # suppose the file name is deep_new_groundtruth.public.10K.bin - ../../scripts/split_groundtruth.pl deep_new_groundtruth.public.10K.bin groundtruth - # two files 'groundtruth.neighbors.ibin' and 'groundtruth.distances.fbin' should be produced - popd - ``` - Besides ground truth files for the whole billion-scale datasets, this site also provides ground truth files for the first 10M or 100M vectors of the base sets. This mean we can use these billion-scale datasets as million-scale datasets. To facilitate this, an optional parameter `subset_size` for dataset can be used. See the next step for further explanation. - -## Generate ground truth - -If you have a dataset, but no corresponding ground truth file, then you can generate ground trunth using the `generate_groundtruth` utility. Example usage: - -```bash -# With existing query file -python -m raft_ann_bench.generate_groundtruth --dataset /dataset/base.fbin --output=groundtruth_dir --queries=/dataset/query.public.10K.fbin - -# With randomly generated queries -python -m raft_ann_bench.generate_groundtruth --dataset /dataset/base.fbin --output=groundtruth_dir --queries=random --n_queries=10000 - -# Using only a subset of the dataset. Define queries by randomly -# selecting vectors from the (subset of the) dataset. -python -m raft_ann_bench.generate_groundtruth --dataset /dataset/base.fbin --nrows=2000000 --output=groundtruth_dir --queries=random-choice --n_queries=10000 -``` \ No newline at end of file diff --git a/docs/source/ann_benchmarks_low_level.md b/docs/source/ann_benchmarks_low_level.md deleted file mode 100644 index 7ba13dec8d..0000000000 --- a/docs/source/ann_benchmarks_low_level.md +++ /dev/null @@ -1,219 +0,0 @@ -### Low-level Scripts and Executables -#### End-to-end Example -An end-to-end example (run from the RAFT source code root directory): -```bash -# (0) get raft sources -git clone https://github.com/rapidsai/raft.git -cd raft - -# (1) prepare a dataset -export PYTHONPATH=python/raft-ann-bench/src:$PYTHONPATH -python -m raft_ann_bench.get_dataset --dataset glove-100-angular --normalize - -# option --normalize is used here to normalize vectors so cosine distance is converted -# to inner product; don't use -n for l2 distance - -# (2) build index -$CONDA_PREFIX/bin/ann/RAFT_IVF_FLAT_ANN_BENCH \ - --data_prefix=datasets \ - --build \ - --benchmark_filter="raft_ivf_flat\..*" \ - python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-100-inner.json - -# (3) search -$CONDA_PREFIX/bin/ann/RAFT_IVF_FLAT_ANN_BENCH\ - --data_prefix=datasets \ - --benchmark_min_time=2s \ - --benchmark_out=ivf_flat_search.csv \ - --benchmark_out_format=csv \ - --benchmark_counters_tabular \ - --search \ - --benchmark_filter="raft_ivf_flat\..*" \ - python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-100-inner.json - - -# optional step: plot QPS-Recall figure using data in ivf_flat_search.csv with your favorite tool -``` - -##### Step 1: Prepare Dataset -Note: the preferred way to download and process smaller (million scale) datasets is to use the `get_dataset` script as demonstrated in the example above. - -A dataset usually has 4 binary files containing database vectors, query vectors, ground truth neighbors and their corresponding distances. For example, Glove-100 dataset has files `base.fbin` (database vectors), `query.fbin` (query vectors), `groundtruth.neighbors.ibin` (ground truth neighbors), and `groundtruth.distances.fbin` (ground truth distances). The first two files are for index building and searching, while the other two are associated with a particular distance and are used for evaluation. - -The file suffixes `.fbin`, `.f16bin`, `.ibin`, `.u8bin`, and `.i8bin` denote that the data type of vectors stored in the file are `float32`, `float16`(a.k.a `half`), `int`, `uint8`, and `int8`, respectively. -These binary files are little-endian and the format is: the first 8 bytes are `num_vectors` (`uint32_t`) and `num_dimensions` (`uint32_t`), and the following `num_vectors * num_dimensions * sizeof(type)` bytes are vectors stored in row-major order. - -Some implementation can take `float16` database and query vectors as inputs and will have better performance. Use `python/raft-ann-bench/src/raft_ann_bench/get_dataset/fbin_to_f16bin.py` to transform dataset from `float32` to `float16` type. - -Commonly used datasets can be downloaded from two websites: -1. Million-scale datasets can be found at the [Data sets](https://github.com/erikbern/ann-benchmarks#data-sets) section of [`ann-benchmarks`](https://github.com/erikbern/ann-benchmarks). - - However, these datasets are in HDF5 format. Use `python/raft-ann-bench/src/raft_ann_bench/get_dataset/fbin_to_f16bin.py/hdf5_to_fbin.py` to transform the format. A few Python packages are required to run it: - ```bash - pip3 install numpy h5py - ``` - The usage of this script is: - ```bash - $ cpp/bench/ann/scripts/hdf5_to_fbin.py - usage: scripts/hdf5_to_fbin.py [-n] .hdf5 - -n: normalize base/query set - outputs: .base.fbin - .query.fbin - .groundtruth.neighbors.ibin - .groundtruth.distances.fbin - ``` - So for an input `.hdf5` file, four output binary files will be produced. See previous section for an example of prepossessing GloVe dataset. - - Most datasets provided by `ann-benchmarks` use `Angular` or `Euclidean` distance. `Angular` denotes cosine distance. However, computing cosine distance reduces to computing inner product by normalizing vectors beforehand. In practice, we can always do the normalization to decrease computation cost, so it's better to measure the performance of inner product rather than cosine distance. The `-n` option of `hdf5_to_fbin.py` can be used to normalize the dataset. - -2. Billion-scale datasets can be found at [`big-ann-benchmarks`](http://big-ann-benchmarks.com). The ground truth file contains both neighbors and distances, thus should be split. A script is provided for this: - ```bash - $ python/raft-ann-bench/src/raft_ann_bench/split_groundtruth/split_groundtruth.pl - usage: split_groundtruth.pl input output_prefix - ``` - Take Deep-1B dataset as an example: - ```bash - pushd - cd cpp/bench/ann - mkdir -p data/deep-1B && cd data/deep-1B - # download manually "Ground Truth" file of "Yandex DEEP" - # suppose the file name is deep_new_groundtruth.public.10K.bin - /path/to/raft/python/raft-ann-bench/src/raft_ann_bench/split_groundtruth/split_groundtruth.pl deep_new_groundtruth.public.10K.bin groundtruth - # two files 'groundtruth.neighbors.ibin' and 'groundtruth.distances.fbin' should be produced - popd - ``` - Besides ground truth files for the whole billion-scale datasets, this site also provides ground truth files for the first 10M or 100M vectors of the base sets. This mean we can use these billion-scale datasets as million-scale datasets. To facilitate this, an optional parameter `subset_size` for dataset can be used. See the next step for further explanation. - - -##### Step 2: Build Index -An index is a data structure to facilitate searching. Different algorithms may use different data structures for their index. We can use `RAFT_IVF_FLAT_ANN_BENCH --build` to build an index and save it to disk. - -To run a benchmark executable, like `RAFT_IVF_FLAT_ANN_BENCH`, a JSON configuration file is required. Refer to [`cpp/bench/ann/conf/glove-100-inner.json`](../../cpp/cpp/bench/ann/conf/glove-100-inner.json) as an example. Configuration file has 3 sections: -* `dataset` section specifies the name and files of a dataset, and also the distance in use. Since the `*_ANN_BENCH` programs are for index building and searching, only `base_file` for database vectors and `query_file` for query vectors are needed. Ground truth files are for evaluation thus not needed. - - To use only a subset of the base dataset, an optional parameter `subset_size` can be specified. It means using only the first `subset_size` vectors of `base_file` as the base dataset. -* `search_basic_param` section specifies basic parameters for searching: - - `k` is the "k" in "k-nn", that is, the number of neighbors (or results) we want from the searching. -* `index` section specifies an array of configurations for index building and searching: - - `build_param` and `search_params` are parameters for building and searching, respectively. `search_params` is an array since we will search with different parameters to get different recall values. - - `file` is the file name of index. Building will save built index to this file, while searching will load this file. - - if `refine_ratio` is specified, refinement, as a post-processing step of search, will be done. It's for algorithms that compress vectors. For example, if `"refine_ratio" : 2` is set, 2`k` results are first computed, then exact distances of them are computed using original uncompressed vectors, and finally top `k` results among them are kept. - - -The usage of `*_ANN_BENCH` can be found by running `*_ANN_BENCH --help` on one of the executables: -```bash -$ ./cpp/build/*_ANN_BENCH --help -benchmark [--benchmark_list_tests={true|false}] - [--benchmark_filter=] - [--benchmark_min_time=`x` OR `s` ] - [--benchmark_min_warmup_time=] - [--benchmark_repetitions=] - [--benchmark_enable_random_interleaving={true|false}] - [--benchmark_report_aggregates_only={true|false}] - [--benchmark_display_aggregates_only={true|false}] - [--benchmark_format=] - [--benchmark_out=] - [--benchmark_out_format=] - [--benchmark_color={auto|true|false}] - [--benchmark_counters_tabular={true|false}] - [--benchmark_context==,...] - [--benchmark_time_unit={ns|us|ms|s}] - [--v=] - [--build|--search] - [--overwrite] - [--data_prefix=] - .json - -Note the non-standard benchmark parameters: - --build: build mode, will build index - --search: search mode, will search using the built index - one and only one of --build and --search should be specified - --overwrite: force overwriting existing index files - --data_prefix=: prepend to dataset file paths specified in the .json. - --override_kv=: override a build/search key one or more times multiplying the number of configurations; you can use this parameter multiple times to get the Cartesian product of benchmark configs. -``` -* `--build`: build index. -* `--search`: do the searching with built index. -* `--overwrite`: by default, the building mode skips building an index if it find out it already exists. This is useful when adding more configurations to the config; only new indices are build without the need to specify an elaborate filtering regex. By supplying `overwrite` flag, you disable this behavior; all indices are build regardless whether they are already stored on disk. -* `--data_prefix`: prepend an arbitrary path to the data file paths. By default, it is equal to `data`. Note, this does not apply to index file paths. -* `--override_kv`: override a build/search key one or more times multiplying the number of configurations. - -In addition to these ANN-specific flags, you can use all of the standard google benchmark flags. Some of the useful flags: -* `--benchmark_filter`: specify subset of benchmarks to run -* `--benchmark_out`, `--benchmark_out_format`: store the output to a file -* `--benchmark_list_tests`: check the available configurations -* `--benchmark_min_time`: specify the minimum duration or number of iterations per case to improve accuracy of the benchmarks. - -Refer to the google benchmark [user guide](https://github.com/google/benchmark/blob/main/docs/user_guide.md#command-line) for more information about the command-line usage. - -##### Step 3: Searching -Use the `--search` flag on any of the `*_ANN_BENCH` executables. Other options are the same as in step 2. - -## Adding a new ANN algorithm -Implementation of a new algorithm should be a class that inherits `class ANN` (defined in `cpp/bench/ann/src/ann.h`) and implements all the pure virtual functions. - -In addition, it should define two `struct`s for building and searching parameters. The searching parameter class should inherit `struct ANN::AnnSearchParam`. Take `class HnswLib` as an example, its definition is: -```c++ -template -class HnswLib : public ANN { -public: - struct BuildParam { - int M; - int ef_construction; - int num_threads; - }; - - using typename ANN::AnnSearchParam; - struct SearchParam : public AnnSearchParam { - int ef; - int num_threads; - }; - - // ... -}; -``` - -The benchmark program uses JSON configuration file. To add the new algorithm to the benchmark, need be able to specify `build_param`, whose value is a JSON object, and `search_params`, whose value is an array of JSON objects, for this algorithm in configuration file. Still take the configuration for `HnswLib` as an example: -```json -{ - "name" : "...", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "/path/to/file", - "search_params" : [ - {"ef":10, "numThreads":1}, - {"ef":20, "numThreads":1}, - {"ef":40, "numThreads":1} - ] -}, -``` - -How to interpret these JSON objects is totally left to the implementation and should be specified in `cpp/bench/ann/src/factory.cuh`: -1. First, add two functions for parsing JSON object to `struct BuildParam` and `struct SearchParam`, respectively: - ```c++ - template - void parse_build_param(const nlohmann::json& conf, - typename cuann::HnswLib::BuildParam& param) { - param.ef_construction = conf.at("efConstruction"); - param.M = conf.at("M"); - if (conf.contains("numThreads")) { - param.num_threads = conf.at("numThreads"); - } - } - - template - void parse_search_param(const nlohmann::json& conf, - typename cuann::HnswLib::SearchParam& param) { - param.ef = conf.at("ef"); - if (conf.contains("numThreads")) { - param.num_threads = conf.at("numThreads"); - } - } - ``` - -2. Next, add corresponding `if` case to functions `create_algo()` and `create_search_param()` by calling parsing functions. The string literal in `if` condition statement must be the same as the value of `algo` in configuration file. For example, - ```c++ - // JSON configuration file contains a line like: "algo" : "hnswlib" - if (algo == "hnswlib") { - // ... - } - ``` diff --git a/docs/source/ann_benchmarks_param_tuning.md b/docs/source/ann_benchmarks_param_tuning.md deleted file mode 100644 index afb4ed18ea..0000000000 --- a/docs/source/ann_benchmarks_param_tuning.md +++ /dev/null @@ -1,178 +0,0 @@ -# ANN Benchmarks Parameter Tuning Guide - -This guide outlines the various parameter settings that can be specified in [RAFT ANN Benchmark](raft_ann_benchmarks.md) json configuration files and explains the impact they have on corresponding algorithms to help inform their settings for benchmarking across desired levels of recall. - - -## RAFT Indexes - -### `raft_brute_force` - -Use RAFT brute-force index for exact search. Brute-force has no further build or search parameters. - -### `raft_ivf_flat` - -IVF-flat uses an inverted-file index, which partitions the vectors into a series of clusters, or lists, storing them in an interleaved format which is optimized for fast distance computation. The searching of an IVF-flat index reduces the total vectors in the index to those within some user-specified nearest clusters called probes. - -IVF-flat is a simple algorithm which won't save any space, but it provides competitive search times even at higher levels of recall. - -| Parameter | Type | Required | Data Type | Default | Description | -|----------------------|------------------|----------|----------------------------|----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `nlist` | `build` | Y | Positive Integer >0 | | Number of clusters to partition the vectors into. Larger values will put less points into each cluster but this will impact index build time as more clusters need to be trained. | -| `niter` | `build` | N | Positive Integer >0 | 20 | Number of clusters to partition the vectors into. Larger values will put less points into each cluster but this will impact index build time as more clusters need to be trained. | -| `ratio` | `build` | N | Positive Integer >0 | 2 | `1/ratio` is the number of training points which should be used to train the clusters. | -| `dataset_memory_type` | `build` | N | ["device", "host", "mmap"] | "mmap" | What memory type should the dataset reside? | -| `query_memory_type` | `search` | N | ["device", "host", "mmap"] | "device | What memory type should the queries reside? | -| `nprobe` | `search` | Y | Positive Integer >0 | | The closest number of clusters to search for each query vector. Larger values will improve recall but will search more points in the index. | - - -### `raft_ivf_pq` - -IVF-pq is an inverted-file index, which partitions the vectors into a series of clusters, or lists, in a similar way to IVF-flat above. The difference is that IVF-PQ uses product quantization to also compress the vectors, giving the index a smaller memory footprint. Unfortunately, higher levels of compression can also shrink recall, which a refinement step can improve when the original vectors are still available. - -| Parameter | Type | Required | Data Type | Default | Description | -|------------------------|----------------|---|----------------------------------|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `nlist` | `build` | Y | Positive Integer >0 | | Number of clusters to partition the vectors into. Larger values will put less points into each cluster but this will impact index build time as more clusters need to be trained. | -| `niter` | `build` | N | Positive Integer >0 | 20 | Number of k-means iterations to use when training the clusters. | -| `ratio` | `build` | N | Positive Integer >0 | 2 | `1/ratio` is the number of training points which should be used to train the clusters. | -| `pq_dim` | `build` | N | Positive Integer. Multiple of 8. | 0 | Dimensionality of the vector after product quantization. When 0, a heuristic is used to select this value. `pq_dim` * `pq_bits` must be a multiple of 8. | -| `pq_bits` | `build` | N | Positive Integer. [4-8] | 8 | Bit length of the vector element after quantization. | -| `codebook_kind` | `build` | N | ["cluster", "subspace"] | "subspace" | Type of codebook. See the [API docs](https://docs.rapids.ai/api/raft/nightly/cpp_api/neighbors_ivf_pq/#_CPPv412codebook_gen) for more detail | -| `dataset_memory_type` | `build` | N | ["device", "host", "mmap"] | "host" | What memory type should the dataset reside? | -| `query_memory_type` | `search` | N | ["device", "host", "mmap"] | "device | What memory type should the queries reside? | -| `nprobe` | `search` | Y | Positive Integer >0 | | The closest number of clusters to search for each query vector. Larger values will improve recall but will search more points in the index. | -| `internalDistanceDtype` | `search` | N | [`float`, `half`] | `half` | The precision to use for the distance computations. Lower precision can increase performance at the cost of accuracy. | -| `smemLutDtype` | `search` | N | [`float`, `half`, `fp8`] | `half` | The precision to use for the lookup table in shared memory. Lower precision can increase performance at the cost of accuracy. | -| `refine_ratio` | `search` | N| Positive Number >=1 | 1 | `refine_ratio * k` nearest neighbors are queried from the index initially and an additional refinement step improves recall by selecting only the best `k` neighbors. | - - -### `raft_cagra` -CAGRA uses a graph-based index, which creates an intermediate, approximate kNN graph using IVF-PQ and then further refining and optimizing to create a final kNN graph. This kNN graph is used by CAGRA as an index for search. - -| Parameter | Type | Required | Data Type | Default | Description | -|-----------------------------|----------------|----------|----------------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `graph_degree` | `build` | N | Positive Integer >0 | 64 | Degree of the final kNN graph index. | -| `intermediate_graph_degree` | `build` | N | Positive Integer >0 | 128 | Degree of the intermediate kNN graph. | -| `graph_build_algo` | `build` | N | ["IVF_PQ", "NN_DESCENT"] | "IVF_PQ" | Algorithm to use for search | -| `dataset_memory_type` | `build` | N | ["device", "host", "mmap"] | "mmap" | What memory type should the dataset reside while constructing the index? | -| `query_memory_type` | `search` | N | ["device", "host", "mmap"] | "device | What memory type should the queries reside? | -| `itopk` | `search_wdith` | N | Positive Integer >0 | 64 | Number of intermediate search results retained during the search. Higher values improve search accuracy at the cost of speed. | -| `search_width` | `search` | N | Positive Integer >0 | 1 | Number of graph nodes to select as the starting point for the search in each iteration. | -| `max_iterations` | `search` | N | Integer >=0 | 0 | Upper limit of search iterations. Auto select when 0. | -| `algo` | `search` | N | string | "auto" | Algorithm to use for search. Possible values: {"auto", "single_cta", "multi_cta", "multi_kernel"} | -| `graph_memory_type` | `search` | N | string | "device" | Memory type to store gaph. Must be one of {"device", "host_pinned", "host_huge_page"}. | -| `internal_dataset_memory_type` | `search` | N | string | "device" | Memory type to store dataset in the index. Must be one of {"device", "host_pinned", "host_huge_page"}. | - -The `graph_memory_type` or `internal_dataset_memory_type` options can be useful for large datasets that do not fit the device memory. Setting `internal_dataset_memory_type` other than `device` has negative impact on search speed. Using `host_huge_page` option is only supported on systems with Heterogeneous Memory Management or on platforms that natively support GPU access to system allocated memory, for example Grace Hopper. - -To fine tune CAGRA index building we can customize IVF-PQ index builder options using the following settings. These take effect only if `graph_build_algo == "IVF_PQ"`. It is recommended to experiment using a separate IVF-PQ index to find the config that gives the largest QPS for large batch. Recall does not need to be very high, since CAGRA further optimizes the kNN neighbor graph. Some of the default values are derived from the dataset size which is assumed to be [n_vecs, dim]. - -| Parameter | Type | Required | Data Type | Default | Description | -|------------------------|----------------|---|----------------------------------|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `ivf_pq_build_nlist` | `build` | N | Positive Integer >0 | n_vecs / 2500 | Number of clusters to partition the vectors into. Larger values will put less points into each cluster but this will impact index build time as more clusters need to be trained. | -| `ivf_pq_build_niter` | `build` | N | Positive Integer >0 | 25 | Number of k-means iterations to use when training the clusters. | -| `ivf_pq_build_ratio` | `build` | N | Positive Integer >0 | 10 | `1/ratio` is the number of training points which should be used to train the clusters. | -| `ivf_pq_build_pq_dim` | `build` | N | Positive Integer. Multiple of 8. | dim/2 rounded up to 8 | Dimensionality of the vector after product quantization. When 0, a heuristic is used to select this value. `pq_dim` * `pq_bits` must be a multiple of 8. | -| `ivf_pq_build_pq_bits` | `build` | N | Positive Integer. [4-8] | 8 | Bit length of the vector element after quantization. | -| `ivf_pq_build_codebook_kind` | `build` | N | ["cluster", "subspace"] | "subspace" | Type of codebook. See the [API docs](https://docs.rapids.ai/api/raft/nightly/cpp_api/neighbors_ivf_pq/#_CPPv412codebook_gen) for more detail | -| `ivf_pq_search_nprobe` | `build` | N | Positive Integer >0 | min(2*dim, nlist) | The closest number of clusters to search for each query vector. | -| `ivf_pq_search_internalDistanceDtype` | `build` | N | [`float`, `half`] | `fp8` | The precision to use for the distance computations. Lower precision can increase performance at the cost of accuracy. | -| `ivf_pq_search_smemLutDtype` | `build` | N | [`float`, `half`, `fp8`] | `half` | The precision to use for the lookup table in shared memory. Lower precision can increase performance at the cost of accuracy. | -| `ivf_pq_search_refine_ratio` | `build` | N| Positive Number >=1 | 2 | `refine_ratio * k` nearest neighbors are queried from the index initially and an additional refinement step improves recall by selecting only the best `k` neighbors. | - -Alternatively, if `graph_build_algo == "NN_DESCENT"`, then we can customize the following parameters - -| Parameter | Type | Required | Data Type | Default | Description | -|-----------------------------|----------------|----------|----------------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `nn_descent_niter` | `build` | N | Positive Integer>0 | 20 | Number of NN Descent iterations. | -| `nn_descent_intermediate_graph_degree` | `build` | N | Positive Integer>0 | `intermediate_graph_degree` * 1.5 | Intermadiate graph degree during NN descent iterations | -| `nn_descent_max_iterations` | `build` | N | Positive Integer>0 | 20 | Alias for `nn_descent_niter` | -| `nn_descent_termination_threshold` | `build` | N | Positive float>0 | 0.0001 | Termination threshold for NN descent. | - -### `raft_cagra_hnswlib` -This is a benchmark that enables interoperability between `CAGRA` built `HNSW` search. It uses the `CAGRA` built graph as the base layer of an `hnswlib` index to search queries only within the base layer (this is enabled with a simple patch to `hnswlib`). - -`build` : Same as `build` of [CAGRA](#raft-cagra) - -`search` : Same as `search` of [hnswlib](#hnswlib) - -## FAISS Indexes - -### `faiss_gpu_flat` - -Use FAISS flat index on the GPU, which performs an exact search using brute-force and doesn't have any further build or search parameters. - -### `faiss_gpu_ivf_flat` - -IVF-flat uses an inverted-file index, which partitions the vectors into a series of clusters, or lists, storing them in an interleaved format which is optimized for fast distance computation. The searching of an IVF-flat index reduces the total vectors in the index to those within some user-specified nearest clusters called probes. - -IVF-flat is a simple algorithm which won't save any space, but it provides competitive search times even at higher levels of recall. - -| Parameter | Type | Required | Data Type | Default | Description | -|-----------|----------------|----------|---------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `nlists` | `build` | Y | Positive Integer >0 | | Number of clusters to partition the vectors into. Larger values will put less points into each cluster but this will impact index build time as more clusters need to be trained. | -| `ratio` | `build` | N | Positive Integer >0 | 2 | `1/ratio` is the number of training points which should be used to train the clusters. | -| `nprobe` | `search` | Y | Positive Integer >0 | | The closest number of clusters to search for each query vector. Larger values will improve recall but will search more points in the index. | - -### `faiss_gpu_ivf_pq` - -IVF-pq is an inverted-file index, which partitions the vectors into a series of clusters, or lists, in a similar way to IVF-flat above. The difference is that IVF-PQ uses product quantization to also compress the vectors, giving the index a smaller memory footprint. Unfortunately, higher levels of compression can also shrink recall, which a refinement step can improve when the original vectors are still available. - -| Parameter | Type | Required | Data Type | Default | Description | -|------------------|----------------|----------|----------------------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `nlist` | `build` | Y | Positive Integer >0 | | Number of clusters to partition the vectors into. Larger values will put less points into each cluster but this will impact index build time as more clusters need to be trained. | -| `ratio` | `build` | N | Positive Integer >0 | 2 | `1/ratio` is the number of training points which should be used to train the clusters. | -| `M_ratio` | `build` | Y | Positive Integer Power of 2 [8-64] | | Ratio of numbeer of chunks or subquantizers for each vector. Computed by `dims` / `M_ratio` | -| `usePrecomputed` | `build` | N | Boolean. Default=`false` | `false` | Use pre-computed lookup tables to speed up search at the cost of increased memory usage. | -| `useFloat16` | `build` | N | Boolean. Default=`false` | `false` | Use half-precision floats for clustering step. | -| `nprobe` | `search` | Y | Positive Integer >0 | | The closest number of clusters to search for each query vector. Larger values will improve recall but will search more points in the index. | -| `refine_ratio` | `search` | N| Positive Number >=1 | 1 | `refine_ratio * k` nearest neighbors are queried from the index initially and an additional refinement step improves recall by selecting only the best `k` neighbors. | - -### `faiss_cpu_flat` - -Use FAISS flat index on the CPU, which performs an exact search using brute-force and doesn't have any further build or search parameters. - - -| Parameter | Type | Required | Data Type | Default | Description | -|-----------|----------------|----------|---------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `numThreads` | `search` | N | Positive Integer >0 | 1 | Number of threads to use for queries. | - -### `faiss_cpu_ivf_flat` - -Use FAISS IVF-Flat index on CPU - -| Parameter | Type | Required | Data Type | Default | Description | -|----------|----------------|----------|---------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `nlist` | `build` | Y | Positive Integer >0 | | Number of clusters to partition the vectors into. Larger values will put less points into each cluster but this will impact index build time as more clusters need to be trained. | -| `ratio` | `build` | N | Positive Integer >0 | 2 | `1/ratio` is the number of training points which should be used to train the clusters. | -| `nprobe` | `search` | Y | Positive Integer >0 | | The closest number of clusters to search for each query vector. Larger values will improve recall but will search more points in the index. | -| `numThreads` | `search` | N | Positive Integer >0 | 1 | Number of threads to use for queries. | - -### `faiss_cpu_ivf_pq` - -Use FAISS IVF-PQ index on CPU - -| Parameter | Type | Required | Data Type | Default | Description | -|------------------|----------------|----------|------------------------------------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `nlist` | `build` | Y | Positive Integer >0 | | Number of clusters to partition the vectors into. Larger values will put less points into each cluster but this will impact index build time as more clusters need to be trained. | -| `ratio` | `build` | N | Positive Integer >0 | 2 | `1/ratio` is the number of training points which should be used to train the clusters. | -| `M` | `build` | Y | Positive Integer Power of 2 [8-64] | | Number of chunks or subquantizers for each vector. | -| `usePrecomputed` | `build` | N | Boolean. Default=`false` | `false` | Use pre-computed lookup tables to speed up search at the cost of increased memory usage. | -| `bitsPerCode` | `build` | N | Positive Integer [4-8] | 8 | Number of bits to use for each code. | -| `nprobe` | `search` | Y | Positive Integer >0 | | The closest number of clusters to search for each query vector. Larger values will improve recall but will search more points in the index. | -| `refine_ratio` | `search` | N| Positive Number >=1 | 1 | `refine_ratio * k` nearest neighbors are queried from the index initially and an additional refinement step improves recall by selecting only the best `k` neighbors. | -| `numThreads` | `search` | N | Positive Integer >0 | 1 | Number of threads to use for queries. | - - -## HNSW - -### `hnswlib` - -| Parameter | Type | Required | Data Type | Default | Description | -|------------------|-----------|----------|--------------------------------------|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `efConstruction` | `build` | Y | Positive Integer >0 | | Controls index time and accuracy. Bigger values increase the index quality. At some point, increasing this will no longer improve the quality. | -| `M` | `build` | Y | Positive Integer often between 2-100 | | Number of bi-directional links create for every new element during construction. Higher values work for higher intrinsic dimensionality and/or high recall, low values can work for datasets with low intrinsic dimensionality and/or low recalls. Also affects the algorithm's memory consumption. | -| `numThreads` | `build` | N | Positive Integer >0 | 1 | Number of threads to use to build the index. | -| `ef` | `search` | Y | Positive Integer >0 | | Size of the dynamic list for the nearest neighbors used for search. Higher value leads to more accurate but slower search. Cannot be lower than `k`. | -| `numThreads` | `search` | N | Positive Integer >0 | 1 | Number of threads to use for queries. | - -Please refer to [HNSW algorithm parameters guide](https://github.com/nmslib/hnswlib/blob/master/ALGO_PARAMS.md) from `hnswlib` to learn more about these arguments. \ No newline at end of file diff --git a/docs/source/build.md b/docs/source/build.md index b9a1832b02..3d059d5a69 100644 --- a/docs/source/build.md +++ b/docs/source/build.md @@ -1,6 +1,6 @@ # Installation -RAFT currently provides libraries for C++ and Python. The C++ libraries, including the header-only and optional shared library, can be installed with Conda. +RAFT currently provides libraries for C++ and Python. The C++ libraries, including the header-only and optional shared library, can be installed with Conda. Both the C++ and Python APIs require CMake to build from source. @@ -34,8 +34,6 @@ The easiest way to install RAFT is through conda and several packages are provid - `libraft` (optional) C++ shared library containing pre-compiled template instantiations and runtime API. - `pylibraft` (optional) Python library - `raft-dask` (optional) Python library for deployment of multi-node multi-GPU algorithms that use the RAFT `raft::comms` abstraction layer in Dask clusters. -- `raft-ann-bench` (optional) Benchmarking tool for easily producing benchmarks that compare RAFT's vector search algorithms against other state-of-the-art implementations. -- `raft-ann-bench-cpu` (optional) Reproducible benchmarking tool similar to above, but doesn't require CUDA to be installed on the machine. Can be used to test in environments with competitive CPUs. Use the following command, depending on your CUDA version, to install all of the RAFT packages with conda (replace `rapidsai` with `rapidsai-nightly` to install more up-to-date but less stable nightly packages). `mamba` is preferred over the `conda` command. ```bash @@ -60,7 +58,7 @@ If installing the C++ APIs Please see [using libraft](https://docs.rapids.ai/api ## Installing Python through Pip -`pylibraft` and `raft-dask` both have packages that can be [installed through pip](https://rapids.ai/pip.html#install). +`pylibraft` and `raft-dask` both have packages that can be [installed through pip](https://rapids.ai/pip.html#install). For CUDA 11 packages: ```bash @@ -74,7 +72,7 @@ pip install pylibraft-cu12 --extra-index-url=https://pypi.nvidia.com pip install raft-dask-cu12 --extra-index-url=https://pypi.nvidia.com ``` -These packages statically build RAFT's pre-compiled instantiations, so the C++ headers and pre-compiled shared library won't be readily available to use in your code. +These packages statically build RAFT's pre-compiled instantiations, so the C++ headers and pre-compiled shared library won't be readily available to use in your code. ## Building C++ and Python from source @@ -124,7 +122,7 @@ The recommended way to build and install RAFT from source is to use the `build.s `build.sh` uses [rapids-cmake](https://github.com/rapidsai/rapids-cmake), which will automatically download any dependencies which are not already installed. It's important to note that while all the headers will be installed and available, some parts of the RAFT API depend on libraries like CUTLASS, which will need to be explicitly enabled in `build.sh`. -The following example will download the needed dependencies and install the RAFT headers into `$INSTALL_PREFIX/include/raft`. +The following example will download the needed dependencies and install the RAFT headers into `$INSTALL_PREFIX/include/raft`. ```bash ./build.sh libraft ``` @@ -201,8 +199,6 @@ It can take sometime to compile all of the benchmarks. You can build individual ./build.sh libraft bench-prims -n --limit-bench=NEIGHBORS_PRIMS_BENCH;DISTANCE_PRIMS_BENCH;LINALG_PRIMS_BENCH ``` -In addition to microbenchmarks for individual primitives, RAFT contains a reproducible benchmarking tool for evaluating the performance of RAFT's vector search algorithms against the existing state-of-the-art. Please refer to the [RAFT ANN Benchmarks](https://docs.rapids.ai/api/raft/nightly/raft_ann_benchmarks/) guide for more information on this tool. - ### Python libraries The Python libraries can be built and installed using the `build.sh` script: @@ -242,7 +238,7 @@ The Python packages can also be uninstalled using the `build.sh` script: ### Using CMake directly -When building RAFT from source, the `build.sh` script offers a nice wrapper around the `cmake` commands to ease the burdens of manually configuring the various available cmake options. When more fine-grained control over the CMake configuration is desired, the `cmake` command can be invoked directly as the below example demonstrates. +When building RAFT from source, the `build.sh` script offers a nice wrapper around the `cmake` commands to ease the burdens of manually configuring the various available cmake options. When more fine-grained control over the CMake configuration is desired, the `cmake` command can be invoked directly as the below example demonstrates. The `CMAKE_INSTALL_PREFIX` installs RAFT into a specific location. The example below installs RAFT into the current Conda environment: ```bash @@ -259,7 +255,6 @@ RAFT's CMake has the following configurable flags available: |---------------------------------|----------------------| --- |------------------------------------------------------------------------------| | BUILD_TESTS | ON, OFF | ON | Compile Googletests | | BUILD_PRIMS_BENCH | ON, OFF | OFF | Compile benchmarks | -| BUILD_ANN_BENCH | ON, OFF | OFF | Compile end-to-end ANN benchmarks | | CUDA_ENABLE_KERNELINFO | ON, OFF | OFF | Enables `kernelinfo` in nvcc. This is useful for `compute-sanitizer` | | CUDA_ENABLE_LINEINFO | ON, OFF | OFF | Enable the -lineinfo option for nvcc | | CUDA_STATIC_RUNTIME | ON, OFF | OFF | Statically link the CUDA runtime | @@ -267,10 +262,10 @@ RAFT's CMake has the following configurable flags available: | DETECT_CONDA_ENV | ON, OFF | ON | Enable detection of conda environment for dependencies | | raft_FIND_COMPONENTS | compiled distributed | | Configures the optional components as a space-separated list | | RAFT_COMPILE_LIBRARY | ON, OFF | ON if either BUILD_TESTS or BUILD_PRIMS_BENCH is ON; otherwise OFF | Compiles all `libraft` shared libraries (these are required for Googletests) | -| RAFT_ENABLE_CUBLAS_DEPENDENCY | ON, OFF | ON | Link against cublas library in `raft::raft` | -| RAFT_ENABLE_CUSOLVER_DEPENDENCY | ON, OFF | ON | Link against cusolver library in `raft::raft` | -| RAFT_ENABLE_CUSPARSE_DEPENDENCY | ON, OFF | ON | Link against cusparse library in `raft::raft` | -| RAFT_ENABLE_CUSOLVER_DEPENDENCY | ON, OFF | ON | Link against curand library in `raft::raft` | +| RAFT_ENABLE_CUBLAS_DEPENDENCY | ON, OFF | ON | Link against cublas library in `raft::raft` | +| RAFT_ENABLE_CUSOLVER_DEPENDENCY | ON, OFF | ON | Link against cusolver library in `raft::raft` | +| RAFT_ENABLE_CUSPARSE_DEPENDENCY | ON, OFF | ON | Link against cusparse library in `raft::raft` | +| RAFT_ENABLE_CUSOLVER_DEPENDENCY | ON, OFF | ON | Link against curand library in `raft::raft` | | RAFT_NVTX | ON, OFF | OFF | Enable NVTX Markers | ### Build documentation @@ -316,4 +311,4 @@ The `raft::raft` CMake target is made available when including RAFT into your CM |-------------|---------------------|----------------------------------------------------------|----------------------------------------| | n/a | `raft::raft` | Full RAFT header library | CUDA toolkit, RMM, NVTX, CCCL, CUTLASS | | compiled | `raft::compiled` | Pre-compiled template instantiations and runtime library | raft::raft | -| distributed | `raft::distributed` | Dependencies for `raft::comms` APIs | raft::raft, UCX, NCCL \ No newline at end of file +| distributed | `raft::distributed` | Dependencies for `raft::comms` APIs | raft::raft, UCX, NCCL diff --git a/docs/source/index.rst b/docs/source/index.rst index bee0e948ff..46ebd1b737 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -63,7 +63,6 @@ While not exhaustive, the following general categories help summarize the accele pylibraft_api.rst using_libraft.md vector_search_tutorial.md - raft_ann_benchmarks.md raft_dask_api.rst using_raft_comms.rst developer_guide.md diff --git a/docs/source/raft_ann_benchmarks.md b/docs/source/raft_ann_benchmarks.md deleted file mode 100644 index 12a94e45ce..0000000000 --- a/docs/source/raft_ann_benchmarks.md +++ /dev/null @@ -1,597 +0,0 @@ -# RAFT ANN Benchmarks - -This project provides a benchmark program for various ANN search implementations. It's especially suitable for comparing GPU implementations as well as comparing GPU against CPU. - -> [!IMPORTANT] -> The vector search and clustering algorithms in RAFT are being migrated to a new library dedicated to vector search called [cuVS](https://github.com/rapidsai/cuvs). As a result, `raft-ann-bench` is being migrated to `cuvs-bench` and will be removed from RAFT altogether in the 24.12 (December) release. - - -## Table of Contents - -- [Installing the benchmarks](#installing-the-benchmarks) - - [Conda](#conda) - - [Docker](#docker) -- [How to run the benchmarks](#how-to-run-the-benchmarks) - - [Step 1: prepare dataset](#step-1-prepare-dataset) - - [Step 2: build and search index](#step-2-build-and-search-index) - - [Step 3: data export](#step-3-data-export) - - [Step 4: plot results](#step-4-plot-results) -- [Running the benchmarks](#running-the-benchmarks) - - [End to end: small-scale (<1M to 10M)](#end-to-end-small-scale-benchmarks-1m-to-10m) - - [End to end: large-scale (>10M)](#end-to-end-large-scale-benchmarks-10m-vectors) - - [Running with Docker containers](#running-with-docker-containers) - - [Evaluating the results](#evaluating-the-results) -- [Creating and customizing dataset configurations](#creating-and-customizing-dataset-configurations) -- [Adding a new ANN algorithm](#adding-a-new-ann-algorithm) -- [Parameter tuning guide](https://docs.rapids.ai/api/raft/nightly/ann_benchmarks_param_tuning/) -- [Wiki-all RAG/LLM Dataset](https://docs.rapids.ai/api/raft/nightly/wiki_all_dataset/) - -## Installing the benchmarks - -There are two main ways pre-compiled benchmarks are distributed: - -- [Conda](#Conda): For users not using containers but want an easy to install and use Python package. Pip wheels are planned to be added as an alternative for users that cannot use conda and prefer to not use containers. -- [Docker](#Docker): Only needs docker and [NVIDIA docker](https://github.com/NVIDIA/nvidia-docker) to use. Provides a single docker run command for basic dataset benchmarking, as well as all the functionality of the conda solution inside the containers. - -## Conda - -If containers are not an option or not preferred, the easiest way to install the ANN benchmarks is through conda. We provide packages for GPU enabled systems, as well for systems without a GPU. We suggest using mamba as it generally leads to a faster install time: - -```bash - -mamba create --name raft_ann_benchmarks -conda activate raft_ann_benchmarks - -# to install GPU package: -mamba install -c rapidsai -c conda-forge -c nvidia raft-ann-bench= cuda-version=11.8* - -# to install CPU package for usage in CPU-only systems: -mamba install -c rapidsai -c conda-forge raft-ann-bench-cpu -``` - -The channel `rapidsai` can easily be substituted `rapidsai-nightly` if nightly benchmarks are desired. The CPU package currently allows to run the HNSW benchmarks. - -Please see the [build instructions](ann_benchmarks_build.md) to build the benchmarks from source. - -## Docker - -We provide images for GPU enabled systems, as well as systems without a GPU. The following images are available: - -- `raft-ann-bench`: Contains GPU and CPU benchmarks, can run all algorithms supported. Will download million-scale datasets as required. Best suited for users that prefer a smaller container size for GPU based systems. Requires the NVIDIA Container Toolkit to run GPU algorithms, can run CPU algorithms without it. -- `raft-ann-bench-datasets`: Contains the GPU and CPU benchmarks with million-scale datasets already included in the container. Best suited for users that want to run multiple million scale datasets already included in the image. -- `raft-ann-bench-cpu`: Contains only CPU benchmarks with minimal size. Best suited for users that want the smallest containers to reproduce benchmarks on systems without a GPU. - -Nightly images are located in [dockerhub](https://hub.docker.com/r/rapidsai/raft-ann-bench/tags), meanwhile release (stable) versions are located in [NGC](https://hub.docker.com/r/rapidsai/raft-ann-bench), starting with release 23.12. - -- The following command pulls the nightly container for python version 10, cuda version 12, and RAFT version 23.10: - -```bash -docker pull rapidsai/raft-ann-bench:24.12a-cuda12.0-py3.10 #substitute raft-ann-bench for the exact desired container. -``` - -The CUDA and python versions can be changed for the supported values: - -Supported CUDA versions: 11.2 and 12.0 -Supported Python versions: 3.9 and 3.10. - -You can see the exact versions as well in the dockerhub site: - -- [RAFT ANN Benchmark images](https://hub.docker.com/r/rapidsai/raft-ann-bench/tags) -- [RAFT ANN Benchmark with datasets preloaded images](https://hub.docker.com/r/rapidsai/raft-ann-bench-cpu/tags) -- [RAFT ANN Benchmark CPU only images](https://hub.docker.com/r/rapidsai/raft-ann-bench-datasets/tags) - -**Note:** GPU containers use the CUDA toolkit from inside the container, the only requirement is a driver installed on the host machine that supports that version. So, for example, CUDA 11.8 containers can run in systems with a CUDA 12.x capable driver. Please also note that the Nvidia-Docker runtime from the [Nvidia Container Toolkit](https://github.com/NVIDIA/nvidia-docker) is required to use GPUs inside docker containers. - -[//]: # (- The following command (only available after RAPIDS 23.10 release) pulls the container:) - -[//]: # () -[//]: # (```bash) - -[//]: # (docker pull nvcr.io/nvidia/rapidsai/raft-ann-bench:24.12-cuda11.8-py3.10 #substitute raft-ann-bench for the exact desired container.) - -[//]: # (```) - -## How to run the benchmarks - -We provide a collection of lightweight Python scripts to run the benchmarks. There are 4 general steps to running the benchmarks and visualizing the results. -1. Prepare Dataset -2. Build Index and Search Index -3. Data Export -4. Plot Results - -### Step 1: Prepare Dataset -The script `raft_ann_bench.get_dataset` will download and unpack the dataset in directory -that the user provides. As of now, only million-scale datasets are supported by this -script. For more information on [datasets and formats](ann_benchmarks_dataset.md). - -The usage of this script is: -```bash -usage: get_dataset.py [-h] [--name NAME] [--dataset-path DATASET_PATH] [--normalize] - -options: - -h, --help show this help message and exit - --dataset DATASET dataset to download (default: glove-100-angular) - --dataset-path DATASET_PATH - path to download dataset (default: ${RAPIDS_DATASET_ROOT_DIR}) - --normalize normalize cosine distance to inner product (default: False) -``` - -When option `normalize` is provided to the script, any dataset that has cosine distances -will be normalized to inner product. So, for example, the dataset `glove-100-angular` -will be written at location `datasets/glove-100-inner/`. - -### Step 2: Build and Search Index -The script `raft_ann_bench.run` will build and search indices for a given dataset and its -specified configuration. - -The usage of the script `raft_ann_bench.run` is: -```bash -usage: __main__.py [-h] [--subset-size SUBSET_SIZE] [-k COUNT] [-bs BATCH_SIZE] [--dataset-configuration DATASET_CONFIGURATION] [--configuration CONFIGURATION] [--dataset DATASET] - [--dataset-path DATASET_PATH] [--build] [--search] [--algorithms ALGORITHMS] [--groups GROUPS] [--algo-groups ALGO_GROUPS] [-f] [-m SEARCH_MODE] - -options: - -h, --help show this help message and exit - --subset-size SUBSET_SIZE - the number of subset rows of the dataset to build the index (default: None) - -k COUNT, --count COUNT - the number of nearest neighbors to search for (default: 10) - -bs BATCH_SIZE, --batch-size BATCH_SIZE - number of query vectors to use in each query trial (default: 10000) - --dataset-configuration DATASET_CONFIGURATION - path to YAML configuration file for datasets (default: None) - --configuration CONFIGURATION - path to YAML configuration file or directory for algorithms Any run groups found in the specified file/directory will automatically override groups of the same name - present in the default configurations, including `base` (default: None) - --dataset DATASET name of dataset (default: glove-100-inner) - --dataset-path DATASET_PATH - path to dataset folder, by default will look in RAPIDS_DATASET_ROOT_DIR if defined, otherwise a datasets subdirectory from the calling directory (default: - os.getcwd()/datasets/) - --build - --search - --algorithms ALGORITHMS - run only comma separated list of named algorithms. If parameters `groups` and `algo-groups are both undefined, then group `base` is run by default (default: None) - --groups GROUPS run only comma separated groups of parameters (default: base) - --algo-groups ALGO_GROUPS - add comma separated . to run. Example usage: "--algo-groups=raft_cagra.large,hnswlib.large" (default: None) - -f, --force re-run algorithms even if their results already exist (default: False) - -m SEARCH_MODE, --search-mode SEARCH_MODE - run search in 'latency' (measure individual batches) or 'throughput' (pipeline batches and measure end-to-end) mode (default: throughput) - -t SEARCH_THREADS, --search-threads SEARCH_THREADS - specify the number threads to use for throughput benchmark. Single value or a pair of min and max separated by ':'. Example --search-threads=1:4. Power of 2 values between 'min' and 'max' will be used. If only 'min' is - specified, then a single test is run with 'min' threads. By default min=1, max=. (default: None) - -r, --dry-run dry-run mode will convert the yaml config for the specified algorithms and datasets to the json format that's consumed by the lower-level c++ binaries and then print the command to run execute the benchmarks but - will not actually execute the command. (default: False) -``` - -`dataset`: name of the dataset to be searched in [datasets.yaml](#yaml-dataset-config) - -`dataset-configuration`: optional filepath to custom dataset YAML config which has an entry for arg `dataset` - -`configuration`: optional filepath to YAML configuration for an algorithm or to directory that contains YAML configurations for several algorithms. [Here's how to configure an algorithm.](#yaml-algo-config) - -`algorithms`: runs all algorithms that it can find in YAML configs found by `configuration`. By default, only `base` group will be run. - -`groups`: run only specific groups of parameters configurations for an algorithm. Groups are defined in YAML configs (see `configuration`), and by default run `base` group - -`algo-groups`: this parameter is helpful to append any specific algorithm+group combination to run the benchmark for in addition to all the arguments from `algorithms` and `groups`. It is of the format `.`, or for example, `raft_cagra.large` - -For every algorithm run by this script, it outputs an index build statistics JSON file in `/result/build/<{algo},{group}.json>` -and an index search statistics JSON file in `/result/search/<{algo},{group},k{k},bs{batch_size}.json>`. NOTE: The filenames will not have ",{group}" if `group = "base"`. - -`dataset-path` : -1. data is read from `/` -2. indices are built in `//index` -3. build/search results are stored in `//result` - -`build` and `search` : if both parameters are not supplied to the script then -it is assumed both are `True`. - -`indices` and `algorithms` : these parameters ensure that the algorithm specified for an index -is available in `algos.yaml` and not disabled, as well as having an associated executable. - -### Step 3: Data Export -The script `raft_ann_bench.data_export` will convert the intermediate JSON outputs produced by `raft_ann_bench.run` to more -easily readable CSV files, which are needed to build charts made by `raft_ann_bench.plot`. - -```bash -usage: data_export.py [-h] [--dataset DATASET] [--dataset-path DATASET_PATH] - -options: - -h, --help show this help message and exit - --dataset DATASET dataset to download (default: glove-100-inner) - --dataset-path DATASET_PATH - path to dataset folder (default: ${RAPIDS_DATASET_ROOT_DIR}) -``` -Build statistics CSV file is stored in `/result/build/<{algo},{group}.csv>` -and index search statistics CSV file in `/result/search/<{algo},{group},k{k},bs{batch_size},{suffix}.csv>`, where suffix has three values: -1. `raw`: All search results are exported -2. `throughput`: Pareto frontier of throughput results is exported -3. `latency`: Pareto frontier of latency results is exported - - -### Step 4: Plot Results -The script `raft_ann_bench.plot` will plot results for all algorithms found in index search statistics -CSV files `/result/search/*.csv`. - -The usage of this script is: -```bash -usage: [-h] [--dataset DATASET] [--dataset-path DATASET_PATH] [--output-filepath OUTPUT_FILEPATH] [--algorithms ALGORITHMS] [--groups GROUPS] [--algo-groups ALGO_GROUPS] - [-k COUNT] [-bs BATCH_SIZE] [--build] [--search] [--x-scale X_SCALE] [--y-scale {linear,log,symlog,logit}] [--x-start X_START] [--mode {throughput,latency}] - [--time-unit {s,ms,us}] [--raw] - -options: - -h, --help show this help message and exit - --dataset DATASET dataset to plot (default: glove-100-inner) - --dataset-path DATASET_PATH - path to dataset folder (default: /home/coder/raft/datasets/) - --output-filepath OUTPUT_FILEPATH - directory for PNG to be saved (default: /home/coder/raft) - --algorithms ALGORITHMS - plot only comma separated list of named algorithms. If parameters `groups` and `algo-groups are both undefined, then group `base` is plot by default - (default: None) - --groups GROUPS plot only comma separated groups of parameters (default: base) - --algo-groups ALGO_GROUPS, --algo-groups ALGO_GROUPS - add comma separated . to plot. Example usage: "--algo-groups=raft_cagra.large,hnswlib.large" (default: None) - -k COUNT, --count COUNT - the number of nearest neighbors to search for (default: 10) - -bs BATCH_SIZE, --batch-size BATCH_SIZE - number of query vectors to use in each query trial (default: 10000) - --build - --search - --x-scale X_SCALE Scale to use when drawing the X-axis. Typically linear, logit or a2 (default: linear) - --y-scale {linear,log,symlog,logit} - Scale to use when drawing the Y-axis (default: linear) - --x-start X_START Recall values to start the x-axis from (default: 0.8) - --mode {throughput,latency} - search mode whose Pareto frontier is used on the y-axis (default: throughput) - --time-unit {s,ms,us} - time unit to plot when mode is latency (default: ms) - --raw Show raw results (not just Pareto frontier) of mode arg (default: False) -``` -`mode`: plots pareto frontier of `throughput` or `latency` results exported in the previous step - -`algorithms`: plots all algorithms that it can find results for the specified `dataset`. By default, only `base` group will be plotted. - -`groups`: plot only specific groups of parameters configurations for an algorithm. Groups are defined in YAML configs (see `configuration`), and by default run `base` group - -`algo-groups`: this parameter is helpful to append any specific algorithm+group combination to plot results for in addition to all the arguments from `algorithms` and `groups`. It is of the format `.`, or for example, `raft_cagra.large` - -The figure below is the resulting plot of running our benchmarks as of August 2023 for a batch size of 10, on an NVIDIA H100 GPU and an Intel Xeon Platinum 8480CL CPU. It presents the throughput (in Queries-Per-Second) performance for every level of recall. - -![Throughput vs recall plot comparing popular ANN algorithms with RAFT's at batch size 10](../../img/raft-vector-search-batch-10.png) - -## Running the benchmarks - -### End to end: small-scale benchmarks (<1M to 10M) - -The steps below demonstrate how to download, install, and run benchmarks on a subset of 10M vectors from the Yandex Deep-1B dataset By default the datasets will be stored and used from the folder indicated by the `RAPIDS_DATASET_ROOT_DIR` environment variable if defined, otherwise a datasets sub-folder from where the script is being called: - -```bash - -# (1) prepare dataset. -python -m raft_ann_bench.get_dataset --dataset deep-image-96-angular --normalize - -# (2) build and search index -python -m raft_ann_bench.run --dataset deep-image-96-inner --algorithms raft_cagra --batch-size 10 -k 10 - -# (3) export data -python -m raft_ann_bench.data_export --dataset deep-image-96-inner - -# (4) plot results -python -m raft_ann_bench.plot --dataset deep-image-96-inner -``` - -Configuration files already exist for the following list of the million-scale datasets. Please refer to [ann-benchmarks datasets](https://github.com/erikbern/ann-benchmarks/#data-sets) for more information, including actual train and sizes. These all work out-of-the-box with the `--dataset` argument. Other million-scale datasets from `ann-benchmarks.com` will work, but will require a json configuration file to be created in `$CONDA_PREFIX/lib/python3.xx/site-packages/raft_ann_bench/run/conf`, or you can specify the `--configuration` option to use a specific file. - -| Dataset Name | Train Rows | Columns | Test Rows | Distance | -|-----|------------|----|----------------|------------| -| `deep-image-96-angular` | 10M | 96 | 10K | Angular | -| `fashion-mnist-784-euclidean` | 60K | 784 | 10K | Euclidean | -| `glove-50-angular` | 1.1M | 50 | 10K | Angular | -| `glove-100-angular` | 1.1M | 100 | 10K | Angular | -| `mnist-784-euclidean` | 60K | 784 | 10K | Euclidean | -| `nytimes-256-angular` | 290K | 256 | 10K | Angular | -| `sift-128-euclidean` | 1M | 128 | 10K | Euclidean| - -All of the datasets above contain ground test datasets with 100 neighbors. Thus `k` for these datasets must be less than or equal to 100. - -### End to end: large-scale benchmarks (>10M vectors) - -`raft_ann_bench.get_dataset` cannot be used to download the [billion-scale datasets](ann_benchmarks_dataset.md#billion-scale) -due to their size. You should instead use our billion-scale datasets guide to download and prepare them. -All other python commands mentioned below work as intended once the -billion-scale dataset has been downloaded. -To download billion-scale datasets, visit [big-ann-benchmarks](http://big-ann-benchmarks.com/neurips21.html) - -We also provide a new dataset called `wiki-all` containing 88 million 768-dimensional vectors. This dataset is meant for benchmarking a realistic retrieval-augmented generation (RAG)/LLM embedding size at scale. It also contains 1M and 10M vector subsets for smaller-scale experiments. See our [Wiki-all Dataset Guide](https://docs.rapids.ai/api/raft/nightly/wiki_all_dataset/) for more information and to download the dataset. - -The steps below demonstrate how to download, install, and run benchmarks on a subset of 100M vectors from the Yandex Deep-1B dataset. Please note that datasets of this scale are recommended for GPUs with larger amounts of memory, such as the A100 or H100. -```bash - -mkdir -p datasets/deep-1B -# (1) prepare dataset -# download manually "Ground Truth" file of "Yandex DEEP" -# suppose the file name is deep_new_groundtruth.public.10K.bin -python -m raft_ann_bench.split_groundtruth --groundtruth datasets/deep-1B/deep_new_groundtruth.public.10K.bin -# two files 'groundtruth.neighbors.ibin' and 'groundtruth.distances.fbin' should be produced - -# (2) build and search index -python -m raft_ann_bench.run --dataset deep-1B --algorithms raft_cagra --batch-size 10 -k 10 - -# (3) export data -python -m raft_ann_bench.data_export --dataset deep-1B - -# (4) plot results -python -m raft_ann_bench.plot --dataset deep-1B -``` - -The usage of `python -m raft_ann_bench.split_groundtruth` is: -```bash -usage: split_groundtruth.py [-h] --groundtruth GROUNDTRUTH - -options: - -h, --help show this help message and exit - --groundtruth GROUNDTRUTH - Path to billion-scale dataset groundtruth file (default: None) -``` - -### Running with Docker containers - -Two methods are provided for running the benchmarks with the Docker containers. - -#### End-to-end run on GPU - -When no other entrypoint is provided, an end-to-end script will run through all the steps in [Running the benchmarks](#running-the-benchmarks) above. - -For GPU-enabled systems, the `DATA_FOLDER` variable should be a local folder where you want datasets stored in `$DATA_FOLDER/datasets` and results in `$DATA_FOLDER/result` (we highly recommend `$DATA_FOLDER` to be a dedicated folder for the datasets and results of the containers): -```bash -export DATA_FOLDER=path/to/store/datasets/and/results -docker run --gpus all --rm -it -u $(id -u) \ - -v $DATA_FOLDER:/data/benchmarks \ - rapidsai/raft-ann-bench:24.12a-cuda11.8-py3.10 \ - "--dataset deep-image-96-angular" \ - "--normalize" \ - "--algorithms raft_cagra,raft_ivf_pq --batch-size 10 -k 10" \ - "" -``` - -Usage of the above command is as follows: - -| Argument | Description | -|-----------------------------------------------------------|----------------------------------------------------------------------------------------------------| -| `rapidsai/raft-ann-bench:24.12a-cuda11.8-py3.10` | Image to use. Can be either `raft-ann-bench` or `raft-ann-bench-datasets` | -| `"--dataset deep-image-96-angular"` | Dataset name | -| `"--normalize"` | Whether to normalize the dataset | -| `"--algorithms raft_cagra,hnswlib --batch-size 10 -k 10"` | Arguments passed to the `run` script, such as the algorithms to benchmark, the batch size, and `k` | -| `""` | Additional (optional) arguments that will be passed to the `plot` script. | - -***Note about user and file permissions:*** The flag `-u $(id -u)` allows the user inside the container to match the `uid` of the user outside the container, allowing the container to read and write to the mounted volume indicated by the `$DATA_FOLDER` variable. - -#### End-to-end run on CPU - -The container arguments in the above section also be used for the CPU-only container, which can be used on systems that don't have a GPU installed. - -***Note:*** the image changes to `raft-ann-bench-cpu` container and the `--gpus all` argument is no longer used: -```bash -export DATA_FOLDER=path/to/store/datasets/and/results -docker run --rm -it -u $(id -u) \ - -v $DATA_FOLDER:/data/benchmarks \ - rapidsai/raft-ann-bench-cpu:24.12a-py3.10 \ - "--dataset deep-image-96-angular" \ - "--normalize" \ - "--algorithms hnswlib --batch-size 10 -k 10" \ - "" -``` - -#### Manually run the scripts inside the container - -All of the `raft-ann-bench` images contain the Conda packages, so they can be used directly by logging directly into the container itself: - -```bash -export DATA_FOLDER=path/to/store/datasets/and/results -docker run --gpus all --rm -it -u $(id -u) \ - --entrypoint /bin/bash \ - --workdir /data/benchmarks \ - -v $DATA_FOLDER:/data/benchmarks \ - rapidsai/raft-ann-bench:24.12a-cuda11.8-py3.10 -``` - -This will drop you into a command line in the container, with the `raft-ann-bench` python package ready to use, as described in the [Running the benchmarks](#running-the-benchmarks) section above: - -``` -(base) root@00b068fbb862:/data/benchmarks# python -m raft_ann_bench.get_dataset --dataset deep-image-96-angular --normalize -``` - -Additionally, the containers can be run in detached mode without any issue. - - -### Evaluating the results - -The benchmarks capture several different measurements. The table below describes each of the measurements for index build benchmarks: - -| Name | Description | -|------------|--------------------------------------------------------| -| Benchmark | A name that uniquely identifies the benchmark instance | -| Time | Wall-time spent training the index | -| CPU | CPU time spent training the index | -| Iterations | Number of iterations (this is usually 1) | -| GPU | GPU time spent building | -| index_size | Number of vectors used to train index | - - -The table below describes each of the measurements for the index search benchmarks. The most important measurements `Latency`, `items_per_second`, `end_to_end`. - -| Name | Description | -|------------|-------------------------------------------------------------------------------------------------------------------------------------------------------| -| Benchmark | A name that uniquely identifies the benchmark instance | -| Time | The wall-clock time of a single iteration (batch) divided by the number of threads. | -| CPU | The average CPU time (user + sys time). This does not include idle time (which can also happen while waiting for GPU sync). | -| Iterations | Total number of batches. This is going to be `total_queries` / `n_queries`. | -| GPU | GPU latency of a single batch (seconds). In throughput mode this is averaged over multiple threads. | -| Latency | Latency of a single batch (seconds), calculated from wall-clock time. In throughput mode this is averaged over multiple threads. | -| Recall | Proportion of correct neighbors to ground truth neighbors. Note this column is only present if groundtruth file is specified in dataset configuration.| -| items_per_second | Total throughput, a.k.a Queries per second (QPS). This is approximately `total_queries` / `end_to_end`. | -| k | Number of neighbors being queried in each iteration | -| end_to_end | Total time taken to run all batches for all iterations | -| n_queries | Total number of query vectors in each batch | -| total_queries | Total number of vectors queries across all iterations ( = `iterations` * `n_queries`) | - -Note the following: -- A slightly different method is used to measure `Time` and `end_to_end`. That is why `end_to_end` = `Time` * `Iterations` holds only approximately. -- The actual table displayed on the screen may differ slightly as the hyper-parameters will also be displayed for each different combination being benchmarked. -- Recall calculation: the number of queries processed per test depends on the number of iterations. Because of this, recall can show slight fluctuations if less neighbors are processed then it is available for the benchmark. - -## Creating and customizing dataset configurations - -A single configuration will often define a set of algorithms, with associated index and search parameters, that can be generalize across datasets. We use YAML to define dataset specific and algorithm specific configurations. - -A default `datasets.yaml` is provided by RAFT in `${RAFT_HOME}/python/raft-ann-bench/src/raft_ann_bench/run/conf` with configurations available for several datasets. Here's a simple example entry for the `sift-128-euclidean` dataset: - -```yaml -- name: sift-128-euclidean - base_file: sift-128-euclidean/base.fbin - query_file: sift-128-euclidean/query.fbin - groundtruth_neighbors_file: sift-128-euclidean/groundtruth.neighbors.ibin - dims: 128 - distance: euclidean -``` - -Configuration files for ANN algorithms supported by `raft-ann-bench` are provided in `${RAFT_HOME}/python/raft-ann-bench/src/raft_ann_bench/run/conf`. `raft_cagra` algorithm configuration looks like: -```yaml -name: raft_cagra -groups: - base: - build: - graph_degree: [32, 64] - intermediate_graph_degree: [64, 96] - graph_build_algo: ["NN_DESCENT"] - search: - itopk: [32, 64, 128] - - large: - build: - graph_degree: [32, 64] - search: - itopk: [32, 64, 128] -``` -The default parameters for which the benchmarks are run can be overridden by creating a custom YAML file for algorithms with a `base` group. - -There config above has 2 fields: -1. `name` - define the name of the algorithm for which the parameters are being specified. -2. `groups` - define a run group which has a particular set of parameters. Each group helps create a cross-product of all hyper-parameter fields for `build` and `search`. - -The table below contains all algorithms supported by RAFT. Each unique algorithm will have its own set of `build` and `search` settings. The [ANN Algorithm Parameter Tuning Guide](ann_benchmarks_param_tuning.md) contains detailed instructions on choosing build and search parameters for each supported algorithm. - -| Library | Algorithms | -|-----------|---------------------------------------------------------------------------------------| -| FAISS GPU | `faiss_gpu_flat`, `faiss_gpu_ivf_flat`, `faiss_gpu_ivf_pq` | -| FAISS CPU | `faiss_cpu_flat`, `faiss_cpu_ivf_flat`, `faiss_cpu_ivf_pq` | -| GGNN | `ggnn` | -| HNSWlib | `hnswlib` | -| RAFT | `raft_brute_force`, `raft_cagra`, `raft_ivf_flat`, `raft_ivf_pq`, `raft_cagra_hnswlib`| - -## Adding a new ANN algorithm - -### Implementation and Configuration -Implementation of a new algorithm should be a C++ class that inherits `class ANN` (defined in `cpp/bench/ann/src/ann.h`) and implements all the pure virtual functions. - -In addition, it should define two `struct`s for building and searching parameters. The searching parameter class should inherit `struct ANN::AnnSearchParam`. Take `class HnswLib` as an example, its definition is: -```c++ -template -class HnswLib : public ANN { -public: - struct BuildParam { - int M; - int ef_construction; - int num_threads; - }; - - using typename ANN::AnnSearchParam; - struct SearchParam : public AnnSearchParam { - int ef; - int num_threads; - }; - - // ... -}; -``` - -The benchmark program uses JSON format in a configuration file to specify indexes to build, along with the build and search parameters. To add the new algorithm to the benchmark, need be able to specify `build_param`, whose value is a JSON object, and `search_params`, whose value is an array of JSON objects, for this algorithm in configuration file. The `build_param` and `search_param` arguments will vary depending on the algorithm. Take the configuration for `HnswLib` as an example: -```json -{ - "name" : "hnswlib.M12.ef500.th32", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "/path/to/file", - "search_params" : [ - {"ef":10, "numThreads":1}, - {"ef":20, "numThreads":1}, - {"ef":40, "numThreads":1}, - ], - "search_result_file" : "/path/to/file" -}, -``` -How to interpret these JSON objects is totally left to the implementation and should be specified in `cpp/bench/ann/src/factory.cuh`: -1. First, add two functions for parsing JSON object to `struct BuildParam` and `struct SearchParam`, respectively: - ```c++ - template - void parse_build_param(const nlohmann::json& conf, - typename cuann::HnswLib::BuildParam& param) { - param.ef_construction = conf.at("efConstruction"); - param.M = conf.at("M"); - if (conf.contains("numThreads")) { - param.num_threads = conf.at("numThreads"); - } - } - - template - void parse_search_param(const nlohmann::json& conf, - typename cuann::HnswLib::SearchParam& param) { - param.ef = conf.at("ef"); - if (conf.contains("numThreads")) { - param.num_threads = conf.at("numThreads"); - } - } - ``` - -2. Next, add corresponding `if` case to functions `create_algo()` (in `cpp/bench/ann/) and `create_search_param()` by calling parsing functions. The string literal in `if` condition statement must be the same as the value of `algo` in configuration file. For example, - ```c++ - // JSON configuration file contains a line like: "algo" : "hnswlib" - if (algo == "hnswlib") { - // ... - } - ``` - - -### Adding a CMake Target -In `raft/cpp/bench/ann/CMakeLists.txt`, we provide a `CMake` function to configure a new Benchmark target with the following signature: -``` -ConfigureAnnBench( - NAME - PATH - INCLUDES - CXXFLAGS - LINKS -) -``` - -To add a target for `HNSWLIB`, we would call the function as: -``` -ConfigureAnnBench( - NAME HNSWLIB PATH bench/ann/src/hnswlib/hnswlib_benchmark.cpp INCLUDES - ${CMAKE_CURRENT_BINARY_DIR}/_deps/hnswlib-src/hnswlib CXXFLAGS "${HNSW_CXX_FLAGS}" -) -``` - -This will create an executable called `HNSWLIB_ANN_BENCH`, which can then be used to run `HNSWLIB` benchmarks. - -Add a new entry to `algos.yaml` to map the name of the algorithm to its binary executable and specify whether the algorithm requires GPU support. -```yaml -raft_ivf_pq: - executable: RAFT_IVF_PQ_ANN_BENCH - requires_gpu: true -``` - -`executable` : specifies the name of the binary that will build/search the index. It is assumed to be -available in `raft/cpp/build/`. -`requires_gpu` : denotes whether an algorithm requires GPU to run. diff --git a/docs/source/vector_search_tutorial.md b/docs/source/vector_search_tutorial.md index d1d5c57700..8f7b2d1bfd 100644 --- a/docs/source/vector_search_tutorial.md +++ b/docs/source/vector_search_tutorial.md @@ -17,7 +17,7 @@ RAFT has several important algorithms for performing vector search on the GPU and this tutorial walks through the primary vector search APIs from start to finish to provide a reference for quick setup and C++ API usage. -This tutorial assumes RAFT has been installed and/or added to your build so that you are able to compile and run RAFT code. If not done already, please follow the [build and install instructions](build.md) and consider taking a look at the [example c++ template project](https://github.com/rapidsai/raft/tree/HEAD/cpp/template) for ready-to-go examples that you can immediately build and start playing with. Also take a look at RAFT's library of [reproducible vector search benchmarks](raft_ann_benchmarks.md) to run benchmarks that compare RAFT against other state-of-the-art nearest neighbors algorithms at scale. +This tutorial assumes RAFT has been installed and/or added to your build so that you are able to compile and run RAFT code. If not done already, please follow the [build and install instructions](build.md) and consider taking a look at the [example c++ template project](https://github.com/rapidsai/raft/tree/HEAD/cpp/template) for ready-to-go examples that you can immediately build and start playing with. For more information about the various APIs demonstrated in this tutorial, along with comprehensive usage examples of all the APIs offered by RAFT, please refer to the [RAFT's C++ API Documentation](https://docs.rapids.ai/api/raft/nightly/cpp_api/). @@ -271,7 +271,7 @@ auto removed_indices = raft::make_device_vector(res, n_removed_indices); raft::core::bitset removed_indices_bitset( res, removed_indices.view(), dataset.extent(0)); -// ... Populate the bitset ... +// ... Populate the bitset ... // search K nearest neighbours according to a bitset filter auto neighbors = raft::make_device_matrix(res, n_queries, k); @@ -406,4 +406,4 @@ The below example specifies the total number of bytes that RAFT can use for temp std::shared_ptr managed_resource; raft::device_resource res(managed_resource, std::make_optional(3 * 1024^3)); -``` \ No newline at end of file +``` diff --git a/docs/source/wiki_all_dataset.md b/docs/source/wiki_all_dataset.md deleted file mode 100644 index c001bdc409..0000000000 --- a/docs/source/wiki_all_dataset.md +++ /dev/null @@ -1,47 +0,0 @@ -# Wiki-all Dataset - -The `wiki-all` dataset was created to stress vector search algorithms at scale with both a large number of vectors and dimensions. The entire dataset contains 88M vectors with 768 dimensions and is meant for testing the types of vectors one would typically encounter in retrieval augmented generation (RAG) workloads. The full dataset is ~251GB in size, which is intentionally larger than the typical memory of GPUs. The massive scale is intended to promote the use of compression and efficient out-of-core methods for both indexing and search. - -The dataset is composed of English wiki texts from [Kaggle](https://www.kaggle.com/datasets/jjinho/wikipedia-20230701) and multi-lingual wiki texts from [Cohere Wikipedia](https://huggingface.co/datasets/Cohere/wikipedia-22-12). - -Cohere's English Texts are older (2022) and smaller than the Kaggle English Wiki texts (2023) so the English texts have been removed from Cohere completely. The final Wiki texts include English Wiki from Kaggle and the other languages from Cohere. The English texts constitute 50% of the total text size. - -To form the final dataset, the Wiki texts were chunked into 85 million 128-token pieces. For reference, Cohere chunks Wiki texts into 104-token pieces. Finally, the embeddings of each chunk were computed using the [paraphrase-multilingual-mpnet-base-v2](https://huggingface.co/sentence-transformers/paraphrase-multilingual-mpnet-base-v2) embedding model. The resulting dataset is an embedding matrix of size 88 million by 768. Also included with the dataset is a query file containing 10k query vectors and a groundtruth file to evaluate nearest neighbors algorithms. - -## Getting the dataset - -### Full dataset - -A version of the dataset is made available in the binary format that can be used directly by the [raft-ann-bench](https://docs.rapids.ai/api/raft/nightly/raft_ann_benchmarks/) tool. The full 88M dataset is ~251GB and the download link below contains tarballs that have been split into multiple parts. - -The following will download all 10 the parts and untar them to a `wiki_all_88M` directory: -```bash -curl -s https://data.rapids.ai/raft/datasets/wiki_all/wiki_all.tar.{00..9} | tar -xf - -C wiki_all_88M/ -``` - -The above has the unfortunate drawback that if the command should fail for any reason, all the parts need to be re-downloaded. The files can also be downloaded individually and then untarred to the directory. Each file is ~27GB and there are 10 of them. - -```bash -curl -s https://data.rapids.ai/raft/datasets/wiki_all/wiki_all.tar.00 -... -curl -s https://data.rapids.ai/raft/datasets/wiki_all/wiki_all.tar.09 - -cat wiki_all.tar.* | tar -xf - -C wiki_all_88M/ -``` - -### 1M and 10M subsets - -Also available are 1M and 10M subsets of the full dataset which are 2.9GB and 29GB, respectively. These subsets also include query sets of 10k vectors and corresponding groundtruth files. - -```bash -curl -s https://data.rapids.ai/raft/datasets/wiki_all_1M/wiki_all_1M.tar -curl -s https://data.rapids.ai/raft/datasets/wiki_all_10M/wiki_all_10M.tar -``` - -## Using the dataset - -After the dataset is downloaded and extracted to the `wiki_all_88M` directory (or `wiki_all_1M`/`wiki_all_10M` depending on whether the subsets are used), the files can be used in the benchmarking tool. The dataset name is `wiki_all` (or `wiki_all_1M`/`wiki_all_10M`), and the benchmarking tool can be used by specifying the appropriate name `--dataset wiki_all_88M` in the scripts. - -## License info - -The English wiki texts available on Kaggle come with the [CC BY-NCSA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/) license and the Cohere wikipedia data set comes with the [Apache 2.0](https://choosealicense.com/licenses/apache-2.0/) license. \ No newline at end of file diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index c286d3debf..3e3cc15221 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -53,7 +53,6 @@ if(NOT raft_FOUND) set(BUILD_TESTS OFF) set(BUILD_PRIMS_BENCH OFF) - set(BUILD_ANN_BENCH OFF) set(RAFT_COMPILE_LIBRARY ON) set(CUDA_STATIC_RUNTIME ON) set(CUDA_STATIC_MATH_LIBRARIES ON) diff --git a/python/raft-ann-bench/LICENSE b/python/raft-ann-bench/LICENSE deleted file mode 120000 index 30cff7403d..0000000000 --- a/python/raft-ann-bench/LICENSE +++ /dev/null @@ -1 +0,0 @@ -../../LICENSE \ No newline at end of file diff --git a/python/raft-ann-bench/pyproject.toml b/python/raft-ann-bench/pyproject.toml deleted file mode 100644 index 0e4fda1f00..0000000000 --- a/python/raft-ann-bench/pyproject.toml +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. - -[build-system] -build-backend = "rapids_build_backend.build" -requires = [ - "rapids-build-backend>=0.3.0,<0.4.0.dev0", - "setuptools", - "wheel", -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. - -[project] -name = "raft-ann-bench" -dynamic = ["version"] -description = "RAFT ANN benchmarks" -authors = [ - { name = "NVIDIA Corporation" }, -] -license = { text = "Apache 2.0" } -requires-python = ">=3.10" -dependencies = [ -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -classifiers = [ - "Intended Audience :: Developers", - "Topic :: Database", - "Topic :: Scientific/Engineering", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", -] - -[project.urls] -Homepage = "https://github.com/rapidsai/raft" - -[tool.setuptools.packages.find] -where = ["src"] - -[tool.setuptools.package-data] -"*" = ["*.*", "VERSION"] - -[tool.isort] -line_length = 79 -multi_line_output = 3 -include_trailing_comma = true -force_grid_wrap = 0 -combine_as_imports = true -order_by_type = true -skip = [ - "thirdparty", - ".eggs", - ".git", - ".hg", - ".mypy_cache", - ".tox", - ".venv", - "_build", - "buck-out", - "build", - "dist", -] - -[tool.setuptools.dynamic] -version = { file = "raft_ann_bench/VERSION" } - -[tool.rapids-build-backend] -build-backend = "setuptools.build_meta" -requires = [] -dependencies-file = "../../dependencies.yaml" -commit-files = ["src/raft_ann_bench/GIT_COMMIT"] -matrix-entry = "cuda_suffixed=true" diff --git a/python/raft-ann-bench/src/raft_ann_bench/VERSION b/python/raft-ann-bench/src/raft_ann_bench/VERSION deleted file mode 120000 index a4e948506b..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/VERSION +++ /dev/null @@ -1 +0,0 @@ -../../../../VERSION \ No newline at end of file diff --git a/python/raft-ann-bench/src/raft_ann_bench/__init__.py b/python/raft-ann-bench/src/raft_ann_bench/__init__.py deleted file mode 100644 index 80a3b3f284..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from ._version import __git_commit__, __version__ diff --git a/python/raft-ann-bench/src/raft_ann_bench/_version.py b/python/raft-ann-bench/src/raft_ann_bench/_version.py deleted file mode 100644 index 0fa0ba80bc..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/_version.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import importlib.resources - -__version__ = ( - importlib.resources.files(__package__) - .joinpath("VERSION") - .read_text() - .strip() -) -try: - __git_commit__ = ( - importlib.resources.files(__package__) - .joinpath("GIT_COMMIT") - .read_text() - .strip() - ) -except FileNotFoundError: - __git_commit__ = "" - -__all__ = ["__version__", "__git_commit__"] diff --git a/python/raft-ann-bench/src/raft_ann_bench/constraints/__init__.py b/python/raft-ann-bench/src/raft_ann_bench/constraints/__init__.py deleted file mode 100644 index e94ee56c92..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/constraints/__init__.py +++ /dev/null @@ -1,77 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -DTYPE_SIZES = {"float": 4, "half": 2, "fp8": 1} - - -def raft_cagra_build_constraints(params, dims): - if "graph_degree" in params and "intermediate_graph_degree" in params: - return params["graph_degree"] <= params["intermediate_graph_degree"] - return True - - -def raft_ivf_pq_build_constraints(params, dims): - if "pq_dim" in params: - return params["pq_dim"] <= dims - return True - - -def raft_ivf_pq_search_constraints(params, build_params, k, batch_size): - ret = True - if "internalDistanceDtype" in params and "smemLutDtype" in params: - ret = ( - DTYPE_SIZES[params["smemLutDtype"]] - <= DTYPE_SIZES[params["internalDistanceDtype"]] - ) - - if "nlist" in build_params and "nprobe" in params: - ret = ret and build_params["nlist"] >= params["nprobe"] - return ret - - -def raft_cagra_search_constraints(params, build_params, k, batch_size): - ret = True - if "itopk" in params: - ret = ret and params["itopk"] >= k - return ret - - -def hnswlib_search_constraints(params, build_params, k, batch_size): - if "ef" in params: - return params["ef"] >= k - - -def faiss_gpu_ivf_pq_build_constraints(params, dims): - ret = True - # M must be defined - ret = params["M"] <= dims and dims % params["M"] == 0 - if "use_raft" in params and params["use_raft"]: - return ret - pq_bits = 8 - if "bitsPerCode" in params: - pq_bits = params["bitsPerCode"] - lookup_table_size = 4 - if "useFloat16" in params and params["useFloat16"]: - lookup_table_size = 2 - # FAISS constraint to check if lookup table fits in shared memory - # for now hard code maximum shared memory per block to 49 kB (the value for A100 and V100) - return ret and lookup_table_size * params["M"] * (2**pq_bits) <= 49152 - - -def faiss_gpu_ivf_pq_search_constraints(params, build_params, k, batch_size): - ret = True - if "nlist" in build_params and "nprobe" in params: - ret = ret and build_params["nlist"] >= params["nprobe"] - return ret diff --git a/python/raft-ann-bench/src/raft_ann_bench/data_export/__main__.py b/python/raft-ann-bench/src/raft_ann_bench/data_export/__main__.py deleted file mode 100644 index c8a6375577..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/data_export/__main__.py +++ /dev/null @@ -1,257 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import argparse -import json -import os -import sys -import traceback -import warnings - -import pandas as pd - -skip_build_cols = set( - [ - "algo_name", - "index_name", - "time", - "name", - "family_index", - "per_family_instance_index", - "run_name", - "run_type", - "repetitions", - "repetition_index", - "iterations", - "real_time", - "time_unit", - "index_size", - ] -) - -skip_search_cols = ( - set(["recall", "qps", "latency", "items_per_second", "Recall", "Latency"]) - | skip_build_cols -) - -metrics = { - "k-nn": { - "description": "Recall", - "worst": float("-inf"), - "lim": [0.0, 1.03], - }, - "throughput": { - "description": "Queries per second (1/s)", - "worst": float("-inf"), - }, - "latency": { - "description": "Search Latency (s)", - "worst": float("inf"), - }, -} - - -def read_file(dataset, dataset_path, method): - dir = os.path.join(dataset_path, dataset, "result", method) - for file in os.listdir(dir): - if file.endswith(".json"): - with open( - os.path.join(dir, file), "r", encoding="ISO-8859-1" - ) as f: - try: - data = json.load(f) - df = pd.DataFrame(data["benchmarks"]) - filename_split = file.split(",") - algo_name = (filename_split[0], filename_split[1]) - yield os.path.join(dir, file), algo_name, df - except Exception as e: - print( - "An error occurred processing file %s (%s). " - "Skipping..." % (file, e) - ) - - -def convert_json_to_csv_build(dataset, dataset_path): - for file, algo_name, df in read_file(dataset, dataset_path, "build"): - try: - if "base" in algo_name[1]: - algo_name = algo_name[0] - else: - algo_name = "_".join(algo_name) - df["name"] = df["name"].str.split("/").str[0] - write = pd.DataFrame( - { - "algo_name": [algo_name] * len(df), - "index_name": df["name"], - "time": df["real_time"], - } - ) - for name in df: - if name not in skip_build_cols: - write[name] = df[name] - write.to_csv(file.replace(".json", ".csv"), index=False) - except Exception as e: - print( - "An error occurred processing file %s (%s). Skipping..." - % (file, e) - ) - traceback.print_exc() - - -def create_pointset(data, xn, yn): - xm, ym = (metrics[xn], metrics[yn]) - rev_y = -1 if ym["worst"] < 0 else 1 - rev_x = -1 if xm["worst"] < 0 else 1 - - y_idx = 3 if yn == "throughput" else 4 - data.sort(key=lambda t: (rev_y * t[y_idx], rev_x * t[2])) - - lines = [] - last_x = xm["worst"] - comparator = ( - (lambda xv, lx: xv > lx) if last_x < 0 else (lambda xv, lx: xv < lx) - ) - for d in data: - if comparator(d[2], last_x): - last_x = d[2] - lines.append(d) - return lines - - -def get_frontier(df, metric): - lines = create_pointset(df.values.tolist(), "k-nn", metric) - return pd.DataFrame(lines, columns=df.columns) - - -def convert_json_to_csv_search(dataset, dataset_path): - for file, algo_name, df in read_file(dataset, dataset_path, "search"): - try: - build_file = os.path.join( - dataset_path, - dataset, - "result", - "build", - f"{','.join(algo_name)}.csv", - ) - print(build_file) - if "base" in algo_name[1]: - algo_name = algo_name[0] - else: - algo_name = "_".join(algo_name) - df["name"] = df["name"].str.split("/").str[0] - try: - write = pd.DataFrame( - { - "algo_name": [algo_name] * len(df), - "index_name": df["name"], - "recall": df["Recall"], - "throughput": df["items_per_second"], - "latency": df["Latency"], - } - ) - except Exception as e: - print( - "Search file %s (%s) missing a key. Skipping..." - % (file, e) - ) - for name in df: - if name not in skip_search_cols: - write[name] = df[name] - - if os.path.exists(build_file): - build_df = pd.read_csv(build_file) - write_ncols = len(write.columns) - write["build time"] = None - write["build threads"] = None - write["build cpu_time"] = None - write["build GPU"] = None - - try: - for col_idx in range(6, len(build_df.columns)): - col_name = build_df.columns[col_idx] - write[col_name] = None - - for s_index, search_row in write.iterrows(): - for b_index, build_row in build_df.iterrows(): - if ( - search_row["index_name"] - == build_row["index_name"] - ): - write.iloc[ - s_index, write_ncols - ] = build_df.iloc[b_index, 2] - write.iloc[ - s_index, write_ncols + 1 : - ] = build_df.iloc[b_index, 3:] - break - except Exception as e: - print( - "Build file %s (%s) missing a key. Skipping..." - % (build_file, e) - ) - else: - warnings.warn( - f"Build CSV not found for {algo_name}, " - f"build params won't be " - "appended in the Search CSV" - ) - - write.to_csv(file.replace(".json", ",raw.csv"), index=False) - throughput = get_frontier(write, "throughput") - throughput.to_csv( - file.replace(".json", ",throughput.csv"), index=False - ) - latency = get_frontier(write, "latency") - latency.to_csv(file.replace(".json", ",latency.csv"), index=False) - except Exception as e: - print( - "An error occurred processing file %s (%s). Skipping..." - % (file, e) - ) - traceback.print_exc() - - -def main(): - - call_path = os.getcwd() - if "RAPIDS_DATASET_ROOT_DIR" in os.environ: - default_dataset_path = os.getenv("RAPIDS_DATASET_ROOT_DIR") - else: - default_dataset_path = os.path.join(call_path, "datasets/") - - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - "--dataset", help="dataset to download", default="glove-100-inner" - ) - parser.add_argument( - "--dataset-path", - help="path to dataset folder", - default=default_dataset_path, - ) - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - args = parser.parse_args() - - convert_json_to_csv_build(args.dataset, args.dataset_path) - convert_json_to_csv_search(args.dataset, args.dataset_path) - - -if __name__ == "__main__": - main() diff --git a/python/raft-ann-bench/src/raft_ann_bench/generate_groundtruth/__main__.py b/python/raft-ann-bench/src/raft_ann_bench/generate_groundtruth/__main__.py deleted file mode 100644 index e6f7aaf99c..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/generate_groundtruth/__main__.py +++ /dev/null @@ -1,240 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import argparse -import os -import sys - -import cupy as cp -import numpy as np -import rmm -from pylibraft.common import DeviceResources -from pylibraft.neighbors.brute_force import knn -from rmm.allocators.cupy import rmm_cupy_allocator - -from .utils import memmap_bin_file, suffix_from_dtype, write_bin - - -def generate_random_queries(n_queries, n_features, dtype=np.float32): - print("Generating random queries") - if np.issubdtype(dtype, np.integer): - queries = cp.random.randint( - 0, 255, size=(n_queries, n_features), dtype=dtype - ) - else: - queries = cp.random.uniform(size=(n_queries, n_features)).astype(dtype) - return queries - - -def choose_random_queries(dataset, n_queries): - print("Choosing random vector from dataset as query vectors") - query_idx = np.random.choice( - dataset.shape[0], size=(n_queries,), replace=False - ) - return dataset[query_idx, :] - - -def calc_truth(dataset, queries, k, metric="sqeuclidean"): - handle = DeviceResources() - n_samples = dataset.shape[0] - n = 500000 # batch size for processing neighbors - i = 0 - indices = None - distances = None - queries = cp.asarray(queries, dtype=cp.float32) - - while i < n_samples: - print("Step {0}/{1}:".format(i // n, n_samples // n)) - n_batch = n if i + n <= n_samples else n_samples - i - - X = cp.asarray(dataset[i : i + n_batch, :], cp.float32) - - D, Ind = knn(X, queries, k, metric=metric, handle=handle) - handle.sync() - - D, Ind = cp.asarray(D), cp.asarray(Ind) - Ind += i # shift neighbor index by offset i - - if distances is None: - distances = D - indices = Ind - else: - distances = cp.concatenate([distances, D], axis=1) - indices = cp.concatenate([indices, Ind], axis=1) - idx = cp.argsort(distances, axis=1)[:, :k] - distances = cp.take_along_axis(distances, idx, axis=1) - indices = cp.take_along_axis(indices, idx, axis=1) - - i += n_batch - - return distances, indices - - -def main(): - pool = rmm.mr.PoolMemoryResource( - rmm.mr.CudaMemoryResource(), initial_pool_size=2**30 - ) - rmm.mr.set_current_device_resource(pool) - cp.cuda.set_allocator(rmm_cupy_allocator) - - parser = argparse.ArgumentParser( - prog="generate_groundtruth", - description="Generate true neighbors using exact NN search. " - "The input and output files are in big-ann-benchmark's binary format.", - epilog="""Example usage - # With existing query file - python -m raft_ann_bench.generate_groundtruth --dataset /dataset/base.\ -fbin --output=groundtruth_dir --queries=/dataset/query.public.10K.fbin - - # With randomly generated queries - python -m raft_ann_bench.generate_groundtruth --dataset /dataset/base.\ -fbin --output=groundtruth_dir --queries=random --n_queries=10000 - - # Using only a subset of the dataset. Define queries by randomly - # selecting vectors from the (subset of the) dataset. - python -m raft_ann_bench.generate_groundtruth --dataset /dataset/base.\ -fbin --nrows=2000000 --cols=128 --output=groundtruth_dir \ ---queries=random-choice --n_queries=10000 - """, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - - parser.add_argument("dataset", type=str, help="input dataset file name") - parser.add_argument( - "--queries", - type=str, - default="random", - help="Queries file name, or one of 'random-choice' or 'random' " - "(default). 'random-choice': select n_queries vectors from the input " - "dataset. 'random': generate n_queries as uniform random numbers.", - ) - parser.add_argument( - "--output", - type=str, - default="", - help="output directory name (default current dir)", - ) - - parser.add_argument( - "--n_queries", - type=int, - default=10000, - help="Number of quries to generate (if no query file is given). " - "Default: 10000.", - ) - - parser.add_argument( - "-N", - "--rows", - default=None, - type=int, - help="use only first N rows from dataset, by default the whole " - "dataset is used", - ) - parser.add_argument( - "-D", - "--cols", - default=None, - type=int, - help="number of features (dataset columns). " - "Default: read from dataset file.", - ) - parser.add_argument( - "--dtype", - type=str, - help="Dataset dtype. When not specified, then derived from extension." - " Supported types: 'float32', 'float16', 'uint8', 'int8'", - ) - - parser.add_argument( - "-k", - type=int, - default=100, - help="Number of neighbors (per query) to calculate", - ) - parser.add_argument( - "--metric", - type=str, - default="sqeuclidean", - help="Metric to use while calculating distances. Valid metrics are " - "those that are accepted by pylibraft.neighbors.brute_force.knn. Most" - " commonly used with RAFT ANN are 'sqeuclidean' and 'inner_product'", - ) - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - args = parser.parse_args() - - if args.rows is not None: - print("Reading subset of the data, nrows=", args.rows) - else: - print("Reading whole dataset") - - # Load input data - dataset = memmap_bin_file( - args.dataset, args.dtype, shape=(args.rows, args.cols) - ) - n_features = dataset.shape[1] - dtype = dataset.dtype - - print( - "Dataset size {:6.1f} GB, shape {}, dtype {}".format( - dataset.size * dataset.dtype.itemsize / 1e9, - dataset.shape, - np.dtype(dtype), - ) - ) - - if len(args.output) > 0: - os.makedirs(args.output, exist_ok=True) - - if args.queries == "random" or args.queries == "random-choice": - if args.n_queries is None: - raise RuntimeError( - "n_queries must be given to generate random queries" - ) - if args.queries == "random": - queries = generate_random_queries( - args.n_queries, n_features, dtype - ) - elif args.queries == "random-choice": - queries = choose_random_queries(dataset, args.n_queries) - - queries_filename = os.path.join( - args.output, "queries" + suffix_from_dtype(dtype) - ) - print("Writing queries file", queries_filename) - write_bin(queries_filename, queries) - else: - print("Reading queries from file", args.queries) - queries = memmap_bin_file(args.queries, dtype) - - print("Calculating true nearest neighbors") - distances, indices = calc_truth(dataset, queries, args.k, args.metric) - - write_bin( - os.path.join(args.output, "groundtruth.neighbors.ibin"), - indices.astype(np.uint32), - ) - write_bin( - os.path.join(args.output, "groundtruth.distances.fbin"), - distances.astype(np.float32), - ) - - -if __name__ == "__main__": - main() diff --git a/python/raft-ann-bench/src/raft_ann_bench/generate_groundtruth/utils.py b/python/raft-ann-bench/src/raft_ann_bench/generate_groundtruth/utils.py deleted file mode 100644 index 3f2dd11a16..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/generate_groundtruth/utils.py +++ /dev/null @@ -1,103 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os - -import numpy as np - - -def dtype_from_filename(filename): - ext = os.path.splitext(filename)[1] - if ext == ".fbin": - return np.float32 - if ext == ".hbin": - return np.float16 - elif ext == ".ibin": - return np.int32 - elif ext == ".u8bin": - return np.ubyte - elif ext == ".i8bin": - return np.byte - else: - raise RuntimeError("Not supported file extension" + ext) - - -def suffix_from_dtype(dtype): - if dtype == np.float32: - return ".fbin" - if dtype == np.float16: - return ".hbin" - elif dtype == np.int32: - return ".ibin" - elif dtype == np.ubyte: - return ".u8bin" - elif dtype == np.byte: - return ".i8bin" - else: - raise RuntimeError("Not supported dtype extension" + dtype) - - -def memmap_bin_file( - bin_file, dtype, shape=None, mode="r", size_dtype=np.uint32 -): - extent_itemsize = np.dtype(size_dtype).itemsize - offset = int(extent_itemsize) * 2 - if bin_file is None: - return None - if dtype is None: - dtype = dtype_from_filename(bin_file) - - if mode[0] == "r": - a = np.memmap(bin_file, mode=mode, dtype=size_dtype, shape=(2,)) - if shape is None: - shape = (a[0], a[1]) - else: - shape = tuple( - [ - aval if sval is None else sval - for aval, sval in zip(a, shape) - ] - ) - - return np.memmap( - bin_file, mode=mode, dtype=dtype, offset=offset, shape=shape - ) - elif mode[0] == "w": - if shape is None: - raise ValueError("Need to specify shape to map file in write mode") - - print("creating file", bin_file) - dirname = os.path.dirname(bin_file) - if len(dirname) > 0: - os.makedirs(dirname, exist_ok=True) - a = np.memmap(bin_file, mode=mode, dtype=size_dtype, shape=(2,)) - a[0] = shape[0] - a[1] = shape[1] - a.flush() - del a - fp = np.memmap( - bin_file, mode="r+", dtype=dtype, offset=offset, shape=shape - ) - return fp - - # print('# {}: shape: {}, dtype: {}'.format(bin_file, shape, dtype)) - - -def write_bin(fname, data): - print("writing", fname, data.shape, data.dtype, "...") - with open(fname, "wb") as f: - np.asarray(data.shape, dtype=np.uint32).tofile(f) - data.tofile(f) diff --git a/python/raft-ann-bench/src/raft_ann_bench/get_dataset/__main__.py b/python/raft-ann-bench/src/raft_ann_bench/get_dataset/__main__.py deleted file mode 100644 index 0a6c37aabc..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/get_dataset/__main__.py +++ /dev/null @@ -1,115 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import os -import subprocess -import sys -from urllib.request import urlretrieve - - -def get_dataset_path(name, ann_bench_data_path): - if not os.path.exists(ann_bench_data_path): - os.mkdir(ann_bench_data_path) - return os.path.join(ann_bench_data_path, f"{name}.hdf5") - - -def download_dataset(url, path): - if not os.path.exists(path): - print(f"downloading {url} -> {path}...") - urlretrieve(url, path) - - -def convert_hdf5_to_fbin(path, normalize): - scripts_path = os.path.dirname(os.path.realpath(__file__)) - ann_bench_scripts_path = os.path.join(scripts_path, "hdf5_to_fbin.py") - print(f"calling script {ann_bench_scripts_path}") - if normalize and "angular" in path: - subprocess.run( - ["python", ann_bench_scripts_path, "-n", "%s" % path], check=True - ) - else: - subprocess.run( - ["python", ann_bench_scripts_path, "%s" % path], check=True - ) - - -def move(name, ann_bench_data_path): - if "angular" in name: - new_name = name.replace("angular", "inner") - else: - new_name = name - new_path = os.path.join(ann_bench_data_path, new_name) - if not os.path.exists(new_path): - os.mkdir(new_path) - for bin_name in [ - "base.fbin", - "query.fbin", - "groundtruth.neighbors.ibin", - "groundtruth.distances.fbin", - ]: - os.rename( - f"{ann_bench_data_path}/{name}.{bin_name}", - f"{new_path}/{bin_name}", - ) - - -def download(name, normalize, ann_bench_data_path): - path = get_dataset_path(name, ann_bench_data_path) - try: - url = f"http://ann-benchmarks.com/{name}.hdf5" - download_dataset(url, path) - - convert_hdf5_to_fbin(path, normalize) - - move(name, ann_bench_data_path) - except Exception: - print(f"Cannot download {url}") - raise - - -def main(): - call_path = os.getcwd() - if "RAPIDS_DATASET_ROOT_DIR" in os.environ: - default_dataset_path = os.getenv("RAPIDS_DATASET_ROOT_DIR") - else: - default_dataset_path = os.path.join(call_path, "datasets/") - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - "--dataset", help="dataset to download", default="glove-100-angular" - ) - parser.add_argument( - "--dataset-path", - help="path to download dataset", - default=default_dataset_path, - ) - parser.add_argument( - "--normalize", - help="normalize cosine distance to inner product", - action="store_true", - ) - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - args = parser.parse_args() - - download(args.dataset, args.normalize, args.dataset_path) - - -if __name__ == "__main__": - main() diff --git a/python/raft-ann-bench/src/raft_ann_bench/get_dataset/fbin_to_f16bin.py b/python/raft-ann-bench/src/raft_ann_bench/get_dataset/fbin_to_f16bin.py deleted file mode 100755 index ee7410e0cc..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/get_dataset/fbin_to_f16bin.py +++ /dev/null @@ -1,49 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from __future__ import absolute_import, division, print_function - -import sys - -import numpy as np - - -def read_fbin(fname): - shape = np.fromfile(fname, dtype=np.uint32, count=2) - if float(shape[0]) * shape[1] * 4 > 2000000000: - data = np.memmap(fname, dtype=np.float32, offset=8, mode="r").reshape( - shape - ) - else: - data = np.fromfile(fname, dtype=np.float32, offset=8).reshape(shape) - return data - - -def write_bin(fname, data): - with open(fname, "wb") as f: - np.asarray(data.shape, dtype=np.uint32).tofile(f) - data.tofile(f) - - -if len(sys.argv) != 3: - print( - "usage: %s input.fbin output.f16bin" % (sys.argv[0]), - file=sys.stderr, - ) - sys.exit(-1) - -data = read_fbin(sys.argv[1]).astype(np.float16) -write_bin(sys.argv[2], data) diff --git a/python/raft-ann-bench/src/raft_ann_bench/get_dataset/hdf5_to_fbin.py b/python/raft-ann-bench/src/raft_ann_bench/get_dataset/hdf5_to_fbin.py deleted file mode 100755 index ba853c63f5..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/get_dataset/hdf5_to_fbin.py +++ /dev/null @@ -1,90 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import sys - -import h5py -import numpy as np - - -def normalize(x): - norm = np.linalg.norm(x, axis=1) - return (x.T / norm).T - - -def write_bin(fname, data): - with open(fname, "wb") as f: - np.asarray(data.shape, dtype=np.uint32).tofile(f) - data.tofile(f) - - -if __name__ == "__main__": - if len(sys.argv) != 2 and len(sys.argv) != 3: - print( - "usage: %s [-n] .hdf5\n" % (sys.argv[0]), - " -n: normalize base/query set\n", - "outputs: .base.fbin\n", - " .query.fbin\n", - " .groundtruth.neighbors.ibin\n", - " .groundtruth.distances.fbin", - file=sys.stderr, - ) - sys.exit(-1) - - need_normalize = False - if len(sys.argv) == 3: - assert sys.argv[1] == "-n" - need_normalize = True - fname_prefix = sys.argv[-1] - assert fname_prefix.endswith(".hdf5") - fname_prefix = fname_prefix[:-5] - - hdf5 = h5py.File(sys.argv[-1], "r") - assert ( - hdf5.attrs["distance"] == "angular" - or hdf5.attrs["distance"] == "euclidean" - ) - assert hdf5["train"].dtype == np.float32 - assert hdf5["test"].dtype == np.float32 - assert hdf5["neighbors"].dtype == np.int32 - assert hdf5["distances"].dtype == np.float32 - - base = hdf5["train"][:] - query = hdf5["test"][:] - if need_normalize: - base = normalize(base) - query = normalize(query) - elif hdf5.attrs["distance"] == "angular": - print( - "warning: input has angular distance, ", - "specify -n to normalize base/query set!\n", - ) - - output_fname = fname_prefix + ".base.fbin" - print("writing", output_fname, "...") - write_bin(output_fname, base) - - output_fname = fname_prefix + ".query.fbin" - print("writing", output_fname, "...") - write_bin(output_fname, query) - - output_fname = fname_prefix + ".groundtruth.neighbors.ibin" - print("writing", output_fname, "...") - write_bin(output_fname, hdf5["neighbors"][:]) - - output_fname = fname_prefix + ".groundtruth.distances.fbin" - print("writing", output_fname, "...") - write_bin(output_fname, hdf5["distances"][:]) diff --git a/python/raft-ann-bench/src/raft_ann_bench/plot/__main__.py b/python/raft-ann-bench/src/raft_ann_bench/plot/__main__.py deleted file mode 100644 index 86fd527f5f..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/plot/__main__.py +++ /dev/null @@ -1,623 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script is inspired by -# 1: https://github.com/erikbern/ann-benchmarks/blob/main/plot.py -# 2: https://github.com/erikbern/ann-benchmarks/blob/main/ann_benchmarks/plotting/utils.py # noqa: E501 -# 3: https://github.com/erikbern/ann-benchmarks/blob/main/ann_benchmarks/plotting/metrics.py # noqa: E501 -# Licence: https://github.com/erikbern/ann-benchmarks/blob/main/LICENSE - -import argparse -import itertools -import os -import sys -from collections import OrderedDict - -import matplotlib as mpl -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd - -mpl.use("Agg") - -metrics = { - "k-nn": { - "description": "Recall", - "worst": float("-inf"), - "lim": [0.0, 1.03], - }, - "throughput": { - "description": "Queries per second (1/s)", - "worst": float("-inf"), - }, - "latency": { - "description": "Search Latency (s)", - "worst": float("inf"), - }, -} - - -def positive_int(input_str: str) -> int: - try: - i = int(input_str) - if i < 1: - raise ValueError - except ValueError: - raise argparse.ArgumentTypeError( - f"{input_str} is not a positive integer" - ) - - return i - - -def positive_float(input_str: str) -> float: - try: - i = float(input_str) - if i < 0.0: - raise ValueError - except ValueError: - raise argparse.ArgumentTypeError( - f"{input_str} is not a positive float" - ) - - return i - - -def generate_n_colors(n): - vs = np.linspace(0.3, 0.9, 7) - colors = [(0.9, 0.4, 0.4, 1.0)] - - def euclidean(a, b): - return sum((x - y) ** 2 for x, y in zip(a, b)) - - while len(colors) < n: - new_color = max( - itertools.product(vs, vs, vs), - key=lambda a: min(euclidean(a, b) for b in colors), - ) - colors.append(new_color + (1.0,)) - return colors - - -def create_linestyles(unique_algorithms): - colors = dict( - zip(unique_algorithms, generate_n_colors(len(unique_algorithms))) - ) - linestyles = dict( - (algo, ["--", "-.", "-", ":"][i % 4]) - for i, algo in enumerate(unique_algorithms) - ) - markerstyles = dict( - (algo, ["+", "<", "o", "*", "x"][i % 5]) - for i, algo in enumerate(unique_algorithms) - ) - faded = dict( - (algo, (r, g, b, 0.3)) for algo, (r, g, b, a) in colors.items() - ) - return dict( - ( - algo, - (colors[algo], faded[algo], linestyles[algo], markerstyles[algo]), - ) - for algo in unique_algorithms - ) - - -def create_plot_search( - all_data, - x_scale, - y_scale, - fn_out, - linestyles, - dataset, - k, - batch_size, - mode, - time_unit, - x_start, -): - xn = "k-nn" - xm, ym = (metrics[xn], metrics[mode]) - xm["lim"][0] = x_start - # Now generate each plot - handles = [] - labels = [] - plt.figure(figsize=(12, 9)) - - # Sorting by mean y-value helps aligning plots with labels - def mean_y(algo): - points = np.array(all_data[algo], dtype=object) - return -np.log(np.array(points[:, 3], dtype=np.float32)).mean() - - # Find range for logit x-scale - min_x, max_x = 1, 0 - for algo in sorted(all_data.keys(), key=mean_y): - points = np.array(all_data[algo], dtype=object) - xs = points[:, 2] - ys = points[:, 3] - min_x = min([min_x] + [x for x in xs if x > 0]) - max_x = max([max_x] + [x for x in xs if x < 1]) - color, faded, linestyle, marker = linestyles[algo] - (handle,) = plt.plot( - xs, - ys, - "-", - label=algo, - color=color, - ms=7, - mew=3, - lw=3, - marker=marker, - ) - handles.append(handle) - - labels.append(algo) - - ax = plt.gca() - y_description = ym["description"] - if mode == "latency": - y_description = y_description.replace("(s)", f"({time_unit})") - ax.set_ylabel(y_description) - ax.set_xlabel("Recall") - # Custom scales of the type --x-scale a3 - if x_scale[0] == "a": - alpha = float(x_scale[1:]) - - def fun(x): - return 1 - (1 - x) ** (1 / alpha) - - def inv_fun(x): - return 1 - (1 - x) ** alpha - - ax.set_xscale("function", functions=(fun, inv_fun)) - if alpha <= 3: - ticks = [inv_fun(x) for x in np.arange(0, 1.2, 0.2)] - plt.xticks(ticks) - if alpha > 3: - from matplotlib import ticker - - ax.xaxis.set_major_formatter(ticker.LogitFormatter()) - # plt.xticks(ticker.LogitLocator().tick_values(min_x, max_x)) - plt.xticks([0, 1 / 2, 1 - 1e-1, 1 - 1e-2, 1 - 1e-3, 1 - 1e-4, 1]) - # Other x-scales - else: - ax.set_xscale(x_scale) - ax.set_yscale(y_scale) - ax.set_title(f"{dataset} k={k} batch_size={batch_size}") - plt.gca().get_position() - # plt.gca().set_position([box.x0, box.y0, box.width * 0.8, box.height]) - ax.legend( - handles, - labels, - loc="center left", - bbox_to_anchor=(1, 0.5), - prop={"size": 9}, - ) - plt.grid(visible=True, which="major", color="0.65", linestyle="-") - plt.setp(ax.get_xminorticklabels(), visible=True) - - # Logit scale has to be a subset of (0,1) - if "lim" in xm and x_scale != "logit": - x0, x1 = xm["lim"] - plt.xlim(max(x0, 0), min(x1, 1)) - elif x_scale == "logit": - plt.xlim(min_x, max_x) - if "lim" in ym: - plt.ylim(ym["lim"]) - - # Workaround for bug https://github.com/matplotlib/matplotlib/issues/6789 - ax.spines["bottom"]._adjust_location() - - print(f"writing search output to {fn_out}") - plt.savefig(fn_out, bbox_inches="tight") - plt.close() - - -def create_plot_build( - build_results, search_results, linestyles, fn_out, dataset, k, batch_size -): - bt_80 = [0] * len(linestyles) - - bt_90 = [0] * len(linestyles) - - bt_95 = [0] * len(linestyles) - - bt_99 = [0] * len(linestyles) - - data = OrderedDict() - colors = OrderedDict() - - # Sorting by mean y-value helps aligning plots with labels - - def mean_y(algo): - points = np.array(search_results[algo], dtype=object) - return -np.log(np.array(points[:, 3], dtype=np.float32)).mean() - - for pos, algo in enumerate(sorted(search_results.keys(), key=mean_y)): - points = np.array(search_results[algo], dtype=object) - # x is recall, ls is algo_name, idxs is index_name - xs = points[:, 2] - ls = points[:, 0] - idxs = points[:, 1] - - len_80, len_90, len_95, len_99 = 0, 0, 0, 0 - for i in range(len(xs)): - if xs[i] >= 0.80 and xs[i] < 0.90: - bt_80[pos] = bt_80[pos] + build_results[(ls[i], idxs[i])][0][2] - len_80 = len_80 + 1 - elif xs[i] >= 0.9 and xs[i] < 0.95: - bt_90[pos] = bt_90[pos] + build_results[(ls[i], idxs[i])][0][2] - len_90 = len_90 + 1 - elif xs[i] >= 0.95 and xs[i] < 0.99: - bt_95[pos] = bt_95[pos] + build_results[(ls[i], idxs[i])][0][2] - len_95 = len_95 + 1 - elif xs[i] >= 0.99: - bt_99[pos] = bt_99[pos] + build_results[(ls[i], idxs[i])][0][2] - len_99 = len_99 + 1 - if len_80 > 0: - bt_80[pos] = bt_80[pos] / len_80 - if len_90 > 0: - bt_90[pos] = bt_90[pos] / len_90 - if len_95 > 0: - bt_95[pos] = bt_95[pos] / len_95 - if len_99 > 0: - bt_99[pos] = bt_99[pos] / len_99 - data[algo] = [ - bt_80[pos], - bt_90[pos], - bt_95[pos], - bt_99[pos], - ] - colors[algo] = linestyles[algo][0] - - index = [ - "@80% Recall", - "@90% Recall", - "@95% Recall", - "@99% Recall", - ] - - df = pd.DataFrame(data, index=index) - df.replace(0.0, np.nan, inplace=True) - df = df.dropna(how="all") - plt.figure(figsize=(12, 9)) - ax = df.plot.bar(rot=0, color=colors) - fig = ax.get_figure() - print(f"writing build output to {fn_out}") - plt.title( - "Average Build Time within Recall Range " - f"for k={k} batch_size={batch_size}" - ) - plt.suptitle(f"{dataset}") - plt.ylabel("Build Time (s)") - fig.savefig(fn_out) - - -def load_lines(results_path, result_files, method, index_key, mode, time_unit): - results = dict() - - for result_filename in result_files: - try: - with open(os.path.join(results_path, result_filename), "r") as f: - lines = f.readlines() - lines = lines[:-1] if lines[-1] == "\n" else lines - - if method == "build": - key_idx = [2] - elif method == "search": - y_idx = 3 if mode == "throughput" else 4 - key_idx = [2, y_idx] - - for line in lines[1:]: - split_lines = line.split(",") - - algo_name = split_lines[0] - index_name = split_lines[1] - - if index_key == "algo": - dict_key = algo_name - elif index_key == "index": - dict_key = (algo_name, index_name) - if dict_key not in results: - results[dict_key] = [] - to_add = [algo_name, index_name] - for key_i in key_idx: - to_add.append(float(split_lines[key_i])) - if ( - mode == "latency" - and time_unit != "s" - and method == "search" - ): - to_add[-1] = ( - to_add[-1] * (10**3) - if time_unit == "ms" - else to_add[-1] * (10**6) - ) - results[dict_key].append(to_add) - except Exception: - print( - f"An error occurred processing file {result_filename}. " - "Skipping..." - ) - - return results - - -def load_all_results( - dataset_path, - algorithms, - groups, - algo_groups, - k, - batch_size, - method, - index_key, - raw, - mode, - time_unit, -): - results_path = os.path.join(dataset_path, "result", method) - result_files = os.listdir(results_path) - if method == "build": - result_files = [ - result_file - for result_file in result_files - if ".csv" in result_file - ] - elif method == "search": - if raw: - suffix = ",raw" - else: - suffix = f",{mode}" - result_files = [ - result_file - for result_file in result_files - if f"{suffix}.csv" in result_file - ] - if len(result_files) == 0: - raise FileNotFoundError(f"No CSV result files found in {results_path}") - - if method == "search": - filter_k_bs = [] - for result_filename in result_files: - filename_split = result_filename.split(",") - if ( - int(filename_split[-3][1:]) == k - and int(filename_split[-2][2:]) == batch_size - ): - filter_k_bs.append(result_filename) - result_files = filter_k_bs - - algo_group_files = [ - result_filename.replace(".csv", "").split(",")[:2] - for result_filename in result_files - ] - algo_group_files = list(zip(*algo_group_files)) - - if len(algorithms) > 0: - final_results = [ - result_files[i] - for i in range(len(result_files)) - if (algo_group_files[0][i] in algorithms) - and (algo_group_files[1][i] in groups) - ] - else: - final_results = [ - result_files[i] - for i in range(len(result_files)) - if (algo_group_files[1][i] in groups) - ] - - if len(algo_groups) > 0: - split_algo_groups = [ - algo_group.split(".") for algo_group in algo_groups - ] - split_algo_groups = list(zip(*split_algo_groups)) - final_algo_groups = [ - result_files[i] - for i in range(len(result_files)) - if (algo_group_files[0][i] in split_algo_groups[0]) - and (algo_group_files[1][i] in split_algo_groups[1]) - ] - final_results = final_results + final_algo_groups - final_results = set(final_results) - - results = load_lines( - results_path, final_results, method, index_key, mode, time_unit - ) - - return results - - -def main(): - call_path = os.getcwd() - if "RAPIDS_DATASET_ROOT_DIR" in os.environ: - default_dataset_path = os.getenv("RAPIDS_DATASET_ROOT_DIR") - else: - default_dataset_path = os.path.join(call_path, "datasets/") - - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - "--dataset", help="dataset to plot", default="glove-100-inner" - ) - parser.add_argument( - "--dataset-path", - help="path to dataset folder", - default=default_dataset_path, - ) - parser.add_argument( - "--output-filepath", - help="directory for PNG to be saved", - default=os.getcwd(), - ) - parser.add_argument( - "--algorithms", - help="plot only comma separated list of named \ - algorithms. If parameters `groups` and `algo-groups \ - are both undefined, then group `base` is plot by default", - default=None, - ) - parser.add_argument( - "--groups", - help="plot only comma separated groups of parameters", - default="base", - ) - parser.add_argument( - "--algo-groups", - "--algo-groups", - help='add comma separated . to plot. \ - Example usage: "--algo-groups=raft_cagra.large,hnswlib.large"', - ) - parser.add_argument( - "-k", - "--count", - default=10, - type=positive_int, - help="the number of nearest neighbors to search for", - ) - parser.add_argument( - "-bs", - "--batch-size", - default=10000, - type=positive_int, - help="number of query vectors to use in each query trial", - ) - parser.add_argument("--build", action="store_true") - parser.add_argument("--search", action="store_true") - parser.add_argument( - "--x-scale", - help="Scale to use when drawing the X-axis. \ - Typically linear, logit or a2", - default="linear", - ) - parser.add_argument( - "--y-scale", - help="Scale to use when drawing the Y-axis", - choices=["linear", "log", "symlog", "logit"], - default="linear", - ) - parser.add_argument( - "--x-start", - help="Recall values to start the x-axis from", - default=0.8, - type=positive_float, - ) - parser.add_argument( - "--mode", - help="search mode whose Pareto frontier is used on the y-axis", - choices=["throughput", "latency"], - default="throughput", - ) - parser.add_argument( - "--time-unit", - help="time unit to plot when mode is latency", - choices=["s", "ms", "us"], - default="ms", - ) - parser.add_argument( - "--raw", - help="Show raw results (not just Pareto frontier) of mode arg", - action="store_true", - ) - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - args = parser.parse_args() - - if args.algorithms: - algorithms = args.algorithms.split(",") - else: - algorithms = [] - groups = args.groups.split(",") - if args.algo_groups: - algo_groups = args.algo_groups.split(",") - else: - algo_groups = [] - k = args.count - batch_size = args.batch_size - if not args.build and not args.search: - build = True - search = True - else: - build = args.build - search = args.search - - search_output_filepath = os.path.join( - args.output_filepath, - f"search-{args.dataset}-k{k}-batch_size{batch_size}.png", - ) - build_output_filepath = os.path.join( - args.output_filepath, - f"build-{args.dataset}-k{k}-batch_size{batch_size}.png", - ) - - search_results = load_all_results( - os.path.join(args.dataset_path, args.dataset), - algorithms, - groups, - algo_groups, - k, - batch_size, - "search", - "algo", - args.raw, - args.mode, - args.time_unit, - ) - linestyles = create_linestyles(sorted(search_results.keys())) - if search: - create_plot_search( - search_results, - args.x_scale, - args.y_scale, - search_output_filepath, - linestyles, - args.dataset, - k, - batch_size, - args.mode, - args.time_unit, - args.x_start, - ) - if build: - build_results = load_all_results( - os.path.join(args.dataset_path, args.dataset), - algorithms, - groups, - algo_groups, - k, - batch_size, - "build", - "index", - args.raw, - args.mode, - args.time_unit, - ) - create_plot_build( - build_results, - search_results, - linestyles, - build_output_filepath, - args.dataset, - k, - batch_size, - ) - - -if __name__ == "__main__": - main() diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/__main__.py b/python/raft-ann-bench/src/raft_ann_bench/run/__main__.py deleted file mode 100644 index c34377d733..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/__main__.py +++ /dev/null @@ -1,614 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import itertools -import json -import os -import subprocess -import sys -import uuid -import warnings -from importlib import import_module - -import yaml - -log_levels = { - "off": 0, - "error": 1, - "warn": 2, - "info": 3, - "debug": 4, - "trace": 5, -} - - -def parse_log_level(level_str): - if level_str not in log_levels: - raise ValueError("Invalid log level: %s" % level_str) - return log_levels[level_str.lower()] - - -def positive_int(input_str: str) -> int: - try: - i = int(input_str) - if i < 1: - raise ValueError - except ValueError: - raise argparse.ArgumentTypeError( - f"{input_str} is not a positive integer" - ) - - return i - - -def merge_build_files(build_dir, build_file, temp_build_file): - - build_dict = {} - - # If build file exists, read it - build_json_path = os.path.join(build_dir, build_file) - tmp_build_json_path = os.path.join(build_dir, temp_build_file) - if os.path.isfile(build_json_path): - try: - with open(build_json_path, "r") as f: - build_dict = json.load(f) - except Exception as e: - print( - "Error loading existing build file: %s (%s)" - % (build_json_path, e) - ) - - temp_build_dict = {} - if os.path.isfile(tmp_build_json_path): - with open(tmp_build_json_path, "r") as f: - temp_build_dict = json.load(f) - else: - raise ValueError("Temp build file not found: %s" % tmp_build_json_path) - - tmp_benchmarks = ( - temp_build_dict["benchmarks"] - if "benchmarks" in temp_build_dict - else {} - ) - benchmarks = build_dict["benchmarks"] if "benchmarks" in build_dict else {} - - # If the build time is absolute 0 then an error occurred - final_bench_dict = {} - for b in benchmarks: - if b["real_time"] > 0: - final_bench_dict[b["name"]] = b - - for tmp_bench in tmp_benchmarks: - if tmp_bench["real_time"] > 0: - final_bench_dict[tmp_bench["name"]] = tmp_bench - - temp_build_dict["benchmarks"] = [v for k, v in final_bench_dict.items()] - with open(build_json_path, "w") as f: - json_str = json.dumps(temp_build_dict, indent=2) - f.write(json_str) - - -def validate_algorithm(algos_conf, algo, gpu_present): - algos_conf_keys = set(algos_conf.keys()) - if gpu_present: - return algo in algos_conf_keys - else: - return ( - algo in algos_conf_keys - and algos_conf[algo]["requires_gpu"] is False - ) - - -def find_executable(algos_conf, algo, group, k, batch_size): - executable = algos_conf[algo]["executable"] - - file_name = (f"{algo},{group}", f"{algo},{group},k{k},bs{batch_size}") - - build_path = os.getenv("RAFT_HOME") - if build_path is not None: - build_path = os.path.join( - build_path, "cpp", "build", "release", executable - ) - if os.path.exists(build_path): - print(f"-- Using RAFT bench from repository in {build_path}. ") - return (executable, build_path, file_name) - - # if there is no build folder present, we look in the conda environment - conda_path = os.getenv("CONDA_PREFIX") - if conda_path is not None: - conda_path = os.path.join(conda_path, "bin", "ann", executable) - if os.path.exists(conda_path): - print("-- Using RAFT bench found in conda environment. ") - return (executable, conda_path, file_name) - - else: - raise FileNotFoundError(executable) - - -def run_build_and_search( - conf_file, - conf_filename, - conf_filedir, - executables_to_run, - dataset_path, - force, - build, - search, - dry_run, - k, - batch_size, - search_threads, - mode="throughput", - raft_log_level="info", -): - for ( - executable, - ann_executable_path, - output_filename, - ) in executables_to_run.keys(): - # Need to write temporary configuration - temp_conf_filename = ( - f"{conf_filename}_{output_filename[1]}_{uuid.uuid1()}.json" - ) - with open(temp_conf_filename, "w") as f: - temp_conf = dict() - temp_conf["dataset"] = conf_file["dataset"] - temp_conf["search_basic_param"] = conf_file["search_basic_param"] - temp_conf["index"] = executables_to_run[ - (executable, ann_executable_path, output_filename) - ]["index"] - json_str = json.dumps(temp_conf, indent=2) - f.write(json_str) - - legacy_result_folder = os.path.join( - dataset_path, conf_file["dataset"]["name"], "result" - ) - os.makedirs(legacy_result_folder, exist_ok=True) - if build: - build_folder = os.path.join(legacy_result_folder, "build") - os.makedirs(build_folder, exist_ok=True) - build_file = f"{output_filename[0]}.json" - temp_build_file = f"{build_file}.lock" - cmd = [ - ann_executable_path, - "--build", - "--data_prefix=" + dataset_path, - "--benchmark_out_format=json", - "--benchmark_counters_tabular=true", - "--benchmark_out=" - + f"{os.path.join(build_folder, temp_build_file)}", - "--raft_log_level=" + f"{parse_log_level(raft_log_level)}", - ] - if force: - cmd = cmd + ["--force"] - cmd = cmd + [temp_conf_filename] - - if dry_run: - print( - "Benchmark command for %s:\n%s\n" - % (output_filename[0], " ".join(cmd)) - ) - else: - try: - subprocess.run(cmd, check=True) - merge_build_files( - build_folder, build_file, temp_build_file - ) - except Exception as e: - print("Error occurred running benchmark: %s" % e) - finally: - os.remove(os.path.join(build_folder, temp_build_file)) - if not search: - os.remove(temp_conf_filename) - - if search: - search_folder = os.path.join(legacy_result_folder, "search") - os.makedirs(search_folder, exist_ok=True) - search_file = f"{output_filename[1]}.json" - cmd = [ - ann_executable_path, - "--search", - "--data_prefix=" + dataset_path, - "--benchmark_counters_tabular=true", - "--override_kv=k:%s" % k, - "--override_kv=n_queries:%s" % batch_size, - "--benchmark_min_warmup_time=1", - "--benchmark_out_format=json", - "--mode=%s" % mode, - "--benchmark_out=" - + f"{os.path.join(search_folder, search_file)}", - "--raft_log_level=" + f"{parse_log_level(raft_log_level)}", - ] - if force: - cmd = cmd + ["--force"] - - if search_threads: - cmd = cmd + ["--threads=%s" % search_threads] - - cmd = cmd + [temp_conf_filename] - if dry_run: - print( - "Benchmark command for %s:\n%s\n" - % (output_filename[1], " ".join(cmd)) - ) - else: - try: - subprocess.run(cmd, check=True) - except Exception as e: - print("Error occurred running benchmark: %s" % e) - finally: - os.remove(temp_conf_filename) - - -def main(): - scripts_path = os.path.dirname(os.path.realpath(__file__)) - call_path = os.getcwd() - - # Read list of allowed algorithms - try: - import rmm # noqa: F401 - - gpu_present = True - except ImportError: - gpu_present = False - - with open(f"{scripts_path}/algos.yaml", "r") as f: - algos_yaml = yaml.safe_load(f) - - if "RAPIDS_DATASET_ROOT_DIR" in os.environ: - default_dataset_path = os.getenv("RAPIDS_DATASET_ROOT_DIR") - else: - default_dataset_path = os.path.join(call_path, "datasets/") - - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - - parser.add_argument( - "--subset-size", - type=positive_int, - help="the number of subset rows of the dataset to build the index", - ) - parser.add_argument( - "-k", - "--count", - default=10, - type=positive_int, - help="the number of nearest neighbors to search for", - ) - parser.add_argument( - "-bs", - "--batch-size", - default=10000, - type=positive_int, - help="number of query vectors to use in each query trial", - ) - parser.add_argument( - "--dataset-configuration", - help="path to YAML configuration file for datasets", - ) - parser.add_argument( - "--configuration", - help="path to YAML configuration file or directory for algorithms\ - Any run groups found in the specified file/directory will \ - automatically override groups of the same name present in the \ - default configurations, including `base`", - ) - parser.add_argument( - "--dataset", - help="name of dataset", - default="glove-100-inner", - ) - parser.add_argument( - "--dataset-path", - help="path to dataset folder, by default will look in " - "RAPIDS_DATASET_ROOT_DIR if defined, otherwise a datasets " - "subdirectory from the calling directory", - default=default_dataset_path, - ) - parser.add_argument("--build", action="store_true") - parser.add_argument("--search", action="store_true") - parser.add_argument( - "--algorithms", - help="run only comma separated list of named \ - algorithms. If parameters `groups` and `algo-groups \ - are both undefined, then group `base` is run by default", - default=None, - ) - parser.add_argument( - "--groups", - help="run only comma separated groups of parameters", - default="base", - ) - parser.add_argument( - "--algo-groups", - help='add comma separated . to run. \ - Example usage: "--algo-groups=raft_cagra.large,hnswlib.large"', - ) - parser.add_argument( - "-f", - "--force", - help="re-run algorithms even if their results \ - already exist", - action="store_true", - ) - - parser.add_argument( - "-m", - "--search-mode", - help="run search in 'latency' (measure individual batches) or " - "'throughput' (pipeline batches and measure end-to-end) mode", - default="latency", - ) - - parser.add_argument( - "-t", - "--search-threads", - help="specify the number threads to use for throughput benchmark." - " Single value or a pair of min and max separated by ':'. " - "Example: --search-threads=1:4. Power of 2 values between 'min' " - "and 'max' will be used. If only 'min' is specified, then a " - "single test is run with 'min' threads. By default min=1, " - "max=.", - default=None, - ) - - parser.add_argument( - "-r", - "--dry-run", - help="dry-run mode will convert the yaml config for the specified " - "algorithms and datasets to the json format that's consumed " - "by the lower-level c++ binaries and then print the command " - "to run execute the benchmarks but will not actually execute " - "the command.", - action="store_true", - ) - parser.add_argument( - "--raft-log-level", - help="Log level, possible values are " - "[off, error, warn, info, debug, trace]. " - "Default: 'info'. Note that 'debug' or more detailed " - "logging level requires that the library is compiled with " - "-DRAFT_ACTIVE_LEVEL= where >= ", - default="info", - ) - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - args = parser.parse_args() - - # If both build and search are not provided, - # run both - if not args.build and not args.search: - build = True - search = True - else: - build = args.build - search = args.search - - dry_run = args.dry_run - - mode = args.search_mode - k = args.count - batch_size = args.batch_size - - # Read configuration file associated to datasets - if args.dataset_configuration: - dataset_conf_f = args.dataset_configuration - else: - dataset_conf_f = os.path.join(scripts_path, "conf", "datasets.yaml") - with open(dataset_conf_f, "r") as f: - dataset_conf_all = yaml.safe_load(f) - - dataset_conf = None - for dataset in dataset_conf_all: - if args.dataset == dataset["name"]: - dataset_conf = dataset - break - if not dataset_conf: - raise ValueError("Could not find a dataset configuration") - - conf_file = dict() - conf_file["dataset"] = dataset_conf - if args.subset_size: - conf_file["dataset"]["subset_size"] = args.subset_size - - conf_file["search_basic_param"] = {} - conf_file["search_basic_param"]["k"] = k - conf_file["search_basic_param"]["batch_size"] = batch_size - - algos_conf_fs = os.listdir(os.path.join(scripts_path, "conf", "algos")) - algos_conf_fs = [ - os.path.join(scripts_path, "conf", "algos", f) - for f in algos_conf_fs - if ".json" not in f - ] - conf_filedir = os.path.join(scripts_path, "conf", "algos") - if args.configuration: - if os.path.isdir(args.configuration): - conf_filedir = args.configuration - algos_conf_fs = algos_conf_fs + [ - os.path.join(args.configuration, f) - for f in os.listdir(args.configuration) - if ".json" not in f - ] - elif os.path.isfile(args.configuration): - conf_filedir = os.path.normpath(args.configuration).split(os.sep) - conf_filedir = os.path.join(*conf_filedir[:-1]) - algos_conf_fs = algos_conf_fs + [args.configuration] - - filter_algos = True if args.algorithms else False - if filter_algos: - allowed_algos = args.algorithms.split(",") - named_groups = args.groups.split(",") - filter_algo_groups = True if args.algo_groups else False - allowed_algo_groups = None - if filter_algo_groups: - allowed_algo_groups = [ - algo_group.split(".") for algo_group in args.algo_groups.split(",") - ] - allowed_algo_groups = list(zip(*allowed_algo_groups)) - algos_conf = dict() - for algo_f in algos_conf_fs: - with open(algo_f, "r") as f: - try: - algo = yaml.safe_load(f) - except Exception as e: - warnings.warn( - f"Could not load YAML config {algo_f} due to " - + e.with_traceback() - ) - continue - insert_algo = True - insert_algo_group = False - if filter_algos: - if algo["name"] not in allowed_algos: - insert_algo = False - if filter_algo_groups: - if algo["name"] in allowed_algo_groups[0]: - insert_algo_group = True - - def add_algo_group(group_list): - if algo["name"] not in algos_conf: - algos_conf[algo["name"]] = {"groups": {}} - for group in algo["groups"].keys(): - if group in group_list: - algos_conf[algo["name"]]["groups"][group] = algo[ - "groups" - ][group] - if "constraints" in algo: - algos_conf[algo["name"]]["constraints"] = algo[ - "constraints" - ] - - if insert_algo: - add_algo_group(named_groups) - if insert_algo_group: - add_algo_group(allowed_algo_groups[1]) - - executables_to_run = dict() - for algo in algos_conf.keys(): - validate_algorithm(algos_yaml, algo, gpu_present) - for group in algos_conf[algo]["groups"].keys(): - executable = find_executable( - algos_yaml, algo, group, k, batch_size - ) - if executable not in executables_to_run: - executables_to_run[executable] = {"index": []} - build_params = algos_conf[algo]["groups"][group]["build"] or {} - search_params = algos_conf[algo]["groups"][group]["search"] or {} - - param_names = [] - param_lists = [] - for param in build_params.keys(): - param_names.append(param) - param_lists.append(build_params[param]) - - all_build_params = itertools.product(*param_lists) - - search_param_names = [] - search_param_lists = [] - for search_param in search_params.keys(): - search_param_names.append(search_param) - search_param_lists.append(search_params[search_param]) - - for params in all_build_params: - index = {"algo": algo, "build_param": {}} - if group != "base": - index_name = f"{algo}_{group}" - else: - index_name = f"{algo}" - for i in range(len(params)): - index["build_param"][param_names[i]] = params[i] - index_name += "." + f"{param_names[i]}{params[i]}" - - if "constraints" in algos_conf[algo]: - if "build" in algos_conf[algo]["constraints"]: - importable = algos_conf[algo]["constraints"]["build"] - importable = importable.split(".") - module = ".".join(importable[:-1]) - func = importable[-1] - validator = import_module(module) - build_constraints = getattr(validator, func) - if "dims" not in conf_file["dataset"]: - raise ValueError( - "`dims` needed for build constraints but not " - "specified in datasets.yaml" - ) - if not build_constraints( - index["build_param"], conf_file["dataset"]["dims"] - ): - continue - index_filename = ( - index_name - if len(index_name) < 128 - else str(hash(index_name)) - ) - index["name"] = index_name - index["file"] = os.path.join( - args.dataset_path, args.dataset, "index", index_filename - ) - index["search_params"] = [] - all_search_params = itertools.product(*search_param_lists) - for search_params in all_search_params: - search_dict = dict() - for i in range(len(search_params)): - search_dict[search_param_names[i]] = search_params[i] - if "constraints" in algos_conf[algo]: - if "search" in algos_conf[algo]["constraints"]: - importable = algos_conf[algo]["constraints"][ - "search" - ] - importable = importable.split(".") - module = ".".join(importable[:-1]) - func = importable[-1] - validator = import_module(module) - search_constraints = getattr(validator, func) - if search_constraints( - search_dict, - index["build_param"], - k, - batch_size, - ): - index["search_params"].append(search_dict) - else: - index["search_params"].append(search_dict) - executables_to_run[executable]["index"].append(index) - - if len(index["search_params"]) == 0: - print("No search parameters were added to configuration") - - run_build_and_search( - conf_file, - f"{args.dataset}", - conf_filedir, - executables_to_run, - args.dataset_path, - args.force, - build, - search, - dry_run, - k, - batch_size, - args.search_threads, - mode, - args.raft_log_level, - ) - - -if __name__ == "__main__": - main() diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/algos.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/algos.yaml deleted file mode 100644 index e382bdcba6..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/algos.yaml +++ /dev/null @@ -1,42 +0,0 @@ -faiss_gpu_flat: - executable: FAISS_GPU_FLAT_ANN_BENCH - requires_gpu: true -faiss_gpu_ivf_flat: - executable: FAISS_GPU_IVF_FLAT_ANN_BENCH - requires_gpu: true -faiss_gpu_ivf_pq: - executable: FAISS_GPU_IVF_PQ_ANN_BENCH - requires_gpu: true -faiss_gpu_ivf_sq: - executable: FAISS_GPU_IVF_PQ_ANN_BENCH - requires_gpu: true -faiss_cpu_flat: - executable: FAISS_CPU_FLAT_ANN_BENCH - requires_gpu: false -faiss_cpu_ivf_flat: - executable: FAISS_CPU_IVF_FLAT_ANN_BENCH - requires_gpu: false -faiss_cpu_ivf_pq: - executable: FAISS_CPU_IVF_PQ_ANN_BENCH - requires_gpu: false -raft_ivf_flat: - executable: RAFT_IVF_FLAT_ANN_BENCH - requires_gpu: true -raft_ivf_pq: - executable: RAFT_IVF_PQ_ANN_BENCH - requires_gpu: true -raft_cagra: - executable: RAFT_CAGRA_ANN_BENCH - requires_gpu: true -raft_brute_force: - executable: RAFT_BRUTE_FORCE_ANN_BENCH - requires_gpu: true -ggnn: - executable: GGNN_ANN_BENCH - requires_gpu: true -hnswlib: - executable: HNSWLIB_ANN_BENCH - requires_gpu: false -raft_cagra_hnswlib: - executable: RAFT_CAGRA_HNSWLIB_ANN_BENCH - requires_gpu: true diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_cpu_flat.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_cpu_flat.yaml deleted file mode 100644 index 25eaf03d40..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_cpu_flat.yaml +++ /dev/null @@ -1,5 +0,0 @@ -name: faiss_cpu_flat -groups: - base: - build: - search: diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_cpu_ivf_flat.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_cpu_ivf_flat.yaml deleted file mode 100644 index 29c145f86d..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_cpu_ivf_flat.yaml +++ /dev/null @@ -1,10 +0,0 @@ -name: faiss_cpu_ivf_flat -groups: - base: - build: - nlist: [2048] - ratio: [10] - useFloat16: [False] - search: - nprobe: [1, 5, 10, 50, 100, 200] - refine_ratio: [1] \ No newline at end of file diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_cpu_ivf_pq.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_cpu_ivf_pq.yaml deleted file mode 100644 index a531ec8294..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_cpu_ivf_pq.yaml +++ /dev/null @@ -1,18 +0,0 @@ -name: faiss_cpu_ivf_pq -groups: - base: - build: - nlist: [1024, 2048, 4096, 8192] - M: [48, 32, 16] - ratio: [10] - bitsPerCode: [8, 6, 5, 4] - search: - nprobe: [1, 5, 10, 50, 100, 200] - large: - build: - nlist: [8192, 16384, 32768, 65536] - M: [48, 32, 16] - ratio: [10] - bitsPerCode: [8, 6, 5, 4] - search: - nprobe: [20, 30, 40, 50, 100, 200, 500, 1000] diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_gpu_flat.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_gpu_flat.yaml deleted file mode 100644 index a722e1b91c..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_gpu_flat.yaml +++ /dev/null @@ -1,5 +0,0 @@ -name: faiss_gpu_flat -groups: - base: - build: - search: diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_gpu_ivf_flat.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_gpu_ivf_flat.yaml deleted file mode 100644 index e4abc35f5c..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_gpu_ivf_flat.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: faiss_gpu_ivf_flat -groups: - base: - build: - nlist: [2048] - ratio: [10] - useFloat16: [False, True] - use_raft: [False] - search: - nprobe: [1, 5, 10, 50, 100, 200] - refine_ratio: [1] -groups: - baseraft: - build: - nlist: [2048] - ratio: [10] - useFloat16: [False, True] - use_raft: [True] - search: - nprobe: [1, 5, 10, 50, 100, 200] - refine_ratio: [1] \ No newline at end of file diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_gpu_ivf_pq.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_gpu_ivf_pq.yaml deleted file mode 100644 index 7560ceaa9c..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_gpu_ivf_pq.yaml +++ /dev/null @@ -1,77 +0,0 @@ -name: faiss_gpu_ivf_pq -constraints: - build: raft-ann-bench.constraints.faiss_gpu_ivf_pq_build_constraints - search: raft-ann-bench.constraints.faiss_gpu_ivf_pq_search_constraints -groups: - base: - build: - nlist: [1024, 2048, 4096, 8192] - M: [64, 32, 16] - ratio: [10] - usePrecomputed: [False, True] - useFloat16: [False, True] - use_raft: [False] - bitsPerCode: [8] - search: - nprobe: [1, 5, 10, 50, 100, 200] - refine_ratio: [1, 2, 4] - baseraft: - build: - nlist: [1024, 2048, 4096, 8192] - M: [64, 32, 16] - ratio: [10] - usePrecomputed: [False] - useFloat16: [False, True] - use_raft: [True] - bitsPerCode: [8, 6, 5, 4] - search: - nprobe: [1, 5, 10, 50, 100, 200] - refine_ratio: [1, 2, 4] - large: - build: - nlist: [8192, 16384, 32768, 65536] - M: [48, 32, 16] - ratio: [4] - usePrecomputed: [False, True] - useFloat16: [False, True] - use_raft: [False] - bitsPerCode: [8] - search: - nprobe: [20, 30, 40, 50, 100, 200, 500, 1000] - refine_ratio: [1, 2, 4] - largeraft: - build: - nlist: [8192, 16384, 32768, 65536] - M: [48, 32, 16] - ratio: [4] - usePrecomputed: [False] - useFloat16: [False, True] - use_raft: [True] - bitsPerCode: [8, 6, 5, 4] - search: - nprobe: [20, 30, 40, 50, 100, 200, 500, 1000] - refine_ratio: [1, 2, 4] - 100M: - build: - nlist: [50000] - M: [48] - ratio: [10] - usePrecomputed: [False, True] - useFloat16: [False, True] - use_raft: [False] - bitsPerCode: [8] - search: - nprobe: [20, 30, 40, 50, 100, 200, 500, 1000] - refine_ratio: [1] - 100Mraft: - build: - nlist: [50000] - M: [48] - ratio: [10] - usePrecomputed: [False, True] - useFloat16: [False, True] - use_raft: [True] - bitsPerCode: [8, 6, 5, 4] - search: - nprobe: [20, 30, 40, 50, 100, 200, 500, 1000] - refine_ratio: [1] \ No newline at end of file diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/hnswlib.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/hnswlib.yaml deleted file mode 100644 index e7a4e6b506..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/hnswlib.yaml +++ /dev/null @@ -1,10 +0,0 @@ -name: hnswlib -constraints: - search: raft_ann_bench.constraints.hnswlib_search_constraints -groups: - base: - build: - M: [12, 16, 24, 36] - efConstruction: [64, 128, 256, 512] - search: - ef: [10, 20, 40, 60, 80, 120, 200, 400, 600, 800] diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_brute_force.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_brute_force.yaml deleted file mode 100644 index da99841f9b..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_brute_force.yaml +++ /dev/null @@ -1,5 +0,0 @@ -name: raft_brute_force -groups: - base: - build: - search: diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_cagra.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_cagra.yaml deleted file mode 100644 index bb66b4b232..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_cagra.yaml +++ /dev/null @@ -1,13 +0,0 @@ -name: raft_cagra -constraints: - build: raft_ann_bench.constraints.raft_cagra_build_constraints - search: raft_ann_bench.constraints.raft_cagra_search_constraints -groups: - base: - build: - graph_degree: [32, 64, 128, 256] - intermediate_graph_degree: [32, 64, 96, 128] - graph_build_algo: ["NN_DESCENT"] - search: - itopk: [32, 64, 128, 256, 512] - search_width: [1, 2, 4, 8, 16, 32, 64] diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_cagra_hnswlib.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_cagra_hnswlib.yaml deleted file mode 100644 index 3ac2d16b68..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_cagra_hnswlib.yaml +++ /dev/null @@ -1,11 +0,0 @@ -name: raft_cagra_hnswlib -constraints: - search: raft_ann_bench.constraints.hnswlib_search_constraints -groups: - base: - build: - graph_degree: [32, 64, 128, 256] - intermediate_graph_degree: [32, 64, 96, 128] - graph_build_algo: ["NN_DESCENT"] - search: - ef: [10, 20, 40, 60, 80, 120, 200, 400, 600, 800] diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_ivf_flat.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_ivf_flat.yaml deleted file mode 100644 index c36a26514d..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_ivf_flat.yaml +++ /dev/null @@ -1,9 +0,0 @@ -name: raft_ivf_flat -groups: - base: - build: - nlist: [1024, 2048, 4096, 8192, 16384, 32000, 64000] - ratio: [1, 2, 4] - niter: [20, 25] - search: - nprobe: [1, 5, 10, 50, 100, 200, 500, 1000, 2000] \ No newline at end of file diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_ivf_pq.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_ivf_pq.yaml deleted file mode 100644 index bcdcde42a2..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_ivf_pq.yaml +++ /dev/null @@ -1,41 +0,0 @@ -name: raft_ivf_pq -constraints: - build: raft_ann_bench.constraints.raft_ivf_pq_build_constraints - search: raft_ann_bench.constraints.raft_ivf_pq_search_constraints -groups: - base: - build: - nlist: [1024, 2048, 4096, 8192] - pq_dim: [64, 32, 16] - pq_bits: [8, 6, 5, 4] - ratio: [10] - niter: [25] - search: - nprobe: [1, 5, 10, 50, 100, 200] - internalDistanceDtype: ["float"] - smemLutDtype: ["float", "fp8", "half"] - refine_ratio: [1, 2, 4] - large: - build: - nlist: [8192, 16384, 32768, 65536] - pq_dim: [48, 32, 16] - pq_bits: [8, 6, 5, 4] - ratio: [4] - niter: [20] - search: - nprobe: [20, 30, 40, 50, 100, 200, 500, 1000] - internalDistanceDtype: ["float"] - smemLutDtype: ["float", "fp8", "half"] - refine_ratio: [1, 2, 4] - 100M: - build: - nlist: [50000] - pq_dim: [48] - pq_bits: [8, 6, 5, 4] - ratio: [10] - niter: [10] - search: - nprobe: [20, 30, 40, 50, 100, 200, 500, 1000] - internalDistanceDtype: ["float"] - smemLutDtype: ["float", "fp8", "half"] - refine_ratio: [1] diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/bigann-100M.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/bigann-100M.json deleted file mode 100644 index 55abca25d2..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/bigann-100M.json +++ /dev/null @@ -1,192 +0,0 @@ -{ - "dataset": { - "name": "bigann-100M", - "base_file": "bigann-1B/base.1B.u8bin", - "subset_size": 100000000, - "query_file": "bigann-1B/query.public.10K.u8bin", - "groundtruth_neighbors_file": "bigann-100M/groundtruth.neighbors.ibin", - "distance": "euclidean" - }, - - "search_basic_param": { - "batch_size": 10000, - "k": 10 - }, - - "index": [ - { - "name": "raft_ivf_pq.dimpq64-cluster5K", - "algo": "raft_ivf_pq", - "build_param": {"niter": 25, "nlist": 5000, "pq_dim": 64, "ratio": 10}, - "file": "bigann-100M/raft_ivf_pq/dimpq64-cluster5K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 1000, "internalDistanceDtype": "half", "smemLutDtype": "half" } - ] - }, - { - "name": "raft_ivf_pq.dimpq64-cluster10K", - "algo": "raft_ivf_pq", - "build_param": {"niter": 25, "nlist": 10000, "pq_dim": 64, "ratio": 10}, - "file": "bigann-100M/raft_ivf_pq/dimpq64-cluster5K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 1000, "internalDistanceDtype": "half", "smemLutDtype": "half" } - ] - }, - { - "name": "hnswlib.M12", - "algo": "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file": "bigann-100M/hnswlib/M12", - "search_params": [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ] - }, - { - "name": "hnswlib.M16", - "algo": "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file": "bigann-100M/hnswlib/M16", - "search_params": [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ] - }, - { - "name": "hnswlib.M24", - "algo": "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file": "bigann-100M/hnswlib/M24", - "search_params": [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ] - }, - { - "name": "hnswlib.M36", - "algo": "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file": "bigann-100M/hnswlib/M36", - "search_params": [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ] - }, - { - "name": "raft_ivf_flat.nlist100K", - "algo": "raft_ivf_flat", - "build_param": {"nlist": 100000, "niter": 25, "ratio": 5}, - "file": "bigann-100M/raft_ivf_flat/nlist100K", - "search_params": [ - {"max_batch":10000, "max_k":10, "nprobe":20}, - {"max_batch":10000, "max_k":10, "nprobe":30}, - {"max_batch":10000, "max_k":10, "nprobe":40}, - {"max_batch":10000, "max_k":10, "nprobe":50}, - {"max_batch":10000, "max_k":10, "nprobe":100}, - {"max_batch":10000, "max_k":10, "nprobe":200}, - {"max_batch":10000, "max_k":10, "nprobe":500}, - {"max_batch":10000, "max_k":10, "nprobe":1000} - ] - }, - { - "name": "raft_cagra.dim32", - "algo": "raft_cagra", - "build_param": {"graph_degree": 32}, - "file": "bigann-100M/raft_cagra/dim32", - "search_params": [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ] - }, - { - "name": "raft_cagra.dim64", - "algo": "raft_cagra", - "build_param": {"graph_degree": 64}, - "file": "bigann-100M/raft_cagra/dim64", - "search_params": [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ] - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/datasets.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/datasets.yaml deleted file mode 100644 index 188d24d20f..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/datasets.yaml +++ /dev/null @@ -1,127 +0,0 @@ -- name: bigann-1B - base_file: bigann-1B/base.1B.u8bin - subset_size: 100000000 - dims: 128 - query_file: bigann-1B/query.public.10K.u8bin - groundtruth_neighbors_file: bigann-1B/groundtruth.neighbors.ibin - distance: euclidean - -- name: deep-1B - base_file: deep-1B/base.1B.fbin - query_file: deep-1B/query.public.10K.fbin - dims: 96 - groundtruth_neighbors_file: deep-1B/groundtruth.neighbors.ibin - distance: inner_product - -- name: bigann-100M - base_file: bigann-100M/base.1B.u8bin - subset_size: 100000000 - dims: 128 - query_file: bigann-100M/query.public.10K.u8bin - groundtruth_neighbors_file: bigann-100M/groundtruth.neighbors.ibin - distance: euclidean - -- name: deep-image-96-inner - base_file: deep-image-96-inner/base.fbin - query_file: deep-image-96-inner/query.fbin - dims: 96 - groundtruth_neighbors_file: deep-image-96-inner/groundtruth.neighbors.ibin - distance: euclidean - -- name: fashion-mnist-784-euclidean - dims: 784 - base_file: fashion-mnist-784-euclidean/base.fbin - query_file: fashion-mnist-784-euclidean/query.fbin - groundtruth_neighbors_file: fashion-mnist-784-euclidean/groundtruth.neighbors.ibin - distance: euclidean - -- name: gist-960-euclidean - dims: 960 - base_file: gist-960-euclidean/base.fbin - query_file: gist-960-euclidean/query.fbin - groundtruth_neighbors_file: gist-960-euclidean/groundtruth.neighbors.ibin - distance: euclidean - -- name: glove-50-angular - dims: 50 - base_file: glove-50-angular/base.fbin - query_file: glove-50-angular/query.fbin - groundtruth_neighbors_file: glove-50-angular/groundtruth.neighbors.ibin - distance: euclidean - -- name: glove-50-inner - dims: 50 - base_file: glove-50-inner/base.fbin - query_file: glove-50-inner/query.fbin - groundtruth_neighbors_file: glove-50-inner/groundtruth.neighbors.ibin - distance: euclidean - -- name: glove-100-angular - dims: 100 - base_file: glove-100-angular/base.fbin - query_file: glove-100-angular/query.fbin - groundtruth_neighbors_file: glove-100-angular/groundtruth.neighbors.ibin - distance: euclidean - -- name: glove-100-inner - dims: 100 - base_file: glove-100-inner/base.fbin - query_file: glove-100-inner/query.fbin - groundtruth_neighbors_file: glove-100-inner/groundtruth.neighbors.ibin - distance: euclidean - -- name: lastfm-65-angular - dims: 65 - base_file: lastfm-65-angular/base.fbin - query_file: lastfm-65-angular/query.fbin - groundtruth_neighbors_file: lastfm-65-angular/groundtruth.neighbors.ibin - distance: euclidean - -- name: mnist-784-euclidean - dims: 784 - base_file: mnist-784-euclidean/base.fbin - query_file: mnist-784-euclidean/query.fbin - groundtruth_neighbors_file: mnist-784-euclidean/groundtruth.neighbors.ibin - distance: euclidean - -- name: nytimes-256-angular - dims: 256 - base_file: nytimes-256-angular/base.fbin - query_file: nytimes-256-angular/query.fbin - groundtruth_neighbors_file: nytimes-256-angular/groundtruth.neighbors.ibin - distance: euclidean - -- name: nytimes-256-inner - dims: 256 - base_file: nytimes-256-inner/base.fbin - query_file: nytimes-256-inner/query.fbin - groundtruth_neighbors_file: nytimes-256-inner/groundtruth.neighbors.ibin - distance: euclidean - -- name: sift-128-euclidean - dims: 128 - base_file: sift-128-euclidean/base.fbin - query_file: sift-128-euclidean/query.fbin - groundtruth_neighbors_file: sift-128-euclidean/groundtruth.neighbors.ibin - distance: euclidean - -- name: wiki_all_1M - dims: 768 - base_file: wiki_all_1M/base.1M.fbin - query_file: wiki_all_1M/queries.fbin - groundtruth_neighbors_file: wiki_all_1M/groundtruth.1M.neighbors.ibin - distance: euclidean - -- name: wiki_all_10M - dims: 768 - base_file: wiki_all_10M/base.10M.fbin - query_file: wiki_all_10M/queries.fbin - groundtruth_neighbors_file: wiki_all_10M/groundtruth.10M.neighbors.ibin - distance: euclidean - -- name: wiki_all_88M - dims: 768 - base_file: wiki_all_88M/base.88M.fbin - query_file: wiki_all_88M/queries.fbin - groundtruth_neighbors_file: wiki_all_88M/groundtruth.88M.neighbors.ibin - distance: euclidean diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/deep-100M.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/deep-100M.json deleted file mode 100644 index ea92a0de18..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/deep-100M.json +++ /dev/null @@ -1,458 +0,0 @@ -{ - "dataset": { - "name": "deep-100M", - "base_file": "deep-100M/base.1B.fbin", - "subset_size": 100000000, - "query_file": "deep-100M/query.public.10K.fbin", - "groundtruth_neighbors_file": "deep-100M/groundtruth.neighbors.ibin", - "distance": "euclidean" - }, - - "search_basic_param": { - "batch_size": 10000, - "k": 10 - }, - - "index": [ - { - "name": "hnswlib.M12", - "algo": "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file": "deep-100M/hnswlib/M12", - "search_params": [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ] - }, - { - "name": "hnswlib.M16", - "algo": "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file": "deep-100M/hnswlib/M16", - "search_params": [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ] - }, - { - "name": "hnswlib.M24", - "algo": "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file": "deep-100M/hnswlib/M24", - "search_params": [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ] - }, - { - "name": "hnswlib.M36", - "algo": "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file": "deep-100M/hnswlib/M36", - "search_params": [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ] - }, - { - "name": "faiss_gpu_ivf_flat.nlist50K", - "algo": "faiss_gpu_ivf_flat", - "build_param": {"nlist":50000}, - "file": "deep-100M/faiss_gpu_ivf_flat/nlist50K", - "search_params": [ - {"nprobe":20}, - {"nprobe":30}, - {"nprobe":40}, - {"nprobe":50}, - {"nprobe":100}, - {"nprobe":200}, - {"nprobe":500}, - {"nprobe":1000} - ] - }, - { - "name": "faiss_gpu_ivf_flat.nlist100K", - "algo": "faiss_gpu_ivf_flat", - "build_param": {"nlist":100000}, - "file": "deep-100M/faiss_gpu_ivf_flat/nlist100K", - "search_params": [ - {"nprobe":20}, - {"nprobe":30}, - {"nprobe":40}, - {"nprobe":50}, - {"nprobe":100}, - {"nprobe":200}, - {"nprobe":500}, - {"nprobe":1000} - ] - }, - { - "name": "faiss_gpu_ivf_flat.nlist200K", - "algo": "faiss_gpu_ivf_flat", - "build_param": {"nlist":200000}, - "file": "deep-100M/faiss_gpu_ivf_flat/nlist200K", - "search_params": [ - {"nprobe":20}, - {"nprobe":30}, - {"nprobe":40}, - {"nprobe":50}, - {"nprobe":100}, - {"nprobe":200}, - {"nprobe":500}, - {"nprobe":1000} - ] - }, - { - "name": "faiss_gpu_ivf_pq.M48-nlist16K", - "algo": "faiss_gpu_ivf_pq", - "build_param": {"nlist":16384, "M":48}, - "file": "deep-100M/faiss_gpu_ivf_pq/M48-nlist16K", - "search_params": [ - {"nprobe":10}, - {"nprobe":20}, - {"nprobe":30}, - {"nprobe":40}, - {"nprobe":50}, - {"nprobe":100}, - {"nprobe":200}, - {"nprobe":500} - ] - }, - { - "name": "faiss_gpu_ivf_pq.M48-nlist50K", - "algo": "faiss_gpu_ivf_pq", - "build_param": {"nlist":50000, "M":48}, - "file": "deep-100M/faiss_gpu_ivf_pq/M48-nlist50K", - "search_params": [ - {"nprobe":20}, - {"nprobe":30}, - {"nprobe":40}, - {"nprobe":50}, - {"nprobe":100}, - {"nprobe":200}, - {"nprobe":500}, - {"nprobe":1000} - ] - }, - { - "name": "faiss_gpu_ivf_pq.M48-nlist100K", - "algo": "faiss_gpu_ivf_pq", - "build_param": {"nlist":100000, "M":48}, - "file": "deep-100M/faiss_gpu_ivf_pq/M48-nlist100K", - "search_params": [ - {"nprobe":20}, - {"nprobe":30}, - {"nprobe":40}, - {"nprobe":50}, - {"nprobe":100}, - {"nprobe":200}, - {"nprobe":500}, - {"nprobe":1000} - ] - }, - { - "name": "faiss_gpu_ivf_pq.M48-nlist200K", - "algo": "faiss_gpu_ivf_pq", - "build_param": {"nlist":200000, "M":48}, - "file": "deep-100M/faiss_gpu_ivf_pq/M48-nlist200K", - "search_params": [ - {"nprobe":20}, - {"nprobe":30}, - {"nprobe":40}, - {"nprobe":50}, - {"nprobe":100}, - {"nprobe":200}, - {"nprobe":500}, - {"nprobe":1000} - ] - }, - - - { - "name": "raft_ivf_flat.nlist50K", - "algo": "raft_ivf_flat", - "build_param": {"nlist": 50000, "niter": 25, "ratio": 5}, - "file": "deep-100M/raft_ivf_flat/nlist50K", - "search_params": [ - {"max_batch":10000, "max_k":10, "nprobe":20}, - {"max_batch":10000, "max_k":10, "nprobe":30}, - {"max_batch":10000, "max_k":10, "nprobe":40}, - {"max_batch":10000, "max_k":10, "nprobe":50}, - {"max_batch":10000, "max_k":10, "nprobe":100}, - {"max_batch":10000, "max_k":10, "nprobe":200}, - {"max_batch":10000, "max_k":10, "nprobe":500}, - {"max_batch":10000, "max_k":10, "nprobe":1000} - ] - }, - { - "name": "raft_ivf_flat.nlist100K", - "algo": "raft_ivf_flat", - "build_param": {"nlist": 100000, "niter": 25, "ratio": 5}, - "file": "deep-100M/raft_ivf_flat/nlist100K", - "search_params": [ - {"max_batch":10000, "max_k":10, "nprobe":20}, - {"max_batch":10000, "max_k":10, "nprobe":30}, - {"max_batch":10000, "max_k":10, "nprobe":40}, - {"max_batch":10000, "max_k":10, "nprobe":50}, - {"max_batch":10000, "max_k":10, "nprobe":100}, - {"max_batch":10000, "max_k":10, "nprobe":200}, - {"max_batch":10000, "max_k":10, "nprobe":500}, - {"max_batch":10000, "max_k":10, "nprobe":1000} - ] - }, - { - "name": "raft_ivf_flat.nlist200K", - "algo": "raft_ivf_flat", - "build_param": {"nlist": 200000, "niter": 25, "ratio": 5}, - "file": "deep-100M/raft_ivf_flat/nlist200K", - "search_params": [ - {"max_batch":10000, "max_k":10, "nprobe":20}, - {"max_batch":10000, "max_k":10, "nprobe":30}, - {"max_batch":10000, "max_k":10, "nprobe":40}, - {"max_batch":10000, "max_k":10, "nprobe":50}, - {"max_batch":10000, "max_k":10, "nprobe":100}, - {"max_batch":10000, "max_k":10, "nprobe":200}, - {"max_batch":10000, "max_k":10, "nprobe":500}, - {"max_batch":10000, "max_k":10, "nprobe":1000} - ] - }, -{ - "name": "raft_ivf_pq.d96b5n50K", - "algo": "raft_ivf_pq", - "build_param": {"nlist": 50000, "pq_dim": 96, "pq_bits": 5, "ratio": 10, "niter": 25}, - "file": "deep-100M/raft_ivf_pq/d96b5n50K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 2 }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 2 }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 2 }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 2 }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 2 }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 2 }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 2 }, - { "nprobe": 2000, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 2 }, - { "nprobe": 5000, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 2 }, - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 2000, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 5000, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 2000, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 5000, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 1000, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 2000, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 5000, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 1000, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 2000, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 5000, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 2 } - ] - }, - { - "name": "raft_ivf_pq.d64b5n50K", - "algo": "raft_ivf_pq", - "build_param": {"nlist": 50000, "pq_dim": 64, "pq_bits": 5, "ratio": 10, "niter": 25}, - "file": "deep-100M/raft_ivf_pq/d64b5n50K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 4 }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 4 }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 4 }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 4 }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 4 }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 4 }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 4 }, - { "nprobe": 2000, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 4 }, - { "nprobe": 5000, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 4 }, - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 2000, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 5000, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 2000, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 5000, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 1000, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 2000, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 5000, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 1000, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 2000, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 5000, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 4 } - ] - }, - { - "name": "raft_ivf_pq.dimpq512-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 512, - "ratio": 1, - "niter": 25 - }, - "file": "index/deep-image-96-angular/raft_ivf_pq/dimpq512-cluster1024-float-float", - "search_params": [ - {"nprobe": 10, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 1024, "internalDistanceDtype": "float", "smemLutDtype": "float"} - ], - "search_result_file": "result/deep-image-96-angular/raft_ivf_pq/dimpq512-cluster1024-float-float" - }, - { - "name": "raft_cagra.dim32", - "algo": "raft_cagra", - "build_param": {"graph_degree": 32, "intermediate_graph_degree": 48}, - "file": "deep-100M/raft_cagra/dim32", - "search_params": [ - {"itopk": 32, "search_width": 1, "max_iterations": 0, "algo": "single_cta"}, - {"itopk": 32, "search_width": 1, "max_iterations": 32, "algo": "single_cta"}, - {"itopk": 64, "search_width": 4, "max_iterations": 16, "algo": "single_cta"}, - {"itopk": 64, "search_width": 1, "max_iterations": 64, "algo": "single_cta"}, - {"itopk": 96, "search_width": 2, "max_iterations": 48, "algo": "single_cta"}, - {"itopk": 128, "search_width": 8, "max_iterations": 16, "algo": "single_cta"}, - {"itopk": 128, "search_width": 2, "max_iterations": 64, "algo": "single_cta"}, - {"itopk": 192, "search_width": 8, "max_iterations": 24, "algo": "single_cta"}, - {"itopk": 192, "search_width": 2, "max_iterations": 96, "algo": "single_cta"}, - {"itopk": 256, "search_width": 8, "max_iterations": 32, "algo": "single_cta"}, - {"itopk": 384, "search_width": 8, "max_iterations": 48, "algo": "single_cta"}, - {"itopk": 512, "search_width": 8, "max_iterations": 64, "algo": "single_cta"} - ] - }, - { - "name": "raft_cagra.dim32.multi_cta", - "algo": "raft_cagra", - "build_param": {"graph_degree": 32, "intermediate_graph_degree": 48}, - "file": "deep-100M/raft_cagra/dim32", - "search_params": [ - {"itopk": 32, "search_width": 1, "max_iterations": 0, "algo": "multi_cta"}, - {"itopk": 32, "search_width": 1, "max_iterations": 32, "algo": "multi_cta"}, - {"itopk": 64, "search_width": 4, "max_iterations": 16, "algo": "multi_cta"}, - {"itopk": 64, "search_width": 1, "max_iterations": 64, "algo": "multi_cta"}, - {"itopk": 96, "search_width": 2, "max_iterations": 48, "algo": "multi_cta"}, - {"itopk": 128, "search_width": 8, "max_iterations": 16, "algo": "multi_cta"}, - {"itopk": 128, "search_width": 2, "max_iterations": 64, "algo": "multi_cta"}, - {"itopk": 192, "search_width": 8, "max_iterations": 24, "algo": "multi_cta"}, - {"itopk": 192, "search_width": 2, "max_iterations": 96, "algo": "multi_cta"}, - {"itopk": 256, "search_width": 8, "max_iterations": 32, "algo": "multi_cta"}, - {"itopk": 384, "search_width": 8, "max_iterations": 48, "algo": "multi_cta"}, - {"itopk": 512, "search_width": 8, "max_iterations": 64, "algo": "multi_cta"} - ] - }, - { - "name": "raft_cagra.dim32.multi_kernel", - "algo": "raft_cagra", - "build_param": {"graph_degree": 32, "intermediate_graph_degree": 48}, - "file": "deep-100M/raft_cagra/dim32", - "search_params": [ - {"itopk": 32, "search_width": 1, "max_iterations": 0, "algo": "multi_kernel"}, - {"itopk": 32, "search_width": 1, "max_iterations": 32, "algo": "multi_kernel"}, - {"itopk": 64, "search_width": 4, "max_iterations": 16, "algo": "multi_kernel"}, - {"itopk": 64, "search_width": 1, "max_iterations": 64, "algo": "multi_kernel"}, - {"itopk": 96, "search_width": 2, "max_iterations": 48, "algo": "multi_kernel"}, - {"itopk": 128, "search_width": 8, "max_iterations": 16, "algo": "multi_kernel"}, - {"itopk": 128, "search_width": 2, "max_iterations": 64, "algo": "multi_kernel"}, - {"itopk": 192, "search_width": 8, "max_iterations": 24, "algo": "multi_kernel"}, - {"itopk": 192, "search_width": 2, "max_iterations": 96, "algo": "multi_kernel"}, - {"itopk": 256, "search_width": 8, "max_iterations": 32, "algo": "multi_kernel"}, - {"itopk": 384, "search_width": 8, "max_iterations": 48, "algo": "multi_kernel"}, - {"itopk": 512, "search_width": 8, "max_iterations": 64, "algo": "multi_kernel"} - ] - }, - { - "name": "raft_cagra.dim64", - "algo": "raft_cagra", - "build_param": {"graph_degree": 64}, - "file": "deep-100M/raft_cagra/dim64", - "search_params": [ - {"itopk": 32, "search_width": 1, "max_iterations": 0}, - {"itopk": 32, "search_width": 1, "max_iterations": 32}, - {"itopk": 64, "search_width": 4, "max_iterations": 16}, - {"itopk": 64, "search_width": 1, "max_iterations": 64}, - {"itopk": 96, "search_width": 2, "max_iterations": 48}, - {"itopk": 128, "search_width": 8, "max_iterations": 16}, - {"itopk": 128, "search_width": 2, "max_iterations": 64}, - {"itopk": 192, "search_width": 8, "max_iterations": 24}, - {"itopk": 192, "search_width": 2, "max_iterations": 96}, - {"itopk": 256, "search_width": 8, "max_iterations": 32}, - {"itopk": 384, "search_width": 8, "max_iterations": 48}, - {"itopk": 512, "search_width": 8, "max_iterations": 64} - ] - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/deep-1B.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/deep-1B.json deleted file mode 100644 index e5190e073e..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/deep-1B.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "dataset": { - "name": "deep-1B", - "base_file": "deep-1B/base.1B.fbin", - "query_file": "deep-1B/query.public.10K.fbin", - "groundtruth_neighbors_file": "deep-1B/groundtruth.neighbors.ibin", - "distance": "inner_product" - }, - - "search_basic_param": { - "batch_size": 10000, - "k": 10 - }, - - "index": [ - { - "name": "faiss_gpu_ivf_pq.M48-nlist50K", - "algo": "faiss_gpu_ivf_pq", - "build_param": {"nlist":50000, "M":48}, - "file": "deep-1B/faiss_gpu_ivf_pq/M48-nlist50K", - "search_params": [ - {"nprobe":1}, - {"nprobe":5}, - {"nprobe":10}, - {"nprobe":50}, - {"nprobe":100}, - {"nprobe":200}, - {"nprobe":500}, - {"nprobe":1000}, - {"nprobe":2000} - ] - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/deep-image-96-inner.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/deep-image-96-inner.json deleted file mode 100644 index 3d69e775a1..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/deep-image-96-inner.json +++ /dev/null @@ -1,1013 +0,0 @@ -{ - "dataset": { - "name": "deep-image-96-inner", - "base_file": "deep-image-96-inner/base.fbin", - "query_file": "deep-image-96-inner/query.fbin", - "groundtruth_neighbors_file": "deep-image-96-inner/groundtruth.neighbors.ibin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 5000, - "k": 10, - "run_count": 3 - }, - "index": [ - { - "name" : "hnswlib.M12", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "index/deep-image-96-inner/hnswlib/M12", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/deep-image-96-inner/hnswlib/M12" - }, - { - "name" : "hnswlib.M16", - "algo" : "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file" : "index/deep-image-96-inner/hnswlib/M16", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/deep-image-96-inner/hnswlib/M16" - }, - { - "name" : "hnswlib.M24", - "algo" : "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file" : "index/deep-image-96-inner/hnswlib/M24", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/deep-image-96-inner/hnswlib/M24" - }, - { - "name" : "hnswlib.M36", - "algo" : "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file" : "index/deep-image-96-inner/hnswlib/M36", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/deep-image-96-inner/hnswlib/M36" - }, - - - - - { - "name": "raft_bfknn", - "algo": "raft_bfknn", - - "build_param": {}, - "file": "index/deep-image-96-inner/raft_bfknn/bfknn", - "search_params": [ - { - "probe": 1 - } - ], - "search_result_file": "result/deep-image-96-inner/raft_bfknn/bfknn" - }, - { - "name": "faiss_gpu_ivf_flat.nlist1024", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 1024 - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_flat/nlist1024" - }, - { - "name": "faiss_gpu_ivf_flat.nlist2048", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 2048 - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_flat/nlist2048", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_flat/nlist2048" - }, - { - "name": "faiss_gpu_ivf_flat.nlist4096", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 4096 - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_flat/nlist4096", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_flat/nlist4096" - }, - { - "name": "faiss_gpu_ivf_flat.nlist8192", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 8192 - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_flat/nlist8192", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_flat/nlist8192" - }, - { - "name": "faiss_gpu_ivf_flat.nlist16384", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 16384 - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_flat/nlist16384" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": true - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_pq/M64-nlist1024", - "search_params": [ - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024.noprecomp", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": false - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_pq/M64-nlist1024.noprecomp", - "search_params": [ - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "fp16" - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_sq/nlist1024-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_sq/nlist1024-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "fp16" - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_sq/nlist2048-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_sq/nlist2048-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "fp16" - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_sq/nlist4096-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_sq/nlist4096-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "fp16" - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_sq/nlist8192-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_sq/nlist8192-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "fp16" - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_sq/nlist16384-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_sq/nlist16384-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "int8" - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_sq/nlist1024-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_sq/nlist1024-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "int8" - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_sq/nlist2048-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_sq/nlist2048-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "int8" - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_sq/nlist4096-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_sq/nlist4096-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "int8" - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_sq/nlist8192-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_sq/nlist8192-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "int8" - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_sq/nlist16384-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_sq/nlist16384-int8" - }, - { - "name": "faiss_gpu_flat", - "algo": "faiss_gpu_flat", - "build_param": {}, - "file": "index/deep-image-96-inner/faiss_gpu_flat/flat", - "search_params": [ - {} - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_flat/flat" - }, - - { - "name": "raft_ivf_pq.dimpq128-cluster1024", - "algo": "raft_ivf_pq", - - "build_param": {"nlist": 1024, "pq_dim": 128, "ratio": 1, "niter": 25 - }, - "file": "index/deep-image-96-inner/raft_ivf_pq/dimpq128-cluster1024", - "search_params": [ - {"nprobe": 10, "internalDistanceDtype": "half", "smemLutDtype": "half"}, - {"nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half"}, - {"nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half"}, - {"nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half"}, - {"nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half"}, - {"nprobe": 1024, "internalDistanceDtype": "half", "smemLutDtype": "half"} - ], - "search_result_file": "result/deep-image-96-inner/raft_ivf_pq/dimpq128-cluster1024" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-float", - "algo": "raft_ivf_pq", - - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/deep-image-96-inner/raft_ivf_pq/dimpq128-cluster1024-float-float", - "search_params": [ - {"nprobe": 1, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 5, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 10, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 1024, "internalDistanceDtype": "float", "smemLutDtype": "float"} - ], - "search_result_file": "result/deep-image-96-inner/raft_ivf_pq/dimpq128-cluster1024-float-float" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-half", - "algo": "raft_ivf_pq", - - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/deep-image-96-inner/raft_ivf_pq/dimpq128-cluster1024-float-half", - "search_params": [ - {"nprobe": 10, "internalDistanceDtype": "float", "smemLutDtype": "half"}, - {"nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "half"}, - {"nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "half"}, - {"nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "half"}, - {"nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "half"}, - {"nprobe": 1024, "internalDistanceDtype": "float", "smemLutDtype": "half"} - ], - "search_result_file": "result/deep-image-96-inner/raft_ivf_pq/dimpq128-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/deep-image-96-inner/raft_ivf_pq/dimpq128-cluster1024-float-fp8", - "search_params": [ - {"nprobe": 10, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 1024, "internalDistanceDtype": "float", "smemLutDtype": "fp8"} - ], - "search_result_file": "result/deep-image-96-inner/raft_ivf_pq/dimpq128-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/deep-image-96-inner/raft_ivf_pq/dimpq64-cluster1024-float-fp8", - "search_params": [ - {"nprobe": 10, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 1024, "internalDistanceDtype": "float", "smemLutDtype": "fp8"} - ], - "search_result_file": "result/deep-image-96-inner/raft_ivf_pq/dimpq64-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/deep-image-96-inner/raft_ivf_pq/dimpq64-cluster1024-float-half", - "search_params": [ - {"nprobe": 10, "internalDistanceDtype": "float", "smemLutDtype": "half"}, - {"nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "half"}, - {"nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "half"}, - {"nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "half"}, - {"nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "half"}, - {"nprobe": 1024, "internalDistanceDtype": "float", "smemLutDtype": "half"} - ], - "search_result_file": "result/deep-image-96-inner/raft_ivf_pq/dimpq64-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq32-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 32, - "ratio": 1, - "niter": 25 - }, - "file": "index/deep-image-96-inner/raft_ivf_pq/dimpq32-cluster1024-float-fp8", - "search_params": [ - {"nprobe": 10, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 1024, "internalDistanceDtype": "float", "smemLutDtype": "fp8"} - ], - "search_result_file": "result/deep-image-96-inner/raft_ivf_pq/dimpq32-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq16-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - - "build_param": { - "nlist": 1024, - "pq_dim": 16, - "ratio": 1, - "niter": 25 - }, - "file": "index/deep-image-96-inner/raft_ivf_pq/dimpq16-cluster1024-float-fp8", - "search_params": [ - {"nprobe": 10, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 1024, "internalDistanceDtype": "float", "smemLutDtype": "fp8"} - ], - "search_result_file": "result/deep-image-96-inner/raft_ivf_pq/dimpq16-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-half-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/deep-image-96-inner/raft_ivf_pq/dimpq128-cluster1024-half-float", - "search_params": [ - {"nprobe": 10, "internalDistanceDtype": "half", "smemLutDtype": "float"}, - {"nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "float"}, - {"nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "float"}, - {"nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "float"}, - {"nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "float"}, - {"nprobe": 1024, "internalDistanceDtype": "half", "smemLutDtype": "float"} - ], - "search_result_file": "result/deep-image-96-inner/raft_ivf_pq/dimpq128-cluster1024-half-float" - }, - { - "name": "raft_ivf_pq.dimpq512-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 512, - "ratio": 1, - "niter": 25 - }, - "file": "index/deep-image-96-inner/raft_ivf_pq/dimpq512-cluster1024-float-float", - "search_params": [ - {"nprobe": 10, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 1024, "internalDistanceDtype": "float", "smemLutDtype": "float"} - ], - "search_result_file": "result/deep-image-96-inner/raft_ivf_pq/dimpq512-cluster1024-float-float" - }, - { - "name": "raft_ivf_flat.nlist1024", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 1024, - "ratio": 1, - "niter": 25 - }, - "file": "index/deep-image-96-inner/raft_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/raft_ivf_flat/nlist1024" - }, - { - "name": "raft_ivf_flat.nlist16384", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 16384, - "ratio": 2, - "niter": 20 - }, - "file": "index/deep-image-96-inner/raft_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/deep-image-96-inner/raft_ivf_flat/nlist16384" - }, - - { - "name" : "raft_cagra.dim32", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 32 - }, - "file" : "index/deep-image-96-inner/raft_cagra/dim32", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/deep-image-96-inner/raft_cagra/dim32" - }, - - { - "name" : "raft_cagra.dim64", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 64 - }, - "file" : "index/deep-image-96-inner/raft_cagra/dim64", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/deep-image-96-inner/raft_cagra/dim64" - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/fashion-mnist-784-euclidean.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/fashion-mnist-784-euclidean.json deleted file mode 100644 index 2c86b0c4ee..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/fashion-mnist-784-euclidean.json +++ /dev/null @@ -1,1352 +0,0 @@ -{ - "dataset": { - "name": "fashion-mnist-784-euclidean", - "base_file": "fashion-mnist-784-euclidean/base.fbin", - "query_file": "fashion-mnist-784-euclidean/query.fbin", - "groundtruth_neighbors_file": "fashion-mnist-784-euclidean/groundtruth.neighbors.ibin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 5000, - "k": 10, - "run_count": 3 - }, - "index": [ - { - "name" : "hnswlib.M12", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "index/fashion-mnist-784-euclidean/hnswlib/M12", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/fashion-mnist-784-euclidean/hnswlib/M12" - }, - { - "name" : "hnswlib.M16", - "algo" : "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file" : "index/fashion-mnist-784-euclidean/hnswlib/M16", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/fashion-mnist-784-euclidean/hnswlib/M16" - }, - { - "name" : "hnswlib.M24", - "algo" : "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file" : "index/fashion-mnist-784-euclidean/hnswlib/M24", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/fashion-mnist-784-euclidean/hnswlib/M24" - }, - { - "name" : "hnswlib.M36", - "algo" : "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file" : "index/fashion-mnist-784-euclidean/hnswlib/M36", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/fashion-mnist-784-euclidean/hnswlib/M36" - }, - - - - - { - "name": "raft_bfknn", - "algo": "raft_bfknn", - "build_param": {}, - "file": "index/fashion-mnist-784-euclidean/raft_bfknn/bfknn", - "search_params": [ - { - "probe": 1 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_bfknn/bfknn" - }, - { - "name": "faiss_gpu_ivf_flat.nlist1024", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 1024 - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_flat/nlist1024" - }, - { - "name": "faiss_gpu_ivf_flat.nlist2048", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 2048 - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_flat/nlist2048", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_flat/nlist2048" - }, - { - "name": "faiss_gpu_ivf_flat.nlist4096", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 4096 - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_flat/nlist4096", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_flat/nlist4096" - }, - { - "name": "faiss_gpu_ivf_flat.nlist8192", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 8192 - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_flat/nlist8192", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_flat/nlist8192" - }, - { - "name": "faiss_gpu_ivf_flat.nlist16384", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 16384 - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_flat/nlist16384" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": true - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_pq/M64-nlist1024", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024.noprecomp", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": false - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_pq/M64-nlist1024.noprecomp", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "fp16" - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist1024-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist1024-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "fp16" - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist2048-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist2048-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "fp16" - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist4096-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist4096-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "fp16" - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist8192-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist8192-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "fp16" - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist16384-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist16384-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "int8" - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist1024-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist1024-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "int8" - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist2048-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist2048-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "int8" - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist4096-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist4096-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "int8" - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist8192-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist8192-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "int8" - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist16384-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist16384-int8" - }, - { - "name": "faiss_gpu_flat", - "algo": "faiss_gpu_flat", - "build_param": {}, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_flat/flat", - "search_params": [ - {} - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_flat/flat" - }, - - { - "name": "raft_ivf_pq.dimpq128-cluster1024", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "numProbes": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 5, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-float" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq64-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq64-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq64-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq64-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq32-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 32, - "ratio": 1, - "niter": 25 - }, - "file": "index/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq32-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq32-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq16-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 16, - "ratio": 1, - "niter": 25 - }, - "file": "index/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq16-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq16-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-half-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-half-float", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-half-float" - }, - { - "name": "raft_ivf_pq.dimpq512-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 512, - "ratio": 1, - "niter": 25 - }, - "file": "index/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq512-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq512-cluster1024-float-float" - }, - { - "name": "raft_ivf_flat.nlist1024", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 1024, - "ratio": 1, - "niter": 25 - }, - "file": "index/fashion-mnist-784-euclidean/raft_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_ivf_flat/nlist1024" - }, - { - "name": "raft_ivf_flat.nlist16384", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 16384, - "ratio": 2, - "niter": 20 - }, - "file": "index/fashion-mnist-784-euclidean/raft_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_ivf_flat/nlist16384" - }, - - { - "name" : "raft_cagra.dim32", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 32 - }, - "file" : "index/fashion-mnist-784-euclidean/raft_cagra/dim32", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/fashion-mnist-784-euclidean/raft_cagra/dim32" - }, - - { - "name" : "raft_cagra.dim64", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 64 - }, - "file" : "index/fashion-mnist-784-euclidean/raft_cagra/dim64", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/fashion-mnist-784-euclidean/raft_cagra/dim64" - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/gist-960-euclidean.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/gist-960-euclidean.json deleted file mode 100644 index c5480900a7..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/gist-960-euclidean.json +++ /dev/null @@ -1,1351 +0,0 @@ -{ - "dataset": { - "name": "gist-960-euclidean", - "base_file": "gist-960-euclidean/base.fbin", - "query_file": "gist-960-euclidean/query.fbin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 5000, - "k": 10, - "run_count": 3 - }, - "index": [ - { - "name" : "hnswlib.M12", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "index/gist-960-euclidean/hnswlib/M12", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/gist-960-euclidean/hnswlib/M12" - }, - { - "name" : "hnswlib.M16", - "algo" : "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file" : "index/gist-960-euclidean/hnswlib/M16", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/gist-960-euclidean/hnswlib/M16" - }, - { - "name" : "hnswlib.M24", - "algo" : "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file" : "index/gist-960-euclidean/hnswlib/M24", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/gist-960-euclidean/hnswlib/M24" - }, - { - "name" : "hnswlib.M36", - "algo" : "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file" : "index/gist-960-euclidean/hnswlib/M36", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/gist-960-euclidean/hnswlib/M36" - }, - - - - - { - "name": "raft_bfknn", - "algo": "raft_bfknn", - "build_param": {}, - "file": "index/gist-960-euclidean/raft_bfknn/bfknn", - "search_params": [ - { - "probe": 1 - } - ], - "search_result_file": "result/gist-960-euclidean/raft_bfknn/bfknn" - }, - { - "name": "faiss_gpu_ivf_flat.nlist1024", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 1024 - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_flat/nlist1024" - }, - { - "name": "faiss_gpu_ivf_flat.nlist2048", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 2048 - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_flat/nlist2048", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_flat/nlist2048" - }, - { - "name": "faiss_gpu_ivf_flat.nlist4096", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 4096 - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_flat/nlist4096", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_flat/nlist4096" - }, - { - "name": "faiss_gpu_ivf_flat.nlist8192", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 8192 - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_flat/nlist8192", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_flat/nlist8192" - }, - { - "name": "faiss_gpu_ivf_flat.nlist16384", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 16384 - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_flat/nlist16384" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": true - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_pq/M64-nlist1024", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024.noprecomp", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": false - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_pq/M64-nlist1024.noprecomp", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "fp16" - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_sq/nlist1024-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_sq/nlist1024-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "fp16" - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_sq/nlist2048-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_sq/nlist2048-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "fp16" - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_sq/nlist4096-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_sq/nlist4096-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "fp16" - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_sq/nlist8192-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_sq/nlist8192-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "fp16" - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_sq/nlist16384-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_sq/nlist16384-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "int8" - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_sq/nlist1024-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_sq/nlist1024-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "int8" - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_sq/nlist2048-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_sq/nlist2048-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "int8" - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_sq/nlist4096-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_sq/nlist4096-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "int8" - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_sq/nlist8192-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_sq/nlist8192-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "int8" - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_sq/nlist16384-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_sq/nlist16384-int8" - }, - { - "name": "faiss_gpu_flat", - "algo": "faiss_gpu_flat", - "build_param": {}, - "file": "index/gist-960-euclidean/faiss_gpu_flat/flat", - "search_params": [ - {} - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_flat/flat" - }, - - { - "name": "raft_ivf_pq.dimpq128-cluster1024", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/gist-960-euclidean/raft_ivf_pq/dimpq128-cluster1024", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/gist-960-euclidean/raft_ivf_pq/dimpq128-cluster1024" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/gist-960-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "numProbes": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 5, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/gist-960-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-float" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/gist-960-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/gist-960-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/gist-960-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/gist-960-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/gist-960-euclidean/raft_ivf_pq/dimpq64-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/gist-960-euclidean/raft_ivf_pq/dimpq64-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/gist-960-euclidean/raft_ivf_pq/dimpq64-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/gist-960-euclidean/raft_ivf_pq/dimpq64-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq32-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 32, - "ratio": 1, - "niter": 25 - }, - "file": "index/gist-960-euclidean/raft_ivf_pq/dimpq32-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/gist-960-euclidean/raft_ivf_pq/dimpq32-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq16-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 16, - "ratio": 1, - "niter": 25 - }, - "file": "index/gist-960-euclidean/raft_ivf_pq/dimpq16-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/gist-960-euclidean/raft_ivf_pq/dimpq16-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-half-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/gist-960-euclidean/raft_ivf_pq/dimpq128-cluster1024-half-float", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/gist-960-euclidean/raft_ivf_pq/dimpq128-cluster1024-half-float" - }, - { - "name": "raft_ivf_pq.dimpq512-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 512, - "ratio": 1, - "niter": 25 - }, - "file": "index/gist-960-euclidean/raft_ivf_pq/dimpq512-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/gist-960-euclidean/raft_ivf_pq/dimpq512-cluster1024-float-float" - }, - { - "name": "raft_ivf_flat.nlist1024", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 1024, - "ratio": 1, - "niter": 25 - }, - "file": "index/gist-960-euclidean/raft_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/raft_ivf_flat/nlist1024" - }, - { - "name": "raft_ivf_flat.nlist16384", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 16384, - "ratio": 2, - "niter": 20 - }, - "file": "index/gist-960-euclidean/raft_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/gist-960-euclidean/raft_ivf_flat/nlist16384" - }, - - { - "name" : "raft_cagra.dim32", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 32 - }, - "file" : "index/gist-960-euclidean/raft_cagra/dim32", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/gist-960-euclidean/raft_cagra/dim32" - }, - - { - "name" : "raft_cagra.dim64", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 64 - }, - "file" : "index/gist-960-euclidean/raft_cagra/dim64", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/gist-960-euclidean/raft_cagra/dim64" - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-100-angular.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-100-angular.json deleted file mode 100644 index 2074ef13a3..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-100-angular.json +++ /dev/null @@ -1,1351 +0,0 @@ -{ - "dataset": { - "name": "glove-100-angular", - "base_file": "glove-100-angular/base.fbin", - "query_file": "glove-100-angular/query.fbin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 5000, - "k": 10, - "run_count": 3 - }, - "index": [ - { - "name" : "hnswlib.M12", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-100-angular/hnswlib/M12", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-100-angular/hnswlib/M12" - }, - { - "name" : "hnswlib.M16", - "algo" : "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-100-angular/hnswlib/M16", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-100-angular/hnswlib/M16" - }, - { - "name" : "hnswlib.M24", - "algo" : "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-100-angular/hnswlib/M24", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-100-angular/hnswlib/M24" - }, - { - "name" : "hnswlib.M36", - "algo" : "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-100-angular/hnswlib/M36", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-100-angular/hnswlib/M36" - }, - - - - - { - "name": "raft_bfknn", - "algo": "raft_bfknn", - "build_param": {}, - "file": "index/glove-100-angular/raft_bfknn/bfknn", - "search_params": [ - { - "probe": 1 - } - ], - "search_result_file": "result/glove-100-angular/raft_bfknn/bfknn" - }, - { - "name": "faiss_gpu_ivf_flat.nlist1024", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 1024 - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_flat/nlist1024" - }, - { - "name": "faiss_gpu_ivf_flat.nlist2048", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 2048 - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_flat/nlist2048", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_flat/nlist2048" - }, - { - "name": "faiss_gpu_ivf_flat.nlist4096", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 4096 - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_flat/nlist4096", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_flat/nlist4096" - }, - { - "name": "faiss_gpu_ivf_flat.nlist8192", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 8192 - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_flat/nlist8192", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_flat/nlist8192" - }, - { - "name": "faiss_gpu_ivf_flat.nlist16384", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 16384 - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_flat/nlist16384" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": true - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_pq/M64-nlist1024", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024.noprecomp", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": false - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_pq/M64-nlist1024.noprecomp", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "fp16" - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_sq/nlist1024-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_sq/nlist1024-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "fp16" - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_sq/nlist2048-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_sq/nlist2048-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "fp16" - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_sq/nlist4096-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_sq/nlist4096-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "fp16" - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_sq/nlist8192-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_sq/nlist8192-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "fp16" - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_sq/nlist16384-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_sq/nlist16384-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "int8" - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_sq/nlist1024-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_sq/nlist1024-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "int8" - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_sq/nlist2048-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_sq/nlist2048-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "int8" - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_sq/nlist4096-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_sq/nlist4096-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "int8" - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_sq/nlist8192-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_sq/nlist8192-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "int8" - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_sq/nlist16384-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_sq/nlist16384-int8" - }, - { - "name": "faiss_gpu_flat", - "algo": "faiss_gpu_flat", - "build_param": {}, - "file": "index/glove-100-angular/faiss_gpu_flat/flat", - "search_params": [ - {} - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_flat/flat" - }, - - { - "name": "raft_ivf_pq.dimpq128-cluster1024", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-angular/raft_ivf_pq/dimpq128-cluster1024", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/glove-100-angular/raft_ivf_pq/dimpq128-cluster1024" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-angular/raft_ivf_pq/dimpq128-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "nprobe": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 5, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/glove-100-angular/raft_ivf_pq/dimpq128-cluster1024-float-float" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-angular/raft_ivf_pq/dimpq128-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/glove-100-angular/raft_ivf_pq/dimpq128-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-angular/raft_ivf_pq/dimpq128-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-100-angular/raft_ivf_pq/dimpq128-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-angular/raft_ivf_pq/dimpq64-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-100-angular/raft_ivf_pq/dimpq64-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-angular/raft_ivf_pq/dimpq64-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/glove-100-angular/raft_ivf_pq/dimpq64-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq32-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 32, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-angular/raft_ivf_pq/dimpq32-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-100-angular/raft_ivf_pq/dimpq32-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq16-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 16, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-angular/raft_ivf_pq/dimpq16-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-100-angular/raft_ivf_pq/dimpq16-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-half-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-angular/raft_ivf_pq/dimpq128-cluster1024-half-float", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/glove-100-angular/raft_ivf_pq/dimpq128-cluster1024-half-float" - }, - { - "name": "raft_ivf_pq.dimpq512-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 512, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-angular/raft_ivf_pq/dimpq512-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/glove-100-angular/raft_ivf_pq/dimpq512-cluster1024-float-float" - }, - { - "name": "raft_ivf_flat.nlist1024", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 1024, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-angular/raft_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/raft_ivf_flat/nlist1024" - }, - { - "name": "raft_ivf_flat.nlist16384", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 16384, - "ratio": 2, - "niter": 20 - }, - "file": "index/glove-100-angular/raft_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-100-angular/raft_ivf_flat/nlist16384" - }, - - { - "name" : "raft_cagra.dim32", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 32 - }, - "file" : "index/glove-100-angular/raft_cagra/dim32", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/glove-100-angular/raft_cagra/dim32" - }, - - { - "name" : "raft_cagra.dim64", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 64 - }, - "file" : "index/glove-100-angular/raft_cagra/dim64", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/glove-100-angular/raft_cagra/dim64" - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-100-inner.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-100-inner.json deleted file mode 100644 index 5da3fa18d3..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-100-inner.json +++ /dev/null @@ -1,1314 +0,0 @@ -{ - "dataset": { - "name": "glove-100-inner", - "base_file": "glove-100-inner/base.fbin", - "query_file": "glove-100-inner/query.fbin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 5000, - "k": 10, - "run_count": 3 - }, - "index": [ - { - "name" : "hnswlib.M12", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-100-inner/hnswlib/M12", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-100-inner/hnswlib/M12" - }, - { - "name" : "hnswlib.M16", - "algo" : "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-100-inner/hnswlib/M16", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-100-inner/hnswlib/M16" - }, - { - "name" : "hnswlib.M24", - "algo" : "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-100-inner/hnswlib/M24", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-100-inner/hnswlib/M24" - }, - { - "name" : "hnswlib.M36", - "algo" : "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-100-inner/hnswlib/M36", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-100-inner/hnswlib/M36" - }, - - - - - { - "name": "raft_bfknn", - "algo": "raft_bfknn", - "build_param": {}, - "file": "index/glove-100-inner/raft_bfknn/bfknn", - "search_params": [ - { - "probe": 1 - } - ], - "search_result_file": "result/glove-100-inner/raft_bfknn/bfknn" - }, - { - "name": "faiss_gpu_ivf_flat.nlist1024", - "algo": "faiss_gpu_ivf_flat", - "build_param": {"nlist":1024}, - "file": "glove-100-inner/faiss_gpu_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_flat/nlist1024" - }, - { - "name": "faiss_gpu_ivf_flat.nlist2048", - "algo": "faiss_gpu_ivf_flat", - "build_param": {"nlist":2048}, - "file": "glove-100-inner/faiss_gpu_ivf_flat/nlist2048", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_flat/nlist2048" - }, - { - "name": "faiss_gpu_ivf_flat.nlist4096", - "algo": "faiss_gpu_ivf_flat", - "build_param": {"nlist":4096}, - "file": "glove-100-inner/faiss_gpu_ivf_flat/nlist4096", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_flat/nlist4096" - }, - { - "name": "faiss_gpu_ivf_flat.nlist8192", - "algo": "faiss_gpu_ivf_flat", - "build_param": {"nlist":8192}, - "file": "glove-100-inner/faiss_gpu_ivf_flat/nlist8192", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_flat/nlist8192" - }, - { - "name": "faiss_gpu_ivf_flat.nlist16384", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 16384 - }, - "file": "index/glove-100-inner/faiss_gpu_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_gpu_ivf_flat/nlist16384" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": true - }, - "file": "index/glove-100-inner/faiss_gpu_ivf_pq/M64-nlist1024", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024.noprecomp", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": false - }, - "file": "index/glove-100-inner/faiss_gpu_ivf_pq/M64-nlist1024.noprecomp", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "fp16" - }, - "file": "index/glove-100-inner/faiss_gpu_ivf_sq/nlist1024-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_gpu_ivf_sq/nlist1024-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist":2048, "quantizer_type":"fp16"}, - "file": "glove-100-inner/faiss_gpu_ivf_sq/nlist2048-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_sq/nlist2048-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist":4096, "quantizer_type":"fp16"}, - "file": "glove-100-inner/faiss_gpu_ivf_sq/nlist4096-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_sq/nlist4096-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist":8192, "quantizer_type":"fp16"}, - "file": "glove-100-inner/faiss_gpu_ivf_sq/nlist8192-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_sq/nlist8192-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist":16384, "quantizer_type":"fp16"}, - "file": "glove-100-inner/faiss_gpu_ivf_sq/nlist16384-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_sq/nlist16384-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist":1024, "quantizer_type":"int8"}, - "file": "glove-100-inner/faiss_gpu_ivf_sq/nlist1024-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_sq/nlist1024-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist":2048, "quantizer_type":"int8"}, - "file": "glove-100-inner/faiss_gpu_ivf_sq/nlist2048-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_sq/nlist2048-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist":4096, "quantizer_type":"int8"}, - "file": "glove-100-inner/faiss_gpu_ivf_sq/nlist4096-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_sq/nlist4096-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist":8192, "quantizer_type":"int8"}, - "file": "glove-100-inner/faiss_gpu_ivf_sq/nlist8192-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_sq/nlist8192-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist":16384, "quantizer_type":"int8"}, - "file": "glove-100-inner/faiss_gpu_ivf_sq/nlist16384-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_sq/nlist16384-int8" - }, - { - "name": "faiss_gpu_flat", - "algo": "faiss_gpu_flat", - "build_param": {}, - "file": "glove-100-inner/faiss_gpu_flat/flat", - "search_params": [{}], - "search_result_file": "result/glove-100-inner/faiss_gpu_flat/flat" - }, - - { - "name": "raft_ivf_pq.dimpq128-cluster1024", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-inner/raft_ivf_pq/dimpq128-cluster1024", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/glove-100-inner/raft_gpu_ivf_pq/dimpq128-cluster1024" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-inner/raft_ivf_pq/dimpq128-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "nprobe": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 5, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/glove-100-inner/raft_ivf_pq/dimpq128-cluster1024-float-float" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-inner/raft_ivf_pq/dimpq128-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/glove-100-inner/raft_ivf_pq/dimpq128-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-inner/raft_ivf_pq/dimpq128-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-100-inner/raft_ivf_pq/dimpq128-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-inner/raft_ivf_pq/dimpq64-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-100-inner/raft_ivf_pq/dimpq64-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-inner/raft_ivf_pq/dimpq64-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/glove-100-inner/raft_ivf_pq/dimpq64-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq32-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 32, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-inner/raft_ivf_pq/dimpq32-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-100-inner/raft_ivf_pq/dimpq32-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq16-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 16, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-inner/raft_ivf_pq/dimpq16-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-100-inner/raft_ivf_pq/dimpq16-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-half-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-inner/raft_ivf_pq/dimpq128-cluster1024-half-float", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/glove-100-inner/raft_ivf_pq/dimpq128-cluster1024-half-float" - }, - { - "name": "raft_ivf_pq.dimpq512-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 512, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-inner/raft_ivf_pq/dimpq512-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/glove-100-inner/raft_ivf_pq/dimpq512-cluster1024-float-float" - }, - { - "name": "raft_ivf_flat.nlist1024", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 1024, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-inner/raft_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/raft_ivf_flat/nlist1024" - }, - { - "name": "raft_ivf_flat.nlist16384", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 16384, - "ratio": 2, - "niter": 20 - }, - "file": "index/glove-100-inner/raft_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-100-inner/raft_ivf_flat/nlist16384" - }, - - { - "name" : "raft_cagra.dim32", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 32 - }, - "file" : "index/glove-100-inner/raft_cagra/dim32", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/glove-100-inner/raft_cagra/dim32" - }, - - { - "name" : "raft_cagra.dim64", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 64 - }, - "file" : "index/glove-100-inner/raft_cagra/dim64", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/glove-100-inner/raft_cagra/dim64" - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-50-angular.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-50-angular.json deleted file mode 100644 index 11fa07c5c9..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-50-angular.json +++ /dev/null @@ -1,1351 +0,0 @@ -{ - "dataset": { - "name": "glove-50-angular", - "base_file": "glove-50-angular/base.fbin", - "query_file": "glove-50-angular/query.fbin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 5000, - "k": 10, - "run_count": 3 - }, - "index": [ - { - "name" : "hnswlib.M12", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-50-angular/hnswlib/M12", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-50-angular/hnswlib/M12" - }, - { - "name" : "hnswlib.M16", - "algo" : "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-50-angular/hnswlib/M16", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-50-angular/hnswlib/M16" - }, - { - "name" : "hnswlib.M24", - "algo" : "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-50-angular/hnswlib/M24", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-50-angular/hnswlib/M24" - }, - { - "name" : "hnswlib.M36", - "algo" : "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-50-angular/hnswlib/M36", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-50-angular/hnswlib/M36" - }, - - - - - { - "name": "raft_bfknn", - "algo": "raft_bfknn", - "build_param": {}, - "file": "index/glove-50-angular/raft_bfknn/bfknn", - "search_params": [ - { - "probe": 1 - } - ], - "search_result_file": "result/glove-50-angular/raft_bfknn/bfknn" - }, - { - "name": "faiss_gpu_ivf_flat.nlist1024", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 1024 - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_flat/nlist1024" - }, - { - "name": "faiss_gpu_ivf_flat.nlist2048", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 2048 - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_flat/nlist2048", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_flat/nlist2048" - }, - { - "name": "faiss_gpu_ivf_flat.nlist4096", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 4096 - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_flat/nlist4096", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_flat/nlist4096" - }, - { - "name": "faiss_gpu_ivf_flat.nlist8192", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 8192 - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_flat/nlist8192", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_flat/nlist8192" - }, - { - "name": "faiss_gpu_ivf_flat.nlist16384", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 16384 - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_flat/nlist16384" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": true - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_pq/M64-nlist1024", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024.noprecomp", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": false - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_pq/M64-nlist1024.noprecomp", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "fp16" - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_sq/nlist1024-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_sq/nlist1024-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "fp16" - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_sq/nlist2048-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_sq/nlist2048-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "fp16" - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_sq/nlist4096-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_sq/nlist4096-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "fp16" - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_sq/nlist8192-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_sq/nlist8192-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "fp16" - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_sq/nlist16384-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_sq/nlist16384-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "int8" - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_sq/nlist1024-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_sq/nlist1024-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "int8" - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_sq/nlist2048-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_sq/nlist2048-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "int8" - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_sq/nlist4096-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_sq/nlist4096-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "int8" - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_sq/nlist8192-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_sq/nlist8192-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "int8" - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_sq/nlist16384-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_sq/nlist16384-int8" - }, - { - "name": "faiss_gpu_flat", - "algo": "faiss_gpu_flat", - "build_param": {}, - "file": "index/glove-50-angular/faiss_gpu_flat/flat", - "search_params": [ - {} - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_flat/flat" - }, - - { - "name": "raft_ivf_pq.dimpq128-cluster1024", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-angular/raft_ivf_pq/dimpq128-cluster1024", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/glove-50-angular/raft_ivf_pq/dimpq128-cluster1024" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-angular/raft_ivf_pq/dimpq128-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "nprobe": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 5, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/glove-50-angular/raft_ivf_pq/dimpq128-cluster1024-float-float" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-angular/raft_ivf_pq/dimpq128-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/glove-50-angular/raft_ivf_pq/dimpq128-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-angular/raft_ivf_pq/dimpq128-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-50-angular/raft_ivf_pq/dimpq128-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-angular/raft_ivf_pq/dimpq64-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-50-angular/raft_ivf_pq/dimpq64-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-angular/raft_ivf_pq/dimpq64-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/glove-50-angular/raft_ivf_pq/dimpq64-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq32-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 32, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-angular/raft_ivf_pq/dimpq32-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-50-angular/raft_ivf_pq/dimpq32-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq16-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 16, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-angular/raft_ivf_pq/dimpq16-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-50-angular/raft_ivf_pq/dimpq16-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-half-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-angular/raft_ivf_pq/dimpq128-cluster1024-half-float", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/glove-50-angular/raft_ivf_pq/dimpq128-cluster1024-half-float" - }, - { - "name": "raft_ivf_pq.dimpq512-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 512, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-angular/raft_ivf_pq/dimpq512-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/glove-50-angular/raft_ivf_pq/dimpq512-cluster1024-float-float" - }, - { - "name": "raft_ivf_flat.nlist1024", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 1024, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-angular/raft_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/raft_ivf_flat/nlist1024" - }, - { - "name": "raft_ivf_flat.nlist16384", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 16384, - "ratio": 2, - "niter": 20 - }, - "file": "index/glove-50-angular/raft_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-50-angular/raft_ivf_flat/nlist16384" - }, - - { - "name" : "raft_cagra.dim32", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 32 - }, - "file" : "index/glove-50-angular/raft_cagra/dim32", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/glove-50-angular/raft_cagra/dim32" - }, - - { - "name" : "raft_cagra.dim64", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 64 - }, - "file" : "index/glove-50-angular/raft_cagra/dim64", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/glove-50-angular/raft_cagra/dim64" - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-50-inner.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-50-inner.json deleted file mode 100644 index 32613b7c16..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-50-inner.json +++ /dev/null @@ -1,1351 +0,0 @@ -{ - "dataset": { - "name": "glove-50-inner", - "base_file": "glove-50-inner/base.fbin", - "query_file": "glove-50-inner/query.fbin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 5000, - "k": 10, - "run_count": 3 - }, - "index": [ - { - "name" : "hnswlib.M12", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-50-inner/hnswlib/M12", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-50-inner/hnswlib/M12" - }, - { - "name" : "hnswlib.M16", - "algo" : "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-50-inner/hnswlib/M16", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-50-inner/hnswlib/M16" - }, - { - "name" : "hnswlib.M24", - "algo" : "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-50-inner/hnswlib/M24", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-50-inner/hnswlib/M24" - }, - { - "name" : "hnswlib.M36", - "algo" : "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-50-inner/hnswlib/M36", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-50-inner/hnswlib/M36" - }, - - - - - { - "name": "raft_bfknn", - "algo": "raft_bfknn", - "build_param": {}, - "file": "index/glove-50-inner/raft_bfknn/bfknn", - "search_params": [ - { - "probe": 1 - } - ], - "search_result_file": "result/glove-50-inner/raft_bfknn/bfknn" - }, - { - "name": "faiss_ivf_flat.nlist1024", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 1024 - }, - "file": "index/glove-50-inner/faiss_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_flat/nlist1024" - }, - { - "name": "faiss_ivf_flat.nlist2048", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 2048 - }, - "file": "index/glove-50-inner/faiss_ivf_flat/nlist2048", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_flat/nlist2048" - }, - { - "name": "faiss_ivf_flat.nlist4096", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 4096 - }, - "file": "index/glove-50-inner/faiss_ivf_flat/nlist4096", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_flat/nlist4096" - }, - { - "name": "faiss_ivf_flat.nlist8192", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 8192 - }, - "file": "index/glove-50-inner/faiss_ivf_flat/nlist8192", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_flat/nlist8192" - }, - { - "name": "faiss_ivf_flat.nlist16384", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 16384 - }, - "file": "index/glove-50-inner/faiss_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_flat/nlist16384" - }, - { - "name": "faiss_ivf_pq.M64-nlist1024", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": true - }, - "file": "index/glove-50-inner/faiss_ivf_pq/M64-nlist1024", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_ivf_pq.M64-nlist1024.noprecomp", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": false - }, - "file": "index/glove-50-inner/faiss_ivf_pq/M64-nlist1024.noprecomp", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_ivf_sq.nlist1024-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "fp16" - }, - "file": "index/glove-50-inner/faiss_ivf_sq/nlist1024-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_sq/nlist1024-fp16" - }, - { - "name": "faiss_ivf_sq.nlist2048-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "fp16" - }, - "file": "index/glove-50-inner/faiss_ivf_sq/nlist2048-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_sq/nlist2048-fp16" - }, - { - "name": "faiss_ivf_sq.nlist4096-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "fp16" - }, - "file": "index/glove-50-inner/faiss_ivf_sq/nlist4096-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_sq/nlist4096-fp16" - }, - { - "name": "faiss_ivf_sq.nlist8192-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "fp16" - }, - "file": "index/glove-50-inner/faiss_ivf_sq/nlist8192-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_sq/nlist8192-fp16" - }, - { - "name": "faiss_ivf_sq.nlist16384-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "fp16" - }, - "file": "index/glove-50-inner/faiss_ivf_sq/nlist16384-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_sq/nlist16384-fp16" - }, - { - "name": "faiss_ivf_sq.nlist1024-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "int8" - }, - "file": "index/glove-50-inner/faiss_ivf_sq/nlist1024-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_sq/nlist1024-int8" - }, - { - "name": "faiss_ivf_sq.nlist2048-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "int8" - }, - "file": "index/glove-50-inner/faiss_ivf_sq/nlist2048-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_sq/nlist2048-int8" - }, - { - "name": "faiss_ivf_sq.nlist4096-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "int8" - }, - "file": "index/glove-50-inner/faiss_ivf_sq/nlist4096-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_sq/nlist4096-int8" - }, - { - "name": "faiss_ivf_sq.nlist8192-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "int8" - }, - "file": "index/glove-50-inner/faiss_ivf_sq/nlist8192-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_sq/nlist8192-int8" - }, - { - "name": "faiss_ivf_sq.nlist16384-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "int8" - }, - "file": "index/glove-50-inner/faiss_ivf_sq/nlist16384-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_sq/nlist16384-int8" - }, - { - "name": "faiss_flat", - "algo": "faiss_gpu_flat", - "build_param": {}, - "file": "index/glove-50-inner/faiss_flat/flat", - "search_params": [ - {} - ], - "search_result_file": "result/glove-50-inner/faiss_flat/flat" - }, - - { - "name": "raft_ivf_pq.dimpq128-cluster1024", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-inner/raft_ivf_pq/dimpq128-cluster1024", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/glove-50-inner/raft_ivf_pq/dimpq128-cluster1024" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-inner/raft_ivf_pq/dimpq128-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "nprobe": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 5, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/glove-50-inner/raft_ivf_pq/dimpq128-cluster1024-float-float" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-inner/raft_ivf_pq/dimpq128-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/glove-50-inner/raft_ivf_pq/dimpq128-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-inner/raft_ivf_pq/dimpq128-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-50-inner/raft_ivf_pq/dimpq128-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-inner/raft_ivf_pq/dimpq64-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-50-inner/raft_ivf_pq/dimpq64-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-inner/raft_ivf_pq/dimpq64-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/glove-50-inner/raft_ivf_pq/dimpq64-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq32-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 32, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-inner/raft_ivf_pq/dimpq32-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-50-inner/raft_ivf_pq/dimpq32-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq16-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 16, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-inner/raft_ivf_pq/dimpq16-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-50-inner/raft_ivf_pq/dimpq16-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-half-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-inner/raft_ivf_pq/dimpq128-cluster1024-half-float", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/glove-50-inner/raft_ivf_pq/dimpq128-cluster1024-half-float" - }, - { - "name": "raft_ivf_pq.dimpq512-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 512, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-inner/raft_ivf_pq/dimpq512-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/glove-50-inner/raft_ivf_pq/dimpq512-cluster1024-float-float" - }, - { - "name": "raft_ivf_flat.nlist1024", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 1024, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-inner/raft_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/raft_ivf_flat/nlist1024" - }, - { - "name": "raft_ivf_flat.nlist16384", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 16384, - "ratio": 2, - "niter": 20 - }, - "file": "index/glove-50-inner/raft_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-50-inner/raft_ivf_flat/nlist16384" - }, - - { - "name" : "raft_cagra.dim32", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 32 - }, - "file" : "index/glove-50-inner/raft_cagra/dim32", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/glove-50-inner/raft_cagra/dim32" - }, - - { - "name" : "raft_cagra.dim64", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 64 - }, - "file" : "index/glove-50-inner/raft_cagra/dim64", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/glove-50-inner/raft_cagra/dim64" - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/lastfm-65-angular.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/lastfm-65-angular.json deleted file mode 100644 index 943d09231a..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/lastfm-65-angular.json +++ /dev/null @@ -1,1351 +0,0 @@ -{ - "dataset": { - "name": "lastfm-65-angular", - "base_file": "lastfm-65-angular/base.fbin", - "query_file": "lastfm-65-angular/query.fbin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 5000, - "k": 10, - "run_count": 3 - }, - "index": [ - { - "name" : "hnswlib.M12", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "index/lastfm-65-angular/hnswlib/M12", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/lastfm-65-angular/hnswlib/M12" - }, - { - "name" : "hnswlib.M16", - "algo" : "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file" : "index/lastfm-65-angular/hnswlib/M16", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/lastfm-65-angular/hnswlib/M16" - }, - { - "name" : "hnswlib.M24", - "algo" : "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file" : "index/lastfm-65-angular/hnswlib/M24", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/lastfm-65-angular/hnswlib/M24" - }, - { - "name" : "hnswlib.M36", - "algo" : "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file" : "index/lastfm-65-angular/hnswlib/M36", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/lastfm-65-angular/hnswlib/M36" - }, - - - - - { - "name": "raft_bfknn", - "algo": "raft_bfknn", - "build_param": {}, - "file": "index/lastfm-65-angular/raft_bfknn/bfknn", - "search_params": [ - { - "probe": 1 - } - ], - "search_result_file": "result/lastfm-65-angular/raft_bfknn/bfknn" - }, - { - "name": "faiss_gpu_ivf_flat.nlist1024", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 1024 - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_flat/nlist1024" - }, - { - "name": "faiss_gpu_ivf_flat.nlist2048", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 2048 - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_flat/nlist2048", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_flat/nlist2048" - }, - { - "name": "faiss_gpu_ivf_flat.nlist4096", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 4096 - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_flat/nlist4096", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_flat/nlist4096" - }, - { - "name": "faiss_gpu_ivf_flat.nlist8192", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 8192 - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_flat/nlist8192", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_flat/nlist8192" - }, - { - "name": "faiss_gpu_ivf_flat.nlist16384", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 16384 - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_flat/nlist16384" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": true - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_pq/M64-nlist1024", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024.noprecomp", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": false - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_pq/M64-nlist1024.noprecomp", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "fp16" - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_sq/nlist1024-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_sq/nlist1024-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "fp16" - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_sq/nlist2048-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_sq/nlist2048-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "fp16" - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_sq/nlist4096-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_sq/nlist4096-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "fp16" - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_sq/nlist8192-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_sq/nlist8192-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "fp16" - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_sq/nlist16384-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_sq/nlist16384-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "int8" - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_sq/nlist1024-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_sq/nlist1024-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "int8" - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_sq/nlist2048-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_sq/nlist2048-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "int8" - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_sq/nlist4096-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_sq/nlist4096-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "int8" - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_sq/nlist8192-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_sq/nlist8192-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "int8" - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_sq/nlist16384-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_sq/nlist16384-int8" - }, - { - "name": "faiss_gpu_flat", - "algo": "faiss_gpu_flat", - "build_param": {}, - "file": "index/lastfm-65-angular/faiss_gpu_flat/flat", - "search_params": [ - {} - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_flat/flat" - }, - - { - "name": "raft_ivf_pq.dimpq128-cluster1024", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/lastfm-65-angular/raft_ivf_pq/dimpq128-cluster1024", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/lastfm-65-angular/raft_ivf_pq/dimpq128-cluster1024" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/lastfm-65-angular/raft_ivf_pq/dimpq128-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "numProbes": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 5, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/lastfm-65-angular/raft_ivf_pq/dimpq128-cluster1024-float-float" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/lastfm-65-angular/raft_ivf_pq/dimpq128-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/lastfm-65-angular/raft_ivf_pq/dimpq128-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/lastfm-65-angular/raft_ivf_pq/dimpq128-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/lastfm-65-angular/raft_ivf_pq/dimpq128-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/lastfm-65-angular/raft_ivf_pq/dimpq64-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/lastfm-65-angular/raft_ivf_pq/dimpq64-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/lastfm-65-angular/raft_ivf_pq/dimpq64-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/lastfm-65-angular/raft_ivf_pq/dimpq64-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq32-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 32, - "ratio": 1, - "niter": 25 - }, - "file": "index/lastfm-65-angular/raft_ivf_pq/dimpq32-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/lastfm-65-angular/raft_ivf_pq/dimpq32-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq16-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 16, - "ratio": 1, - "niter": 25 - }, - "file": "index/lastfm-65-angular/raft_ivf_pq/dimpq16-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/lastfm-65-angular/raft_ivf_pq/dimpq16-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-half-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/lastfm-65-angular/raft_ivf_pq/dimpq128-cluster1024-half-float", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/lastfm-65-angular/raft_ivf_pq/dimpq128-cluster1024-half-float" - }, - { - "name": "raft_ivf_pq.dimpq512-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 512, - "ratio": 1, - "niter": 25 - }, - "file": "index/lastfm-65-angular/raft_ivf_pq/dimpq512-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/lastfm-65-angular/raft_ivf_pq/dimpq512-cluster1024-float-float" - }, - { - "name": "raft_ivf_flat.nlist1024", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 1024, - "ratio": 1, - "niter": 25 - }, - "file": "index/lastfm-65-angular/raft_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/raft_ivf_flat/nlist1024" - }, - { - "name": "raft_ivf_flat.nlist16384", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 16384, - "ratio": 2, - "niter": 20 - }, - "file": "index/lastfm-65-angular/raft_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/lastfm-65-angular/raft_ivf_flat/nlist16384" - }, - - { - "name" : "raft_cagra.dim32", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 32 - }, - "file" : "index/lastfm-65-angular/raft_cagra/dim32", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/lastfm-65-angular/raft_cagra/dim32" - }, - - { - "name" : "raft_cagra.dim64", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 64 - }, - "file" : "index/lastfm-65-angular/raft_cagra/dim64", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/lastfm-65-angular/raft_cagra/dim64" - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/mnist-784-euclidean.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/mnist-784-euclidean.json deleted file mode 100644 index 04e7ecb469..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/mnist-784-euclidean.json +++ /dev/null @@ -1,1352 +0,0 @@ -{ - "dataset": { - "name": "mnist-784-euclidean", - "base_file": "mnist-784-euclidean/base.fbin", - "query_file": "mnist-784-euclidean/query.fbin", - "groundtruth_neighbors_file": "mnist-784-euclidean/groundtruth.neighbors.ibin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 5000, - "k": 10, - "run_count": 3 - }, - "index": [ - { - "name" : "hnswlib.M12", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "index/mnist-784-euclidean/hnswlib/M12", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/mnist-784-euclidean/hnswlib/M12" - }, - { - "name" : "hnswlib.M16", - "algo" : "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file" : "index/mnist-784-euclidean/hnswlib/M16", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/mnist-784-euclidean/hnswlib/M16" - }, - { - "name" : "hnswlib.M24", - "algo" : "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file" : "index/mnist-784-euclidean/hnswlib/M24", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/mnist-784-euclidean/hnswlib/M24" - }, - { - "name" : "hnswlib.M36", - "algo" : "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file" : "index/mnist-784-euclidean/hnswlib/M36", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/mnist-784-euclidean/hnswlib/M36" - }, - - - - - { - "name": "raft_bfknn", - "algo": "raft_bfknn", - "build_param": {}, - "file": "index/mnist-784-euclidean/raft_bfknn/bfknn", - "search_params": [ - { - "probe": 1 - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_bfknn/bfknn" - }, - { - "name": "faiss_gpu_ivf_flat.nlist1024", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 1024 - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_flat/nlist1024" - }, - { - "name": "faiss_gpu_ivf_flat.nlist2048", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 2048 - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_flat/nlist2048", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_flat/nlist2048" - }, - { - "name": "faiss_gpu_ivf_flat.nlist4096", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 4096 - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_flat/nlist4096", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_flat/nlist4096" - }, - { - "name": "faiss_gpu_ivf_flat.nlist8192", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 8192 - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_flat/nlist8192", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_flat/nlist8192" - }, - { - "name": "faiss_gpu_ivf_flat.nlist16384", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 16384 - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_flat/nlist16384" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": true - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_pq/M64-nlist1024", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024.noprecomp", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": false - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_pq/M64-nlist1024.noprecomp", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "fp16" - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist1024-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist1024-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "fp16" - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist2048-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist2048-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "fp16" - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist4096-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist4096-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "fp16" - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist8192-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist8192-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "fp16" - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist16384-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist16384-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "int8" - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist1024-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist1024-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "int8" - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist2048-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist2048-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "int8" - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist4096-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist4096-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "int8" - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist8192-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist8192-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "int8" - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist16384-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist16384-int8" - }, - { - "name": "faiss_gpu_flat", - "algo": "faiss_gpu_flat", - "build_param": {}, - "file": "index/mnist-784-euclidean/faiss_gpu_flat/flat", - "search_params": [ - {} - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_flat/flat" - }, - - { - "name": "raft_ivf_pq.dimpq128-cluster1024", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "numProbes": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 5, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-float" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/mnist-784-euclidean/raft_ivf_pq/dimpq64-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_ivf_pq/dimpq64-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/mnist-784-euclidean/raft_ivf_pq/dimpq64-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_ivf_pq/dimpq64-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq32-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 32, - "ratio": 1, - "niter": 25 - }, - "file": "index/mnist-784-euclidean/raft_ivf_pq/dimpq32-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_ivf_pq/dimpq32-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq16-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 16, - "ratio": 1, - "niter": 25 - }, - "file": "index/mnist-784-euclidean/raft_ivf_pq/dimpq16-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_ivf_pq/dimpq16-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-half-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-half-float", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-half-float" - }, - { - "name": "raft_ivf_pq.dimpq512-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 512, - "ratio": 1, - "niter": 25 - }, - "file": "index/mnist-784-euclidean/raft_ivf_pq/dimpq512-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_ivf_pq/dimpq512-cluster1024-float-float" - }, - { - "name": "raft_ivf_flat.nlist1024", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 1024, - "ratio": 1, - "niter": 25 - }, - "file": "index/mnist-784-euclidean/raft_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_ivf_flat/nlist1024" - }, - { - "name": "raft_ivf_flat.nlist16384", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 16384, - "ratio": 2, - "niter": 20 - }, - "file": "index/mnist-784-euclidean/raft_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_ivf_flat/nlist16384" - }, - - { - "name" : "raft_cagra.dim32", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 32 - }, - "file" : "index/mnist-784-euclidean/raft_cagra/dim32", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/mnist-784-euclidean/raft_cagra/dim32" - }, - - { - "name" : "raft_cagra.dim64", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 64 - }, - "file" : "index/mnist-784-euclidean/raft_cagra/dim64", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/mnist-784-euclidean/raft_cagra/dim64" - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/nytimes-256-angular.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/nytimes-256-angular.json deleted file mode 100644 index df2a16f1f8..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/nytimes-256-angular.json +++ /dev/null @@ -1,1352 +0,0 @@ -{ - "dataset": { - "name": "nytimes-256-angular", - "base_file": "nytimes-256-angular/base.fbin", - "query_file": "nytimes-256-angular/query.fbin", - "groundtruth_neighbors_file": "nytimes-256-angular/groundtruth.neighbors.ibin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 5000, - "k": 10, - "run_count": 3 - }, - "index": [ - { - "name" : "hnswlib.M12", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "index/nytimes-256-angular/hnswlib/M12", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/nytimes-256-angular/hnswlib/M12" - }, - { - "name" : "hnswlib.M16", - "algo" : "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file" : "index/nytimes-256-angular/hnswlib/M16", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/nytimes-256-angular/hnswlib/M16" - }, - { - "name" : "hnswlib.M24", - "algo" : "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file" : "index/nytimes-256-angular/hnswlib/M24", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/nytimes-256-angular/hnswlib/M24" - }, - { - "name" : "hnswlib.M36", - "algo" : "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file" : "index/nytimes-256-angular/hnswlib/M36", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/nytimes-256-angular/hnswlib/M36" - }, - - - - - { - "name": "raft_bfknn", - "algo": "raft_bfknn", - "build_param": {}, - "file": "index/nytimes-256-angular/raft_bfknn/bfknn", - "search_params": [ - { - "probe": 1 - } - ], - "search_result_file": "result/nytimes-256-angular/raft_bfknn/bfknn" - }, - { - "name": "faiss_gpu_ivf_flat.nlist1024", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 1024 - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_flat/nlist1024" - }, - { - "name": "faiss_gpu_ivf_flat.nlist2048", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 2048 - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_flat/nlist2048", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_flat/nlist2048" - }, - { - "name": "faiss_gpu_ivf_flat.nlist4096", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 4096 - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_flat/nlist4096", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_flat/nlist4096" - }, - { - "name": "faiss_gpu_ivf_flat.nlist8192", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 8192 - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_flat/nlist8192", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_flat/nlist8192" - }, - { - "name": "faiss_gpu_ivf_flat.nlist16384", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 16384 - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_flat/nlist16384" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": true - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_pq/M64-nlist1024", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024.noprecomp", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": false - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_pq/M64-nlist1024.noprecomp", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "fp16" - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_sq/nlist1024-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_sq/nlist1024-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "fp16" - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_sq/nlist2048-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_sq/nlist2048-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "fp16" - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_sq/nlist4096-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_sq/nlist4096-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "fp16" - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_sq/nlist8192-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_sq/nlist8192-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "fp16" - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_sq/nlist16384-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_sq/nlist16384-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "int8" - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_sq/nlist1024-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_sq/nlist1024-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "int8" - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_sq/nlist2048-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_sq/nlist2048-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "int8" - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_sq/nlist4096-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_sq/nlist4096-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "int8" - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_sq/nlist8192-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_sq/nlist8192-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "int8" - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_sq/nlist16384-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_sq/nlist16384-int8" - }, - { - "name": "faiss_gpu_flat", - "algo": "faiss_gpu_flat", - "build_param": {}, - "file": "index/nytimes-256-angular/faiss_gpu_flat/flat", - "search_params": [ - {} - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_flat/flat" - }, - - { - "name": "raft_ivf_pq.dimpq128-cluster1024", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-angular/raft_ivf_pq/dimpq128-cluster1024", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/nytimes-256-angular/raft_ivf_pq/dimpq128-cluster1024" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-angular/raft_ivf_pq/dimpq128-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "numProbes": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 5, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/nytimes-256-angular/raft_ivf_pq/dimpq128-cluster1024-float-float" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-angular/raft_ivf_pq/dimpq128-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/nytimes-256-angular/raft_ivf_pq/dimpq128-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-angular/raft_ivf_pq/dimpq128-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/nytimes-256-angular/raft_ivf_pq/dimpq128-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-angular/raft_ivf_pq/dimpq64-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/nytimes-256-angular/raft_ivf_pq/dimpq64-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-angular/raft_ivf_pq/dimpq64-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/nytimes-256-angular/raft_ivf_pq/dimpq64-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq32-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 32, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-angular/raft_ivf_pq/dimpq32-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/nytimes-256-angular/raft_ivf_pq/dimpq32-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq16-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 16, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-angular/raft_ivf_pq/dimpq16-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/nytimes-256-angular/raft_ivf_pq/dimpq16-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-half-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-angular/raft_ivf_pq/dimpq128-cluster1024-half-float", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/nytimes-256-angular/raft_ivf_pq/dimpq128-cluster1024-half-float" - }, - { - "name": "raft_ivf_pq.dimpq512-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 512, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-angular/raft_ivf_pq/dimpq512-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/nytimes-256-angular/raft_ivf_pq/dimpq512-cluster1024-float-float" - }, - { - "name": "raft_ivf_flat.nlist1024", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 1024, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-angular/raft_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/raft_ivf_flat/nlist1024" - }, - { - "name": "raft_ivf_flat.nlist16384", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 16384, - "ratio": 2, - "niter": 20 - }, - "file": "index/nytimes-256-angular/raft_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/nytimes-256-angular/raft_ivf_flat/nlist16384" - }, - - { - "name" : "raft_cagra.dim32", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 32 - }, - "file" : "index/nytimes-256-angular/raft_cagra/dim32", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/nytimes-256-angular/raft_cagra/dim32" - }, - - { - "name" : "raft_cagra.dim64", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 64 - }, - "file" : "index/nytimes-256-angular/raft_cagra/dim64", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/nytimes-256-angular/raft_cagra/dim64" - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/nytimes-256-inner.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/nytimes-256-inner.json deleted file mode 100644 index 18942a95c3..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/nytimes-256-inner.json +++ /dev/null @@ -1,1352 +0,0 @@ -{ - "dataset": { - "name": "nytimes-256-inner", - "base_file": "nytimes-256-inner/base.fbin", - "query_file": "nytimes-256-inner/query.fbin", - "groundtruth_neighbors_file": "nytimes-256-inner/groundtruth.neighbors.ibin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 5000, - "k": 10, - "run_count": 3 - }, - "index": [ - { - "name" : "hnswlib.M12", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "index/nytimes-256-inner/hnswlib/M12", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/nytimes-256-inner/hnswlib/M12" - }, - { - "name" : "hnswlib.M16", - "algo" : "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file" : "index/nytimes-256-inner/hnswlib/M16", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/nytimes-256-inner/hnswlib/M16" - }, - { - "name" : "hnswlib.M24", - "algo" : "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file" : "index/nytimes-256-inner/hnswlib/M24", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/nytimes-256-inner/hnswlib/M24" - }, - { - "name" : "hnswlib.M36", - "algo" : "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file" : "index/nytimes-256-inner/hnswlib/M36", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/nytimes-256-inner/hnswlib/M36" - }, - - - - - { - "name": "raft_bfknn", - "algo": "raft_bfknn", - "build_param": {}, - "file": "index/nytimes-256-inner/raft_bfknn/bfknn", - "search_params": [ - { - "probe": 1 - } - ], - "search_result_file": "result/nytimes-256-inner/raft_bfknn/bfknn" - }, - { - "name": "faiss_ivf_flat.nlist1024", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 1024 - }, - "file": "index/nytimes-256-inner/faiss_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_flat/nlist1024" - }, - { - "name": "faiss_ivf_flat.nlist2048", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 2048 - }, - "file": "index/nytimes-256-inner/faiss_ivf_flat/nlist2048", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_flat/nlist2048" - }, - { - "name": "faiss_ivf_flat.nlist4096", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 4096 - }, - "file": "index/nytimes-256-inner/faiss_ivf_flat/nlist4096", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_flat/nlist4096" - }, - { - "name": "faiss_ivf_flat.nlist8192", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 8192 - }, - "file": "index/nytimes-256-inner/faiss_ivf_flat/nlist8192", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_flat/nlist8192" - }, - { - "name": "faiss_ivf_flat.nlist16384", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 16384 - }, - "file": "index/nytimes-256-inner/faiss_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_flat/nlist16384" - }, - { - "name": "faiss_ivf_pq.M64-nlist1024", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": true - }, - "file": "index/nytimes-256-inner/faiss_ivf_pq/M64-nlist1024", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_ivf_pq.M64-nlist1024.noprecomp", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": false - }, - "file": "index/nytimes-256-inner/faiss_ivf_pq/M64-nlist1024.noprecomp", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_ivf_sq.nlist1024-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "fp16" - }, - "file": "index/nytimes-256-inner/faiss_ivf_sq/nlist1024-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_sq/nlist1024-fp16" - }, - { - "name": "faiss_ivf_sq.nlist2048-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "fp16" - }, - "file": "index/nytimes-256-inner/faiss_ivf_sq/nlist2048-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_sq/nlist2048-fp16" - }, - { - "name": "faiss_ivf_sq.nlist4096-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "fp16" - }, - "file": "index/nytimes-256-inner/faiss_ivf_sq/nlist4096-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_sq/nlist4096-fp16" - }, - { - "name": "faiss_ivf_sq.nlist8192-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "fp16" - }, - "file": "index/nytimes-256-inner/faiss_ivf_sq/nlist8192-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_sq/nlist8192-fp16" - }, - { - "name": "faiss_ivf_sq.nlist16384-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "fp16" - }, - "file": "index/nytimes-256-inner/faiss_ivf_sq/nlist16384-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_sq/nlist16384-fp16" - }, - { - "name": "faiss_ivf_sq.nlist1024-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "int8" - }, - "file": "index/nytimes-256-inner/faiss_ivf_sq/nlist1024-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_sq/nlist1024-int8" - }, - { - "name": "faiss_ivf_sq.nlist2048-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "int8" - }, - "file": "index/nytimes-256-inner/faiss_ivf_sq/nlist2048-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_sq/nlist2048-int8" - }, - { - "name": "faiss_ivf_sq.nlist4096-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "int8" - }, - "file": "index/nytimes-256-inner/faiss_ivf_sq/nlist4096-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_sq/nlist4096-int8" - }, - { - "name": "faiss_ivf_sq.nlist8192-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "int8" - }, - "file": "index/nytimes-256-inner/faiss_ivf_sq/nlist8192-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_sq/nlist8192-int8" - }, - { - "name": "faiss_ivf_sq.nlist16384-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "int8" - }, - "file": "index/nytimes-256-inner/faiss_ivf_sq/nlist16384-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_sq/nlist16384-int8" - }, - { - "name": "faiss_flat", - "algo": "faiss_gpu_flat", - "build_param": {}, - "file": "index/nytimes-256-inner/faiss_flat/flat", - "search_params": [ - {} - ], - "search_result_file": "result/nytimes-256-inner/faiss_flat/flat" - }, - - { - "name": "raft_ivf_pq.dimpq128-cluster1024", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-inner/raft_ivf_pq/dimpq128-cluster1024", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/nytimes-256-inner/raft_ivf_pq/dimpq128-cluster1024" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-inner/raft_ivf_pq/dimpq128-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "numProbes": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 5, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/nytimes-256-inner/raft_ivf_pq/dimpq128-cluster1024-float-float" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-inner/raft_ivf_pq/dimpq128-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/nytimes-256-inner/raft_ivf_pq/dimpq128-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-inner/raft_ivf_pq/dimpq128-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/nytimes-256-inner/raft_ivf_pq/dimpq128-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-inner/raft_ivf_pq/dimpq64-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/nytimes-256-inner/raft_ivf_pq/dimpq64-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-inner/raft_ivf_pq/dimpq64-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/nytimes-256-inner/raft_ivf_pq/dimpq64-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq32-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 32, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-inner/raft_ivf_pq/dimpq32-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/nytimes-256-inner/raft_ivf_pq/dimpq32-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq16-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 16, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-inner/raft_ivf_pq/dimpq16-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/nytimes-256-inner/raft_ivf_pq/dimpq16-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-half-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-inner/raft_ivf_pq/dimpq128-cluster1024-half-float", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/nytimes-256-inner/raft_ivf_pq/dimpq128-cluster1024-half-float" - }, - { - "name": "raft_ivf_pq.dimpq512-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 512, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-inner/raft_ivf_pq/dimpq512-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/nytimes-256-inner/raft_ivf_pq/dimpq512-cluster1024-float-float" - }, - { - "name": "raft_ivf_flat.nlist1024", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 1024, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-inner/raft_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/raft_ivf_flat/nlist1024" - }, - { - "name": "raft_ivf_flat.nlist16384", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 16384, - "ratio": 2, - "niter": 20 - }, - "file": "index/nytimes-256-inner/raft_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/nytimes-256-inner/raft_ivf_flat/nlist16384" - }, - - { - "name" : "raft_cagra.dim32", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 32 - }, - "file" : "index/nytimes-256-inner/raft_cagra/dim32", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/nytimes-256-inner/raft_cagra/dim32" - }, - - { - "name" : "raft_cagra.dim64", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 64 - }, - "file" : "index/nytimes-256-inner/raft_cagra/dim64", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/nytimes-256-inner/raft_cagra/dim64" - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/sift-128-euclidean.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/sift-128-euclidean.json deleted file mode 100644 index 791261251a..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/sift-128-euclidean.json +++ /dev/null @@ -1,498 +0,0 @@ -{ - "dataset": { - "name": "sift-128-euclidean", - "base_file": "sift-128-euclidean/base.fbin", - "query_file": "sift-128-euclidean/query.fbin", - "groundtruth_neighbors_file": "sift-128-euclidean/groundtruth.neighbors.ibin", - "distance": "euclidean" - }, - - "search_basic_param": { - "batch_size": 5000, - "k": 10 - }, - - "index": [ - { - "name": "hnswlib.M12", - "algo": "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file": "sift-128-euclidean/hnswlib/M12", - "search_params": [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ] - }, - { - "name": "hnswlib.M16", - "algo": "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file": "sift-128-euclidean/hnswlib/M16", - "search_params": [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ] - }, - { - "name": "hnswlib.M24", - "algo": "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file": "sift-128-euclidean/hnswlib/M24", - "search_params": [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ] - }, - { - "name": "hnswlib.M36", - "algo": "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file": "sift-128-euclidean/hnswlib/M36", - "search_params": [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ] - }, - { - "name": "raft_bfknn", - "algo": "raft_bfknn", - "build_param": {}, - "file": "sift-128-euclidean/raft_bfknn/bfknn", - "search_params": [{"probe": 1}] - }, - { - "name": "faiss_gpu_ivf_flat.nlist1024", - "algo": "faiss_gpu_ivf_flat", - "build_param": {"nlist": 1024}, - "file": "sift-128-euclidean/faiss_gpu_ivf_flat/nlist1024", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_flat.nlist2048", - "algo": "faiss_gpu_ivf_flat", - "build_param": {"nlist": 2048}, - "file": "sift-128-euclidean/faiss_gpu_ivf_flat/nlist2048", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_flat.nlist4096", - "algo": "faiss_gpu_ivf_flat", - "build_param": {"nlist": 4096}, - "file": "sift-128-euclidean/faiss_gpu_ivf_flat/nlist4096", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_flat.nlist8192", - "algo": "faiss_gpu_ivf_flat", - "build_param": {"nlist": 8192}, - "file": "sift-128-euclidean/faiss_gpu_ivf_flat/nlist8192", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_flat.nlist16384", - "algo": "faiss_gpu_ivf_flat", - "build_param": {"nlist": 16384}, - "file": "sift-128-euclidean/faiss_gpu_ivf_flat/nlist16384", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000}, - {"nprobe": 2000} - ] - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024", - "algo": "faiss_gpu_ivf_pq", - "build_param": {"nlist": 1024, "M": 64, "useFloat16": true, "usePrecomputed": true}, - "file": "sift-128-euclidean/faiss_gpu_ivf_pq/M64-nlist1024", - "search_params": [ - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024.noprecomp", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": false - }, - "file": "sift-128-euclidean/faiss_gpu_ivf_pq/M64-nlist1024.noprecomp", - "search_params": [ - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist": 1024, "quantizer_type": "fp16"}, - "file": "sift-128-euclidean/faiss_gpu_ivf_sq/nlist1024-fp16", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist": 2048, "quantizer_type": "fp16"}, - "file": "sift-128-euclidean/faiss_gpu_ivf_sq/nlist2048-fp16", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist": 4096, "quantizer_type": "fp16"}, - "file": "sift-128-euclidean/faiss_gpu_ivf_sq/nlist4096-fp16", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist": 8192, "quantizer_type": "fp16"}, - "file": "sift-128-euclidean/faiss_gpu_ivf_sq/nlist8192-fp16", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist": 16384, "quantizer_type": "fp16"}, - "file": "sift-128-euclidean/faiss_gpu_ivf_sq/nlist16384-fp16", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000}, - {"nprobe": 2000} - ] - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist": 1024, "quantizer_type": "int8"}, - "file": "sift-128-euclidean/faiss_gpu_ivf_sq/nlist1024-int8", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist": 2048,"quantizer_type": "int8"}, - "file": "sift-128-euclidean/faiss_gpu_ivf_sq/nlist2048-int8", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist": 4096, "quantizer_type": "int8"}, - "file": "sift-128-euclidean/faiss_gpu_ivf_sq/nlist4096-int8", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist": 8192, "quantizer_type": "int8"}, - "file": "sift-128-euclidean/faiss_gpu_ivf_sq/nlist8192-int8", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist": 16384, "quantizer_type": "int8"}, - "file": "sift-128-euclidean/faiss_gpu_ivf_sq/nlist16384-int8", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000}, - {"nprobe": 2000} - ] - }, - { - "name": "faiss_gpu_flat", - "algo": "faiss_gpu_flat", - "build_param": {}, - "file": "sift-128-euclidean/faiss_gpu_flat/flat", - "search_params": [{}] - }, - { - "name": "raft_ivf_pq.dimpq64-bitpq8-cluster1K", - "algo": "raft_ivf_pq", - "build_param": {"niter": 25, "nlist": 1000, "pq_dim": 64, "pq_bits": 8, "ratio": 1}, - "file": "sift-128-euclidean/raft_ivf_pq/dimpq64-bitpq8-cluster1K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 1000, "internalDistanceDtype": "half", "smemLutDtype": "half" } - ] - }, - { - "name": "raft_ivf_pq.dimpq128-bitpq6-cluster1K", - "algo": "raft_ivf_pq", - "build_param": {"niter": 25, "nlist": 1000, "pq_dim": 128, "pq_bits": 6, "ratio": 1}, - "file": "sift-128-euclidean/raft_ivf_pq/dimpq128-bitpq6-cluster1K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 1000, "internalDistanceDtype": "half", "smemLutDtype": "half" } - ] - }, - { - "name": "raft_ivf_flat.nlist1024", - "algo": "raft_ivf_flat", - "build_param": {"nlist": 1024, "ratio": 1, "niter": 25}, - "file": "sift-128-euclidean/raft_ivf_flat/nlist1024", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "raft_ivf_flat.nlist16384", - "algo": "raft_ivf_flat", - "build_param": {"nlist": 16384, "ratio": 2, "niter": 20}, - "file": "sift-128-euclidean/raft_ivf_flat/nlist16384", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000}, - {"nprobe": 2000} - ] - }, - { - "name": "raft_cagra.dim32", - "algo": "raft_cagra", - "build_param": {"graph_degree": 32}, - "file": "sift-128-euclidean/raft_cagra/dim32", - "search_params": [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ] - }, - { - "name": "raft_cagra.dim64", - "algo": "raft_cagra", - "build_param": {"graph_degree": 64}, - "file": "sift-128-euclidean/raft_cagra/dim64", - "search_params": [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ] - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/wiki_all_10M.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/wiki_all_10M.json deleted file mode 100644 index e5f77e7858..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/wiki_all_10M.json +++ /dev/null @@ -1,200 +0,0 @@ -{ - "dataset": { - "name": "wiki_all_10M", - "base_file": "wiki_all_10M/base.88M.fbin", - "query_file": "wiki_all_10M/queries.fbin", - "groundtruth_neighbors_file": "wiki_all_10M/groundtruth.88M.neighbors.ibin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 10000, - "k": 10 - }, - "index": [ - { - "name": "hnswlib.M16.ef50", - "algo": "hnswlib", - "build_param": { "M": 16, "efConstruction": 50, "numThreads": 56 }, - "file": "wiki_all_10M/hnswlib/M16.ef50", - "search_params": [ - { "ef": 10, "numThreads": 56 }, - { "ef": 20, "numThreads": 56 }, - { "ef": 40, "numThreads": 56 }, - { "ef": 60, "numThreads": 56 }, - { "ef": 80, "numThreads": 56 }, - { "ef": 120, "numThreads": 56 }, - { "ef": 200, "numThreads": 56 }, - { "ef": 400, "numThreads": 56 }, - { "ef": 600, "numThreads": 56 }, - { "ef": 800, "numThreads": 56 } - ] - }, - { - "name": "faiss_ivf_pq.M32-nlist16K", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "M": 32, - "nlist": 16384, - "ratio": 2 - }, - "file": "wiki_all_10M/faiss_ivf_pq/M32-nlist16K_ratio2", - "search_params": [ - { "nprobe": 10 }, - { "nprobe": 20 }, - { "nprobe": 30 }, - { "nprobe": 40 }, - { "nprobe": 50 }, - { "nprobe": 100 }, - { "nprobe": 200 }, - { "nprobe": 500 } - ] - }, - { - "name": "faiss_ivf_pq.M64-nlist16K", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "M": 64, - "nlist": 16384, - "ratio": 2 - }, - "file": "wiki_all_10M/faiss_ivf_pq/M64-nlist16K_ratio2", - "search_params": [ - { "nprobe": 10 }, - { "nprobe": 20 }, - { "nprobe": 30 }, - { "nprobe": 40 }, - { "nprobe": 50 }, - { "nprobe": 100 }, - { "nprobe": 200 }, - { "nprobe": 500 } - ] - }, - { - "name": "raft_ivf_pq.d128-nlist16K", - "algo": "raft_ivf_pq", - "build_param": { - "pq_dim": 128, - "pq_bits": 8, - "nlist": 16384, - "niter": 10, - "ratio": 10 - }, - "file": "wiki_all_10M/raft_ivf_pq/d128-nlist16K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 } - ] - }, - { - "name": "raft_ivf_pq.d64-nlist16K", - "algo": "raft_ivf_pq", - "build_param": { - "pq_dim": 64, - "pq_bits": 8, - "nlist": 16384, - "niter": 10, - "ratio": 10 - }, - "file": "wiki_all_10M/raft_ivf_pq/d64-nlist16K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 } - ] - }, - { - "name": "raft_ivf_pq.d32-nlist16K", - "algo": "raft_ivf_pq", - "build_param": { - "pq_dim": 32, - "pq_bits": 8, - "nlist": 16384, - "niter": 10, - "ratio": 10 - }, - "file": "wiki_all_10M/raft_ivf_pq/d32-nlist16K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 } - ] - }, - { - "name": "raft_ivf_pq.d32X-nlist16K", - "algo": "raft_ivf_pq", - "build_param": { - "pq_dim": 32, - "pq_bits": 8, - "nlist": 16384, - "niter": 10, - "ratio": 10 - }, - "file": "wiki_all_10M/raft_ivf_pq/d32-nlist16K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 } - - ] - }, - { - "name": "raft_cagra.dim32.multi_cta", - "algo": "raft_cagra", - "build_param": { "graph_degree": 32, "intermediate_graph_degree": 48 }, - "file": "wiki_all_10M/raft_cagra/dim32.ibin", - "search_params": [ - { "itopk": 32, "search_width": 1, "max_iterations": 0, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 32, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 36, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 40, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 44, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 48, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 2, "max_iterations": 16, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 2, "max_iterations": 24, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 2, "max_iterations": 26, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 2, "max_iterations": 32, "algo": "multi_cta" }, - { "itopk": 64, "search_width": 4, "max_iterations": 16, "algo": "multi_cta" }, - { "itopk": 64, "search_width": 1, "max_iterations": 64, "algo": "multi_cta" }, - { "itopk": 96, "search_width": 2, "max_iterations": 48, "algo": "multi_cta" }, - { "itopk": 128, "search_width": 8, "max_iterations": 16, "algo": "multi_cta" }, - { "itopk": 128, "search_width": 2, "max_iterations": 64, "algo": "multi_cta" }, - { "itopk": 192, "search_width": 8, "max_iterations": 24, "algo": "multi_cta" }, - { "itopk": 192, "search_width": 2, "max_iterations": 96, "algo": "multi_cta" }, - { "itopk": 256, "search_width": 8, "max_iterations": 32, "algo": "multi_cta" }, - { "itopk": 384, "search_width": 8, "max_iterations": 48, "algo": "multi_cta" }, - { "itopk": 512, "search_width": 8, "max_iterations": 64, "algo": "multi_cta" } - ] - } - - ] -} - diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/wiki_all_1M.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/wiki_all_1M.json deleted file mode 100644 index 2d1ec1e322..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/wiki_all_1M.json +++ /dev/null @@ -1,216 +0,0 @@ -{ - "dataset": { - "name": "wiki_all_1M", - "base_file": "wiki_all_1M/base.1M.fbin", - "subset_size": 1000000, - "query_file": "wiki_all_1M/queries.fbin", - "groundtruth_neighbors_file": "wiki_all_1M/groundtruth.1M.neighbors.ibin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 10000, - "k": 10 - }, - "index": [ - { - "name": "hnswlib.M16.ef50", - "algo": "hnswlib", - "build_param": { "M": 16, "efConstruction": 50, "numThreads": 56 }, - "file": "wiki_all_1M/hnswlib/M16.ef50", - "search_params": [ - { "ef": 10, "numThreads": 56 }, - { "ef": 20, "numThreads": 56 }, - { "ef": 40, "numThreads": 56 }, - { "ef": 60, "numThreads": 56 }, - { "ef": 80, "numThreads": 56 }, - { "ef": 120, "numThreads": 56 }, - { "ef": 200, "numThreads": 56 }, - { "ef": 400, "numThreads": 56 }, - { "ef": 600, "numThreads": 56 }, - { "ef": 800, "numThreads": 56 } - ] - }, - { - "name": "faiss_ivf_pq.M32-nlist16K", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "M": 32, - "nlist": 16384, - "ratio": 2 - }, - "file": "wiki_all_1M/faiss_ivf_pq/M32-nlist16K_ratio2", - "search_params": [ - { "nprobe": 10 }, - { "nprobe": 20 }, - { "nprobe": 30 }, - { "nprobe": 40 }, - { "nprobe": 50 }, - { "nprobe": 100 }, - { "nprobe": 200 }, - { "nprobe": 500 } - ] - }, - { - "name": "faiss_ivf_pq.M64-nlist16K", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "M": 64, - "nlist": 16384, - "ratio": 2 - }, - "file": "wiki_all_1M/faiss_ivf_pq/M64-nlist16K_ratio2", - "search_params": [ - { "nprobe": 10 }, - { "nprobe": 20 }, - { "nprobe": 30 }, - { "nprobe": 40 }, - { "nprobe": 50 }, - { "nprobe": 100 }, - { "nprobe": 200 }, - { "nprobe": 500 } - ] - }, - { - "name": "raft_ivf_pq.d128-nlist16K", - "algo": "raft_ivf_pq", - "build_param": { - "pq_dim": 128, - "pq_bits": 8, - "nlist": 16384, - "niter": 10, - "ratio": 10 - }, - "file": "wiki_all_1M/raft_ivf_pq/d128-nlist16K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 } - ] - }, - { - "name": "raft_ivf_pq.d64-nlist16K", - "algo": "raft_ivf_pq", - "build_param": { - "pq_dim": 64, - "pq_bits": 8, - "nlist": 16384, - "niter": 10, - "ratio": 10 - }, - "file": "wiki_all_1M/raft_ivf_pq/d64-nlist16K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 } - ] - }, - { - "name": "raft_ivf_pq.d32-nlist16K", - "algo": "raft_ivf_pq", - "build_param": { - "pq_dim": 32, - "pq_bits": 8, - "nlist": 16384, - "niter": 10, - "ratio": 10 - }, - "file": "wiki_all_1M/raft_ivf_pq/d32-nlist16K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 } - ] - }, - { - "name": "raft_ivf_pq.d32X-nlist16K", - "algo": "raft_ivf_pq", - "build_param": { - "pq_dim": 32, - "pq_bits": 8, - "nlist": 16384, - "niter": 10, - "ratio": 10 - }, - "file": "wiki_all_1M/raft_ivf_pq/d32-nlist16K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 } - - ] - }, - { - "name": "raft_cagra.dim32.multi_cta", - "algo": "raft_cagra", - "build_param": { "graph_degree": 32, - "intermediate_graph_degree": 48, - "graph_build_algo": "NN_DESCENT", - "ivf_pq_build_pq_dim": 32, - "ivf_pq_build_pq_bits": 8, - "ivf_pq_build_nlist": 16384, - "ivf_pq_build_niter": 10, - "ivf_pq_build_ratio": 10, - "ivf_pq_search_nprobe": 30, - "ivf_pq_search_internalDistanceDtype": "half", - "ivf_pq_search_smemLutDtype": "half", - "ivf_pq_search_refine_ratio": 8, - "nn_descent_max_iterations": 10, - "nn_descent_intermediate_graph_degree": 72, - "nn_descent_termination_threshold": 0.001 - }, - "file": "wiki_all_1M/raft_cagra/dim32.ibin", - "search_params": [ - { "itopk": 32, "search_width": 1, "max_iterations": 0, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 32, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 36, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 40, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 44, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 48, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 2, "max_iterations": 16, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 2, "max_iterations": 24, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 2, "max_iterations": 26, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 2, "max_iterations": 32, "algo": "multi_cta" }, - { "itopk": 64, "search_width": 4, "max_iterations": 16, "algo": "multi_cta" }, - { "itopk": 64, "search_width": 1, "max_iterations": 64, "algo": "multi_cta" }, - { "itopk": 96, "search_width": 2, "max_iterations": 48, "algo": "multi_cta" }, - { "itopk": 128, "search_width": 8, "max_iterations": 16, "algo": "multi_cta" }, - { "itopk": 128, "search_width": 2, "max_iterations": 64, "algo": "multi_cta" }, - { "itopk": 192, "search_width": 8, "max_iterations": 24, "algo": "multi_cta" }, - { "itopk": 192, "search_width": 2, "max_iterations": 96, "algo": "multi_cta" }, - { "itopk": 256, "search_width": 8, "max_iterations": 32, "algo": "multi_cta" }, - { "itopk": 384, "search_width": 8, "max_iterations": 48, "algo": "multi_cta" }, - { "itopk": 512, "search_width": 8, "max_iterations": 64, "algo": "multi_cta" } - ] - } - - ] -} - diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/wiki_all_88M.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/wiki_all_88M.json deleted file mode 100644 index e50b40f554..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/wiki_all_88M.json +++ /dev/null @@ -1,200 +0,0 @@ -{ - "dataset": { - "name": "wiki_all_88M", - "base_file": "wiki_all_88M/base.88M.fbin", - "query_file": "wiki_all_88M/queries.fbin", - "groundtruth_neighbors_file": "wiki_all_88M/groundtruth.88M.neighbors.ibin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 10000, - "k": 10 - }, - "index": [ - { - "name": "hnswlib.M16.ef50", - "algo": "hnswlib", - "build_param": { "M": 16, "efConstruction": 50, "numThreads": 56 }, - "file": "wiki_all_88M/hnswlib/M16.ef50", - "search_params": [ - { "ef": 10, "numThreads": 56 }, - { "ef": 20, "numThreads": 56 }, - { "ef": 40, "numThreads": 56 }, - { "ef": 60, "numThreads": 56 }, - { "ef": 80, "numThreads": 56 }, - { "ef": 120, "numThreads": 56 }, - { "ef": 200, "numThreads": 56 }, - { "ef": 400, "numThreads": 56 }, - { "ef": 600, "numThreads": 56 }, - { "ef": 800, "numThreads": 56 } - ] - }, - { - "name": "faiss_ivf_pq.M32-nlist16K", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "M": 32, - "nlist": 16384, - "ratio": 2 - }, - "file": "wiki_all_88M/faiss_ivf_pq/M32-nlist16K_ratio2", - "search_params": [ - { "nprobe": 10 }, - { "nprobe": 20 }, - { "nprobe": 30 }, - { "nprobe": 40 }, - { "nprobe": 50 }, - { "nprobe": 100 }, - { "nprobe": 200 }, - { "nprobe": 500 } - ] - }, - { - "name": "faiss_ivf_pq.M64-nlist16K", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "M": 64, - "nlist": 16384, - "ratio": 2 - }, - "file": "wiki_all_88M/faiss_ivf_pq/M64-nlist16K_ratio2", - "search_params": [ - { "nprobe": 10 }, - { "nprobe": 20 }, - { "nprobe": 30 }, - { "nprobe": 40 }, - { "nprobe": 50 }, - { "nprobe": 100 }, - { "nprobe": 200 }, - { "nprobe": 500 } - ] - }, - { - "name": "raft_ivf_pq.d128-nlist16K", - "algo": "raft_ivf_pq", - "build_param": { - "pq_dim": 128, - "pq_bits": 8, - "nlist": 16384, - "niter": 10, - "ratio": 10 - }, - "file": "wiki_all_88M/raft_ivf_pq/d128-nlist16K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 } - ] - }, - { - "name": "raft_ivf_pq.d64-nlist16K", - "algo": "raft_ivf_pq", - "build_param": { - "pq_dim": 64, - "pq_bits": 8, - "nlist": 16384, - "niter": 10, - "ratio": 10 - }, - "file": "wiki_all_88M/raft_ivf_pq/d64-nlist16K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 } - ] - }, - { - "name": "raft_ivf_pq.d32-nlist16K", - "algo": "raft_ivf_pq", - "build_param": { - "pq_dim": 32, - "pq_bits": 8, - "nlist": 16384, - "niter": 10, - "ratio": 10 - }, - "file": "wiki_all_88M/raft_ivf_pq/d32-nlist16K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 } - ] - }, - { - "name": "raft_ivf_pq.d32X-nlist16K", - "algo": "raft_ivf_pq", - "build_param": { - "pq_dim": 32, - "pq_bits": 8, - "nlist": 16384, - "niter": 10, - "ratio": 10 - }, - "file": "wiki_all_88M/raft_ivf_pq/d32-nlist16K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 } - - ] - }, - { - "name": "raft_cagra.dim32.multi_cta", - "algo": "raft_cagra", - "build_param": { "graph_degree": 32, "intermediate_graph_degree": 48 }, - "file": "wiki_all_88M/raft_cagra/dim32.ibin", - "search_params": [ - { "itopk": 32, "search_width": 1, "max_iterations": 0, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 32, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 36, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 40, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 44, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 48, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 2, "max_iterations": 16, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 2, "max_iterations": 24, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 2, "max_iterations": 26, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 2, "max_iterations": 32, "algo": "multi_cta" }, - { "itopk": 64, "search_width": 4, "max_iterations": 16, "algo": "multi_cta" }, - { "itopk": 64, "search_width": 1, "max_iterations": 64, "algo": "multi_cta" }, - { "itopk": 96, "search_width": 2, "max_iterations": 48, "algo": "multi_cta" }, - { "itopk": 128, "search_width": 8, "max_iterations": 16, "algo": "multi_cta" }, - { "itopk": 128, "search_width": 2, "max_iterations": 64, "algo": "multi_cta" }, - { "itopk": 192, "search_width": 8, "max_iterations": 24, "algo": "multi_cta" }, - { "itopk": 192, "search_width": 2, "max_iterations": 96, "algo": "multi_cta" }, - { "itopk": 256, "search_width": 8, "max_iterations": 32, "algo": "multi_cta" }, - { "itopk": 384, "search_width": 8, "max_iterations": 48, "algo": "multi_cta" }, - { "itopk": 512, "search_width": 8, "max_iterations": 64, "algo": "multi_cta" } - ] - } - - ] -} - diff --git a/python/raft-ann-bench/src/raft_ann_bench/split_groundtruth/__main__.py b/python/raft-ann-bench/src/raft_ann_bench/split_groundtruth/__main__.py deleted file mode 100644 index c65360ebb0..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/split_groundtruth/__main__.py +++ /dev/null @@ -1,57 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import os -import subprocess -import sys - - -def split_groundtruth(groundtruth_filepath): - ann_bench_scripts_path = os.path.join( - os.path.dirname(os.path.realpath(__file__)), "split_groundtruth.pl" - ) - pwd = os.getcwd() - path_to_groundtruth = os.path.normpath(groundtruth_filepath).split(os.sep) - if len(path_to_groundtruth) > 1: - os.chdir(os.path.join(*path_to_groundtruth[:-1])) - groundtruth_filename = path_to_groundtruth[-1] - subprocess.run( - [ann_bench_scripts_path, groundtruth_filename, "groundtruth"], - check=True, - ) - os.chdir(pwd) - - -def main(): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - "--groundtruth", - help="Path to billion-scale dataset groundtruth file", - required=True, - ) - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - args = parser.parse_args() - - split_groundtruth(args.groundtruth) - - -if __name__ == "__main__": - main() diff --git a/python/raft-ann-bench/src/raft_ann_bench/split_groundtruth/split_groundtruth.pl b/python/raft-ann-bench/src/raft_ann_bench/split_groundtruth/split_groundtruth.pl deleted file mode 100755 index b0a59f806c..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/split_groundtruth/split_groundtruth.pl +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/perl - -# ============================================================================= -# Copyright (c) 2020-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. - -use warnings; -use strict; -use autodie qw(open close); - - -@ARGV == 2 - or die "usage: $0 input output_prefix\n"; - -open my $fh, '<:raw', $ARGV[0]; - -my $raw; -read($fh, $raw, 8); -my ($nrows, $dim) = unpack('LL', $raw); - -my $expected_size = 8 + $nrows * $dim * (4 + 4); -my $size = (stat($fh))[7]; -$size == $expected_size - or die("error: expected size is $expected_size, but actual size is $size\n"); - - -open my $fh_out1, '>:raw', "$ARGV[1].neighbors.ibin"; -open my $fh_out2, '>:raw', "$ARGV[1].distances.fbin"; - -print {$fh_out1} $raw; -print {$fh_out2} $raw; - -read($fh, $raw, $nrows * $dim * 4); -print {$fh_out1} $raw; -read($fh, $raw, $nrows * $dim * 4); -print {$fh_out2} $raw; diff --git a/python/raft-dask/CMakeLists.txt b/python/raft-dask/CMakeLists.txt index 197ddae05f..9ebbaa5298 100644 --- a/python/raft-dask/CMakeLists.txt +++ b/python/raft-dask/CMakeLists.txt @@ -45,7 +45,6 @@ if(NOT raft_FOUND) # raft-dask doesn't actually use raft libraries, it just needs the headers, so we can turn off all # library compilation and we don't need to install anything here. set(BUILD_TESTS OFF) - set(BUILD_ANN_BENCH OFF) set(BUILD_PRIMS_BENCH OFF) set(RAFT_COMPILE_LIBRARIES OFF) set(RAFT_COMPILE_DIST_LIBRARY OFF) From 1881b4f62bca970917219622f05955132c684441 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Mon, 18 Nov 2024 09:46:55 -0500 Subject: [PATCH 53/79] 2412 remove libraft vss instantiations (#2498) We are keeping random ball cover headers in RAFT for 24.12, and random ball cover depends on distances and brute-force. Because of this, we're going to leave all of the VSS headers in RAFT for the time being, and will remove them all in a future PR once RBC is formally migrated to cuVS. The tests, benchmarks, and instantiations for all of these APIs will be removed, though, so while the actual headers can still be used, they are no longer being tested and could fail without warning. I've also included a note to users in the README about this, stating to use at their own risk. Authors: - Corey J. Nolet (https://github.com/cjnolet) - Bradley Dice (https://github.com/bdice) Approvers: - Ben Frederickson (https://github.com/benfred) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/raft/pull/2498 --- .github/workflows/build.yaml | 1 - .github/workflows/pr.yaml | 9 - README.md | 26 +- build.sh | 15 +- .../recipes/libraft/build_libraft_template.sh | 5 - conda/recipes/libraft/meta.yaml | 54 - cpp/CMakeLists.txt | 274 ---- cpp/bench/prims/CMakeLists.txt | 88 +- cpp/bench/prims/cluster/kmeans.cu | 125 -- cpp/bench/prims/cluster/kmeans_balanced.cu | 100 -- cpp/bench/prims/distance/distance_common.cuh | 94 -- cpp/bench/prims/distance/distance_cosine.cu | 23 - cpp/bench/prims/distance/distance_exp_l2.cu | 24 - cpp/bench/prims/distance/distance_l1.cu | 23 - cpp/bench/prims/distance/distance_unexp_l2.cu | 24 - cpp/bench/prims/distance/fused_l2_nn.cu | 163 -- cpp/bench/prims/distance/kernels.cu | 122 -- cpp/bench/prims/distance/masked_nn.cu | 264 ---- .../prims/distance/tune_pairwise/bench.cu | 155 -- .../prims/distance/tune_pairwise/kernel.cu | 89 -- .../prims/distance/tune_pairwise/kernel.cuh | 44 - cpp/bench/prims/neighbors/cagra_bench.cuh | 208 --- cpp/bench/prims/neighbors/knn.cuh | 516 ------- .../knn/brute_force_float_int64_t.cu | 23 - .../knn/brute_force_float_uint32_t.cu | 23 - .../neighbors/knn/cagra_float_uint32_t.cu | 23 - .../knn/ivf_flat_filter_float_int64_t.cu | 24 - .../neighbors/knn/ivf_flat_float_int64_t.cu | 23 - .../neighbors/knn/ivf_flat_int8_t_int64_t.cu | 23 - .../neighbors/knn/ivf_flat_uint8_t_int64_t.cu | 23 - .../knn/ivf_pq_filter_float_int64_t.cu | 25 - .../neighbors/knn/ivf_pq_float_int64_t.cu | 23 - .../neighbors/knn/ivf_pq_int8_t_int64_t.cu | 23 - .../neighbors/knn/ivf_pq_uint8_t_int64_t.cu | 23 - cpp/bench/prims/neighbors/refine.cuh | 111 -- .../prims/neighbors/refine_float_int64_t.cu | 26 - .../prims/neighbors/refine_uint8_t_int64_t.cu | 26 - cpp/cmake/patches/hnswlib.diff | 188 --- cpp/cmake/patches/hnswlib_override.json | 16 - cpp/cmake/thirdparty/get_hnswlib.cmake | 88 -- cpp/include/raft/cluster/specializations.cuh | 24 - .../detail/pairwise_matrix/dispatch-ext.cuh | 198 --- .../detail/pairwise_matrix/dispatch.cuh | 8 +- cpp/include/raft/distance/distance-ext.cuh | 1117 -------------- cpp/include/raft/distance/distance.cuh | 8 +- cpp/include/raft/distance/fused_l2_nn-ext.cuh | 83 - cpp/include/raft/distance/fused_l2_nn.cuh | 8 +- cpp/include/raft/distance/specializations.cuh | 24 - .../distance/specializations/distance.cuh | 24 - .../specializations/fused_l2_nn_min.cuh | 24 - .../raft/matrix/detail/select_k-ext.cuh | 72 - cpp/include/raft/matrix/detail/select_k.cuh | 8 +- .../specializations/detail/select_k.cuh | 24 - cpp/include/raft/neighbors/ball_cover-ext.cuh | 162 -- cpp/include/raft/neighbors/ball_cover.cuh | 6 - .../raft/neighbors/brute_force-ext.cuh | 193 --- cpp/include/raft/neighbors/brute_force.cuh | 12 +- cpp/include/raft/neighbors/ivf_flat-ext.cuh | 257 ---- cpp/include/raft/neighbors/ivf_flat.cuh | 8 +- cpp/include/raft/neighbors/ivf_pq-ext.cuh | 200 --- cpp/include/raft/neighbors/ivf_pq.cuh | 8 +- cpp/include/raft/neighbors/refine-ext.cuh | 85 - cpp/include/raft/neighbors/refine.cuh | 8 +- .../raft/neighbors/specializations.cuh | 24 - .../neighbors/specializations/ball_cover.cuh | 24 - .../neighbors/specializations/brute_force.cuh | 24 - .../detail/ball_cover_lowdim.hpp | 86 -- .../detail/ivf_pq_compute_similarity.cuh | 24 - .../specializations/fused_l2_knn.cuh | 24 - .../neighbors/specializations/ivf_flat.cuh | 24 - .../raft/neighbors/specializations/ivf_pq.cuh | 24 - .../raft/neighbors/specializations/refine.cuh | 24 - .../raft/sparse/neighbors/specializations.cuh | 24 - .../knn/detail/ball_cover/registers-ext.cuh | 197 --- .../knn/detail/ball_cover/registers.cuh | 9 +- .../spatial/knn/detail/fused_l2_knn-ext.cuh | 75 - .../raft/spatial/knn/detail/fused_l2_knn.cuh | 8 +- .../raft/spatial/knn/specializations.cuh | 24 - .../raft/spatial/knn/specializations/knn.cuh | 24 - ...pq_compute_similarity_filters_test-ext.cuh | 181 --- .../neighbors/ivf_pq_search_test-ext.cuh | 89 -- .../raft_internal/neighbors/naive_knn.cuh | 124 -- .../raft_internal/neighbors/refine_helper.cuh | 158 -- .../pairwise_matrix/dispatch_00_generate.py | 199 --- ...patch_canberra_double_double_double_int.cu | 55 - ...dispatch_canberra_float_float_float_int.cu | 50 - ...ch_correlation_double_double_double_int.cu | 55 - ...patch_correlation_float_float_float_int.cu | 55 - ...ispatch_cosine_double_double_double_int.cu | 51 - .../dispatch_cosine_float_float_float_int.cu | 51 - .../dispatch_dice_double_double_double_int.cu | 51 - .../dispatch_dice_float_float_float_int.cu | 51 - ...ing_unexpanded_double_double_double_int.cu | 50 - ...amming_unexpanded_float_float_float_int.cu | 50 - ...inger_expanded_double_double_double_int.cu | 55 - ...ellinger_expanded_float_float_float_int.cu | 50 - ...jensen_shannon_double_double_double_int.cu | 55 - ...ch_jensen_shannon_float_float_float_int.cu | 55 - ..._kl_divergence_double_double_double_int.cu | 50 - ...tch_kl_divergence_float_float_float_int.cu | 50 - .../dispatch_l1_double_double_double_int.cu | 50 - .../dispatch_l1_float_float_float_int.cu | 50 - ...ch_l2_expanded_double_double_double_int.cu | 51 - ...patch_l2_expanded_float_float_float_int.cu | 51 - ..._l2_unexpanded_double_double_double_int.cu | 55 - ...tch_l2_unexpanded_float_float_float_int.cu | 50 - ...dispatch_l_inf_double_double_double_int.cu | 50 - .../dispatch_l_inf_float_float_float_int.cu | 50 - ..._lp_unexpanded_double_double_double_int.cu | 55 - ...tch_lp_unexpanded_float_float_float_int.cu | 50 - .../detail/pairwise_matrix/dispatch_rbf.cu | 64 - ...tch_russel_rao_double_double_double_int.cu | 55 - ...spatch_russel_rao_float_float_float_int.cu | 50 - cpp/src/distance/distance.cu | 982 ------------ cpp/src/distance/fused_distance_nn.cu | 53 - cpp/src/distance/fused_l2_nn.cu | 55 - .../matrix/detail/select_k_double_int64_t.cu | 35 - .../matrix/detail/select_k_double_uint32_t.cu | 37 - cpp/src/matrix/detail/select_k_float_int32.cu | 35 - .../matrix/detail/select_k_float_int64_t.cu | 35 - .../matrix/detail/select_k_float_uint32_t.cu | 35 - .../matrix/detail/select_k_half_int64_t.cu | 35 - .../matrix/detail/select_k_half_uint32_t.cu | 35 - cpp/src/neighbors/ball_cover.cu | 85 - cpp/src/neighbors/brute_force_00_generate.py | 106 -- .../brute_force_fused_l2_knn_float_int64_t.cu | 46 - .../neighbors/brute_force_knn_index_float.cu | 79 - .../brute_force_knn_int64_t_float_int64_t.cu | 48 - .../brute_force_knn_int64_t_float_uint32_t.cu | 48 - .../brute_force_knn_int_float_int.cu | 48 - ...brute_force_knn_uint32_t_float_uint32_t.cu | 48 - .../cagra/q_search_multi_cta_00_generate.py | 84 - ...float_uint32_dim1024_t32_8pq_2subd_half.cu | 37 - ...float_uint32_dim1024_t32_8pq_4subd_half.cu | 37 - ...a_float_uint32_dim128_t8_8pq_2subd_half.cu | 37 - ...a_float_uint32_dim128_t8_8pq_4subd_half.cu | 37 - ..._float_uint32_dim256_t16_8pq_2subd_half.cu | 37 - ..._float_uint32_dim256_t16_8pq_4subd_half.cu | 37 - ..._float_uint32_dim512_t32_8pq_2subd_half.cu | 37 - ..._float_uint32_dim512_t32_8pq_4subd_half.cu | 37 - ...float_uint64_dim1024_t32_8pq_2subd_half.cu | 37 - ...float_uint64_dim1024_t32_8pq_4subd_half.cu | 37 - ...a_float_uint64_dim128_t8_8pq_2subd_half.cu | 37 - ...a_float_uint64_dim128_t8_8pq_4subd_half.cu | 37 - ..._float_uint64_dim256_t16_8pq_2subd_half.cu | 37 - ..._float_uint64_dim256_t16_8pq_4subd_half.cu | 37 - ..._float_uint64_dim512_t32_8pq_2subd_half.cu | 37 - ..._float_uint64_dim512_t32_8pq_4subd_half.cu | 37 - ..._half_uint32_dim1024_t32_8pq_2subd_half.cu | 37 - ..._half_uint32_dim1024_t32_8pq_4subd_half.cu | 37 - ...ta_half_uint32_dim128_t8_8pq_2subd_half.cu | 37 - ...ta_half_uint32_dim128_t8_8pq_4subd_half.cu | 37 - ...a_half_uint32_dim256_t16_8pq_2subd_half.cu | 37 - ...a_half_uint32_dim256_t16_8pq_4subd_half.cu | 37 - ...a_half_uint32_dim512_t32_8pq_2subd_half.cu | 37 - ...a_half_uint32_dim512_t32_8pq_4subd_half.cu | 37 - ..._half_uint64_dim1024_t32_8pq_2subd_half.cu | 37 - ..._half_uint64_dim1024_t32_8pq_4subd_half.cu | 37 - ...ta_half_uint64_dim128_t8_8pq_2subd_half.cu | 37 - ...ta_half_uint64_dim128_t8_8pq_4subd_half.cu | 37 - ...a_half_uint64_dim256_t16_8pq_2subd_half.cu | 37 - ...a_half_uint64_dim256_t16_8pq_4subd_half.cu | 37 - ...a_half_uint64_dim512_t32_8pq_2subd_half.cu | 37 - ...a_half_uint64_dim512_t32_8pq_4subd_half.cu | 37 - ..._int8_uint32_dim1024_t32_8pq_2subd_half.cu | 37 - ..._int8_uint32_dim1024_t32_8pq_4subd_half.cu | 37 - ...ta_int8_uint32_dim128_t8_8pq_2subd_half.cu | 37 - ...ta_int8_uint32_dim128_t8_8pq_4subd_half.cu | 37 - ...a_int8_uint32_dim256_t16_8pq_2subd_half.cu | 37 - ...a_int8_uint32_dim256_t16_8pq_4subd_half.cu | 37 - ...a_int8_uint32_dim512_t32_8pq_2subd_half.cu | 37 - ...a_int8_uint32_dim512_t32_8pq_4subd_half.cu | 37 - ...uint8_uint32_dim1024_t32_8pq_2subd_half.cu | 37 - ...uint8_uint32_dim1024_t32_8pq_4subd_half.cu | 37 - ...a_uint8_uint32_dim128_t8_8pq_2subd_half.cu | 37 - ...a_uint8_uint32_dim128_t8_8pq_4subd_half.cu | 37 - ..._uint8_uint32_dim256_t16_8pq_2subd_half.cu | 37 - ..._uint8_uint32_dim256_t16_8pq_4subd_half.cu | 37 - ..._uint8_uint32_dim512_t32_8pq_2subd_half.cu | 37 - ..._uint8_uint32_dim512_t32_8pq_4subd_half.cu | 37 - .../cagra/q_search_single_cta_00_generate.py | 89 -- ...float_uint32_dim1024_t32_8pq_2subd_half.cu | 37 - ...float_uint32_dim1024_t32_8pq_4subd_half.cu | 37 - ...a_float_uint32_dim128_t8_8pq_2subd_half.cu | 37 - ...a_float_uint32_dim128_t8_8pq_4subd_half.cu | 37 - ..._float_uint32_dim256_t16_8pq_2subd_half.cu | 37 - ..._float_uint32_dim256_t16_8pq_4subd_half.cu | 37 - ..._float_uint32_dim512_t32_8pq_2subd_half.cu | 37 - ..._float_uint32_dim512_t32_8pq_4subd_half.cu | 37 - ...float_uint64_dim1024_t32_8pq_2subd_half.cu | 37 - ...float_uint64_dim1024_t32_8pq_4subd_half.cu | 37 - ...a_float_uint64_dim128_t8_8pq_2subd_half.cu | 37 - ...a_float_uint64_dim128_t8_8pq_4subd_half.cu | 37 - ..._float_uint64_dim256_t16_8pq_2subd_half.cu | 37 - ..._float_uint64_dim256_t16_8pq_4subd_half.cu | 37 - ..._float_uint64_dim512_t32_8pq_2subd_half.cu | 37 - ..._float_uint64_dim512_t32_8pq_4subd_half.cu | 37 - ..._half_uint32_dim1024_t32_8pq_2subd_half.cu | 37 - ..._half_uint32_dim1024_t32_8pq_4subd_half.cu | 37 - ...ta_half_uint32_dim128_t8_8pq_2subd_half.cu | 37 - ...ta_half_uint32_dim128_t8_8pq_4subd_half.cu | 37 - ...a_half_uint32_dim256_t16_8pq_2subd_half.cu | 37 - ...a_half_uint32_dim256_t16_8pq_4subd_half.cu | 37 - ...a_half_uint32_dim512_t32_8pq_2subd_half.cu | 37 - ...a_half_uint32_dim512_t32_8pq_4subd_half.cu | 37 - ..._half_uint64_dim1024_t32_8pq_2subd_half.cu | 37 - ..._half_uint64_dim1024_t32_8pq_4subd_half.cu | 37 - ...ta_half_uint64_dim128_t8_8pq_2subd_half.cu | 37 - ...ta_half_uint64_dim128_t8_8pq_4subd_half.cu | 37 - ...a_half_uint64_dim256_t16_8pq_2subd_half.cu | 37 - ...a_half_uint64_dim256_t16_8pq_4subd_half.cu | 37 - ...a_half_uint64_dim512_t32_8pq_2subd_half.cu | 37 - ...a_half_uint64_dim512_t32_8pq_4subd_half.cu | 37 - ..._int8_uint32_dim1024_t32_8pq_2subd_half.cu | 37 - ..._int8_uint32_dim1024_t32_8pq_4subd_half.cu | 37 - ...ta_int8_uint32_dim128_t8_8pq_2subd_half.cu | 37 - ...ta_int8_uint32_dim128_t8_8pq_4subd_half.cu | 37 - ...a_int8_uint32_dim256_t16_8pq_2subd_half.cu | 37 - ...a_int8_uint32_dim256_t16_8pq_4subd_half.cu | 37 - ...a_int8_uint32_dim512_t32_8pq_2subd_half.cu | 37 - ...a_int8_uint32_dim512_t32_8pq_4subd_half.cu | 37 - ...uint8_uint32_dim1024_t32_8pq_2subd_half.cu | 37 - ...uint8_uint32_dim1024_t32_8pq_4subd_half.cu | 37 - ...a_uint8_uint32_dim128_t8_8pq_2subd_half.cu | 37 - ...a_uint8_uint32_dim128_t8_8pq_4subd_half.cu | 37 - ..._uint8_uint32_dim256_t16_8pq_2subd_half.cu | 37 - ..._uint8_uint32_dim256_t16_8pq_4subd_half.cu | 37 - ..._uint8_uint32_dim512_t32_8pq_2subd_half.cu | 37 - ..._uint8_uint32_dim512_t32_8pq_4subd_half.cu | 37 - .../detail/cagra/search_multi_cta.cuh | 52 - .../cagra/search_multi_cta_00_generate.py | 78 - ...arch_multi_cta_float_uint32_dim1024_t32.cu | 37 - ...search_multi_cta_float_uint32_dim128_t8.cu | 37 - ...earch_multi_cta_float_uint32_dim256_t16.cu | 37 - ...earch_multi_cta_float_uint32_dim512_t32.cu | 37 - ...arch_multi_cta_float_uint64_dim1024_t32.cu | 37 - ...search_multi_cta_float_uint64_dim128_t8.cu | 37 - ...earch_multi_cta_float_uint64_dim256_t16.cu | 37 - ...earch_multi_cta_float_uint64_dim512_t32.cu | 37 - ...earch_multi_cta_half_uint32_dim1024_t32.cu | 37 - .../search_multi_cta_half_uint32_dim128_t8.cu | 37 - ...search_multi_cta_half_uint32_dim256_t16.cu | 37 - ...search_multi_cta_half_uint32_dim512_t32.cu | 37 - ...earch_multi_cta_half_uint64_dim1024_t32.cu | 37 - .../search_multi_cta_half_uint64_dim128_t8.cu | 37 - ...search_multi_cta_half_uint64_dim256_t16.cu | 37 - ...search_multi_cta_half_uint64_dim512_t32.cu | 37 - ...earch_multi_cta_int8_uint32_dim1024_t32.cu | 37 - .../search_multi_cta_int8_uint32_dim128_t8.cu | 37 - ...search_multi_cta_int8_uint32_dim256_t16.cu | 37 - ...search_multi_cta_int8_uint32_dim512_t32.cu | 37 - ...arch_multi_cta_uint8_uint32_dim1024_t32.cu | 37 - ...search_multi_cta_uint8_uint32_dim128_t8.cu | 37 - ...earch_multi_cta_uint8_uint32_dim256_t16.cu | 37 - ...earch_multi_cta_uint8_uint32_dim512_t32.cu | 37 - .../detail/cagra/search_single_cta.cuh | 53 - .../cagra/search_single_cta_00_generate.py | 82 - ...rch_single_cta_float_uint32_dim1024_t32.cu | 37 - ...earch_single_cta_float_uint32_dim128_t8.cu | 37 - ...arch_single_cta_float_uint32_dim256_t16.cu | 37 - ...arch_single_cta_float_uint32_dim512_t32.cu | 37 - ...rch_single_cta_float_uint64_dim1024_t32.cu | 37 - ...earch_single_cta_float_uint64_dim128_t8.cu | 37 - ...arch_single_cta_float_uint64_dim256_t16.cu | 37 - ...arch_single_cta_float_uint64_dim512_t32.cu | 37 - ...arch_single_cta_half_uint32_dim1024_t32.cu | 37 - ...search_single_cta_half_uint32_dim128_t8.cu | 37 - ...earch_single_cta_half_uint32_dim256_t16.cu | 37 - ...earch_single_cta_half_uint32_dim512_t32.cu | 37 - ...arch_single_cta_half_uint64_dim1024_t32.cu | 37 - ...search_single_cta_half_uint64_dim128_t8.cu | 37 - ...earch_single_cta_half_uint64_dim256_t16.cu | 37 - ...earch_single_cta_half_uint64_dim512_t32.cu | 37 - ...arch_single_cta_int8_uint32_dim1024_t32.cu | 37 - ...search_single_cta_int8_uint32_dim128_t8.cu | 37 - ...earch_single_cta_int8_uint32_dim256_t16.cu | 37 - ...earch_single_cta_int8_uint32_dim512_t32.cu | 37 - ...rch_single_cta_uint8_uint32_dim1024_t32.cu | 37 - ...earch_single_cta_uint8_uint32_dim128_t8.cu | 37 - ...arch_single_cta_uint8_uint32_dim256_t16.cu | 37 - ...arch_single_cta_uint8_uint32_dim512_t32.cu | 37 - ...at_interleaved_scan_float_float_int64_t.cu | 44 - ...flat_interleaved_scan_half_half_int64_t.cu | 46 - ...interleaved_scan_int8_t_int32_t_int64_t.cu | 44 - ...terleaved_scan_uint8_t_uint32_t_int64_t.cu | 44 - cpp/src/neighbors/detail/ivf_flat_search.cu | 42 - .../ivf_pq_compute_similarity_00_generate.py | 85 - .../ivf_pq_compute_similarity_float_float.cu | 28 - ...compute_similarity_float_float_bitset32.cu | 28 - ...compute_similarity_float_float_bitset64.cu | 28 - ...q_compute_similarity_float_float_filt32.cu | 28 - ...f_pq_compute_similarity_float_fp8_false.cu | 28 - ...ute_similarity_float_fp8_false_bitset32.cu | 28 - ...ute_similarity_float_fp8_false_bitset64.cu | 28 - ...mpute_similarity_float_fp8_false_filt32.cu | 28 - ...vf_pq_compute_similarity_float_fp8_true.cu | 28 - ...pute_similarity_float_fp8_true_bitset32.cu | 28 - ...pute_similarity_float_fp8_true_bitset64.cu | 28 - ...ompute_similarity_float_fp8_true_filt32.cu | 28 - .../ivf_pq_compute_similarity_float_half.cu | 28 - ..._compute_similarity_float_half_bitset32.cu | 28 - ..._compute_similarity_float_half_bitset64.cu | 28 - ...pq_compute_similarity_float_half_filt32.cu | 28 - ...vf_pq_compute_similarity_half_fp8_false.cu | 28 - ...pute_similarity_half_fp8_false_bitset32.cu | 28 - ...pute_similarity_half_fp8_false_bitset64.cu | 28 - ...ompute_similarity_half_fp8_false_filt32.cu | 28 - ...ivf_pq_compute_similarity_half_fp8_true.cu | 28 - ...mpute_similarity_half_fp8_true_bitset32.cu | 28 - ...mpute_similarity_half_fp8_true_bitset64.cu | 28 - ...compute_similarity_half_fp8_true_filt32.cu | 28 - .../ivf_pq_compute_similarity_half_half.cu | 28 - ...q_compute_similarity_half_half_bitset32.cu | 28 - ...q_compute_similarity_half_half_bitset64.cu | 28 - ..._pq_compute_similarity_half_half_filt32.cu | 28 - .../ivf_pq_search_filtering_float_int64_t.cu | 43 - .../detail/refine_host_float_float.cpp | 30 - .../detail/refine_host_half_float.cpp | 31 - .../detail/refine_host_int8_t_float.cpp | 29 - .../detail/refine_host_uint8_t_float.cpp | 30 - cpp/src/neighbors/ivf_flat_00_generate.py | 175 --- .../neighbors/ivf_flat_build_float_int64_t.cu | 62 - .../ivf_flat_build_int8_t_int64_t.cu | 62 - .../ivf_flat_build_uint8_t_int64_t.cu | 62 - .../ivf_flat_extend_float_int64_t.cu | 71 - .../ivf_flat_extend_int8_t_int64_t.cu | 71 - .../ivf_flat_extend_uint8_t_int64_t.cu | 71 - .../ivf_flat_search_float_int64_t.cu | 51 - .../ivf_flat_search_int8_t_int64_t.cu | 51 - .../ivf_flat_search_uint8_t_int64_t.cu | 51 - .../neighbors/ivfpq_build_float_int64_t.cu | 36 - cpp/src/neighbors/ivfpq_build_half_int64_t.cu | 38 - .../neighbors/ivfpq_build_int8_t_int64_t.cu | 36 - .../neighbors/ivfpq_build_uint8_t_int64_t.cu | 36 - .../neighbors/ivfpq_extend_float_int64_t.cu | 50 - .../neighbors/ivfpq_extend_half_int64_t.cu | 52 - .../neighbors/ivfpq_extend_int8_t_int64_t.cu | 50 - .../neighbors/ivfpq_extend_uint8_t_int64_t.cu | 50 - .../neighbors/ivfpq_search_float_int64_t.cu | 43 - .../neighbors/ivfpq_search_half_int64_t.cu | 45 - .../neighbors/ivfpq_search_int8_t_int64_t.cu | 43 - .../neighbors/ivfpq_search_uint8_t_int64_t.cu | 43 - cpp/src/neighbors/refine_00_generate.py | 79 - cpp/src/neighbors/refine_float_float.cu | 54 - cpp/src/neighbors/refine_half_float.cu | 50 - cpp/src/neighbors/refine_int8_t_float.cu | 50 - cpp/src/neighbors/refine_uint8_t_float.cu | 50 - cpp/src/raft_runtime/cluster/cluster_cost.cuh | 87 -- .../cluster/cluster_cost_double.cu | 34 - .../cluster/cluster_cost_float.cu | 34 - .../raft_runtime/cluster/kmeans_fit_double.cu | 33 - .../raft_runtime/cluster/kmeans_fit_float.cu | 33 - .../cluster/kmeans_init_plus_plus_double.cu | 31 - .../cluster/kmeans_init_plus_plus_float.cu | 31 - .../raft_runtime/cluster/update_centroids.cuh | 72 - .../cluster/update_centroids_double.cu | 47 - .../cluster/update_centroids_float.cu | 47 - .../distance/fused_distance_min_arg.cu | 56 - .../distance/fused_distance_min_arg.hpp | 144 -- .../raft_runtime/distance/fused_l2_min_arg.cu | 59 - .../distance/pairwise_distance.cu | 52 - .../matrix/select_k_float_int64_t.cu | 36 - .../brute_force_knn_int64_t_float.cu | 47 - cpp/src/raft_runtime/neighbors/cagra_build.cu | 85 - .../raft_runtime/neighbors/cagra_search.cu | 43 - .../raft_runtime/neighbors/cagra_serialize.cu | 86 -- .../neighbors/eps_neighborhood.cu | 101 -- cpp/src/raft_runtime/neighbors/hnsw.cpp | 80 - .../raft_runtime/neighbors/ivf_flat_build.cu | 63 - .../raft_runtime/neighbors/ivf_flat_search.cu | 41 - .../neighbors/ivf_flat_serialize.cu | 66 - cpp/src/raft_runtime/neighbors/ivfpq_build.cu | 60 - .../neighbors/ivfpq_deserialize.cu | 31 - .../neighbors/ivfpq_search_float_int64_t.cu | 38 - .../neighbors/ivfpq_search_int8_t_int64_t.cu | 38 - .../neighbors/ivfpq_search_uint8_t_int64_t.cu | 38 - .../raft_runtime/neighbors/ivfpq_serialize.cu | 31 - .../neighbors/refine_d_int64_t_float.cu | 33 - .../neighbors/refine_d_int64_t_int8_t.cu | 33 - .../neighbors/refine_d_int64_t_uint8_t.cu | 33 - .../neighbors/refine_h_int64_t_float.cu | 34 - .../neighbors/refine_h_int64_t_int8_t.cu | 33 - .../neighbors/refine_h_int64_t_uint8_t.cu | 33 - .../knn/detail/ball_cover/registers.cu | 94 -- .../ball_cover/registers_00_generate.py | 164 -- .../registers_eps_pass_euclidean.cu | 60 - .../ball_cover/registers_pass_one_2d_dist.cu | 49 - .../registers_pass_one_2d_euclidean.cu | 49 - .../registers_pass_one_2d_haversine.cu | 49 - .../ball_cover/registers_pass_one_3d_dist.cu | 49 - .../registers_pass_one_3d_euclidean.cu | 49 - .../registers_pass_one_3d_haversine.cu | 49 - .../ball_cover/registers_pass_two_2d_dist.cu | 49 - .../registers_pass_two_2d_euclidean.cu | 49 - .../registers_pass_two_2d_haversine.cu | 49 - .../ball_cover/registers_pass_two_3d_dist.cu | 49 - .../registers_pass_two_3d_euclidean.cu | 49 - .../registers_pass_two_3d_haversine.cu | 49 - .../knn/detail/fused_l2_knn_int32_t_float.cu | 43 - .../knn/detail/fused_l2_knn_int64_t_float.cu | 43 - .../knn/detail/fused_l2_knn_uint32_t_float.cu | 44 - cpp/template/CMakeLists.txt | 44 - cpp/template/README.md | 18 - cpp/template/build.sh | 41 - .../cmake/thirdparty/fetch_rapids.cmake | 21 - cpp/template/src/cagra_example.cu | 91 -- cpp/template/src/common.cuh | 97 -- cpp/template/src/ivf_flat_example.cu | 161 -- cpp/template/src/ivf_pq_example.cu | 116 -- cpp/test/CMakeLists.txt | 198 +-- cpp/test/cluster/cluster_solvers.cu | 104 -- .../cluster/cluster_solvers_deprecated.cu | 59 - cpp/test/cluster/kmeans.cu | 363 ----- cpp/test/cluster/kmeans_balanced.cu | 240 --- cpp/test/cluster/kmeans_find_k.cu | 142 -- cpp/test/cluster/linkage.cu | 674 -------- cpp/test/cluster/spectral.cu | 109 -- cpp/test/distance/dist_adj.cu | 196 --- cpp/test/distance/dist_adj.cuh | 72 - .../distance/dist_adj_distance_instance.cu | 65 - cpp/test/distance/dist_adj_threshold.cuh | 36 - cpp/test/distance/dist_canberra.cu | 70 - cpp/test/distance/dist_correlation.cu | 94 -- cpp/test/distance/dist_cos.cu | 112 -- cpp/test/distance/dist_dice.cu | 112 -- cpp/test/distance/dist_hamming.cu | 71 - cpp/test/distance/dist_hellinger.cu | 71 - cpp/test/distance/dist_inner_product.cu | 74 - cpp/test/distance/dist_jensen_shannon.cu | 71 - cpp/test/distance/dist_kl_divergence.cu | 71 - cpp/test/distance/dist_l1.cu | 70 - cpp/test/distance/dist_l2_exp.cu | 115 -- cpp/test/distance/dist_l2_sqrt_exp.cu | 74 - cpp/test/distance/dist_l2_unexp.cu | 71 - cpp/test/distance/dist_l_inf.cu | 70 - cpp/test/distance/dist_lp_unexp.cu | 71 - cpp/test/distance/dist_russell_rao.cu | 71 - cpp/test/distance/distance_base.cuh | 708 --------- cpp/test/distance/fused_cosine_nn.cu | 420 ----- cpp/test/distance/fused_l2_nn.cu | 437 ------ cpp/test/distance/gram.cu | 174 --- cpp/test/distance/gram_base.cuh | 90 -- cpp/test/distance/masked_nn.cu | 438 ------ .../distance/masked_nn_compress_to_bits.cu | 220 --- cpp/test/neighbors/ann_brute_force.cuh | 253 --- .../neighbors/ann_brute_force/test_float.cu | 28 - cpp/test/neighbors/ann_cagra.cuh | 949 ------------ .../ann_cagra/search_kernel_uint64_t.cuh | 155 -- .../neighbors/ann_cagra/test_float_int64_t.cu | 29 - .../ann_cagra/test_float_uint32_t.cu | 40 - .../neighbors/ann_cagra/test_half_int64_t.cu | 29 - .../neighbors/ann_cagra/test_half_uint32_t.cu | 40 - .../ann_cagra/test_int8_t_uint32_t.cu | 38 - .../ann_cagra/test_uint8_t_uint32_t.cu | 40 - cpp/test/neighbors/ann_cagra_vpq.cuh | 336 ---- .../ann_cagra_vpq/test_float_int64_t.cu | 29 - .../ann_cagra_vpq/test_float_uint32_t.cu | 28 - cpp/test/neighbors/ann_ivf_flat.cuh | 675 -------- .../ann_ivf_flat/test_filter_float_int64_t.cu | 29 - .../ann_ivf_flat/test_float_int64_t.cu | 32 - .../ann_ivf_flat/test_int8_t_int64_t.cu | 28 - .../ann_ivf_flat/test_uint8_t_int64_t.cu | 28 - cpp/test/neighbors/ann_ivf_pq.cuh | 1095 ------------- .../ann_ivf_pq/ivf_pq_build_float_uint32_t.cu | 37 - .../ann_ivf_pq/ivf_pq_build_test-ext.cuh | 38 - .../ivf_pq_search_float_uint32_t.cu | 67 - .../ann_ivf_pq/test_filter_float_int64_t.cu | 28 - .../ann_ivf_pq/test_filter_int8_t_int64_t.cu | 29 - .../ann_ivf_pq/test_float_int64_t.cu | 27 - .../ann_ivf_pq/test_float_uint32_t.cu | 34 - .../ann_ivf_pq/test_int8_t_int64_t.cu | 28 - .../ann_ivf_pq/test_uint8_t_int64_t.cu | 27 - cpp/test/neighbors/ann_nn_descent.cuh | 332 ---- .../test_batch_float_uint32_t.cu | 30 - .../ann_nn_descent/test_float_uint32_t.cu | 28 - .../ann_nn_descent/test_int8_t_uint32_t.cu | 28 - .../ann_nn_descent/test_uint8_t_uint32_t.cu | 28 - cpp/test/neighbors/ann_utils.cuh | 335 ---- cpp/test/neighbors/fused_l2_knn.cu | 173 --- cpp/test/neighbors/knn.cu | 197 --- cpp/test/neighbors/refine.cu | 129 -- cpp/test/neighbors/tiled_knn.cu | 352 ----- cpp/test/sparse/gram.cu | 332 ---- cpp/test/sparse/neighbors/brute_force.cu | 179 --- .../sparse/neighbors/cross_component_nn.cu | 1036 ------------- cpp/test/sparse/neighbors/knn_graph.cu | 129 -- cpp/test/stats/neighborhood_recall.cu | 177 --- cpp/test/stats/silhouette_score.cu | 230 --- cpp/test/stats/trustworthiness.cu | 354 ----- docs/source/build.md | 35 +- docs/source/cpp_api.rst | 3 - docs/source/cpp_api/cluster.rst | 18 - docs/source/cpp_api/cluster_kmeans.rst | 13 - .../cpp_api/cluster_kmeans_balanced.rst | 13 - docs/source/cpp_api/cluster_slhc.rst | 13 - docs/source/cpp_api/cluster_spectral.rst | 13 - docs/source/cpp_api/distance.rst | 28 - docs/source/cpp_api/distance_1nn.rst | 24 - docs/source/cpp_api/distance_pairwise.rst | 17 - docs/source/cpp_api/matrix.rst | 1 - docs/source/cpp_api/matrix_selection.rst | 69 - docs/source/cpp_api/neighbors.rst | 19 - docs/source/cpp_api/neighbors_ball_cover.rst | 17 - docs/source/cpp_api/neighbors_brute_force.rst | 18 - docs/source/cpp_api/neighbors_cagra.rst | 31 - .../neighbors_epsilon_neighborhood.rst | 15 - docs/source/cpp_api/neighbors_hnsw.rst | 29 - docs/source/cpp_api/neighbors_ivf_flat.rst | 37 - docs/source/cpp_api/neighbors_ivf_pq.rst | 48 - docs/source/cpp_api/sparse.rst | 2 - docs/source/cpp_api/sparse_distance.rst | 7 - docs/source/cpp_api/sparse_neighbors.rst | 7 - docs/source/cpp_api/stats.rst | 1 - docs/source/cpp_api/stats_neighborhood.rst | 30 - docs/source/index.rst | 7 - docs/source/pylibraft_api.rst | 4 - docs/source/pylibraft_api/cluster.rst | 20 - docs/source/pylibraft_api/distance.rst | 15 - docs/source/pylibraft_api/matrix.rst | 11 - docs/source/pylibraft_api/neighbors.rst | 99 -- docs/source/quick_start.md | 16 +- docs/source/using_libraft.md | 64 - docs/source/vector_search_tutorial.md | 409 ----- .../VectorSearch_QuestionRetrieval.ipynb | 632 -------- notebooks/ivf_flat_example.ipynb | 674 -------- notebooks/tutorial_ivf_pq.ipynb | 1365 ----------------- notebooks/utils.py | 102 -- python/pylibraft/CMakeLists.txt | 18 +- .../pylibraft/cluster/CMakeLists.txt | 24 - .../pylibraft/pylibraft/cluster/__init__.pxd | 14 - .../pylibraft/pylibraft/cluster/__init__.py | 30 - .../pylibraft/cluster/cpp/__init__.pxd | 0 .../pylibraft/cluster/cpp/__init__.py | 0 .../pylibraft/cluster/cpp/kmeans.pxd | 106 -- .../pylibraft/cluster/cpp/kmeans_types.pxd | 44 - python/pylibraft/pylibraft/cluster/kmeans.pyx | 589 ------- .../pylibraft/distance/CMakeLists.txt | 24 - .../pylibraft/pylibraft/distance/__init__.pxd | 14 - .../pylibraft/pylibraft/distance/__init__.py | 24 - .../pylibraft/distance/distance_type.pxd | 44 - .../pylibraft/distance/fused_distance_nn.pyx | 200 --- .../pylibraft/distance/fused_l2_nn.pyx | 193 --- .../pylibraft/distance/pairwise_distance.pyx | 242 --- .../pylibraft/pylibraft/matrix/CMakeLists.txt | 24 - .../pylibraft/pylibraft/matrix/__init__.pxd | 14 - python/pylibraft/pylibraft/matrix/__init__.py | 18 - .../pylibraft/matrix/cpp/__init__.pxd | 0 .../pylibraft/matrix/cpp/__init__.py | 14 - .../pylibraft/matrix/cpp/select_k.pxd | 39 - .../pylibraft/pylibraft/matrix/select_k.pyx | 133 -- .../pylibraft/neighbors/CMakeLists.txt | 28 - .../pylibraft/neighbors/__init__.pxd | 14 - .../pylibraft/pylibraft/neighbors/__init__.py | 32 - .../pylibraft/neighbors/brute_force.pyx | 269 ---- .../pylibraft/neighbors/cagra/CMakeLists.txt | 24 - .../pylibraft/neighbors/cagra/__init__.pxd | 0 .../pylibraft/neighbors/cagra/__init__.py | 26 - .../pylibraft/neighbors/cagra/cagra.pxd | 39 - .../pylibraft/neighbors/cagra/cagra.pyx | 900 ----------- .../neighbors/cagra/cpp/__init__.pxd | 0 .../pylibraft/neighbors/cagra/cpp/__init__.py | 14 - .../pylibraft/neighbors/cagra/cpp/c_cagra.pxd | 255 --- .../pylibraft/pylibraft/neighbors/common.pxd | 24 - .../pylibraft/pylibraft/neighbors/common.pyx | 63 - .../pylibraft/neighbors/cpp/__init__.pxd | 0 .../pylibraft/neighbors/cpp/__init__.py | 14 - .../pylibraft/neighbors/cpp/brute_force.pxd | 68 - .../pylibraft/neighbors/cpp/hnsw.pxd | 82 - .../pylibraft/pylibraft/neighbors/cpp/rbc.pxd | 84 - python/pylibraft/pylibraft/neighbors/hnsw.pyx | 490 ------ .../neighbors/ivf_flat/CMakeLists.txt | 24 - .../pylibraft/neighbors/ivf_flat/__init__.pxd | 0 .../pylibraft/neighbors/ivf_flat/__init__.py | 36 - .../neighbors/ivf_flat/cpp/__init__.pxd | 0 .../neighbors/ivf_flat/cpp/__init__.py | 14 - .../neighbors/ivf_flat/cpp/c_ivf_flat.pxd | 183 --- .../pylibraft/neighbors/ivf_flat/ivf_flat.pyx | 821 ---------- .../pylibraft/neighbors/ivf_pq/CMakeLists.txt | 24 - .../pylibraft/neighbors/ivf_pq/__init__.pxd | 0 .../pylibraft/neighbors/ivf_pq/__init__.py | 36 - .../neighbors/ivf_pq/cpp/__init__.pxd | 0 .../neighbors/ivf_pq/cpp/__init__.py | 14 - .../neighbors/ivf_pq/cpp/c_ivf_pq.pxd | 178 --- .../pylibraft/neighbors/ivf_pq/ivf_pq.pxd | 25 - .../pylibraft/neighbors/ivf_pq/ivf_pq.pyx | 797 ---------- python/pylibraft/pylibraft/neighbors/rbc.pyx | 241 --- .../pylibraft/pylibraft/neighbors/refine.pyx | 375 ----- python/pylibraft/pylibraft/test/ann_utils.py | 35 - .../pylibraft/test/test_brute_force.py | 111 -- python/pylibraft/pylibraft/test/test_cagra.py | 292 ---- .../pylibraft/pylibraft/test/test_distance.py | 80 - .../pylibraft/pylibraft/test/test_doctests.py | 17 +- .../pylibraft/test/test_eps_neighborhood.py | 102 -- .../test/test_fused_distance_argmin.py | 69 - .../pylibraft/test/test_fused_l2_argmin.py | 53 - .../pylibraft/pylibraft/test/test_handle.py | 38 +- python/pylibraft/pylibraft/test/test_hnsw.py | 98 -- .../pylibraft/pylibraft/test/test_ivf_flat.py | 518 ------- .../pylibraft/pylibraft/test/test_ivf_pq.py | 550 ------- .../pylibraft/pylibraft/test/test_kmeans.py | 206 --- .../pylibraft/pylibraft/test/test_refine.py | 233 --- .../pylibraft/pylibraft/test/test_select_k.py | 54 - setup.cfg | 2 +- 603 files changed, 90 insertions(+), 50080 deletions(-) delete mode 100644 conda/recipes/libraft/build_libraft_template.sh delete mode 100644 cpp/bench/prims/cluster/kmeans.cu delete mode 100644 cpp/bench/prims/cluster/kmeans_balanced.cu delete mode 100644 cpp/bench/prims/distance/distance_common.cuh delete mode 100644 cpp/bench/prims/distance/distance_cosine.cu delete mode 100644 cpp/bench/prims/distance/distance_exp_l2.cu delete mode 100644 cpp/bench/prims/distance/distance_l1.cu delete mode 100644 cpp/bench/prims/distance/distance_unexp_l2.cu delete mode 100644 cpp/bench/prims/distance/fused_l2_nn.cu delete mode 100644 cpp/bench/prims/distance/kernels.cu delete mode 100644 cpp/bench/prims/distance/masked_nn.cu delete mode 100644 cpp/bench/prims/distance/tune_pairwise/bench.cu delete mode 100644 cpp/bench/prims/distance/tune_pairwise/kernel.cu delete mode 100644 cpp/bench/prims/distance/tune_pairwise/kernel.cuh delete mode 100644 cpp/bench/prims/neighbors/cagra_bench.cuh delete mode 100644 cpp/bench/prims/neighbors/knn.cuh delete mode 100644 cpp/bench/prims/neighbors/knn/brute_force_float_int64_t.cu delete mode 100644 cpp/bench/prims/neighbors/knn/brute_force_float_uint32_t.cu delete mode 100644 cpp/bench/prims/neighbors/knn/cagra_float_uint32_t.cu delete mode 100644 cpp/bench/prims/neighbors/knn/ivf_flat_filter_float_int64_t.cu delete mode 100644 cpp/bench/prims/neighbors/knn/ivf_flat_float_int64_t.cu delete mode 100644 cpp/bench/prims/neighbors/knn/ivf_flat_int8_t_int64_t.cu delete mode 100644 cpp/bench/prims/neighbors/knn/ivf_flat_uint8_t_int64_t.cu delete mode 100644 cpp/bench/prims/neighbors/knn/ivf_pq_filter_float_int64_t.cu delete mode 100644 cpp/bench/prims/neighbors/knn/ivf_pq_float_int64_t.cu delete mode 100644 cpp/bench/prims/neighbors/knn/ivf_pq_int8_t_int64_t.cu delete mode 100644 cpp/bench/prims/neighbors/knn/ivf_pq_uint8_t_int64_t.cu delete mode 100644 cpp/bench/prims/neighbors/refine.cuh delete mode 100644 cpp/bench/prims/neighbors/refine_float_int64_t.cu delete mode 100644 cpp/bench/prims/neighbors/refine_uint8_t_int64_t.cu delete mode 100644 cpp/cmake/patches/hnswlib.diff delete mode 100644 cpp/cmake/patches/hnswlib_override.json delete mode 100644 cpp/cmake/thirdparty/get_hnswlib.cmake delete mode 100644 cpp/include/raft/cluster/specializations.cuh delete mode 100644 cpp/include/raft/distance/detail/pairwise_matrix/dispatch-ext.cuh delete mode 100644 cpp/include/raft/distance/distance-ext.cuh delete mode 100644 cpp/include/raft/distance/fused_l2_nn-ext.cuh delete mode 100644 cpp/include/raft/distance/specializations.cuh delete mode 100644 cpp/include/raft/distance/specializations/distance.cuh delete mode 100644 cpp/include/raft/distance/specializations/fused_l2_nn_min.cuh delete mode 100644 cpp/include/raft/matrix/detail/select_k-ext.cuh delete mode 100644 cpp/include/raft/matrix/specializations/detail/select_k.cuh delete mode 100644 cpp/include/raft/neighbors/ball_cover-ext.cuh delete mode 100644 cpp/include/raft/neighbors/brute_force-ext.cuh delete mode 100644 cpp/include/raft/neighbors/ivf_flat-ext.cuh delete mode 100644 cpp/include/raft/neighbors/ivf_pq-ext.cuh delete mode 100644 cpp/include/raft/neighbors/refine-ext.cuh delete mode 100644 cpp/include/raft/neighbors/specializations.cuh delete mode 100644 cpp/include/raft/neighbors/specializations/ball_cover.cuh delete mode 100644 cpp/include/raft/neighbors/specializations/brute_force.cuh delete mode 100644 cpp/include/raft/neighbors/specializations/detail/ball_cover_lowdim.hpp delete mode 100644 cpp/include/raft/neighbors/specializations/detail/ivf_pq_compute_similarity.cuh delete mode 100644 cpp/include/raft/neighbors/specializations/fused_l2_knn.cuh delete mode 100644 cpp/include/raft/neighbors/specializations/ivf_flat.cuh delete mode 100644 cpp/include/raft/neighbors/specializations/ivf_pq.cuh delete mode 100644 cpp/include/raft/neighbors/specializations/refine.cuh delete mode 100644 cpp/include/raft/sparse/neighbors/specializations.cuh delete mode 100644 cpp/include/raft/spatial/knn/detail/ball_cover/registers-ext.cuh delete mode 100644 cpp/include/raft/spatial/knn/detail/fused_l2_knn-ext.cuh delete mode 100644 cpp/include/raft/spatial/knn/specializations.cuh delete mode 100644 cpp/include/raft/spatial/knn/specializations/knn.cuh delete mode 100644 cpp/internal/raft_internal/neighbors/ivf_pq_compute_similarity_filters_test-ext.cuh delete mode 100644 cpp/internal/raft_internal/neighbors/ivf_pq_search_test-ext.cuh delete mode 100644 cpp/internal/raft_internal/neighbors/naive_knn.cuh delete mode 100644 cpp/internal/raft_internal/neighbors/refine_helper.cuh delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_00_generate.py delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_canberra_double_double_double_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_canberra_float_float_float_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_correlation_double_double_double_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_correlation_float_float_float_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_cosine_double_double_double_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_cosine_float_float_float_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_dice_double_double_double_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_dice_float_float_float_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_hamming_unexpanded_double_double_double_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_hamming_unexpanded_float_float_float_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_hellinger_expanded_double_double_double_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_hellinger_expanded_float_float_float_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_jensen_shannon_double_double_double_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_jensen_shannon_float_float_float_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_kl_divergence_double_double_double_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_kl_divergence_float_float_float_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_l1_double_double_double_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_l1_float_float_float_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_l2_expanded_double_double_double_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_l2_expanded_float_float_float_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_l2_unexpanded_double_double_double_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_l2_unexpanded_float_float_float_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_l_inf_double_double_double_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_l_inf_float_float_float_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_lp_unexpanded_double_double_double_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_lp_unexpanded_float_float_float_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_rbf.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_russel_rao_double_double_double_int.cu delete mode 100644 cpp/src/distance/detail/pairwise_matrix/dispatch_russel_rao_float_float_float_int.cu delete mode 100644 cpp/src/distance/distance.cu delete mode 100644 cpp/src/distance/fused_distance_nn.cu delete mode 100644 cpp/src/distance/fused_l2_nn.cu delete mode 100644 cpp/src/matrix/detail/select_k_double_int64_t.cu delete mode 100644 cpp/src/matrix/detail/select_k_double_uint32_t.cu delete mode 100644 cpp/src/matrix/detail/select_k_float_int32.cu delete mode 100644 cpp/src/matrix/detail/select_k_float_int64_t.cu delete mode 100644 cpp/src/matrix/detail/select_k_float_uint32_t.cu delete mode 100644 cpp/src/matrix/detail/select_k_half_int64_t.cu delete mode 100644 cpp/src/matrix/detail/select_k_half_uint32_t.cu delete mode 100644 cpp/src/neighbors/ball_cover.cu delete mode 100644 cpp/src/neighbors/brute_force_00_generate.py delete mode 100644 cpp/src/neighbors/brute_force_fused_l2_knn_float_int64_t.cu delete mode 100644 cpp/src/neighbors/brute_force_knn_index_float.cu delete mode 100644 cpp/src/neighbors/brute_force_knn_int64_t_float_int64_t.cu delete mode 100644 cpp/src/neighbors/brute_force_knn_int64_t_float_uint32_t.cu delete mode 100644 cpp/src/neighbors/brute_force_knn_int_float_int.cu delete mode 100644 cpp/src/neighbors/brute_force_knn_uint32_t_float_uint32_t.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_00_generate.py delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_00_generate.py delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta.cuh delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_00_generate.py delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim1024_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim128_t8.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim256_t16.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim512_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim1024_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim128_t8.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim256_t16.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim512_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim1024_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim128_t8.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim256_t16.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim512_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim1024_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim128_t8.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim256_t16.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim512_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim1024_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim128_t8.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim256_t16.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim512_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim1024_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim128_t8.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim256_t16.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim512_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta.cuh delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_00_generate.py delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim1024_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim128_t8.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim256_t16.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim512_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim1024_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim128_t8.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim256_t16.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim512_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim1024_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim128_t8.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim256_t16.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim512_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim1024_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim128_t8.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim256_t16.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim512_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim1024_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim128_t8.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim256_t16.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim512_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim1024_t32.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim128_t8.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim256_t16.cu delete mode 100644 cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim512_t32.cu delete mode 100644 cpp/src/neighbors/detail/ivf_flat_interleaved_scan_float_float_int64_t.cu delete mode 100644 cpp/src/neighbors/detail/ivf_flat_interleaved_scan_half_half_int64_t.cu delete mode 100644 cpp/src/neighbors/detail/ivf_flat_interleaved_scan_int8_t_int32_t_int64_t.cu delete mode 100644 cpp/src/neighbors/detail/ivf_flat_interleaved_scan_uint8_t_uint32_t_int64_t.cu delete mode 100644 cpp/src/neighbors/detail/ivf_flat_search.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_00_generate.py delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float_bitset32.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float_bitset64.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float_filt32.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_bitset32.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_bitset64.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_filt32.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_bitset32.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_bitset64.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_filt32.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half_bitset32.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half_bitset64.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half_filt32.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_bitset32.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_bitset64.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_filt32.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_bitset32.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_bitset64.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_filt32.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half_bitset32.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half_bitset64.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half_filt32.cu delete mode 100644 cpp/src/neighbors/detail/ivf_pq_search_filtering_float_int64_t.cu delete mode 100644 cpp/src/neighbors/detail/refine_host_float_float.cpp delete mode 100644 cpp/src/neighbors/detail/refine_host_half_float.cpp delete mode 100644 cpp/src/neighbors/detail/refine_host_int8_t_float.cpp delete mode 100644 cpp/src/neighbors/detail/refine_host_uint8_t_float.cpp delete mode 100644 cpp/src/neighbors/ivf_flat_00_generate.py delete mode 100644 cpp/src/neighbors/ivf_flat_build_float_int64_t.cu delete mode 100644 cpp/src/neighbors/ivf_flat_build_int8_t_int64_t.cu delete mode 100644 cpp/src/neighbors/ivf_flat_build_uint8_t_int64_t.cu delete mode 100644 cpp/src/neighbors/ivf_flat_extend_float_int64_t.cu delete mode 100644 cpp/src/neighbors/ivf_flat_extend_int8_t_int64_t.cu delete mode 100644 cpp/src/neighbors/ivf_flat_extend_uint8_t_int64_t.cu delete mode 100644 cpp/src/neighbors/ivf_flat_search_float_int64_t.cu delete mode 100644 cpp/src/neighbors/ivf_flat_search_int8_t_int64_t.cu delete mode 100644 cpp/src/neighbors/ivf_flat_search_uint8_t_int64_t.cu delete mode 100644 cpp/src/neighbors/ivfpq_build_float_int64_t.cu delete mode 100644 cpp/src/neighbors/ivfpq_build_half_int64_t.cu delete mode 100644 cpp/src/neighbors/ivfpq_build_int8_t_int64_t.cu delete mode 100644 cpp/src/neighbors/ivfpq_build_uint8_t_int64_t.cu delete mode 100644 cpp/src/neighbors/ivfpq_extend_float_int64_t.cu delete mode 100644 cpp/src/neighbors/ivfpq_extend_half_int64_t.cu delete mode 100644 cpp/src/neighbors/ivfpq_extend_int8_t_int64_t.cu delete mode 100644 cpp/src/neighbors/ivfpq_extend_uint8_t_int64_t.cu delete mode 100644 cpp/src/neighbors/ivfpq_search_float_int64_t.cu delete mode 100644 cpp/src/neighbors/ivfpq_search_half_int64_t.cu delete mode 100644 cpp/src/neighbors/ivfpq_search_int8_t_int64_t.cu delete mode 100644 cpp/src/neighbors/ivfpq_search_uint8_t_int64_t.cu delete mode 100644 cpp/src/neighbors/refine_00_generate.py delete mode 100644 cpp/src/neighbors/refine_float_float.cu delete mode 100644 cpp/src/neighbors/refine_half_float.cu delete mode 100644 cpp/src/neighbors/refine_int8_t_float.cu delete mode 100644 cpp/src/neighbors/refine_uint8_t_float.cu delete mode 100644 cpp/src/raft_runtime/cluster/cluster_cost.cuh delete mode 100644 cpp/src/raft_runtime/cluster/cluster_cost_double.cu delete mode 100644 cpp/src/raft_runtime/cluster/cluster_cost_float.cu delete mode 100644 cpp/src/raft_runtime/cluster/kmeans_fit_double.cu delete mode 100644 cpp/src/raft_runtime/cluster/kmeans_fit_float.cu delete mode 100644 cpp/src/raft_runtime/cluster/kmeans_init_plus_plus_double.cu delete mode 100644 cpp/src/raft_runtime/cluster/kmeans_init_plus_plus_float.cu delete mode 100644 cpp/src/raft_runtime/cluster/update_centroids.cuh delete mode 100644 cpp/src/raft_runtime/cluster/update_centroids_double.cu delete mode 100644 cpp/src/raft_runtime/cluster/update_centroids_float.cu delete mode 100644 cpp/src/raft_runtime/distance/fused_distance_min_arg.cu delete mode 100644 cpp/src/raft_runtime/distance/fused_distance_min_arg.hpp delete mode 100644 cpp/src/raft_runtime/distance/fused_l2_min_arg.cu delete mode 100644 cpp/src/raft_runtime/distance/pairwise_distance.cu delete mode 100644 cpp/src/raft_runtime/matrix/select_k_float_int64_t.cu delete mode 100644 cpp/src/raft_runtime/neighbors/brute_force_knn_int64_t_float.cu delete mode 100644 cpp/src/raft_runtime/neighbors/cagra_build.cu delete mode 100644 cpp/src/raft_runtime/neighbors/cagra_search.cu delete mode 100644 cpp/src/raft_runtime/neighbors/cagra_serialize.cu delete mode 100644 cpp/src/raft_runtime/neighbors/eps_neighborhood.cu delete mode 100644 cpp/src/raft_runtime/neighbors/hnsw.cpp delete mode 100644 cpp/src/raft_runtime/neighbors/ivf_flat_build.cu delete mode 100644 cpp/src/raft_runtime/neighbors/ivf_flat_search.cu delete mode 100644 cpp/src/raft_runtime/neighbors/ivf_flat_serialize.cu delete mode 100644 cpp/src/raft_runtime/neighbors/ivfpq_build.cu delete mode 100644 cpp/src/raft_runtime/neighbors/ivfpq_deserialize.cu delete mode 100644 cpp/src/raft_runtime/neighbors/ivfpq_search_float_int64_t.cu delete mode 100644 cpp/src/raft_runtime/neighbors/ivfpq_search_int8_t_int64_t.cu delete mode 100644 cpp/src/raft_runtime/neighbors/ivfpq_search_uint8_t_int64_t.cu delete mode 100644 cpp/src/raft_runtime/neighbors/ivfpq_serialize.cu delete mode 100644 cpp/src/raft_runtime/neighbors/refine_d_int64_t_float.cu delete mode 100644 cpp/src/raft_runtime/neighbors/refine_d_int64_t_int8_t.cu delete mode 100644 cpp/src/raft_runtime/neighbors/refine_d_int64_t_uint8_t.cu delete mode 100644 cpp/src/raft_runtime/neighbors/refine_h_int64_t_float.cu delete mode 100644 cpp/src/raft_runtime/neighbors/refine_h_int64_t_int8_t.cu delete mode 100644 cpp/src/raft_runtime/neighbors/refine_h_int64_t_uint8_t.cu delete mode 100644 cpp/src/spatial/knn/detail/ball_cover/registers.cu delete mode 100644 cpp/src/spatial/knn/detail/ball_cover/registers_00_generate.py delete mode 100644 cpp/src/spatial/knn/detail/ball_cover/registers_eps_pass_euclidean.cu delete mode 100644 cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_dist.cu delete mode 100644 cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_euclidean.cu delete mode 100644 cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_haversine.cu delete mode 100644 cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_dist.cu delete mode 100644 cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_euclidean.cu delete mode 100644 cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_haversine.cu delete mode 100644 cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_dist.cu delete mode 100644 cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_euclidean.cu delete mode 100644 cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_haversine.cu delete mode 100644 cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_dist.cu delete mode 100644 cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_euclidean.cu delete mode 100644 cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_haversine.cu delete mode 100644 cpp/src/spatial/knn/detail/fused_l2_knn_int32_t_float.cu delete mode 100644 cpp/src/spatial/knn/detail/fused_l2_knn_int64_t_float.cu delete mode 100644 cpp/src/spatial/knn/detail/fused_l2_knn_uint32_t_float.cu delete mode 100644 cpp/template/CMakeLists.txt delete mode 100644 cpp/template/README.md delete mode 100755 cpp/template/build.sh delete mode 100644 cpp/template/cmake/thirdparty/fetch_rapids.cmake delete mode 100644 cpp/template/src/cagra_example.cu delete mode 100644 cpp/template/src/common.cuh delete mode 100644 cpp/template/src/ivf_flat_example.cu delete mode 100644 cpp/template/src/ivf_pq_example.cu delete mode 100644 cpp/test/cluster/cluster_solvers.cu delete mode 100644 cpp/test/cluster/cluster_solvers_deprecated.cu delete mode 100644 cpp/test/cluster/kmeans.cu delete mode 100644 cpp/test/cluster/kmeans_balanced.cu delete mode 100644 cpp/test/cluster/kmeans_find_k.cu delete mode 100644 cpp/test/cluster/linkage.cu delete mode 100644 cpp/test/cluster/spectral.cu delete mode 100644 cpp/test/distance/dist_adj.cu delete mode 100644 cpp/test/distance/dist_adj.cuh delete mode 100644 cpp/test/distance/dist_adj_distance_instance.cu delete mode 100644 cpp/test/distance/dist_adj_threshold.cuh delete mode 100644 cpp/test/distance/dist_canberra.cu delete mode 100644 cpp/test/distance/dist_correlation.cu delete mode 100644 cpp/test/distance/dist_cos.cu delete mode 100644 cpp/test/distance/dist_dice.cu delete mode 100644 cpp/test/distance/dist_hamming.cu delete mode 100644 cpp/test/distance/dist_hellinger.cu delete mode 100644 cpp/test/distance/dist_inner_product.cu delete mode 100644 cpp/test/distance/dist_jensen_shannon.cu delete mode 100644 cpp/test/distance/dist_kl_divergence.cu delete mode 100644 cpp/test/distance/dist_l1.cu delete mode 100644 cpp/test/distance/dist_l2_exp.cu delete mode 100644 cpp/test/distance/dist_l2_sqrt_exp.cu delete mode 100644 cpp/test/distance/dist_l2_unexp.cu delete mode 100644 cpp/test/distance/dist_l_inf.cu delete mode 100644 cpp/test/distance/dist_lp_unexp.cu delete mode 100644 cpp/test/distance/dist_russell_rao.cu delete mode 100644 cpp/test/distance/distance_base.cuh delete mode 100644 cpp/test/distance/fused_cosine_nn.cu delete mode 100644 cpp/test/distance/fused_l2_nn.cu delete mode 100644 cpp/test/distance/gram.cu delete mode 100644 cpp/test/distance/gram_base.cuh delete mode 100644 cpp/test/distance/masked_nn.cu delete mode 100644 cpp/test/distance/masked_nn_compress_to_bits.cu delete mode 100644 cpp/test/neighbors/ann_brute_force.cuh delete mode 100644 cpp/test/neighbors/ann_brute_force/test_float.cu delete mode 100644 cpp/test/neighbors/ann_cagra.cuh delete mode 100644 cpp/test/neighbors/ann_cagra/search_kernel_uint64_t.cuh delete mode 100644 cpp/test/neighbors/ann_cagra/test_float_int64_t.cu delete mode 100644 cpp/test/neighbors/ann_cagra/test_float_uint32_t.cu delete mode 100644 cpp/test/neighbors/ann_cagra/test_half_int64_t.cu delete mode 100644 cpp/test/neighbors/ann_cagra/test_half_uint32_t.cu delete mode 100644 cpp/test/neighbors/ann_cagra/test_int8_t_uint32_t.cu delete mode 100644 cpp/test/neighbors/ann_cagra/test_uint8_t_uint32_t.cu delete mode 100644 cpp/test/neighbors/ann_cagra_vpq.cuh delete mode 100644 cpp/test/neighbors/ann_cagra_vpq/test_float_int64_t.cu delete mode 100644 cpp/test/neighbors/ann_cagra_vpq/test_float_uint32_t.cu delete mode 100644 cpp/test/neighbors/ann_ivf_flat.cuh delete mode 100644 cpp/test/neighbors/ann_ivf_flat/test_filter_float_int64_t.cu delete mode 100644 cpp/test/neighbors/ann_ivf_flat/test_float_int64_t.cu delete mode 100644 cpp/test/neighbors/ann_ivf_flat/test_int8_t_int64_t.cu delete mode 100644 cpp/test/neighbors/ann_ivf_flat/test_uint8_t_int64_t.cu delete mode 100644 cpp/test/neighbors/ann_ivf_pq.cuh delete mode 100644 cpp/test/neighbors/ann_ivf_pq/ivf_pq_build_float_uint32_t.cu delete mode 100644 cpp/test/neighbors/ann_ivf_pq/ivf_pq_build_test-ext.cuh delete mode 100644 cpp/test/neighbors/ann_ivf_pq/ivf_pq_search_float_uint32_t.cu delete mode 100644 cpp/test/neighbors/ann_ivf_pq/test_filter_float_int64_t.cu delete mode 100644 cpp/test/neighbors/ann_ivf_pq/test_filter_int8_t_int64_t.cu delete mode 100644 cpp/test/neighbors/ann_ivf_pq/test_float_int64_t.cu delete mode 100644 cpp/test/neighbors/ann_ivf_pq/test_float_uint32_t.cu delete mode 100644 cpp/test/neighbors/ann_ivf_pq/test_int8_t_int64_t.cu delete mode 100644 cpp/test/neighbors/ann_ivf_pq/test_uint8_t_int64_t.cu delete mode 100644 cpp/test/neighbors/ann_nn_descent.cuh delete mode 100644 cpp/test/neighbors/ann_nn_descent/test_batch_float_uint32_t.cu delete mode 100644 cpp/test/neighbors/ann_nn_descent/test_float_uint32_t.cu delete mode 100644 cpp/test/neighbors/ann_nn_descent/test_int8_t_uint32_t.cu delete mode 100644 cpp/test/neighbors/ann_nn_descent/test_uint8_t_uint32_t.cu delete mode 100644 cpp/test/neighbors/ann_utils.cuh delete mode 100644 cpp/test/neighbors/fused_l2_knn.cu delete mode 100644 cpp/test/neighbors/knn.cu delete mode 100644 cpp/test/neighbors/refine.cu delete mode 100644 cpp/test/neighbors/tiled_knn.cu delete mode 100644 cpp/test/sparse/gram.cu delete mode 100644 cpp/test/sparse/neighbors/brute_force.cu delete mode 100644 cpp/test/sparse/neighbors/cross_component_nn.cu delete mode 100644 cpp/test/sparse/neighbors/knn_graph.cu delete mode 100644 cpp/test/stats/neighborhood_recall.cu delete mode 100644 cpp/test/stats/silhouette_score.cu delete mode 100644 cpp/test/stats/trustworthiness.cu delete mode 100644 docs/source/cpp_api/cluster.rst delete mode 100644 docs/source/cpp_api/cluster_kmeans.rst delete mode 100644 docs/source/cpp_api/cluster_kmeans_balanced.rst delete mode 100644 docs/source/cpp_api/cluster_slhc.rst delete mode 100644 docs/source/cpp_api/cluster_spectral.rst delete mode 100644 docs/source/cpp_api/distance.rst delete mode 100644 docs/source/cpp_api/distance_1nn.rst delete mode 100644 docs/source/cpp_api/distance_pairwise.rst delete mode 100644 docs/source/cpp_api/matrix_selection.rst delete mode 100644 docs/source/cpp_api/neighbors.rst delete mode 100644 docs/source/cpp_api/neighbors_ball_cover.rst delete mode 100644 docs/source/cpp_api/neighbors_brute_force.rst delete mode 100644 docs/source/cpp_api/neighbors_cagra.rst delete mode 100644 docs/source/cpp_api/neighbors_epsilon_neighborhood.rst delete mode 100644 docs/source/cpp_api/neighbors_hnsw.rst delete mode 100644 docs/source/cpp_api/neighbors_ivf_flat.rst delete mode 100644 docs/source/cpp_api/neighbors_ivf_pq.rst delete mode 100644 docs/source/cpp_api/sparse_distance.rst delete mode 100644 docs/source/cpp_api/sparse_neighbors.rst delete mode 100644 docs/source/cpp_api/stats_neighborhood.rst delete mode 100644 docs/source/pylibraft_api/cluster.rst delete mode 100644 docs/source/pylibraft_api/distance.rst delete mode 100644 docs/source/pylibraft_api/matrix.rst delete mode 100644 docs/source/pylibraft_api/neighbors.rst delete mode 100644 docs/source/using_libraft.md delete mode 100644 docs/source/vector_search_tutorial.md delete mode 100644 notebooks/VectorSearch_QuestionRetrieval.ipynb delete mode 100644 notebooks/ivf_flat_example.ipynb delete mode 100644 notebooks/tutorial_ivf_pq.ipynb delete mode 100644 notebooks/utils.py delete mode 100644 python/pylibraft/pylibraft/cluster/CMakeLists.txt delete mode 100644 python/pylibraft/pylibraft/cluster/__init__.pxd delete mode 100644 python/pylibraft/pylibraft/cluster/__init__.py delete mode 100644 python/pylibraft/pylibraft/cluster/cpp/__init__.pxd delete mode 100644 python/pylibraft/pylibraft/cluster/cpp/__init__.py delete mode 100644 python/pylibraft/pylibraft/cluster/cpp/kmeans.pxd delete mode 100644 python/pylibraft/pylibraft/cluster/cpp/kmeans_types.pxd delete mode 100644 python/pylibraft/pylibraft/cluster/kmeans.pyx delete mode 100644 python/pylibraft/pylibraft/distance/CMakeLists.txt delete mode 100644 python/pylibraft/pylibraft/distance/__init__.pxd delete mode 100644 python/pylibraft/pylibraft/distance/__init__.py delete mode 100644 python/pylibraft/pylibraft/distance/distance_type.pxd delete mode 100644 python/pylibraft/pylibraft/distance/fused_distance_nn.pyx delete mode 100644 python/pylibraft/pylibraft/distance/fused_l2_nn.pyx delete mode 100644 python/pylibraft/pylibraft/distance/pairwise_distance.pyx delete mode 100644 python/pylibraft/pylibraft/matrix/CMakeLists.txt delete mode 100644 python/pylibraft/pylibraft/matrix/__init__.pxd delete mode 100644 python/pylibraft/pylibraft/matrix/__init__.py delete mode 100644 python/pylibraft/pylibraft/matrix/cpp/__init__.pxd delete mode 100644 python/pylibraft/pylibraft/matrix/cpp/__init__.py delete mode 100644 python/pylibraft/pylibraft/matrix/cpp/select_k.pxd delete mode 100644 python/pylibraft/pylibraft/matrix/select_k.pyx delete mode 100644 python/pylibraft/pylibraft/neighbors/CMakeLists.txt delete mode 100644 python/pylibraft/pylibraft/neighbors/__init__.pxd delete mode 100644 python/pylibraft/pylibraft/neighbors/__init__.py delete mode 100644 python/pylibraft/pylibraft/neighbors/brute_force.pyx delete mode 100644 python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt delete mode 100644 python/pylibraft/pylibraft/neighbors/cagra/__init__.pxd delete mode 100644 python/pylibraft/pylibraft/neighbors/cagra/__init__.py delete mode 100644 python/pylibraft/pylibraft/neighbors/cagra/cagra.pxd delete mode 100644 python/pylibraft/pylibraft/neighbors/cagra/cagra.pyx delete mode 100644 python/pylibraft/pylibraft/neighbors/cagra/cpp/__init__.pxd delete mode 100644 python/pylibraft/pylibraft/neighbors/cagra/cpp/__init__.py delete mode 100644 python/pylibraft/pylibraft/neighbors/cagra/cpp/c_cagra.pxd delete mode 100644 python/pylibraft/pylibraft/neighbors/common.pxd delete mode 100644 python/pylibraft/pylibraft/neighbors/common.pyx delete mode 100644 python/pylibraft/pylibraft/neighbors/cpp/__init__.pxd delete mode 100644 python/pylibraft/pylibraft/neighbors/cpp/__init__.py delete mode 100644 python/pylibraft/pylibraft/neighbors/cpp/brute_force.pxd delete mode 100644 python/pylibraft/pylibraft/neighbors/cpp/hnsw.pxd delete mode 100644 python/pylibraft/pylibraft/neighbors/cpp/rbc.pxd delete mode 100644 python/pylibraft/pylibraft/neighbors/hnsw.pyx delete mode 100644 python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt delete mode 100644 python/pylibraft/pylibraft/neighbors/ivf_flat/__init__.pxd delete mode 100644 python/pylibraft/pylibraft/neighbors/ivf_flat/__init__.py delete mode 100644 python/pylibraft/pylibraft/neighbors/ivf_flat/cpp/__init__.pxd delete mode 100644 python/pylibraft/pylibraft/neighbors/ivf_flat/cpp/__init__.py delete mode 100644 python/pylibraft/pylibraft/neighbors/ivf_flat/cpp/c_ivf_flat.pxd delete mode 100644 python/pylibraft/pylibraft/neighbors/ivf_flat/ivf_flat.pyx delete mode 100644 python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt delete mode 100644 python/pylibraft/pylibraft/neighbors/ivf_pq/__init__.pxd delete mode 100644 python/pylibraft/pylibraft/neighbors/ivf_pq/__init__.py delete mode 100644 python/pylibraft/pylibraft/neighbors/ivf_pq/cpp/__init__.pxd delete mode 100644 python/pylibraft/pylibraft/neighbors/ivf_pq/cpp/__init__.py delete mode 100644 python/pylibraft/pylibraft/neighbors/ivf_pq/cpp/c_ivf_pq.pxd delete mode 100644 python/pylibraft/pylibraft/neighbors/ivf_pq/ivf_pq.pxd delete mode 100644 python/pylibraft/pylibraft/neighbors/ivf_pq/ivf_pq.pyx delete mode 100644 python/pylibraft/pylibraft/neighbors/rbc.pyx delete mode 100644 python/pylibraft/pylibraft/neighbors/refine.pyx delete mode 100644 python/pylibraft/pylibraft/test/ann_utils.py delete mode 100644 python/pylibraft/pylibraft/test/test_brute_force.py delete mode 100644 python/pylibraft/pylibraft/test/test_cagra.py delete mode 100644 python/pylibraft/pylibraft/test/test_distance.py delete mode 100644 python/pylibraft/pylibraft/test/test_eps_neighborhood.py delete mode 100755 python/pylibraft/pylibraft/test/test_fused_distance_argmin.py delete mode 100644 python/pylibraft/pylibraft/test/test_fused_l2_argmin.py delete mode 100644 python/pylibraft/pylibraft/test/test_hnsw.py delete mode 100644 python/pylibraft/pylibraft/test/test_ivf_flat.py delete mode 100644 python/pylibraft/pylibraft/test/test_ivf_pq.py delete mode 100644 python/pylibraft/pylibraft/test/test_kmeans.py delete mode 100644 python/pylibraft/pylibraft/test/test_refine.py delete mode 100644 python/pylibraft/pylibraft/test/test_select_k.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index db379c9d47..945589dc12 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -52,7 +52,6 @@ jobs: branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} - skip_upload_pkgs: libraft-template docs-build: if: github.ref_type == 'branch' needs: python-build diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 82e56cd95d..47951783ba 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -43,16 +43,8 @@ jobs: - '!README.md' - '!docs/**' - '!img/**' - - '!notebooks/**' - '!python/**' - '!thirdparty/LICENSES/**' - test_notebooks: - - '**' - - '!.devcontainer/**' - - '!.pre-commit-config.yaml' - - '!CONTRIBUTING.md' - - '!README.md' - - '!thirdparty/LICENSES/**' test_python: - '**' - '!.devcontainer/**' @@ -61,7 +53,6 @@ jobs: - '!README.md' - '!docs/**' - '!img/**' - - '!notebooks/**' - '!thirdparty/LICENSES/**' checks: secrets: inherit diff --git a/README.md b/README.md index 7f43eb89dc..898c5c22c3 100755 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ #
 RAFT: Reusable Accelerated Functions and Tools for Vector Search and More
> [!IMPORTANT] -> The vector search and clustering algorithms in RAFT are being migrated to a new library dedicated to vector search called [cuVS](https://github.com/rapidsai/cuvs). We will continue to support the vector search algorithms in RAFT during this move, but will no longer update them after the RAPIDS 24.06 (June) release. We plan to complete the migration by RAPIDS 24.10 (October) release and will be removing them altogether in the 24.12 (December) release. +> The vector search and clustering algorithms in RAFT have been formally migrated to a new library dedicated to vector search called [cuVS](https://github.com/rapidsai/cuvs). The headers for the vector search and clustering algorithms in RAFT will remain for a bried period, but will no longer be tested, benchmarked, included in the pre-compiled libraft binary, or otherwise updated after the 24.12 (December 2024) release. We will be removing these headers altogether in a future release. It is strongly suggested to use cuVS for these routines, which include any headers in the `distance`, `neighbors`, `cluster` and `spatial` directories, and use the RAFT versions at your own risk. ![RAFT tech stack](img/raft-tech-stack-vss.png) @@ -27,7 +27,6 @@ - [RAFT Reference Documentation](https://docs.rapids.ai/api/raft/stable/): API Documentation. - [RAFT Getting Started](./docs/source/quick_start.md): Getting started with RAFT. - [Build and Install RAFT](./docs/source/build.md): Instructions for installing and building RAFT. -- [Example Notebooks](./notebooks): Example jupyter notebooks - [RAPIDS Community](https://rapids.ai/community.html): Get help, contribute, and collaborate. - [GitHub repository](https://github.com/rapidsai/raft): Download the RAFT source code. - [Issue tracker](https://github.com/rapidsai/raft/issues): Report issues or request features. @@ -120,13 +119,13 @@ auto metric = raft::distance::DistanceType::L2SqrtExpanded; raft::distance::pairwise_distance(handle, input.view(), input.view(), output.view(), metric); ``` -It's also possible to create `raft::device_mdspan` views to invoke the same API with raw pointers and shape information: +It's also possible to create `raft::device_mdspan` views to invoke the same API with raw pointers and shape information. Take this example from the [NVIDIA cuVS](https://github.com/rapidsai/cuvs) library: ```c++ #include #include #include -#include +#include raft::device_resources handle; @@ -147,8 +146,8 @@ auto output_view = raft::make_device_matrix_view(output, n_samples, n_samples); raft::random::make_blobs(handle, input_view, labels_view); -auto metric = raft::distance::DistanceType::L2SqrtExpanded; -raft::distance::pairwise_distance(handle, input_view, input_view, output_view, metric); +auto metric = cuvs::distance::DistanceType::L2SqrtExpanded; +cuvs::distance::pairwise_distance(handle, input_view, input_view, output_view, metric); ``` @@ -156,12 +155,12 @@ raft::distance::pairwise_distance(handle, input_view, input_view, output_view, m The `pylibraft` package contains a Python API for RAFT algorithms and primitives. `pylibraft` integrates nicely into other libraries by being very lightweight with minimal dependencies and accepting any object that supports the `__cuda_array_interface__`, such as [CuPy's ndarray](https://docs.cupy.dev/en/stable/user_guide/interoperability.html#rmm). The number of RAFT algorithms exposed in this package is continuing to grow from release to release. -The example below demonstrates computing the pairwise Euclidean distances between CuPy arrays. Note that CuPy is not a required dependency for `pylibraft`. +The example below demonstrates computing the pairwise Euclidean distances between CuPy arrays using the [NVIDIA cuVS](https://github.com/rapidsai/cuvs) library. Note that CuPy is not a required dependency for `pylibraft`. ```python import cupy as cp -from pylibraft.distance import pairwise_distance +from cuvs.distance import pairwise_distance n_samples = 5000 n_features = 50 @@ -208,7 +207,7 @@ pylibraft.config.set_output_as(lambda device_ndarray: return device_ndarray.copy ```python import cupy as cp -from pylibraft.distance import pairwise_distance +from cuvs.distance import pairwise_distance n_samples = 5000 n_features = 50 @@ -230,7 +229,6 @@ RAFT's C++ and Python libraries can both be installed through Conda and the Pyth The easiest way to install RAFT is through conda and several packages are provided. - `libraft-headers` C++ headers -- `libraft` (optional) C++ shared library containing pre-compiled template instantiations and runtime API. - `pylibraft` (optional) Python library - `raft-dask` (optional) Python library for deployment of multi-node multi-GPU algorithms that use the RAFT `raft::comms` abstraction layer in Dask clusters. @@ -253,8 +251,6 @@ You can also install the conda packages individually using the `mamba` command a mamba install -c rapidsai -c conda-forge -c nvidia libraft libraft-headers cuda-version=12.5 ``` -If installing the C++ APIs please see [using libraft](https://docs.rapids.ai/api/raft/nightly/using_libraft/) for more information on using the pre-compiled shared library. You can also refer to the [example C++ template project](https://github.com/rapidsai/raft/tree/branch-24.12/cpp/template) for a ready-to-go CMake configuration that you can drop into your project and build against installed RAFT development artifacts above. - ### Installing Python through Pip `pylibraft` and `raft-dask` both have experimental packages that can be [installed through pip](https://rapids.ai/pip.html#install): @@ -263,12 +259,10 @@ pip install pylibraft-cu11 --extra-index-url=https://pypi.nvidia.com pip install raft-dask-cu11 --extra-index-url=https://pypi.nvidia.com ``` -These packages statically build RAFT's pre-compiled instantiations and so the C++ headers and pre-compiled shared library won't be readily available to use in your code. +These packages statically build RAFT's pre-compiled instantiations and so the C++ headers won't be readily available to use in your code. The [build instructions](https://docs.rapids.ai/api/raft/nightly/build/) contain more details on building RAFT from source and including it in downstream projects. You can also find a more comprehensive version of the above CPM code snippet the [Building RAFT C++ and Python from source](https://docs.rapids.ai/api/raft/nightly/build/#building-c-and-python-from-source) section of the build instructions. -You can find an example [RAFT project template](cpp/template/README.md) in the `cpp/template` directory, which demonstrates how to build a new application with RAFT or incorporate RAFT into an existing CMake project. - ## Contributing @@ -282,7 +276,7 @@ When citing RAFT generally, please consider referencing this Github project. title={Rapidsai/raft: RAFT contains fundamental widely-used algorithms and primitives for data science, Graph and machine learning.}, url={https://github.com/rapidsai/raft}, journal={GitHub}, - publisher={Nvidia RAPIDS}, + publisher={NVIDIA RAPIDS}, author={Rapidsai}, year={2022} } diff --git a/build.sh b/build.sh index d54a8895a3..a95cb8ee23 100755 --- a/build.sh +++ b/build.sh @@ -18,7 +18,7 @@ ARGS=$* # scripts, and that this script resides in the repo dir! REPODIR=$(cd $(dirname $0); pwd) -VALIDARGS="clean libraft pylibraft raft-dask docs tests template bench-prims clean --uninstall -v -g -n --compile-lib --compile-static-lib --allgpuarch --no-nvtx --show_depr_warn --incl-cache-stats --time -h" +VALIDARGS="clean libraft pylibraft raft-dask docs tests bench-prims clean --uninstall -v -g -n --compile-lib --compile-static-lib --allgpuarch --no-nvtx --show_depr_warn --incl-cache-stats --time -h" HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool=] [--limit-tests=] [--limit-bench-prims=] [--build-metrics=] where is: clean - remove all existing build artifacts and configuration (start over) @@ -29,7 +29,6 @@ HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool= is: -v - verbose build mode @@ -73,8 +72,8 @@ INSTALL_TARGET=install BUILD_REPORT_METRICS="" BUILD_REPORT_INCL_CACHE_STATS=OFF -TEST_TARGETS="CLUSTER_TEST;CORE_TEST;DISTANCE_TEST;LABEL_TEST;LINALG_TEST;MATRIX_TEST;NEIGHBORS_TEST;NEIGHBORS_ANN_BRUTE_FORCE_TEST;NEIGHBORS_ANN_CAGRA_TEST;NEIGHBORS_ANN_NN_DESCENT_TEST;NEIGHBORS_ANN_IVF_TEST;RANDOM_TEST;SOLVERS_TEST;SPARSE_TEST;SPARSE_DIST_TEST;SPARSE_NEIGHBORS_TEST;STATS_TEST;UTILS_TEST" -BENCH_TARGETS="CLUSTER_BENCH;CORE_BENCH;NEIGHBORS_BENCH;DISTANCE_BENCH;LINALG_BENCH;MATRIX_BENCH;SPARSE_BENCH;RANDOM_BENCH" +TEST_TARGETS="CORE_TEST;LABEL_TEST;LINALG_TEST;MATRIX_TEST;RANDOM_TEST;SOLVERS_TEST;SPARSE_TEST;STATS_TEST;UTILS_TEST" +BENCH_TARGETS="CORE_BENCH;LINALG_BENCH;MATRIX_BENCH;SPARSE_BENCH;RANDOM_BENCH" CACHE_ARGS="" NVTX=ON @@ -480,11 +479,3 @@ if hasArg docs; then sphinx-build -b html source _html fi -################################################################################ -# Initiate build for example RAFT application template (if needed) - -if hasArg template; then - pushd ${REPODIR}/cpp/template - ./build.sh - popd -fi diff --git a/conda/recipes/libraft/build_libraft_template.sh b/conda/recipes/libraft/build_libraft_template.sh deleted file mode 100644 index 86c0fa11b6..0000000000 --- a/conda/recipes/libraft/build_libraft_template.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2022-2024, NVIDIA CORPORATION. - -# Just building template so we verify it uses libraft.so and fail if it doesn't build -./build.sh template --no-nvtx diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index a075308500..503c4cb6fb 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -322,57 +322,3 @@ outputs: home: https://rapids.ai/ license: Apache-2.0 summary: libraft tests - - name: libraft-template - version: {{ version }} - script: build_libraft_template.sh - build: - script_env: *script_env - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - ignore_run_exports_from: - {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} - {% else %} - - {{ compiler('cuda') }} - - cuda-cudart-dev - - libcublas-dev - {% endif %} - requirements: - build: - - {{ compiler('c') }} - - {{ compiler('cxx') }} - {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} ={{ cuda_version }} - {% else %} - - {{ compiler('cuda') }} - {% endif %} - - cuda-version ={{ cuda_version }} - - cmake {{ cmake_version }} - - ninja - - {{ stdlib("c") }} - host: - - {{ pin_subpackage('libraft', exact=True) }} - - {{ pin_subpackage('libraft-headers', exact=True) }} - - cuda-version ={{ cuda_version }} - {% if cuda_major == "11" %} - - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - - libcublas {{ cuda11_libcublas_host_version }} - - libcublas-dev {{ cuda11_libcublas_host_version }} - {% else %} - - cuda-cudart-dev - - cuda-profiler-api - - libcublas-dev - {% endif %} - run: - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - {% if cuda_major == "11" %} - - cudatoolkit - {% else %} - - cuda-cudart - - libcublas - {% endif %} - - {{ pin_subpackage('libraft', exact=True) }} - about: - home: https://rapids.ai/ - license: Apache-2.0 - summary: libraft template diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index f4c18d53a8..4ed9529a36 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -49,7 +49,6 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) option(BUILD_SHARED_LIBS "Build raft shared libraries" ON) option(BUILD_TESTS "Build raft unit-tests" ON) option(BUILD_PRIMS_BENCH "Build raft C++ benchmark tests" OFF) -option(BUILD_CAGRA_HNSWLIB "Build CAGRA+hnswlib interface" ON) option(CUDA_ENABLE_KERNELINFO "Enable kernel resource usage info" OFF) option(CUDA_ENABLE_LINEINFO "Enable the -lineinfo option for nvcc (useful for cuda-memcheck / profiler)" OFF @@ -171,10 +170,6 @@ if(BUILD_PRIMS_BENCH) rapids_cpm_gbench(BUILD_STATIC) endif() -if(BUILD_CAGRA_HNSWLIB) - include(cmake/thirdparty/get_hnswlib.cmake) -endif() - # ################################################################################################## # * raft --------------------------------------------------------------------- add_library(raft INTERFACE) @@ -183,9 +178,6 @@ add_library(raft::raft ALIAS raft) target_include_directories( raft INTERFACE "$" "$" ) -if(BUILD_CAGRA_HNSWLIB) - target_link_libraries(raft INTERFACE hnswlib::hnswlib) -endif() # Keep RAFT as lightweight as possible. Only CUDA libs and rmm should be used in global target. target_link_libraries(raft INTERFACE rmm::rmm cuco::cuco nvidia::cutlass::cutlass CCCL::CCCL) @@ -271,277 +263,11 @@ if(RAFT_COMPILE_LIBRARY) add_library( raft_objs OBJECT src/core/logger.cpp - src/distance/detail/pairwise_matrix/dispatch_canberra_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_canberra_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_correlation_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_correlation_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_cosine_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_cosine_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_dice_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_dice_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_hamming_unexpanded_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_hamming_unexpanded_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_hellinger_expanded_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_hellinger_expanded_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_jensen_shannon_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_jensen_shannon_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_kl_divergence_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_kl_divergence_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_l1_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_l1_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_l2_expanded_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_l2_expanded_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_l2_unexpanded_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_l2_unexpanded_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_l_inf_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_l_inf_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_lp_unexpanded_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_lp_unexpanded_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_rbf.cu - src/distance/detail/pairwise_matrix/dispatch_russel_rao_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_russel_rao_float_float_float_int.cu - src/distance/distance.cu - src/distance/fused_l2_nn.cu - src/distance/fused_distance_nn.cu src/linalg/detail/coalesced_reduction.cu - src/matrix/detail/select_k_double_int64_t.cu - src/matrix/detail/select_k_double_uint32_t.cu - src/matrix/detail/select_k_float_int64_t.cu - src/matrix/detail/select_k_float_uint32_t.cu - src/matrix/detail/select_k_float_int32.cu - src/matrix/detail/select_k_half_int64_t.cu - src/matrix/detail/select_k_half_uint32_t.cu - src/neighbors/ball_cover.cu - src/neighbors/brute_force_fused_l2_knn_float_int64_t.cu - src/neighbors/brute_force_knn_int64_t_float_int64_t.cu - src/neighbors/brute_force_knn_int64_t_float_uint32_t.cu - src/neighbors/brute_force_knn_int_float_int.cu - src/neighbors/brute_force_knn_uint32_t_float_uint32_t.cu - src/neighbors/brute_force_knn_index_float.cu - src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim128_t8.cu - src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim256_t16.cu - src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim512_t32.cu - src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim1024_t32.cu - src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim128_t8.cu - src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim256_t16.cu - src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim512_t32.cu - src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim1024_t32.cu - src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim128_t8.cu - src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim256_t16.cu - src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim512_t32.cu - src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim1024_t32.cu - src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim128_t8.cu - src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim256_t16.cu - src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim512_t32.cu - src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim1024_t32.cu - src/neighbors/detail/cagra/search_single_cta_float_uint32_dim128_t8.cu - src/neighbors/detail/cagra/search_single_cta_float_uint32_dim256_t16.cu - src/neighbors/detail/cagra/search_single_cta_float_uint32_dim512_t32.cu - src/neighbors/detail/cagra/search_single_cta_float_uint32_dim1024_t32.cu - src/neighbors/detail/cagra/search_single_cta_half_uint32_dim128_t8.cu - src/neighbors/detail/cagra/search_single_cta_half_uint32_dim256_t16.cu - src/neighbors/detail/cagra/search_single_cta_half_uint32_dim512_t32.cu - src/neighbors/detail/cagra/search_single_cta_half_uint32_dim1024_t32.cu - src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim128_t8.cu - src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim256_t16.cu - src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim512_t32.cu - src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim1024_t32.cu - src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim128_t8.cu - src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim256_t16.cu - src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim512_t32.cu - src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim1024_t32.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/ivf_flat_interleaved_scan_float_float_int64_t.cu - src/neighbors/detail/ivf_flat_interleaved_scan_half_half_int64_t.cu - src/neighbors/detail/ivf_flat_interleaved_scan_int8_t_int32_t_int64_t.cu - src/neighbors/detail/ivf_flat_interleaved_scan_uint8_t_uint32_t_int64_t.cu - src/neighbors/detail/ivf_flat_search.cu - src/neighbors/detail/ivf_pq_compute_similarity_float_float.cu - src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false.cu - src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true.cu - src/neighbors/detail/ivf_pq_compute_similarity_float_half.cu - src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false.cu - src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true.cu - src/neighbors/detail/ivf_pq_compute_similarity_half_half.cu - src/neighbors/detail/refine_host_float_float.cpp - src/neighbors/detail/refine_host_half_float.cpp - src/neighbors/detail/refine_host_int8_t_float.cpp - src/neighbors/detail/refine_host_uint8_t_float.cpp - src/neighbors/ivf_flat_build_float_int64_t.cu - src/neighbors/ivf_flat_build_int8_t_int64_t.cu - src/neighbors/ivf_flat_build_uint8_t_int64_t.cu - src/neighbors/ivf_flat_extend_float_int64_t.cu - src/neighbors/ivf_flat_extend_int8_t_int64_t.cu - src/neighbors/ivf_flat_extend_uint8_t_int64_t.cu - src/neighbors/ivf_flat_search_float_int64_t.cu - src/neighbors/ivf_flat_search_int8_t_int64_t.cu - src/neighbors/ivf_flat_search_uint8_t_int64_t.cu - src/neighbors/ivfpq_build_float_int64_t.cu - src/neighbors/ivfpq_build_half_int64_t.cu - src/neighbors/ivfpq_build_int8_t_int64_t.cu - src/neighbors/ivfpq_build_uint8_t_int64_t.cu - src/neighbors/ivfpq_extend_float_int64_t.cu - src/neighbors/ivfpq_extend_half_int64_t.cu - src/neighbors/ivfpq_extend_int8_t_int64_t.cu - src/neighbors/ivfpq_extend_uint8_t_int64_t.cu - src/neighbors/ivfpq_search_float_int64_t.cu - src/neighbors/ivfpq_search_half_int64_t.cu - src/neighbors/ivfpq_search_int8_t_int64_t.cu - src/neighbors/ivfpq_search_uint8_t_int64_t.cu - src/neighbors/refine_float_float.cu - src/neighbors/refine_half_float.cu - src/neighbors/refine_int8_t_float.cu - src/neighbors/refine_uint8_t_float.cu - src/raft_runtime/cluster/cluster_cost.cuh - src/raft_runtime/cluster/cluster_cost_double.cu - src/raft_runtime/cluster/cluster_cost_float.cu - src/raft_runtime/cluster/kmeans_fit_double.cu - src/raft_runtime/cluster/kmeans_fit_float.cu - src/raft_runtime/cluster/kmeans_init_plus_plus_double.cu - src/raft_runtime/cluster/kmeans_init_plus_plus_float.cu - src/raft_runtime/cluster/update_centroids.cuh - src/raft_runtime/cluster/update_centroids_double.cu - src/raft_runtime/cluster/update_centroids_float.cu - src/raft_runtime/distance/fused_distance_min_arg.cu - src/raft_runtime/distance/fused_l2_min_arg.cu - src/raft_runtime/distance/pairwise_distance.cu - src/raft_runtime/matrix/select_k_float_int64_t.cu - src/raft_runtime/neighbors/brute_force_knn_int64_t_float.cu - src/raft_runtime/neighbors/cagra_build.cu - src/raft_runtime/neighbors/cagra_search.cu - src/raft_runtime/neighbors/cagra_serialize.cu - src/raft_runtime/neighbors/eps_neighborhood.cu - $<$:src/raft_runtime/neighbors/hnsw.cpp> - src/raft_runtime/neighbors/ivf_flat_build.cu - src/raft_runtime/neighbors/ivf_flat_search.cu - src/raft_runtime/neighbors/ivf_flat_serialize.cu - src/raft_runtime/neighbors/ivfpq_build.cu - src/raft_runtime/neighbors/ivfpq_deserialize.cu - src/raft_runtime/neighbors/ivfpq_search_float_int64_t.cu - src/raft_runtime/neighbors/ivfpq_search_int8_t_int64_t.cu - src/raft_runtime/neighbors/ivfpq_search_uint8_t_int64_t.cu - src/raft_runtime/neighbors/ivfpq_serialize.cu - src/raft_runtime/neighbors/refine_d_int64_t_float.cu - src/raft_runtime/neighbors/refine_d_int64_t_int8_t.cu - src/raft_runtime/neighbors/refine_d_int64_t_uint8_t.cu - src/raft_runtime/neighbors/refine_h_int64_t_float.cu - src/raft_runtime/neighbors/refine_h_int64_t_int8_t.cu - src/raft_runtime/neighbors/refine_h_int64_t_uint8_t.cu src/raft_runtime/random/rmat_rectangular_generator_int64_double.cu src/raft_runtime/random/rmat_rectangular_generator_int64_float.cu src/raft_runtime/random/rmat_rectangular_generator_int_double.cu src/raft_runtime/random/rmat_rectangular_generator_int_float.cu - src/spatial/knn/detail/ball_cover/registers_eps_pass_euclidean.cu - src/spatial/knn/detail/ball_cover/registers_pass_one_2d_dist.cu - src/spatial/knn/detail/ball_cover/registers_pass_one_2d_euclidean.cu - src/spatial/knn/detail/ball_cover/registers_pass_one_2d_haversine.cu - src/spatial/knn/detail/ball_cover/registers_pass_one_3d_dist.cu - src/spatial/knn/detail/ball_cover/registers_pass_one_3d_euclidean.cu - src/spatial/knn/detail/ball_cover/registers_pass_one_3d_haversine.cu - src/spatial/knn/detail/ball_cover/registers_pass_two_2d_dist.cu - src/spatial/knn/detail/ball_cover/registers_pass_two_2d_euclidean.cu - src/spatial/knn/detail/ball_cover/registers_pass_two_2d_haversine.cu - src/spatial/knn/detail/ball_cover/registers_pass_two_3d_dist.cu - src/spatial/knn/detail/ball_cover/registers_pass_two_3d_euclidean.cu - src/spatial/knn/detail/ball_cover/registers_pass_two_3d_haversine.cu - src/spatial/knn/detail/fused_l2_knn_int32_t_float.cu - src/spatial/knn/detail/fused_l2_knn_int64_t_float.cu - src/spatial/knn/detail/fused_l2_knn_uint32_t_float.cu ) set_target_properties( raft_objs diff --git a/cpp/bench/prims/CMakeLists.txt b/cpp/bench/prims/CMakeLists.txt index 52c63ad73b..cf03a36612 100644 --- a/cpp/bench/prims/CMakeLists.txt +++ b/cpp/bench/prims/CMakeLists.txt @@ -74,49 +74,9 @@ function(ConfigureBench) endfunction() if(BUILD_PRIMS_BENCH) - ConfigureBench( - NAME - CORE_BENCH - PATH - core/bitset.cu - core/copy.cu - main.cpp - ) + ConfigureBench(NAME CORE_BENCH PATH core/bitset.cu core/copy.cu main.cpp) - ConfigureBench( - NAME - UTIL_BENCH - PATH - util/popc.cu - main.cpp - ) - - ConfigureBench( - NAME CLUSTER_BENCH PATH cluster/kmeans_balanced.cu cluster/kmeans.cu - main.cpp OPTIONAL LIB EXPLICIT_INSTANTIATE_ONLY - ) - - ConfigureBench( - NAME TUNE_DISTANCE PATH distance/tune_pairwise/kernel.cu - distance/tune_pairwise/bench.cu main.cpp - ) - - ConfigureBench( - NAME - DISTANCE_BENCH - PATH - distance/distance_cosine.cu - distance/distance_exp_l2.cu - distance/distance_l1.cu - distance/distance_unexp_l2.cu - distance/fused_l2_nn.cu - distance/masked_nn.cu - distance/kernels.cu - main.cpp - OPTIONAL - LIB - EXPLICIT_INSTANTIATE_ONLY - ) + ConfigureBench(NAME UTIL_BENCH PATH util/popc.cu main.cpp) ConfigureBench( NAME @@ -137,54 +97,18 @@ if(BUILD_PRIMS_BENCH) ) ConfigureBench( - NAME MATRIX_BENCH PATH matrix/argmin.cu matrix/gather.cu - matrix/select_k.cu main.cpp OPTIONAL LIB EXPLICIT_INSTANTIATE_ONLY + NAME MATRIX_BENCH PATH matrix/argmin.cu matrix/gather.cu matrix/select_k.cu main.cpp OPTIONAL + LIB EXPLICIT_INSTANTIATE_ONLY ) ConfigureBench( - NAME RANDOM_BENCH PATH random/make_blobs.cu random/permute.cu - random/rng.cu random/subsample.cu main.cpp - ) - - ConfigureBench( - NAME - SPARSE_BENCH - PATH - sparse/bitmap_to_csr.cu - sparse/convert_csr.cu - sparse/select_k_csr.cu + NAME RANDOM_BENCH PATH random/make_blobs.cu random/permute.cu random/rng.cu random/subsample.cu main.cpp ) ConfigureBench( - NAME - NEIGHBORS_BENCH - PATH - neighbors/knn/brute_force_float_int64_t.cu - neighbors/knn/brute_force_float_uint32_t.cu - neighbors/knn/cagra_float_uint32_t.cu - neighbors/knn/ivf_flat_filter_float_int64_t.cu - neighbors/knn/ivf_flat_float_int64_t.cu - neighbors/knn/ivf_flat_int8_t_int64_t.cu - neighbors/knn/ivf_flat_uint8_t_int64_t.cu - neighbors/knn/ivf_pq_float_int64_t.cu - neighbors/knn/ivf_pq_filter_float_int64_t.cu - neighbors/knn/ivf_pq_int8_t_int64_t.cu - neighbors/knn/ivf_pq_uint8_t_int64_t.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_search_filtering_float_int64_t.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_float_bitset64.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_bitset64.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_bitset64.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_half_bitset64.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_bitset64.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_bitset64.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_half_half_bitset64.cu - neighbors/refine_float_int64_t.cu - neighbors/refine_uint8_t_int64_t.cu + NAME SPARSE_BENCH PATH sparse/bitmap_to_csr.cu sparse/convert_csr.cu sparse/select_k_csr.cu main.cpp - OPTIONAL - LIB - EXPLICIT_INSTANTIATE_ONLY ) endif() diff --git a/cpp/bench/prims/cluster/kmeans.cu b/cpp/bench/prims/cluster/kmeans.cu deleted file mode 100644 index 6387211135..0000000000 --- a/cpp/bench/prims/cluster/kmeans.cu +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include - -namespace raft::bench::cluster { - -struct KMeansBenchParams { - DatasetParams data; - BlobsParams blobs; - raft::cluster::KMeansParams kmeans; -}; - -inline auto operator<<(std::ostream& os, const KMeansBenchParams& p) -> std::ostream& -{ - os << p.data.rows << "#" << p.data.cols << "#" << p.kmeans.n_clusters; - return os; -} - -template -struct KMeans : public BlobsFixture { - KMeans(const KMeansBenchParams& p) - : BlobsFixture(p.data, p.blobs), - params(p), - centroids(this->handle), - labels(this->handle) - { - } - - void run_benchmark(::benchmark::State& state) override - { - std::ostringstream label_stream; - label_stream << params; - state.SetLabel(label_stream.str()); - - raft::device_matrix_view X_view = this->X.view(); - std::optional> opt_weights_view = std::nullopt; - std::optional> centroids_view = - std::make_optional>(centroids.view()); - raft::device_vector_view labels_view = labels.view(); - raft::host_scalar_view inertia_view = raft::make_host_scalar_view(&inertia); - raft::host_scalar_view n_iter_view = raft::make_host_scalar_view(&n_iter); - - this->loop_on_state(state, [&]() { - raft::cluster::kmeans_fit_predict(this->handle, - params.kmeans, - X_view, - opt_weights_view, - centroids_view, - labels_view, - inertia_view, - n_iter_view); - }); - } - - void allocate_temp_buffers(const ::benchmark::State& state) override - { - centroids = - raft::make_device_matrix(this->handle, params.kmeans.n_clusters, params.data.cols); - labels = raft::make_device_vector(this->handle, params.data.rows); - } - - private: - KMeansBenchParams params; - raft::device_matrix centroids; - raft::device_vector labels; - T inertia; - IndexT n_iter; -}; // struct KMeans - -std::vector getKMeansInputs() -{ - std::vector out; - KMeansBenchParams p; - p.data.row_major = true; - p.blobs.cluster_std = 1.0; - p.blobs.shuffle = false; - p.blobs.center_box_min = -10.0; - p.blobs.center_box_max = 10.0; - p.blobs.seed = 12345ULL; - p.kmeans.init = raft::cluster::KMeansParams::KMeansPlusPlus; - p.kmeans.max_iter = 300; - p.kmeans.tol = 1e-4; - p.kmeans.verbosity = RAFT_LEVEL_INFO; - p.kmeans.metric = raft::distance::DistanceType::L2Expanded; - p.kmeans.inertia_check = true; - std::vector> row_cols_k = { - {1000000, 20, 1000}, - {3000000, 50, 20}, - {10000000, 50, 5}, - }; - for (auto& rck : row_cols_k) { - p.data.rows = std::get<0>(rck); - p.data.cols = std::get<1>(rck); - p.blobs.n_clusters = std::get<2>(rck); - p.kmeans.n_clusters = std::get<2>(rck); - out.push_back(p); - } - return out; -} - -// note(lsugy): commenting out int64_t because the templates are not compiled in the distance -// library, resulting in long compilation times. -RAFT_BENCH_REGISTER((KMeans), "", getKMeansInputs()); -RAFT_BENCH_REGISTER((KMeans), "", getKMeansInputs()); -// RAFT_BENCH_REGISTER((KMeans), "", getKMeansInputs()); -// RAFT_BENCH_REGISTER((KMeans), "", getKMeansInputs()); - -} // namespace raft::bench::cluster diff --git a/cpp/bench/prims/cluster/kmeans_balanced.cu b/cpp/bench/prims/cluster/kmeans_balanced.cu deleted file mode 100644 index dc05783989..0000000000 --- a/cpp/bench/prims/cluster/kmeans_balanced.cu +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include - -namespace raft::bench::cluster { - -struct KMeansBalancedBenchParams { - DatasetParams data; - uint32_t n_lists; - raft::cluster::kmeans_balanced_params kb_params; -}; - -template -struct KMeansBalanced : public fixture { - KMeansBalanced(const KMeansBalancedBenchParams& p) : params(p), X(handle), centroids(handle) {} - - void run_benchmark(::benchmark::State& state) override - { - this->loop_on_state(state, [this]() { - raft::device_matrix_view X_view = this->X.view(); - raft::device_matrix_view centroids_view = this->centroids.view(); - raft::cluster::kmeans_balanced::fit( - this->handle, this->params.kb_params, X_view, centroids_view); - }); - } - - void allocate_data(const ::benchmark::State& state) override - { - X = raft::make_device_matrix(handle, params.data.rows, params.data.cols); - - raft::random::RngState rng{1234}; - constexpr T kRangeMax = std::is_integral_v ? std::numeric_limits::max() : T(1); - constexpr T kRangeMin = std::is_integral_v ? std::numeric_limits::min() : T(-1); - if constexpr (std::is_integral_v) { - raft::random::uniformInt( - handle, rng, X.data_handle(), params.data.rows * params.data.cols, kRangeMin, kRangeMax); - } else { - raft::random::uniform( - handle, rng, X.data_handle(), params.data.rows * params.data.cols, kRangeMin, kRangeMax); - } - resource::sync_stream(handle, stream); - } - - void allocate_temp_buffers(const ::benchmark::State& state) override - { - centroids = - raft::make_device_matrix(this->handle, params.n_lists, params.data.cols); - } - - private: - KMeansBalancedBenchParams params; - raft::device_matrix X; - raft::device_matrix centroids; -}; // struct KMeansBalanced - -std::vector getKMeansBalancedInputs() -{ - std::vector out; - KMeansBalancedBenchParams p; - p.data.row_major = true; - p.kb_params.n_iters = 20; - p.kb_params.metric = raft::distance::DistanceType::L2Expanded; - std::vector> row_cols = { - {100000, 128}, {1000000, 128}, {10000000, 128}, - // The following dataset sizes are too large for most GPUs. - // {100000000, 128}, - }; - for (auto& rc : row_cols) { - p.data.rows = rc.first; - p.data.cols = rc.second; - for (auto n_lists : std::vector({1000, 10000, 100000})) { - p.n_lists = n_lists; - out.push_back(p); - } - } - return out; -} - -// Note: the datasets sizes are too large for 32-bit index types. -RAFT_BENCH_REGISTER((KMeansBalanced), "", getKMeansBalancedInputs()); - -} // namespace raft::bench::cluster diff --git a/cpp/bench/prims/distance/distance_common.cuh b/cpp/bench/prims/distance/distance_common.cuh deleted file mode 100644 index 8368062168..0000000000 --- a/cpp/bench/prims/distance/distance_common.cuh +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include - -#include - -namespace raft::bench::distance { - -struct distance_params { - int m, n, k; - bool isRowMajor; -}; // struct distance_params - -template -struct distance : public fixture { - distance(const distance_params& p) - : params(p), - x(p.m * p.k, stream), - y(p.n * p.k, stream), - out(p.m * p.n, stream), - workspace(0, stream) - { - RAFT_CUDA_TRY(cudaMemsetAsync(x.data(), 0, x.size() * sizeof(T), stream)); - RAFT_CUDA_TRY(cudaMemsetAsync(y.data(), 0, y.size() * sizeof(T), stream)); - RAFT_CUDA_TRY(cudaMemsetAsync(out.data(), 0, out.size() * sizeof(T), stream)); - worksize = raft::distance::getWorkspaceSize( - x.data(), y.data(), params.m, params.n, params.k); - workspace.resize(worksize, stream); - } - - void run_benchmark(::benchmark::State& state) override - { - loop_on_state(state, [this]() { - raft::distance::distance(handle, - x.data(), - y.data(), - out.data(), - params.m, - params.n, - params.k, - (void*)workspace.data(), - worksize, - params.isRowMajor); - }); - } - - private: - distance_params params; - rmm::device_uvector x, y, out; - rmm::device_uvector workspace; - size_t worksize; -}; // struct Distance - -const std::vector dist_input_vecs{ - {32, 16384, 16384, true}, {64, 16384, 16384, true}, {128, 16384, 16384, true}, - {256, 16384, 16384, true}, {512, 16384, 16384, true}, {1024, 16384, 16384, true}, - {16384, 32, 16384, true}, {16384, 64, 16384, true}, {16384, 128, 16384, true}, - {16384, 256, 16384, true}, {16384, 512, 16384, true}, {16384, 1024, 16384, true}, - {16384, 16384, 32, true}, {16384, 16384, 64, true}, {16384, 16384, 128, true}, - {16384, 16384, 256, true}, {16384, 16384, 512, true}, {16384, 16384, 1024, true}, - {16384, 16384, 16384, true}, {32, 16384, 16384, false}, {64, 16384, 16384, false}, - {128, 16384, 16384, false}, {256, 16384, 16384, false}, {512, 16384, 16384, false}, - {1024, 16384, 16384, false}, {16384, 32, 16384, false}, {16384, 64, 16384, false}, - {16384, 128, 16384, false}, {16384, 256, 16384, false}, {16384, 512, 16384, false}, - {16384, 1024, 16384, false}, {16384, 16384, 32, false}, {16384, 16384, 64, false}, - {16384, 16384, 128, false}, {16384, 16384, 256, false}, {16384, 16384, 512, false}, - {16384, 16384, 1024, false}, {16384, 16384, 16384, false} - -}; - -#define DIST_BENCH_REGISTER(Name, Metric) \ - using Name##F = distance; \ - RAFT_BENCH_REGISTER(Name##F, "", dist_input_vecs); \ - using Name##D = distance; \ - RAFT_BENCH_REGISTER(Name##D, "", dist_input_vecs); - -} // namespace raft::bench::distance diff --git a/cpp/bench/prims/distance/distance_cosine.cu b/cpp/bench/prims/distance/distance_cosine.cu deleted file mode 100644 index c8ac8067c8..0000000000 --- a/cpp/bench/prims/distance/distance_cosine.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "distance_common.cuh" - -namespace raft::bench::distance { - -DIST_BENCH_REGISTER(DistanceCosine, raft::distance::DistanceType::CosineExpanded); - -} // namespace raft::bench::distance diff --git a/cpp/bench/prims/distance/distance_exp_l2.cu b/cpp/bench/prims/distance/distance_exp_l2.cu deleted file mode 100644 index 52b7fff05c..0000000000 --- a/cpp/bench/prims/distance/distance_exp_l2.cu +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "distance_common.cuh" - -namespace raft::bench::distance { - -DIST_BENCH_REGISTER(DistanceL2Sq, raft::distance::DistanceType::L2Expanded); -DIST_BENCH_REGISTER(DistanceL2Sqrt, raft::distance::DistanceType::L2SqrtExpanded); - -} // namespace raft::bench::distance diff --git a/cpp/bench/prims/distance/distance_l1.cu b/cpp/bench/prims/distance/distance_l1.cu deleted file mode 100644 index e80db48ef0..0000000000 --- a/cpp/bench/prims/distance/distance_l1.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "distance_common.cuh" - -namespace raft::bench::distance { - -DIST_BENCH_REGISTER(DistanceL1, raft::distance::DistanceType::L1); - -} // namespace raft::bench::distance diff --git a/cpp/bench/prims/distance/distance_unexp_l2.cu b/cpp/bench/prims/distance/distance_unexp_l2.cu deleted file mode 100644 index 7ac1a8a4b5..0000000000 --- a/cpp/bench/prims/distance/distance_unexp_l2.cu +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "distance_common.cuh" - -namespace raft::bench::distance { - -DIST_BENCH_REGISTER(DistanceUnexpL2Sq, raft::distance::DistanceType::L2Unexpanded); -DIST_BENCH_REGISTER(DistanceUnexpL2Sqrt, raft::distance::DistanceType::L2SqrtUnexpanded); - -} // namespace raft::bench::distance diff --git a/cpp/bench/prims/distance/fused_l2_nn.cu b/cpp/bench/prims/distance/fused_l2_nn.cu deleted file mode 100644 index a263bef6ba..0000000000 --- a/cpp/bench/prims/distance/fused_l2_nn.cu +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include -#include - -#include - -namespace raft::bench::distance { - -struct fusedl2nn_inputs { - int64_t m, n, k; -}; // struct fusedl2nn_inputs - -inline auto operator<<(std::ostream& os, const fusedl2nn_inputs& p) -> std::ostream& -{ - os << p.m << "#" << p.n << "#" << p.k; - return os; -} - -template -struct fusedl2nn : public fixture { - fusedl2nn(const fusedl2nn_inputs& p) - : params(p), - workspace(this->handle), - x(this->handle), - y(this->handle), - x_norm(this->handle), - y_norm(this->handle), - out(this->handle) - { - } - - void allocate_data(const ::benchmark::State& state) override - { - x = raft::make_device_matrix(handle, params.m, params.k); - y = raft::make_device_matrix(handle, params.n, params.k); - x_norm = raft::make_device_vector(handle, params.m); - y_norm = raft::make_device_vector(handle, params.n); - out = raft::make_device_vector(handle, params.m); - - raft::random::RngState rng{1234}; - raft::random::uniform( - handle, rng, x.data_handle(), params.m * params.k, (DataT)-1.0, (DataT)1.0); - raft::random::uniform( - handle, rng, y.data_handle(), params.n * params.k, (DataT)-1.0, (DataT)1.0); - - // Pre-compute norms - raft::linalg::rowNorm(x_norm.data_handle(), - x.data_handle(), - params.k, - params.m, - raft::linalg::L2Norm, - true, - stream); - raft::linalg::rowNorm(y_norm.data_handle(), - y.data_handle(), - params.k, - params.n, - raft::linalg::L2Norm, - true, - stream); - resource::sync_stream(handle, stream); - } - - void allocate_temp_buffers(const ::benchmark::State& state) override - { - workspace = raft::make_device_vector(handle, params.m * sizeof(IdxT)); - } - - void run_benchmark(::benchmark::State& state) override - { - std::ostringstream label_stream; - label_stream << params; - state.SetLabel(label_stream.str()); - - loop_on_state(state, [this]() { - raft::distance::fusedL2NNMinReduce(out.data_handle(), - x.data_handle(), - y.data_handle(), - x_norm.data_handle(), - y_norm.data_handle(), - static_cast(params.m), - static_cast(params.n), - static_cast(params.k), - (void*)workspace.data_handle(), - false, - true, - stream); - }); - - int64_t num_flops = 2 * params.m * params.n * params.k; - - int64_t read_elts = params.n * params.k + params.m * params.k; - int64_t write_elts = params.m; - - state.counters["FLOP/s"] = benchmark::Counter( - num_flops, benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::OneK::kIs1000); - - state.counters["BW Wr"] = benchmark::Counter(write_elts * sizeof(OutT), - benchmark::Counter::kIsIterationInvariantRate, - benchmark::Counter::OneK::kIs1000); - state.counters["BW Rd"] = benchmark::Counter(read_elts * sizeof(DataT), - benchmark::Counter::kIsIterationInvariantRate, - benchmark::Counter::OneK::kIs1000); - } - - private: - fusedl2nn_inputs params; - raft::device_matrix x, y; - raft::device_vector x_norm, y_norm; - raft::device_vector out; - raft::device_vector workspace; -}; // struct fusedl2nn - -template -std::vector getFusedL2NNInputs() -{ - std::vector inputs; - std::vector m_list = {100000, 1000000}; - if constexpr (sizeof(IdxT) == 8) { m_list.push_back(10000000); } - std::vector n_list = {100, 1000, 10000}; - std::vector k_list = {64, 128, 256}; - for (auto m : m_list) { - for (auto n : n_list) { - for (auto k : k_list) { - inputs.push_back({m, n, k}); - } - } - } - return inputs; -} - -#define FUSEDL2NN_BENCH(DataT, IdxT, OutT) \ - RAFT_BENCH_REGISTER((fusedl2nn), "", getFusedL2NNInputs()) - -FUSEDL2NN_BENCH(float, int, float); -FUSEDL2NN_BENCH(double, int, double); -FUSEDL2NN_BENCH(float, int, (raft::KeyValuePair)); -FUSEDL2NN_BENCH(double, int, (raft::KeyValuePair)); -FUSEDL2NN_BENCH(float, int64_t, float); -FUSEDL2NN_BENCH(double, int64_t, double); -FUSEDL2NN_BENCH(float, int64_t, (raft::KeyValuePair)); -FUSEDL2NN_BENCH(double, int64_t, (raft::KeyValuePair)); - -} // namespace raft::bench::distance diff --git a/cpp/bench/prims/distance/kernels.cu b/cpp/bench/prims/distance/kernels.cu deleted file mode 100644 index eb86330637..0000000000 --- a/cpp/bench/prims/distance/kernels.cu +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (c) 2019-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include - -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -namespace raft::bench::distance::kernels { - -using namespace raft::distance::kernels; -struct GramTestParams { - int m; // m parameter of the GEMM - int k; // k parameter of the GEMM - int n; // n parameter of the GEMM - KernelParams kernel_params; - bool is_row_major; -}; // struct GramTestParams - -template -struct GramMatrix : public fixture { - GramMatrix(const GramTestParams& p) - : params(p), handle(stream), A(0, stream), B(0, stream), C(0, stream) - { - kernel = std::unique_ptr>( - KernelFactory::create(p.kernel_params, resource::get_cublas_handle(handle))); - - A.resize(params.m * params.k, stream); - B.resize(params.k * params.n, stream); - C.resize(params.m * params.n, stream); - raft::random::RngState rng(123456ULL); - raft::random::uniform(handle, rng, A.data(), params.m * params.k, T(-1.0), T(1.0)); - raft::random::uniform(handle, rng, B.data(), params.k * params.n, T(-1.0), T(1.0)); - } - - ~GramMatrix() - { - A.release(); - B.release(); - C.release(); - } - - void run_benchmark(::benchmark::State& state) override - { - if (!this->kernel) { state.SkipWithError("Kernel matrix is not initialized"); } - loop_on_state(state, [this]() { - (*this->kernel)(A.data(), - this->params.m, - this->params.k, - B.data(), - this->params.n, - C.data(), - this->params.is_row_major, - this->stream); - }); - } - - private: - const raft::device_resources handle; - std::unique_ptr> kernel; - GramTestParams params; - - rmm::device_uvector A; // input matrix A, size [m * k] - rmm::device_uvector B; // input matrix B, size [n * k] - rmm::device_uvector C; // output matrix C, size [m*n] -}; - -static std::vector getInputs() -{ - std::vector param_vec; - std::vector kernel_params{KernelParams{LINEAR, 3, 1, 0}, - KernelParams{POLYNOMIAL, 2, 1.3, 1}, - KernelParams{TANH, 2, 0.5, 2.4}, - KernelParams{RBF, 2, 0.5, 0}}; - struct TestSize { - int m; - int k; - int n; - }; - std::vector data_size{{4096, 10, 1024}, - {4096, 100, 1024}, - {4096, 1000, 1024}, - {4096, 10000, 1024}, - {100000, 10, 1024}, - {100000, 100, 1024}, - {100000, 1000, 1024}}; - - param_vec.reserve(kernel_params.size() * data_size.size()); - for (TestSize s : data_size) { - for (auto kernel : kernel_params) { - for (bool row_major : {false, true}) { - param_vec.push_back(GramTestParams{s.m, s.k, s.n, kernel, row_major}); - } - } - } - return param_vec; -} - -RAFT_BENCH_REGISTER(GramMatrix, "", getInputs()); -RAFT_BENCH_REGISTER(GramMatrix, "", getInputs()); - -} // namespace raft::bench::distance::kernels diff --git a/cpp/bench/prims/distance/masked_nn.cu b/cpp/bench/prims/distance/masked_nn.cu deleted file mode 100644 index 979d438b67..0000000000 --- a/cpp/bench/prims/distance/masked_nn.cu +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace raft::bench::distance::masked_nn { - -// Introduce various sparsity patterns -enum AdjacencyPattern { - checkerboard = 0, - checkerboard_4 = 1, - checkerboard_64 = 2, - all_true = 3, - all_false = 4 -}; - -struct Params { - int m, n, k, num_groups; - AdjacencyPattern pattern; -}; // struct Params - -RAFT_KERNEL init_adj(AdjacencyPattern pattern, - int n, - raft::device_matrix_view adj, - raft::device_vector_view group_idxs) -{ - int m = adj.extent(0); - int num_groups = adj.extent(1); - - for (int idx_m = blockIdx.y * blockDim.y + threadIdx.y; idx_m < m; - idx_m += blockDim.y * gridDim.y) { - for (int idx_g = blockIdx.x * blockDim.x + threadIdx.x; idx_g < num_groups; - idx_g += blockDim.x * gridDim.x) { - switch (pattern) { - case checkerboard: adj(idx_m, idx_g) = (idx_m + idx_g) % 2; break; - case checkerboard_4: adj(idx_m, idx_g) = (idx_m / 4 + idx_g) % 2; break; - case checkerboard_64: adj(idx_m, idx_g) = (idx_m / 64 + idx_g) % 2; break; - case all_true: adj(idx_m, idx_g) = true; break; - case all_false: adj(idx_m, idx_g) = false; break; - default: assert(false && "unknown pattern"); - } - } - } - // Each group is of size n / num_groups. - // - // - group_idxs[j] indicates the start of group j + 1 (i.e. is the inclusive - // scan of the group lengths) - // - // - The first group always starts at index zero, so we do not store it. - // - // - The group_idxs[num_groups - 1] should always equal n. - - if (blockIdx.y == 0 && threadIdx.y == 0) { - const int g_stride = blockDim.x * gridDim.x; - for (int idx_g = blockIdx.x * blockDim.x + threadIdx.x; idx_g < num_groups; idx_g += g_stride) { - group_idxs(idx_g) = (idx_g + 1) * (n / num_groups); - } - group_idxs(num_groups - 1) = n; - } -} - -template -struct masked_l2_nn : public fixture { - using DataT = T; - using IdxT = int; - using OutT = raft::KeyValuePair; - using RedOpT = raft::distance::MinAndDistanceReduceOp; - using PairRedOpT = raft::distance::KVPMinReduce; - using ParamT = raft::distance::masked_l2_nn_params; - - // Parameters - Params params; - // Data - raft::device_vector out; - raft::device_matrix x, y; - raft::device_vector xn, yn; - raft::device_matrix adj; - raft::device_vector group_idxs; - - masked_l2_nn(const Params& p) - : params(p), - out{raft::make_device_vector(handle, p.m)}, - x{raft::make_device_matrix(handle, p.m, p.k)}, - y{raft::make_device_matrix(handle, p.n, p.k)}, - xn{raft::make_device_vector(handle, p.m)}, - yn{raft::make_device_vector(handle, p.n)}, - adj{raft::make_device_matrix(handle, p.m, p.num_groups)}, - group_idxs{raft::make_device_vector(handle, p.num_groups)} - { - raft::random::RngState r(123456ULL); - - uniform(handle, r, x.data_handle(), p.m * p.k, T(-1.0), T(1.0)); - uniform(handle, r, y.data_handle(), p.n * p.k, T(-1.0), T(1.0)); - raft::linalg::rowNorm( - xn.data_handle(), x.data_handle(), p.k, p.m, raft::linalg::L2Norm, true, stream); - raft::linalg::rowNorm( - yn.data_handle(), y.data_handle(), p.k, p.n, raft::linalg::L2Norm, true, stream); - raft::distance::initialize, int>( - handle, out.data_handle(), p.m, std::numeric_limits::max(), RedOpT{}); - - dim3 block(32, 32); - dim3 grid(10, 10); - init_adj<<>>(p.pattern, p.n, adj.view(), group_idxs.view()); - RAFT_CUDA_TRY(cudaGetLastError()); - } - - void run_benchmark(::benchmark::State& state) override - { - bool init_out = true; - bool sqrt = false; - ParamT masked_l2_params{RedOpT{}, PairRedOpT{}, sqrt, init_out}; - - loop_on_state(state, [this, masked_l2_params]() { - // It is sufficient to only benchmark the L2-squared metric - raft::distance::masked_l2_nn(handle, - masked_l2_params, - x.view(), - y.view(), - xn.view(), - yn.view(), - adj.view(), - group_idxs.view(), - out.view()); - }); - - // Virtual flop count if no skipping had occurred. - size_t virtual_flops = size_t(2) * size_t(params.m) * size_t(params.n) * size_t(params.k); - - int64_t read_elts = params.n * params.k + params.m * params.k; - int64_t write_elts = params.m; - - // Virtual min flops is the number of flops that would have been executed if - // the algorithm had actually skipped each computation that it could have - // skipped. - size_t virtual_min_flops = 0; - switch (params.pattern) { - case checkerboard: - case checkerboard_4: - case checkerboard_64: virtual_min_flops = virtual_flops / 2; break; - case all_true: virtual_min_flops = virtual_flops; break; - case all_false: virtual_min_flops = 0; break; - default: assert(false && "unknown pattern"); - } - - // VFLOP/s is the "virtual" flop count that would have executed if there was - // no adjacency pattern. This is useful for comparing to fusedL2NN - state.counters["VFLOP/s"] = benchmark::Counter(virtual_flops, - benchmark::Counter::kIsIterationInvariantRate, - benchmark::Counter::OneK::kIs1000); - // Virtual min flops is the number of flops that would have been executed if - // the algorithm had actually skipped each computation that it could have - // skipped. - state.counters["VminFLOP/s"] = benchmark::Counter(virtual_min_flops, - benchmark::Counter::kIsIterationInvariantRate, - benchmark::Counter::OneK::kIs1000); - - state.counters["BW Wr"] = benchmark::Counter(write_elts * sizeof(OutT), - benchmark::Counter::kIsIterationInvariantRate, - benchmark::Counter::OneK::kIs1000); - state.counters["BW Rd"] = benchmark::Counter(read_elts * sizeof(DataT), - benchmark::Counter::kIsIterationInvariantRate, - benchmark::Counter::OneK::kIs1000); - - state.counters["m"] = benchmark::Counter(params.m); - state.counters["n"] = benchmark::Counter(params.n); - state.counters["k"] = benchmark::Counter(params.k); - state.counters["num_groups"] = benchmark::Counter(params.num_groups); - state.counters["group size"] = benchmark::Counter(params.n / params.num_groups); - state.counters["Pat"] = benchmark::Counter(static_cast(params.pattern)); - - state.counters["SM count"] = raft::getMultiProcessorCount(); - } -}; - -const std::vector masked_l2_nn_input_vecs = { - // Very fat matrices... - {32, 16384, 16384, 32, AdjacencyPattern::checkerboard}, - {64, 16384, 16384, 32, AdjacencyPattern::checkerboard}, - {128, 16384, 16384, 32, AdjacencyPattern::checkerboard}, - {256, 16384, 16384, 32, AdjacencyPattern::checkerboard}, - {512, 16384, 16384, 32, AdjacencyPattern::checkerboard}, - {1024, 16384, 16384, 32, AdjacencyPattern::checkerboard}, - {16384, 32, 16384, 32, AdjacencyPattern::checkerboard}, - {16384, 64, 16384, 32, AdjacencyPattern::checkerboard}, - {16384, 128, 16384, 32, AdjacencyPattern::checkerboard}, - {16384, 256, 16384, 32, AdjacencyPattern::checkerboard}, - {16384, 512, 16384, 32, AdjacencyPattern::checkerboard}, - {16384, 1024, 16384, 32, AdjacencyPattern::checkerboard}, - - // Representative matrices... - {16384, 16384, 32, 32, AdjacencyPattern::checkerboard}, - {16384, 16384, 64, 32, AdjacencyPattern::checkerboard}, - {16384, 16384, 128, 32, AdjacencyPattern::checkerboard}, - {16384, 16384, 256, 32, AdjacencyPattern::checkerboard}, - {16384, 16384, 512, 32, AdjacencyPattern::checkerboard}, - {16384, 16384, 1024, 32, AdjacencyPattern::checkerboard}, - {16384, 16384, 16384, 32, AdjacencyPattern::checkerboard}, - - {16384, 16384, 32, 32, AdjacencyPattern::checkerboard_4}, - {16384, 16384, 64, 32, AdjacencyPattern::checkerboard_4}, - {16384, 16384, 128, 32, AdjacencyPattern::checkerboard_4}, - {16384, 16384, 256, 32, AdjacencyPattern::checkerboard_4}, - {16384, 16384, 512, 32, AdjacencyPattern::checkerboard_4}, - {16384, 16384, 1024, 32, AdjacencyPattern::checkerboard_4}, - {16384, 16384, 16384, 32, AdjacencyPattern::checkerboard_4}, - - {16384, 16384, 32, 32, AdjacencyPattern::checkerboard_64}, - {16384, 16384, 64, 32, AdjacencyPattern::checkerboard_64}, - {16384, 16384, 128, 32, AdjacencyPattern::checkerboard_64}, - {16384, 16384, 256, 32, AdjacencyPattern::checkerboard_64}, - {16384, 16384, 512, 32, AdjacencyPattern::checkerboard_64}, - {16384, 16384, 1024, 32, AdjacencyPattern::checkerboard_64}, - {16384, 16384, 16384, 32, AdjacencyPattern::checkerboard_64}, - - {16384, 16384, 32, 32, AdjacencyPattern::all_true}, - {16384, 16384, 64, 32, AdjacencyPattern::all_true}, - {16384, 16384, 128, 32, AdjacencyPattern::all_true}, - {16384, 16384, 256, 32, AdjacencyPattern::all_true}, - {16384, 16384, 512, 32, AdjacencyPattern::all_true}, - {16384, 16384, 1024, 32, AdjacencyPattern::all_true}, - {16384, 16384, 16384, 32, AdjacencyPattern::all_true}, - - {16384, 16384, 32, 32, AdjacencyPattern::all_false}, - {16384, 16384, 64, 32, AdjacencyPattern::all_false}, - {16384, 16384, 128, 32, AdjacencyPattern::all_false}, - {16384, 16384, 256, 32, AdjacencyPattern::all_false}, - {16384, 16384, 512, 32, AdjacencyPattern::all_false}, - {16384, 16384, 1024, 32, AdjacencyPattern::all_false}, - {16384, 16384, 16384, 32, AdjacencyPattern::all_false}, -}; - -RAFT_BENCH_REGISTER(masked_l2_nn, "", masked_l2_nn_input_vecs); -// We don't benchmark double to keep compile times in check when not using the -// distance library. - -} // namespace raft::bench::distance::masked_nn diff --git a/cpp/bench/prims/distance/tune_pairwise/bench.cu b/cpp/bench/prims/distance/tune_pairwise/bench.cu deleted file mode 100644 index 81105cdefe..0000000000 --- a/cpp/bench/prims/distance/tune_pairwise/bench.cu +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Tuning benchmarks. -// -// Goals: -// -// 1. Fast compile times to maintain iteration speed. -// 2. Create benchmarks that can inform the design of the kernels. -// -// Non-goals: -// -// 1. Measure every distance operation. Instead measures just one distance -// operation at the same time. -// 2. Be useful for finding performance regressions. This is handled by the -// normal benchmarks. -// -// So far, both goals are partly achieved. -// -// RE (1), COMPILE TIMES: kernel.cu is fast to compile. This file is not. -// When the internals of a pairwise distance kernel is changed, this file is not -// recompiled. -// -// RE 2, benchmarks with intent: this file contains a benchmark to check the -// maximal throughput of a kernel. Measuring other things, like performance on -// skinny or wide matrices is not yet implemented. - -#include "kernel.cuh" // launch_kernel - -#include // RAFT_BENCH_REGISTER - -#include // pairwise_matrix_params - -#include // rmm::device_uvector - -#include // std::min -#include // std::vector - -namespace raft::bench::distance::tune { - -// Max throughput benchmark. -// -// Goal: Measure the maximum distances/sec that can be computed. -// -// To achieve this, we make sure that: -// -// - Input data size is a multiple of the block tile size. -// -// - Perfect distribution of work between SMs, i.e. the number of block tiles is -// a large multiple (num_waves) of the number of blocks (#SMs * occupancy). -// -// - Multiple iterations over Kblk are executed (num_k_iters). -struct throughput_param { - int num_waves; - int occupancy; - int num_k_iters; -}; - -const std::vector throughput_params{ - // 32 waves, requested occupancy of 4, and 32 k iterations typically achieves - // maximum throughput. No need to pick higher values. - {32, 4, 32}, -}; - -struct throughput_bench : public fixture { - const throughput_param p; - - throughput_bench(const throughput_param& p_) : p(p_) {} - - void run_benchmark(::benchmark::State& state) override - { - // Get block size: - int block_m, block_n, block_k; - get_block_size(block_m, block_n, block_k); - - // Determine number of blocks that will be launched. This informs the size - // of the inputs as well as the grid size. - const int num_sms = raft::getMultiProcessorCount(); - const int max_occupancy = get_max_occupancy(); - const int occupancy = std::min(p.occupancy, max_occupancy); - const int num_blocks = occupancy * num_sms; - dim3 grid(num_blocks); - - // Create input sizes that are a multiple of the block tile size. - size_t m = block_m; - size_t n = block_n * p.num_waves * num_blocks; - size_t k = block_k * p.num_k_iters; - - // DataT, OutT, IdxT, etc, are defined in tuned_kernel.cuh - rmm::device_uvector x_vec(m * k, stream); - rmm::device_uvector y_vec(n * k, stream); - rmm::device_uvector x_norm_vec(m, stream); - rmm::device_uvector y_norm_vec(n, stream); - rmm::device_uvector out_vec(m * n, stream); - - auto x = x_vec.data(); - auto y = y_vec.data(); - auto x_norm = x_norm_vec.data(); - auto y_norm = y_norm_vec.data(); - auto out = out_vec.data(); - FinOpT fin_op{}; - - // Create kernel parameter struct. Flip x and y if column major. - IdxT ldx = row_major ? k : m; - IdxT ldy = row_major ? k : n; - IdxT ld_out = row_major ? n : m; - - // Template parameters of pairwise_matrix_params are defined in kernel.cuh - pairwise_matrix_params kparams{ - IdxT(m), IdxT(n), IdxT(k), ldx, ldy, ld_out, x, y, x_norm, y_norm, out, fin_op, row_major}; - - // Run benchmark - loop_on_state(state, [&]() { launch_kernel(kparams, grid, stream); }); - - // Report metrics. We don't report flop/s because we do not know for each - // distance operation how many flops it costs. For L2_unexp and l1, we can - // double this number to get the flop/s. For l2 expanded, core_ops/s should - // equal flop/s (modulo the sqrt and subtracting from the norm). - size_t num_core_ops = m * n * k; - size_t read_elts = n * k + m * k; - size_t write_elts = m * n; - - state.counters["m"] = benchmark::Counter(m); - state.counters["n"] = benchmark::Counter(n); - state.counters["k"] = benchmark::Counter(k); - state.counters["occupancy"] = benchmark::Counter(occupancy); - state.counters["# waves"] = benchmark::Counter(p.num_waves); - state.counters["# k iters"] = benchmark::Counter(p.num_k_iters); - - state.counters["core_ops/s"] = benchmark::Counter(num_core_ops, - benchmark::Counter::kIsIterationInvariantRate, - benchmark::Counter::OneK::kIs1000); - - state.counters["BW"] = benchmark::Counter(write_elts * sizeof(OutT) + read_elts * sizeof(DataT), - benchmark::Counter::kIsIterationInvariantRate, - benchmark::Counter::OneK::kIs1000); - } -}; - -RAFT_BENCH_REGISTER(throughput_bench, "", throughput_params); - -} // namespace raft::bench::distance::tune diff --git a/cpp/bench/prims/distance/tune_pairwise/kernel.cu b/cpp/bench/prims/distance/tune_pairwise/kernel.cu deleted file mode 100644 index 42173c51f5..0000000000 --- a/cpp/bench/prims/distance/tune_pairwise/kernel.cu +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "kernel.cuh" - -#include // pairwise_matrix_sm60_wrapper -#include // raft::linalg::Policy4x4 -#include // raft::util::arch::SM_compute_arch - -namespace raft::bench::distance::tune { - -// Distance op -using OpT = raft::distance::detail::ops::lp_unexp_distance_op; -constexpr float metric_arg = 2.0; -OpT distance_op{metric_arg}; - -// Kernel policy -constexpr int vec_len = 1; -using Policy = typename raft::linalg::Policy4x4::Policy; - -// Architecture -namespace arch = raft::util::arch; -constexpr auto sm_compat_range = arch::SM_range(arch::SM_min(), arch::SM_future()); - -void launch_kernel(pairwise_matrix_params params, dim3 grid, cudaStream_t stream) -{ - dim3 block(Policy::Nthreads); - int smem_size = OpT::shared_mem_size(); - - // Obtain function pointer to kernel - auto kernel = raft::distance::detail::pairwise_matrix_kernel; - - kernel<<>>(distance_op, params); - RAFT_CUDA_TRY(cudaGetLastError()); -} - -void get_block_size(int& m, int& n, int& k) -{ - m = Policy::Mblk; - n = Policy::Nblk; - k = Policy::Kblk; -} - -void* get_kernel_ptr() -{ - auto kernel = raft::distance::detail::pairwise_matrix_kernel; - return reinterpret_cast(kernel); -} - -int get_max_occupancy() -{ - void* kernel_ptr = get_kernel_ptr(); - int max_occupancy; - int smem_size = OpT::shared_mem_size(); - - RAFT_CUDA_TRY(cudaOccupancyMaxActiveBlocksPerMultiprocessor( - &max_occupancy, kernel_ptr, Policy::Nthreads, smem_size)); - - return max_occupancy; -} - -} // namespace raft::bench::distance::tune diff --git a/cpp/bench/prims/distance/tune_pairwise/kernel.cuh b/cpp/bench/prims/distance/tune_pairwise/kernel.cuh deleted file mode 100644 index 5da54a343c..0000000000 --- a/cpp/bench/prims/distance/tune_pairwise/kernel.cuh +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // lp_unexp_distance_op -#include // pairwise_matrix_params - -namespace raft::bench::distance::tune { - -// Launch one specific kernel with the following template parameters -constexpr bool row_major = true; -using DataT = float; -using AccT = float; -using OutT = DataT; -using IdxT = int; - -using FinOpT = raft::identity_op; - -using pairwise_matrix_params = - raft::distance::detail::pairwise_matrix_params; - -// Launches kernel -void launch_kernel(pairwise_matrix_params, dim3, cudaStream_t); - -// Describes the block size that is decided by the policy -void get_block_size(int& m, int& n, int& k); - -int get_max_occupancy(); - -} // namespace raft::bench::distance::tune diff --git a/cpp/bench/prims/neighbors/cagra_bench.cuh b/cpp/bench/prims/neighbors/cagra_bench.cuh deleted file mode 100644 index acbeba375a..0000000000 --- a/cpp/bench/prims/neighbors/cagra_bench.cuh +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#include -#include -#include -#include - -#include - -#include - -namespace raft::bench::neighbors { - -struct params { - /** Size of the dataset. */ - size_t n_samples; - /** Number of dimensions in the dataset. */ - int n_dims; - /** The batch size -- number of KNN searches. */ - int n_queries; - /** Number of nearest neighbours to find for every probe. */ - int k; - /** kNN graph degree*/ - int degree; - int itopk_size; - int block_size; - int search_width; - int max_iterations; - /** Ratio of removed indices. */ - double removed_ratio; -}; - -template -struct CagraBench : public fixture { - explicit CagraBench(const params& ps) - : fixture(true), - params_(ps), - queries_(make_device_matrix(handle, ps.n_queries, ps.n_dims)), - dataset_(make_device_matrix(handle, ps.n_samples, ps.n_dims)), - knn_graph_(make_device_matrix(handle, ps.n_samples, ps.degree)), - removed_indices_bitset_(handle, ps.n_samples) - { - // Generate random dataset and queriees - raft::random::RngState state{42}; - constexpr T kRangeMax = std::is_integral_v ? std::numeric_limits::max() : T(1); - constexpr T kRangeMin = std::is_integral_v ? std::numeric_limits::min() : T(-1); - if constexpr (std::is_integral_v) { - raft::random::uniformInt( - handle, state, dataset_.data_handle(), dataset_.size(), kRangeMin, kRangeMax); - raft::random::uniformInt( - handle, state, queries_.data_handle(), queries_.size(), kRangeMin, kRangeMax); - } else { - raft::random::uniform( - handle, state, dataset_.data_handle(), dataset_.size(), kRangeMin, kRangeMax); - raft::random::uniform( - handle, state, queries_.data_handle(), queries_.size(), kRangeMin, kRangeMax); - } - - // Generate random knn graph - - raft::random::uniformInt( - handle, state, knn_graph_.data_handle(), knn_graph_.size(), 0, ps.n_samples - 1); - - auto metric = raft::distance::DistanceType::L2Expanded; - - auto removed_indices = - raft::make_device_vector(handle, ps.removed_ratio * ps.n_samples); - thrust::sequence( - resource::get_thrust_policy(handle), - thrust::device_pointer_cast(removed_indices.data_handle()), - thrust::device_pointer_cast(removed_indices.data_handle() + removed_indices.extent(0))); - removed_indices_bitset_.set(handle, removed_indices.view()); - index_.emplace(raft::neighbors::cagra::index( - handle, metric, make_const_mdspan(dataset_.view()), make_const_mdspan(knn_graph_.view()))); - } - - void run_benchmark(::benchmark::State& state) override - { - raft::neighbors::cagra::search_params search_params; - search_params.max_queries = 1024; - search_params.itopk_size = params_.itopk_size; - search_params.team_size = 0; - search_params.thread_block_size = params_.block_size; - search_params.search_width = params_.search_width; - - auto indices = make_device_matrix(handle, params_.n_queries, params_.k); - auto distances = make_device_matrix(handle, params_.n_queries, params_.k); - auto ind_v = make_device_matrix_view( - indices.data_handle(), params_.n_queries, params_.k); - auto dist_v = make_device_matrix_view( - distances.data_handle(), params_.n_queries, params_.k); - - auto queries_v = make_const_mdspan(queries_.view()); - if (params_.removed_ratio > 0) { - auto filter = raft::neighbors::filtering::bitset_filter(removed_indices_bitset_.view()); - loop_on_state(state, [&]() { - raft::neighbors::cagra::search_with_filtering( - this->handle, search_params, *this->index_, queries_v, ind_v, dist_v, filter); - }); - } else { - loop_on_state(state, [&]() { - raft::neighbors::cagra::search( - this->handle, search_params, *this->index_, queries_v, ind_v, dist_v); - }); - } - - double data_size = params_.n_samples * params_.n_dims * sizeof(T); - double graph_size = params_.n_samples * params_.degree * sizeof(IdxT); - - int iterations = params_.max_iterations; - if (iterations == 0) { - // see search_plan_impl::adjust_search_params() - double r = params_.itopk_size / static_cast(params_.search_width); - iterations = 1 + std::min(r * 1.1, r + 10); - } - state.counters["dataset (GiB)"] = data_size / (1 << 30); - state.counters["graph (GiB)"] = graph_size / (1 << 30); - state.counters["n_rows"] = params_.n_samples; - state.counters["n_cols"] = params_.n_dims; - state.counters["degree"] = params_.degree; - state.counters["n_queries"] = params_.n_queries; - state.counters["k"] = params_.k; - state.counters["itopk_size"] = params_.itopk_size; - state.counters["block_size"] = params_.block_size; - state.counters["search_width"] = params_.search_width; - state.counters["iterations"] = iterations; - state.counters["removed_ratio"] = params_.removed_ratio; - } - - private: - const params params_; - std::optional> index_; - raft::device_matrix queries_; - raft::device_matrix dataset_; - raft::device_matrix knn_graph_; - raft::core::bitset removed_indices_bitset_; -}; - -inline const std::vector generate_inputs() -{ - std::vector inputs = - raft::util::itertools::product({2000000ull}, // n_samples - {128, 256, 512, 1024}, // dataset dim - {1000}, // n_queries - {32}, // k - {64}, // knn graph degree - {64}, // itopk_size - {0}, // block_size - {1}, // search_width - {0}, // max_iterations - {0.0} // removed_ratio - ); - auto inputs2 = raft::util::itertools::product({2000000ull, 10000000ull}, // n_samples - {128}, // dataset dim - {1000}, // n_queries - {32}, // k - {64}, // knn graph degree - {64}, // itopk_size - {64, 128, 256, 512, 1024}, // block_size - {1}, // search_width - {0}, // max_iterations - {0.0} // removed_ratio - ); - inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); - - inputs2 = raft::util::itertools::product( - {2000000ull, 10000000ull}, // n_samples - {128}, // dataset dim - {1, 10, 10000}, // n_queries - {255}, // k - {64}, // knn graph degree - {300}, // itopk_size - {256}, // block_size - {2}, // search_width - {0}, // max_iterations - {0.0, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64} // removed_ratio - ); - inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); - return inputs; -} - -const std::vector kCagraInputs = generate_inputs(); - -#define CAGRA_REGISTER(ValT, IdxT, inputs) \ - namespace BENCHMARK_PRIVATE_NAME(knn) { \ - using AnnCagra = CagraBench; \ - RAFT_BENCH_REGISTER(AnnCagra, #ValT "/" #IdxT, inputs); \ - } - -} // namespace raft::bench::neighbors diff --git a/cpp/bench/prims/neighbors/knn.cuh b/cpp/bench/prims/neighbors/knn.cuh deleted file mode 100644 index 6499078623..0000000000 --- a/cpp/bench/prims/neighbors/knn.cuh +++ /dev/null @@ -1,516 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include - -#include - -namespace raft::bench::spatial { - -struct params { - /** Size of the dataset. */ - size_t n_samples; - /** Number of dimensions in the dataset. */ - size_t n_dims; - /** The batch size -- number of KNN searches. */ - size_t n_queries; - /** Number of nearest neighbours to find for every probe. */ - size_t k; - /** Ratio of removed indices. */ - double removed_ratio; -}; - -inline auto operator<<(std::ostream& os, const params& p) -> std::ostream& -{ - os << p.n_samples << "#" << p.n_dims << "#" << p.n_queries << "#" << p.k << "#" - << p.removed_ratio; - return os; -} - -enum class TransferStrategy { NO_COPY, COPY_PLAIN, COPY_PINNED, MAP_PINNED, MANAGED }; // NOLINT -enum class Scope { BUILD, SEARCH, BUILD_SEARCH }; // NOLINT - -inline auto operator<<(std::ostream& os, const TransferStrategy& ts) -> std::ostream& -{ - switch (ts) { - case TransferStrategy::NO_COPY: os << "NO_COPY"; break; - case TransferStrategy::COPY_PLAIN: os << "COPY_PLAIN"; break; - case TransferStrategy::COPY_PINNED: os << "COPY_PINNED"; break; - case TransferStrategy::MAP_PINNED: os << "MAP_PINNED"; break; - case TransferStrategy::MANAGED: os << "MANAGED"; break; - default: os << "UNKNOWN"; - } - return os; -} - -inline auto operator<<(std::ostream& os, const Scope& s) -> std::ostream& -{ - switch (s) { - case Scope::BUILD: os << "BUILD"; break; - case Scope::SEARCH: os << "SEARCH"; break; - case Scope::BUILD_SEARCH: os << "BUILD_SEARCH"; break; - default: os << "UNKNOWN"; - } - return os; -} - -struct device_resource { - public: - explicit device_resource(bool managed) : managed_(managed) - { - if (managed_) { - res_ = new rmm::mr::managed_memory_resource(); - } else { - res_ = rmm::mr::get_current_device_resource(); - } - } - - ~device_resource() - { - if (managed_) { delete res_; } - } - - [[nodiscard]] auto get() const -> rmm::device_async_resource_ref { return res_; } - - private: - const bool managed_; - rmm::mr::device_memory_resource* res_; -}; - -template -struct host_uvector { - host_uvector(size_t n, bool pinned) : n_(n) - { - if (pinned) { - res_ = new rmm::mr::pinned_memory_resource(); - } else { - res_ = new rmm::mr::new_delete_resource(); - } - arr_ = static_cast(res_->allocate(n_ * sizeof(T))); - } - - ~host_uvector() noexcept - { - res_->deallocate(arr_, n_ * sizeof(T)); - delete res_; - } - - auto data() -> T* { return arr_; } - [[nodiscard]] auto size() const -> size_t { return n_; } - - private: - rmm::mr::host_memory_resource* res_; - size_t n_; - T* arr_; -}; - -template -struct ivf_flat_knn { - using dist_t = float; - - std::optional> index; - raft::neighbors::ivf_flat::index_params index_params; - raft::neighbors::ivf_flat::search_params search_params; - params ps; - - ivf_flat_knn(const raft::device_resources& handle, const params& ps, const ValT* data) : ps(ps) - { - index_params.n_lists = 4096; - index_params.metric = raft::distance::DistanceType::L2Expanded; - index.emplace(raft::neighbors::ivf_flat::build( - handle, index_params, data, IdxT(ps.n_samples), uint32_t(ps.n_dims))); - } - - void search(const raft::device_resources& handle, - const ValT* search_items, - dist_t* out_dists, - IdxT* out_idxs) - { - search_params.n_probes = 20; - raft::neighbors::ivf_flat::search(handle, - search_params, - *index, - search_items, - ps.n_queries, - ps.k, - out_idxs, - out_dists, - resource::get_workspace_resource(handle)); - } -}; - -template -struct ivf_pq_knn { - using dist_t = float; - - std::optional> index; - raft::neighbors::ivf_pq::index_params index_params; - raft::neighbors::ivf_pq::search_params search_params; - params ps; - - ivf_pq_knn(const raft::device_resources& handle, const params& ps, const ValT* data) : ps(ps) - { - index_params.n_lists = 4096; - index_params.metric = raft::distance::DistanceType::L2Expanded; - auto data_view = raft::make_device_matrix_view(data, ps.n_samples, ps.n_dims); - index.emplace(raft::neighbors::ivf_pq::build(handle, index_params, data_view)); - } - - void search(const raft::device_resources& handle, - const ValT* search_items, - dist_t* out_dists, - IdxT* out_idxs) - { - search_params.n_probes = 20; - auto queries_view = - raft::make_device_matrix_view(search_items, ps.n_queries, ps.n_dims); - auto idxs_view = raft::make_device_matrix_view(out_idxs, ps.n_queries, ps.k); - auto dists_view = - raft::make_device_matrix_view(out_dists, ps.n_queries, ps.k); - raft::neighbors::ivf_pq::search( - handle, search_params, *index, queries_view, idxs_view, dists_view); - } -}; - -template -struct brute_force_knn { - using dist_t = ValT; - - ValT* index; - params ps; - - brute_force_knn(const raft::device_resources& handle, const params& ps, const ValT* data) - : index(const_cast(data)), ps(ps) - { - } - - void search(const raft::device_resources& handle, - const ValT* search_items, - dist_t* out_dists, - IdxT* out_idxs) - { - std::vector input{index}; - std::vector sizes{ps.n_samples}; - raft::spatial::knn::brute_force_knn(handle, - input, - sizes, - ps.n_dims, - const_cast(search_items), - ps.n_queries, - out_idxs, - out_dists, - ps.k); - } -}; - -template -struct ivf_flat_filter_knn { - using dist_t = float; - - std::optional> index; - raft::neighbors::ivf_flat::index_params index_params; - raft::neighbors::ivf_flat::search_params search_params; - raft::core::bitset removed_indices_bitset_; - params ps; - - ivf_flat_filter_knn(const raft::device_resources& handle, const params& ps, const ValT* data) - : ps(ps), removed_indices_bitset_(handle, ps.n_samples) - { - index_params.n_lists = 4096; - index_params.metric = raft::distance::DistanceType::L2Expanded; - index.emplace(raft::neighbors::ivf_flat::build( - handle, index_params, data, IdxT(ps.n_samples), uint32_t(ps.n_dims))); - auto removed_indices = - raft::make_device_vector(handle, ps.removed_ratio * ps.n_samples); - thrust::sequence( - resource::get_thrust_policy(handle), - thrust::device_pointer_cast(removed_indices.data_handle()), - thrust::device_pointer_cast(removed_indices.data_handle() + removed_indices.extent(0))); - removed_indices_bitset_.set(handle, removed_indices.view()); - } - - void search(const raft::device_resources& handle, - const ValT* search_items, - dist_t* out_dists, - IdxT* out_idxs) - { - search_params.n_probes = 20; - auto queries_view = - raft::make_device_matrix_view(search_items, ps.n_queries, ps.n_dims); - auto neighbors_view = raft::make_device_matrix_view(out_idxs, ps.n_queries, ps.k); - auto distance_view = raft::make_device_matrix_view(out_dists, ps.n_queries, ps.k); - auto filter = raft::neighbors::filtering::bitset_filter(removed_indices_bitset_.view()); - - if (ps.removed_ratio > 0) { - raft::neighbors::ivf_flat::search_with_filtering( - handle, search_params, *index, queries_view, neighbors_view, distance_view, filter); - } else { - raft::neighbors::ivf_flat::search( - handle, search_params, *index, queries_view, neighbors_view, distance_view); - } - } -}; - -template -struct ivf_pq_filter_knn { - using dist_t = float; - - std::optional> index; - raft::neighbors::ivf_pq::index_params index_params; - raft::neighbors::ivf_pq::search_params search_params; - raft::core::bitset removed_indices_bitset_; - params ps; - - ivf_pq_filter_knn(const raft::device_resources& handle, const params& ps, const ValT* data) - : ps(ps), removed_indices_bitset_(handle, ps.n_samples) - { - index_params.n_lists = 4096; - index_params.metric = raft::distance::DistanceType::L2Expanded; - auto data_view = raft::make_device_matrix_view(data, ps.n_samples, ps.n_dims); - index.emplace(raft::neighbors::ivf_pq::build(handle, index_params, data_view)); - auto removed_indices = - raft::make_device_vector(handle, ps.removed_ratio * ps.n_samples); - thrust::sequence( - resource::get_thrust_policy(handle), - thrust::device_pointer_cast(removed_indices.data_handle()), - thrust::device_pointer_cast(removed_indices.data_handle() + removed_indices.extent(0))); - removed_indices_bitset_.set(handle, removed_indices.view()); - } - - void search(const raft::device_resources& handle, - const ValT* search_items, - dist_t* out_dists, - IdxT* out_idxs) - { - search_params.n_probes = 20; - auto queries_view = - raft::make_device_matrix_view(search_items, ps.n_queries, ps.n_dims); - auto neighbors_view = - raft::make_device_matrix_view(out_idxs, ps.n_queries, ps.k); - auto distance_view = - raft::make_device_matrix_view(out_dists, ps.n_queries, ps.k); - auto filter = raft::neighbors::filtering::bitset_filter(removed_indices_bitset_.view()); - - if (ps.removed_ratio > 0) { - raft::neighbors::ivf_pq::search_with_filtering( - handle, search_params, *index, queries_view, neighbors_view, distance_view, filter); - } else { - raft::neighbors::ivf_pq::search( - handle, search_params, *index, queries_view, neighbors_view, distance_view); - } - } -}; - -template -struct knn : public fixture { - explicit knn(const params& p, const TransferStrategy& strategy, const Scope& scope) - : fixture(true), - params_(p), - strategy_(strategy), - scope_(scope), - dev_mem_res_(strategy == TransferStrategy::MANAGED), - data_host_(0), - search_items_(p.n_queries * p.n_dims, stream), - out_dists_(p.n_queries * p.k, stream), - out_idxs_(p.n_queries * p.k, stream) - { - raft::random::RngState state{42}; - gen_data(state, search_items_, search_items_.size(), stream); - try { - size_t total_size = p.n_samples * p.n_dims; - data_host_.resize(total_size); - constexpr size_t kGenMinibatchSize = 1024 * 1024 * 1024; - rmm::device_uvector d(std::min(kGenMinibatchSize, total_size), stream); - for (size_t offset = 0; offset < total_size; offset += kGenMinibatchSize) { - size_t actual_size = std::min(total_size - offset, kGenMinibatchSize); - gen_data(state, d, actual_size, stream); - copy(data_host_.data() + offset, d.data(), actual_size, stream); - } - } catch (std::bad_alloc& e) { - data_does_not_fit_ = true; - } - } - - template - void gen_data(raft::random::RngState& state, // NOLINT - rmm::device_uvector& vec, - size_t n, - rmm::cuda_stream_view stream) - { - constexpr T kRangeMax = std::is_integral_v ? std::numeric_limits::max() : T(1); - constexpr T kRangeMin = std::is_integral_v ? std::numeric_limits::min() : T(-1); - if constexpr (std::is_integral_v) { - raft::random::uniformInt(handle, state, vec.data(), n, kRangeMin, kRangeMax); - } else { - raft::random::uniform(handle, state, vec.data(), n, kRangeMin, kRangeMax); - } - } - - void run_benchmark(::benchmark::State& state) override - { - if (data_does_not_fit_) { - state.SkipWithError("The data size is too big to fit into the host memory."); - } - if (scope_ == Scope::SEARCH && strategy_ != TransferStrategy::NO_COPY) { - state.SkipWithError( - "When benchmarking without index building (Scope::SEARCH), the data must be already on the " - "device (TransferStrategy::NO_COPY)"); - } - - try { - std::ostringstream label_stream; - label_stream << params_ << "#" << strategy_ << "#" << scope_; - state.SetLabel(label_stream.str()); - raft::device_resources handle(stream); - std::optional index; - - if (scope_ == Scope::SEARCH) { // also implies TransferStrategy::NO_COPY - rmm::device_uvector data(data_host_.size(), stream); - copy(data.data(), data_host_.data(), data_host_.size(), stream); - index.emplace(handle, params_, data.data()); - stream.synchronize(); - } - - // benchmark loop - for (auto _ : state) { - // managed or plain device memory initialized anew every time - rmm::device_uvector data(data_host_.size(), stream, dev_mem_res_.get()); - ValT* data_ptr = data.data(); - size_t allocation_size = data_host_.size() * sizeof(ValT); - - // Non-benchmarked part: using different methods to copy the data if necessary - switch (strategy_) { - case TransferStrategy::NO_COPY: // copy data to GPU before starting the timer. - copy(data_ptr, data_host_.data(), data_host_.size(), stream); - break; - case TransferStrategy::COPY_PINNED: - RAFT_CUDA_TRY( - cudaHostRegister(data_host_.data(), allocation_size, cudaHostRegisterDefault)); - break; - case TransferStrategy::MAP_PINNED: - RAFT_CUDA_TRY( - cudaHostRegister(data_host_.data(), allocation_size, cudaHostRegisterMapped)); - RAFT_CUDA_TRY(cudaHostGetDevicePointer(&data_ptr, data_host_.data(), 0)); - break; - case TransferStrategy::MANAGED: // sic! using std::memcpy rather than cuda copy - RAFT_CUDA_TRY(cudaMemAdvise(data_ptr, - allocation_size, - cudaMemAdviseSetPreferredLocation, - resource::get_device_id(handle))); - RAFT_CUDA_TRY(cudaMemAdvise(data_ptr, - allocation_size, - cudaMemAdviseSetAccessedBy, - resource::get_device_id(handle))); - RAFT_CUDA_TRY(cudaMemAdvise(data_ptr, - allocation_size, - cudaMemAdviseSetReadMostly, - resource::get_device_id(handle))); - std::memcpy(data_ptr, data_host_.data(), allocation_size); - break; - default: break; - } - - flush_L2_cache(); - { - // Timer synchronizes the stream, so all prior gpu work should be done before it sets off. - cuda_event_timer timer(state, stream); - switch (strategy_) { - case TransferStrategy::COPY_PLAIN: - case TransferStrategy::COPY_PINNED: - copy(data_ptr, data_host_.data(), data_host_.size(), stream); - default: break; - } - - if (scope_ != Scope::SEARCH) { index.emplace(handle, params_, data_ptr); } - if (scope_ != Scope::BUILD) { - index->search(handle, search_items_.data(), out_dists_.data(), out_idxs_.data()); - } - } - - if (scope_ != Scope::SEARCH) { index.reset(); } - - switch (strategy_) { - case TransferStrategy::COPY_PINNED: - case TransferStrategy::MAP_PINNED: - RAFT_CUDA_TRY(cudaHostUnregister(data_host_.data())); - break; - default: break; - } - } - } catch (raft::exception& e) { - state.SkipWithError(e.what()); - } catch (std::bad_alloc& e) { - state.SkipWithError(e.what()); - } - } - - private: - const params params_; - const TransferStrategy strategy_; - const Scope scope_; - device_resource dev_mem_res_; - bool data_does_not_fit_ = false; - - std::vector data_host_; - rmm::device_uvector search_items_; - rmm::device_uvector out_dists_; - rmm::device_uvector out_idxs_; -}; - -inline const std::vector kInputs{ - {2000000, 128, 1000, 32, 0}, {10000000, 128, 1000, 32, 0}, {10000, 8192, 1000, 32, 0}}; - -const std::vector kInputsFilter = - raft::util::itertools::product({size_t(10000000)}, // n_samples - {size_t(128)}, // n_dim - {size_t(1000)}, // n_queries - {size_t(255)}, // k - {0.0, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64} // removed_ratio - ); -inline const std::vector kAllStrategies{ - TransferStrategy::NO_COPY, TransferStrategy::MAP_PINNED, TransferStrategy::MANAGED}; -inline const std::vector kNoCopyOnly{TransferStrategy::NO_COPY}; - -inline const std::vector kScopeFull{Scope::BUILD_SEARCH}; -inline const std::vector kAllScopes{Scope::BUILD_SEARCH, Scope::SEARCH, Scope::BUILD}; - -#define KNN_REGISTER(ValT, IdxT, ImplT, inputs, strats, scope) \ - namespace BENCHMARK_PRIVATE_NAME(knn) { \ - using KNN = knn>; \ - RAFT_BENCH_REGISTER(KNN, #ValT "/" #IdxT "/" #ImplT, inputs, strats, scope); \ - } - -} // namespace raft::bench::spatial diff --git a/cpp/bench/prims/neighbors/knn/brute_force_float_int64_t.cu b/cpp/bench/prims/neighbors/knn/brute_force_float_int64_t.cu deleted file mode 100644 index 7df0599670..0000000000 --- a/cpp/bench/prims/neighbors/knn/brute_force_float_int64_t.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../knn.cuh" - -namespace raft::bench::spatial { - -KNN_REGISTER(float, int64_t, brute_force_knn, kInputs, kAllStrategies, kScopeFull); - -} // namespace raft::bench::spatial diff --git a/cpp/bench/prims/neighbors/knn/brute_force_float_uint32_t.cu b/cpp/bench/prims/neighbors/knn/brute_force_float_uint32_t.cu deleted file mode 100644 index 9704d39e76..0000000000 --- a/cpp/bench/prims/neighbors/knn/brute_force_float_uint32_t.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../knn.cuh" - -namespace raft::bench::spatial { - -KNN_REGISTER(float, uint32_t, brute_force_knn, kInputs, kAllStrategies, kScopeFull); - -} // namespace raft::bench::spatial diff --git a/cpp/bench/prims/neighbors/knn/cagra_float_uint32_t.cu b/cpp/bench/prims/neighbors/knn/cagra_float_uint32_t.cu deleted file mode 100644 index 5d762f6e85..0000000000 --- a/cpp/bench/prims/neighbors/knn/cagra_float_uint32_t.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../cagra_bench.cuh" - -namespace raft::bench::neighbors { - -CAGRA_REGISTER(float, uint32_t, kCagraInputs); - -} // namespace raft::bench::neighbors diff --git a/cpp/bench/prims/neighbors/knn/ivf_flat_filter_float_int64_t.cu b/cpp/bench/prims/neighbors/knn/ivf_flat_filter_float_int64_t.cu deleted file mode 100644 index bf5118ceae..0000000000 --- a/cpp/bench/prims/neighbors/knn/ivf_flat_filter_float_int64_t.cu +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#undef RAFT_EXPLICIT_INSTANTIATE_ONLY // Enable instantiation of search with filter -#include "../knn.cuh" - -namespace raft::bench::spatial { - -KNN_REGISTER(float, int64_t, ivf_flat_filter_knn, kInputsFilter, kNoCopyOnly, kScopeFull); - -} // namespace raft::bench::spatial diff --git a/cpp/bench/prims/neighbors/knn/ivf_flat_float_int64_t.cu b/cpp/bench/prims/neighbors/knn/ivf_flat_float_int64_t.cu deleted file mode 100644 index fbbb4f9acc..0000000000 --- a/cpp/bench/prims/neighbors/knn/ivf_flat_float_int64_t.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../knn.cuh" - -namespace raft::bench::spatial { - -KNN_REGISTER(float, int64_t, ivf_flat_knn, kInputs, kNoCopyOnly, kAllScopes); - -} // namespace raft::bench::spatial diff --git a/cpp/bench/prims/neighbors/knn/ivf_flat_int8_t_int64_t.cu b/cpp/bench/prims/neighbors/knn/ivf_flat_int8_t_int64_t.cu deleted file mode 100644 index 7067dbe1b6..0000000000 --- a/cpp/bench/prims/neighbors/knn/ivf_flat_int8_t_int64_t.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../knn.cuh" - -namespace raft::bench::spatial { - -KNN_REGISTER(int8_t, int64_t, ivf_flat_knn, kInputs, kNoCopyOnly, kAllScopes); - -} // namespace raft::bench::spatial diff --git a/cpp/bench/prims/neighbors/knn/ivf_flat_uint8_t_int64_t.cu b/cpp/bench/prims/neighbors/knn/ivf_flat_uint8_t_int64_t.cu deleted file mode 100644 index 91fada3c28..0000000000 --- a/cpp/bench/prims/neighbors/knn/ivf_flat_uint8_t_int64_t.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../knn.cuh" - -namespace raft::bench::spatial { - -KNN_REGISTER(uint8_t, int64_t, ivf_flat_knn, kInputs, kNoCopyOnly, kAllScopes); - -} // namespace raft::bench::spatial diff --git a/cpp/bench/prims/neighbors/knn/ivf_pq_filter_float_int64_t.cu b/cpp/bench/prims/neighbors/knn/ivf_pq_filter_float_int64_t.cu deleted file mode 100644 index 1840eca99d..0000000000 --- a/cpp/bench/prims/neighbors/knn/ivf_pq_filter_float_int64_t.cu +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../knn.cuh" - -#include -#include -namespace raft::bench::spatial { - -KNN_REGISTER(float, int64_t, ivf_pq_filter_knn, kInputsFilter, kNoCopyOnly, kScopeFull); - -} // namespace raft::bench::spatial diff --git a/cpp/bench/prims/neighbors/knn/ivf_pq_float_int64_t.cu b/cpp/bench/prims/neighbors/knn/ivf_pq_float_int64_t.cu deleted file mode 100644 index 83c4973c3a..0000000000 --- a/cpp/bench/prims/neighbors/knn/ivf_pq_float_int64_t.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../knn.cuh" - -namespace raft::bench::spatial { - -KNN_REGISTER(float, int64_t, ivf_pq_knn, kInputs, kNoCopyOnly, kAllScopes); - -} // namespace raft::bench::spatial diff --git a/cpp/bench/prims/neighbors/knn/ivf_pq_int8_t_int64_t.cu b/cpp/bench/prims/neighbors/knn/ivf_pq_int8_t_int64_t.cu deleted file mode 100644 index 4ea281b11a..0000000000 --- a/cpp/bench/prims/neighbors/knn/ivf_pq_int8_t_int64_t.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../knn.cuh" - -namespace raft::bench::spatial { - -KNN_REGISTER(int8_t, int64_t, ivf_pq_knn, kInputs, kNoCopyOnly, kAllScopes); - -} // namespace raft::bench::spatial diff --git a/cpp/bench/prims/neighbors/knn/ivf_pq_uint8_t_int64_t.cu b/cpp/bench/prims/neighbors/knn/ivf_pq_uint8_t_int64_t.cu deleted file mode 100644 index 3313a49ba2..0000000000 --- a/cpp/bench/prims/neighbors/knn/ivf_pq_uint8_t_int64_t.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../knn.cuh" - -namespace raft::bench::spatial { - -KNN_REGISTER(uint8_t, int64_t, ivf_pq_knn, kInputs, kNoCopyOnly, kAllScopes); - -} // namespace raft::bench::spatial diff --git a/cpp/bench/prims/neighbors/refine.cuh b/cpp/bench/prims/neighbors/refine.cuh deleted file mode 100644 index 0360babd82..0000000000 --- a/cpp/bench/prims/neighbors/refine.cuh +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include - -#include -#include - -using namespace raft::neighbors; - -namespace raft::bench::neighbors { - -template -inline auto operator<<(std::ostream& os, const RefineInputs& p) -> std::ostream& -{ - os << p.n_rows << "#" << p.dim << "#" << p.n_queries << "#" << p.k0 << "#" << p.k << "#" - << (p.host_data ? "host" : "device"); - return os; -} - -template -class RefineAnn : public fixture { - public: - RefineAnn(RefineInputs p) : data(handle_, p) {} - - void run_benchmark(::benchmark::State& state) override - { - std::ostringstream label_stream; - label_stream << data.p; - state.SetLabel(label_stream.str()); - - auto old_mr = rmm::mr::get_current_device_resource(); - rmm::mr::pool_memory_resource pool_mr( - old_mr, rmm::percent_of_free_device_memory(50)); - rmm::mr::set_current_device_resource(&pool_mr); - - if (data.p.host_data) { - loop_on_state(state, [this]() { - raft::neighbors::refine(handle_, - data.dataset_host.view(), - data.queries_host.view(), - data.candidates_host.view(), - data.refined_indices_host.view(), - data.refined_distances_host.view(), - data.p.metric); - }); - } else { - loop_on_state(state, [&]() { - raft::neighbors::refine(handle_, - data.dataset.view(), - data.queries.view(), - data.candidates.view(), - data.refined_indices.view(), - data.refined_distances.view(), - data.p.metric); - }); - } - rmm::mr::set_current_device_resource(old_mr); - } - - private: - raft::device_resources handle_; - RefineHelper data; -}; - -template -std::vector> getInputs() -{ - std::vector> out; - raft::distance::DistanceType metric = raft::distance::DistanceType::L2Expanded; - for (bool host_data : {true, false}) { - for (T n_queries : {1000, 10000}) { - for (T dim : {128, 512}) { - out.push_back(RefineInputs{n_queries, 2000000, dim, 32, 128, metric, host_data}); - out.push_back(RefineInputs{n_queries, 2000000, dim, 10, 40, metric, host_data}); - } - } - } - return out; -} - -} // namespace raft::bench::neighbors diff --git a/cpp/bench/prims/neighbors/refine_float_int64_t.cu b/cpp/bench/prims/neighbors/refine_float_int64_t.cu deleted file mode 100644 index d69a157eca..0000000000 --- a/cpp/bench/prims/neighbors/refine_float_int64_t.cu +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "refine.cuh" - -#include - -using namespace raft::neighbors; - -namespace raft::bench::neighbors { -using refine_float_int64 = RefineAnn; -RAFT_BENCH_REGISTER(refine_float_int64, "", getInputs()); -} // namespace raft::bench::neighbors diff --git a/cpp/bench/prims/neighbors/refine_uint8_t_int64_t.cu b/cpp/bench/prims/neighbors/refine_uint8_t_int64_t.cu deleted file mode 100644 index 9da536b6c7..0000000000 --- a/cpp/bench/prims/neighbors/refine_uint8_t_int64_t.cu +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "refine.cuh" - -#include - -using namespace raft::neighbors; - -namespace raft::bench::neighbors { -using refine_uint8_int64 = RefineAnn; -RAFT_BENCH_REGISTER(refine_uint8_int64, "", getInputs()); -} // namespace raft::bench::neighbors diff --git a/cpp/cmake/patches/hnswlib.diff b/cpp/cmake/patches/hnswlib.diff deleted file mode 100644 index e7f89a8cc9..0000000000 --- a/cpp/cmake/patches/hnswlib.diff +++ /dev/null @@ -1,188 +0,0 @@ ---- a/hnswlib/hnswalg.h -+++ b/hnswlib/hnswalg.h -@@ -3,6 +3,7 @@ - #include "visited_list_pool.h" - #include "hnswlib.h" - #include -+#include - #include - #include - #include -@@ -16,6 +17,8 @@ namespace hnswlib { - template - class HierarchicalNSW : public AlgorithmInterface { - public: -+ bool base_layer_only{false}; -+ int num_seeds=32; - static const tableint max_update_element_locks = 65536; - HierarchicalNSW(SpaceInterface *s) { - } -@@ -56,7 +59,7 @@ namespace hnswlib { - visited_list_pool_ = new VisitedListPool(1, max_elements); - - //initializations for special treatment of the first node -- enterpoint_node_ = -1; -+ enterpoint_node_ = std::numeric_limits::max(); - maxlevel_ = -1; - - linkLists_ = (char **) malloc(sizeof(void *) * max_elements_); -@@ -527,7 +530,7 @@ namespace hnswlib { - tableint *datal = (tableint *) (data + 1); - for (int i = 0; i < size; i++) { - tableint cand = datal[i]; -- if (cand < 0 || cand > max_elements_) -+ if (cand > max_elements_) - throw std::runtime_error("cand error"); - dist_t d = fstdistfunc_(query_data, getDataByInternalId(cand), dist_func_param_); - -@@ -1067,7 +1070,7 @@ namespace hnswlib { - tableint *datal = (tableint *) (data + 1); - for (int i = 0; i < size; i++) { - tableint cand = datal[i]; -- if (cand < 0 || cand > max_elements_) -+ if (cand > max_elements_) - throw std::runtime_error("cand error"); - dist_t d = fstdistfunc_(data_point, getDataByInternalId(cand), dist_func_param_); - if (d < curdist) { -@@ -1119,28 +1122,41 @@ namespace hnswlib { - tableint currObj = enterpoint_node_; - dist_t curdist = fstdistfunc_(query_data, getDataByInternalId(enterpoint_node_), dist_func_param_); - -- for (int level = maxlevel_; level > 0; level--) { -- bool changed = true; -- while (changed) { -- changed = false; -- unsigned int *data; -+ if (base_layer_only) { -+ // You can increase the number of seeds when testing large-scale dataset, num_seeds = 48 for 100M-scale -+ for (int i = 0; i < num_seeds; i++) { -+ tableint obj = i * (max_elements_ / num_seeds); -+ dist_t dist = fstdistfunc_(query_data, getDataByInternalId(obj), dist_func_param_); -+ if (dist < curdist) { -+ curdist = dist; -+ currObj = obj; -+ } -+ } -+ } -+ else{ -+ for (int level = maxlevel_; level > 0; level--) { -+ bool changed = true; -+ while (changed) { -+ changed = false; -+ unsigned int *data; - -- data = (unsigned int *) get_linklist(currObj, level); -- int size = getListCount(data); -- metric_hops++; -- metric_distance_computations+=size; -+ data = (unsigned int *) get_linklist(currObj, level); -+ int size = getListCount(data); -+ metric_hops++; -+ metric_distance_computations+=size; - -- tableint *datal = (tableint *) (data + 1); -- for (int i = 0; i < size; i++) { -- tableint cand = datal[i]; -- if (cand < 0 || cand > max_elements_) -- throw std::runtime_error("cand error"); -- dist_t d = fstdistfunc_(query_data, getDataByInternalId(cand), dist_func_param_); -+ tableint *datal = (tableint *) (data + 1); -+ for (int i = 0; i < size; i++) { -+ tableint cand = datal[i]; -+ if (cand > max_elements_) -+ throw std::runtime_error("cand error"); -+ dist_t d = fstdistfunc_(query_data, getDataByInternalId(cand), dist_func_param_); - -- if (d < curdist) { -- curdist = d; -- currObj = cand; -- changed = true; -+ if (d < curdist) { -+ curdist = d; -+ currObj = cand; -+ changed = true; -+ } - } - } - } -diff --git a/hnswlib/space_l2.h b/hnswlib/space_l2.h -index 4413537..c3240f3 100644 ---- a/hnswlib/space_l2.h -+++ b/hnswlib/space_l2.h -@@ -252,13 +252,14 @@ namespace hnswlib { - ~L2Space() {} - }; - -+ template - static int - L2SqrI4x(const void *__restrict pVect1, const void *__restrict pVect2, const void *__restrict qty_ptr) { - - size_t qty = *((size_t *) qty_ptr); - int res = 0; -- unsigned char *a = (unsigned char *) pVect1; -- unsigned char *b = (unsigned char *) pVect2; -+ T *a = (T *) pVect1; -+ T *b = (T *) pVect2; - - qty = qty >> 2; - for (size_t i = 0; i < qty; i++) { -@@ -279,11 +280,12 @@ namespace hnswlib { - return (res); - } - -+ template - static int L2SqrI(const void* __restrict pVect1, const void* __restrict pVect2, const void* __restrict qty_ptr) { - size_t qty = *((size_t*)qty_ptr); - int res = 0; -- unsigned char* a = (unsigned char*)pVect1; -- unsigned char* b = (unsigned char*)pVect2; -+ T* a = (T*)pVect1; -+ T* b = (T*)pVect2; - - for(size_t i = 0; i < qty; i++) - { -@@ -294,6 +296,7 @@ namespace hnswlib { - return (res); - } - -+ template - class L2SpaceI : public SpaceInterface { - - DISTFUNC fstdistfunc_; -@@ -302,10 +305,10 @@ namespace hnswlib { - public: - L2SpaceI(size_t dim) { - if(dim % 4 == 0) { -- fstdistfunc_ = L2SqrI4x; -+ fstdistfunc_ = L2SqrI4x; - } - else { -- fstdistfunc_ = L2SqrI; -+ fstdistfunc_ = L2SqrI; - } - dim_ = dim; - data_size_ = dim * sizeof(unsigned char); -diff --git a/hnswlib/visited_list_pool.h b/hnswlib/visited_list_pool.h -index 5e1a4a5..4195ebd 100644 ---- a/hnswlib/visited_list_pool.h -+++ b/hnswlib/visited_list_pool.h -@@ -3,6 +3,7 @@ - #include - #include - #include -+#include - - namespace hnswlib { - typedef unsigned short int vl_type; -@@ -14,7 +15,7 @@ namespace hnswlib { - unsigned int numelements; - - VisitedList(int numelements1) { -- curV = -1; -+ curV = std::numeric_limits::max(); - numelements = numelements1; - mass = new vl_type[numelements]; - } --- -2.43.0 - diff --git a/cpp/cmake/patches/hnswlib_override.json b/cpp/cmake/patches/hnswlib_override.json deleted file mode 100644 index d6ab8a18a5..0000000000 --- a/cpp/cmake/patches/hnswlib_override.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "packages" : { - "hnswlib" : { - "version": "0.6.2", - "git_url": "https://github.com/nmslib/hnswlib.git", - "git_tag": "v${version}", - "patches" : [ - { - "file" : "${current_json_dir}/hnswlib.diff", - "issue" : "Correct compilation issues", - "fixed_in" : "" - } - ] - } - } -} diff --git a/cpp/cmake/thirdparty/get_hnswlib.cmake b/cpp/cmake/thirdparty/get_hnswlib.cmake deleted file mode 100644 index 6ef493336f..0000000000 --- a/cpp/cmake/thirdparty/get_hnswlib.cmake +++ /dev/null @@ -1,88 +0,0 @@ -#============================================================================= -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -function(find_and_configure_hnswlib) - set(oneValueArgs) - - include(${rapids-cmake-dir}/cpm/package_override.cmake) - set(patch_dir "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../patches") - rapids_cpm_package_override("${patch_dir}/hnswlib_override.json") - - include("${rapids-cmake-dir}/cpm/detail/package_details.cmake") - rapids_cpm_package_details(hnswlib version repository tag shallow exclude) - - include("${rapids-cmake-dir}/cpm/detail/generate_patch_command.cmake") - rapids_cpm_generate_patch_command(hnswlib ${version} patch_command) - - rapids_cpm_find( - hnswlib ${version} - GLOBAL_TARGETS hnswlib hnswlib::hnswlib - CPM_ARGS - GIT_REPOSITORY ${repository} - GIT_TAG ${tag} - GIT_SHALLOW ${shallow} ${patch_command} - EXCLUDE_FROM_ALL ${exclude} - DOWNLOAD_ONLY ON - ) - - include("${rapids-cmake-dir}/cpm/detail/display_patch_status.cmake") - rapids_cpm_display_patch_status(hnswlib) - - if(NOT TARGET hnswlib::hnswlib) - add_library(hnswlib INTERFACE ) - add_library(hnswlib::hnswlib ALIAS hnswlib) - target_include_directories(hnswlib INTERFACE - "$" - "$") - endif() - - if(hnswlib_ADDED) - # write build export rules - install(TARGETS hnswlib EXPORT hnswlib-exports) - if(NOT exclude) - install(DIRECTORY "${hnswlib_SOURCE_DIR}/hnswlib/" DESTINATION include/hnswlib) - - # write install export rules - rapids_export( - INSTALL hnswlib - VERSION ${version} - EXPORT_SET hnswlib-exports - GLOBAL_TARGETS hnswlib - NAMESPACE hnswlib::) - endif() - - rapids_export( - BUILD hnswlib - VERSION ${version} - EXPORT_SET hnswlib-exports - GLOBAL_TARGETS hnswlib - NAMESPACE hnswlib::) - - include("${rapids-cmake-dir}/export/package.cmake") - rapids_export_package(INSTALL hnswlib raft-exports VERSION ${version} GLOBAL_TARGETS hnswlib hnswlib::hnswlib) - rapids_export_package(BUILD hnswlib raft-exports VERSION ${version} GLOBAL_TARGETS hnswlib hnswlib::hnswlib) - - - # When using RAFT from the build dir, ensure hnswlib is also found in RAFT's build dir. This - # line adds `set(hnswlib_ROOT "${CMAKE_CURRENT_LIST_DIR}")` to build/raft-dependencies.cmake - include("${rapids-cmake-dir}/export/find_package_root.cmake") - rapids_export_find_package_root( - BUILD hnswlib [=[${CMAKE_CURRENT_LIST_DIR}]=] EXPORT_SET raft-exports - ) - endif() -endfunction() - -find_and_configure_hnswlib() diff --git a/cpp/include/raft/cluster/specializations.cuh b/cpp/include/raft/cluster/specializations.cuh deleted file mode 100644 index e85b05575f..0000000000 --- a/cpp/include/raft/cluster/specializations.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/distance/detail/pairwise_matrix/dispatch-ext.cuh b/cpp/include/raft/distance/detail/pairwise_matrix/dispatch-ext.cuh deleted file mode 100644 index bced721ec8..0000000000 --- a/cpp/include/raft/distance/detail/pairwise_matrix/dispatch-ext.cuh +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include // raft::identity_op -#include // ops::* -#include // ops::has_cutlass_op -#include // rbf_fin_op -#include // pairwise_matrix_params -#include // RAFT_EXPLICIT - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft::distance::detail { - -template -void pairwise_matrix_dispatch(OpT distance_op, - IdxT m, - IdxT n, - IdxT k, - const DataT* x, - const DataT* y, - const DataT* x_norm, - const DataT* y_norm, - OutT* out, - FinOpT fin_op, - cudaStream_t stream, - bool is_row_major) RAFT_EXPLICIT; - -}; // namespace raft::distance::detail - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - extern template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -/* - * Hierarchy of instantiations: - * - * This file defines extern template instantiations of the distance kernels. The - * instantiation of the public API is handled in raft/distance/distance-ext.cuh. - * - * After adding an instance here, make sure to also add the instance there. - */ - -// The following two instances are used in the RBF kernel object. Note the use of int64_t for the -// index type. -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_unexp_distance_op, - float, - float, - float, - raft::distance::kernels::detail::rbf_fin_op, - int64_t); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_unexp_distance_op, - double, - double, - double, - raft::distance::kernels::detail::rbf_fin_op, - int64_t); - -// Rest of instances -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::canberra_distance_op, float, float, float, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::canberra_distance_op, - double, - double, - double, - raft::identity_op, - int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::correlation_distance_op, - float, - float, - float, - raft::identity_op, - int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::correlation_distance_op, - double, - double, - double, - raft::identity_op, - int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::cosine_distance_op, float, float, float, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::cosine_distance_op, double, double, double, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::dice_distance_op, float, float, float, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::dice_distance_op, double, double, double, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::hamming_distance_op, float, float, float, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::hamming_distance_op, double, double, double, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::hellinger_distance_op, float, float, float, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::hellinger_distance_op, - double, - double, - double, - raft::identity_op, - int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::jensen_shannon_distance_op, - float, - float, - float, - raft::identity_op, - int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::jensen_shannon_distance_op, - double, - double, - double, - raft::identity_op, - int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::kl_divergence_op, float, float, float, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::kl_divergence_op, double, double, double, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l1_distance_op, float, float, float, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l1_distance_op, double, double, double, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_exp_distance_op, float, float, float, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_exp_distance_op, double, double, double, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_unexp_distance_op, float, float, float, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_unexp_distance_op, - double, - double, - double, - raft::identity_op, - int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l_inf_distance_op, float, float, float, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l_inf_distance_op, double, double, double, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::lp_unexp_distance_op, float, float, float, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::lp_unexp_distance_op, - double, - double, - double, - raft::identity_op, - int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::russel_rao_distance_op, float, float, float, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::russel_rao_distance_op, - double, - double, - double, - raft::identity_op, - int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/include/raft/distance/detail/pairwise_matrix/dispatch.cuh b/cpp/include/raft/distance/detail/pairwise_matrix/dispatch.cuh index 4a52b7ebe7..5bc70ca4aa 100644 --- a/cpp/include/raft/distance/detail/pairwise_matrix/dispatch.cuh +++ b/cpp/include/raft/distance/detail/pairwise_matrix/dispatch.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "dispatch-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "dispatch-ext.cuh" -#endif diff --git a/cpp/include/raft/distance/distance-ext.cuh b/cpp/include/raft/distance/distance-ext.cuh deleted file mode 100644 index dcbfbfdbc3..0000000000 --- a/cpp/include/raft/distance/distance-ext.cuh +++ /dev/null @@ -1,1117 +0,0 @@ -/* - * Copyright (c) 2018-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include // raft::device_matrix_view -#include // raft::identity_op -#include // raft::resources -#include // rbf_fin_op -#include // raft::distance::DistanceType -#include // RAFT_EXPLICIT - -#include // rmm::device_uvector - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft { -namespace distance { - -template -[[deprecated("Use cuVS instead")]] void distance(raft::resources const& handle, - const DataT* x, - const DataT* y, - OutT* dist, - IdxT m, - IdxT n, - IdxT k, - void* workspace, - size_t worksize, - FinalLambda fin_op, - bool isRowMajor = true, - DataT metric_arg = 2.0f) RAFT_EXPLICIT; - -template -[[deprecated("Use cuVS instead")]] void distance(raft::resources const& handle, - const DataT* x, - const DataT* y, - OutT* dist, - IdxT m, - IdxT n, - IdxT k, - void* workspace, - size_t worksize, - bool isRowMajor = true, - DataT metric_arg = 2.0f) RAFT_EXPLICIT; - -template -[[deprecated("Use cuVS instead")]] size_t getWorkspaceSize( - const DataT* x, const DataT* y, IdxT m, IdxT n, IdxT k) RAFT_EXPLICIT; - -template -size_t getWorkspaceSize(raft::device_matrix_view const& x, - raft::device_matrix_view const& y) RAFT_EXPLICIT; - -template -[[deprecated("Use cuVS instead")]] void distance(raft::resources const& handle, - const DataT* x, - const DataT* y, - OutT* dist, - IdxT m, - IdxT n, - IdxT k, - bool isRowMajor = true, - DataT metric_arg = 2.0f) RAFT_EXPLICIT; - -template -[[deprecated("Use cuVS instead")]] void pairwise_distance(raft::resources const& handle, - const Type* x, - const Type* y, - Type* dist, - IdxT m, - IdxT n, - IdxT k, - rmm::device_uvector& workspace, - raft::distance::DistanceType metric, - bool isRowMajor = true, - Type metric_arg = 2.0f) RAFT_EXPLICIT; - -template -[[deprecated("Use cuVS instead")]] void pairwise_distance(raft::resources const& handle, - const Type* x, - const Type* y, - Type* dist, - IdxT m, - IdxT n, - IdxT k, - raft::distance::DistanceType metric, - bool isRowMajor = true, - Type metric_arg = 2.0f) RAFT_EXPLICIT; - -template -[[deprecated("Use cuVS instead")]] void distance( - raft::resources const& handle, - raft::device_matrix_view const x, - raft::device_matrix_view const y, - raft::device_matrix_view dist, - DataT metric_arg = 2.0f) RAFT_EXPLICIT; - -template -[[deprecated("Use cuVS instead")]] void pairwise_distance( - raft::resources const& handle, - device_matrix_view const x, - device_matrix_view const y, - device_matrix_view dist, - raft::distance::DistanceType metric, - Type metric_arg = 2.0f) RAFT_EXPLICIT; - -}; // namespace distance -}; // namespace raft - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -/* - * Hierarchy of instantiations: - * - * This file defines the extern template instantiations for the public API of - * raft::distance. To improve compile times, the extern template instantiation - * of the distance kernels is handled in - * distance/detail/pairwise_matrix/dispatch-ext.cuh. - * - * After adding an instance here, make sure to also add the instance to - * dispatch-ext.cuh and the corresponding .cu files. - */ - -#define instantiate_raft_distance_distance(DT, DataT, AccT, OutT, FinalLambda, IdxT) \ - extern template void raft::distance::distance( \ - raft::resources const& handle, \ - const DataT* x, \ - const DataT* y, \ - OutT* dist, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - void* workspace, \ - size_t worksize, \ - FinalLambda fin_op, \ - bool isRowMajor, \ - DataT metric_arg) - -// The following two instances are used in test/distance/gram.cu. Note the use -// of int64_t for the index type. -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Unexpanded, - float, - float, - float, - raft::distance::kernels::detail::rbf_fin_op, - int64_t); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Unexpanded, - double, - double, - double, - raft::distance::kernels::detail::rbf_fin_op, - int64_t); - -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, - double, - double, - double, - raft::identity_op, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, double, double, double, raft::identity_op, int); - -#undef instantiate_raft_distance_distance - -// Same, but without raft::identity_op -#define instantiate_raft_distance_distance(DT, DataT, AccT, OutT, IdxT) \ - extern template void raft::distance::distance( \ - raft::resources const& handle, \ - const DataT* x, \ - const DataT* y, \ - OutT* dist, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - void* workspace, \ - size_t worksize, \ - bool isRowMajor, \ - DataT metric_arg) - -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, double, double, double, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L1, float, float, float, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L1, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, double, double, double, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::Linf, float, float, float, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::Linf, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, double, double, double, int); - -#undef instantiate_raft_distance_distance - -// Same, but without workspace -#define instantiate_raft_distance_distance(DT, DataT, AccT, OutT, IdxT) \ - extern template void raft::distance::distance( \ - raft::resources const& handle, \ - const DataT* x, \ - const DataT* y, \ - OutT* dist, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - bool isRowMajor, \ - DataT metric_arg) - -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, double, double, double, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L1, float, float, float, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L1, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, double, double, double, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::Linf, float, float, float, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::Linf, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, double, double, double, int); - -#undef instantiate_raft_distance_distance - -#define instantiate_raft_distance_getWorkspaceSize(DistT, DataT, AccT, OutT, IdxT) \ - extern template size_t raft::distance::getWorkspaceSize( \ - const DataT* x, const DataT* y, IdxT m, IdxT n, IdxT k) - -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::CorrelationExpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::CorrelationExpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::CosineExpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::CosineExpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::DiceExpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::DiceExpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::HammingUnexpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::HammingUnexpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::HellingerExpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::HellingerExpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::InnerProduct, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::InnerProduct, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::JensenShannon, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::JensenShannon, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::KLDivergence, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::KLDivergence, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2SqrtExpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2SqrtExpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2SqrtUnexpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2SqrtUnexpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Unexpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Linf, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Linf, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::LpUnexpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::LpUnexpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::RusselRaoExpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::RusselRaoExpanded, double, double, double, int); - -#undef instantiate_raft_distance_getWorkspaceSize - -#define instantiate_raft_distance_getWorkspaceSize(DistT, DataT, AccT, OutT, IdxT, layout) \ - extern template size_t raft::distance::getWorkspaceSize( \ - raft::device_matrix_view const& x, \ - raft::device_matrix_view const& y) - -// We could consider not taking template parameters for this function. The -// number of instantiations seems a bit excessive.. -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, double, double, double, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, double, double, double, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CorrelationExpanded, - float, - float, - float, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CorrelationExpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CorrelationExpanded, - float, - float, - float, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CorrelationExpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CosineExpanded, - float, - float, - float, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CosineExpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CosineExpanded, - float, - float, - float, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CosineExpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::DiceExpanded, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::DiceExpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::DiceExpanded, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::DiceExpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HammingUnexpanded, - float, - float, - float, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HammingUnexpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HammingUnexpanded, - float, - float, - float, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HammingUnexpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HellingerExpanded, - float, - float, - float, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HellingerExpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HellingerExpanded, - float, - float, - float, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HellingerExpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::InnerProduct, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::InnerProduct, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::InnerProduct, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::InnerProduct, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::JensenShannon, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::JensenShannon, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::JensenShannon, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::JensenShannon, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::KLDivergence, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::KLDivergence, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::KLDivergence, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::KLDivergence, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, double, double, double, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, double, double, double, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, double, double, double, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, double, double, double, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtExpanded, - float, - float, - float, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtExpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtExpanded, - float, - float, - float, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtExpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtUnexpanded, - float, - float, - float, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtUnexpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtUnexpanded, - float, - float, - float, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtUnexpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2Unexpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int, raft::layout_f_contiguous); - -#undef instantiate_raft_distance_getWorkspaceSize - -#define instantiate_raft_distance_pairwise_distance(DataT, IdxT) \ - extern template void raft::distance::pairwise_distance(raft::resources const& handle, \ - const DataT* x, \ - const DataT* y, \ - DataT* dist, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - rmm::device_uvector& workspace, \ - raft::distance::DistanceType metric, \ - bool isRowMajor, \ - DataT metric_arg) - -instantiate_raft_distance_pairwise_distance(float, int); -instantiate_raft_distance_pairwise_distance(double, int); - -#undef instantiate_raft_distance_pairwise_distance - -// Same, but without workspace -#define instantiate_raft_distance_pairwise_distance(DataT, IdxT) \ - extern template void raft::distance::pairwise_distance(raft::resources const& handle, \ - const DataT* x, \ - const DataT* y, \ - DataT* dist, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - raft::distance::DistanceType metric, \ - bool isRowMajor, \ - DataT metric_arg) - -instantiate_raft_distance_pairwise_distance(float, int); -instantiate_raft_distance_pairwise_distance(double, int); - -#undef instantiate_raft_distance_pairwise_distance - -// Version with mdspan -#define instantiate_raft_distance_distance(DistT, DataT, AccT, OutT, layout, IdxT) \ - extern template void raft::distance::distance( \ - raft::resources const& handle, \ - raft::device_matrix_view const x, \ - raft::device_matrix_view const y, \ - raft::device_matrix_view dist, \ - DataT metric_arg) - -// Again, we might want to consider reigning in the number of instantiations... -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CosineExpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CosineExpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CosineExpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CosineExpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::DiceExpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::DiceExpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HammingUnexpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HammingUnexpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HammingUnexpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HammingUnexpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HellingerExpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HellingerExpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HellingerExpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HellingerExpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::InnerProduct, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::InnerProduct, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::JensenShannon, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::JensenShannon, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::KLDivergence, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::KLDivergence, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, double, double, double, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, double, double, double, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtExpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtExpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtExpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtExpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtUnexpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtUnexpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtUnexpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtUnexpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Unexpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Unexpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, double, double, double, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, double, double, double, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::LpUnexpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::LpUnexpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::RusselRaoExpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::RusselRaoExpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::RusselRaoExpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::RusselRaoExpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); - -#undef instantiate_raft_distance_distance - -#define instantiate_raft_distance_pairwise_distance(DataT, layout, IdxT) \ - extern template void raft::distance::pairwise_distance( \ - raft::resources const& handle, \ - raft::device_matrix_view const x, \ - raft::device_matrix_view const y, \ - raft::device_matrix_view dist, \ - raft::distance::DistanceType metric, \ - DataT metric_arg) - -instantiate_raft_distance_pairwise_distance(float, raft::layout_c_contiguous, int); -instantiate_raft_distance_pairwise_distance(float, raft::layout_f_contiguous, int); -instantiate_raft_distance_pairwise_distance(double, raft::layout_c_contiguous, int); -instantiate_raft_distance_pairwise_distance(double, raft::layout_f_contiguous, int); - -#undef instantiate_raft_distance_pairwise_distance diff --git a/cpp/include/raft/distance/distance.cuh b/cpp/include/raft/distance/distance.cuh index de70cd4691..dc81e22b18 100644 --- a/cpp/include/raft/distance/distance.cuh +++ b/cpp/include/raft/distance/distance.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "distance-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "distance-ext.cuh" -#endif diff --git a/cpp/include/raft/distance/fused_l2_nn-ext.cuh b/cpp/include/raft/distance/fused_l2_nn-ext.cuh deleted file mode 100644 index d0ac83cd51..0000000000 --- a/cpp/include/raft/distance/fused_l2_nn-ext.cuh +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // raft::KeyValuePair -#include // raft::resources -#include // include initialize and reduce operations -#include // RAFT_EXPLICIT - -#include // int64_t - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft { -namespace distance { - -template -void fusedL2NNMinReduce(OutT* min, - const DataT* x, - const DataT* y, - const DataT* xn, - const DataT* yn, - IdxT m, - IdxT n, - IdxT k, - void* workspace, - bool sqrt, - bool initOutBuffer, - cudaStream_t stream) RAFT_EXPLICIT; - -} // namespace distance -} // namespace raft - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_distance_fusedL2NNMinReduce(DataT, OutT, IdxT) \ - extern template void raft::distance::fusedL2NNMinReduce(OutT * min, \ - const DataT* x, \ - const DataT* y, \ - const DataT* xn, \ - const DataT* yn, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - void* workspace, \ - bool sqrt, \ - bool initOutBuffer, \ - cudaStream_t stream) - -instantiate_raft_distance_fusedL2NNMinReduce(double, double, int); -instantiate_raft_distance_fusedL2NNMinReduce(double, double, int64_t); -instantiate_raft_distance_fusedL2NNMinReduce(float, float, int); -instantiate_raft_distance_fusedL2NNMinReduce(float, float, int64_t); - -// We can't have comma's in the macro expansion, so we use the COMMA macro: -#define COMMA , - -instantiate_raft_distance_fusedL2NNMinReduce(double, raft::KeyValuePair, int); -instantiate_raft_distance_fusedL2NNMinReduce(double, - raft::KeyValuePair, - int64_t); -instantiate_raft_distance_fusedL2NNMinReduce(float, raft::KeyValuePair, int); -instantiate_raft_distance_fusedL2NNMinReduce(float, - raft::KeyValuePair, - int64_t); - -#undef COMMA - -#undef instantiate_raft_distance_fusedL2NNMinReduce diff --git a/cpp/include/raft/distance/fused_l2_nn.cuh b/cpp/include/raft/distance/fused_l2_nn.cuh index b1a3551323..c17439f942 100644 --- a/cpp/include/raft/distance/fused_l2_nn.cuh +++ b/cpp/include/raft/distance/fused_l2_nn.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "fused_l2_nn-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "fused_l2_nn-ext.cuh" -#endif diff --git a/cpp/include/raft/distance/specializations.cuh b/cpp/include/raft/distance/specializations.cuh deleted file mode 100644 index cba059154f..0000000000 --- a/cpp/include/raft/distance/specializations.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/distance/specializations/distance.cuh b/cpp/include/raft/distance/specializations/distance.cuh deleted file mode 100644 index cba059154f..0000000000 --- a/cpp/include/raft/distance/specializations/distance.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/distance/specializations/fused_l2_nn_min.cuh b/cpp/include/raft/distance/specializations/fused_l2_nn_min.cuh deleted file mode 100644 index e85b05575f..0000000000 --- a/cpp/include/raft/distance/specializations/fused_l2_nn_min.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/matrix/detail/select_k-ext.cuh b/cpp/include/raft/matrix/detail/select_k-ext.cuh deleted file mode 100644 index 6db1a5acac..0000000000 --- a/cpp/include/raft/matrix/detail/select_k-ext.cuh +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include // RAFT_EXPLICIT - -#include // __half - -#include // uint32_t - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft::matrix::detail { - -template -void select_k(raft::resources const& handle, - const T* in_val, - const IdxT* in_idx, - size_t batch_size, - size_t len, - int k, - T* out_val, - IdxT* out_idx, - bool select_min, - bool sorted = false, - SelectAlgo algo = SelectAlgo::kAuto, - const IdxT* len_i = nullptr) RAFT_EXPLICIT; -} // namespace raft::matrix::detail - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_matrix_detail_select_k(T, IdxT) \ - extern template void raft::matrix::detail::select_k(raft::resources const& handle, \ - const T* in_val, \ - const IdxT* in_idx, \ - size_t batch_size, \ - size_t len, \ - int k, \ - T* out_val, \ - IdxT* out_idx, \ - bool select_min, \ - bool sorted, \ - raft::matrix::SelectAlgo algo, \ - const IdxT* len_i) -instantiate_raft_matrix_detail_select_k(__half, uint32_t); -instantiate_raft_matrix_detail_select_k(__half, int64_t); -instantiate_raft_matrix_detail_select_k(float, int64_t); -instantiate_raft_matrix_detail_select_k(float, uint32_t); -// needed for brute force knn -instantiate_raft_matrix_detail_select_k(float, int); -// We did not have these two for double before, but there are tests for them. We -// therefore include them here. -instantiate_raft_matrix_detail_select_k(double, int64_t); -instantiate_raft_matrix_detail_select_k(double, uint32_t); - -#undef instantiate_raft_matrix_detail_select_k diff --git a/cpp/include/raft/matrix/detail/select_k.cuh b/cpp/include/raft/matrix/detail/select_k.cuh index 711169984b..31a4b54a94 100644 --- a/cpp/include/raft/matrix/detail/select_k.cuh +++ b/cpp/include/raft/matrix/detail/select_k.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "select_k-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "select_k-ext.cuh" -#endif diff --git a/cpp/include/raft/matrix/specializations/detail/select_k.cuh b/cpp/include/raft/matrix/specializations/detail/select_k.cuh deleted file mode 100644 index c61d65dcaf..0000000000 --- a/cpp/include/raft/matrix/specializations/detail/select_k.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/neighbors/ball_cover-ext.cuh b/cpp/include/raft/neighbors/ball_cover-ext.cuh deleted file mode 100644 index 2a83a57749..0000000000 --- a/cpp/include/raft/neighbors/ball_cover-ext.cuh +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include // raft::distance::DistanceType -#include // BallCoverIndex -#include // RAFT_EXPLICIT - -#include // uint32_t - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft::neighbors::ball_cover { - -template -void build_index(raft::resources const& handle, - BallCoverIndex& index) RAFT_EXPLICIT; - -template -void all_knn_query(raft::resources const& handle, - BallCoverIndex& index, - int_t k, - idx_t* inds, - value_t* dists, - bool perform_post_filtering = true, - float weight = 1.0) RAFT_EXPLICIT; - -template -void all_knn_query(raft::resources const& handle, - BallCoverIndex& index, - raft::device_matrix_view inds, - raft::device_matrix_view dists, - int_t k, - bool perform_post_filtering = true, - float weight = 1.0) RAFT_EXPLICIT; - -template -void knn_query(raft::resources const& handle, - const BallCoverIndex& index, - int_t k, - const value_t* query, - int_t n_query_pts, - idx_t* inds, - value_t* dists, - bool perform_post_filtering = true, - float weight = 1.0) RAFT_EXPLICIT; - -template -void knn_query(raft::resources const& handle, - const BallCoverIndex& index, - raft::device_matrix_view query, - raft::device_matrix_view inds, - raft::device_matrix_view dists, - int_t k, - bool perform_post_filtering = true, - float weight = 1.0) RAFT_EXPLICIT; - -template -void eps_nn(raft::resources const& handle, - const BallCoverIndex& index, - raft::device_matrix_view adj, - raft::device_vector_view vd, - raft::device_matrix_view query, - value_t eps) RAFT_EXPLICIT; - -template -void eps_nn(raft::resources const& handle, - const BallCoverIndex& index, - raft::device_vector_view adj_ia, - raft::device_vector_view adj_ja, - raft::device_vector_view vd, - raft::device_matrix_view query, - value_t eps, - std::optional> max_k = std::nullopt) - RAFT_EXPLICIT; - -} // namespace raft::neighbors::ball_cover - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_neighbors_ball_cover(idx_t, value_t, int_t, matrix_idx_t) \ - extern template void \ - raft::neighbors::ball_cover::build_index( \ - raft::resources const& handle, \ - raft::neighbors::ball_cover::BallCoverIndex& index); \ - \ - extern template void \ - raft::neighbors::ball_cover::all_knn_query( \ - raft::resources const& handle, \ - raft::neighbors::ball_cover::BallCoverIndex& index, \ - int_t k, \ - idx_t* inds, \ - value_t* dists, \ - bool perform_post_filtering, \ - float weight); \ - \ - extern template void raft::neighbors::ball_cover::eps_nn( \ - raft::resources const& handle, \ - const raft::neighbors::ball_cover::BallCoverIndex& index, \ - raft::device_matrix_view adj, \ - raft::device_vector_view vd, \ - raft::device_matrix_view query, \ - value_t eps); \ - \ - extern template void raft::neighbors::ball_cover::eps_nn( \ - raft::resources const& handle, \ - const raft::neighbors::ball_cover::BallCoverIndex& index, \ - raft::device_vector_view adj_ia, \ - raft::device_vector_view adj_ja, \ - raft::device_vector_view vd, \ - raft::device_matrix_view query, \ - value_t eps, \ - std::optional> max_k); \ - \ - extern template void \ - raft::neighbors::ball_cover::all_knn_query( \ - raft::resources const& handle, \ - raft::neighbors::ball_cover::BallCoverIndex& index, \ - raft::device_matrix_view inds, \ - raft::device_matrix_view dists, \ - int_t k, \ - bool perform_post_filtering, \ - float weight); \ - \ - extern template void raft::neighbors::ball_cover::knn_query( \ - raft::resources const& handle, \ - const raft::neighbors::ball_cover::BallCoverIndex& index, \ - int_t k, \ - const value_t* query, \ - int_t n_query_pts, \ - idx_t* inds, \ - value_t* dists, \ - bool perform_post_filtering, \ - float weight); \ - \ - extern template void \ - raft::neighbors::ball_cover::knn_query( \ - raft::resources const& handle, \ - const raft::neighbors::ball_cover::BallCoverIndex& index, \ - raft::device_matrix_view query, \ - raft::device_matrix_view inds, \ - raft::device_matrix_view dists, \ - int_t k, \ - bool perform_post_filtering, \ - float weight); - -instantiate_raft_neighbors_ball_cover(int64_t, float, int64_t, int64_t); - -#undef instantiate_raft_neighbors_ball_cover diff --git a/cpp/include/raft/neighbors/ball_cover.cuh b/cpp/include/raft/neighbors/ball_cover.cuh index 09938020b9..cc74113f7e 100644 --- a/cpp/include/raft/neighbors/ball_cover.cuh +++ b/cpp/include/raft/neighbors/ball_cover.cuh @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "ball_cover-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "ball_cover-ext.cuh" -#endif diff --git a/cpp/include/raft/neighbors/brute_force-ext.cuh b/cpp/include/raft/neighbors/brute_force-ext.cuh deleted file mode 100644 index 4055c253c8..0000000000 --- a/cpp/include/raft/neighbors/brute_force-ext.cuh +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Copyright (c) 2020-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // raft::device_matrix_view -#include // raft::identity_op -#include // raft::resources -#include // raft::distance::DistanceType -#include -#include // RAFT_EXPLICIT - -#include - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft::neighbors::brute_force { - -template -inline void knn_merge_parts( - raft::resources const& handle, - raft::device_matrix_view in_keys, - raft::device_matrix_view in_values, - raft::device_matrix_view out_keys, - raft::device_matrix_view out_values, - size_t n_samples, - std::optional> translations = std::nullopt) RAFT_EXPLICIT; - -template -index build(raft::resources const& res, - mdspan, row_major, Accessor> dataset, - raft::distance::DistanceType metric = distance::DistanceType::L2Unexpanded, - T metric_arg = 0.0) RAFT_EXPLICIT; - -template -index build(raft::resources const& res, - index_params const& params, - mdspan, row_major, Accessor> dataset) RAFT_EXPLICIT; - -template -void search(raft::resources const& res, - const index& idx, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances) RAFT_EXPLICIT; - -template -void search(raft::resources const& res, - search_params const& params, - const index& idx, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances) RAFT_EXPLICIT; - -template -void knn(raft::resources const& handle, - std::vector> index, - raft::device_matrix_view search, - raft::device_matrix_view indices, - raft::device_matrix_view distances, - distance::DistanceType metric = distance::DistanceType::L2Unexpanded, - std::optional metric_arg = std::make_optional(2.0f), - std::optional global_id_offset = std::nullopt, - epilogue_op distance_epilogue = raft::identity_op()) RAFT_EXPLICIT; - -template -void fused_l2_knn(raft::resources const& handle, - raft::device_matrix_view index, - raft::device_matrix_view query, - raft::device_matrix_view out_inds, - raft::device_matrix_view out_dists, - raft::distance::DistanceType metric) RAFT_EXPLICIT; - -} // namespace raft::neighbors::brute_force - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -// No extern template for raft::neighbors::brute_force::knn_merge_parts - -#define instantiate_raft_neighbors_brute_force_knn( \ - idx_t, value_t, matrix_idx, index_layout, search_layout, epilogue_op) \ - extern template void raft::neighbors::brute_force:: \ - knn( \ - raft::resources const& handle, \ - std::vector> index, \ - raft::device_matrix_view search, \ - raft::device_matrix_view indices, \ - raft::device_matrix_view distances, \ - raft::distance::DistanceType metric, \ - std::optional metric_arg, \ - std::optional global_id_offset, \ - epilogue_op distance_epilogue); - -instantiate_raft_neighbors_brute_force_knn( - int64_t, float, uint32_t, raft::row_major, raft::row_major, raft::identity_op); -instantiate_raft_neighbors_brute_force_knn( - int64_t, float, int64_t, raft::row_major, raft::row_major, raft::identity_op); -instantiate_raft_neighbors_brute_force_knn( - int, float, int, raft::row_major, raft::row_major, raft::identity_op); -instantiate_raft_neighbors_brute_force_knn( - uint32_t, float, uint32_t, raft::row_major, raft::row_major, raft::identity_op); - -#undef instantiate_raft_neighbors_brute_force_knn - -namespace raft::neighbors::brute_force { - -extern template void search( - raft::resources const& res, - const raft::neighbors::brute_force::index& idx, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances); - -extern template void search( - raft::resources const& res, - search_params const& params, - const raft::neighbors::brute_force::index& idx, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances); - -extern template void search( - raft::resources const& res, - const raft::neighbors::brute_force::index& idx, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances); - -extern template void search( - raft::resources const& res, - search_params const& params, - const raft::neighbors::brute_force::index& idx, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances); - -extern template raft::neighbors::brute_force::index build( - raft::resources const& res, - raft::device_matrix_view dataset, - raft::distance::DistanceType metric, - float metric_arg); - -extern template raft::neighbors::brute_force::index build( - raft::resources const& res, - index_params const& params, - raft::device_matrix_view dataset); - -extern template raft::neighbors::brute_force::index build( - raft::resources const& res, - raft::host_matrix_view dataset, - raft::distance::DistanceType metric, - float metric_arg); - -extern template raft::neighbors::brute_force::index build( - raft::resources const& res, - index_params const& params, - raft::host_matrix_view dataset); -} // namespace raft::neighbors::brute_force - -#define instantiate_raft_neighbors_brute_force_fused_l2_knn( \ - value_t, idx_t, idx_layout, query_layout) \ - extern template void raft::neighbors::brute_force::fused_l2_knn( \ - raft::resources const& handle, \ - raft::device_matrix_view index, \ - raft::device_matrix_view query, \ - raft::device_matrix_view out_inds, \ - raft::device_matrix_view out_dists, \ - raft::distance::DistanceType metric); - -instantiate_raft_neighbors_brute_force_fused_l2_knn(float, - int64_t, - raft::row_major, - raft::row_major) - -#undef instantiate_raft_neighbors_brute_force_fused_l2_knn diff --git a/cpp/include/raft/neighbors/brute_force.cuh b/cpp/include/raft/neighbors/brute_force.cuh index 331ea55540..d01b7052cb 100644 --- a/cpp/include/raft/neighbors/brute_force.cuh +++ b/cpp/include/raft/neighbors/brute_force.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,18 +14,12 @@ * limitations under the License. */ #pragma once -#include - -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "brute_force-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "brute_force-ext.cuh" -#endif #include +#include + namespace raft::neighbors::brute_force { /** * @brief Make a brute force query over batches of k diff --git a/cpp/include/raft/neighbors/ivf_flat-ext.cuh b/cpp/include/raft/neighbors/ivf_flat-ext.cuh deleted file mode 100644 index 12ab0dc3a6..0000000000 --- a/cpp/include/raft/neighbors/ivf_flat-ext.cuh +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // raft::device_matrix_view -#include // raft::resources -#include -#include // raft::neighbors::ivf_flat::index -#include // RAFT_EXPLICIT - -#include - -#include // int64_t - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft::neighbors::ivf_flat { - -template -auto build(raft::resources const& handle, - const index_params& params, - const T* dataset, - IdxT n_rows, - uint32_t dim) -> index RAFT_EXPLICIT; - -template -auto build(raft::resources const& handle, - const index_params& params, - raft::device_matrix_view dataset) - -> index RAFT_EXPLICIT; - -template -void build(raft::resources const& handle, - const index_params& params, - raft::device_matrix_view dataset, - raft::neighbors::ivf_flat::index& idx) RAFT_EXPLICIT; - -template -auto build(raft::resources const& handle, - const index_params& params, - raft::host_matrix_view dataset) - -> index RAFT_EXPLICIT; - -template -void build(raft::resources const& handle, - const index_params& params, - raft::host_matrix_view dataset, - raft::neighbors::ivf_flat::index& idx) RAFT_EXPLICIT; - -template -auto extend(raft::resources const& handle, - const index& orig_index, - const T* new_vectors, - const IdxT* new_indices, - IdxT n_rows) -> index RAFT_EXPLICIT; - -template -auto extend(raft::resources const& handle, - raft::device_matrix_view new_vectors, - std::optional> new_indices, - const index& orig_index) -> index RAFT_EXPLICIT; - -template -void extend(raft::resources const& handle, - index* index, - const T* new_vectors, - const IdxT* new_indices, - IdxT n_rows) RAFT_EXPLICIT; - -template -void extend(raft::resources const& handle, - raft::device_matrix_view new_vectors, - std::optional> new_indices, - index* index) RAFT_EXPLICIT; - -template -auto extend(raft::resources const& handle, - raft::host_matrix_view new_vectors, - std::optional> new_indices, - const raft::neighbors::ivf_flat::index& orig_index) - -> raft::neighbors::ivf_flat::index RAFT_EXPLICIT; - -template -void extend(raft::resources const& handle, - raft::host_matrix_view new_vectors, - std::optional> new_indices, - index* index) RAFT_EXPLICIT; - -template -void search_with_filtering(raft::resources const& handle, - const search_params& params, - const index& index, - const T* queries, - uint32_t n_queries, - uint32_t k, - IdxT* neighbors, - float* distances, - rmm::device_async_resource_ref mr, - IvfSampleFilterT sample_filter = IvfSampleFilterT()) RAFT_EXPLICIT; - -template -void search(raft::resources const& handle, - const search_params& params, - const index& index, - const T* queries, - uint32_t n_queries, - uint32_t k, - IdxT* neighbors, - float* distances, - rmm::device_async_resource_ref mr) RAFT_EXPLICIT; - -template -void search_with_filtering(raft::resources const& handle, - const search_params& params, - const index& index, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances, - IvfSampleFilterT sample_filter = IvfSampleFilterT()) RAFT_EXPLICIT; - -template -void search(raft::resources const& handle, - const search_params& params, - const index& index, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances) RAFT_EXPLICIT; - -} // namespace raft::neighbors::ivf_flat - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_neighbors_ivf_flat_build(T, IdxT) \ - extern template auto raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - const T* dataset, \ - IdxT n_rows, \ - uint32_t dim) \ - ->raft::neighbors::ivf_flat::index; \ - \ - extern template auto raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::device_matrix_view dataset) \ - ->raft::neighbors::ivf_flat::index; \ - \ - extern template void raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::device_matrix_view dataset, \ - raft::neighbors::ivf_flat::index& idx); \ - \ - extern template auto raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::host_matrix_view dataset) \ - ->raft::neighbors::ivf_flat::index; \ - \ - extern template void raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::host_matrix_view dataset, \ - raft::neighbors::ivf_flat::index& idx); - -instantiate_raft_neighbors_ivf_flat_build(float, int64_t); -instantiate_raft_neighbors_ivf_flat_build(int8_t, int64_t); -instantiate_raft_neighbors_ivf_flat_build(uint8_t, int64_t); -#undef instantiate_raft_neighbors_ivf_flat_build - -#define instantiate_raft_neighbors_ivf_flat_extend(T, IdxT) \ - extern template auto raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index& orig_index, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows) \ - ->raft::neighbors::ivf_flat::index; \ - \ - extern template auto raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_flat::index& orig_index) \ - ->raft::neighbors::ivf_flat::index; \ - \ - extern template void raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::neighbors::ivf_flat::index* index, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows); \ - \ - extern template void raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_flat::index* index); \ - \ - extern template void raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::host_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_flat::index* index); \ - \ - extern template auto raft::neighbors::ivf_flat::extend( \ - const raft::resources& handle, \ - raft::host_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_flat::index& idx) \ - ->raft::neighbors::ivf_flat::index; - -instantiate_raft_neighbors_ivf_flat_extend(float, int64_t); -instantiate_raft_neighbors_ivf_flat_extend(int8_t, int64_t); -instantiate_raft_neighbors_ivf_flat_extend(uint8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_flat_extend - -#define instantiate_raft_neighbors_ivf_flat_search(T, IdxT) \ - extern template void raft::neighbors::ivf_flat::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::search_params& params, \ - const raft::neighbors::ivf_flat::index& index, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances, \ - rmm::device_async_resource_ref mr); \ - \ - extern template void raft::neighbors::ivf_flat::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::search_params& params, \ - const raft::neighbors::ivf_flat::index& index, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances); - -instantiate_raft_neighbors_ivf_flat_search(float, int64_t); -instantiate_raft_neighbors_ivf_flat_search(int8_t, int64_t); -instantiate_raft_neighbors_ivf_flat_search(uint8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_flat_search diff --git a/cpp/include/raft/neighbors/ivf_flat.cuh b/cpp/include/raft/neighbors/ivf_flat.cuh index 8fd9628a41..d5c8f26fb3 100644 --- a/cpp/include/raft/neighbors/ivf_flat.cuh +++ b/cpp/include/raft/neighbors/ivf_flat.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "ivf_flat-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "ivf_flat-ext.cuh" -#endif diff --git a/cpp/include/raft/neighbors/ivf_pq-ext.cuh b/cpp/include/raft/neighbors/ivf_pq-ext.cuh deleted file mode 100644 index 620f4a244f..0000000000 --- a/cpp/include/raft/neighbors/ivf_pq-ext.cuh +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // raft::device_matrix_view -#include // raft::resources -#include // raft::neighbors::ivf_pq::index -#include // RAFT_EXPLICIT - -#include // int64_t - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft::neighbors::ivf_pq { - -template -index build(raft::resources const& handle, - const index_params& params, - raft::device_matrix_view dataset) RAFT_EXPLICIT; - -template -index extend(raft::resources const& handle, - raft::device_matrix_view new_vectors, - std::optional> new_indices, - const index& idx) RAFT_EXPLICIT; - -template -void extend(raft::resources const& handle, - raft::device_matrix_view new_vectors, - std::optional> new_indices, - index* idx) RAFT_EXPLICIT; - -template -void search_with_filtering(raft::resources const& handle, - const search_params& params, - const index& idx, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances, - IvfSampleFilterT sample_filter) RAFT_EXPLICIT; - -template -void search(raft::resources const& handle, - const search_params& params, - const index& idx, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances) RAFT_EXPLICIT; - -template -auto build(raft::resources const& handle, - const index_params& params, - const T* dataset, - IdxT n_rows, - uint32_t dim) -> index RAFT_EXPLICIT; - -template -auto extend(raft::resources const& handle, - const index& idx, - const T* new_vectors, - const IdxT* new_indices, - IdxT n_rows) -> index RAFT_EXPLICIT; - -template -void extend(raft::resources const& handle, - index* idx, - const T* new_vectors, - const IdxT* new_indices, - IdxT n_rows) RAFT_EXPLICIT; - -template -void search_with_filtering(raft::resources const& handle, - const raft::neighbors::ivf_pq::search_params& params, - const index& idx, - const T* queries, - uint32_t n_queries, - uint32_t k, - IdxT* neighbors, - float* distances, - IvfSampleFilterT sample_filter = IvfSampleFilterT{}) RAFT_EXPLICIT; - -template -void search(raft::resources const& handle, - const raft::neighbors::ivf_pq::search_params& params, - const index& idx, - const T* queries, - uint32_t n_queries, - uint32_t k, - IdxT* neighbors, - float* distances) RAFT_EXPLICIT; - -} // namespace raft::neighbors::ivf_pq - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_neighbors_ivf_pq_build(T, IdxT) \ - extern template raft::neighbors::ivf_pq::index raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - raft::device_matrix_view dataset); \ - \ - extern template auto raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - const T* dataset, \ - IdxT n_rows, \ - uint32_t dim) \ - ->raft::neighbors::ivf_pq::index; - -instantiate_raft_neighbors_ivf_pq_build(float, int64_t); -instantiate_raft_neighbors_ivf_pq_build(half, int64_t); -instantiate_raft_neighbors_ivf_pq_build(int8_t, int64_t); -instantiate_raft_neighbors_ivf_pq_build(uint8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_build - -#define instantiate_raft_neighbors_ivf_pq_extend(T, IdxT) \ - extern template raft::neighbors::ivf_pq::index raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_pq::index& idx); \ - \ - extern template void raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_pq::index* idx); \ - \ - extern template auto raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows) \ - ->raft::neighbors::ivf_pq::index; \ - \ - extern template void raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::neighbors::ivf_pq::index* idx, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows); - -instantiate_raft_neighbors_ivf_pq_extend(float, int64_t); -instantiate_raft_neighbors_ivf_pq_extend(half, int64_t); -instantiate_raft_neighbors_ivf_pq_extend(int8_t, int64_t); -instantiate_raft_neighbors_ivf_pq_extend(uint8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_extend - -#define instantiate_raft_neighbors_ivf_pq_search(T, IdxT) \ - extern template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances); \ - \ - extern template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances); \ - \ - extern template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances) - -instantiate_raft_neighbors_ivf_pq_search(float, int64_t); -instantiate_raft_neighbors_ivf_pq_search(half, int64_t); -instantiate_raft_neighbors_ivf_pq_search(int8_t, int64_t); -instantiate_raft_neighbors_ivf_pq_search(uint8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_search diff --git a/cpp/include/raft/neighbors/ivf_pq.cuh b/cpp/include/raft/neighbors/ivf_pq.cuh index 2d20638f00..6608912c50 100644 --- a/cpp/include/raft/neighbors/ivf_pq.cuh +++ b/cpp/include/raft/neighbors/ivf_pq.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "ivf_pq-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "ivf_pq-ext.cuh" -#endif diff --git a/cpp/include/raft/neighbors/refine-ext.cuh b/cpp/include/raft/neighbors/refine-ext.cuh deleted file mode 100644 index 216e1b9ab5..0000000000 --- a/cpp/include/raft/neighbors/refine-ext.cuh +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // raft::device_matrix_view -#include // // raft::host_matrix_view -#include // raft::resources -#include // raft::distance::DistanceType -#include // RAFT_EXPLICIT - -#include // int64_t - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft::neighbors { - -template -[[deprecated("Use cuVS instead")]] void refine( - raft::resources const& handle, - raft::device_matrix_view dataset, - raft::device_matrix_view queries, - raft::device_matrix_view neighbor_candidates, - raft::device_matrix_view indices, - raft::device_matrix_view distances, - raft::distance::DistanceType metric = distance::DistanceType::L2Unexpanded) RAFT_EXPLICIT; - -template -[[deprecated("Use cuVS instead")]] void refine( - raft::resources const& handle, - raft::host_matrix_view dataset, - raft::host_matrix_view queries, - raft::host_matrix_view neighbor_candidates, - raft::host_matrix_view indices, - raft::host_matrix_view distances, - raft::distance::DistanceType metric = distance::DistanceType::L2Unexpanded) RAFT_EXPLICIT; - -} // namespace raft::neighbors - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_neighbors_refine_d(idx_t, data_t, distance_t, matrix_idx) \ - extern template void raft::neighbors::refine( \ - raft::resources const& handle, \ - raft::device_matrix_view dataset, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbor_candidates, \ - raft::device_matrix_view indices, \ - raft::device_matrix_view distances, \ - raft::distance::DistanceType metric); - -#define instantiate_raft_neighbors_refine_h(idx_t, data_t, distance_t, matrix_idx) \ - extern template void raft::neighbors::refine( \ - raft::resources const& handle, \ - raft::host_matrix_view dataset, \ - raft::host_matrix_view queries, \ - raft::host_matrix_view neighbor_candidates, \ - raft::host_matrix_view indices, \ - raft::host_matrix_view distances, \ - raft::distance::DistanceType metric); - -instantiate_raft_neighbors_refine_d(int64_t, float, float, int64_t); -instantiate_raft_neighbors_refine_d(int64_t, int8_t, float, int64_t); -instantiate_raft_neighbors_refine_d(int64_t, uint8_t, float, int64_t); - -instantiate_raft_neighbors_refine_h(int64_t, float, float, int64_t); -instantiate_raft_neighbors_refine_h(uint32_t, float, float, int64_t); -instantiate_raft_neighbors_refine_h(int64_t, int8_t, float, int64_t); -instantiate_raft_neighbors_refine_h(int64_t, uint8_t, float, int64_t); - -#undef instantiate_raft_neighbors_refine_d -#undef instantiate_raft_neighbors_refine_h diff --git a/cpp/include/raft/neighbors/refine.cuh b/cpp/include/raft/neighbors/refine.cuh index 15f2b02928..83a98097e0 100644 --- a/cpp/include/raft/neighbors/refine.cuh +++ b/cpp/include/raft/neighbors/refine.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "refine-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "refine-ext.cuh" -#endif diff --git a/cpp/include/raft/neighbors/specializations.cuh b/cpp/include/raft/neighbors/specializations.cuh deleted file mode 100644 index cba059154f..0000000000 --- a/cpp/include/raft/neighbors/specializations.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/neighbors/specializations/ball_cover.cuh b/cpp/include/raft/neighbors/specializations/ball_cover.cuh deleted file mode 100644 index cba059154f..0000000000 --- a/cpp/include/raft/neighbors/specializations/ball_cover.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/neighbors/specializations/brute_force.cuh b/cpp/include/raft/neighbors/specializations/brute_force.cuh deleted file mode 100644 index cba059154f..0000000000 --- a/cpp/include/raft/neighbors/specializations/brute_force.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/neighbors/specializations/detail/ball_cover_lowdim.hpp b/cpp/include/raft/neighbors/specializations/detail/ball_cover_lowdim.hpp deleted file mode 100644 index fbe38fd261..0000000000 --- a/cpp/include/raft/neighbors/specializations/detail/ball_cover_lowdim.hpp +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include - -namespace raft { -namespace spatial { -namespace knn { -namespace detail { - -extern template void rbc_low_dim_pass_one( - raft::resources const& handle, - const BallCoverIndex& index, - const float* query, - const std::uint32_t n_query_rows, - std::uint32_t k, - const std::int64_t* R_knn_inds, - const float* R_knn_dists, - DistFunc& dfunc, - std::int64_t* inds, - float* dists, - float weight, - std::uint32_t* dists_counter); - -extern template void rbc_low_dim_pass_two( - raft::resources const& handle, - const BallCoverIndex& index, - const float* query, - const std::uint32_t n_query_rows, - std::uint32_t k, - const std::int64_t* R_knn_inds, - const float* R_knn_dists, - DistFunc& dfunc, - std::int64_t* inds, - float* dists, - float weight, - std::uint32_t* post_dists_counter); - -extern template void rbc_low_dim_pass_one( - raft::resources const& handle, - const BallCoverIndex& index, - const float* query, - const std::uint32_t n_query_rows, - std::uint32_t k, - const std::int64_t* R_knn_inds, - const float* R_knn_dists, - DistFunc& dfunc, - std::int64_t* inds, - float* dists, - float weight, - std::uint32_t* dists_counter); - -extern template void rbc_low_dim_pass_two( - raft::resources const& handle, - const BallCoverIndex& index, - const float* query, - const std::uint32_t n_query_rows, - std::uint32_t k, - const std::int64_t* R_knn_inds, - const float* R_knn_dists, - DistFunc& dfunc, - std::int64_t* inds, - float* dists, - float weight, - std::uint32_t* post_dists_counter); - -}; // namespace detail -}; // namespace knn -}; // namespace spatial -}; // namespace raft \ No newline at end of file diff --git a/cpp/include/raft/neighbors/specializations/detail/ivf_pq_compute_similarity.cuh b/cpp/include/raft/neighbors/specializations/detail/ivf_pq_compute_similarity.cuh deleted file mode 100644 index e85b05575f..0000000000 --- a/cpp/include/raft/neighbors/specializations/detail/ivf_pq_compute_similarity.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/neighbors/specializations/fused_l2_knn.cuh b/cpp/include/raft/neighbors/specializations/fused_l2_knn.cuh deleted file mode 100644 index cba059154f..0000000000 --- a/cpp/include/raft/neighbors/specializations/fused_l2_knn.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/neighbors/specializations/ivf_flat.cuh b/cpp/include/raft/neighbors/specializations/ivf_flat.cuh deleted file mode 100644 index c61d65dcaf..0000000000 --- a/cpp/include/raft/neighbors/specializations/ivf_flat.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/neighbors/specializations/ivf_pq.cuh b/cpp/include/raft/neighbors/specializations/ivf_pq.cuh deleted file mode 100644 index e85b05575f..0000000000 --- a/cpp/include/raft/neighbors/specializations/ivf_pq.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/neighbors/specializations/refine.cuh b/cpp/include/raft/neighbors/specializations/refine.cuh deleted file mode 100644 index e85b05575f..0000000000 --- a/cpp/include/raft/neighbors/specializations/refine.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/sparse/neighbors/specializations.cuh b/cpp/include/raft/sparse/neighbors/specializations.cuh deleted file mode 100644 index e85b05575f..0000000000 --- a/cpp/include/raft/sparse/neighbors/specializations.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/spatial/knn/detail/ball_cover/registers-ext.cuh b/cpp/include/raft/spatial/knn/detail/ball_cover/registers-ext.cuh deleted file mode 100644 index 70df6d0165..0000000000 --- a/cpp/include/raft/spatial/knn/detail/ball_cover/registers-ext.cuh +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "../../ball_cover_types.hpp" // BallCoverIndex -#include "registers_types.cuh" // DistFunc - -#include //RAFT_EXPLICIT - -#include // uint32_t - -#if defined(RAFT_EXPLICIT_INSTANTIATE_ONLY) - -namespace raft::spatial::knn::detail { - -template -void rbc_low_dim_pass_one(raft::resources const& handle, - const BallCoverIndex& index, - const value_t* query, - const value_int n_query_rows, - value_int k, - const value_idx* R_knn_inds, - const value_t* R_knn_dists, - dist_func& dfunc, - value_idx* inds, - value_t* dists, - float weight, - value_int* dists_counter) RAFT_EXPLICIT; - -template -void rbc_low_dim_pass_two(raft::resources const& handle, - const BallCoverIndex& index, - const value_t* query, - const value_int n_query_rows, - value_int k, - const value_idx* R_knn_inds, - const value_t* R_knn_dists, - dist_func& dfunc, - value_idx* inds, - value_t* dists, - float weight, - value_int* post_dists_counter) RAFT_EXPLICIT; - -template -void rbc_eps_pass(raft::resources const& handle, - const BallCoverIndex& index, - const value_t* query, - const value_int n_query_rows, - value_t eps, - const value_t* R_dists, - dist_func& dfunc, - bool* adj, - value_idx* vd) RAFT_EXPLICIT; - -template -void rbc_eps_pass(raft::resources const& handle, - const BallCoverIndex& index, - const value_t* query, - const value_int n_query_rows, - value_t eps, - value_int* max_k, - const value_t* R_dists, - dist_func& dfunc, - value_idx* adj_ia, - value_idx* adj_ja, - value_idx* vd) RAFT_EXPLICIT; - -}; // namespace raft::spatial::knn::detail - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - extern template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_one( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - extern template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_two( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -#define instantiate_raft_spatial_knn_detail_rbc_eps_pass( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdist_func) \ - extern template void \ - raft::spatial::knn::detail::rbc_eps_pass( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_t eps, \ - const Mvalue_t* R_dists, \ - Mdist_func& dfunc, \ - bool* adj, \ - Mvalue_idx* vd); \ - \ - extern template void \ - raft::spatial::knn::detail::rbc_eps_pass( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_t eps, \ - Mvalue_int* max_k, \ - const Mvalue_t* R_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* adj_ia, \ - Mvalue_idx* adj_ja, \ - Mvalue_idx* vd); - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 2, raft::spatial::knn::detail::HaversineFunc); -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 3, raft::spatial::knn::detail::HaversineFunc); -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 2, raft::spatial::knn::detail::EuclideanFunc); -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 3, raft::spatial::knn::detail::EuclideanFunc); -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 2, raft::spatial::knn::detail::DistFunc); -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 3, raft::spatial::knn::detail::DistFunc); - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 2, raft::spatial::knn::detail::HaversineFunc); -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 3, raft::spatial::knn::detail::HaversineFunc); -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 2, raft::spatial::knn::detail::EuclideanFunc); -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 3, raft::spatial::knn::detail::EuclideanFunc); -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 2, raft::spatial::knn::detail::DistFunc); -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 3, raft::spatial::knn::detail::DistFunc); - -instantiate_raft_spatial_knn_detail_rbc_eps_pass( - std::int64_t, float, std::int64_t, std::int64_t, raft::spatial::knn::detail::EuclideanSqFunc); - -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one -#undef instantiate_raft_spatial_knn_detail_rbc_eps_pass diff --git a/cpp/include/raft/spatial/knn/detail/ball_cover/registers.cuh b/cpp/include/raft/spatial/knn/detail/ball_cover/registers.cuh index 8bd57b47cc..1c13b6b59f 100644 --- a/cpp/include/raft/spatial/knn/detail/ball_cover/registers.cuh +++ b/cpp/include/raft/spatial/knn/detail/ball_cover/registers.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,11 +14,4 @@ * limitations under the License. */ #pragma once - -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "registers-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "registers-ext.cuh" -#endif diff --git a/cpp/include/raft/spatial/knn/detail/fused_l2_knn-ext.cuh b/cpp/include/raft/spatial/knn/detail/fused_l2_knn-ext.cuh deleted file mode 100644 index 1968bdb205..0000000000 --- a/cpp/include/raft/spatial/knn/detail/fused_l2_knn-ext.cuh +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include // DistanceType -#include // RAFT_EXPLICIT - -#include // size_t -#include // uint32_t - -#if defined(RAFT_EXPLICIT_INSTANTIATE_ONLY) - -namespace raft::spatial::knn::detail { - -template -void fusedL2Knn(size_t D, - value_idx* out_inds, - value_t* out_dists, - const value_t* index, - const value_t* query, - size_t n_index_rows, - size_t n_query_rows, - int k, - bool rowMajorIndex, - bool rowMajorQuery, - cudaStream_t stream, - raft::distance::DistanceType metric, - const value_t* index_norms = NULL, - const value_t* query_norms = NULL) RAFT_EXPLICIT; - -} // namespace raft::spatial::knn::detail - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_spatial_knn_detail_fusedL2Knn(Mvalue_idx, Mvalue_t, MusePrevTopKs) \ - extern template void \ - raft::spatial::knn::detail::fusedL2Knn( \ - size_t D, \ - Mvalue_idx * out_inds, \ - Mvalue_t * out_dists, \ - const Mvalue_t* index, \ - const Mvalue_t* query, \ - size_t n_index_rows, \ - size_t n_query_rows, \ - int k, \ - bool rowMajorIndex, \ - bool rowMajorQuery, \ - cudaStream_t stream, \ - raft::distance::DistanceType metric, \ - const Mvalue_t* index_norms, \ - const Mvalue_t* query_norms); - -instantiate_raft_spatial_knn_detail_fusedL2Knn(int32_t, float, true); -instantiate_raft_spatial_knn_detail_fusedL2Knn(int32_t, float, false); -instantiate_raft_spatial_knn_detail_fusedL2Knn(int64_t, float, true); -instantiate_raft_spatial_knn_detail_fusedL2Knn(int64_t, float, false); - -// These are used by brute_force_knn: -instantiate_raft_spatial_knn_detail_fusedL2Knn(uint32_t, float, true); -instantiate_raft_spatial_knn_detail_fusedL2Knn(uint32_t, float, false); - -#undef instantiate_raft_spatial_knn_detail_fusedL2Knn diff --git a/cpp/include/raft/spatial/knn/detail/fused_l2_knn.cuh b/cpp/include/raft/spatial/knn/detail/fused_l2_knn.cuh index 8cc02c7c78..a45a00730c 100644 --- a/cpp/include/raft/spatial/knn/detail/fused_l2_knn.cuh +++ b/cpp/include/raft/spatial/knn/detail/fused_l2_knn.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "fused_l2_knn-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "fused_l2_knn-ext.cuh" -#endif diff --git a/cpp/include/raft/spatial/knn/specializations.cuh b/cpp/include/raft/spatial/knn/specializations.cuh deleted file mode 100644 index cba059154f..0000000000 --- a/cpp/include/raft/spatial/knn/specializations.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/spatial/knn/specializations/knn.cuh b/cpp/include/raft/spatial/knn/specializations/knn.cuh deleted file mode 100644 index cba059154f..0000000000 --- a/cpp/include/raft/spatial/knn/specializations/knn.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/internal/raft_internal/neighbors/ivf_pq_compute_similarity_filters_test-ext.cuh b/cpp/internal/raft_internal/neighbors/ivf_pq_compute_similarity_filters_test-ext.cuh deleted file mode 100644 index aa14ab19b8..0000000000 --- a/cpp/internal/raft_internal/neighbors/ivf_pq_compute_similarity_filters_test-ext.cuh +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // RAFT_WEAK_FUNCTION -#include // raft::distance::DistanceType -#include -#include // raft::neighbors::ivf_pq::detail::fp_8bit -#include // none_ivf_sample_filter -#include // none_ivf_sample_filter - -#include // rmm::cuda_stream_view - -#include // __half - -#define instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( \ - OutT, LutT, IvfSampleFilterT) \ - extern template auto \ - raft::neighbors::ivf_pq::detail::compute_similarity_select( \ - const cudaDeviceProp& dev_props, \ - bool manage_local_topk, \ - int locality_hint, \ - double preferred_shmem_carveout, \ - uint32_t pq_bits, \ - uint32_t pq_dim, \ - uint32_t precomp_data_count, \ - uint32_t n_queries, \ - uint32_t n_probes, \ - uint32_t topk) \ - ->raft::neighbors::ivf_pq::detail::selected; \ - \ - extern template void \ - raft::neighbors::ivf_pq::detail::compute_similarity_run( \ - raft::neighbors::ivf_pq::detail::selected s, \ - rmm::cuda_stream_view stream, \ - uint32_t dim, \ - uint32_t n_probes, \ - uint32_t pq_dim, \ - uint32_t n_queries, \ - uint32_t queries_offset, \ - raft::distance::DistanceType metric, \ - raft::neighbors::ivf_pq::codebook_gen codebook_kind, \ - uint32_t topk, \ - uint32_t max_samples, \ - const float* cluster_centers, \ - const float* pq_centers, \ - const uint8_t* const* pq_dataset, \ - const uint32_t* cluster_labels, \ - const uint32_t* _chunk_indices, \ - const float* queries, \ - const uint32_t* index_list, \ - float* query_kths, \ - IvfSampleFilterT sample_filter, \ - LutT* lut_scores, \ - OutT* _out_scores, \ - uint32_t* _out_indices); - -#define COMMA , -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - float, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); - -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - float, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - float, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); -#undef COMMA - -#undef instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select diff --git a/cpp/internal/raft_internal/neighbors/ivf_pq_search_test-ext.cuh b/cpp/internal/raft_internal/neighbors/ivf_pq_search_test-ext.cuh deleted file mode 100644 index 1e6f4f9976..0000000000 --- a/cpp/internal/raft_internal/neighbors/ivf_pq_search_test-ext.cuh +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // raft::device_matrix_view -#include // raft::resources -#include -#include // raft::neighbors::ivf_pq::index -#include -#include - -#include - -#include - -#include // int64_t - -#define instantiate_raft_neighbors_ivf_pq_search(T, IdxT) \ - extern template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances); \ - \ - extern template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances); \ - \ - extern template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances) - -instantiate_raft_neighbors_ivf_pq_search(float, uint32_t); - -#undef instantiate_raft_neighbors_ivf_pq_search - -#define instantiate_raft_neighbors_ivf_pq_search_with_filtering(T, IdxT, FilterT) \ - extern template void raft::neighbors::ivf_pq::search_with_filtering( \ - raft::resources const& handle, \ - const search_params& params, \ - const index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances, \ - FilterT sample_filter) - -#define COMMA , -instantiate_raft_neighbors_ivf_pq_search_with_filtering( - float, uint32_t, raft::neighbors::filtering::bitset_filter); - -instantiate_raft_neighbors_ivf_pq_search_with_filtering( - float, uint32_t, raft::neighbors::filtering::none_ivf_sample_filter); - -instantiate_raft_neighbors_ivf_pq_search_with_filtering( - float, int64_t, raft::neighbors::filtering::bitset_filter); - -instantiate_raft_neighbors_ivf_pq_search_with_filtering( - int8_t, int64_t, raft::neighbors::filtering::bitset_filter); - -#undef COMMA -#undef instantiate_raft_neighbors_ivf_pq_search_with_filtering diff --git a/cpp/internal/raft_internal/neighbors/naive_knn.cuh b/cpp/internal/raft_internal/neighbors/naive_knn.cuh deleted file mode 100644 index c14a8e3e9f..0000000000 --- a/cpp/internal/raft_internal/neighbors/naive_knn.cuh +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include - -namespace raft::neighbors { - -template -RAFT_KERNEL naive_distance_kernel(EvalT* dist, - const DataT* x, - const DataT* y, - IdxT m, - IdxT n, - IdxT k, - raft::distance::DistanceType metric) -{ - IdxT midx = IdxT(threadIdx.x) + IdxT(blockIdx.x) * IdxT(blockDim.x); - if (midx >= m) return; - IdxT grid_size = IdxT(blockDim.y) * IdxT(gridDim.y); - for (IdxT nidx = threadIdx.y + blockIdx.y * blockDim.y; nidx < n; nidx += grid_size) { - EvalT acc = EvalT(0); - for (IdxT i = 0; i < k; ++i) { - IdxT xidx = i + midx * k; - IdxT yidx = i + nidx * k; - auto xv = EvalT(x[xidx]); - auto yv = EvalT(y[yidx]); - switch (metric) { - case raft::distance::DistanceType::InnerProduct: { - acc += xv * yv; - } break; - case raft::distance::DistanceType::L2SqrtExpanded: - case raft::distance::DistanceType::L2SqrtUnexpanded: - case raft::distance::DistanceType::L2Expanded: - case raft::distance::DistanceType::L2Unexpanded: { - auto diff = xv - yv; - acc += diff * diff; - } break; - default: break; - } - } - switch (metric) { - case raft::distance::DistanceType::L2SqrtExpanded: - case raft::distance::DistanceType::L2SqrtUnexpanded: { - acc = raft::sqrt(acc); - } break; - default: break; - } - dist[midx * n + nidx] = acc; - } -} - -/** - * Naive, but flexible bruteforce KNN search. - * - * TODO: either replace this with brute_force_knn or with distance+select_k - * when either distance or brute_force_knn support 8-bit int inputs. - */ -template -void naive_knn(raft::resources const& handle, - EvalT* dist_topk, - IdxT* indices_topk, - const DataT* x, - const DataT* y, - size_t n_inputs, - size_t input_len, - size_t dim, - uint32_t k, - raft::distance::DistanceType type) -{ - auto mr = resource::get_workspace_resource(handle); - auto stream = raft::resource::get_cuda_stream(handle); - dim3 block_dim(16, 32, 1); - // maximum reasonable grid size in `y` direction - auto grid_y = - static_cast(std::min(raft::ceildiv(input_len, block_dim.y), 32768)); - - // bound the memory used by this function - size_t max_batch_size = - std::min(n_inputs, raft::ceildiv(size_t(1) << size_t(27), input_len)); - rmm::device_uvector dist(max_batch_size * input_len, stream, mr); - - for (size_t offset = 0; offset < n_inputs; offset += max_batch_size) { - size_t batch_size = std::min(max_batch_size, n_inputs - offset); - dim3 grid_dim(raft::ceildiv(batch_size, block_dim.x), grid_y, 1); - - naive_distance_kernel<<>>( - dist.data(), x + offset * dim, y, batch_size, input_len, dim, type); - - matrix::detail::select_k(handle, - dist.data(), - nullptr, - batch_size, - input_len, - static_cast(k), - dist_topk + offset * k, - indices_topk + offset * k, - type != raft::distance::DistanceType::InnerProduct); - } - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); -} - -} // namespace raft::neighbors diff --git a/cpp/internal/raft_internal/neighbors/refine_helper.cuh b/cpp/internal/raft_internal/neighbors/refine_helper.cuh deleted file mode 100644 index 665ec23d8e..0000000000 --- a/cpp/internal/raft_internal/neighbors/refine_helper.cuh +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include - -namespace raft::neighbors { - -template -struct RefineInputs { - IdxT n_queries; - IdxT n_rows; - IdxT dim; - IdxT k; // after refinement - IdxT k0; // initial k before refinement (k0 >= k). - raft::distance::DistanceType metric; - bool host_data; -}; - -/** Helper class to allocate arrays and generate input data for refinement test and benchmark. */ -template -class RefineHelper { - public: - RefineHelper(const raft::resources& handle, RefineInputs params) - : handle_(handle), - stream_(resource::get_cuda_stream(handle)), - p(params), - dataset(handle), - queries(handle), - refined_distances(handle), - refined_indices(handle), - candidates(handle), - dataset_host(handle), - queries_host(handle), - candidates_host(handle), - refined_distances_host(handle), - refined_indices_host(handle) - { - raft::random::RngState rng(1234ULL); - - dataset = raft::make_device_matrix(handle_, p.n_rows, p.dim); - queries = raft::make_device_matrix(handle_, p.n_queries, p.dim); - if constexpr (std::is_same{}) { - raft::random::uniform( - handle, rng, dataset.data_handle(), dataset.size(), DataT(-10.0), DataT(10.0)); - raft::random::uniform( - handle, rng, queries.data_handle(), queries.size(), DataT(-10.0), DataT(10.0)); - } else { - raft::random::uniformInt( - handle, rng, dataset.data_handle(), dataset.size(), DataT(1), DataT(20)); - raft::random::uniformInt( - handle, rng, queries.data_handle(), queries.size(), DataT(1), DataT(20)); - } - - refined_distances = raft::make_device_matrix(handle_, p.n_queries, p.k); - refined_indices = raft::make_device_matrix(handle_, p.n_queries, p.k); - - // Generate candidate vectors - { - candidates = raft::make_device_matrix(handle_, p.n_queries, p.k0); - rmm::device_uvector distances_tmp(p.n_queries * p.k0, stream_); - naive_knn(handle_, - distances_tmp.data(), - candidates.data_handle(), - queries.data_handle(), - dataset.data_handle(), - p.n_queries, - p.n_rows, - p.dim, - p.k0, - p.metric); - resource::sync_stream(handle_, stream_); - } - - if (p.host_data) { - dataset_host = raft::make_host_matrix(p.n_rows, p.dim); - queries_host = raft::make_host_matrix(p.n_queries, p.dim); - candidates_host = raft::make_host_matrix(p.n_queries, p.k0); - - raft::copy(dataset_host.data_handle(), dataset.data_handle(), dataset.size(), stream_); - raft::copy(queries_host.data_handle(), queries.data_handle(), queries.size(), stream_); - raft::copy( - candidates_host.data_handle(), candidates.data_handle(), candidates.size(), stream_); - - refined_distances_host = raft::make_host_matrix(p.n_queries, p.k); - refined_indices_host = raft::make_host_matrix(p.n_queries, p.k); - resource::sync_stream(handle_, stream_); - } - - // Generate ground thruth for testing. - { - rmm::device_uvector distances_dev(p.n_queries * p.k, stream_); - rmm::device_uvector indices_dev(p.n_queries * p.k, stream_); - naive_knn(handle_, - distances_dev.data(), - indices_dev.data(), - queries.data_handle(), - dataset.data_handle(), - p.n_queries, - p.n_rows, - p.dim, - p.k, - p.metric); - true_refined_distances_host.resize(p.n_queries * p.k); - true_refined_indices_host.resize(p.n_queries * p.k); - raft::copy(true_refined_indices_host.data(), indices_dev.data(), indices_dev.size(), stream_); - raft::copy( - true_refined_distances_host.data(), distances_dev.data(), distances_dev.size(), stream_); - resource::sync_stream(handle_, stream_); - } - } - - public: - RefineInputs p; - const raft::resources& handle_; - rmm::cuda_stream_view stream_; - - raft::device_matrix dataset; - raft::device_matrix queries; - raft::device_matrix candidates; // Neighbor candidate indices - raft::device_matrix refined_indices; - raft::device_matrix refined_distances; - - raft::host_matrix dataset_host; - raft::host_matrix queries_host; - raft::host_matrix candidates_host; - raft::host_matrix refined_indices_host; - raft::host_matrix refined_distances_host; - - std::vector true_refined_indices_host; - std::vector true_refined_distances_host; -}; -} // namespace raft::neighbors diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_00_generate.py b/cpp/src/distance/detail/pairwise_matrix/dispatch_00_generate.py deleted file mode 100644 index 6adff0eee1..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_00_generate.py +++ /dev/null @@ -1,199 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# NOTE: this template is not perfectly formatted. Use pre-commit to get -# everything in shape again. -header = """/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -""" - - -macro = """ -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \\ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \\ - template void raft::distance::detail:: \\ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \\ - OpT distance_op, \\ - IdxT m, \\ - IdxT n, \\ - IdxT k, \\ - const DataT* x, \\ - const DataT* y, \\ - const DataT* x_norm, \\ - const DataT* y_norm, \\ - OutT* out, \\ - FinOpT fin_op, \\ - cudaStream_t stream, \\ - bool is_row_major) -""" - -data_type_instances = [ - dict( - DataT="float", - AccT="float", - OutT="float", - IdxT="int", - ), - dict( - DataT="double", - AccT="double", - OutT="double", - IdxT="int", - ), -] - -op_instances = [ - dict( - path_prefix="canberra", - OpT="raft::distance::detail::ops::canberra_distance_op", - archs = [60], - ), - dict( - path_prefix="correlation", - OpT="raft::distance::detail::ops::correlation_distance_op", - archs = [60], - ), - dict( - path_prefix="cosine", - OpT="raft::distance::detail::ops::cosine_distance_op", - archs = [60, 80], - ), - dict( - path_prefix="dice", - OpT="raft::distance::detail::ops::dice_distance_op", - archs = [60, 80], - ), - dict( - path_prefix="hamming_unexpanded", - OpT="raft::distance::detail::ops::hamming_distance_op", - archs = [60], - ), - dict( - path_prefix="hellinger_expanded", - OpT="raft::distance::detail::ops::hellinger_distance_op", - archs = [60], - ), - # inner product is handled by cublas. - dict( - path_prefix="jensen_shannon", - OpT="raft::distance::detail::ops::jensen_shannon_distance_op", - archs = [60], - ), - dict( - path_prefix="kl_divergence", - OpT="raft::distance::detail::ops::kl_divergence_op", - archs = [60], - ), - dict( - path_prefix="l1", - OpT="raft::distance::detail::ops::l1_distance_op", - archs = [60], - ), - dict( - path_prefix="l2_expanded", - OpT="raft::distance::detail::ops::l2_exp_distance_op", - archs = [60, 80], - ), - dict( - path_prefix="l2_unexpanded", - OpT="raft::distance::detail::ops::l2_unexp_distance_op", - archs = [60], - ), - dict( - path_prefix="l_inf", - OpT="raft::distance::detail::ops::l_inf_distance_op", - archs = [60], - ), - dict( - path_prefix="lp_unexpanded", - OpT="raft::distance::detail::ops::lp_unexp_distance_op", - archs = [60], - ), - dict( - path_prefix="russel_rao", - OpT="raft::distance::detail::ops::russel_rao_distance_op", - archs = [60], - ), -] - -def arch_headers(archs): - include_headers ="\n".join([ - f"#include " - for arch in archs - ]) - return include_headers - - - -for op in op_instances: - for dt in data_type_instances: - DataT, AccT, OutT, IdxT = (dt[k] for k in ["DataT", "AccT", "OutT", "IdxT"]); - path = f"dispatch_{op['path_prefix']}_{DataT}_{AccT}_{OutT}_{IdxT}.cu" - with open(path, "w") as f: - f.write(header) - f.write(arch_headers(op["archs"])) - f.write(macro) - - OpT = op['OpT'] - FinOpT = "raft::identity_op" - f.write(f"\ninstantiate_raft_distance_detail_pairwise_matrix_dispatch({OpT}, {DataT}, {AccT}, {OutT}, {FinOpT}, {IdxT});\n") - f.write("\n#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch\n") - print(f"src/distance/detail/pairwise_matrix/{path}") - -# Dispatch kernels for with the RBF fin op. -with open("dispatch_rbf.cu", "w") as f: - OpT="raft::distance::detail::ops::l2_unexp_distance_op" - archs = [60] - - f.write(header) - f.write("#include // rbf_fin_op\n") - f.write(arch_headers(archs)) - f.write(macro) - - for dt in data_type_instances: - DataT, AccT, OutT, IdxT = (dt[k] for k in ["DataT", "AccT", "OutT", "IdxT"]); - IdxT = "int64_t" # overwrite IdxT - - FinOpT = f"raft::distance::kernels::detail::rbf_fin_op<{DataT}>" - f.write(f"\ninstantiate_raft_distance_detail_pairwise_matrix_dispatch({OpT}, {DataT}, {AccT}, {OutT}, {FinOpT}, {IdxT});\n") - - f.write("\n#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch\n") - -print("src/distance/detail/pairwise_matrix/dispatch_rbf.cu") diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_canberra_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_canberra_double_double_double_int.cu deleted file mode 100644 index 41db12e9ae..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_canberra_double_double_double_int.cu +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::canberra_distance_op, - double, - double, - double, - raft::identity_op, - int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_canberra_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_canberra_float_float_float_int.cu deleted file mode 100644 index f038e53381..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_canberra_float_float_float_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::canberra_distance_op, float, float, float, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_correlation_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_correlation_double_double_double_int.cu deleted file mode 100644 index 52e4cc02d8..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_correlation_double_double_double_int.cu +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::correlation_distance_op, - double, - double, - double, - raft::identity_op, - int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_correlation_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_correlation_float_float_float_int.cu deleted file mode 100644 index c9481d6c22..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_correlation_float_float_float_int.cu +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::correlation_distance_op, - float, - float, - float, - raft::identity_op, - int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_cosine_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_cosine_double_double_double_int.cu deleted file mode 100644 index 517858125b..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_cosine_double_double_double_int.cu +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::cosine_distance_op, double, double, double, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_cosine_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_cosine_float_float_float_int.cu deleted file mode 100644 index 62f1d9874b..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_cosine_float_float_float_int.cu +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::cosine_distance_op, float, float, float, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_dice_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_dice_double_double_double_int.cu deleted file mode 100644 index a259f8b3b0..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_dice_double_double_double_int.cu +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::dice_distance_op, double, double, double, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_dice_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_dice_float_float_float_int.cu deleted file mode 100644 index e89f8b422c..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_dice_float_float_float_int.cu +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::dice_distance_op, float, float, float, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_hamming_unexpanded_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_hamming_unexpanded_double_double_double_int.cu deleted file mode 100644 index 500f7b4a9c..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_hamming_unexpanded_double_double_double_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::hamming_distance_op, double, double, double, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_hamming_unexpanded_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_hamming_unexpanded_float_float_float_int.cu deleted file mode 100644 index 3be7586b43..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_hamming_unexpanded_float_float_float_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::hamming_distance_op, float, float, float, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_hellinger_expanded_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_hellinger_expanded_double_double_double_int.cu deleted file mode 100644 index 023134ddff..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_hellinger_expanded_double_double_double_int.cu +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::hellinger_distance_op, - double, - double, - double, - raft::identity_op, - int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_hellinger_expanded_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_hellinger_expanded_float_float_float_int.cu deleted file mode 100644 index e438f121f2..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_hellinger_expanded_float_float_float_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::hellinger_distance_op, float, float, float, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_jensen_shannon_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_jensen_shannon_double_double_double_int.cu deleted file mode 100644 index 31c5003ad6..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_jensen_shannon_double_double_double_int.cu +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::jensen_shannon_distance_op, - double, - double, - double, - raft::identity_op, - int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_jensen_shannon_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_jensen_shannon_float_float_float_int.cu deleted file mode 100644 index e78c1c320a..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_jensen_shannon_float_float_float_int.cu +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::jensen_shannon_distance_op, - float, - float, - float, - raft::identity_op, - int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_kl_divergence_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_kl_divergence_double_double_double_int.cu deleted file mode 100644 index 5b95df9614..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_kl_divergence_double_double_double_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::kl_divergence_op, double, double, double, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_kl_divergence_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_kl_divergence_float_float_float_int.cu deleted file mode 100644 index fb72c91b73..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_kl_divergence_float_float_float_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::kl_divergence_op, float, float, float, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_l1_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_l1_double_double_double_int.cu deleted file mode 100644 index cac5acad92..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_l1_double_double_double_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l1_distance_op, double, double, double, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_l1_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_l1_float_float_float_int.cu deleted file mode 100644 index 78aa097961..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_l1_float_float_float_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l1_distance_op, float, float, float, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_l2_expanded_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_l2_expanded_double_double_double_int.cu deleted file mode 100644 index c8d922f6fa..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_l2_expanded_double_double_double_int.cu +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_exp_distance_op, double, double, double, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_l2_expanded_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_l2_expanded_float_float_float_int.cu deleted file mode 100644 index 20cf57f898..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_l2_expanded_float_float_float_int.cu +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_exp_distance_op, float, float, float, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_l2_unexpanded_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_l2_unexpanded_double_double_double_int.cu deleted file mode 100644 index eadd0d2c2b..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_l2_unexpanded_double_double_double_int.cu +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_unexp_distance_op, - double, - double, - double, - raft::identity_op, - int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_l2_unexpanded_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_l2_unexpanded_float_float_float_int.cu deleted file mode 100644 index e4b5dd3a86..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_l2_unexpanded_float_float_float_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_unexp_distance_op, float, float, float, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_l_inf_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_l_inf_double_double_double_int.cu deleted file mode 100644 index 45d021bce9..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_l_inf_double_double_double_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l_inf_distance_op, double, double, double, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_l_inf_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_l_inf_float_float_float_int.cu deleted file mode 100644 index ba48e52a18..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_l_inf_float_float_float_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l_inf_distance_op, float, float, float, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_lp_unexpanded_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_lp_unexpanded_double_double_double_int.cu deleted file mode 100644 index ffa58793d9..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_lp_unexpanded_double_double_double_int.cu +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::lp_unexp_distance_op, - double, - double, - double, - raft::identity_op, - int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_lp_unexpanded_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_lp_unexpanded_float_float_float_int.cu deleted file mode 100644 index 915c68f05f..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_lp_unexpanded_float_float_float_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::lp_unexp_distance_op, float, float, float, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_rbf.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_rbf.cu deleted file mode 100644 index 15855cea0a..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_rbf.cu +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // rbf_fin_op -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_unexp_distance_op, - float, - float, - float, - raft::distance::kernels::detail::rbf_fin_op, - int64_t); - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_unexp_distance_op, - double, - double, - double, - raft::distance::kernels::detail::rbf_fin_op, - int64_t); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_russel_rao_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_russel_rao_double_double_double_int.cu deleted file mode 100644 index db45dc8b94..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_russel_rao_double_double_double_int.cu +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::russel_rao_distance_op, - double, - double, - double, - raft::identity_op, - int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_russel_rao_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_russel_rao_float_float_float_int.cu deleted file mode 100644 index a2a5a9fafe..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_russel_rao_float_float_float_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::russel_rao_distance_op, float, float, float, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/distance.cu b/cpp/src/distance/distance.cu deleted file mode 100644 index 8fe0bf2007..0000000000 --- a/cpp/src/distance/distance.cu +++ /dev/null @@ -1,982 +0,0 @@ -/* - * Copyright (c) 2018-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include // rbf_fin_op -#include - -/* - * Hierarchy of instantiations: - * - * This file defines the template instantiations for the public API of - * raft::distance. To improve compile times, the compilation of the distance - * kernels is handled in distance/detail/pairwise_matrix/dispatch_*.cu. - * - */ - -#define instantiate_raft_distance_distance(DT, DataT, AccT, OutT, FinalLambda, IdxT) \ - template void raft::distance::distance( \ - raft::resources const& handle, \ - const DataT* x, \ - const DataT* y, \ - OutT* dist, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - void* workspace, \ - size_t worksize, \ - FinalLambda fin_op, \ - bool isRowMajor, \ - DataT metric_arg) - -// The following two instances are used in test/distance/gram.cu. Note the use -// of int64_t for the index type. -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Unexpanded, - float, - float, - float, - raft::distance::kernels::detail::rbf_fin_op, - int64_t); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Unexpanded, - double, - double, - double, - raft::distance::kernels::detail::rbf_fin_op, - int64_t); - -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, - double, - double, - double, - raft::identity_op, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, double, double, double, raft::identity_op, int); - -#undef instantiate_raft_distance_distance - -// Same, but without raft::identity_op -#define instantiate_raft_distance_distance(DT, DataT, AccT, OutT, IdxT) \ - template void raft::distance::distance( \ - raft::resources const& handle, \ - const DataT* x, \ - const DataT* y, \ - OutT* dist, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - void* workspace, \ - size_t worksize, \ - bool isRowMajor, \ - DataT metric_arg) - -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, double, double, double, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L1, float, float, float, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L1, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, double, double, double, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::Linf, float, float, float, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::Linf, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, double, double, double, int); - -#undef instantiate_raft_distance_distance - -// Same, but without workspace -#define instantiate_raft_distance_distance(DT, DataT, AccT, OutT, IdxT) \ - template void raft::distance::distance( \ - raft::resources const& handle, \ - const DataT* x, \ - const DataT* y, \ - OutT* dist, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - bool isRowMajor, \ - DataT metric_arg) - -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, double, double, double, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L1, float, float, float, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L1, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, double, double, double, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::Linf, float, float, float, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::Linf, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, double, double, double, int); - -#undef instantiate_raft_distance_distance - -#define instantiate_raft_distance_getWorkspaceSize(DistT, DataT, AccT, OutT, IdxT) \ - template size_t raft::distance::getWorkspaceSize( \ - const DataT* x, const DataT* y, IdxT m, IdxT n, IdxT k) - -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::CorrelationExpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::CorrelationExpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::CosineExpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::CosineExpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::DiceExpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::DiceExpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::HammingUnexpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::HammingUnexpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::HellingerExpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::HellingerExpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::InnerProduct, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::InnerProduct, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::JensenShannon, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::JensenShannon, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::KLDivergence, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::KLDivergence, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2SqrtExpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2SqrtExpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2SqrtUnexpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2SqrtUnexpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Unexpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Linf, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Linf, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::LpUnexpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::LpUnexpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::RusselRaoExpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::RusselRaoExpanded, double, double, double, int); - -#undef instantiate_raft_distance_getWorkspaceSize - -#define instantiate_raft_distance_getWorkspaceSize(DistT, DataT, AccT, OutT, IdxT, layout) \ - template size_t raft::distance::getWorkspaceSize( \ - raft::device_matrix_view const& x, \ - raft::device_matrix_view const& y) - -// We could consider not taking template parameters for this function. The -// number of instantiations seems a bit excessive.. -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, double, double, double, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, double, double, double, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CorrelationExpanded, - float, - float, - float, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CorrelationExpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CorrelationExpanded, - float, - float, - float, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CorrelationExpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CosineExpanded, - float, - float, - float, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CosineExpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CosineExpanded, - float, - float, - float, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CosineExpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::DiceExpanded, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::DiceExpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::DiceExpanded, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::DiceExpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HammingUnexpanded, - float, - float, - float, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HammingUnexpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HammingUnexpanded, - float, - float, - float, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HammingUnexpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HellingerExpanded, - float, - float, - float, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HellingerExpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HellingerExpanded, - float, - float, - float, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HellingerExpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::InnerProduct, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::InnerProduct, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::InnerProduct, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::InnerProduct, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::JensenShannon, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::JensenShannon, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::JensenShannon, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::JensenShannon, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::KLDivergence, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::KLDivergence, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::KLDivergence, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::KLDivergence, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, double, double, double, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, double, double, double, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, double, double, double, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, double, double, double, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtExpanded, - float, - float, - float, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtExpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtExpanded, - float, - float, - float, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtExpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtUnexpanded, - float, - float, - float, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtUnexpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtUnexpanded, - float, - float, - float, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtUnexpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2Unexpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int, raft::layout_f_contiguous); - -#undef instantiate_raft_distance_getWorkspaceSize - -#define instantiate_raft_distance_pairwise_distance(DataT, IdxT) \ - template void raft::distance::pairwise_distance(raft::resources const& handle, \ - const DataT* x, \ - const DataT* y, \ - DataT* dist, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - rmm::device_uvector& workspace, \ - raft::distance::DistanceType metric, \ - bool isRowMajor, \ - DataT metric_arg) - -instantiate_raft_distance_pairwise_distance(float, int); -instantiate_raft_distance_pairwise_distance(double, int); - -#undef instantiate_raft_distance_pairwise_distance - -// Same, but without workspace -#define instantiate_raft_distance_pairwise_distance(DataT, IdxT) \ - template void raft::distance::pairwise_distance(raft::resources const& handle, \ - const DataT* x, \ - const DataT* y, \ - DataT* dist, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - raft::distance::DistanceType metric, \ - bool isRowMajor, \ - DataT metric_arg) - -instantiate_raft_distance_pairwise_distance(float, int); -instantiate_raft_distance_pairwise_distance(double, int); - -#undef instantiate_raft_distance_pairwise_distance - -// Version with mdspan -#define instantiate_raft_distance_distance(DistT, DataT, AccT, OutT, layout, IdxT) \ - template void raft::distance::distance( \ - raft::resources const& handle, \ - raft::device_matrix_view const x, \ - raft::device_matrix_view const y, \ - raft::device_matrix_view dist, \ - DataT metric_arg) - -// Again, we might want to consider reigning in the number of instantiations... -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CosineExpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CosineExpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CosineExpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CosineExpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::DiceExpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::DiceExpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HammingUnexpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HammingUnexpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HammingUnexpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HammingUnexpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HellingerExpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HellingerExpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HellingerExpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HellingerExpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::InnerProduct, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::InnerProduct, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::JensenShannon, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::JensenShannon, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::KLDivergence, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::KLDivergence, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, double, double, double, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, double, double, double, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtExpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtExpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtExpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtExpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtUnexpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtUnexpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtUnexpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtUnexpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Unexpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Unexpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, double, double, double, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, double, double, double, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::LpUnexpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::LpUnexpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::RusselRaoExpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::RusselRaoExpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::RusselRaoExpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::RusselRaoExpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); - -#undef instantiate_raft_distance_distance - -#define instantiate_raft_distance_pairwise_distance(DataT, layout, IdxT) \ - template void raft::distance::pairwise_distance( \ - raft::resources const& handle, \ - raft::device_matrix_view const x, \ - raft::device_matrix_view const y, \ - raft::device_matrix_view dist, \ - raft::distance::DistanceType metric, \ - DataT metric_arg) - -instantiate_raft_distance_pairwise_distance(float, raft::layout_c_contiguous, int); -instantiate_raft_distance_pairwise_distance(float, raft::layout_f_contiguous, int); -instantiate_raft_distance_pairwise_distance(double, raft::layout_c_contiguous, int); -instantiate_raft_distance_pairwise_distance(double, raft::layout_f_contiguous, int); - -#undef instantiate_raft_distance_pairwise_distance diff --git a/cpp/src/distance/fused_distance_nn.cu b/cpp/src/distance/fused_distance_nn.cu deleted file mode 100644 index dc722d929c..0000000000 --- a/cpp/src/distance/fused_distance_nn.cu +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include // raft::KeyValuePair -#include - -#include // int64_t - -#define instantiate_raft_distance_fusedDistanceNNMinReduce(DataT, OutT, IdxT) \ - template void raft::distance::fusedDistanceNNMinReduce( \ - OutT * min, \ - const DataT* x, \ - const DataT* y, \ - const DataT* xn, \ - const DataT* yn, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - void* workspace, \ - bool sqrt, \ - bool initOutBuffer, \ - bool isRowMajor, \ - raft::distance::DistanceType metric, \ - float metric_arg, \ - cudaStream_t stream) - -instantiate_raft_distance_fusedDistanceNNMinReduce(float, float, int); -instantiate_raft_distance_fusedDistanceNNMinReduce(float, float, int64_t); - -// We can't have comma's in the macro expansion, so we use the COMMA macro: -#define COMMA , - -instantiate_raft_distance_fusedDistanceNNMinReduce(float, raft::KeyValuePair, int); -instantiate_raft_distance_fusedDistanceNNMinReduce(float, - raft::KeyValuePair, - int64_t); - -#undef COMMA - -#undef instantiate_raft_distance_fusedDistanceNNMinReduce diff --git a/cpp/src/distance/fused_l2_nn.cu b/cpp/src/distance/fused_l2_nn.cu deleted file mode 100644 index f29ab08dc1..0000000000 --- a/cpp/src/distance/fused_l2_nn.cu +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include // raft::KeyValuePair -#include - -#include // int64_t - -#define instantiate_raft_distance_fusedL2NNMinReduce(DataT, OutT, IdxT) \ - template void raft::distance::fusedL2NNMinReduce(OutT * min, \ - const DataT* x, \ - const DataT* y, \ - const DataT* xn, \ - const DataT* yn, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - void* workspace, \ - bool sqrt, \ - bool initOutBuffer, \ - cudaStream_t stream) - -instantiate_raft_distance_fusedL2NNMinReduce(double, double, int); -instantiate_raft_distance_fusedL2NNMinReduce(double, double, int64_t); -instantiate_raft_distance_fusedL2NNMinReduce(float, float, int); -instantiate_raft_distance_fusedL2NNMinReduce(float, float, int64_t); - -// We can't have comma's in the macro expansion, so we use the COMMA macro: -#define COMMA , - -instantiate_raft_distance_fusedL2NNMinReduce(double, raft::KeyValuePair, int); -instantiate_raft_distance_fusedL2NNMinReduce(double, - raft::KeyValuePair, - int64_t); -instantiate_raft_distance_fusedL2NNMinReduce(float, raft::KeyValuePair, int); -instantiate_raft_distance_fusedL2NNMinReduce(float, - raft::KeyValuePair, - int64_t); - -#undef COMMA - -#undef instantiate_raft_distance_fusedL2NNMinReduce diff --git a/cpp/src/matrix/detail/select_k_double_int64_t.cu b/cpp/src/matrix/detail/select_k_double_int64_t.cu deleted file mode 100644 index bf234aacbf..0000000000 --- a/cpp/src/matrix/detail/select_k_double_int64_t.cu +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#define instantiate_raft_matrix_detail_select_k(T, IdxT) \ - template void raft::matrix::detail::select_k(raft::resources const& handle, \ - const T* in_val, \ - const IdxT* in_idx, \ - size_t batch_size, \ - size_t len, \ - int k, \ - T* out_val, \ - IdxT* out_idx, \ - bool select_min, \ - bool sorted, \ - raft::matrix::SelectAlgo algo, \ - const IdxT* len_i) - -instantiate_raft_matrix_detail_select_k(double, int64_t); - -#undef instantiate_raft_matrix_detail_select_k diff --git a/cpp/src/matrix/detail/select_k_double_uint32_t.cu b/cpp/src/matrix/detail/select_k_double_uint32_t.cu deleted file mode 100644 index 7f0511a76a..0000000000 --- a/cpp/src/matrix/detail/select_k_double_uint32_t.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include // uint32_t - -#define instantiate_raft_matrix_detail_select_k(T, IdxT) \ - template void raft::matrix::detail::select_k(raft::resources const& handle, \ - const T* in_val, \ - const IdxT* in_idx, \ - size_t batch_size, \ - size_t len, \ - int k, \ - T* out_val, \ - IdxT* out_idx, \ - bool select_min, \ - bool sorted, \ - raft::matrix::SelectAlgo algo, \ - const IdxT* len_i) - -instantiate_raft_matrix_detail_select_k(double, uint32_t); - -#undef instantiate_raft_matrix_detail_select_k diff --git a/cpp/src/matrix/detail/select_k_float_int32.cu b/cpp/src/matrix/detail/select_k_float_int32.cu deleted file mode 100644 index e68b1e32df..0000000000 --- a/cpp/src/matrix/detail/select_k_float_int32.cu +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#define instantiate_raft_matrix_detail_select_k(T, IdxT) \ - template void raft::matrix::detail::select_k(raft::resources const& handle, \ - const T* in_val, \ - const IdxT* in_idx, \ - size_t batch_size, \ - size_t len, \ - int k, \ - T* out_val, \ - IdxT* out_idx, \ - bool select_min, \ - bool sorted, \ - raft::matrix::SelectAlgo algo, \ - const IdxT* len_i) - -instantiate_raft_matrix_detail_select_k(float, int); - -#undef instantiate_raft_matrix_detail_select_k diff --git a/cpp/src/matrix/detail/select_k_float_int64_t.cu b/cpp/src/matrix/detail/select_k_float_int64_t.cu deleted file mode 100644 index 5aa40d8c9d..0000000000 --- a/cpp/src/matrix/detail/select_k_float_int64_t.cu +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#define instantiate_raft_matrix_detail_select_k(T, IdxT) \ - template void raft::matrix::detail::select_k(raft::resources const& handle, \ - const T* in_val, \ - const IdxT* in_idx, \ - size_t batch_size, \ - size_t len, \ - int k, \ - T* out_val, \ - IdxT* out_idx, \ - bool select_min, \ - bool sorted, \ - raft::matrix::SelectAlgo algo, \ - const IdxT* len_i) - -instantiate_raft_matrix_detail_select_k(float, int64_t); - -#undef instantiate_raft_matrix_detail_select_k diff --git a/cpp/src/matrix/detail/select_k_float_uint32_t.cu b/cpp/src/matrix/detail/select_k_float_uint32_t.cu deleted file mode 100644 index 9aba147edf..0000000000 --- a/cpp/src/matrix/detail/select_k_float_uint32_t.cu +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#define instantiate_raft_matrix_detail_select_k(T, IdxT) \ - template void raft::matrix::detail::select_k(raft::resources const& handle, \ - const T* in_val, \ - const IdxT* in_idx, \ - size_t batch_size, \ - size_t len, \ - int k, \ - T* out_val, \ - IdxT* out_idx, \ - bool select_min, \ - bool sorted, \ - raft::matrix::SelectAlgo algo, \ - const IdxT* len_i) - -instantiate_raft_matrix_detail_select_k(float, uint32_t); - -#undef instantiate_raft_matrix_detail_select_k diff --git a/cpp/src/matrix/detail/select_k_half_int64_t.cu b/cpp/src/matrix/detail/select_k_half_int64_t.cu deleted file mode 100644 index bc513e4aeb..0000000000 --- a/cpp/src/matrix/detail/select_k_half_int64_t.cu +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#define instantiate_raft_matrix_detail_select_k(T, IdxT) \ - template void raft::matrix::detail::select_k(raft::resources const& handle, \ - const T* in_val, \ - const IdxT* in_idx, \ - size_t batch_size, \ - size_t len, \ - int k, \ - T* out_val, \ - IdxT* out_idx, \ - bool select_min, \ - bool sorted, \ - raft::matrix::SelectAlgo algo, \ - const IdxT* len_i) - -instantiate_raft_matrix_detail_select_k(__half, int64_t); - -#undef instantiate_raft_matrix_detail_select_k diff --git a/cpp/src/matrix/detail/select_k_half_uint32_t.cu b/cpp/src/matrix/detail/select_k_half_uint32_t.cu deleted file mode 100644 index e46c7d46bb..0000000000 --- a/cpp/src/matrix/detail/select_k_half_uint32_t.cu +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#define instantiate_raft_matrix_detail_select_k(T, IdxT) \ - template void raft::matrix::detail::select_k(raft::resources const& handle, \ - const T* in_val, \ - const IdxT* in_idx, \ - size_t batch_size, \ - size_t len, \ - int k, \ - T* out_val, \ - IdxT* out_idx, \ - bool select_min, \ - bool sorted, \ - raft::matrix::SelectAlgo algo, \ - const IdxT* len_i) - -instantiate_raft_matrix_detail_select_k(__half, uint32_t); - -#undef instantiate_raft_matrix_detail_select_k diff --git a/cpp/src/neighbors/ball_cover.cu b/cpp/src/neighbors/ball_cover.cu deleted file mode 100644 index e9c78f8e7c..0000000000 --- a/cpp/src/neighbors/ball_cover.cu +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include - -#define instantiate_raft_neighbors_ball_cover(idx_t, value_t, int_t, matrix_idx_t) \ - template void raft::neighbors::ball_cover::build_index( \ - raft::resources const& handle, \ - raft::neighbors::ball_cover::BallCoverIndex& index); \ - \ - template void raft::neighbors::ball_cover::eps_nn( \ - raft::resources const& handle, \ - const raft::neighbors::ball_cover::BallCoverIndex& index, \ - raft::device_matrix_view adj, \ - raft::device_vector_view vd, \ - raft::device_matrix_view query, \ - value_t eps); \ - \ - template void raft::neighbors::ball_cover::eps_nn( \ - raft::resources const& handle, \ - const raft::neighbors::ball_cover::BallCoverIndex& index, \ - raft::device_vector_view ia, \ - raft::device_vector_view ja, \ - raft::device_vector_view vd, \ - raft::device_matrix_view query, \ - value_t eps, \ - std::optional> max_k); \ - \ - template void raft::neighbors::ball_cover::all_knn_query( \ - raft::resources const& handle, \ - raft::neighbors::ball_cover::BallCoverIndex& index, \ - int_t k, \ - idx_t* inds, \ - value_t* dists, \ - bool perform_post_filtering, \ - float weight); \ - \ - template void raft::neighbors::ball_cover::all_knn_query( \ - raft::resources const& handle, \ - raft::neighbors::ball_cover::BallCoverIndex& index, \ - raft::device_matrix_view inds, \ - raft::device_matrix_view dists, \ - int_t k, \ - bool perform_post_filtering, \ - float weight); \ - \ - template void raft::neighbors::ball_cover::knn_query( \ - raft::resources const& handle, \ - const raft::neighbors::ball_cover::BallCoverIndex& index, \ - int_t k, \ - const value_t* query, \ - int_t n_query_pts, \ - idx_t* inds, \ - value_t* dists, \ - bool perform_post_filtering, \ - float weight); \ - \ - template void raft::neighbors::ball_cover::knn_query( \ - raft::resources const& handle, \ - const raft::neighbors::ball_cover::BallCoverIndex& index, \ - raft::device_matrix_view query, \ - raft::device_matrix_view inds, \ - raft::device_matrix_view dists, \ - int_t k, \ - bool perform_post_filtering, \ - float weight); - -instantiate_raft_neighbors_ball_cover(int64_t, float, int64_t, int64_t); - -#undef instantiate_raft_neighbors_ball_cover diff --git a/cpp/src/neighbors/brute_force_00_generate.py b/cpp/src/neighbors/brute_force_00_generate.py deleted file mode 100644 index 8ed05dc4c2..0000000000 --- a/cpp/src/neighbors/brute_force_00_generate.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -header = """ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by brute_force_00_generate.py - * - * Make changes there and run in this directory: - * - * > python brute_force_00_generate.py - * - */ - -#include -#include - -""" - -knn_macro = """ -#define instantiate_raft_neighbors_brute_force_knn(idx_t, value_t, matrix_idx, index_layout, search_layout, epilogue_op) \\ - template void raft::neighbors::brute_force::knn( \\ - raft::resources const& handle, \\ - std::vector> index, \\ - raft::device_matrix_view search, \\ - raft::device_matrix_view indices, \\ - raft::device_matrix_view distances, \\ - raft::distance::DistanceType metric, \\ - std::optional metric_arg, \\ - std::optional global_id_offset, \\ - epilogue_op distance_epilogue); - -""" - -fused_l2_knn_macro = """ -#define instantiate_raft_neighbors_brute_force_fused_l2_knn(value_t, idx_t, idx_layout, query_layout) \\ - template void raft::neighbors::brute_force::fused_l2_knn( \\ - raft::resources const& handle, \\ - raft::device_matrix_view index, \\ - raft::device_matrix_view query, \\ - raft::device_matrix_view out_inds, \\ - raft::device_matrix_view out_dists, \\ - raft::distance::DistanceType metric); - -""" - -knn_types = dict( - int64_t_float_uint32_t=("int64_t","float","uint32_t"), - int64_t_float_int64_t=("int64_t","float","int64_t"), - int_float_int=("int","float","int"), - uint32_t_float_uint32_t=("uint32_t","float","uint32_t"), -) - -fused_l2_knn_types = dict( - float_int64_t=("float", "int64_t"), -) - -# knn -for type_path, (idx_t, value_t, matrix_idx) in knn_types.items(): - path = f"brute_force_knn_{type_path}.cu" - with open(path, "w") as f: - f.write(header) - f.write(knn_macro) - f.write(f"instantiate_raft_neighbors_brute_force_knn({idx_t},{value_t},{matrix_idx},raft::row_major,raft::row_major,raft::identity_op);\n\n") - f.write("#undef instantiate_raft_neighbors_brute_force_knn\n") - - # For pasting into CMakeLists.txt - print(f"src/neighbors/{path}") - -#fused_l2_knn -for type_path, (value_t, idx_t) in fused_l2_knn_types.items(): - path = f"brute_force_fused_l2_knn_{type_path}.cu" - with open(path, "w") as f: - f.write(header) - f.write(fused_l2_knn_macro) - f.write(f"instantiate_raft_neighbors_brute_force_fused_l2_knn({value_t},{idx_t},raft::row_major,raft::row_major);\n\n") - f.write("#undef instantiate_raft_neighbors_brute_force_fused_l2_knn\n") - - # For pasting into CMakeLists.txt - print(f"src/neighbors/{path}") diff --git a/cpp/src/neighbors/brute_force_fused_l2_knn_float_int64_t.cu b/cpp/src/neighbors/brute_force_fused_l2_knn_float_int64_t.cu deleted file mode 100644 index 4269d27225..0000000000 --- a/cpp/src/neighbors/brute_force_fused_l2_knn_float_int64_t.cu +++ /dev/null @@ -1,46 +0,0 @@ - -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by brute_force_00_generate.py - * - * Make changes there and run in this directory: - * - * > python brute_force_00_generate.py - * - */ - -#include - -#include - -#define instantiate_raft_neighbors_brute_force_fused_l2_knn( \ - value_t, idx_t, idx_layout, query_layout) \ - template void raft::neighbors::brute_force::fused_l2_knn( \ - raft::resources const& handle, \ - raft::device_matrix_view index, \ - raft::device_matrix_view query, \ - raft::device_matrix_view out_inds, \ - raft::device_matrix_view out_dists, \ - raft::distance::DistanceType metric); - -instantiate_raft_neighbors_brute_force_fused_l2_knn(float, - int64_t, - raft::row_major, - raft::row_major); - -#undef instantiate_raft_neighbors_brute_force_fused_l2_knn diff --git a/cpp/src/neighbors/brute_force_knn_index_float.cu b/cpp/src/neighbors/brute_force_knn_index_float.cu deleted file mode 100644 index de94be4c09..0000000000 --- a/cpp/src/neighbors/brute_force_knn_index_float.cu +++ /dev/null @@ -1,79 +0,0 @@ - -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include - -template void raft::neighbors::brute_force::search( - raft::resources const& res, - const raft::neighbors::brute_force::index& idx, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances); - -template void raft::neighbors::brute_force::search( - raft::resources const& res, - raft::neighbors::brute_force::search_params const& params, - const raft::neighbors::brute_force::index& idx, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances); - -template void raft::neighbors::brute_force::search( - raft::resources const& res, - const raft::neighbors::brute_force::index& idx, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances); - -template void raft::neighbors::brute_force::search( - raft::resources const& res, - raft::neighbors::brute_force::search_params const& params, - const raft::neighbors::brute_force::index& idx, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances); - -template raft::neighbors::brute_force::index raft::neighbors::brute_force:: - build::accessor_type>( - raft::resources const& res, - raft::host_matrix_view dataset, - raft::distance::DistanceType metric, - float metric_arg); - -template raft::neighbors::brute_force::index raft::neighbors::brute_force:: - build::accessor_type>( - raft::resources const& res, - raft::device_matrix_view dataset, - raft::distance::DistanceType metric, - float metric_arg); - -template raft::neighbors::brute_force::index raft::neighbors::brute_force:: - build::accessor_type>( - raft::resources const& res, - raft::neighbors::brute_force::index_params const& params, - raft::host_matrix_view dataset); - -template raft::neighbors::brute_force::index raft::neighbors::brute_force:: - build::accessor_type>( - raft::resources const& res, - raft::neighbors::brute_force::index_params const& params, - raft::device_matrix_view dataset); diff --git a/cpp/src/neighbors/brute_force_knn_int64_t_float_int64_t.cu b/cpp/src/neighbors/brute_force_knn_int64_t_float_int64_t.cu deleted file mode 100644 index 1c08cf8e82..0000000000 --- a/cpp/src/neighbors/brute_force_knn_int64_t_float_int64_t.cu +++ /dev/null @@ -1,48 +0,0 @@ - -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by brute_force_00_generate.py - * - * Make changes there and run in this directory: - * - * > python brute_force_00_generate.py - * - */ - -#include - -#include - -#define instantiate_raft_neighbors_brute_force_knn( \ - idx_t, value_t, matrix_idx, index_layout, search_layout, epilogue_op) \ - template void raft::neighbors::brute_force:: \ - knn( \ - raft::resources const& handle, \ - std::vector> index, \ - raft::device_matrix_view search, \ - raft::device_matrix_view indices, \ - raft::device_matrix_view distances, \ - raft::distance::DistanceType metric, \ - std::optional metric_arg, \ - std::optional global_id_offset, \ - epilogue_op distance_epilogue); - -instantiate_raft_neighbors_brute_force_knn( - int64_t, float, int64_t, raft::row_major, raft::row_major, raft::identity_op); - -#undef instantiate_raft_neighbors_brute_force_knn diff --git a/cpp/src/neighbors/brute_force_knn_int64_t_float_uint32_t.cu b/cpp/src/neighbors/brute_force_knn_int64_t_float_uint32_t.cu deleted file mode 100644 index 809cf6eec0..0000000000 --- a/cpp/src/neighbors/brute_force_knn_int64_t_float_uint32_t.cu +++ /dev/null @@ -1,48 +0,0 @@ - -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by brute_force_00_generate.py - * - * Make changes there and run in this directory: - * - * > python brute_force_00_generate.py - * - */ - -#include - -#include - -#define instantiate_raft_neighbors_brute_force_knn( \ - idx_t, value_t, matrix_idx, index_layout, search_layout, epilogue_op) \ - template void raft::neighbors::brute_force:: \ - knn( \ - raft::resources const& handle, \ - std::vector> index, \ - raft::device_matrix_view search, \ - raft::device_matrix_view indices, \ - raft::device_matrix_view distances, \ - raft::distance::DistanceType metric, \ - std::optional metric_arg, \ - std::optional global_id_offset, \ - epilogue_op distance_epilogue); - -instantiate_raft_neighbors_brute_force_knn( - int64_t, float, uint32_t, raft::row_major, raft::row_major, raft::identity_op); - -#undef instantiate_raft_neighbors_brute_force_knn diff --git a/cpp/src/neighbors/brute_force_knn_int_float_int.cu b/cpp/src/neighbors/brute_force_knn_int_float_int.cu deleted file mode 100644 index 2ffa864dea..0000000000 --- a/cpp/src/neighbors/brute_force_knn_int_float_int.cu +++ /dev/null @@ -1,48 +0,0 @@ - -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by brute_force_00_generate.py - * - * Make changes there and run in this directory: - * - * > python brute_force_00_generate.py - * - */ - -#include - -#include - -#define instantiate_raft_neighbors_brute_force_knn( \ - idx_t, value_t, matrix_idx, index_layout, search_layout, epilogue_op) \ - template void raft::neighbors::brute_force:: \ - knn( \ - raft::resources const& handle, \ - std::vector> index, \ - raft::device_matrix_view search, \ - raft::device_matrix_view indices, \ - raft::device_matrix_view distances, \ - raft::distance::DistanceType metric, \ - std::optional metric_arg, \ - std::optional global_id_offset, \ - epilogue_op distance_epilogue); - -instantiate_raft_neighbors_brute_force_knn( - int, float, int, raft::row_major, raft::row_major, raft::identity_op); - -#undef instantiate_raft_neighbors_brute_force_knn diff --git a/cpp/src/neighbors/brute_force_knn_uint32_t_float_uint32_t.cu b/cpp/src/neighbors/brute_force_knn_uint32_t_float_uint32_t.cu deleted file mode 100644 index dde92765b5..0000000000 --- a/cpp/src/neighbors/brute_force_knn_uint32_t_float_uint32_t.cu +++ /dev/null @@ -1,48 +0,0 @@ - -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by brute_force_00_generate.py - * - * Make changes there and run in this directory: - * - * > python brute_force_00_generate.py - * - */ - -#include - -#include - -#define instantiate_raft_neighbors_brute_force_knn( \ - idx_t, value_t, matrix_idx, index_layout, search_layout, epilogue_op) \ - template void raft::neighbors::brute_force:: \ - knn( \ - raft::resources const& handle, \ - std::vector> index, \ - raft::device_matrix_view search, \ - raft::device_matrix_view indices, \ - raft::device_matrix_view distances, \ - raft::distance::DistanceType metric, \ - std::optional metric_arg, \ - std::optional global_id_offset, \ - epilogue_op distance_epilogue); - -instantiate_raft_neighbors_brute_force_knn( - uint32_t, float, uint32_t, raft::row_major, raft::row_major, raft::identity_op); - -#undef instantiate_raft_neighbors_brute_force_knn diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_00_generate.py b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_00_generate.py deleted file mode 100644 index e827c06be5..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_00_generate.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -header = """/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -""" - -trailer = """ -} // namespace raft::neighbors::cagra::detail::multi_cta_search -""" - -mxdim_team = [(128, 8), (256, 16), (512, 32), (1024, 32)] -pq_bits = [8] -subspace_dims = [2, 4] -# block = [(64, 16), (128, 8), (256, 4), (512, 2), (1024, 1)] -# mxelem = [64, 128, 256] -load_types = ["uint4"] -code_book_types = ["half"] -search_types = dict( - float_uint32=( - "float", - "uint32_t", - "float", - ), # data_t, vec_idx_t, distance_t - half_uint32=("half", "uint32_t", "float"), - int8_uint32=("int8_t", "uint32_t", "float"), - uint8_uint32=("uint8_t", "uint32_t", "float"), - float_uint64=("float", "uint64_t", "float"), - half_uint64=("half", "uint64_t", "float"), -) -# knn -for type_path, (data_t, idx_t, distance_t) in search_types.items(): - for (mxdim, team) in mxdim_team: - for code_book_t in code_book_types: - for subspace_dim in subspace_dims: - for pq_bit in pq_bits: - path = f"q_search_multi_cta_{type_path}_dim{mxdim}_t{team}_{pq_bit}pq_{subspace_dim}subd_{code_book_t}.cu" - with open(path, "w") as f: - f.write(header) - f.write( - f"instantiate_kernel_selection(\n {team}, {mxdim}, raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t<{data_t} COMMA {code_book_t} COMMA {pq_bit} COMMA {subspace_dim} COMMA {distance_t} COMMA {idx_t}>, raft::neighbors::filtering::none_cagra_sample_filter);\n" - ) - f.write(trailer) - # For pasting into CMakeLists.txt - print(f"src/neighbors/detail/cagra/{path}") diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index 0bd386144c..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index cd891b8e97..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index 66e8357498..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index eb84983f9e..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index c66f8a0ae3..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index 2a1783944c..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 9fa74f1134..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index 8fc91b5a10..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index 4e68c00525..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index 5fe526ae47..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index 64c89a880a..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index c3e2427f57..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 0a8826df1c..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index 8019bec3e3..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 1a2a364037..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index 2f661538e6..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index aec486769f..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index 03f27085d8..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index 119d1f2921..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index 666c676e87..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index e53b456a54..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index 2aee739141..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index daa442b514..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index a19346d19b..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index 1c1d5381c9..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index b7402a3c38..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index f493b83bee..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index 8efcbe0650..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index cb770f44ba..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index 0fd8ab809c..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 50cf198883..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index 1548ed831e..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index c60ea7c87d..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index 4a68e1e43c..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index df9fabd6a5..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index 77075b0a44..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 374af8b56b..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index ddb80458fd..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 14e5c5d3dc..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index 3c1776760a..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index e5a0a8882c..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index cee80390e8..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index 88678bf4ff..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index baa7ee358a..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 5c44f052f2..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index 127a065fb5..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index fcf6985f97..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index f361e771b5..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_00_generate.py b/cpp/src/neighbors/detail/cagra/q_search_single_cta_00_generate.py deleted file mode 100644 index 418d528a82..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_00_generate.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -header = """/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -""" - -trailer = """ -} // namespace raft::neighbors::cagra::detail::single_cta_search -""" - -mxdim_team = [(128, 8), (256, 16), (512, 32), (1024, 32)] -# block = [(64, 16), (128, 8), (256, 4), (512, 2), (1024, 1)] -# itopk_candidates = [64, 128, 256] -# itopk_size = [64, 128, 256, 512] -# mxelem = [64, 128, 256] - -pq_bits = [8] -subspace_dims = [2, 4] - -# rblock = [(256, 4), (512, 2), (1024, 1)] -# rcandidates = [32] -# rsize = [256, 512] -code_book_types = ["half"] - -search_types = dict( - float_uint32=("float", "uint32_t", "float"), # data_t, idx_t, distance_t - half_uint32=("half", "uint32_t", "float"), - int8_uint32=("int8_t", "uint32_t", "float"), - uint8_uint32=("uint8_t", "uint32_t", "float"), - float_uint64=("float", "uint64_t", "float"), - half_uint64=("half", "uint64_t", "float"), -) - -# knn -for type_path, (data_t, idx_t, distance_t) in search_types.items(): - for (mxdim, team) in mxdim_team: - for code_book_t in code_book_types: - for subspace_dim in subspace_dims: - for pq_bit in pq_bits: - path = f"q_search_single_cta_{type_path}_dim{mxdim}_t{team}_{pq_bit}pq_{subspace_dim}subd_{code_book_t}.cu" - with open(path, "w") as f: - f.write(header) - f.write( - f"instantiate_kernel_selection(\n {team}, {mxdim}, raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t<{data_t} COMMA {code_book_t} COMMA {pq_bit} COMMA {subspace_dim} COMMA {distance_t} COMMA {idx_t}>, raft::neighbors::filtering::none_cagra_sample_filter);\n" - ) - - f.write(trailer) - # For pasting into CMakeLists.txt - print(f"src/neighbors/detail/cagra/{path}") diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index d61ad0ce15..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index 410d2377ec..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index 60cd58bab9..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index dfe5e6f14e..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 9a5d862276..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index d92ab50a58..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index aac197d590..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index f38a10e6d0..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index 5523e63038..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index b06ef3d4fd..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index 1fddee0e06..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index 2aee442186..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 7a15e85280..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index efba46c248..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 990582f18b..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index a55907c66f..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index 55fd749720..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index 4b4063652a..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index bae83dc0fa..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index 99492db344..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 797142e317..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index 9a36c35ae0..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index e0a01e84cc..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index 14de1b8941..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index b1d50fb445..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index c189a91764..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index 8693ee3716..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index 216ffd1ec5..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 36985d218b..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index 8d55fe2b09..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 2fdb1cbc20..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index 6dc3dc2ca8..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index 21f8633033..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index 1a3867e06f..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index 9cbb16188a..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index 305a1754bc..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 900e1b69d9..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index a0bb2259f0..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 09d36a39a0..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index dc9cbb2b56..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index c5508a38e2..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index 7024425155..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index 68687bc9cf..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index 60efc55a30..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index b2dfaac5fe..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index 891e9ef7cc..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 91e617204c..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index a01d497676..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta.cuh b/cpp/src/neighbors/detail/cagra/search_multi_cta.cuh deleted file mode 100644 index 542fdaad1f..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta.cuh +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { - -#define instantiate_kernel_selection(TEAM_SIZE, MAX_DATASET_DIM, DATASET_DESC_T, SAMPLE_FILTER_T) \ - template void select_and_run( \ - DATASET_DESC_T dataset_desc, \ - raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ - const uint32_t num_queries, \ - const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ - uint32_t topk, \ - uint32_t block_size, \ - uint32_t result_buffer_size, \ - uint32_t smem_size, \ - int64_t hash_bitlen, \ - typename DATASET_DESC_T::INDEX_T* hashmap_ptr, \ - uint32_t num_cta_per_query, \ - uint32_t num_random_samplings, \ - uint64_t rand_xor_mask, \ - uint32_t num_seeds, \ - size_t itopk_size, \ - size_t search_width, \ - size_t min_iterations, \ - size_t max_iterations, \ - SAMPLE_FILTER_T sample_filter, \ - raft::distance::DistanceType metric, \ - cudaStream_t stream); - -#define COMMA , - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_00_generate.py b/cpp/src/neighbors/detail/cagra/search_multi_cta_00_generate.py deleted file mode 100644 index 6f023c39f1..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_00_generate.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -header = """/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -""" - -trailer = """ -} // namespace raft::neighbors::cagra::detail::multi_cta_search -""" - -mxdim_team = [(128, 8), (256, 16), (512, 32), (1024, 32)] -# block = [(64, 16), (128, 8), (256, 4), (512, 2), (1024, 1)] -# mxelem = [64, 128, 256] -load_types = ["uint4"] -search_types = dict( - float_uint32=( - "float", - "uint32_t", - "float", - ), # data_t, vec_idx_t, distance_t - half_uint32=("half", "uint32_t", "float"), - int8_uint32=("int8_t", "uint32_t", "float"), - uint8_uint32=("uint8_t", "uint32_t", "float"), - float_uint64=("float", "uint64_t", "float"), - half_uint64=("half", "uint64_t", "float"), -) -# knn -for type_path, (data_t, idx_t, distance_t) in search_types.items(): - for (mxdim, team) in mxdim_team: - path = f"search_multi_cta_{type_path}_dim{mxdim}_t{team}.cu" - with open(path, "w") as f: - f.write(header) - f.write( - f"instantiate_kernel_selection(\n {team}, {mxdim}, raft::neighbors::cagra::detail::standard_dataset_descriptor_t<{data_t} COMMA {idx_t} COMMA {distance_t}>, raft::neighbors::filtering::none_cagra_sample_filter);\n" - ) - f.write(trailer) - # For pasting into CMakeLists.txt - print(f"src/neighbors/detail/cagra/{path}") diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim1024_t32.cu deleted file mode 100644 index 0e28d7a876..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim128_t8.cu deleted file mode 100644 index 5e5e80a5de..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim128_t8.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim256_t16.cu deleted file mode 100644 index 9039496968..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim512_t32.cu deleted file mode 100644 index fe1c7e77e5..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim1024_t32.cu deleted file mode 100644 index 7ef36baf7d..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim128_t8.cu deleted file mode 100644 index da51c16314..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim128_t8.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim256_t16.cu deleted file mode 100644 index 99a4f7feb7..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim512_t32.cu deleted file mode 100644 index 50cdc97dd7..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim1024_t32.cu deleted file mode 100644 index b2d9cdb600..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim128_t8.cu deleted file mode 100644 index d756b295b7..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim128_t8.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim256_t16.cu deleted file mode 100644 index b1e998762c..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim512_t32.cu deleted file mode 100644 index e712de6390..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim1024_t32.cu deleted file mode 100644 index 282de4a851..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim128_t8.cu deleted file mode 100644 index 71ef968575..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim128_t8.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim256_t16.cu deleted file mode 100644 index 7c88406d71..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim512_t32.cu deleted file mode 100644 index 360635dddb..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim1024_t32.cu deleted file mode 100644 index 3f129bd7cf..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim128_t8.cu deleted file mode 100644 index 053b73275e..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim128_t8.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim256_t16.cu deleted file mode 100644 index a1bb20369a..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim512_t32.cu deleted file mode 100644 index dbbc8bdd21..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim1024_t32.cu deleted file mode 100644 index 125499e319..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim128_t8.cu deleted file mode 100644 index f2117c4f80..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim128_t8.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim256_t16.cu deleted file mode 100644 index 8e5ba0f98f..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim512_t32.cu deleted file mode 100644 index bea7d25392..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta.cuh b/cpp/src/neighbors/detail/cagra/search_single_cta.cuh deleted file mode 100644 index 855b104670..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta.cuh +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { - -#define instantiate_kernel_selection(TEAM_SIZE, MAX_DATASET_DIM, DATASET_DESC_T, SAMPLE_FILTER_T) \ - template void select_and_run( \ - DATASET_DESC_T dataset_desc, \ - raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ - const uint32_t num_queries, \ - const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ - uint32_t topk, \ - uint32_t num_itopk_candidates, \ - uint32_t block_size, \ - uint32_t smem_size, \ - int64_t hash_bitlen, \ - typename DATASET_DESC_T::INDEX_T* hashmap_ptr, \ - size_t small_hash_bitlen, \ - size_t small_hash_reset_interval, \ - uint32_t num_random_samplings, \ - uint64_t rand_xor_mask, \ - uint32_t num_seeds, \ - size_t itopk_size, \ - size_t search_width, \ - size_t min_iterations, \ - size_t max_iterations, \ - SAMPLE_FILTER_T sample_filter, \ - raft::distance::DistanceType metric, \ - cudaStream_t stream); - -#define COMMA , - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_00_generate.py b/cpp/src/neighbors/detail/cagra/search_single_cta_00_generate.py deleted file mode 100644 index 0e809e4dc3..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_00_generate.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -header = """/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -""" - -trailer = """ -} // namespace raft::neighbors::cagra::detail::single_cta_search -""" - -mxdim_team = [(128, 8), (256, 16), (512, 32), (1024, 32)] -# block = [(64, 16), (128, 8), (256, 4), (512, 2), (1024, 1)] -# itopk_candidates = [64, 128, 256] -# itopk_size = [64, 128, 256, 512] -# mxelem = [64, 128, 256] - -# rblock = [(256, 4), (512, 2), (1024, 1)] -# rcandidates = [32] -# rsize = [256, 512] - -search_types = dict( - float_uint32=("float", "uint32_t", "float"), # data_t, idx_t, distance_t - half_uint32=("half", "uint32_t", "float"), - int8_uint32=("int8_t", "uint32_t", "float"), - uint8_uint32=("uint8_t", "uint32_t", "float"), - float_uint64=("float", "uint64_t", "float"), - half_uint64=("half", "uint64_t", "float"), -) - -# knn -for type_path, (data_t, idx_t, distance_t) in search_types.items(): - for (mxdim, team) in mxdim_team: - path = f"search_single_cta_{type_path}_dim{mxdim}_t{team}.cu" - with open(path, "w") as f: - f.write(header) - f.write( - f"instantiate_kernel_selection(\n {team}, {mxdim}, raft::neighbors::cagra::detail::standard_dataset_descriptor_t<{data_t} COMMA {idx_t} COMMA {distance_t}>, raft::neighbors::filtering::none_cagra_sample_filter);\n" - ) - - f.write(trailer) - # For pasting into CMakeLists.txt - print(f"src/neighbors/detail/cagra/{path}") diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim1024_t32.cu deleted file mode 100644 index 8a9fc408ee..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim128_t8.cu deleted file mode 100644 index c6f7c90c69..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim128_t8.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim256_t16.cu deleted file mode 100644 index 2766286673..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim512_t32.cu deleted file mode 100644 index 98ee189766..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim1024_t32.cu deleted file mode 100644 index c3ea39a729..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim128_t8.cu deleted file mode 100644 index a53457656c..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim128_t8.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim256_t16.cu deleted file mode 100644 index 52318efb85..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim512_t32.cu deleted file mode 100644 index 6451fdc7f3..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim1024_t32.cu deleted file mode 100644 index e927fd0878..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim128_t8.cu deleted file mode 100644 index 3f3d22ee08..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim128_t8.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim256_t16.cu deleted file mode 100644 index a84e5b8bd7..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim512_t32.cu deleted file mode 100644 index af4248865b..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim1024_t32.cu deleted file mode 100644 index 16bd0cb647..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim128_t8.cu deleted file mode 100644 index afc59c8a59..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim128_t8.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim256_t16.cu deleted file mode 100644 index 147d31cf85..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim512_t32.cu deleted file mode 100644 index 5624a71c3c..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim1024_t32.cu deleted file mode 100644 index 761fb705ba..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim128_t8.cu deleted file mode 100644 index 84b76cba53..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim128_t8.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim256_t16.cu deleted file mode 100644 index 598fff9cdf..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim512_t32.cu deleted file mode 100644 index e7a1a9d9c6..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim1024_t32.cu deleted file mode 100644 index d40b9285fc..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim128_t8.cu deleted file mode 100644 index 073bb350da..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim128_t8.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim256_t16.cu deleted file mode 100644 index 29b0224b4d..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim512_t32.cu deleted file mode 100644 index d9601de2ad..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_float_float_int64_t.cu b/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_float_float_int64_t.cu deleted file mode 100644 index 5ac820e0dd..0000000000 --- a/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_float_float_int64_t.cu +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#define instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( \ - T, AccT, IdxT, IvfSampleFilterT) \ - template void \ - raft::neighbors::ivf_flat::detail::ivfflat_interleaved_scan( \ - const raft::neighbors::ivf_flat::index& index, \ - const T* queries, \ - const uint32_t* coarse_query_results, \ - const uint32_t n_queries, \ - const uint32_t queries_offset, \ - const raft::distance::DistanceType metric, \ - const uint32_t n_probes, \ - const uint32_t k, \ - const uint32_t max_samples, \ - const uint32_t* chunk_indices, \ - const bool select_min, \ - IvfSampleFilterT sample_filter, \ - uint32_t* neighbors, \ - float* distances, \ - uint32_t& grid_dim_x, \ - rmm::cuda_stream_view stream) - -instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( - float, float, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); - -#undef instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan diff --git a/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_half_half_int64_t.cu b/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_half_half_int64_t.cu deleted file mode 100644 index 4d847cdeb1..0000000000 --- a/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_half_half_int64_t.cu +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include - -#define instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( \ - T, AccT, IdxT, IvfSampleFilterT) \ - template void \ - raft::neighbors::ivf_flat::detail::ivfflat_interleaved_scan( \ - const raft::neighbors::ivf_flat::index& index, \ - const T* queries, \ - const uint32_t* coarse_query_results, \ - const uint32_t n_queries, \ - const uint32_t queries_offset, \ - const raft::distance::DistanceType metric, \ - const uint32_t n_probes, \ - const uint32_t k, \ - const uint32_t max_samples, \ - const uint32_t* chunk_indices, \ - const bool select_min, \ - IvfSampleFilterT sample_filter, \ - uint32_t* neighbors, \ - float* distances, \ - uint32_t& grid_dim_x, \ - rmm::cuda_stream_view stream) - -instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( - half, half, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); - -#undef instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan diff --git a/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_int8_t_int32_t_int64_t.cu b/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_int8_t_int32_t_int64_t.cu deleted file mode 100644 index 8a0e8f0118..0000000000 --- a/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_int8_t_int32_t_int64_t.cu +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#define instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( \ - T, AccT, IdxT, IvfSampleFilterT) \ - template void \ - raft::neighbors::ivf_flat::detail::ivfflat_interleaved_scan( \ - const raft::neighbors::ivf_flat::index& index, \ - const T* queries, \ - const uint32_t* coarse_query_results, \ - const uint32_t n_queries, \ - const uint32_t queries_offset, \ - const raft::distance::DistanceType metric, \ - const uint32_t n_probes, \ - const uint32_t k, \ - const uint32_t max_samples, \ - const uint32_t* chunk_indices, \ - const bool select_min, \ - IvfSampleFilterT sample_filter, \ - uint32_t* neighbors, \ - float* distances, \ - uint32_t& grid_dim_x, \ - rmm::cuda_stream_view stream) - -instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( - int8_t, int32_t, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); - -#undef instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan diff --git a/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_uint8_t_uint32_t_int64_t.cu b/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_uint8_t_uint32_t_int64_t.cu deleted file mode 100644 index 7cad992e2b..0000000000 --- a/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_uint8_t_uint32_t_int64_t.cu +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#define instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( \ - T, AccT, IdxT, IvfSampleFilterT) \ - template void \ - raft::neighbors::ivf_flat::detail::ivfflat_interleaved_scan( \ - const raft::neighbors::ivf_flat::index& index, \ - const T* queries, \ - const uint32_t* coarse_query_results, \ - const uint32_t n_queries, \ - const uint32_t queries_offset, \ - const raft::distance::DistanceType metric, \ - const uint32_t n_probes, \ - const uint32_t k, \ - const uint32_t max_samples, \ - const uint32_t* chunk_indices, \ - const bool select_min, \ - IvfSampleFilterT sample_filter, \ - uint32_t* neighbors, \ - float* distances, \ - uint32_t& grid_dim_x, \ - rmm::cuda_stream_view stream) - -instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( - uint8_t, uint32_t, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); - -#undef instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan diff --git a/cpp/src/neighbors/detail/ivf_flat_search.cu b/cpp/src/neighbors/detail/ivf_flat_search.cu deleted file mode 100644 index 336bea19b6..0000000000 --- a/cpp/src/neighbors/detail/ivf_flat_search.cu +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include - -#define instantiate_raft_neighbors_ivf_flat_detail_search(T, IdxT, IvfSampleFilterT) \ - template void raft::neighbors::ivf_flat::detail::search( \ - raft::resources const& handle, \ - const search_params& params, \ - const raft::neighbors::ivf_flat::index& index, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances, \ - rmm::device_async_resource_ref mr, \ - IvfSampleFilterT sample_filter) - -instantiate_raft_neighbors_ivf_flat_detail_search( - float, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); -instantiate_raft_neighbors_ivf_flat_detail_search( - int8_t, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); -instantiate_raft_neighbors_ivf_flat_detail_search( - uint8_t, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); - -#undef instantiate_raft_neighbors_ivf_flat_detail_search diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_00_generate.py b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_00_generate.py deleted file mode 100644 index 9825a48f81..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_00_generate.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -header = """/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -""" - -none_filter_int64 = "raft::neighbors::filtering::ivf_to_sample_filter" \ - "" -none_filter_int32 = "raft::neighbors::filtering::ivf_to_sample_filter" \ - "" -bitset_filter32 = "raft::neighbors::filtering::ivf_to_sample_filter" \ - ">" -bitset_filter64 = "raft::neighbors::filtering::ivf_to_sample_filter" \ - ">" - -types = dict( - half_fp8_false=("half", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>", none_filter_int64), - half_fp8_true=("half", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>", none_filter_int64), - half_half=("half", "half", none_filter_int64), - float_half=("float", "half", none_filter_int64), - float_float= ("float", "float", none_filter_int64), - float_fp8_false=("float", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>", none_filter_int64), - float_fp8_true=("float", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>", none_filter_int64), - half_fp8_false_filt32=("half", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>", none_filter_int32), - half_fp8_true_filt32=("half", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>", none_filter_int32), - half_half_filt32=("half", "half", none_filter_int32), - float_half_filt32=("float", "half", none_filter_int32), - float_float_filt32= ("float", "float", none_filter_int32), - float_fp8_false_filt32=("float", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>", none_filter_int32), - float_fp8_true_filt32=("float", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>", none_filter_int32), - half_fp8_false_bitset32=("half", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>", bitset_filter32), - half_fp8_true_bitset32=("half", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>", bitset_filter32), - half_half_bitset32=("half", "half", bitset_filter32), - float_half_bitset32=("float", "half", bitset_filter32), - float_float_bitset32= ("float", "float", bitset_filter32), - float_fp8_false_bitset32=("float", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>", bitset_filter32), - float_fp8_true_bitset32=("float", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>", bitset_filter32), - half_fp8_false_bitset64=("half", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>", bitset_filter64), - half_fp8_true_bitset64=("half", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>", bitset_filter64), - half_half_bitset64=("half", "half", bitset_filter64), - float_half_bitset64=("float", "half", bitset_filter64), - float_float_bitset64= ("float", "float", bitset_filter64), - float_fp8_false_bitset64=("float", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>", bitset_filter64), - float_fp8_true_bitset64=("float", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>", bitset_filter64) -) - -for path_key, (OutT, LutT, FilterT) in types.items(): - path = f"ivf_pq_compute_similarity_{path_key}.cu" - with open(path, "w") as f: - f.write(header) - f.write(f"instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select({OutT}, {LutT}, {FilterT});\n") - print(f"src/neighbors/detail/{path}") diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float.cu deleted file mode 100644 index db51608ae1..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - float, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float_bitset32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float_bitset32.cu deleted file mode 100644 index caaf40abdf..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float_bitset32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - float, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float_bitset64.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float_bitset64.cu deleted file mode 100644 index 7801c25e9f..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float_bitset64.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - float, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float_filt32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float_filt32.cu deleted file mode 100644 index 45ae348849..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float_filt32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - float, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false.cu deleted file mode 100644 index 2f5bcf8f92..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_bitset32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_bitset32.cu deleted file mode 100644 index e7f2c44254..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_bitset32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_bitset64.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_bitset64.cu deleted file mode 100644 index 01b6900bb8..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_bitset64.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_filt32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_filt32.cu deleted file mode 100644 index 9f8d453364..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_filt32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true.cu deleted file mode 100644 index 06d21bcd50..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_bitset32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_bitset32.cu deleted file mode 100644 index 8b733a23c1..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_bitset32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_bitset64.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_bitset64.cu deleted file mode 100644 index 77e4f9a023..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_bitset64.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_filt32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_filt32.cu deleted file mode 100644 index 3e036e3df4..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_filt32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half.cu deleted file mode 100644 index ff42f5e041..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half_bitset32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half_bitset32.cu deleted file mode 100644 index 40b6313865..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half_bitset32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half_bitset64.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half_bitset64.cu deleted file mode 100644 index 9cedabdb11..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half_bitset64.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half_filt32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half_filt32.cu deleted file mode 100644 index 61422bbc36..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half_filt32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false.cu deleted file mode 100644 index d2064cfe97..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_bitset32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_bitset32.cu deleted file mode 100644 index 1127f39f71..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_bitset32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_bitset64.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_bitset64.cu deleted file mode 100644 index 0330bf58d6..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_bitset64.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_filt32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_filt32.cu deleted file mode 100644 index d20f7921d5..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_filt32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true.cu deleted file mode 100644 index 9dc954406e..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_bitset32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_bitset32.cu deleted file mode 100644 index 9131fa25a8..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_bitset32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_bitset64.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_bitset64.cu deleted file mode 100644 index 8b4521b31b..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_bitset64.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_filt32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_filt32.cu deleted file mode 100644 index 71b63cf4a0..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_filt32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half.cu deleted file mode 100644 index f527d879be..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half_bitset32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half_bitset32.cu deleted file mode 100644 index 8e1962e2bb..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half_bitset32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half_bitset64.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half_bitset64.cu deleted file mode 100644 index e9671703e7..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half_bitset64.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half_filt32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half_filt32.cu deleted file mode 100644 index b66a07d1a9..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half_filt32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_search_filtering_float_int64_t.cu b/cpp/src/neighbors/detail/ivf_pq_search_filtering_float_int64_t.cu deleted file mode 100644 index 39af78f12e..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_search_filtering_float_int64_t.cu +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include // raft::device_matrix_view -#include // raft::resources -#include -#include // raft::neighbors::ivf_pq::index -#include -#include - -#include - -#include // int64_t - -#define instantiate_raft_neighbors_ivf_pq_search_with_filtering(T, IdxT, FilterT) \ - template void raft::neighbors::ivf_pq::search_with_filtering( \ - raft::resources const& handle, \ - const search_params& params, \ - const index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances, \ - FilterT sample_filter) - -#define COMMA , -instantiate_raft_neighbors_ivf_pq_search_with_filtering( - float, int64_t, raft::neighbors::filtering::bitset_filter); - -#undef COMMA -#undef instantiate_raft_neighbors_ivf_pq_search_with_filtering diff --git a/cpp/src/neighbors/detail/refine_host_float_float.cpp b/cpp/src/neighbors/detail/refine_host_float_float.cpp deleted file mode 100644 index 09dcae9c3a..0000000000 --- a/cpp/src/neighbors/detail/refine_host_float_float.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include - -#define instantiate_raft_neighbors_refine(IdxT, DataT, DistanceT, ExtentsT) \ - template void raft::neighbors::detail::refine_host( \ - raft::host_matrix_view dataset, \ - raft::host_matrix_view queries, \ - raft::host_matrix_view neighbor_candidates, \ - raft::host_matrix_view indices, \ - raft::host_matrix_view distances, \ - distance::DistanceType metric); - -instantiate_raft_neighbors_refine(int64_t, float, float, int64_t); -instantiate_raft_neighbors_refine(uint32_t, float, float, int64_t); - -#undef instantiate_raft_neighbors_refine diff --git a/cpp/src/neighbors/detail/refine_host_half_float.cpp b/cpp/src/neighbors/detail/refine_host_half_float.cpp deleted file mode 100644 index d9fb2864fe..0000000000 --- a/cpp/src/neighbors/detail/refine_host_half_float.cpp +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include - -#include - -#define instantiate_raft_neighbors_refine(IdxT, DataT, DistanceT, ExtentsT) \ - template void raft::neighbors::detail::refine_host( \ - raft::host_matrix_view dataset, \ - raft::host_matrix_view queries, \ - raft::host_matrix_view neighbor_candidates, \ - raft::host_matrix_view indices, \ - raft::host_matrix_view distances, \ - distance::DistanceType metric); - -instantiate_raft_neighbors_refine(int64_t, half, float, int64_t); - -#undef instantiate_raft_neighbors_refine diff --git a/cpp/src/neighbors/detail/refine_host_int8_t_float.cpp b/cpp/src/neighbors/detail/refine_host_int8_t_float.cpp deleted file mode 100644 index 334a3e8cb6..0000000000 --- a/cpp/src/neighbors/detail/refine_host_int8_t_float.cpp +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#define instantiate_raft_neighbors_refine(IdxT, DataT, DistanceT, ExtentsT) \ - template void raft::neighbors::detail::refine_host( \ - raft::host_matrix_view dataset, \ - raft::host_matrix_view queries, \ - raft::host_matrix_view neighbor_candidates, \ - raft::host_matrix_view indices, \ - raft::host_matrix_view distances, \ - distance::DistanceType metric); -instantiate_raft_neighbors_refine(int64_t, int8_t, float, int64_t); - -#undef instantiate_raft_neighbors_refine diff --git a/cpp/src/neighbors/detail/refine_host_uint8_t_float.cpp b/cpp/src/neighbors/detail/refine_host_uint8_t_float.cpp deleted file mode 100644 index 43d93e5f2e..0000000000 --- a/cpp/src/neighbors/detail/refine_host_uint8_t_float.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#define instantiate_raft_neighbors_refine(IdxT, DataT, DistanceT, ExtentsT) \ - template void raft::neighbors::detail::refine_host( \ - raft::host_matrix_view dataset, \ - raft::host_matrix_view queries, \ - raft::host_matrix_view neighbor_candidates, \ - raft::host_matrix_view indices, \ - raft::host_matrix_view distances, \ - distance::DistanceType metric); - -instantiate_raft_neighbors_refine(int64_t, uint8_t, float, int64_t); - -#undef instantiate_raft_neighbors_refine diff --git a/cpp/src/neighbors/ivf_flat_00_generate.py b/cpp/src/neighbors/ivf_flat_00_generate.py deleted file mode 100644 index 7b55cad4de..0000000000 --- a/cpp/src/neighbors/ivf_flat_00_generate.py +++ /dev/null @@ -1,175 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -header = """/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_flat_00_generate.py - * - * Make changes there and run in this directory: - * - * > python ivf_flat_00_generate.py - * - */ - -#include -""" - -types = dict( - float_int64_t=("float", "int64_t"), - int8_t_int64_t=("int8_t", "int64_t"), - uint8_t_int64_t=("uint8_t", "int64_t"), -) - -build_macro = """ -#define instantiate_raft_neighbors_ivf_flat_build(T, IdxT) \\ - template auto raft::neighbors::ivf_flat::build( \\ - raft::resources const& handle, \\ - const raft::neighbors::ivf_flat::index_params& params, \\ - const T* dataset, \\ - IdxT n_rows, \\ - uint32_t dim) \\ - ->raft::neighbors::ivf_flat::index; \\ - \\ - template auto raft::neighbors::ivf_flat::build( \\ - raft::resources const& handle, \\ - const raft::neighbors::ivf_flat::index_params& params, \\ - raft::device_matrix_view dataset) \\ - ->raft::neighbors::ivf_flat::index; \\ - \\ - template void raft::neighbors::ivf_flat::build( \\ - raft::resources const& handle, \\ - const raft::neighbors::ivf_flat::index_params& params, \\ - raft::device_matrix_view dataset, \\ - raft::neighbors::ivf_flat::index& idx); \\ - \\ - template auto raft::neighbors::ivf_flat::build( \\ - raft::resources const& handle, \\ - const raft::neighbors::ivf_flat::index_params& params, \\ - raft::host_matrix_view dataset) \\ - ->raft::neighbors::ivf_flat::index; \\ - \\ - template void raft::neighbors::ivf_flat::build( \\ - raft::resources const& handle, \\ - const raft::neighbors::ivf_flat::index_params& params, \\ - raft::host_matrix_view dataset, \\ - raft::neighbors::ivf_flat::index& idx); -""" - -extend_macro = """ -#define instantiate_raft_neighbors_ivf_flat_extend(T, IdxT) \\ - template auto raft::neighbors::ivf_flat::extend( \\ - raft::resources const& handle, \\ - const raft::neighbors::ivf_flat::index& orig_index, \\ - const T* new_vectors, \\ - const IdxT* new_indices, \\ - IdxT n_rows) \\ - ->raft::neighbors::ivf_flat::index; \\ - \\ - template auto raft::neighbors::ivf_flat::extend( \\ - raft::resources const& handle, \\ - raft::device_matrix_view new_vectors, \\ - std::optional> new_indices, \\ - const raft::neighbors::ivf_flat::index& orig_index) \\ - ->raft::neighbors::ivf_flat::index; \\ - \\ - template void raft::neighbors::ivf_flat::extend( \\ - raft::resources const& handle, \\ - raft::neighbors::ivf_flat::index* index, \\ - const T* new_vectors, \\ - const IdxT* new_indices, \\ - IdxT n_rows); \\ - \\ - template void raft::neighbors::ivf_flat::extend( \\ - raft::resources const& handle, \\ - raft::device_matrix_view new_vectors, \\ - std::optional> new_indices, \\ - raft::neighbors::ivf_flat::index* index); \\ - \\ - template auto raft::neighbors::ivf_flat::extend( \\ - const raft::resources& handle, \\ - raft::host_matrix_view new_vectors, \\ - std::optional> new_indices, \\ - const raft::neighbors::ivf_flat::index& idx) \\ - -> raft::neighbors::ivf_flat::index; \\ - \\ - template void raft::neighbors::ivf_flat::extend( \\ - raft::resources const& handle, \\ - raft::host_matrix_view new_vectors, \\ - std::optional> new_indices, \\ - raft::neighbors::ivf_flat::index* index); -""" - -search_macro = """ -#define instantiate_raft_neighbors_ivf_flat_search(T, IdxT) \\ - template void raft::neighbors::ivf_flat::search( \\ - raft::resources const& handle, \\ - const raft::neighbors::ivf_flat::search_params& params, \\ - const raft::neighbors::ivf_flat::index& index, \\ - const T* queries, \\ - uint32_t n_queries, \\ - uint32_t k, \\ - IdxT* neighbors, \\ - float* distances, \\ - rmm::device_async_resource_ref mr); \\ - \\ - template void raft::neighbors::ivf_flat::search( \\ - raft::resources const& handle, \\ - const raft::neighbors::ivf_flat::search_params& params, \\ - const raft::neighbors::ivf_flat::index& index, \\ - raft::device_matrix_view queries, \\ - raft::device_matrix_view neighbors, \\ - raft::device_matrix_view distances); -""" - -macros = dict( - build=dict( - definition=build_macro, - name="instantiate_raft_neighbors_ivf_flat_build", - ), - extend=dict( - definition=extend_macro, - name="instantiate_raft_neighbors_ivf_flat_extend", - ), - search=dict( - definition=search_macro, - name="instantiate_raft_neighbors_ivf_flat_search", - ), -) - -for type_path, (T, IdxT) in types.items(): - for macro_path, macro in macros.items(): - path = f"ivf_flat_{macro_path}_{type_path}.cu" - with open(path, "w") as f: - f.write(header) - f.write(macro["definition"]) - - f.write(f"{macro['name']}({T}, {IdxT});\n\n") - f.write(f"#undef {macro['name']}\n") - - print(f"src/neighbors/{path}") diff --git a/cpp/src/neighbors/ivf_flat_build_float_int64_t.cu b/cpp/src/neighbors/ivf_flat_build_float_int64_t.cu deleted file mode 100644 index cf3cb6b1b2..0000000000 --- a/cpp/src/neighbors/ivf_flat_build_float_int64_t.cu +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_flat_00_generate.py - * - * Make changes there and run in this directory: - * - * > python ivf_flat_00_generate.py - * - */ - -#include - -#define instantiate_raft_neighbors_ivf_flat_build(T, IdxT) \ - template auto raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - const T* dataset, \ - IdxT n_rows, \ - uint32_t dim) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template auto raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::device_matrix_view dataset) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template void raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::device_matrix_view dataset, \ - raft::neighbors::ivf_flat::index& idx); \ - \ - template auto raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::host_matrix_view dataset) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template void raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::host_matrix_view dataset, \ - raft::neighbors::ivf_flat::index& idx); -instantiate_raft_neighbors_ivf_flat_build(float, int64_t); - -#undef instantiate_raft_neighbors_ivf_flat_build diff --git a/cpp/src/neighbors/ivf_flat_build_int8_t_int64_t.cu b/cpp/src/neighbors/ivf_flat_build_int8_t_int64_t.cu deleted file mode 100644 index e1cf64907e..0000000000 --- a/cpp/src/neighbors/ivf_flat_build_int8_t_int64_t.cu +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_flat_00_generate.py - * - * Make changes there and run in this directory: - * - * > python ivf_flat_00_generate.py - * - */ - -#include - -#define instantiate_raft_neighbors_ivf_flat_build(T, IdxT) \ - template auto raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - const T* dataset, \ - IdxT n_rows, \ - uint32_t dim) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template auto raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::device_matrix_view dataset) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template void raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::device_matrix_view dataset, \ - raft::neighbors::ivf_flat::index& idx); \ - \ - template auto raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::host_matrix_view dataset) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template void raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::host_matrix_view dataset, \ - raft::neighbors::ivf_flat::index& idx); -instantiate_raft_neighbors_ivf_flat_build(int8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_flat_build diff --git a/cpp/src/neighbors/ivf_flat_build_uint8_t_int64_t.cu b/cpp/src/neighbors/ivf_flat_build_uint8_t_int64_t.cu deleted file mode 100644 index 26d1647954..0000000000 --- a/cpp/src/neighbors/ivf_flat_build_uint8_t_int64_t.cu +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_flat_00_generate.py - * - * Make changes there and run in this directory: - * - * > python ivf_flat_00_generate.py - * - */ - -#include - -#define instantiate_raft_neighbors_ivf_flat_build(T, IdxT) \ - template auto raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - const T* dataset, \ - IdxT n_rows, \ - uint32_t dim) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template auto raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::device_matrix_view dataset) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template void raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::device_matrix_view dataset, \ - raft::neighbors::ivf_flat::index& idx); \ - \ - template auto raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::host_matrix_view dataset) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template void raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::host_matrix_view dataset, \ - raft::neighbors::ivf_flat::index& idx); -instantiate_raft_neighbors_ivf_flat_build(uint8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_flat_build diff --git a/cpp/src/neighbors/ivf_flat_extend_float_int64_t.cu b/cpp/src/neighbors/ivf_flat_extend_float_int64_t.cu deleted file mode 100644 index 16472c6692..0000000000 --- a/cpp/src/neighbors/ivf_flat_extend_float_int64_t.cu +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_flat_00_generate.py - * - * Make changes there and run in this directory: - * - * > python ivf_flat_00_generate.py - * - */ - -#include - -#define instantiate_raft_neighbors_ivf_flat_extend(T, IdxT) \ - template auto raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index& orig_index, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template auto raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_flat::index& orig_index) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template void raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::neighbors::ivf_flat::index* index, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows); \ - \ - template void raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_flat::index* index); \ - \ - template auto raft::neighbors::ivf_flat::extend( \ - const raft::resources& handle, \ - raft::host_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_flat::index& idx) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template void raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::host_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_flat::index* index); -instantiate_raft_neighbors_ivf_flat_extend(float, int64_t); - -#undef instantiate_raft_neighbors_ivf_flat_extend diff --git a/cpp/src/neighbors/ivf_flat_extend_int8_t_int64_t.cu b/cpp/src/neighbors/ivf_flat_extend_int8_t_int64_t.cu deleted file mode 100644 index d98b5225c3..0000000000 --- a/cpp/src/neighbors/ivf_flat_extend_int8_t_int64_t.cu +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_flat_00_generate.py - * - * Make changes there and run in this directory: - * - * > python ivf_flat_00_generate.py - * - */ - -#include - -#define instantiate_raft_neighbors_ivf_flat_extend(T, IdxT) \ - template auto raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index& orig_index, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template auto raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_flat::index& orig_index) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template void raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::neighbors::ivf_flat::index* index, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows); \ - \ - template void raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_flat::index* index); \ - \ - template auto raft::neighbors::ivf_flat::extend( \ - const raft::resources& handle, \ - raft::host_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_flat::index& idx) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template void raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::host_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_flat::index* index); -instantiate_raft_neighbors_ivf_flat_extend(int8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_flat_extend diff --git a/cpp/src/neighbors/ivf_flat_extend_uint8_t_int64_t.cu b/cpp/src/neighbors/ivf_flat_extend_uint8_t_int64_t.cu deleted file mode 100644 index 520c3be536..0000000000 --- a/cpp/src/neighbors/ivf_flat_extend_uint8_t_int64_t.cu +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_flat_00_generate.py - * - * Make changes there and run in this directory: - * - * > python ivf_flat_00_generate.py - * - */ - -#include - -#define instantiate_raft_neighbors_ivf_flat_extend(T, IdxT) \ - template auto raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index& orig_index, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template auto raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_flat::index& orig_index) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template void raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::neighbors::ivf_flat::index* index, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows); \ - \ - template void raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_flat::index* index); \ - \ - template auto raft::neighbors::ivf_flat::extend( \ - const raft::resources& handle, \ - raft::host_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_flat::index& idx) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template void raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::host_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_flat::index* index); -instantiate_raft_neighbors_ivf_flat_extend(uint8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_flat_extend diff --git a/cpp/src/neighbors/ivf_flat_search_float_int64_t.cu b/cpp/src/neighbors/ivf_flat_search_float_int64_t.cu deleted file mode 100644 index e5cfe14e3f..0000000000 --- a/cpp/src/neighbors/ivf_flat_search_float_int64_t.cu +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_flat_00_generate.py - * - * Make changes there and run in this directory: - * - * > python ivf_flat_00_generate.py - * - */ - -#include - -#include - -#define instantiate_raft_neighbors_ivf_flat_search(T, IdxT) \ - template void raft::neighbors::ivf_flat::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::search_params& params, \ - const raft::neighbors::ivf_flat::index& index, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances, \ - rmm::device_async_resource_ref mr); \ - \ - template void raft::neighbors::ivf_flat::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::search_params& params, \ - const raft::neighbors::ivf_flat::index& index, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances); -instantiate_raft_neighbors_ivf_flat_search(float, int64_t); - -#undef instantiate_raft_neighbors_ivf_flat_search diff --git a/cpp/src/neighbors/ivf_flat_search_int8_t_int64_t.cu b/cpp/src/neighbors/ivf_flat_search_int8_t_int64_t.cu deleted file mode 100644 index 35792a78a8..0000000000 --- a/cpp/src/neighbors/ivf_flat_search_int8_t_int64_t.cu +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_flat_00_generate.py - * - * Make changes there and run in this directory: - * - * > python ivf_flat_00_generate.py - * - */ - -#include - -#include - -#define instantiate_raft_neighbors_ivf_flat_search(T, IdxT) \ - template void raft::neighbors::ivf_flat::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::search_params& params, \ - const raft::neighbors::ivf_flat::index& index, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances, \ - rmm::device_async_resource_ref mr); \ - \ - template void raft::neighbors::ivf_flat::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::search_params& params, \ - const raft::neighbors::ivf_flat::index& index, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances); -instantiate_raft_neighbors_ivf_flat_search(int8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_flat_search diff --git a/cpp/src/neighbors/ivf_flat_search_uint8_t_int64_t.cu b/cpp/src/neighbors/ivf_flat_search_uint8_t_int64_t.cu deleted file mode 100644 index 663e52cb99..0000000000 --- a/cpp/src/neighbors/ivf_flat_search_uint8_t_int64_t.cu +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_flat_00_generate.py - * - * Make changes there and run in this directory: - * - * > python ivf_flat_00_generate.py - * - */ - -#include - -#include - -#define instantiate_raft_neighbors_ivf_flat_search(T, IdxT) \ - template void raft::neighbors::ivf_flat::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::search_params& params, \ - const raft::neighbors::ivf_flat::index& index, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances, \ - rmm::device_async_resource_ref mr); \ - \ - template void raft::neighbors::ivf_flat::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::search_params& params, \ - const raft::neighbors::ivf_flat::index& index, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances); -instantiate_raft_neighbors_ivf_flat_search(uint8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_flat_search diff --git a/cpp/src/neighbors/ivfpq_build_float_int64_t.cu b/cpp/src/neighbors/ivfpq_build_float_int64_t.cu deleted file mode 100644 index 8281abb62e..0000000000 --- a/cpp/src/neighbors/ivfpq_build_float_int64_t.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index - -#define instantiate_raft_neighbors_ivf_pq_build(T, IdxT) \ - template raft::neighbors::ivf_pq::index raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - raft::device_matrix_view dataset); \ - \ - template auto raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - const T* dataset, \ - IdxT n_rows, \ - uint32_t dim) \ - ->raft::neighbors::ivf_pq::index; - -instantiate_raft_neighbors_ivf_pq_build(float, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_build diff --git a/cpp/src/neighbors/ivfpq_build_half_int64_t.cu b/cpp/src/neighbors/ivfpq_build_half_int64_t.cu deleted file mode 100644 index aacb2d7198..0000000000 --- a/cpp/src/neighbors/ivfpq_build_half_int64_t.cu +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index - -#include - -#define instantiate_raft_neighbors_ivf_pq_build(T, IdxT) \ - template raft::neighbors::ivf_pq::index raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - raft::device_matrix_view dataset); \ - \ - template auto raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - const T* dataset, \ - IdxT n_rows, \ - uint32_t dim) \ - ->raft::neighbors::ivf_pq::index; - -instantiate_raft_neighbors_ivf_pq_build(half, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_build diff --git a/cpp/src/neighbors/ivfpq_build_int8_t_int64_t.cu b/cpp/src/neighbors/ivfpq_build_int8_t_int64_t.cu deleted file mode 100644 index 5f79ee3033..0000000000 --- a/cpp/src/neighbors/ivfpq_build_int8_t_int64_t.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index - -#define instantiate_raft_neighbors_ivf_pq_build(T, IdxT) \ - template raft::neighbors::ivf_pq::index raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - raft::device_matrix_view dataset); \ - \ - template auto raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - const T* dataset, \ - IdxT n_rows, \ - uint32_t dim) \ - ->raft::neighbors::ivf_pq::index; - -instantiate_raft_neighbors_ivf_pq_build(int8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_build diff --git a/cpp/src/neighbors/ivfpq_build_uint8_t_int64_t.cu b/cpp/src/neighbors/ivfpq_build_uint8_t_int64_t.cu deleted file mode 100644 index 49866ba09a..0000000000 --- a/cpp/src/neighbors/ivfpq_build_uint8_t_int64_t.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index - -#define instantiate_raft_neighbors_ivf_pq_build(T, IdxT) \ - template raft::neighbors::ivf_pq::index raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - raft::device_matrix_view dataset); \ - \ - template auto raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - const T* dataset, \ - IdxT n_rows, \ - uint32_t dim) \ - ->raft::neighbors::ivf_pq::index; - -instantiate_raft_neighbors_ivf_pq_build(uint8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_build diff --git a/cpp/src/neighbors/ivfpq_extend_float_int64_t.cu b/cpp/src/neighbors/ivfpq_extend_float_int64_t.cu deleted file mode 100644 index 6ee6cb3879..0000000000 --- a/cpp/src/neighbors/ivfpq_extend_float_int64_t.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index - -#define instantiate_raft_neighbors_ivf_pq_extend(T, IdxT) \ - template raft::neighbors::ivf_pq::index raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_pq::index& idx); \ - \ - template void raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_pq::index* idx); \ - \ - template auto raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows) \ - ->raft::neighbors::ivf_pq::index; \ - \ - template void raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::neighbors::ivf_pq::index* idx, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows); - -instantiate_raft_neighbors_ivf_pq_extend(float, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_extend diff --git a/cpp/src/neighbors/ivfpq_extend_half_int64_t.cu b/cpp/src/neighbors/ivfpq_extend_half_int64_t.cu deleted file mode 100644 index 85477ca4a0..0000000000 --- a/cpp/src/neighbors/ivfpq_extend_half_int64_t.cu +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index - -#include - -#define instantiate_raft_neighbors_ivf_pq_extend(T, IdxT) \ - template raft::neighbors::ivf_pq::index raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_pq::index& idx); \ - \ - template void raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_pq::index* idx); \ - \ - template auto raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows) \ - ->raft::neighbors::ivf_pq::index; \ - \ - template void raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::neighbors::ivf_pq::index* idx, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows); - -instantiate_raft_neighbors_ivf_pq_extend(half, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_extend diff --git a/cpp/src/neighbors/ivfpq_extend_int8_t_int64_t.cu b/cpp/src/neighbors/ivfpq_extend_int8_t_int64_t.cu deleted file mode 100644 index aefeba2aa6..0000000000 --- a/cpp/src/neighbors/ivfpq_extend_int8_t_int64_t.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index - -#define instantiate_raft_neighbors_ivf_pq_extend(T, IdxT) \ - template raft::neighbors::ivf_pq::index raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_pq::index& idx); \ - \ - template void raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_pq::index* idx); \ - \ - template auto raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows) \ - ->raft::neighbors::ivf_pq::index; \ - \ - template void raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::neighbors::ivf_pq::index* idx, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows); - -instantiate_raft_neighbors_ivf_pq_extend(int8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_extend diff --git a/cpp/src/neighbors/ivfpq_extend_uint8_t_int64_t.cu b/cpp/src/neighbors/ivfpq_extend_uint8_t_int64_t.cu deleted file mode 100644 index e3a6dd365b..0000000000 --- a/cpp/src/neighbors/ivfpq_extend_uint8_t_int64_t.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index - -#define instantiate_raft_neighbors_ivf_pq_extend(T, IdxT) \ - template raft::neighbors::ivf_pq::index raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_pq::index& idx); \ - \ - template void raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_pq::index* idx); \ - \ - template auto raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows) \ - ->raft::neighbors::ivf_pq::index; \ - \ - template void raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::neighbors::ivf_pq::index* idx, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows); - -instantiate_raft_neighbors_ivf_pq_extend(uint8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_extend diff --git a/cpp/src/neighbors/ivfpq_search_float_int64_t.cu b/cpp/src/neighbors/ivfpq_search_float_int64_t.cu deleted file mode 100644 index 2d15167099..0000000000 --- a/cpp/src/neighbors/ivfpq_search_float_int64_t.cu +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index - -#include - -#define instantiate_raft_neighbors_ivf_pq_search(T, IdxT) \ - template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances); \ - \ - template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances) - -instantiate_raft_neighbors_ivf_pq_search(float, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_search diff --git a/cpp/src/neighbors/ivfpq_search_half_int64_t.cu b/cpp/src/neighbors/ivfpq_search_half_int64_t.cu deleted file mode 100644 index c9a380e21f..0000000000 --- a/cpp/src/neighbors/ivfpq_search_half_int64_t.cu +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index - -#include - -#include - -#define instantiate_raft_neighbors_ivf_pq_search(T, IdxT) \ - template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances); \ - \ - template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances) - -instantiate_raft_neighbors_ivf_pq_search(half, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_search diff --git a/cpp/src/neighbors/ivfpq_search_int8_t_int64_t.cu b/cpp/src/neighbors/ivfpq_search_int8_t_int64_t.cu deleted file mode 100644 index e85c98d8dd..0000000000 --- a/cpp/src/neighbors/ivfpq_search_int8_t_int64_t.cu +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index - -#include - -#define instantiate_raft_neighbors_ivf_pq_search(T, IdxT) \ - template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances); \ - \ - template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances) - -instantiate_raft_neighbors_ivf_pq_search(int8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_search diff --git a/cpp/src/neighbors/ivfpq_search_uint8_t_int64_t.cu b/cpp/src/neighbors/ivfpq_search_uint8_t_int64_t.cu deleted file mode 100644 index 42653254e9..0000000000 --- a/cpp/src/neighbors/ivfpq_search_uint8_t_int64_t.cu +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index - -#include - -#define instantiate_raft_neighbors_ivf_pq_search(T, IdxT) \ - template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances); \ - \ - template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances) - -instantiate_raft_neighbors_ivf_pq_search(uint8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_search diff --git a/cpp/src/neighbors/refine_00_generate.py b/cpp/src/neighbors/refine_00_generate.py deleted file mode 100644 index fd11f4d5c3..0000000000 --- a/cpp/src/neighbors/refine_00_generate.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -header = """ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by refine_00_generate.py - * - * Make changes there and run in this directory: - * - * > python refine_00_generate.py - * - */ - -#include - -#define instantiate_raft_neighbors_refine(idx_t, data_t, distance_t, matrix_idx) \\ - template void raft::neighbors::refine( \\ - raft::resources const& handle, \\ - raft::device_matrix_view dataset, \\ - raft::device_matrix_view queries, \\ - raft::device_matrix_view neighbor_candidates, \\ - raft::device_matrix_view indices, \\ - raft::device_matrix_view distances, \\ - raft::distance::DistanceType metric); \\ - \\ - template void raft::neighbors::refine( \\ - raft::resources const& handle, \\ - raft::host_matrix_view dataset, \\ - raft::host_matrix_view queries, \\ - raft::host_matrix_view neighbor_candidates, \\ - raft::host_matrix_view indices, \\ - raft::host_matrix_view distances, \\ - raft::distance::DistanceType metric); - -""" - -types = dict( - float_float= ("float", "float"), - half_float= ("half", "float"), - int8_t_float=("int8_t", "float"), - uint8_t_float=("uint8_t", "float"), -) - -for type_path, (data_t, distance_t) in types.items(): - path = f"refine_{type_path}.cu" - with open(path, "w") as f: - f.write(header) - f.write(f"instantiate_raft_neighbors_refine(int64_t, {data_t}, {distance_t}, int64_t);\n\n") - f.write(f"#undef instantiate_raft_neighbors_refine\n") - - # for pasting into CMakeLists.txt - print(f"src/neighbors/{path}") diff --git a/cpp/src/neighbors/refine_float_float.cu b/cpp/src/neighbors/refine_float_float.cu deleted file mode 100644 index 75851eeedb..0000000000 --- a/cpp/src/neighbors/refine_float_float.cu +++ /dev/null @@ -1,54 +0,0 @@ - -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by refine_00_generate.py - * - * Make changes there and run in this directory: - * - * > python refine_00_generate.py - * - */ - -#include - -#define instantiate_raft_neighbors_refine_d(idx_t, data_t, distance_t, matrix_idx) \ - template void raft::neighbors::refine( \ - raft::resources const& handle, \ - raft::device_matrix_view dataset, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbor_candidates, \ - raft::device_matrix_view indices, \ - raft::device_matrix_view distances, \ - raft::distance::DistanceType metric); - -#define instantiate_raft_neighbors_refine_h(idx_t, data_t, distance_t, matrix_idx) \ - template void raft::neighbors::refine( \ - raft::resources const& handle, \ - raft::host_matrix_view dataset, \ - raft::host_matrix_view queries, \ - raft::host_matrix_view neighbor_candidates, \ - raft::host_matrix_view indices, \ - raft::host_matrix_view distances, \ - raft::distance::DistanceType metric); - -instantiate_raft_neighbors_refine_d(int64_t, float, float, int64_t); -instantiate_raft_neighbors_refine_h(int64_t, float, float, int64_t); -instantiate_raft_neighbors_refine_h(uint32_t, float, float, int64_t); - -#undef instantiate_raft_neighbors_refine_d -#undef instantiate_raft_neighbors_refine_h diff --git a/cpp/src/neighbors/refine_half_float.cu b/cpp/src/neighbors/refine_half_float.cu deleted file mode 100644 index c323951b82..0000000000 --- a/cpp/src/neighbors/refine_half_float.cu +++ /dev/null @@ -1,50 +0,0 @@ - -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by refine_00_generate.py - * - * Make changes there and run in this directory: - * - * > python refine_00_generate.py - * - */ - -#include - -#define instantiate_raft_neighbors_refine(idx_t, data_t, distance_t, matrix_idx) \ - template void raft::neighbors::refine( \ - raft::resources const& handle, \ - raft::device_matrix_view dataset, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbor_candidates, \ - raft::device_matrix_view indices, \ - raft::device_matrix_view distances, \ - raft::distance::DistanceType metric); \ - \ - template void raft::neighbors::refine( \ - raft::resources const& handle, \ - raft::host_matrix_view dataset, \ - raft::host_matrix_view queries, \ - raft::host_matrix_view neighbor_candidates, \ - raft::host_matrix_view indices, \ - raft::host_matrix_view distances, \ - raft::distance::DistanceType metric); - -instantiate_raft_neighbors_refine(int64_t, half, float, int64_t); - -#undef instantiate_raft_neighbors_refine diff --git a/cpp/src/neighbors/refine_int8_t_float.cu b/cpp/src/neighbors/refine_int8_t_float.cu deleted file mode 100644 index 6ed1f86db3..0000000000 --- a/cpp/src/neighbors/refine_int8_t_float.cu +++ /dev/null @@ -1,50 +0,0 @@ - -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by refine_00_generate.py - * - * Make changes there and run in this directory: - * - * > python refine_00_generate.py - * - */ - -#include - -#define instantiate_raft_neighbors_refine(idx_t, data_t, distance_t, matrix_idx) \ - template void raft::neighbors::refine( \ - raft::resources const& handle, \ - raft::device_matrix_view dataset, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbor_candidates, \ - raft::device_matrix_view indices, \ - raft::device_matrix_view distances, \ - raft::distance::DistanceType metric); \ - \ - template void raft::neighbors::refine( \ - raft::resources const& handle, \ - raft::host_matrix_view dataset, \ - raft::host_matrix_view queries, \ - raft::host_matrix_view neighbor_candidates, \ - raft::host_matrix_view indices, \ - raft::host_matrix_view distances, \ - raft::distance::DistanceType metric); - -instantiate_raft_neighbors_refine(int64_t, int8_t, float, int64_t); - -#undef instantiate_raft_neighbors_refine diff --git a/cpp/src/neighbors/refine_uint8_t_float.cu b/cpp/src/neighbors/refine_uint8_t_float.cu deleted file mode 100644 index dac3c68b9f..0000000000 --- a/cpp/src/neighbors/refine_uint8_t_float.cu +++ /dev/null @@ -1,50 +0,0 @@ - -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by refine_00_generate.py - * - * Make changes there and run in this directory: - * - * > python refine_00_generate.py - * - */ - -#include - -#define instantiate_raft_neighbors_refine(idx_t, data_t, distance_t, matrix_idx) \ - template void raft::neighbors::refine( \ - raft::resources const& handle, \ - raft::device_matrix_view dataset, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbor_candidates, \ - raft::device_matrix_view indices, \ - raft::device_matrix_view distances, \ - raft::distance::DistanceType metric); \ - \ - template void raft::neighbors::refine( \ - raft::resources const& handle, \ - raft::host_matrix_view dataset, \ - raft::host_matrix_view queries, \ - raft::host_matrix_view neighbor_candidates, \ - raft::host_matrix_view indices, \ - raft::host_matrix_view distances, \ - raft::distance::DistanceType metric); - -instantiate_raft_neighbors_refine(int64_t, uint8_t, float, int64_t); - -#undef instantiate_raft_neighbors_refine diff --git a/cpp/src/raft_runtime/cluster/cluster_cost.cuh b/cpp/src/raft_runtime/cluster/cluster_cost.cuh deleted file mode 100644 index 325a460ab9..0000000000 --- a/cpp/src/raft_runtime/cluster/cluster_cost.cuh +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace raft::runtime::cluster::kmeans { -template -void cluster_cost(raft::resources const& handle, - const ElementType* X, - IndexType n_samples, - IndexType n_features, - IndexType n_clusters, - const ElementType* centroids, - ElementType* cost) -{ - rmm::device_uvector workspace(n_samples * sizeof(IndexType), - resource::get_cuda_stream(handle)); - - rmm::device_uvector x_norms(n_samples, resource::get_cuda_stream(handle)); - rmm::device_uvector centroid_norms(n_clusters, resource::get_cuda_stream(handle)); - raft::linalg::rowNorm(x_norms.data(), - X, - n_features, - n_samples, - raft::linalg::L2Norm, - true, - resource::get_cuda_stream(handle)); - raft::linalg::rowNorm(centroid_norms.data(), - centroids, - n_features, - n_clusters, - raft::linalg::L2Norm, - true, - resource::get_cuda_stream(handle)); - - auto min_cluster_distance = - raft::make_device_vector>(handle, n_samples); - raft::distance::fusedL2NNMinReduce(min_cluster_distance.data_handle(), - X, - centroids, - x_norms.data(), - centroid_norms.data(), - n_samples, - n_clusters, - n_features, - (void*)workspace.data(), - false, - true, - resource::get_cuda_stream(handle)); - - auto distances = raft::make_device_vector(handle, n_samples); - thrust::transform(resource::get_thrust_policy(handle), - min_cluster_distance.data_handle(), - min_cluster_distance.data_handle() + n_samples, - distances.data_handle(), - raft::value_op{}); - - rmm::device_scalar device_cost(0, resource::get_cuda_stream(handle)); - raft::cluster::kmeans::cluster_cost(handle, - distances.view(), - workspace, - make_device_scalar_view(device_cost.data()), - raft::add_op{}); - - raft::update_host(cost, device_cost.data(), 1, resource::get_cuda_stream(handle)); -} -} // namespace raft::runtime::cluster::kmeans diff --git a/cpp/src/raft_runtime/cluster/cluster_cost_double.cu b/cpp/src/raft_runtime/cluster/cluster_cost_double.cu deleted file mode 100644 index abe07c50e7..0000000000 --- a/cpp/src/raft_runtime/cluster/cluster_cost_double.cu +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "cluster_cost.cuh" - -#include -#include - -namespace raft::runtime::cluster::kmeans { - -void cluster_cost(raft::resources const& handle, - const double* X, - int n_samples, - int n_features, - int n_clusters, - const double* centroids, - double* cost) -{ - cluster_cost(handle, X, n_samples, n_features, n_clusters, centroids, cost); -} -} // namespace raft::runtime::cluster::kmeans diff --git a/cpp/src/raft_runtime/cluster/cluster_cost_float.cu b/cpp/src/raft_runtime/cluster/cluster_cost_float.cu deleted file mode 100644 index 1634788b62..0000000000 --- a/cpp/src/raft_runtime/cluster/cluster_cost_float.cu +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "cluster_cost.cuh" - -#include -#include - -namespace raft::runtime::cluster::kmeans { - -void cluster_cost(raft::resources const& handle, - const float* X, - int n_samples, - int n_features, - int n_clusters, - const float* centroids, - float* cost) -{ - cluster_cost(handle, X, n_samples, n_features, n_clusters, centroids, cost); -} -} // namespace raft::runtime::cluster::kmeans diff --git a/cpp/src/raft_runtime/cluster/kmeans_fit_double.cu b/cpp/src/raft_runtime/cluster/kmeans_fit_double.cu deleted file mode 100644 index 0711f6c974..0000000000 --- a/cpp/src/raft_runtime/cluster/kmeans_fit_double.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -namespace raft::runtime::cluster::kmeans { - -void fit(raft::resources const& handle, - const raft::cluster::kmeans::KMeansParams& params, - raft::device_matrix_view X, - std::optional> sample_weight, - raft::device_matrix_view centroids, - raft::host_scalar_view inertia, - raft::host_scalar_view n_iter) -{ - raft::cluster::kmeans::fit( - handle, params, X, sample_weight, centroids, inertia, n_iter); -} -} // namespace raft::runtime::cluster::kmeans diff --git a/cpp/src/raft_runtime/cluster/kmeans_fit_float.cu b/cpp/src/raft_runtime/cluster/kmeans_fit_float.cu deleted file mode 100644 index f98a87d906..0000000000 --- a/cpp/src/raft_runtime/cluster/kmeans_fit_float.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -namespace raft::runtime::cluster::kmeans { - -void fit(raft::resources const& handle, - const raft::cluster::kmeans::KMeansParams& params, - raft::device_matrix_view X, - std::optional> sample_weight, - raft::device_matrix_view centroids, - raft::host_scalar_view inertia, - raft::host_scalar_view n_iter) -{ - raft::cluster::kmeans::fit( - handle, params, X, sample_weight, centroids, inertia, n_iter); -} -} // namespace raft::runtime::cluster::kmeans diff --git a/cpp/src/raft_runtime/cluster/kmeans_init_plus_plus_double.cu b/cpp/src/raft_runtime/cluster/kmeans_init_plus_plus_double.cu deleted file mode 100644 index 6c7563e457..0000000000 --- a/cpp/src/raft_runtime/cluster/kmeans_init_plus_plus_double.cu +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -namespace raft::runtime::cluster::kmeans { - -void init_plus_plus(raft::resources const& handle, - const raft::cluster::kmeans::KMeansParams& params, - raft::device_matrix_view X, - raft::device_matrix_view centroids) -{ - rmm::device_uvector workspace(0, resource::get_cuda_stream(handle)); - raft::cluster::kmeans::init_plus_plus(handle, params, X, centroids, workspace); -} -} // namespace raft::runtime::cluster::kmeans diff --git a/cpp/src/raft_runtime/cluster/kmeans_init_plus_plus_float.cu b/cpp/src/raft_runtime/cluster/kmeans_init_plus_plus_float.cu deleted file mode 100644 index 99894f4ef7..0000000000 --- a/cpp/src/raft_runtime/cluster/kmeans_init_plus_plus_float.cu +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -namespace raft::runtime::cluster::kmeans { - -void init_plus_plus(raft::resources const& handle, - const raft::cluster::kmeans::KMeansParams& params, - raft::device_matrix_view X, - raft::device_matrix_view centroids) -{ - rmm::device_uvector workspace(0, resource::get_cuda_stream(handle)); - raft::cluster::kmeans::init_plus_plus(handle, params, X, centroids, workspace); -} -} // namespace raft::runtime::cluster::kmeans diff --git a/cpp/src/raft_runtime/cluster/update_centroids.cuh b/cpp/src/raft_runtime/cluster/update_centroids.cuh deleted file mode 100644 index e0dec4bdcf..0000000000 --- a/cpp/src/raft_runtime/cluster/update_centroids.cuh +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include - -namespace raft::runtime::cluster::kmeans { - -template -void update_centroids(raft::resources const& handle, - const DataT* X, - int n_samples, - int n_features, - int n_clusters, - const DataT* sample_weights, - const DataT* centroids, - const IndexT* labels, - DataT* new_centroids, - DataT* weight_per_cluster) -{ - auto X_view = raft::make_device_matrix_view(X, n_samples, n_features); - auto centroids_view = - raft::make_device_matrix_view(centroids, n_clusters, n_features); - - rmm::device_uvector sample_weights_uvec(0, resource::get_cuda_stream(handle)); - if (sample_weights == nullptr) { - sample_weights_uvec.resize(n_samples, resource::get_cuda_stream(handle)); - DataT weight = 1.0 / n_samples; - thrust::fill(resource::get_thrust_policy(handle), - sample_weights_uvec.data(), - sample_weights_uvec.data() + n_samples, - weight); - } - auto sample_weights_view = raft::make_device_vector_view( - sample_weights == nullptr ? sample_weights_uvec.data() : sample_weights, n_samples); - - auto new_centroids_view = - raft::make_device_matrix_view(new_centroids, n_clusters, n_features); - rmm::device_uvector weight_per_cluster_uvec(0, resource::get_cuda_stream(handle)); - if (weight_per_cluster == nullptr) { - weight_per_cluster_uvec.resize(n_clusters, resource::get_cuda_stream(handle)); - } - auto weight_per_cluster_view = raft::make_device_vector_view( - weight_per_cluster == nullptr ? weight_per_cluster_uvec.data() : weight_per_cluster, - n_clusters); - - raft::cluster::kmeans::update_centroids(handle, - X_view, - sample_weights_view, - centroids_view, - labels, - weight_per_cluster_view, - new_centroids_view); -} -} // namespace raft::runtime::cluster::kmeans \ No newline at end of file diff --git a/cpp/src/raft_runtime/cluster/update_centroids_double.cu b/cpp/src/raft_runtime/cluster/update_centroids_double.cu deleted file mode 100644 index 0bc6e33273..0000000000 --- a/cpp/src/raft_runtime/cluster/update_centroids_double.cu +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "update_centroids.cuh" - -#include -#include - -namespace raft::runtime::cluster::kmeans { - -void update_centroids(raft::resources const& handle, - const double* X, - int n_samples, - int n_features, - int n_clusters, - const double* sample_weights, - const double* centroids, - const int* labels, - double* new_centroids, - double* weight_per_cluster) -{ - update_centroids(handle, - X, - n_samples, - n_features, - n_clusters, - sample_weights, - centroids, - labels, - new_centroids, - weight_per_cluster); -} - -} // namespace raft::runtime::cluster::kmeans \ No newline at end of file diff --git a/cpp/src/raft_runtime/cluster/update_centroids_float.cu b/cpp/src/raft_runtime/cluster/update_centroids_float.cu deleted file mode 100644 index 3f98e3fb38..0000000000 --- a/cpp/src/raft_runtime/cluster/update_centroids_float.cu +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "update_centroids.cuh" - -#include -#include - -namespace raft::runtime::cluster::kmeans { - -void update_centroids(raft::resources const& handle, - const float* X, - int n_samples, - int n_features, - int n_clusters, - const float* sample_weights, - const float* centroids, - const int* labels, - float* new_centroids, - float* weight_per_cluster) -{ - update_centroids(handle, - X, - n_samples, - n_features, - n_clusters, - sample_weights, - centroids, - labels, - new_centroids, - weight_per_cluster); -} - -} // namespace raft::runtime::cluster::kmeans \ No newline at end of file diff --git a/cpp/src/raft_runtime/distance/fused_distance_min_arg.cu b/cpp/src/raft_runtime/distance/fused_distance_min_arg.cu deleted file mode 100644 index dfdff4e94b..0000000000 --- a/cpp/src/raft_runtime/distance/fused_distance_min_arg.cu +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fused_distance_min_arg.hpp" - -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace raft::runtime::distance { - -void fused_distance_nn_min_arg(raft::resources const& handle, - int* min, - const float* x, - const float* y, - int m, - int n, - int k, - bool sqrt, - raft::distance::DistanceType metric, - bool isRowMajor, - float metric_arg) -{ - switch (metric) { - case raft::distance::DistanceType::CosineExpanded: - compute_fused_cosine_nn_min_arg(handle, min, x, y, m, n, k, sqrt); - break; - case raft::distance::DistanceType::L2Expanded: - case raft::distance::DistanceType::L2SqrtExpanded: - compute_fused_l2_nn_min_arg(handle, min, x, y, m, n, k, sqrt); - break; - default: assert("only Cosine/L2 metric is supported with fusedDistanceNN\n"); break; - } -} - -} // end namespace raft::runtime::distance diff --git a/cpp/src/raft_runtime/distance/fused_distance_min_arg.hpp b/cpp/src/raft_runtime/distance/fused_distance_min_arg.hpp deleted file mode 100644 index 6452752a79..0000000000 --- a/cpp/src/raft_runtime/distance/fused_distance_min_arg.hpp +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace raft::runtime::distance { - -template -struct KeyValueIndexOp { - __host__ __device__ __forceinline__ IndexT - operator()(const raft::KeyValuePair& a) const - { - return a.key; - } -}; - -template -void compute_fused_l2_nn_min_arg(raft::resources const& handle, - idx_t* min, - const value_t* x, - const value_t* y, - idx_t m, - idx_t n, - idx_t k, - bool sqrt) -{ - rmm::device_uvector workspace(m, resource::get_cuda_stream(handle)); - auto kvp = raft::make_device_vector>(handle, m); - constexpr bool is_row_major = true; - - rmm::device_uvector x_norms(m, resource::get_cuda_stream(handle)); - rmm::device_uvector y_norms(n, resource::get_cuda_stream(handle)); - raft::linalg::rowNorm( - x_norms.data(), x, k, m, raft::linalg::L2Norm, is_row_major, resource::get_cuda_stream(handle)); - raft::linalg::rowNorm( - y_norms.data(), y, k, n, raft::linalg::L2Norm, is_row_major, resource::get_cuda_stream(handle)); - - raft::distance::fusedL2NNMinReduce(kvp.data_handle(), - x, - y, - x_norms.data(), - y_norms.data(), - m, - n, - k, - (void*)workspace.data(), - sqrt, - true, - resource::get_cuda_stream(handle)); - - KeyValueIndexOp conversion_op; - thrust::transform(resource::get_thrust_policy(handle), - kvp.data_handle(), - kvp.data_handle() + m, - min, - conversion_op); - resource::sync_stream(handle); -} - -template -void compute_fused_cosine_nn_min_arg(raft::resources const& handle, - idx_t* min, - const value_t* x, - const value_t* y, - idx_t m, - idx_t n, - idx_t k, - bool sqrt) -{ - rmm::device_uvector workspace(m, resource::get_cuda_stream(handle)); - auto kvp = raft::make_device_vector>(handle, m); - - rmm::device_uvector x_norms(m, resource::get_cuda_stream(handle)); - rmm::device_uvector y_norms(n, resource::get_cuda_stream(handle)); - constexpr bool is_row_major = true; - raft::linalg::rowNorm(x_norms.data(), - x, - k, - m, - raft::linalg::L2Norm, - is_row_major, - resource::get_cuda_stream(handle), - raft::sqrt_op{}); - raft::linalg::rowNorm(y_norms.data(), - y, - k, - n, - raft::linalg::L2Norm, - is_row_major, - resource::get_cuda_stream(handle), - raft::sqrt_op{}); - - raft::distance::fusedDistanceNNMinReduce(kvp.data_handle(), - x, - y, - x_norms.data(), - y_norms.data(), - m, - n, - k, - (void*)workspace.data(), - sqrt, - true, - is_row_major, - raft::distance::DistanceType::CosineExpanded, - 0.0f, - resource::get_cuda_stream(handle)); - - KeyValueIndexOp conversion_op; - thrust::transform(resource::get_thrust_policy(handle), - kvp.data_handle(), - kvp.data_handle() + m, - min, - conversion_op); - resource::sync_stream(handle); -} - -} // end namespace raft::runtime::distance diff --git a/cpp/src/raft_runtime/distance/fused_l2_min_arg.cu b/cpp/src/raft_runtime/distance/fused_l2_min_arg.cu deleted file mode 100644 index 870757dca1..0000000000 --- a/cpp/src/raft_runtime/distance/fused_l2_min_arg.cu +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fused_distance_min_arg.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace raft::runtime::distance { - -[[deprecated("use fused_distance_nn_min_arg instead")]] void fused_l2_nn_min_arg( - raft::resources const& handle, - int* min, - const float* x, - const float* y, - int m, - int n, - int k, - bool sqrt) -{ - compute_fused_l2_nn_min_arg(handle, min, x, y, m, n, k, sqrt); -} - -[[deprecated("use fused_distance_nn_min_arg instead")]] void fused_l2_nn_min_arg( - raft::resources const& handle, - int* min, - const double* x, - const double* y, - int m, - int n, - int k, - bool sqrt) -{ - compute_fused_l2_nn_min_arg(handle, min, x, y, m, n, k, sqrt); -} - -} // end namespace raft::runtime::distance diff --git a/cpp/src/raft_runtime/distance/pairwise_distance.cu b/cpp/src/raft_runtime/distance/pairwise_distance.cu deleted file mode 100644 index 868a243b02..0000000000 --- a/cpp/src/raft_runtime/distance/pairwise_distance.cu +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -namespace raft::runtime::distance { - -void pairwise_distance(raft::resources const& handle, - float* x, - float* y, - float* dists, - int m, - int n, - int k, - raft::distance::DistanceType metric, - bool isRowMajor, - float metric_arg) -{ - raft::distance::pairwise_distance( - handle, x, y, dists, m, n, k, metric, isRowMajor, metric_arg); -} - -void pairwise_distance(raft::resources const& handle, - double* x, - double* y, - double* dists, - int m, - int n, - int k, - raft::distance::DistanceType metric, - bool isRowMajor, - float metric_arg) -{ - raft::distance::pairwise_distance( - handle, x, y, dists, m, n, k, metric, isRowMajor, metric_arg); -} -} // namespace raft::runtime::distance \ No newline at end of file diff --git a/cpp/src/raft_runtime/matrix/select_k_float_int64_t.cu b/cpp/src/raft_runtime/matrix/select_k_float_int64_t.cu deleted file mode 100644 index 551a51f6b6..0000000000 --- a/cpp/src/raft_runtime/matrix/select_k_float_int64_t.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include - -#include - -namespace raft::runtime::matrix { - -void select_k(const resources& handle, - raft::device_matrix_view in_val, - std::optional> in_idx, - raft::device_matrix_view out_val, - raft::device_matrix_view out_idx, - bool select_min) -{ - raft::matrix::select_k(handle, in_val, in_idx, out_val, out_idx, select_min); -} -} // namespace raft::runtime::matrix diff --git a/cpp/src/raft_runtime/neighbors/brute_force_knn_int64_t_float.cu b/cpp/src/raft_runtime/neighbors/brute_force_knn_int64_t_float.cu deleted file mode 100644 index 3752e9218e..0000000000 --- a/cpp/src/raft_runtime/neighbors/brute_force_knn_int64_t_float.cu +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include - -#include - -namespace raft::runtime::neighbors::brute_force { - -#define RAFT_INST_BFKNN(IDX_T, DATA_T, MATRIX_IDX_T, INDEX_LAYOUT, SEARCH_LAYOUT) \ - void knn(raft::resources const& handle, \ - raft::device_matrix_view index, \ - raft::device_matrix_view search, \ - raft::device_matrix_view indices, \ - raft::device_matrix_view distances, \ - distance::DistanceType metric, \ - std::optional metric_arg, \ - std::optional global_id_offset) \ - { \ - std::vector> vec; \ - vec.push_back(index); \ - raft::neighbors::brute_force::knn( \ - handle, vec, search, indices, distances, metric, metric_arg, global_id_offset); \ - } - -RAFT_INST_BFKNN(int64_t, float, int64_t, raft::row_major, raft::row_major); - -#undef RAFT_INST_BFKNN - -} // namespace raft::runtime::neighbors::brute_force diff --git a/cpp/src/raft_runtime/neighbors/cagra_build.cu b/cpp/src/raft_runtime/neighbors/cagra_build.cu deleted file mode 100644 index 9268551b8c..0000000000 --- a/cpp/src/raft_runtime/neighbors/cagra_build.cu +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include - -#include - -namespace raft::runtime::neighbors::cagra { - -#define RAFT_INST_CAGRA_BUILD(T, IdxT) \ - auto build(raft::resources const& handle, \ - const raft::neighbors::cagra::index_params& params, \ - raft::device_matrix_view dataset) \ - ->raft::neighbors::cagra::index \ - { \ - return raft::neighbors::cagra::build(handle, params, dataset); \ - } \ - \ - auto build(raft::resources const& handle, \ - const raft::neighbors::cagra::index_params& params, \ - raft::host_matrix_view dataset) \ - ->raft::neighbors::cagra::index \ - { \ - return raft::neighbors::cagra::build(handle, params, dataset); \ - } \ - \ - void build_device(raft::resources const& handle, \ - const raft::neighbors::cagra::index_params& params, \ - raft::device_matrix_view dataset, \ - raft::neighbors::cagra::index& idx) \ - { \ - idx = build(handle, params, dataset); \ - } \ - \ - void build_host(raft::resources const& handle, \ - const raft::neighbors::cagra::index_params& params, \ - raft::host_matrix_view dataset, \ - raft::neighbors::cagra::index& idx) \ - { \ - idx = build(handle, params, dataset); \ - } - -RAFT_INST_CAGRA_BUILD(float, uint32_t); -RAFT_INST_CAGRA_BUILD(half, uint32_t); -RAFT_INST_CAGRA_BUILD(int8_t, uint32_t); -RAFT_INST_CAGRA_BUILD(uint8_t, uint32_t); - -#undef RAFT_INST_CAGRA_BUILD - -#define RAFT_INST_CAGRA_OPTIMIZE(IdxT) \ - void optimize_device(raft::resources const& handle, \ - raft::device_matrix_view knn_graph, \ - raft::host_matrix_view new_graph) \ - { \ - raft::neighbors::cagra::optimize(handle, knn_graph, new_graph); \ - } \ - void optimize_host(raft::resources const& handle, \ - raft::host_matrix_view knn_graph, \ - raft::host_matrix_view new_graph) \ - { \ - raft::neighbors::cagra::optimize(handle, knn_graph, new_graph); \ - } - -RAFT_INST_CAGRA_OPTIMIZE(uint32_t); - -#undef RAFT_INST_CAGRA_OPTIMIZE - -} // namespace raft::runtime::neighbors::cagra diff --git a/cpp/src/raft_runtime/neighbors/cagra_search.cu b/cpp/src/raft_runtime/neighbors/cagra_search.cu deleted file mode 100644 index b550d2e521..0000000000 --- a/cpp/src/raft_runtime/neighbors/cagra_search.cu +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include - -#include - -namespace raft::runtime::neighbors::cagra { - -#define RAFT_INST_CAGRA_SEARCH(T, IdxT) \ - void search(raft::resources const& handle, \ - raft::neighbors::cagra::search_params const& params, \ - const raft::neighbors::cagra::index& index, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances) \ - { \ - raft::neighbors::cagra::search(handle, params, index, queries, neighbors, distances); \ - } - -RAFT_INST_CAGRA_SEARCH(float, uint32_t); -RAFT_INST_CAGRA_SEARCH(half, uint32_t); -RAFT_INST_CAGRA_SEARCH(int8_t, uint32_t); -RAFT_INST_CAGRA_SEARCH(uint8_t, uint32_t); - -#undef RAFT_INST_CAGRA_SEARCH - -} // namespace raft::runtime::neighbors::cagra diff --git a/cpp/src/raft_runtime/neighbors/cagra_serialize.cu b/cpp/src/raft_runtime/neighbors/cagra_serialize.cu deleted file mode 100644 index 0677575e23..0000000000 --- a/cpp/src/raft_runtime/neighbors/cagra_serialize.cu +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include - -#include - -#include -#include - -namespace raft::runtime::neighbors::cagra { - -#define RAFT_INST_CAGRA_SERIALIZE(DTYPE) \ - void serialize_file(raft::resources const& handle, \ - const std::string& filename, \ - const raft::neighbors::cagra::index& index, \ - bool include_dataset) \ - { \ - raft::neighbors::cagra::serialize(handle, filename, index, include_dataset); \ - }; \ - \ - void deserialize_file(raft::resources const& handle, \ - const std::string& filename, \ - raft::neighbors::cagra::index* index) \ - { \ - if (!index) { RAFT_FAIL("Invalid index pointer"); } \ - *index = raft::neighbors::cagra::deserialize(handle, filename); \ - }; \ - void serialize(raft::resources const& handle, \ - std::string& str, \ - const raft::neighbors::cagra::index& index, \ - bool include_dataset) \ - { \ - std::stringstream os; \ - raft::neighbors::cagra::serialize(handle, os, index, include_dataset); \ - str = os.str(); \ - } \ - \ - void serialize_to_hnswlib_file(raft::resources const& handle, \ - const std::string& filename, \ - const raft::neighbors::cagra::index& index) \ - { \ - raft::neighbors::cagra::serialize_to_hnswlib(handle, filename, index); \ - }; \ - void serialize_to_hnswlib(raft::resources const& handle, \ - std::string& str, \ - const raft::neighbors::cagra::index& index) \ - { \ - std::stringstream os; \ - raft::neighbors::cagra::serialize_to_hnswlib(handle, os, index); \ - str = os.str(); \ - } \ - \ - void deserialize(raft::resources const& handle, \ - const std::string& str, \ - raft::neighbors::cagra::index* index) \ - { \ - std::istringstream is(str); \ - if (!index) { RAFT_FAIL("Invalid index pointer"); } \ - *index = raft::neighbors::cagra::deserialize(handle, is); \ - } - -RAFT_INST_CAGRA_SERIALIZE(float); -RAFT_INST_CAGRA_SERIALIZE(half); -RAFT_INST_CAGRA_SERIALIZE(int8_t); -RAFT_INST_CAGRA_SERIALIZE(uint8_t); - -#undef RAFT_INST_CAGRA_SERIALIZE -} // namespace raft::runtime::neighbors::cagra diff --git a/cpp/src/raft_runtime/neighbors/eps_neighborhood.cu b/cpp/src/raft_runtime/neighbors/eps_neighborhood.cu deleted file mode 100644 index 23cb6fd790..0000000000 --- a/cpp/src/raft_runtime/neighbors/eps_neighborhood.cu +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include - -#include - -namespace raft::runtime::neighbors::epsilon_neighborhood { - -#define RAFT_INST_BFEPSN(IDX_T, DATA_T, MATRIX_IDX_T, INDEX_LAYOUT, SEARCH_LAYOUT) \ - void eps_neighbors(raft::resources const& handle, \ - raft::device_matrix_view index, \ - raft::device_matrix_view search, \ - raft::device_matrix_view adj, \ - raft::device_vector_view vd, \ - DATA_T eps) \ - { \ - raft::neighbors::epsilon_neighborhood::eps_neighbors_l2sq( \ - handle, search, index, adj, vd, eps* eps); \ - } - -RAFT_INST_BFEPSN(int64_t, float, int64_t, raft::row_major, raft::row_major); - -#undef RAFT_INST_BFEPSN - -#define RAFT_INST_RBCEPSN(IDX_T, DATA_T, INT_T, MATRIX_IDX_T, INDEX_LAYOUT, SEARCH_LAYOUT) \ - void eps_neighbors_rbc( \ - raft::resources const& handle, \ - raft::device_matrix_view index, \ - raft::device_matrix_view search, \ - raft::device_matrix_view adj, \ - raft::device_vector_view vd, \ - DATA_T eps) \ - { \ - raft::neighbors::ball_cover::BallCoverIndex rbc_index( \ - handle, \ - index.data_handle(), \ - index.extent(0), \ - index.extent(1), \ - raft::distance::DistanceType::L2SqrtUnexpanded); \ - raft::neighbors::ball_cover::build_index(handle, rbc_index); \ - raft::neighbors::ball_cover::eps_nn(handle, rbc_index, adj, vd, search, eps); \ - } \ - void build_rbc_index( \ - raft::resources const& handle, \ - raft::neighbors::ball_cover::BallCoverIndex& rbc_index) \ - { \ - raft::neighbors::ball_cover::build_index(handle, rbc_index); \ - } \ - void eps_neighbors_rbc_pass1( \ - raft::resources const& handle, \ - raft::neighbors::ball_cover::BallCoverIndex rbc_index, \ - raft::device_matrix_view search, \ - raft::device_vector_view adj_ia, \ - raft::device_vector_view vd, \ - DATA_T eps) \ - { \ - raft::neighbors::ball_cover::eps_nn( \ - handle, \ - rbc_index, \ - adj_ia, \ - raft::make_device_vector_view(nullptr, 0), \ - vd, \ - search, \ - eps); \ - } \ - void eps_neighbors_rbc_pass2( \ - raft::resources const& handle, \ - raft::neighbors::ball_cover::BallCoverIndex rbc_index, \ - raft::device_matrix_view search, \ - raft::device_vector_view adj_ia, \ - raft::device_vector_view adj_ja, \ - raft::device_vector_view vd, \ - DATA_T eps) \ - { \ - raft::neighbors::ball_cover::eps_nn(handle, rbc_index, adj_ia, adj_ja, vd, search, eps); \ - } - -RAFT_INST_RBCEPSN(int64_t, float, int64_t, int64_t, raft::row_major, raft::row_major); - -#undef RAFT_INST_RBCEPSN - -} // namespace raft::runtime::neighbors::epsilon_neighborhood diff --git a/cpp/src/raft_runtime/neighbors/hnsw.cpp b/cpp/src/raft_runtime/neighbors/hnsw.cpp deleted file mode 100644 index 5356e708d2..0000000000 --- a/cpp/src/raft_runtime/neighbors/hnsw.cpp +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include -#include - -#include -#include -#include - -namespace raft::neighbors::hnsw { -#define RAFT_INST_HNSW(T) \ - template <> \ - std::unique_ptr> from_cagra( \ - raft::resources const& res, raft::neighbors::cagra::index cagra_index) \ - { \ - std::random_device dev; \ - std::mt19937 rng(dev()); \ - std::uniform_int_distribution dist(0); \ - auto uuid = std::to_string(dist(rng)); \ - std::string filepath = "/tmp/" + uuid + ".bin"; \ - raft::runtime::neighbors::cagra::serialize_to_hnswlib(res, filepath, cagra_index); \ - auto hnsw_index = raft::runtime::neighbors::hnsw::deserialize_file( \ - res, filepath, cagra_index.dim(), cagra_index.metric()); \ - std::filesystem::remove(filepath); \ - return hnsw_index; \ - } - -RAFT_INST_HNSW(float); -RAFT_INST_HNSW(int8_t); -RAFT_INST_HNSW(uint8_t); -#undef RAFT_INST_HNSW -} // namespace raft::neighbors::hnsw - -namespace raft::runtime::neighbors::hnsw { - -#define RAFT_INST_HNSW(T) \ - void search(raft::resources const& handle, \ - raft::neighbors::hnsw::search_params const& params, \ - const raft::neighbors::hnsw::index& index, \ - raft::host_matrix_view queries, \ - raft::host_matrix_view neighbors, \ - raft::host_matrix_view distances) \ - { \ - raft::neighbors::hnsw::search(handle, params, index, queries, neighbors, distances); \ - } \ - \ - template <> \ - std::unique_ptr> deserialize_file( \ - raft::resources const& handle, \ - const std::string& filename, \ - int dim, \ - raft::distance::DistanceType metric) \ - { \ - return raft::neighbors::hnsw::deserialize(handle, filename, dim, metric); \ - } - -RAFT_INST_HNSW(float); -RAFT_INST_HNSW(int8_t); -RAFT_INST_HNSW(uint8_t); - -#undef RAFT_INST_HNSW - -} // namespace raft::runtime::neighbors::hnsw diff --git a/cpp/src/raft_runtime/neighbors/ivf_flat_build.cu b/cpp/src/raft_runtime/neighbors/ivf_flat_build.cu deleted file mode 100644 index b0c4ec3a49..0000000000 --- a/cpp/src/raft_runtime/neighbors/ivf_flat_build.cu +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include - -namespace raft::runtime::neighbors::ivf_flat { - -#define RAFT_INST_BUILD_EXTEND(T, IdxT) \ - auto build(raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::device_matrix_view dataset) \ - ->raft::neighbors::ivf_flat::index \ - { \ - return raft::neighbors::ivf_flat::build(handle, params, dataset); \ - } \ - auto extend(raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_flat::index& orig_index) \ - ->raft::neighbors::ivf_flat::index \ - { \ - return raft::neighbors::ivf_flat::extend( \ - handle, new_vectors, new_indices, orig_index); \ - } \ - \ - void build(raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::device_matrix_view dataset, \ - raft::neighbors::ivf_flat::index& idx) \ - { \ - idx = build(handle, params, dataset); \ - } \ - \ - void extend(raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_flat::index* idx) \ - { \ - raft::neighbors::ivf_flat::extend(handle, new_vectors, new_indices, idx); \ - } - -RAFT_INST_BUILD_EXTEND(float, int64_t); -RAFT_INST_BUILD_EXTEND(int8_t, int64_t); -RAFT_INST_BUILD_EXTEND(uint8_t, int64_t); - -#undef RAFT_INST_BUILD_EXTEND - -} // namespace raft::runtime::neighbors::ivf_flat diff --git a/cpp/src/raft_runtime/neighbors/ivf_flat_search.cu b/cpp/src/raft_runtime/neighbors/ivf_flat_search.cu deleted file mode 100644 index 6f26e8fc5f..0000000000 --- a/cpp/src/raft_runtime/neighbors/ivf_flat_search.cu +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include - -namespace raft::runtime::neighbors::ivf_flat { - -#define RAFT_INST_SEARCH(T, IdxT) \ - void search(raft::resources const& handle, \ - raft::neighbors::ivf_flat::search_params const& params, \ - const raft::neighbors::ivf_flat::index& index, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances) \ - { \ - raft::neighbors::ivf_flat::search( \ - handle, params, index, queries, neighbors, distances); \ - } - -RAFT_INST_SEARCH(float, int64_t); -RAFT_INST_SEARCH(int8_t, int64_t); -RAFT_INST_SEARCH(uint8_t, int64_t); - -#undef RAFT_INST_SEARCH - -} // namespace raft::runtime::neighbors::ivf_flat diff --git a/cpp/src/raft_runtime/neighbors/ivf_flat_serialize.cu b/cpp/src/raft_runtime/neighbors/ivf_flat_serialize.cu deleted file mode 100644 index 576112f0a4..0000000000 --- a/cpp/src/raft_runtime/neighbors/ivf_flat_serialize.cu +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include - -#include -#include - -namespace raft::runtime::neighbors::ivf_flat { - -#define RAFT_IVF_FLAT_SERIALIZE_INST(DTYPE) \ - void serialize_file(raft::resources const& handle, \ - const std::string& filename, \ - const raft::neighbors::ivf_flat::index& index) \ - { \ - raft::neighbors::ivf_flat::serialize(handle, filename, index); \ - }; \ - \ - void deserialize_file(raft::resources const& handle, \ - const std::string& filename, \ - raft::neighbors::ivf_flat::index* index) \ - { \ - if (!index) { RAFT_FAIL("Invalid index pointer"); } \ - *index = raft::neighbors::ivf_flat::deserialize(handle, filename); \ - }; \ - void serialize(raft::resources const& handle, \ - std::string& str, \ - const raft::neighbors::ivf_flat::index& index) \ - { \ - std::stringstream os; \ - raft::neighbors::ivf_flat::serialize(handle, os, index); \ - str = os.str(); \ - } \ - \ - void deserialize(raft::resources const& handle, \ - const std::string& str, \ - raft::neighbors::ivf_flat::index* index) \ - { \ - std::istringstream is(str); \ - if (!index) { RAFT_FAIL("Invalid index pointer"); } \ - *index = raft::neighbors::ivf_flat::deserialize(handle, is); \ - } - -RAFT_IVF_FLAT_SERIALIZE_INST(float); -RAFT_IVF_FLAT_SERIALIZE_INST(int8_t); -RAFT_IVF_FLAT_SERIALIZE_INST(uint8_t); - -#undef RAFT_IVF_FLAT_SERIALIZE_INST -} // namespace raft::runtime::neighbors::ivf_flat diff --git a/cpp/src/raft_runtime/neighbors/ivfpq_build.cu b/cpp/src/raft_runtime/neighbors/ivfpq_build.cu deleted file mode 100644 index d5c6cd4d28..0000000000 --- a/cpp/src/raft_runtime/neighbors/ivfpq_build.cu +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include - -namespace raft::runtime::neighbors::ivf_pq { - -#define RAFT_INST_BUILD_EXTEND(T, IdxT) \ - raft::neighbors::ivf_pq::index build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - raft::device_matrix_view dataset) \ - { \ - return raft::neighbors::ivf_pq::build(handle, params, dataset); \ - } \ - void build(raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - raft::device_matrix_view dataset, \ - raft::neighbors::ivf_pq::index* idx) \ - { \ - *idx = raft::neighbors::ivf_pq::build(handle, params, dataset); \ - } \ - raft::neighbors::ivf_pq::index extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_pq::index& idx) \ - { \ - return raft::neighbors::ivf_pq::extend(handle, new_vectors, new_indices, idx); \ - } \ - void extend(raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_pq::index* idx) \ - { \ - raft::neighbors::ivf_pq::extend(handle, new_vectors, new_indices, idx); \ - } - -RAFT_INST_BUILD_EXTEND(float, int64_t); -RAFT_INST_BUILD_EXTEND(int8_t, int64_t); -RAFT_INST_BUILD_EXTEND(uint8_t, int64_t); - -#undef RAFT_INST_BUILD_EXTEND - -} // namespace raft::runtime::neighbors::ivf_pq diff --git a/cpp/src/raft_runtime/neighbors/ivfpq_deserialize.cu b/cpp/src/raft_runtime/neighbors/ivfpq_deserialize.cu deleted file mode 100644 index 7a2383281e..0000000000 --- a/cpp/src/raft_runtime/neighbors/ivfpq_deserialize.cu +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include - -namespace raft::runtime::neighbors::ivf_pq { - -void deserialize(raft::resources const& handle, - const std::string& filename, - raft::neighbors::ivf_pq::index* index) -{ - if (!index) { RAFT_FAIL("Invalid index pointer"); } - *index = raft::neighbors::ivf_pq::deserialize(handle, filename); -}; -} // namespace raft::runtime::neighbors::ivf_pq diff --git a/cpp/src/raft_runtime/neighbors/ivfpq_search_float_int64_t.cu b/cpp/src/raft_runtime/neighbors/ivfpq_search_float_int64_t.cu deleted file mode 100644 index 22e4b64387..0000000000 --- a/cpp/src/raft_runtime/neighbors/ivfpq_search_float_int64_t.cu +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include - -namespace raft::runtime::neighbors::ivf_pq { - -#define RAFT_SEARCH_INST(T, IdxT) \ - void search(raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances) \ - { \ - raft::neighbors::ivf_pq::search(handle, params, idx, queries, neighbors, distances); \ - } - -RAFT_SEARCH_INST(float, int64_t); - -#undef RAFT_INST_SEARCH - -} // namespace raft::runtime::neighbors::ivf_pq diff --git a/cpp/src/raft_runtime/neighbors/ivfpq_search_int8_t_int64_t.cu b/cpp/src/raft_runtime/neighbors/ivfpq_search_int8_t_int64_t.cu deleted file mode 100644 index db7b3ce209..0000000000 --- a/cpp/src/raft_runtime/neighbors/ivfpq_search_int8_t_int64_t.cu +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include - -namespace raft::runtime::neighbors::ivf_pq { - -#define RAFT_SEARCH_INST(T, IdxT) \ - void search(raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances) \ - { \ - raft::neighbors::ivf_pq::search(handle, params, idx, queries, neighbors, distances); \ - } - -RAFT_SEARCH_INST(int8_t, int64_t); - -#undef RAFT_INST_SEARCH - -} // namespace raft::runtime::neighbors::ivf_pq diff --git a/cpp/src/raft_runtime/neighbors/ivfpq_search_uint8_t_int64_t.cu b/cpp/src/raft_runtime/neighbors/ivfpq_search_uint8_t_int64_t.cu deleted file mode 100644 index 6a9a2888e0..0000000000 --- a/cpp/src/raft_runtime/neighbors/ivfpq_search_uint8_t_int64_t.cu +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include - -namespace raft::runtime::neighbors::ivf_pq { - -#define RAFT_SEARCH_INST(T, IdxT) \ - void search(raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances) \ - { \ - raft::neighbors::ivf_pq::search(handle, params, idx, queries, neighbors, distances); \ - } - -RAFT_SEARCH_INST(uint8_t, int64_t); - -#undef RAFT_INST_SEARCH - -} // namespace raft::runtime::neighbors::ivf_pq diff --git a/cpp/src/raft_runtime/neighbors/ivfpq_serialize.cu b/cpp/src/raft_runtime/neighbors/ivfpq_serialize.cu deleted file mode 100644 index 9dea8a3b60..0000000000 --- a/cpp/src/raft_runtime/neighbors/ivfpq_serialize.cu +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include - -namespace raft::runtime::neighbors::ivf_pq { - -void serialize(raft::resources const& handle, - const std::string& filename, - const raft::neighbors::ivf_pq::index& index) -{ - raft::neighbors::ivf_pq::serialize(handle, filename, index); -}; - -} // namespace raft::runtime::neighbors::ivf_pq diff --git a/cpp/src/raft_runtime/neighbors/refine_d_int64_t_float.cu b/cpp/src/raft_runtime/neighbors/refine_d_int64_t_float.cu deleted file mode 100644 index a146eba875..0000000000 --- a/cpp/src/raft_runtime/neighbors/refine_d_int64_t_float.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -namespace raft::runtime::neighbors { - -void refine(raft::resources const& handle, - raft::device_matrix_view dataset, - raft::device_matrix_view queries, - raft::device_matrix_view neighbor_candidates, - raft::device_matrix_view indices, - raft::device_matrix_view distances, - distance::DistanceType metric) -{ - raft::neighbors::refine( - handle, dataset, queries, neighbor_candidates, indices, distances, metric); -} - -} // namespace raft::runtime::neighbors diff --git a/cpp/src/raft_runtime/neighbors/refine_d_int64_t_int8_t.cu b/cpp/src/raft_runtime/neighbors/refine_d_int64_t_int8_t.cu deleted file mode 100644 index c840acf3df..0000000000 --- a/cpp/src/raft_runtime/neighbors/refine_d_int64_t_int8_t.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -namespace raft::runtime::neighbors { - -void refine(raft::resources const& handle, - raft::device_matrix_view dataset, - raft::device_matrix_view queries, - raft::device_matrix_view neighbor_candidates, - raft::device_matrix_view indices, - raft::device_matrix_view distances, - distance::DistanceType metric) -{ - raft::neighbors::refine( - handle, dataset, queries, neighbor_candidates, indices, distances, metric); -} - -} // namespace raft::runtime::neighbors diff --git a/cpp/src/raft_runtime/neighbors/refine_d_int64_t_uint8_t.cu b/cpp/src/raft_runtime/neighbors/refine_d_int64_t_uint8_t.cu deleted file mode 100644 index 6ad8d9a38c..0000000000 --- a/cpp/src/raft_runtime/neighbors/refine_d_int64_t_uint8_t.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -namespace raft::runtime::neighbors { - -void refine(raft::resources const& handle, - raft::device_matrix_view dataset, - raft::device_matrix_view queries, - raft::device_matrix_view neighbor_candidates, - raft::device_matrix_view indices, - raft::device_matrix_view distances, - distance::DistanceType metric) -{ - raft::neighbors::refine( - handle, dataset, queries, neighbor_candidates, indices, distances, metric); -} - -} // namespace raft::runtime::neighbors diff --git a/cpp/src/raft_runtime/neighbors/refine_h_int64_t_float.cu b/cpp/src/raft_runtime/neighbors/refine_h_int64_t_float.cu deleted file mode 100644 index 3d186c017c..0000000000 --- a/cpp/src/raft_runtime/neighbors/refine_h_int64_t_float.cu +++ /dev/null @@ -1,34 +0,0 @@ - -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -namespace raft::runtime::neighbors { - -void refine(raft::resources const& handle, - raft::host_matrix_view dataset, - raft::host_matrix_view queries, - raft::host_matrix_view neighbor_candidates, - raft::host_matrix_view indices, - raft::host_matrix_view distances, - distance::DistanceType metric) -{ - raft::neighbors::refine( - handle, dataset, queries, neighbor_candidates, indices, distances, metric); -} - -} // namespace raft::runtime::neighbors diff --git a/cpp/src/raft_runtime/neighbors/refine_h_int64_t_int8_t.cu b/cpp/src/raft_runtime/neighbors/refine_h_int64_t_int8_t.cu deleted file mode 100644 index 93237d11d5..0000000000 --- a/cpp/src/raft_runtime/neighbors/refine_h_int64_t_int8_t.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -namespace raft::runtime::neighbors { - -void refine(raft::resources const& handle, - raft::host_matrix_view dataset, - raft::host_matrix_view queries, - raft::host_matrix_view neighbor_candidates, - raft::host_matrix_view indices, - raft::host_matrix_view distances, - distance::DistanceType metric) -{ - raft::neighbors::refine( - handle, dataset, queries, neighbor_candidates, indices, distances, metric); -} - -} // namespace raft::runtime::neighbors diff --git a/cpp/src/raft_runtime/neighbors/refine_h_int64_t_uint8_t.cu b/cpp/src/raft_runtime/neighbors/refine_h_int64_t_uint8_t.cu deleted file mode 100644 index 91771e171f..0000000000 --- a/cpp/src/raft_runtime/neighbors/refine_h_int64_t_uint8_t.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -namespace raft::runtime::neighbors { - -void refine(raft::resources const& handle, - raft::host_matrix_view dataset, - raft::host_matrix_view queries, - raft::host_matrix_view neighbor_candidates, - raft::host_matrix_view indices, - raft::host_matrix_view distances, - distance::DistanceType metric) -{ - raft::neighbors::refine( - handle, dataset, queries, neighbor_candidates, indices, distances, metric); -} - -} // namespace raft::runtime::neighbors diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers.cu b/cpp/src/spatial/knn/detail/ball_cover/registers.cu deleted file mode 100644 index 31595272b6..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers.cu +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_one( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - raft::spatial::knn::detail::DistFunc& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_two( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - raft::spatial::knn::detail::DistFunc& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -#define instantiate_raft_spatial_knn_detail_rbc_eps_pass( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx) \ - template void \ - raft::spatial::knn::detail::rbc_eps_pass( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_t eps, \ - const Mvalue_t* R_dists, \ - raft::spatial::knn::detail::DistFunc& dfunc, \ - bool* adj, \ - Mvalue_idx* vd); \ - \ - template void \ - raft::spatial::knn::detail::rbc_eps_pass( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_t eps, \ - const Mvalue_t* R_dists, \ - raft::spatial::knn::detail::DistFunc& dfunc, \ - Mvalue_idx* ia, \ - Mvalue_idx* ja, \ - Mvalue_idx* vd) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 2); -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 3); - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 2); -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 3); - -instantiate_raft_spatial_knn_detail_rbc_eps_pass(std::int64_t, float, std::int64_t, std::int64_t); - -#undef instantiate_raft_spatial_knn_detail_rbc_eps_pass -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_00_generate.py b/cpp/src/spatial/knn/detail/ball_cover/registers_00_generate.py deleted file mode 100644 index 10d9c95ece..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_00_generate.py +++ /dev/null @@ -1,164 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -header = """/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include // int64_t -#include - -""" - - -macro_pass_one = """ -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( \\ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \\ - template void \\ - raft::spatial::knn::detail::rbc_low_dim_pass_one( \\ - raft::resources const& handle, \\ - const BallCoverIndex& index, \\ - const Mvalue_t* query, \\ - const Mvalue_int n_query_rows, \\ - Mvalue_int k, \\ - const Mvalue_idx* R_knn_inds, \\ - const Mvalue_t* R_knn_dists, \\ - Mdist_func& dfunc, \\ - Mvalue_idx* inds, \\ - Mvalue_t* dists, \\ - float weight, \\ - Mvalue_int* dists_counter) - -""" - -macro_pass_two = """ -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( \\ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \\ - template void \\ - raft::spatial::knn::detail::rbc_low_dim_pass_two( \\ - raft::resources const& handle, \\ - const BallCoverIndex& index, \\ - const Mvalue_t* query, \\ - const Mvalue_int n_query_rows, \\ - Mvalue_int k, \\ - const Mvalue_idx* R_knn_inds, \\ - const Mvalue_t* R_knn_dists, \\ - Mdist_func& dfunc, \\ - Mvalue_idx* inds, \\ - Mvalue_t* dists, \\ - float weight, \\ - Mvalue_int* dists_counter) - -""" - -macro_pass_eps = """ -#define instantiate_raft_spatial_knn_detail_rbc_eps_pass( \\ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdist_func) \\ - template void \\ - raft::spatial::knn::detail::rbc_eps_pass( \\ - raft::resources const& handle, \\ - const BallCoverIndex& index, \\ - const Mvalue_t* query, \\ - const Mvalue_int n_query_rows, \\ - Mvalue_t eps, \\ - const Mvalue_t* R_dists, \\ - Mdist_func& dfunc, \\ - bool* adj, \\ - Mvalue_idx* vd); \\ - \\ - template void \\ - raft::spatial::knn::detail::rbc_eps_pass( \\ - raft::resources const& handle, \\ - const BallCoverIndex& index, \\ - const Mvalue_t* query, \\ - const Mvalue_int n_query_rows, \\ - Mvalue_t eps, \\ - Mvalue_int* max_k, \\ - const Mvalue_t* R_dists, \\ - Mdist_func& dfunc, \\ - Mvalue_idx* adj_ia, \\ - Mvalue_idx* adj_ja, \\ - Mvalue_idx* vd) - -""" - - -distances = dict( - haversine="raft::spatial::knn::detail::HaversineFunc", - euclidean="raft::spatial::knn::detail::EuclideanFunc", - dist="raft::spatial::knn::detail::DistFunc", -) - -euclideanSq="raft::spatial::knn::detail::EuclideanSqFunc", - -types = dict( - int64_float=("std::int64_t", "float"), - #int64_double=("std::int64_t", "double"), -) - -for k, v in distances.items(): - for dim in [2, 3]: - path = f"registers_pass_one_{dim}d_{k}.cu" - with open(path, "w") as f: - f.write(header) - f.write(macro_pass_one) - for type_path, (int_t, data_t) in types.items(): - f.write(f"instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one(\n") - f.write(f" {int_t}, {data_t}, {int_t}, {int_t}, {dim}, {v});\n") - f.write("#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one\n") - print(f"src/spatial/knn/detail/ball_cover/{path}") - -for k, v in distances.items(): - for dim in [2, 3]: - path = f"registers_pass_two_{dim}d_{k}.cu" - with open(path, "w") as f: - f.write(header) - f.write(macro_pass_two) - for type_path, (int_t, data_t) in types.items(): - f.write(f"instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two(\n") - f.write(f" {int_t}, {data_t}, {int_t}, {int_t}, {dim}, {v});\n") - f.write("#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two\n") - print(f"src/spatial/knn/detail/ball_cover/{path}") - -path="registers_eps_pass_euclidean.cu" -with open(path, "w") as f: - f.write(header) - f.write(macro_pass_eps) - for type_path, (int_t, data_t) in types.items(): - f.write(f"instantiate_raft_spatial_knn_detail_rbc_eps_pass(\n") - f.write(f" {int_t}, {data_t}, {int_t}, {int_t}, {euclideanSq});\n") - f.write("#undef instantiate_raft_spatial_knn_detail_rbc_eps_pass\n") - print(f"src/spatial/knn/detail/ball_cover/{path}") - diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_eps_pass_euclidean.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_eps_pass_euclidean.cu deleted file mode 100644 index 2a88862b2c..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_eps_pass_euclidean.cu +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_eps_pass( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdist_func) \ - template void \ - raft::spatial::knn::detail::rbc_eps_pass( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_t eps, \ - const Mvalue_t* R_dists, \ - Mdist_func& dfunc, \ - bool* adj, \ - Mvalue_idx* vd); \ - \ - template void \ - raft::spatial::knn::detail::rbc_eps_pass( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_t eps, \ - Mvalue_int* max_k, \ - const Mvalue_t* R_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* adj_ia, \ - Mvalue_idx* adj_ja, \ - Mvalue_idx* vd) - -instantiate_raft_spatial_knn_detail_rbc_eps_pass( - std::int64_t, float, std::int64_t, std::int64_t, raft::spatial::knn::detail::EuclideanSqFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_eps_pass diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_dist.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_dist.cu deleted file mode 100644 index 8c46888e05..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_dist.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_one( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 2, raft::spatial::knn::detail::DistFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_euclidean.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_euclidean.cu deleted file mode 100644 index c5a6970a94..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_euclidean.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_one( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 2, raft::spatial::knn::detail::EuclideanFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_haversine.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_haversine.cu deleted file mode 100644 index 19eb4b681b..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_haversine.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_one( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 2, raft::spatial::knn::detail::HaversineFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_dist.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_dist.cu deleted file mode 100644 index 6f878e1296..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_dist.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_one( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 3, raft::spatial::knn::detail::DistFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_euclidean.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_euclidean.cu deleted file mode 100644 index e8fbcd0528..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_euclidean.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_one( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 3, raft::spatial::knn::detail::EuclideanFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_haversine.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_haversine.cu deleted file mode 100644 index 57b8c790ef..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_haversine.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_one( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 3, raft::spatial::knn::detail::HaversineFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_dist.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_dist.cu deleted file mode 100644 index 6e1cc1e35e..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_dist.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_two( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 2, raft::spatial::knn::detail::DistFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_euclidean.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_euclidean.cu deleted file mode 100644 index d1123aff7d..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_euclidean.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_two( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 2, raft::spatial::knn::detail::EuclideanFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_haversine.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_haversine.cu deleted file mode 100644 index 3e118e2f8f..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_haversine.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_two( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 2, raft::spatial::knn::detail::HaversineFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_dist.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_dist.cu deleted file mode 100644 index 674f532e6c..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_dist.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_two( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 3, raft::spatial::knn::detail::DistFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_euclidean.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_euclidean.cu deleted file mode 100644 index dd3a422baa..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_euclidean.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_two( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 3, raft::spatial::knn::detail::EuclideanFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_haversine.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_haversine.cu deleted file mode 100644 index 0b05ca91a9..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_haversine.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_two( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 3, raft::spatial::knn::detail::HaversineFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two diff --git a/cpp/src/spatial/knn/detail/fused_l2_knn_int32_t_float.cu b/cpp/src/spatial/knn/detail/fused_l2_knn_int32_t_float.cu deleted file mode 100644 index aba67b8a3d..0000000000 --- a/cpp/src/spatial/knn/detail/fused_l2_knn_int32_t_float.cu +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include // DistanceType -#include - -#include // size_t -#include // int_Xt - -#define instantiate_raft_spatial_knn_detail_fusedL2Knn(Mvalue_idx, Mvalue_t, MusePrevTopKs) \ - template void raft::spatial::knn::detail::fusedL2Knn( \ - size_t D, \ - Mvalue_idx * out_inds, \ - Mvalue_t * out_dists, \ - const Mvalue_t* index, \ - const Mvalue_t* query, \ - size_t n_index_rows, \ - size_t n_query_rows, \ - int k, \ - bool rowMajorIndex, \ - bool rowMajorQuery, \ - cudaStream_t stream, \ - raft::distance::DistanceType metric, \ - const Mvalue_t* index_norms, \ - const Mvalue_t* query_norms) - -instantiate_raft_spatial_knn_detail_fusedL2Knn(int32_t, float, true); -instantiate_raft_spatial_knn_detail_fusedL2Knn(int32_t, float, false); - -#undef instantiate_raft_spatial_knn_detail_fusedL2Knn diff --git a/cpp/src/spatial/knn/detail/fused_l2_knn_int64_t_float.cu b/cpp/src/spatial/knn/detail/fused_l2_knn_int64_t_float.cu deleted file mode 100644 index c9d0c49727..0000000000 --- a/cpp/src/spatial/knn/detail/fused_l2_knn_int64_t_float.cu +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include // DistanceType -#include - -#include // size_t -#include // int_Xt - -#define instantiate_raft_spatial_knn_detail_fusedL2Knn(Mvalue_idx, Mvalue_t, MusePrevTopKs) \ - template void raft::spatial::knn::detail::fusedL2Knn( \ - size_t D, \ - Mvalue_idx * out_inds, \ - Mvalue_t * out_dists, \ - const Mvalue_t* index, \ - const Mvalue_t* query, \ - size_t n_index_rows, \ - size_t n_query_rows, \ - int k, \ - bool rowMajorIndex, \ - bool rowMajorQuery, \ - cudaStream_t stream, \ - raft::distance::DistanceType metric, \ - const Mvalue_t* index_norms, \ - const Mvalue_t* query_norms) - -instantiate_raft_spatial_knn_detail_fusedL2Knn(int64_t, float, true); -instantiate_raft_spatial_knn_detail_fusedL2Knn(int64_t, float, false); - -#undef instantiate_raft_spatial_knn_detail_fusedL2Knn diff --git a/cpp/src/spatial/knn/detail/fused_l2_knn_uint32_t_float.cu b/cpp/src/spatial/knn/detail/fused_l2_knn_uint32_t_float.cu deleted file mode 100644 index e6e9765151..0000000000 --- a/cpp/src/spatial/knn/detail/fused_l2_knn_uint32_t_float.cu +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include // DistanceType -#include - -#include // size_t -#include // int_Xt - -#define instantiate_raft_spatial_knn_detail_fusedL2Knn(Mvalue_idx, Mvalue_t, MusePrevTopKs) \ - template void raft::spatial::knn::detail::fusedL2Knn( \ - size_t D, \ - Mvalue_idx * out_inds, \ - Mvalue_t * out_dists, \ - const Mvalue_t* index, \ - const Mvalue_t* query, \ - size_t n_index_rows, \ - size_t n_query_rows, \ - int k, \ - bool rowMajorIndex, \ - bool rowMajorQuery, \ - cudaStream_t stream, \ - raft::distance::DistanceType metric, \ - const Mvalue_t* index_norms, \ - const Mvalue_t* query_norms) - -// These are used by brute_force_knn: -instantiate_raft_spatial_knn_detail_fusedL2Knn(uint32_t, float, true); -instantiate_raft_spatial_knn_detail_fusedL2Knn(uint32_t, float, false); - -#undef instantiate_raft_spatial_knn_detail_fusedL2Knn diff --git a/cpp/template/CMakeLists.txt b/cpp/template/CMakeLists.txt deleted file mode 100644 index 40a3795ed1..0000000000 --- a/cpp/template/CMakeLists.txt +++ /dev/null @@ -1,44 +0,0 @@ -# ============================================================================= -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. - -cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) - -# ------------- configure rapids-cmake --------------# - -include(cmake/thirdparty/fetch_rapids.cmake) -include(rapids-cmake) -include(rapids-cpm) -include(rapids-cuda) -include(rapids-export) -include(rapids-find) - -# ------------- configure project --------------# - -rapids_cuda_init_architectures(test_raft) - -project(test_raft LANGUAGES CXX CUDA) - -# ------------- configure raft -----------------# - -rapids_cpm_init() -include(cmake/thirdparty/get_raft.cmake) - -# -------------- compile tasks ----------------- # -add_executable(CAGRA_EXAMPLE src/cagra_example.cu) -target_link_libraries(CAGRA_EXAMPLE PRIVATE raft::raft raft::compiled) - -add_executable(IVF_FLAT_EXAMPLE src/ivf_flat_example.cu) -target_link_libraries(IVF_FLAT_EXAMPLE PRIVATE raft::raft raft::compiled) - -add_executable(IVF_PQ_EXAMPLE src/ivf_pq_example.cu) -target_link_libraries(IVF_PQ_EXAMPLE PRIVATE raft::raft raft::compiled) diff --git a/cpp/template/README.md b/cpp/template/README.md deleted file mode 100644 index 05ec48964f..0000000000 --- a/cpp/template/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Example RAFT Project Template - -This template project provides a drop-in sample to either start building a new application with, or using RAFT in an existing CMake project. - -First, please refer to our [installation docs](https://docs.rapids.ai/api/raft/stable/build.html#cuda-gpu-requirements) for the minimum requirements to use RAFT. - -Once the minimum requirements are satisfied, this example template application can be built with the provided `build.sh` script. This is a bash script that calls the appropriate CMake commands, so you can look into it to see the typical CMake based build workflow. - -This directory (`RAFT_SOURCE/cpp/template`) can be copied directly in order to build a new application with RAFT. - -RAFT can be integrated into an existing CMake project by copying the contents in the `configure rapids-cmake` and `configure raft` sections of the provided `CMakeLists.txt` into your project, along with `cmake/thirdparty/get_raft.cmake`. - -Make sure to link against the appropriate Cmake targets. Use `raft::raft`to add make the headers available and `raft::compiled` when utilizing the shared library. - -```cmake -target_link_libraries(your_app_target PRIVATE raft::raft raft::compiled) -``` - diff --git a/cpp/template/build.sh b/cpp/template/build.sh deleted file mode 100755 index 49c17f7499..0000000000 --- a/cpp/template/build.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2023-2024, NVIDIA CORPORATION. - -# raft empty project template build script - -# Abort script on first error -set -e - -PARALLEL_LEVEL=${PARALLEL_LEVEL:=`nproc`} - -BUILD_TYPE=Release -BUILD_DIR=build/ - -RAFT_REPO_REL="" -EXTRA_CMAKE_ARGS="" -set -e - - -if [[ ${RAFT_REPO_REL} != "" ]]; then - RAFT_REPO_PATH="`readlink -f \"${RAFT_REPO_REL}\"`" - EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCPM_raft_SOURCE=${RAFT_REPO_PATH}" -fi - -if [ "$1" == "clean" ]; then - rm -rf build - exit 0 -fi - -mkdir -p $BUILD_DIR -cd $BUILD_DIR - -cmake \ - -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ - -DRAFT_NVTX=OFF \ - -DCMAKE_CUDA_ARCHITECTURES="NATIVE" \ - -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ - ${EXTRA_CMAKE_ARGS} \ - ../ - -cmake --build . -j${PARALLEL_LEVEL} diff --git a/cpp/template/cmake/thirdparty/fetch_rapids.cmake b/cpp/template/cmake/thirdparty/fetch_rapids.cmake deleted file mode 100644 index 6f4c627ed4..0000000000 --- a/cpp/template/cmake/thirdparty/fetch_rapids.cmake +++ /dev/null @@ -1,21 +0,0 @@ -# ============================================================================= -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. - -# Use this variable to update RAPIDS and RAFT versions -set(RAPIDS_VERSION "24.12") - -if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/RAFT_RAPIDS.cmake) - file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-${RAPIDS_VERSION}/RAPIDS.cmake - ${CMAKE_CURRENT_BINARY_DIR}/RAFT_RAPIDS.cmake) -endif() -include(${CMAKE_CURRENT_BINARY_DIR}/RAFT_RAPIDS.cmake) diff --git a/cpp/template/src/cagra_example.cu b/cpp/template/src/cagra_example.cu deleted file mode 100644 index 3c1be8b4f8..0000000000 --- a/cpp/template/src/cagra_example.cu +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "common.cuh" - -#include -#include -#include -#include - -#include -#include - -#include - -void cagra_build_search_simple(raft::device_resources const& dev_resources, - raft::device_matrix_view dataset, - raft::device_matrix_view queries) -{ - using namespace raft::neighbors; - - int64_t topk = 12; - int64_t n_queries = queries.extent(0); - - // create output arrays - auto neighbors = raft::make_device_matrix(dev_resources, n_queries, topk); - auto distances = raft::make_device_matrix(dev_resources, n_queries, topk); - - // use default index parameters - cagra::index_params index_params; - - std::cout << "Building CAGRA index (search graph)" << std::endl; - auto index = cagra::build(dev_resources, index_params, dataset); - - std::cout << "CAGRA index has " << index.size() << " vectors" << std::endl; - std::cout << "CAGRA graph has degree " << index.graph_degree() << ", graph size [" - << index.graph().extent(0) << ", " << index.graph().extent(1) << "]" << std::endl; - - // use default search parameters - cagra::search_params search_params; - // search K nearest neighbors - cagra::search( - dev_resources, search_params, index, queries, neighbors.view(), distances.view()); - - // The call to ivf_flat::search is asynchronous. Before accessing the data, sync by calling - // raft::resource::sync_stream(dev_resources); - - print_results(dev_resources, neighbors.view(), distances.view()); -} - -int main() -{ - raft::device_resources dev_resources; - - // Set pool memory resource with 1 GiB initial pool size. All allocations use the same pool. - rmm::mr::pool_memory_resource pool_mr( - rmm::mr::get_current_device_resource(), 1024 * 1024 * 1024ull); - rmm::mr::set_current_device_resource(&pool_mr); - - // Alternatively, one could define a pool allocator for temporary arrays (used within RAFT - // algorithms). In that case only the internal arrays would use the pool, any other allocation - // uses the default RMM memory resource. Here is how to change the workspace memory resource to - // a pool with 2 GiB upper limit. - // raft::resource::set_workspace_to_pool_resource(dev_resources, 2 * 1024 * 1024 * 1024ull); - - // Create input arrays. - int64_t n_samples = 10000; - int64_t n_dim = 90; - int64_t n_queries = 10; - auto dataset = raft::make_device_matrix(dev_resources, n_samples, n_dim); - auto queries = raft::make_device_matrix(dev_resources, n_queries, n_dim); - generate_dataset(dev_resources, dataset.view(), queries.view()); - - // Simple build and search example. - cagra_build_search_simple(dev_resources, - raft::make_const_mdspan(dataset.view()), - raft::make_const_mdspan(queries.view())); -} diff --git a/cpp/template/src/common.cuh b/cpp/template/src/common.cuh deleted file mode 100644 index 3057257537..0000000000 --- a/cpp/template/src/common.cuh +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include - -// Fill dataset and queries with synthetic data. -void generate_dataset(raft::device_resources const& dev_resources, - raft::device_matrix_view dataset, - raft::device_matrix_view queries) -{ - auto labels = raft::make_device_vector(dev_resources, dataset.extent(0)); - raft::random::make_blobs(dev_resources, dataset, labels.view()); - raft::random::RngState r(1234ULL); - raft::random::uniform(dev_resources, - r, - raft::make_device_vector_view(queries.data_handle(), queries.size()), - -1.0f, - 1.0f); -} - -// Copy the results to host and print them -template -void print_results(raft::device_resources const& dev_resources, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances) -{ - int64_t topk = neighbors.extent(1); - auto neighbors_host = raft::make_host_matrix(neighbors.extent(0), topk); - auto distances_host = raft::make_host_matrix(distances.extent(0), topk); - - cudaStream_t stream = raft::resource::get_cuda_stream(dev_resources); - - raft::copy(neighbors_host.data_handle(), neighbors.data_handle(), neighbors.size(), stream); - raft::copy(distances_host.data_handle(), distances.data_handle(), distances.size(), stream); - - // The calls to RAFT algorithms and raft::copy is asynchronous. - // We need to sync the stream before accessing the data. - raft::resource::sync_stream(dev_resources, stream); - - for (int query_id = 0; query_id < neighbors.extent(0); query_id++) { - std::cout << "Query " << query_id << " neighbor indices: "; - raft::print_host_vector("", &neighbors_host(query_id, 0), topk, std::cout); - std::cout << "Query " << query_id << " neighbor distances: "; - raft::print_host_vector("", &distances_host(query_id, 0), topk, std::cout); - } -} - -/** Subsample the dataset to create a training set*/ -raft::device_matrix subsample( - raft::device_resources const& dev_resources, - raft::device_matrix_view dataset, - raft::device_vector_view data_indices, - float fraction) -{ - int64_t n_samples = dataset.extent(0); - int64_t n_dim = dataset.extent(1); - int64_t n_train = n_samples * fraction; - auto trainset = raft::make_device_matrix(dev_resources, n_train, n_dim); - - int seed = 137; - raft::random::RngState rng(seed); - auto train_indices = raft::make_device_vector(dev_resources, n_train); - - raft::random::sample_without_replacement( - dev_resources, rng, data_indices, std::nullopt, train_indices.view(), std::nullopt); - - raft::matrix::copy_rows( - dev_resources, dataset, trainset.view(), raft::make_const_mdspan(train_indices.view())); - - return trainset; -} diff --git a/cpp/template/src/ivf_flat_example.cu b/cpp/template/src/ivf_flat_example.cu deleted file mode 100644 index 60694aea0f..0000000000 --- a/cpp/template/src/ivf_flat_example.cu +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "common.cuh" - -#include -#include -#include -#include -#include - -#include -#include - -#include -#include -#include - -#include -#include - -void ivf_flat_build_search_simple(raft::device_resources const& dev_resources, - raft::device_matrix_view dataset, - raft::device_matrix_view queries) -{ - using namespace raft::neighbors; - - ivf_flat::index_params index_params; - index_params.n_lists = 1024; - index_params.kmeans_trainset_fraction = 0.1; - index_params.metric = raft::distance::DistanceType::L2Expanded; - - std::cout << "Building IVF-Flat index" << std::endl; - auto index = ivf_flat::build(dev_resources, index_params, dataset); - - std::cout << "Number of clusters " << index.n_lists() << ", number of vectors added to index " - << index.size() << std::endl; - - // Create output arrays. - int64_t topk = 10; - int64_t n_queries = queries.extent(0); - auto neighbors = raft::make_device_matrix(dev_resources, n_queries, topk); - auto distances = raft::make_device_matrix(dev_resources, n_queries, topk); - - // Set search parameters. - ivf_flat::search_params search_params; - search_params.n_probes = 50; - - // Search K nearest neighbors for each of the queries. - ivf_flat::search( - dev_resources, search_params, index, queries, neighbors.view(), distances.view()); - - // The call to ivf_flat::search is asynchronous. Before accessing the data, sync by calling - // raft::resource::sync_stream(dev_resources); - - print_results(dev_resources, neighbors.view(), distances.view()); -} - -void ivf_flat_build_extend_search(raft::device_resources const& dev_resources, - raft::device_matrix_view dataset, - raft::device_matrix_view queries) -{ - using namespace raft::neighbors; - - // Define dataset indices. - auto data_indices = raft::make_device_vector(dev_resources, dataset.extent(0)); - thrust::counting_iterator first(0); - thrust::device_ptr ptr(data_indices.data_handle()); - thrust::copy( - raft::resource::get_thrust_policy(dev_resources), first, first + dataset.extent(0), ptr); - - // Sub-sample the dataset to create a training set. - auto trainset = - subsample(dev_resources, dataset, raft::make_const_mdspan(data_indices.view()), 0.1); - - ivf_flat::index_params index_params; - index_params.n_lists = 100; - index_params.metric = raft::distance::DistanceType::L2Expanded; - index_params.add_data_on_build = false; - - std::cout << "\nRun k-means clustering using the training set" << std::endl; - auto index = - ivf_flat::build(dev_resources, index_params, raft::make_const_mdspan(trainset.view())); - - std::cout << "Number of clusters " << index.n_lists() << ", number of vectors added to index " - << index.size() << std::endl; - - std::cout << "Filling index with the dataset vectors" << std::endl; - index = ivf_flat::extend(dev_resources, - dataset, - std::make_optional(raft::make_const_mdspan(data_indices.view())), - index); - - std::cout << "Index size after addin dataset vectors " << index.size() << std::endl; - - // Set search parameters. - ivf_flat::search_params search_params; - search_params.n_probes = 10; - - // Create output arrays. - int64_t topk = 10; - int64_t n_queries = queries.extent(0); - auto neighbors = raft::make_device_matrix(dev_resources, n_queries, topk); - auto distances = raft::make_device_matrix(dev_resources, n_queries, topk); - - // Search K nearest neighbors for each queries. - ivf_flat::search( - dev_resources, search_params, index, queries, neighbors.view(), distances.view()); - - // The call to ivf_flat::search is asynchronous. Before accessing the data, sync using: - // raft::resource::sync_stream(dev_resources); - - print_results(dev_resources, neighbors.view(), distances.view()); -} - -int main() -{ - raft::device_resources dev_resources; - - // Set pool memory resource with 1 GiB initial pool size. All allocations use the same pool. - rmm::mr::pool_memory_resource pool_mr( - rmm::mr::get_current_device_resource(), 1024 * 1024 * 1024ull); - rmm::mr::set_current_device_resource(&pool_mr); - - // Alternatively, one could define a pool allocator for temporary arrays (used within RAFT - // algorithms). In that case only the internal arrays would use the pool, any other allocation - // uses the default RMM memory resource. Here is how to change the workspace memory resource to - // a pool with 2 GiB upper limit. - // raft::resource::set_workspace_to_pool_resource(dev_resources, 2 * 1024 * 1024 * 1024ull); - - // Create input arrays. - int64_t n_samples = 10000; - int64_t n_dim = 3; - int64_t n_queries = 10; - auto dataset = raft::make_device_matrix(dev_resources, n_samples, n_dim); - auto queries = raft::make_device_matrix(dev_resources, n_queries, n_dim); - generate_dataset(dev_resources, dataset.view(), queries.view()); - - // Simple build and search example. - ivf_flat_build_search_simple(dev_resources, - raft::make_const_mdspan(dataset.view()), - raft::make_const_mdspan(queries.view())); - - // Build and extend example. - ivf_flat_build_extend_search(dev_resources, - raft::make_const_mdspan(dataset.view()), - raft::make_const_mdspan(queries.view())); -} diff --git a/cpp/template/src/ivf_pq_example.cu b/cpp/template/src/ivf_pq_example.cu deleted file mode 100644 index 4bc0ba4348..0000000000 --- a/cpp/template/src/ivf_pq_example.cu +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "common.cuh" - -#include -#include -#include -#include - -#include -#include - -#include - -void ivf_pq_build_search(raft::device_resources const& dev_resources, - raft::device_matrix_view dataset, - raft::device_matrix_view queries) -{ - using namespace raft::neighbors; // NOLINT - - ivf_pq::index_params index_params; - index_params.n_lists = 1024; - index_params.kmeans_trainset_fraction = 0.1; - index_params.metric = raft::distance::DistanceType::L2Expanded; - index_params.pq_bits = 8; - index_params.pq_dim = 2; - - std::cout << "Building IVF-PQ index" << std::endl; - auto index = ivf_pq::build(dev_resources, index_params, dataset); - - std::cout << "Number of clusters " << index.n_lists() << ", number of vectors added to index " - << index.size() << std::endl; - - // Set search parameters. - ivf_pq::search_params search_params; - search_params.n_probes = 50; - // Set the internal search precision to 16-bit floats; - // usually, this improves the performance at a slight cost to the recall. - search_params.internal_distance_dtype = CUDA_R_16F; - search_params.lut_dtype = CUDA_R_16F; - - // Create output arrays. - int64_t topk = 10; - int64_t n_queries = queries.extent(0); - auto neighbors = raft::make_device_matrix(dev_resources, n_queries, topk); - auto distances = raft::make_device_matrix(dev_resources, n_queries, topk); - - // Search K nearest neighbors for each of the queries. - ivf_pq::search( - dev_resources, search_params, index, queries, neighbors.view(), distances.view()); - - // Re-ranking operation: refine the initial search results by computing exact distances - int64_t topk_refined = 7; - auto neighbors_refined = - raft::make_device_matrix(dev_resources, n_queries, topk_refined); - auto distances_refined = raft::make_device_matrix(dev_resources, n_queries, topk_refined); - - // Note, refinement requires the original dataset and the queries. - // Don't forget to specify the same distance metric as used by the index. - raft::neighbors::refine(dev_resources, - dataset, - queries, - raft::make_const_mdspan(neighbors.view()), - neighbors_refined.view(), - distances_refined.view(), - index.metric()); - - // Show both the original and the refined results - std::cout << std::endl << "Original results:" << std::endl; - print_results(dev_resources, neighbors.view(), distances.view()); - std::cout << std::endl << "Refined results:" << std::endl; - print_results(dev_resources, neighbors_refined.view(), distances_refined.view()); -} - -int main() -{ - raft::device_resources dev_resources; - - // Set pool memory resource with 1 GiB initial pool size. All allocations use the same pool. - rmm::mr::pool_memory_resource pool_mr( - rmm::mr::get_current_device_resource(), 1024 * 1024 * 1024ull); - rmm::mr::set_current_device_resource(&pool_mr); - - // Alternatively, one could define a pool allocator for temporary arrays (used within RAFT - // algorithms). In that case only the internal arrays would use the pool, any other allocation - // uses the default RMM memory resource. Here is how to change the workspace memory resource to - // a pool with 2 GiB upper limit. - // raft::resource::set_workspace_to_pool_resource(dev_resources, 2 * 1024 * 1024 * 1024ull); - - // Create input arrays. - int64_t n_samples = 10000; - int64_t n_dim = 3; - int64_t n_queries = 10; - auto dataset = raft::make_device_matrix(dev_resources, n_samples, n_dim); - auto queries = raft::make_device_matrix(dev_resources, n_queries, n_dim); - generate_dataset(dev_resources, dataset.view(), queries.view()); - - // Simple build and search example. - ivf_pq_build_search(dev_resources, - raft::make_const_mdspan(dataset.view()), - raft::make_const_mdspan(queries.view())); -} diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 5d504d2100..a387e9ce09 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -90,16 +90,7 @@ endfunction() # ################################################################################################## # test sources ################################################################################## # ################################################################################################## - -# ################################################################################################## -# * distance tests ------------------------------------------------------------------------- - if(BUILD_TESTS) - ConfigureTest( - NAME CLUSTER_TEST PATH cluster/kmeans.cu cluster/kmeans_balanced.cu cluster/kmeans_find_k.cu - cluster/cluster_solvers.cu cluster/linkage.cu cluster/spectral.cu LIB EXPLICIT_INSTANTIATE_ONLY - ) - ConfigureTest( NAME CORE_TEST @@ -139,58 +130,7 @@ if(BUILD_TESTS) NOCUDA ) - ConfigureTest( - NAME - DISTANCE_TEST - PATH - distance/dist_adj.cu - distance/dist_adj_distance_instance.cu - distance/dist_canberra.cu - distance/dist_correlation.cu - distance/dist_cos.cu - distance/dist_dice.cu - distance/dist_hamming.cu - distance/dist_hellinger.cu - distance/dist_inner_product.cu - distance/dist_jensen_shannon.cu - distance/dist_kl_divergence.cu - distance/dist_l1.cu - distance/dist_l2_exp.cu - distance/dist_l2_unexp.cu - distance/dist_l2_sqrt_exp.cu - distance/dist_l_inf.cu - distance/dist_lp_unexp.cu - distance/dist_russell_rao.cu - distance/masked_nn.cu - distance/masked_nn_compress_to_bits.cu - distance/fused_l2_nn.cu - distance/fused_cosine_nn.cu - distance/gram.cu - LIB - EXPLICIT_INSTANTIATE_ONLY - ) - - list( - APPEND - EXT_HEADER_TEST_SOURCES - ext_headers/raft_neighbors_brute_force.cu - ext_headers/raft_distance_distance.cu - ext_headers/raft_distance_detail_pairwise_matrix_dispatch.cu - ext_headers/raft_matrix_detail_select_k.cu - ext_headers/raft_neighbors_ball_cover.cu - ext_headers/raft_spatial_knn_detail_fused_l2_knn.cu - ext_headers/raft_distance_fused_l2_nn.cu - ext_headers/raft_neighbors_ivf_pq.cu - ext_headers/raft_neighbors_ivf_flat.cu - ext_headers/raft_core_logger.cpp - ext_headers/raft_neighbors_refine.cu - ext_headers/raft_neighbors_detail_ivf_flat_search.cu - ext_headers/raft_linalg_detail_coalesced_reduction.cu - ext_headers/raft_sparse_matrix_detail_select_k.cu - ext_headers/raft_spatial_knn_detail_ball_cover_registers.cu - ext_headers/raft_neighbors_detail_ivf_flat_interleaved_scan.cu - ext_headers/raft_neighbors_detail_ivf_pq_compute_similarity.cu - ) + list(APPEND EXT_HEADER_TEST_SOURCES ext_headers/raft_core_logger.cpp) # Test that the split headers compile in isolation with: # @@ -292,8 +232,8 @@ if(BUILD_TESTS) ) ConfigureTest( - NAME SOLVERS_TEST PATH cluster/cluster_solvers_deprecated.cu linalg/eigen_solvers.cu lap/lap.cu - sparse/mst.cu LIB EXPLICIT_INSTANTIATE_ONLY + NAME SOLVERS_TEST PATH linalg/eigen_solvers.cu lap/lap.cu sparse/mst.cu LIB + EXPLICIT_INSTANTIATE_ONLY ) ConfigureTest( @@ -322,133 +262,8 @@ if(BUILD_TESTS) ) ConfigureTest( - NAME SPARSE_DIST_TEST PATH sparse/dist_coo_spmv.cu sparse/distance.cu sparse/gram.cu LIB - EXPLICIT_INSTANTIATE_ONLY - ) - - ConfigureTest( - NAME SPARSE_NEIGHBORS_TEST PATH sparse/neighbors/cross_component_nn.cu - sparse/neighbors/brute_force.cu sparse/neighbors/knn_graph.cu LIB EXPLICIT_INSTANTIATE_ONLY - ) - - ConfigureTest( - NAME - NEIGHBORS_TEST - PATH - neighbors/knn.cu - neighbors/fused_l2_knn.cu - neighbors/tiled_knn.cu - neighbors/haversine.cu - neighbors/ball_cover.cu - neighbors/epsilon_neighborhood.cu - neighbors/refine.cu - LIB - EXPLICIT_INSTANTIATE_ONLY - ) - - ConfigureTest( - NAME NEIGHBORS_ANN_BRUTE_FORCE_TEST PATH neighbors/ann_brute_force/test_float.cu LIB - EXPLICIT_INSTANTIATE_ONLY GPUS 1 PERCENT 100 - ) - - ConfigureTest( - NAME - NEIGHBORS_ANN_CAGRA_TEST - PATH - neighbors/ann_cagra/test_float_uint32_t.cu - neighbors/ann_cagra/test_half_uint32_t.cu - neighbors/ann_cagra/test_int8_t_uint32_t.cu - neighbors/ann_cagra/test_uint8_t_uint32_t.cu - neighbors/ann_cagra/test_float_int64_t.cu - neighbors/ann_cagra/test_half_int64_t.cu - neighbors/ann_cagra_vpq/test_float_int64_t.cu - neighbors/ann_cagra_vpq/test_float_uint32_t.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim128_t8.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim256_t16.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim512_t32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim1024_t32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim128_t8.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim256_t16.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim512_t32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim1024_t32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim128_t8.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim256_t16.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim512_t32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim1024_t32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim128_t8.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim256_t16.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim512_t32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim1024_t32.cu - LIB - EXPLICIT_INSTANTIATE_ONLY - GPUS - 1 - PERCENT - 100 - ) - - ConfigureTest( - NAME - NEIGHBORS_ANN_IVF_TEST - PATH - neighbors/ann_ivf_flat/test_filter_float_int64_t.cu - neighbors/ann_ivf_flat/test_float_int64_t.cu - neighbors/ann_ivf_flat/test_int8_t_int64_t.cu - neighbors/ann_ivf_flat/test_uint8_t_int64_t.cu - neighbors/ann_ivf_pq/ivf_pq_build_float_uint32_t.cu - neighbors/ann_ivf_pq/ivf_pq_search_float_uint32_t.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_search_filtering_float_int64_t.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_float_filt32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_filt32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_filt32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_half_filt32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_filt32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_filt32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_half_half_filt32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_float_bitset32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_bitset32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_bitset32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_half_bitset32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_bitset32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_bitset32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_half_half_bitset32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_float_bitset64.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_bitset64.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_bitset64.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_half_bitset64.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_bitset64.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_bitset64.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_half_half_bitset64.cu - neighbors/ann_ivf_pq/test_float_uint32_t.cu - neighbors/ann_ivf_pq/test_float_int64_t.cu - neighbors/ann_ivf_pq/test_int8_t_int64_t.cu - neighbors/ann_ivf_pq/test_uint8_t_int64_t.cu - neighbors/ann_ivf_pq/test_filter_float_int64_t.cu - neighbors/ann_ivf_pq/test_filter_int8_t_int64_t.cu - LIB - EXPLICIT_INSTANTIATE_ONLY - GPUS - 1 - PERCENT - 100 - ) - - ConfigureTest( - NAME - NEIGHBORS_ANN_NN_DESCENT_TEST - PATH - neighbors/ann_nn_descent/test_float_uint32_t.cu - neighbors/ann_nn_descent/test_int8_t_uint32_t.cu - neighbors/ann_nn_descent/test_uint8_t_uint32_t.cu - # TODO: Investigate why this test is failing Reference issue - # https://github.com/rapidsai/raft/issues/2450 - # neighbors/ann_nn_descent/test_batch_float_uint32_t.cu - LIB - EXPLICIT_INSTANTIATE_ONLY - GPUS - 1 - PERCENT - 100 + NAME NEIGHBORS_TEST PATH neighbors/haversine.cu neighbors/ball_cover.cu + neighbors/epsilon_neighborhood.cu LIB EXPLICIT_INSTANTIATE_ONLY ) ConfigureTest( @@ -471,14 +286,11 @@ if(BUILD_TESTS) stats/mean_center.cu stats/minmax.cu stats/mutual_info_score.cu - stats/neighborhood_recall.cu stats/r2_score.cu stats/rand_index.cu stats/regression_metrics.cu - stats/silhouette_score.cu stats/stddev.cu stats/sum.cu - stats/trustworthiness.cu stats/weighted_mean.cu stats/v_measure.cu LIB diff --git a/cpp/test/cluster/cluster_solvers.cu b/cpp/test/cluster/cluster_solvers.cu deleted file mode 100644 index cc0a381bbf..0000000000 --- a/cpp/test/cluster/cluster_solvers.cu +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright (c) 2020-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include - -#include -#include - -namespace raft { -namespace spectral { - -TEST(Raft, ClusterSolvers) -{ - using namespace matrix; - using index_type = int; - using value_type = double; - - raft::resources h; - - index_type maxiter{100}; - value_type tol{1.0e-10}; - unsigned long long seed{100110021003}; - - auto stream = resource::get_cuda_stream(h); - - index_type n{100}; - index_type d{10}; - index_type k{5}; - - // nullptr expected to trigger exceptions: - // - value_type* eigvecs{nullptr}; - index_type* codes{nullptr}; - - cluster_solver_config_t cfg{k, maxiter, tol, seed}; - - kmeans_solver_t cluster_solver{cfg}; - - EXPECT_ANY_THROW(cluster_solver.solve(h, n, d, eigvecs, codes)); -} - -TEST(Raft, ModularitySolvers) -{ - using namespace matrix; - using index_type = int; - using value_type = double; - - raft::resources h; - ASSERT_EQ(0, resource::get_device_id(h)); - - index_type neigvs{10}; - index_type maxiter{100}; - index_type restart_iter{10}; - value_type tol{1.0e-10}; - bool reorthog{true}; - - // nullptr expected to trigger exceptions: - // - index_type* clusters{nullptr}; - value_type* eigvals{nullptr}; - value_type* eigvecs{nullptr}; - - unsigned long long seed{100110021003}; - - eigen_solver_config_t eig_cfg{ - neigvs, maxiter, restart_iter, tol, reorthog, seed}; - lanczos_solver_t eig_solver{eig_cfg}; - - index_type k{5}; - - cluster_solver_config_t clust_cfg{k, maxiter, tol, seed}; - kmeans_solver_t cluster_solver{clust_cfg}; - - auto stream = resource::get_cuda_stream(h); - sparse_matrix_t sm{h, nullptr, nullptr, nullptr, 0, 0}; - - EXPECT_ANY_THROW(spectral::modularity_maximization( - h, sm, eig_solver, cluster_solver, clusters, eigvals, eigvecs)); - - value_type modularity{0}; - EXPECT_ANY_THROW(spectral::analyzeModularity(h, sm, k, clusters, modularity)); -} - -} // namespace spectral -} // namespace raft diff --git a/cpp/test/cluster/cluster_solvers_deprecated.cu b/cpp/test/cluster/cluster_solvers_deprecated.cu deleted file mode 100644 index 954e3e5bb6..0000000000 --- a/cpp/test/cluster/cluster_solvers_deprecated.cu +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2020-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include - -#include -#include - -namespace raft { -namespace spectral { - -TEST(Raft, ClusterSolvers) -{ - using namespace matrix; - using index_type = int; - using value_type = double; - - raft::resources h; - - index_type maxiter{100}; - value_type tol{1.0e-10}; - unsigned long long seed{100110021003}; - - auto stream = resource::get_cuda_stream(h); - - index_type n{100}; - index_type d{10}; - index_type k{5}; - - // nullptr expected to trigger exceptions: - // - value_type* eigvecs{nullptr}; - index_type* codes{nullptr}; - - cluster_solver_config_deprecated_t cfg{k, maxiter, tol, seed}; - kmeans_solver_deprecated_t cluster_solver{cfg}; - - EXPECT_ANY_THROW(cluster_solver.solve(h, n, d, eigvecs, codes)); -} - -} // namespace spectral -} // namespace raft diff --git a/cpp/test/cluster/kmeans.cu b/cpp/test/cluster/kmeans.cu deleted file mode 100644 index 33c17c527d..0000000000 --- a/cpp/test/cluster/kmeans.cu +++ /dev/null @@ -1,363 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include - -#include -#include - -namespace raft { - -template -struct KmeansInputs { - int n_row; - int n_col; - int n_clusters; - T tol; - bool weighted; -}; - -template -void run_cluster_cost(const raft::resources& handle, - raft::device_vector_view minClusterDistance, - rmm::device_uvector& workspace, - raft::device_scalar_view clusterCost) -{ - raft::cluster::kmeans::cluster_cost( - handle, minClusterDistance, workspace, clusterCost, raft::add_op{}); -} - -template -class KmeansTest : public ::testing::TestWithParam> { - protected: - KmeansTest() - : d_labels(0, resource::get_cuda_stream(handle)), - d_labels_ref(0, resource::get_cuda_stream(handle)), - d_centroids(0, resource::get_cuda_stream(handle)), - d_sample_weight(0, resource::get_cuda_stream(handle)) - { - } - - void apiTest() - { - testparams = ::testing::TestWithParam>::GetParam(); - - auto stream = resource::get_cuda_stream(handle); - int n_samples = testparams.n_row; - int n_features = testparams.n_col; - params.n_clusters = testparams.n_clusters; - params.tol = testparams.tol; - params.n_init = 1; - params.rng_state.seed = 1; - params.oversampling_factor = 0; - - raft::random::RngState rng(params.rng_state.seed, params.rng_state.type); - - auto X = raft::make_device_matrix(handle, n_samples, n_features); - auto labels = raft::make_device_vector(handle, n_samples); - - raft::random::make_blobs(X.data_handle(), - labels.data_handle(), - n_samples, - n_features, - params.n_clusters, - stream, - true, - nullptr, - nullptr, - T(1.0), - false, - (T)-10.0f, - (T)10.0f, - (uint64_t)1234); - d_labels.resize(n_samples, stream); - d_labels_ref.resize(n_samples, stream); - d_centroids.resize(params.n_clusters * n_features, stream); - raft::copy(d_labels_ref.data(), labels.data_handle(), n_samples, stream); - rmm::device_uvector d_sample_weight(n_samples, stream); - thrust::fill( - thrust::cuda::par.on(stream), d_sample_weight.data(), d_sample_weight.data() + n_samples, 1); - auto weight_view = - raft::make_device_vector_view(d_sample_weight.data(), n_samples); - - T inertia = 0; - int n_iter = 0; - rmm::device_uvector workspace(0, stream); - rmm::device_uvector L2NormBuf_OR_DistBuf(0, stream); - rmm::device_uvector inRankCp(0, stream); - auto X_view = raft::make_const_mdspan(X.view()); - auto centroids_view = - raft::make_device_matrix_view(d_centroids.data(), params.n_clusters, n_features); - auto miniX = raft::make_device_matrix(handle, n_samples / 4, n_features); - - // Initialize kmeans on a portion of X - raft::cluster::kmeans::shuffle_and_gather( - handle, - X_view, - raft::make_device_matrix_view(miniX.data_handle(), miniX.extent(0), miniX.extent(1)), - miniX.extent(0), - params.rng_state.seed); - - raft::cluster::kmeans::init_plus_plus( - handle, params, raft::make_const_mdspan(miniX.view()), centroids_view, workspace); - - auto minClusterDistance = raft::make_device_vector(handle, n_samples); - auto minClusterAndDistance = - raft::make_device_vector, int>(handle, n_samples); - auto L2NormX = raft::make_device_vector(handle, n_samples); - auto clusterCostBefore = raft::make_device_scalar(handle, 0); - auto clusterCostAfter = raft::make_device_scalar(handle, 0); - - raft::linalg::rowNorm(L2NormX.data_handle(), - X.data_handle(), - X.extent(1), - X.extent(0), - raft::linalg::L2Norm, - true, - stream); - - raft::cluster::kmeans::min_cluster_distance(handle, - X_view, - centroids_view, - minClusterDistance.view(), - L2NormX.view(), - L2NormBuf_OR_DistBuf, - params.metric, - params.batch_samples, - params.batch_centroids, - workspace); - - run_cluster_cost(handle, minClusterDistance.view(), workspace, clusterCostBefore.view()); - - // Run a fit of kmeans - raft::cluster::kmeans::fit_main(handle, - params, - X_view, - weight_view, - centroids_view, - raft::make_host_scalar_view(&inertia), - raft::make_host_scalar_view(&n_iter), - workspace); - - // Check that the cluster cost decreased - raft::cluster::kmeans::min_cluster_distance(handle, - X_view, - centroids_view, - minClusterDistance.view(), - L2NormX.view(), - L2NormBuf_OR_DistBuf, - params.metric, - params.batch_samples, - params.batch_centroids, - workspace); - - run_cluster_cost(handle, minClusterDistance.view(), workspace, clusterCostAfter.view()); - T h_clusterCostBefore = T(0); - T h_clusterCostAfter = T(0); - raft::update_host(&h_clusterCostBefore, clusterCostBefore.data_handle(), 1, stream); - raft::update_host(&h_clusterCostAfter, clusterCostAfter.data_handle(), 1, stream); - ASSERT_TRUE(h_clusterCostAfter < h_clusterCostBefore); - - // Count samples in clusters using 2 methods and compare them - // Fill minClusterAndDistance - raft::cluster::kmeans::min_cluster_and_distance( - handle, - X_view, - raft::make_device_matrix_view( - d_centroids.data(), params.n_clusters, n_features), - minClusterAndDistance.view(), - L2NormX.view(), - L2NormBuf_OR_DistBuf, - params.metric, - params.batch_samples, - params.batch_centroids, - workspace); - raft::cluster::kmeans::KeyValueIndexOp conversion_op; - cub::TransformInputIterator, - raft::KeyValuePair*> - itr(minClusterAndDistance.data_handle(), conversion_op); - - auto sampleCountInCluster = raft::make_device_vector(handle, params.n_clusters); - auto weigthInCluster = raft::make_device_vector(handle, params.n_clusters); - auto newCentroids = raft::make_device_matrix(handle, params.n_clusters, n_features); - raft::cluster::kmeans::update_centroids(handle, - X_view, - weight_view, - raft::make_device_matrix_view( - d_centroids.data(), params.n_clusters, n_features), - itr, - weigthInCluster.view(), - newCentroids.view()); - raft::cluster::kmeans::count_samples_in_cluster(handle, - params, - X_view, - L2NormX.view(), - newCentroids.view(), - workspace, - sampleCountInCluster.view()); - - ASSERT_TRUE(devArrMatch(sampleCountInCluster.data_handle(), - weigthInCluster.data_handle(), - params.n_clusters, - CompareApprox(params.tol))); - } - - void basicTest() - { - testparams = ::testing::TestWithParam>::GetParam(); - - int n_samples = testparams.n_row; - int n_features = testparams.n_col; - params.n_clusters = testparams.n_clusters; - params.tol = testparams.tol; - params.n_init = 5; - params.rng_state.seed = 1; - params.oversampling_factor = 0; - - auto X = raft::make_device_matrix(handle, n_samples, n_features); - auto labels = raft::make_device_vector(handle, n_samples); - auto stream = resource::get_cuda_stream(handle); - - raft::random::make_blobs(X.data_handle(), - labels.data_handle(), - n_samples, - n_features, - params.n_clusters, - stream, - true, - nullptr, - nullptr, - T(1.0), - false, - (T)-10.0f, - (T)10.0f, - (uint64_t)1234); - - d_labels.resize(n_samples, stream); - d_labels_ref.resize(n_samples, stream); - d_centroids.resize(params.n_clusters * n_features, stream); - - std::optional> d_sw = std::nullopt; - auto d_centroids_view = - raft::make_device_matrix_view(d_centroids.data(), params.n_clusters, n_features); - if (testparams.weighted) { - d_sample_weight.resize(n_samples, stream); - d_sw = std::make_optional( - raft::make_device_vector_view(d_sample_weight.data(), n_samples)); - thrust::fill(thrust::cuda::par.on(stream), - d_sample_weight.data(), - d_sample_weight.data() + n_samples, - 1); - } - - raft::copy(d_labels_ref.data(), labels.data_handle(), n_samples, stream); - - T inertia = 0; - int n_iter = 0; - auto X_view = raft::make_const_mdspan(X.view()); - - raft::cluster::kmeans_fit_predict( - handle, - params, - X_view, - d_sw, - d_centroids_view, - raft::make_device_vector_view(d_labels.data(), n_samples), - raft::make_host_scalar_view(&inertia), - raft::make_host_scalar_view(&n_iter)); - - resource::sync_stream(handle, stream); - - score = raft::stats::adjusted_rand_index( - d_labels_ref.data(), d_labels.data(), n_samples, resource::get_cuda_stream(handle)); - - if (score < 1.0) { - std::stringstream ss; - ss << "Expected: " << raft::arr2Str(d_labels_ref.data(), 25, "d_labels_ref", stream); - std::cout << (ss.str().c_str()) << '\n'; - ss.str(std::string()); - ss << "Actual: " << raft::arr2Str(d_labels.data(), 25, "d_labels", stream); - std::cout << (ss.str().c_str()) << '\n'; - std::cout << "Score = " << score << '\n'; - } - } - - void SetUp() override - { - basicTest(); - apiTest(); - } - - protected: - raft::resources handle; - KmeansInputs testparams; - rmm::device_uvector d_labels; - rmm::device_uvector d_labels_ref; - rmm::device_uvector d_centroids; - rmm::device_uvector d_sample_weight; - double score; - raft::cluster::KMeansParams params; -}; - -const std::vector> inputsf2 = {{1000, 32, 5, 0.0001f, true}, - {1000, 32, 5, 0.0001f, false}, - {1000, 100, 20, 0.0001f, true}, - {1000, 100, 20, 0.0001f, false}, - {10000, 32, 10, 0.0001f, true}, - {10000, 32, 10, 0.0001f, false}, - {10000, 100, 50, 0.0001f, true}, - {10000, 100, 50, 0.0001f, false}, - {10000, 500, 100, 0.0001f, true}, - {10000, 500, 100, 0.0001f, false}}; - -const std::vector> inputsd2 = {{1000, 32, 5, 0.0001, true}, - {1000, 32, 5, 0.0001, false}, - {1000, 100, 20, 0.0001, true}, - {1000, 100, 20, 0.0001, false}, - {10000, 32, 10, 0.0001, true}, - {10000, 32, 10, 0.0001, false}, - {10000, 100, 50, 0.0001, true}, - {10000, 100, 50, 0.0001, false}, - {10000, 500, 100, 0.0001, true}, - {10000, 500, 100, 0.0001, false}}; - -typedef KmeansTest KmeansTestF; -TEST_P(KmeansTestF, Result) { ASSERT_TRUE(score == 1.0); } - -typedef KmeansTest KmeansTestD; -TEST_P(KmeansTestD, Result) { ASSERT_TRUE(score == 1.0); } - -INSTANTIATE_TEST_CASE_P(KmeansTests, KmeansTestF, ::testing::ValuesIn(inputsf2)); - -INSTANTIATE_TEST_CASE_P(KmeansTests, KmeansTestD, ::testing::ValuesIn(inputsd2)); - -} // namespace raft diff --git a/cpp/test/cluster/kmeans_balanced.cu b/cpp/test/cluster/kmeans_balanced.cu deleted file mode 100644 index 5009eaf122..0000000000 --- a/cpp/test/cluster/kmeans_balanced.cu +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include - -#include -#include - -/* This test takes advantage of the fact that make_blobs generates balanced clusters. - * It doesn't currently test whether the algorithm can make balanced clusters with an imbalanced - * dataset. - */ - -namespace raft { - -template -struct KmeansBalancedInputs { - IdxT n_rows; - IdxT n_cols; - IdxT n_clusters; - raft::cluster::kmeans_balanced_params kb_params; - MathT tol; -}; - -template -::std::ostream& operator<<(::std::ostream& os, const KmeansBalancedInputs& p) -{ - os << "{ " << p.n_rows << ", " << p.n_cols << ", " << p.n_clusters << ", " << p.kb_params.n_iters - << static_cast(p.kb_params.metric) << '}' << std::endl; - return os; -} - -template -class KmeansBalancedTest : public ::testing::TestWithParam> { - protected: - KmeansBalancedTest() - : stream(resource::get_cuda_stream(handle)), - d_labels(0, stream), - d_labels_ref(0, stream), - d_centroids(0, stream) - { - } - - void basicTest() - { - MappingOpT op{}; - - auto p = ::testing::TestWithParam>::GetParam(); - - auto X = raft::make_device_matrix(handle, p.n_rows, p.n_cols); - auto blob_labels = raft::make_device_vector(handle, p.n_rows); - - MathT* blobs_ptr; - rmm::device_uvector blobs(0, stream); - if constexpr (!std::is_same_v) { - blobs.resize(p.n_rows * p.n_cols, stream); - blobs_ptr = blobs.data(); - } else { - blobs_ptr = X.data_handle(); - } - - raft::random::make_blobs(blobs_ptr, - blob_labels.data_handle(), - p.n_rows, - p.n_cols, - p.n_clusters, - stream, - true, - nullptr, - nullptr, - MathT{0.1}, - true, - MathT{-1}, - MathT{1}, - (uint64_t)1234); - - // Convert blobs dataset to DataT if necessary - if constexpr (!std::is_same_v) { - raft::linalg::unaryOp( - X.data_handle(), blobs.data(), p.n_rows * p.n_cols, op.reverse_op, stream); - } - - d_labels.resize(p.n_rows, stream); - d_labels_ref.resize(p.n_rows, stream); - d_centroids.resize(p.n_clusters * p.n_cols, stream); - - raft::linalg::unaryOp( - d_labels_ref.data(), blob_labels.data_handle(), p.n_rows, raft::cast_op(), stream); - - auto X_view = - raft::make_device_matrix_view(X.data_handle(), X.extent(0), X.extent(1)); - auto d_centroids_view = - raft::make_device_matrix_view(d_centroids.data(), p.n_clusters, p.n_cols); - auto d_labels_view = raft::make_device_vector_view(d_labels.data(), p.n_rows); - - raft::cluster::kmeans_balanced::fit_predict( - handle, p.kb_params, X_view, d_centroids_view, d_labels_view, op); - - resource::sync_stream(handle, stream); - - score = raft::stats::adjusted_rand_index( - d_labels_ref.data(), d_labels.data(), p.n_rows, resource::get_cuda_stream(handle)); - - if (score < 1.0) { - std::stringstream ss; - ss << "Expected: " << raft::arr2Str(d_labels_ref.data(), 25, "d_labels_ref", stream); - std::cout << (ss.str().c_str()) << '\n'; - ss.str(std::string()); - ss << "Actual: " << raft::arr2Str(d_labels.data(), 25, "d_labels", stream); - std::cout << (ss.str().c_str()) << '\n'; - std::cout << "Score = " << score << '\n'; - } - } - - void SetUp() override { basicTest(); } - - protected: - raft::handle_t handle; - cudaStream_t stream; - rmm::device_uvector d_labels; - rmm::device_uvector d_labels_ref; - rmm::device_uvector d_centroids; - double score; -}; - -template -std::vector> get_kmeans_balanced_inputs() -{ - std::vector> out; - KmeansBalancedInputs p; - p.kb_params.n_iters = 20; - p.kb_params.metric = raft::distance::DistanceType::L2Expanded; - p.tol = MathT{0.0001}; - std::vector> row_cols_k = {{1000, 32, 5}, - {1000, 100, 20}, - {10000, 32, 10}, - {10000, 100, 50}, - {10000, 500, 100}, - {1000000, 128, 10}}; - for (auto& rck : row_cols_k) { - p.n_rows = static_cast(std::get<0>(rck)); - p.n_cols = static_cast(std::get<1>(rck)); - p.n_clusters = static_cast(std::get<2>(rck)); - out.push_back(p); - } - return out; -} - -const auto inputsf_i32 = get_kmeans_balanced_inputs(); -const auto inputsd_i32 = get_kmeans_balanced_inputs(); -const auto inputsf_i64 = get_kmeans_balanced_inputs(); -const auto inputsd_i64 = get_kmeans_balanced_inputs(); - -#define KB_TEST(test_type, test_name, test_inputs) \ - typedef RAFT_DEPAREN(test_type) test_name; \ - TEST_P(test_name, Result) { ASSERT_TRUE(score == 1.0); } \ - INSTANTIATE_TEST_CASE_P(KmeansBalancedTests, test_name, ::testing::ValuesIn(test_inputs)) - -/* - * First set of tests: no conversion - */ - -KB_TEST((KmeansBalancedTest), - KmeansBalancedTestFFU32I32, - inputsf_i32); -KB_TEST((KmeansBalancedTest), - KmeansBalancedTestDDU32I32, - inputsd_i32); -KB_TEST((KmeansBalancedTest), - KmeansBalancedTestFFU32I64, - inputsf_i64); -KB_TEST((KmeansBalancedTest), - KmeansBalancedTestDDU32I64, - inputsd_i64); -KB_TEST((KmeansBalancedTest), - KmeansBalancedTestFFI32I32, - inputsf_i32); -KB_TEST((KmeansBalancedTest), - KmeansBalancedTestFFI32I64, - inputsf_i64); -KB_TEST((KmeansBalancedTest), - KmeansBalancedTestFFI64I32, - inputsf_i32); -KB_TEST((KmeansBalancedTest), - KmeansBalancedTestFFI64I64, - inputsf_i64); - -/* - * Second set of tests: integer dataset with conversion - */ - -template -struct i2f_scaler { - // Note: with a scaling factor of 42, and generating blobs with centers between -1 and 1 with a - // standard deviation of 0.1, it's statistically very unlikely that we'd overflow - const raft::compose_op, raft::cast_op> op{ - raft::div_const_op{42}, raft::cast_op{}}; - const raft::compose_op, raft::mul_const_op> reverse_op{ - raft::cast_op{}, raft::mul_const_op{42}}; - - RAFT_INLINE_FUNCTION auto operator()(const DataT& x) const { return op(x); }; -}; - -KB_TEST((KmeansBalancedTest>), - KmeansBalancedTestFI8U32I32, - inputsf_i32); -KB_TEST((KmeansBalancedTest>), - KmeansBalancedTestDI8U32I32, - inputsd_i32); - -} // namespace raft diff --git a/cpp/test/cluster/kmeans_find_k.cu b/cpp/test/cluster/kmeans_find_k.cu deleted file mode 100644 index 8e05ad3695..0000000000 --- a/cpp/test/cluster/kmeans_find_k.cu +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.h" - -#include -#include -#include -#include -#include -#include - -#include - -#include -#include - -namespace raft { - -template -struct KmeansFindKInputs { - int n_row; - int n_col; - int n_clusters; - T tol; - bool weighted; -}; - -template -class KmeansFindKTest : public ::testing::TestWithParam> { - protected: - KmeansFindKTest() - : stream(resource::get_cuda_stream(handle)), best_k(raft::make_host_scalar(0)) - { - } - - void basicTest() - { - testparams = ::testing::TestWithParam>::GetParam(); - - int n_samples = testparams.n_row; - int n_features = testparams.n_col; - int n_clusters = testparams.n_clusters; - - auto X = raft::make_device_matrix(handle, n_samples, n_features); - auto labels = raft::make_device_vector(handle, n_samples); - - raft::random::make_blobs(X.data_handle(), - labels.data_handle(), - n_samples, - n_features, - n_clusters, - stream, - true, - nullptr, - nullptr, - T(.001), - false, - (T)-10.0f, - (T)10.0f, - (uint64_t)1234); - - auto inertia = raft::make_host_scalar(0); - auto n_iter = raft::make_host_scalar(0); - - auto X_view = - raft::make_device_matrix_view(X.data_handle(), X.extent(0), X.extent(1)); - - raft::cluster::kmeans::find_k( - handle, X_view, best_k.view(), inertia.view(), n_iter.view(), n_clusters); - - resource::sync_stream(handle, stream); - } - - void SetUp() override { basicTest(); } - - protected: - raft::resources handle; - cudaStream_t stream; - KmeansFindKInputs testparams; - raft::host_scalar best_k; -}; - -const std::vector> inputsf2 = {{1000, 32, 8, 0.001f, true}, - {1000, 32, 8, 0.001f, false}, - {1000, 100, 20, 0.001f, true}, - {1000, 100, 20, 0.001f, false}, - {10000, 32, 10, 0.001f, true}, - {10000, 32, 10, 0.001f, false}, - {10000, 100, 50, 0.001f, true}, - {10000, 100, 50, 0.001f, false}, - {10000, 500, 100, 0.001f, true}, - {10000, 500, 100, 0.001f, false}}; - -const std::vector> inputsd2 = {{1000, 32, 5, 0.0001, true}, - {1000, 32, 5, 0.0001, false}, - {1000, 100, 20, 0.0001, true}, - {1000, 100, 20, 0.0001, false}, - {10000, 32, 10, 0.0001, true}, - {10000, 32, 10, 0.0001, false}, - {10000, 100, 50, 0.0001, true}, - {10000, 100, 50, 0.0001, false}, - {10000, 500, 100, 0.0001, true}, - {10000, 500, 100, 0.0001, false}}; - -typedef KmeansFindKTest KmeansFindKTestF; -TEST_P(KmeansFindKTestF, Result) -{ - if (best_k.view()[0] != testparams.n_clusters) { - std::cout << best_k.view()[0] << " " << testparams.n_clusters << std::endl; - } - ASSERT_TRUE(best_k.view()[0] == testparams.n_clusters); -} - -typedef KmeansFindKTest KmeansFindKTestD; -TEST_P(KmeansFindKTestD, Result) -{ - if (best_k.view()[0] != testparams.n_clusters) { - std::cout << best_k.view()[0] << " " << testparams.n_clusters << std::endl; - } - - ASSERT_TRUE(best_k.view()[0] == testparams.n_clusters); -} - -INSTANTIATE_TEST_CASE_P(KmeansFindKTests, KmeansFindKTestF, ::testing::ValuesIn(inputsf2)); - -INSTANTIATE_TEST_CASE_P(KmeansFindKTests, KmeansFindKTestD, ::testing::ValuesIn(inputsd2)); - -} // namespace raft diff --git a/cpp/test/cluster/linkage.cu b/cpp/test/cluster/linkage.cu deleted file mode 100644 index ba7ed4254e..0000000000 --- a/cpp/test/cluster/linkage.cu +++ /dev/null @@ -1,674 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// XXX: We allow the instantiation of masked_l2_nn here: -// raft::linkage::FixConnectivitiesRedOp red_op(params.n_row); -// raft::linkage::cross_component_nn( -// handle, out_edges, data.data(), colors.data(), params.n_row, params.n_col, red_op); -// -// TODO: consider adding this to libraft.so or creating an instance in a -// separate translation unit for this test. -#undef RAFT_EXPLICIT_INSTANTIATE_ONLY - -#include "../test_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include - -namespace raft { - -using namespace std; - -template -struct LinkageInputs { - IdxT n_row; - IdxT n_col; - - std::vector data; - - std::vector expected_labels; - - int n_clusters; - - bool use_knn; - - int c; -}; - -/** - * @brief kernel to calculate the values of a and b - * @param firstClusterArray: the array of classes of type T - * @param secondClusterArray: the array of classes of type T - * @param size: the size of the data points - * @param a: number of pairs of points that both the clusters have classified the same - * @param b: number of pairs of points that both the clusters have classified differently - */ -template -RAFT_KERNEL computeTheNumerator( - const T* firstClusterArray, const T* secondClusterArray, uint64_t size, uint64_t* a, uint64_t* b) -{ - // calculating the indices of pairs of datapoints compared by the current thread - uint64_t j = threadIdx.x + blockIdx.x * blockDim.x; - uint64_t i = threadIdx.y + blockIdx.y * blockDim.y; - - // thread-local variables to count a and b - uint64_t myA = 0, myB = 0; - - if (i < size && j < size && j < i) { - // checking if the pair have been classified the same by both the clusters - if (firstClusterArray[i] == firstClusterArray[j] && - secondClusterArray[i] == secondClusterArray[j]) { - ++myA; - } - - // checking if the pair have been classified differently by both the clusters - else if (firstClusterArray[i] != firstClusterArray[j] && - secondClusterArray[i] != secondClusterArray[j]) { - ++myB; - } - } - - // specialize blockReduce for a 2D block of 1024 threads of type uint64_t - typedef cub::BlockReduce - BlockReduce; - - // Allocate shared memory for blockReduce - __shared__ typename BlockReduce::TempStorage temp_storage; - - // summing up thread-local counts specific to a block - myA = BlockReduce(temp_storage).Sum(myA); - __syncthreads(); - myB = BlockReduce(temp_storage).Sum(myB); - __syncthreads(); - - // executed once per block - if (threadIdx.x == 0 && threadIdx.y == 0) { - raft::myAtomicAdd((unsigned long long int*)a, myA); - raft::myAtomicAdd((unsigned long long int*)b, myB); - } -} - -/** - * @brief Function to calculate RandIndex - * more info on rand index - * @param firstClusterArray: the array of classes of type T - * @param secondClusterArray: the array of classes of type T - * @param size: the size of the data points of type uint64_t - * @param stream: the cudaStream object - */ -template -double compute_rand_index(T* firstClusterArray, - T* secondClusterArray, - uint64_t size, - cudaStream_t stream) -{ - // rand index for size less than 2 is not defined - ASSERT(size >= 2, "Rand Index for size less than 2 not defined!"); - - // allocating and initializing memory for a and b in the GPU - rmm::device_uvector arr_buf(2, stream); - RAFT_CUDA_TRY(cudaMemsetAsync(arr_buf.data(), 0, 2 * sizeof(uint64_t), stream)); - - // kernel configuration - static const int BLOCK_DIM_Y = 16, BLOCK_DIM_X = 16; - dim3 numThreadsPerBlock(BLOCK_DIM_X, BLOCK_DIM_Y); - dim3 numBlocks(raft::ceildiv(size, numThreadsPerBlock.x), - raft::ceildiv(size, numThreadsPerBlock.y)); - - // calling the kernel - computeTheNumerator<<>>( - firstClusterArray, secondClusterArray, size, arr_buf.data(), arr_buf.data() + 1); - - // synchronizing and updating the calculated values of a and b from device to host - uint64_t ab_host[2] = {0}; - raft::update_host(ab_host, arr_buf.data(), 2, stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - - // error handling - RAFT_CUDA_TRY(cudaGetLastError()); - - // denominator - uint64_t nChooseTwo = size * (size - 1) / 2; - - // calculating the rand_index - return (double)(((double)(ab_host[0] + ab_host[1])) / (double)nChooseTwo); -} - -template -::std::ostream& operator<<(::std::ostream& os, const LinkageInputs& dims) -{ - return os; -} - -template -class LinkageTest : public ::testing::TestWithParam> { - public: - LinkageTest() - : params(::testing::TestWithParam>::GetParam()), - labels(0, resource::get_cuda_stream(handle)), - labels_ref(0, resource::get_cuda_stream(handle)) - { - } - - protected: - void basicTest() - { - auto stream = resource::get_cuda_stream(handle); - - labels.resize(params.n_row, stream); - labels_ref.resize(params.n_row, stream); - rmm::device_uvector data(params.n_row * params.n_col, stream); - - raft::copy(data.data(), params.data.data(), data.size(), stream); - raft::copy(labels_ref.data(), params.expected_labels.data(), params.n_row, stream); - - rmm::device_uvector out_children(params.n_row * 2, stream); - - auto data_view = raft::make_device_matrix_view( - data.data(), params.n_row, params.n_col); - auto dendrogram_view = - raft::make_device_matrix_view(out_children.data(), params.n_row, 2); - auto labels_view = raft::make_device_vector_view(labels.data(), params.n_row); - - if (params.use_knn) { - raft::cluster::hierarchy:: - single_linkage( - handle, - data_view, - dendrogram_view, - labels_view, - raft::distance::DistanceType::L2SqrtExpanded, - params.n_clusters, - std::make_optional(params.c)); - - } else { - raft::cluster::hierarchy:: - single_linkage( - handle, - data_view, - dendrogram_view, - labels_view, - raft::distance::DistanceType::L2SqrtExpanded, - params.n_clusters, - std::make_optional(params.c)); - } - - resource::sync_stream(handle, stream); - - score = compute_rand_index(labels.data(), labels_ref.data(), params.n_row, stream); - } - - void SetUp() override { basicTest(); } - - protected: - raft::resources handle; - - LinkageInputs params; - rmm::device_uvector labels, labels_ref; - double score; -}; - -const std::vector> linkage_inputsf2 = { - // Test n_clusters == n_points - {10, - 5, - {0.21390334, 0.50261639, 0.91036676, 0.59166485, 0.71162682, 0.10248392, 0.77782677, 0.43772379, - 0.4035871, 0.3282796, 0.47544681, 0.59862974, 0.12319357, 0.06239463, 0.28200272, 0.1345717, - 0.50498218, 0.5113505, 0.16233086, 0.62165332, 0.42281548, 0.933117, 0.41386077, 0.23264562, - 0.73325968, 0.37537541, 0.70719873, 0.14522645, 0.73279625, 0.9126674, 0.84854131, 0.28890216, - 0.85267903, 0.74703138, 0.83842071, 0.34942792, 0.27864171, 0.70911132, 0.21338564, 0.32035554, - 0.73788331, 0.46926692, 0.57570162, 0.42559178, 0.87120209, 0.22734951, 0.01847905, 0.75549396, - 0.76166195, 0.66613745}, - {9, 8, 7, 6, 5, 4, 3, 2, 1, 0}, - 10, - true, - -1}, - // // Test outlier points - {9, - 2, - {-1, -50, 3, 4, 5000, 10000, 1, 3, 4, 5, 0.000005, 0.00002, 2000000, 500000, 10, 50, 30, 5}, - {6, 0, 5, 0, 0, 4, 3, 2, 1}, - 7, - true, - -1}, - - // Test n_clusters == (n_points / 2) - {10, - 5, - {0.21390334, 0.50261639, 0.91036676, 0.59166485, 0.71162682, 0.10248392, 0.77782677, 0.43772379, - 0.4035871, 0.3282796, 0.47544681, 0.59862974, 0.12319357, 0.06239463, 0.28200272, 0.1345717, - 0.50498218, 0.5113505, 0.16233086, 0.62165332, 0.42281548, 0.933117, 0.41386077, 0.23264562, - 0.73325968, 0.37537541, 0.70719873, 0.14522645, 0.73279625, 0.9126674, 0.84854131, 0.28890216, - 0.85267903, 0.74703138, 0.83842071, 0.34942792, 0.27864171, 0.70911132, 0.21338564, 0.32035554, - 0.73788331, 0.46926692, 0.57570162, 0.42559178, 0.87120209, 0.22734951, 0.01847905, 0.75549396, - 0.76166195, 0.66613745}, - {1, 0, 4, 0, 0, 3, 2, 0, 2, 1}, - 5, - true, - -1}, - - // Test n_points == 100 - {100, - 10, - {6.26168372e-01, 9.30437651e-01, 6.02450208e-01, 2.73025296e-01, 9.53050619e-01, 3.32164396e-01, - 6.88942598e-01, 5.79163537e-01, 6.70341547e-01, 2.70140602e-02, 9.30429671e-01, 7.17721157e-01, - 9.89948537e-01, 7.75253347e-01, 1.34491522e-02, 2.48522428e-02, 3.51413378e-01, 7.64405834e-01, - 7.86373507e-01, 7.18748577e-01, 8.66998621e-01, 6.80316582e-01, 2.51288712e-01, 4.91078420e-01, - 3.76246281e-01, 4.86828710e-01, 5.67464772e-01, 5.30734742e-01, 8.99478296e-01, 7.66699088e-01, - 9.49339111e-01, 3.55248484e-01, 9.06046929e-01, 4.48407772e-01, 6.96395305e-01, 2.44277335e-01, - 7.74840000e-01, 5.21046603e-01, 4.66423971e-02, 5.12019638e-02, 8.95019614e-01, 5.28956953e-01, - 4.31536306e-01, 5.83857744e-01, 4.41787364e-01, 4.68656523e-01, 5.73971433e-01, 6.79989654e-01, - 3.19650588e-01, 6.12579596e-01, 6.49126442e-02, 8.39131142e-01, 2.85252117e-01, 5.84848929e-01, - 9.46507115e-01, 8.58440748e-01, 3.61528940e-01, 2.44215959e-01, 3.80101125e-01, 4.57128957e-02, - 8.82216988e-01, 8.31498633e-01, 7.23474381e-01, 7.75788607e-01, 1.40864146e-01, 6.62092382e-01, - 5.13985168e-01, 3.00686418e-01, 8.70109949e-01, 2.43187753e-01, 2.89391938e-01, 2.84214238e-01, - 8.70985521e-01, 8.77491176e-01, 6.72537226e-01, 3.30929686e-01, 1.85934324e-01, 9.16222614e-01, - 6.18239142e-01, 2.64768597e-01, 5.76145451e-01, 8.62961369e-01, 6.84757925e-01, 7.60549082e-01, - 1.27645356e-01, 4.51004673e-01, 3.92292980e-01, 4.63170803e-01, 4.35449330e-02, 2.17583404e-01, - 5.71832605e-02, 2.06763039e-01, 3.70116249e-01, 2.09750028e-01, 6.17283019e-01, 8.62549231e-01, - 9.84156240e-02, 2.66249156e-01, 3.87635103e-01, 2.85591012e-02, 4.24826068e-01, 4.45795088e-01, - 6.86227676e-01, 1.08848960e-01, 5.96731841e-02, 3.71770228e-01, 1.91548833e-01, 6.95136078e-01, - 9.00700636e-01, 8.76363105e-01, 2.67334632e-01, 1.80619709e-01, 7.94060419e-01, 1.42854171e-02, - 1.09372387e-01, 8.74028108e-01, 6.46403232e-01, 4.86588834e-01, 5.93446175e-02, 6.11886291e-01, - 8.83865057e-01, 3.15879821e-01, 2.27043992e-01, 9.76764951e-01, 6.15620336e-01, 9.76199360e-01, - 2.40548962e-01, 3.21795663e-01, 8.75087904e-02, 8.11234663e-01, 6.96070480e-01, 8.12062321e-01, - 1.21958818e-01, 3.44348628e-02, 8.72630414e-01, 3.06162776e-01, 1.76043529e-02, 9.45894971e-01, - 5.33896401e-01, 6.21642973e-01, 4.93062535e-01, 4.48984262e-01, 2.24560379e-01, 4.24052195e-02, - 4.43447610e-01, 8.95646149e-01, 6.05220676e-01, 1.81840491e-01, 9.70831206e-01, 2.12563586e-02, - 6.92582693e-01, 7.55946922e-01, 7.95086143e-01, 6.05328941e-01, 3.99350764e-01, 4.32846636e-01, - 9.81114529e-01, 4.98266428e-01, 6.37127930e-03, 1.59085889e-01, 6.34682067e-05, 5.59429440e-01, - 7.38827633e-01, 8.93214770e-01, 2.16494306e-01, 9.35430573e-02, 4.75665868e-02, 7.80503518e-01, - 7.86240041e-01, 7.06854594e-01, 2.13725879e-02, 7.68246091e-01, 4.50234808e-01, 5.21231104e-01, - 5.01989826e-03, 4.22081572e-02, 1.65337732e-01, 8.54134740e-01, 4.99430262e-01, 8.94525601e-01, - 1.14028379e-01, 3.69739861e-01, 1.32955599e-01, 2.65563824e-01, 2.52811151e-01, 1.44792843e-01, - 6.88449594e-01, 4.44921417e-01, 8.23296587e-01, 1.93266317e-01, 1.19033309e-01, 1.36368966e-01, - 3.42600285e-01, 5.64505195e-01, 5.57594559e-01, 7.44257892e-01, 8.38231569e-02, 4.11548847e-01, - 3.21010077e-01, 8.55081359e-01, 4.30105779e-01, 1.16229135e-01, 9.87731964e-02, 3.14712335e-01, - 4.50880592e-01, 2.72289598e-01, 6.31615256e-01, 8.97432958e-01, 4.44764250e-01, 8.03776440e-01, - 2.68767748e-02, 2.43374608e-01, 4.02141103e-01, 4.98881209e-01, 5.33173003e-01, 8.82890436e-01, - 7.16149148e-01, 4.19664401e-01, 2.29335357e-01, 2.88637806e-01, 3.44696803e-01, 6.78171906e-01, - 5.69849716e-01, 5.86454477e-01, 3.54474989e-01, 9.03876540e-01, 6.45980000e-01, 6.34887593e-01, - 7.88039746e-02, 2.04814126e-01, 7.82251754e-01, 2.43147074e-01, 7.50951808e-01, 1.72799092e-02, - 2.95349590e-01, 6.57991826e-01, 8.81214312e-01, 5.73970708e-01, 2.77610881e-01, 1.82155097e-01, - 7.69797417e-02, 6.44792402e-01, 9.46950998e-01, 7.73064845e-01, 6.04733624e-01, 5.80094567e-01, - 1.67498426e-01, 2.66514296e-01, 6.50140368e-01, 1.91170299e-01, 2.08752199e-01, 3.01664091e-01, - 9.85033484e-01, 2.92909152e-01, 8.65816607e-01, 1.85222119e-01, 2.28814559e-01, 1.34286382e-02, - 2.89234322e-01, 8.18668708e-01, 4.71706924e-01, 9.23199803e-01, 2.80879188e-01, 1.47319284e-01, - 4.13915748e-01, 9.31274932e-02, 6.66322195e-01, 9.66953974e-01, 3.19405786e-01, 6.69486551e-01, - 5.03096313e-02, 6.95225201e-01, 5.78469859e-01, 6.29481655e-01, 1.39252534e-01, 1.22564968e-01, - 6.80663678e-01, 6.34607157e-01, 6.42765834e-01, 1.57127410e-02, 2.92132086e-01, 5.24423878e-01, - 4.68676824e-01, 2.86003928e-01, 7.18608322e-01, 8.95617933e-01, 5.48844309e-01, 1.74517278e-01, - 5.24379196e-01, 2.13526524e-01, 5.88375435e-01, 9.88560185e-01, 4.17435771e-01, 6.14438688e-01, - 9.53760881e-01, 5.27151288e-01, 7.03017278e-01, 3.44448559e-01, 4.47059676e-01, 2.83414901e-01, - 1.98979011e-01, 4.24917361e-01, 5.73172761e-01, 2.32398853e-02, 1.65887230e-01, 4.05552785e-01, - 9.29665524e-01, 2.26135696e-01, 9.20563384e-01, 7.65259963e-01, 4.54820075e-01, 8.97710267e-01, - 3.78559302e-03, 9.15219382e-01, 3.55705698e-01, 6.94905124e-01, 8.58540202e-01, 3.89790666e-01, - 2.49478206e-01, 7.93679304e-01, 4.75830027e-01, 4.40425353e-01, 3.70579459e-01, 1.40578049e-01, - 1.70386675e-01, 7.04056121e-01, 4.85963102e-01, 9.68450060e-01, 6.77178001e-01, 2.65934654e-01, - 2.58915007e-01, 6.70052890e-01, 2.61945109e-01, 8.46207759e-01, 1.01928951e-01, 2.85611334e-01, - 2.45776933e-01, 2.66658783e-01, 3.71724077e-01, 4.34319025e-01, 4.24407347e-01, 7.15417683e-01, - 8.07997684e-01, 1.64296275e-01, 6.01638065e-01, 8.60606804e-02, 2.68719187e-01, 5.11764101e-01, - 9.75844338e-01, 7.81226782e-01, 2.20925515e-01, 7.18135040e-01, 9.82395577e-01, 8.39160243e-01, - 9.08058083e-01, 6.88010677e-01, 8.14271847e-01, 5.12460821e-01, 1.17311345e-01, 5.96075228e-01, - 9.17455497e-01, 2.12052706e-01, 7.04074603e-01, 8.72872565e-02, 8.76047818e-01, 6.96235046e-01, - 8.54801557e-01, 2.49729159e-01, 9.76594604e-01, 2.87386363e-01, 2.36461559e-02, 9.94075254e-01, - 4.25193986e-01, 7.61869994e-01, 5.13334255e-01, 6.44711165e-02, 8.92156689e-01, 3.55235167e-01, - 1.08154647e-01, 8.78446825e-01, 2.43833016e-01, 9.23071293e-01, 2.72724115e-01, 9.46631338e-01, - 3.74510294e-01, 4.08451278e-02, 9.78392777e-01, 3.65079221e-01, 6.37199516e-01, 5.51144906e-01, - 5.25978080e-01, 1.42803678e-01, 4.05451674e-01, 7.79788219e-01, 6.26009784e-01, 3.35249497e-01, - 1.43159543e-02, 1.80363779e-01, 5.05096904e-01, 2.82619947e-01, 5.83561392e-01, 3.10951324e-01, - 8.73223968e-01, 4.38545619e-01, 4.81348800e-01, 6.68497085e-01, 3.79345401e-01, 9.58832501e-01, - 1.89869550e-01, 2.34083070e-01, 2.94066207e-01, 5.74892667e-02, 6.92106828e-02, 9.61127686e-02, - 6.72650672e-02, 8.47345378e-01, 2.80916761e-01, 7.32177357e-03, 9.80785961e-01, 5.73192225e-02, - 8.48781331e-01, 8.83225408e-01, 7.34398275e-01, 7.70381941e-01, 6.20778343e-01, 8.96822048e-01, - 5.40732486e-01, 3.69704071e-01, 5.77305837e-01, 2.08221827e-01, 7.34275341e-01, 1.06110900e-01, - 3.49496706e-01, 8.34948910e-01, 1.56403291e-02, 6.78576376e-01, 8.96141268e-01, 5.94835119e-01, - 1.43943153e-01, 3.49618530e-01, 2.10440392e-01, 3.46585620e-01, 1.05153093e-01, 3.45446174e-01, - 2.72177079e-01, 7.07946300e-01, 4.33717726e-02, 3.31232203e-01, 3.91874320e-01, 4.76338141e-01, - 6.22777789e-01, 2.95989228e-02, 4.32855769e-01, 7.61049310e-01, 3.63279149e-01, 9.47210350e-01, - 6.43721247e-01, 6.58025802e-01, 1.05247633e-02, 5.29974442e-01, 7.30675767e-01, 4.30041079e-01, - 6.62634841e-01, 8.25936616e-01, 9.91253704e-01, 6.79399281e-01, 5.44177006e-01, 7.52876048e-01, - 3.32139049e-01, 7.98732398e-01, 7.38865223e-01, 9.16055132e-01, 6.11736493e-01, 9.63672879e-01, - 1.83778839e-01, 7.27558919e-02, 5.91602822e-01, 3.25235484e-01, 2.34741217e-01, 9.52346277e-01, - 9.18556407e-01, 9.35373324e-01, 6.89209070e-01, 2.56049054e-01, 6.17975395e-01, 7.82285691e-01, - 9.84983432e-01, 6.62322741e-01, 2.04144457e-01, 3.98446577e-01, 1.38918297e-01, 3.05919921e-01, - 3.14043787e-01, 5.91072666e-01, 7.44703771e-01, 8.92272567e-01, 9.78017873e-01, 9.01203161e-01, - 1.41526372e-01, 4.14878484e-01, 6.80683651e-01, 5.01733152e-02, 8.14635389e-01, 2.27926375e-01, - 9.03269815e-01, 8.68443745e-01, 9.86939190e-01, 7.40779486e-01, 2.61005311e-01, 3.19276232e-01, - 9.69509248e-01, 1.11908818e-01, 4.49198556e-01, 1.27056715e-01, 3.84064823e-01, 5.14591811e-01, - 2.10747488e-01, 9.53884090e-01, 8.43167950e-01, 4.51187972e-01, 3.75331782e-01, 6.23566461e-01, - 3.55290379e-01, 2.95705968e-01, 1.69622690e-01, 1.42981830e-01, 2.72180991e-01, 9.46468040e-01, - 3.70932500e-01, 9.94292830e-01, 4.62587505e-01, 7.14817405e-01, 2.45370540e-02, 3.00906377e-01, - 5.75768304e-01, 9.71448393e-01, 6.95574827e-02, 3.93693854e-01, 5.29306116e-01, 5.04694554e-01, - 6.73797120e-02, 6.76596969e-01, 5.50948898e-01, 3.24909641e-01, 7.70337719e-01, 6.51842631e-03, - 3.03264879e-01, 7.61037886e-03, 2.72289601e-01, 1.50502041e-01, 6.71103888e-02, 7.41503703e-01, - 1.92088941e-01, 2.19043977e-01, 9.09320161e-01, 2.37993569e-01, 6.18107973e-02, 8.31447852e-01, - 2.23355609e-01, 1.84789435e-01, 4.16104518e-01, 4.21573859e-01, 8.72446305e-02, 2.97294197e-01, - 4.50328256e-01, 8.72199917e-01, 2.51279916e-01, 4.86219272e-01, 7.57071329e-01, 4.85655942e-01, - 1.06187277e-01, 4.92341327e-01, 1.46017513e-01, 5.25421017e-01, 4.22637906e-01, 2.24685018e-01, - 8.72648431e-01, 5.54051490e-01, 1.80745062e-01, 2.12756336e-01, 5.20883169e-01, 7.60363654e-01, - 8.30254678e-01, 5.00003328e-01, 4.69017439e-01, 6.38105527e-01, 3.50638261e-02, 5.22217353e-02, - 9.06516882e-02, 8.52975842e-01, 1.19985883e-01, 3.74926753e-01, 6.50302066e-01, 1.98875727e-01, - 6.28362507e-02, 4.32693501e-01, 3.10500685e-01, 6.20732833e-01, 4.58503272e-01, 3.20790034e-01, - 7.91284868e-01, 7.93054570e-01, 2.93406765e-01, 8.95399023e-01, 1.06441034e-01, 7.53085241e-02, - 8.67523104e-01, 1.47963482e-01, 1.25584706e-01, 3.81545040e-02, 6.34338619e-01, 1.76368938e-02, - 5.75553531e-02, 5.31607516e-01, 2.63869588e-01, 9.41945823e-01, 9.24028838e-02, 5.21496463e-01, - 7.74866558e-01, 5.65210610e-01, 7.28015327e-02, 6.51963790e-01, 8.94727453e-01, 4.49571590e-01, - 1.29932405e-01, 8.64026259e-01, 9.92599934e-01, 7.43721560e-01, 8.87300215e-01, 1.06369925e-01, - 8.11335531e-01, 7.87734900e-01, 9.87344678e-01, 5.32502820e-01, 4.42612382e-01, 9.64041183e-01, - 1.66085871e-01, 1.12937664e-01, 5.24423470e-01, 6.54689333e-01, 4.59119726e-01, 5.22774091e-01, - 3.08722276e-02, 6.26979315e-01, 4.49754105e-01, 8.07495757e-01, 2.34199499e-01, 1.67765675e-01, - 9.22168418e-01, 3.73210378e-01, 8.04432575e-01, 5.61890354e-01, 4.47025593e-01, 6.43155678e-01, - 2.40407640e-01, 5.91631279e-01, 1.59369206e-01, 7.75799090e-01, 8.32067212e-01, 5.59791576e-02, - 6.39105224e-01, 4.85274738e-01, 2.12630838e-01, 2.81431312e-02, 7.16205363e-01, 6.83885011e-01, - 5.23869697e-01, 9.99418314e-01, 8.35331599e-01, 4.69877463e-02, 6.74712562e-01, 7.99273684e-01, - 2.77001890e-02, 5.75809742e-01, 2.78513031e-01, 8.36209905e-01, 7.25472379e-01, 4.87173943e-01, - 7.88311357e-01, 9.64676177e-01, 1.75752651e-01, 4.98112580e-01, 8.08850418e-02, 6.40981131e-01, - 4.06647450e-01, 8.46539387e-01, 2.12620694e-01, 9.11012851e-01, 8.25041445e-01, 8.90065575e-01, - 9.63626055e-01, 5.96689242e-01, 1.63372670e-01, 4.51640148e-01, 3.43026542e-01, 5.80658851e-01, - 2.82327625e-01, 4.75535418e-01, 6.27760926e-01, 8.46314115e-01, 9.61961932e-01, 3.19806094e-01, - 5.05508062e-01, 5.28102944e-01, 6.13045057e-01, 7.44714938e-01, 1.50586073e-01, 7.91878033e-01, - 4.89839179e-01, 3.10496849e-01, 8.82309038e-01, 2.86922314e-01, 4.84687559e-01, 5.20838630e-01, - 4.62955493e-01, 2.38185305e-01, 5.47259907e-02, 7.10916137e-01, 7.31887202e-01, 6.25602317e-01, - 8.77741168e-01, 4.19881322e-01, 4.81222328e-01, 1.28224501e-01, 2.46034010e-01, 3.34971854e-01, - 7.37216484e-01, 5.62134821e-02, 7.14089724e-01, 9.85549393e-01, 4.66295827e-01, 3.08722434e-03, - 4.70237690e-01, 2.66524167e-01, 7.93875484e-01, 4.54795911e-02, 8.09702944e-01, 1.47709735e-02, - 1.70082405e-01, 6.35905179e-01, 3.75379109e-01, 4.30315011e-01, 3.15788760e-01, 5.58065230e-01, - 2.24643800e-01, 2.42142981e-01, 6.57283636e-01, 3.34921891e-01, 1.26588975e-01, 7.68064155e-01, - 9.43856291e-01, 4.47518596e-01, 5.44453573e-01, 9.95764932e-01, 7.16444391e-01, 8.51019765e-01, - 1.01179183e-01, 4.45473958e-01, 4.60327322e-01, 4.96895844e-02, 4.72907738e-01, 5.58987444e-01, - 3.41027487e-01, 1.56175026e-01, 7.58283148e-01, 6.83600909e-01, 2.14623396e-01, 3.27348880e-01, - 3.92517893e-01, 6.70418431e-01, 5.16440832e-01, 8.63140348e-01, 5.73277464e-01, 3.46608058e-01, - 7.39396341e-01, 7.20852434e-01, 2.35653246e-02, 3.89935659e-01, 7.53783745e-01, 6.34563528e-01, - 8.79339335e-01, 7.41599159e-02, 5.62433904e-01, 6.15553852e-01, 4.56956324e-01, 5.20047447e-01, - 5.26845015e-02, 5.58471266e-01, 1.63632233e-01, 5.38936665e-02, 6.49593683e-01, 2.56838748e-01, - 8.99035326e-01, 7.20847756e-01, 5.68954684e-01, 7.43684755e-01, 5.70924238e-01, 3.82318724e-01, - 4.89328290e-01, 5.62208561e-01, 4.97540804e-02, 4.18011085e-01, 6.88041565e-01, 2.16234653e-01, - 7.89548214e-01, 8.46136387e-01, 8.46816189e-01, 1.73842353e-01, 6.11627842e-02, 8.44440559e-01, - 4.50646654e-01, 3.74785037e-01, 4.87196697e-01, 4.56276448e-01, 9.13284391e-01, 4.15715464e-01, - 7.13597697e-01, 1.23641270e-02, 5.10031271e-01, 4.74601930e-02, 2.55731159e-01, 3.22090006e-01, - 1.91165703e-01, 4.51170940e-01, 7.50843157e-01, 4.42420576e-01, 4.25380660e-01, 4.50667257e-01, - 6.55689206e-01, 9.68257670e-02, 1.96528793e-01, 8.97343028e-01, 4.99940904e-01, 6.65504083e-01, - 9.41828079e-01, 4.54397338e-01, 5.61893331e-01, 5.09839880e-01, 4.53117514e-01, 8.96804127e-02, - 1.74888861e-01, 6.65641378e-01, 2.81668336e-01, 1.89532742e-01, 5.61668382e-01, 8.68330157e-02, - 8.25092797e-01, 5.18106324e-01, 1.71904024e-01, 3.68385523e-01, 1.62005436e-01, 7.48507399e-01, - 9.30274827e-01, 2.38198517e-01, 9.52222901e-01, 5.23587800e-01, 6.94384557e-01, 1.09338652e-01, - 4.83356794e-01, 2.73050402e-01, 3.68027050e-01, 5.92366466e-01, 1.83192289e-01, 8.60376029e-01, - 7.13926203e-01, 8.16750052e-01, 1.57890291e-01, 6.25691951e-01, 5.24831646e-01, 1.73873797e-01, - 1.02429784e-01, 9.17488471e-01, 4.03584434e-01, 9.31170884e-01, 2.79386137e-01, 8.77745206e-01, - 2.45200576e-01, 1.28896951e-01, 3.15713052e-01, 5.27874291e-01, 2.16444335e-01, 7.03883817e-01, - 7.74738919e-02, 8.42422142e-01, 3.75598924e-01, 3.51002411e-01, 6.22752776e-01, 4.82407943e-01, - 7.43107867e-01, 9.46182666e-01, 9.44344819e-01, 3.28124763e-01, 1.06147431e-01, 1.65102684e-01, - 3.84060507e-01, 2.91057722e-01, 7.68173662e-02, 1.03543651e-01, 6.76698940e-01, 1.43141994e-01, - 7.21342202e-01, 6.69471294e-03, 9.07298311e-01, 5.57080171e-01, 8.10954489e-01, 4.11120526e-01, - 2.06407453e-01, 2.59590556e-01, 7.58512718e-01, 5.79873897e-01, 2.92875650e-01, 2.83686529e-01, - 2.42829343e-01, 9.19323719e-01, 3.46832864e-01, 3.58238858e-01, 7.42827585e-01, 2.05760059e-01, - 9.58438860e-01, 5.66326411e-01, 6.60292846e-01, 5.61095078e-02, 6.79465531e-01, 7.05118513e-01, - 4.44713264e-01, 2.09732933e-01, 5.22732436e-01, 1.74396512e-01, 5.29356748e-01, 4.38475687e-01, - 4.94036404e-01, 4.09785794e-01, 6.40025507e-01, 5.79371821e-01, 1.57726118e-01, 6.04572263e-01, - 5.41072639e-01, 5.18847173e-01, 1.97093284e-01, 8.91767002e-01, 4.29050835e-01, 8.25490570e-01, - 3.87699807e-01, 4.50705808e-01, 2.49371643e-01, 3.36074898e-01, 9.29925118e-01, 6.65393649e-01, - 9.07275994e-01, 3.73075859e-01, 4.14044139e-03, 2.37463702e-01, 2.25893784e-01, 2.46900245e-01, - 4.50350196e-01, 3.48618117e-01, 5.07193932e-01, 5.23435142e-01, 8.13611417e-01, 8.92715622e-01, - 1.02623450e-01, 3.06088345e-01, 7.80461650e-01, 2.21453645e-01, 2.01419652e-01, 2.84254457e-01, - 3.68286735e-01, 7.39358243e-01, 8.97879394e-01, 9.81599566e-01, 7.56526442e-01, 7.37645545e-01, - 4.23976657e-02, 8.25922012e-01, 2.60956996e-01, 2.90702065e-01, 8.98388344e-01, 3.03733299e-01, - 8.49071471e-01, 3.45835425e-01, 7.65458276e-01, 5.68094872e-01, 8.93770930e-01, 9.93161641e-01, - 5.63368667e-02, 4.26548945e-01, 5.46745780e-01, 5.75674571e-01, 7.94599487e-01, 7.18935553e-02, - 4.46492976e-01, 6.40240123e-01, 2.73246969e-01, 2.00465968e-01, 1.30718835e-01, 1.92492005e-01, - 1.96617189e-01, 6.61271644e-01, 8.12687657e-01, 8.66342445e-01 - - }, - {0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - 10, - true, - -4}, - {10, - 5, - {0.21390334, 0.50261639, 0.91036676, 0.59166485, 0.71162682, 0.10248392, 0.77782677, 0.43772379, - 0.4035871, 0.3282796, 0.47544681, 0.59862974, 0.12319357, 0.06239463, 0.28200272, 0.1345717, - 0.50498218, 0.5113505, 0.16233086, 0.62165332, 0.42281548, 0.933117, 0.41386077, 0.23264562, - 0.73325968, 0.37537541, 0.70719873, 0.14522645, 0.73279625, 0.9126674, 0.84854131, 0.28890216, - 0.85267903, 0.74703138, 0.83842071, 0.34942792, 0.27864171, 0.70911132, 0.21338564, 0.32035554, - 0.73788331, 0.46926692, 0.57570162, 0.42559178, 0.87120209, 0.22734951, 0.01847905, 0.75549396, - 0.76166195, 0.66613745}, - {9, 8, 7, 6, 5, 4, 3, 2, 1, 0}, - 10, - false, - 5}, - // Test outlier points - {9, - 2, - {-1, -50, 3, 4, 5000, 10000, 1, 3, 4, 5, 0.000005, 0.00002, 2000000, 500000, 10, 50, 30, 5}, - {6, 0, 5, 0, 0, 4, 3, 2, 1}, - 7, - false, - 5}, - - // Test n_clusters == (n_points / 2) - {10, - 5, - {0.21390334, 0.50261639, 0.91036676, 0.59166485, 0.71162682, 0.10248392, 0.77782677, 0.43772379, - 0.4035871, 0.3282796, 0.47544681, 0.59862974, 0.12319357, 0.06239463, 0.28200272, 0.1345717, - 0.50498218, 0.5113505, 0.16233086, 0.62165332, 0.42281548, 0.933117, 0.41386077, 0.23264562, - 0.73325968, 0.37537541, 0.70719873, 0.14522645, 0.73279625, 0.9126674, 0.84854131, 0.28890216, - 0.85267903, 0.74703138, 0.83842071, 0.34942792, 0.27864171, 0.70911132, 0.21338564, 0.32035554, - 0.73788331, 0.46926692, 0.57570162, 0.42559178, 0.87120209, 0.22734951, 0.01847905, 0.75549396, - 0.76166195, 0.66613745}, - {1, 0, 4, 0, 0, 3, 2, 0, 2, 1}, - 5, - false, - 10}, - - // Test n_points == 100 - {100, - 10, - {6.26168372e-01, 9.30437651e-01, 6.02450208e-01, 2.73025296e-01, 9.53050619e-01, 3.32164396e-01, - 6.88942598e-01, 5.79163537e-01, 6.70341547e-01, 2.70140602e-02, 9.30429671e-01, 7.17721157e-01, - 9.89948537e-01, 7.75253347e-01, 1.34491522e-02, 2.48522428e-02, 3.51413378e-01, 7.64405834e-01, - 7.86373507e-01, 7.18748577e-01, 8.66998621e-01, 6.80316582e-01, 2.51288712e-01, 4.91078420e-01, - 3.76246281e-01, 4.86828710e-01, 5.67464772e-01, 5.30734742e-01, 8.99478296e-01, 7.66699088e-01, - 9.49339111e-01, 3.55248484e-01, 9.06046929e-01, 4.48407772e-01, 6.96395305e-01, 2.44277335e-01, - 7.74840000e-01, 5.21046603e-01, 4.66423971e-02, 5.12019638e-02, 8.95019614e-01, 5.28956953e-01, - 4.31536306e-01, 5.83857744e-01, 4.41787364e-01, 4.68656523e-01, 5.73971433e-01, 6.79989654e-01, - 3.19650588e-01, 6.12579596e-01, 6.49126442e-02, 8.39131142e-01, 2.85252117e-01, 5.84848929e-01, - 9.46507115e-01, 8.58440748e-01, 3.61528940e-01, 2.44215959e-01, 3.80101125e-01, 4.57128957e-02, - 8.82216988e-01, 8.31498633e-01, 7.23474381e-01, 7.75788607e-01, 1.40864146e-01, 6.62092382e-01, - 5.13985168e-01, 3.00686418e-01, 8.70109949e-01, 2.43187753e-01, 2.89391938e-01, 2.84214238e-01, - 8.70985521e-01, 8.77491176e-01, 6.72537226e-01, 3.30929686e-01, 1.85934324e-01, 9.16222614e-01, - 6.18239142e-01, 2.64768597e-01, 5.76145451e-01, 8.62961369e-01, 6.84757925e-01, 7.60549082e-01, - 1.27645356e-01, 4.51004673e-01, 3.92292980e-01, 4.63170803e-01, 4.35449330e-02, 2.17583404e-01, - 5.71832605e-02, 2.06763039e-01, 3.70116249e-01, 2.09750028e-01, 6.17283019e-01, 8.62549231e-01, - 9.84156240e-02, 2.66249156e-01, 3.87635103e-01, 2.85591012e-02, 4.24826068e-01, 4.45795088e-01, - 6.86227676e-01, 1.08848960e-01, 5.96731841e-02, 3.71770228e-01, 1.91548833e-01, 6.95136078e-01, - 9.00700636e-01, 8.76363105e-01, 2.67334632e-01, 1.80619709e-01, 7.94060419e-01, 1.42854171e-02, - 1.09372387e-01, 8.74028108e-01, 6.46403232e-01, 4.86588834e-01, 5.93446175e-02, 6.11886291e-01, - 8.83865057e-01, 3.15879821e-01, 2.27043992e-01, 9.76764951e-01, 6.15620336e-01, 9.76199360e-01, - 2.40548962e-01, 3.21795663e-01, 8.75087904e-02, 8.11234663e-01, 6.96070480e-01, 8.12062321e-01, - 1.21958818e-01, 3.44348628e-02, 8.72630414e-01, 3.06162776e-01, 1.76043529e-02, 9.45894971e-01, - 5.33896401e-01, 6.21642973e-01, 4.93062535e-01, 4.48984262e-01, 2.24560379e-01, 4.24052195e-02, - 4.43447610e-01, 8.95646149e-01, 6.05220676e-01, 1.81840491e-01, 9.70831206e-01, 2.12563586e-02, - 6.92582693e-01, 7.55946922e-01, 7.95086143e-01, 6.05328941e-01, 3.99350764e-01, 4.32846636e-01, - 9.81114529e-01, 4.98266428e-01, 6.37127930e-03, 1.59085889e-01, 6.34682067e-05, 5.59429440e-01, - 7.38827633e-01, 8.93214770e-01, 2.16494306e-01, 9.35430573e-02, 4.75665868e-02, 7.80503518e-01, - 7.86240041e-01, 7.06854594e-01, 2.13725879e-02, 7.68246091e-01, 4.50234808e-01, 5.21231104e-01, - 5.01989826e-03, 4.22081572e-02, 1.65337732e-01, 8.54134740e-01, 4.99430262e-01, 8.94525601e-01, - 1.14028379e-01, 3.69739861e-01, 1.32955599e-01, 2.65563824e-01, 2.52811151e-01, 1.44792843e-01, - 6.88449594e-01, 4.44921417e-01, 8.23296587e-01, 1.93266317e-01, 1.19033309e-01, 1.36368966e-01, - 3.42600285e-01, 5.64505195e-01, 5.57594559e-01, 7.44257892e-01, 8.38231569e-02, 4.11548847e-01, - 3.21010077e-01, 8.55081359e-01, 4.30105779e-01, 1.16229135e-01, 9.87731964e-02, 3.14712335e-01, - 4.50880592e-01, 2.72289598e-01, 6.31615256e-01, 8.97432958e-01, 4.44764250e-01, 8.03776440e-01, - 2.68767748e-02, 2.43374608e-01, 4.02141103e-01, 4.98881209e-01, 5.33173003e-01, 8.82890436e-01, - 7.16149148e-01, 4.19664401e-01, 2.29335357e-01, 2.88637806e-01, 3.44696803e-01, 6.78171906e-01, - 5.69849716e-01, 5.86454477e-01, 3.54474989e-01, 9.03876540e-01, 6.45980000e-01, 6.34887593e-01, - 7.88039746e-02, 2.04814126e-01, 7.82251754e-01, 2.43147074e-01, 7.50951808e-01, 1.72799092e-02, - 2.95349590e-01, 6.57991826e-01, 8.81214312e-01, 5.73970708e-01, 2.77610881e-01, 1.82155097e-01, - 7.69797417e-02, 6.44792402e-01, 9.46950998e-01, 7.73064845e-01, 6.04733624e-01, 5.80094567e-01, - 1.67498426e-01, 2.66514296e-01, 6.50140368e-01, 1.91170299e-01, 2.08752199e-01, 3.01664091e-01, - 9.85033484e-01, 2.92909152e-01, 8.65816607e-01, 1.85222119e-01, 2.28814559e-01, 1.34286382e-02, - 2.89234322e-01, 8.18668708e-01, 4.71706924e-01, 9.23199803e-01, 2.80879188e-01, 1.47319284e-01, - 4.13915748e-01, 9.31274932e-02, 6.66322195e-01, 9.66953974e-01, 3.19405786e-01, 6.69486551e-01, - 5.03096313e-02, 6.95225201e-01, 5.78469859e-01, 6.29481655e-01, 1.39252534e-01, 1.22564968e-01, - 6.80663678e-01, 6.34607157e-01, 6.42765834e-01, 1.57127410e-02, 2.92132086e-01, 5.24423878e-01, - 4.68676824e-01, 2.86003928e-01, 7.18608322e-01, 8.95617933e-01, 5.48844309e-01, 1.74517278e-01, - 5.24379196e-01, 2.13526524e-01, 5.88375435e-01, 9.88560185e-01, 4.17435771e-01, 6.14438688e-01, - 9.53760881e-01, 5.27151288e-01, 7.03017278e-01, 3.44448559e-01, 4.47059676e-01, 2.83414901e-01, - 1.98979011e-01, 4.24917361e-01, 5.73172761e-01, 2.32398853e-02, 1.65887230e-01, 4.05552785e-01, - 9.29665524e-01, 2.26135696e-01, 9.20563384e-01, 7.65259963e-01, 4.54820075e-01, 8.97710267e-01, - 3.78559302e-03, 9.15219382e-01, 3.55705698e-01, 6.94905124e-01, 8.58540202e-01, 3.89790666e-01, - 2.49478206e-01, 7.93679304e-01, 4.75830027e-01, 4.40425353e-01, 3.70579459e-01, 1.40578049e-01, - 1.70386675e-01, 7.04056121e-01, 4.85963102e-01, 9.68450060e-01, 6.77178001e-01, 2.65934654e-01, - 2.58915007e-01, 6.70052890e-01, 2.61945109e-01, 8.46207759e-01, 1.01928951e-01, 2.85611334e-01, - 2.45776933e-01, 2.66658783e-01, 3.71724077e-01, 4.34319025e-01, 4.24407347e-01, 7.15417683e-01, - 8.07997684e-01, 1.64296275e-01, 6.01638065e-01, 8.60606804e-02, 2.68719187e-01, 5.11764101e-01, - 9.75844338e-01, 7.81226782e-01, 2.20925515e-01, 7.18135040e-01, 9.82395577e-01, 8.39160243e-01, - 9.08058083e-01, 6.88010677e-01, 8.14271847e-01, 5.12460821e-01, 1.17311345e-01, 5.96075228e-01, - 9.17455497e-01, 2.12052706e-01, 7.04074603e-01, 8.72872565e-02, 8.76047818e-01, 6.96235046e-01, - 8.54801557e-01, 2.49729159e-01, 9.76594604e-01, 2.87386363e-01, 2.36461559e-02, 9.94075254e-01, - 4.25193986e-01, 7.61869994e-01, 5.13334255e-01, 6.44711165e-02, 8.92156689e-01, 3.55235167e-01, - 1.08154647e-01, 8.78446825e-01, 2.43833016e-01, 9.23071293e-01, 2.72724115e-01, 9.46631338e-01, - 3.74510294e-01, 4.08451278e-02, 9.78392777e-01, 3.65079221e-01, 6.37199516e-01, 5.51144906e-01, - 5.25978080e-01, 1.42803678e-01, 4.05451674e-01, 7.79788219e-01, 6.26009784e-01, 3.35249497e-01, - 1.43159543e-02, 1.80363779e-01, 5.05096904e-01, 2.82619947e-01, 5.83561392e-01, 3.10951324e-01, - 8.73223968e-01, 4.38545619e-01, 4.81348800e-01, 6.68497085e-01, 3.79345401e-01, 9.58832501e-01, - 1.89869550e-01, 2.34083070e-01, 2.94066207e-01, 5.74892667e-02, 6.92106828e-02, 9.61127686e-02, - 6.72650672e-02, 8.47345378e-01, 2.80916761e-01, 7.32177357e-03, 9.80785961e-01, 5.73192225e-02, - 8.48781331e-01, 8.83225408e-01, 7.34398275e-01, 7.70381941e-01, 6.20778343e-01, 8.96822048e-01, - 5.40732486e-01, 3.69704071e-01, 5.77305837e-01, 2.08221827e-01, 7.34275341e-01, 1.06110900e-01, - 3.49496706e-01, 8.34948910e-01, 1.56403291e-02, 6.78576376e-01, 8.96141268e-01, 5.94835119e-01, - 1.43943153e-01, 3.49618530e-01, 2.10440392e-01, 3.46585620e-01, 1.05153093e-01, 3.45446174e-01, - 2.72177079e-01, 7.07946300e-01, 4.33717726e-02, 3.31232203e-01, 3.91874320e-01, 4.76338141e-01, - 6.22777789e-01, 2.95989228e-02, 4.32855769e-01, 7.61049310e-01, 3.63279149e-01, 9.47210350e-01, - 6.43721247e-01, 6.58025802e-01, 1.05247633e-02, 5.29974442e-01, 7.30675767e-01, 4.30041079e-01, - 6.62634841e-01, 8.25936616e-01, 9.91253704e-01, 6.79399281e-01, 5.44177006e-01, 7.52876048e-01, - 3.32139049e-01, 7.98732398e-01, 7.38865223e-01, 9.16055132e-01, 6.11736493e-01, 9.63672879e-01, - 1.83778839e-01, 7.27558919e-02, 5.91602822e-01, 3.25235484e-01, 2.34741217e-01, 9.52346277e-01, - 9.18556407e-01, 9.35373324e-01, 6.89209070e-01, 2.56049054e-01, 6.17975395e-01, 7.82285691e-01, - 9.84983432e-01, 6.62322741e-01, 2.04144457e-01, 3.98446577e-01, 1.38918297e-01, 3.05919921e-01, - 3.14043787e-01, 5.91072666e-01, 7.44703771e-01, 8.92272567e-01, 9.78017873e-01, 9.01203161e-01, - 1.41526372e-01, 4.14878484e-01, 6.80683651e-01, 5.01733152e-02, 8.14635389e-01, 2.27926375e-01, - 9.03269815e-01, 8.68443745e-01, 9.86939190e-01, 7.40779486e-01, 2.61005311e-01, 3.19276232e-01, - 9.69509248e-01, 1.11908818e-01, 4.49198556e-01, 1.27056715e-01, 3.84064823e-01, 5.14591811e-01, - 2.10747488e-01, 9.53884090e-01, 8.43167950e-01, 4.51187972e-01, 3.75331782e-01, 6.23566461e-01, - 3.55290379e-01, 2.95705968e-01, 1.69622690e-01, 1.42981830e-01, 2.72180991e-01, 9.46468040e-01, - 3.70932500e-01, 9.94292830e-01, 4.62587505e-01, 7.14817405e-01, 2.45370540e-02, 3.00906377e-01, - 5.75768304e-01, 9.71448393e-01, 6.95574827e-02, 3.93693854e-01, 5.29306116e-01, 5.04694554e-01, - 6.73797120e-02, 6.76596969e-01, 5.50948898e-01, 3.24909641e-01, 7.70337719e-01, 6.51842631e-03, - 3.03264879e-01, 7.61037886e-03, 2.72289601e-01, 1.50502041e-01, 6.71103888e-02, 7.41503703e-01, - 1.92088941e-01, 2.19043977e-01, 9.09320161e-01, 2.37993569e-01, 6.18107973e-02, 8.31447852e-01, - 2.23355609e-01, 1.84789435e-01, 4.16104518e-01, 4.21573859e-01, 8.72446305e-02, 2.97294197e-01, - 4.50328256e-01, 8.72199917e-01, 2.51279916e-01, 4.86219272e-01, 7.57071329e-01, 4.85655942e-01, - 1.06187277e-01, 4.92341327e-01, 1.46017513e-01, 5.25421017e-01, 4.22637906e-01, 2.24685018e-01, - 8.72648431e-01, 5.54051490e-01, 1.80745062e-01, 2.12756336e-01, 5.20883169e-01, 7.60363654e-01, - 8.30254678e-01, 5.00003328e-01, 4.69017439e-01, 6.38105527e-01, 3.50638261e-02, 5.22217353e-02, - 9.06516882e-02, 8.52975842e-01, 1.19985883e-01, 3.74926753e-01, 6.50302066e-01, 1.98875727e-01, - 6.28362507e-02, 4.32693501e-01, 3.10500685e-01, 6.20732833e-01, 4.58503272e-01, 3.20790034e-01, - 7.91284868e-01, 7.93054570e-01, 2.93406765e-01, 8.95399023e-01, 1.06441034e-01, 7.53085241e-02, - 8.67523104e-01, 1.47963482e-01, 1.25584706e-01, 3.81545040e-02, 6.34338619e-01, 1.76368938e-02, - 5.75553531e-02, 5.31607516e-01, 2.63869588e-01, 9.41945823e-01, 9.24028838e-02, 5.21496463e-01, - 7.74866558e-01, 5.65210610e-01, 7.28015327e-02, 6.51963790e-01, 8.94727453e-01, 4.49571590e-01, - 1.29932405e-01, 8.64026259e-01, 9.92599934e-01, 7.43721560e-01, 8.87300215e-01, 1.06369925e-01, - 8.11335531e-01, 7.87734900e-01, 9.87344678e-01, 5.32502820e-01, 4.42612382e-01, 9.64041183e-01, - 1.66085871e-01, 1.12937664e-01, 5.24423470e-01, 6.54689333e-01, 4.59119726e-01, 5.22774091e-01, - 3.08722276e-02, 6.26979315e-01, 4.49754105e-01, 8.07495757e-01, 2.34199499e-01, 1.67765675e-01, - 9.22168418e-01, 3.73210378e-01, 8.04432575e-01, 5.61890354e-01, 4.47025593e-01, 6.43155678e-01, - 2.40407640e-01, 5.91631279e-01, 1.59369206e-01, 7.75799090e-01, 8.32067212e-01, 5.59791576e-02, - 6.39105224e-01, 4.85274738e-01, 2.12630838e-01, 2.81431312e-02, 7.16205363e-01, 6.83885011e-01, - 5.23869697e-01, 9.99418314e-01, 8.35331599e-01, 4.69877463e-02, 6.74712562e-01, 7.99273684e-01, - 2.77001890e-02, 5.75809742e-01, 2.78513031e-01, 8.36209905e-01, 7.25472379e-01, 4.87173943e-01, - 7.88311357e-01, 9.64676177e-01, 1.75752651e-01, 4.98112580e-01, 8.08850418e-02, 6.40981131e-01, - 4.06647450e-01, 8.46539387e-01, 2.12620694e-01, 9.11012851e-01, 8.25041445e-01, 8.90065575e-01, - 9.63626055e-01, 5.96689242e-01, 1.63372670e-01, 4.51640148e-01, 3.43026542e-01, 5.80658851e-01, - 2.82327625e-01, 4.75535418e-01, 6.27760926e-01, 8.46314115e-01, 9.61961932e-01, 3.19806094e-01, - 5.05508062e-01, 5.28102944e-01, 6.13045057e-01, 7.44714938e-01, 1.50586073e-01, 7.91878033e-01, - 4.89839179e-01, 3.10496849e-01, 8.82309038e-01, 2.86922314e-01, 4.84687559e-01, 5.20838630e-01, - 4.62955493e-01, 2.38185305e-01, 5.47259907e-02, 7.10916137e-01, 7.31887202e-01, 6.25602317e-01, - 8.77741168e-01, 4.19881322e-01, 4.81222328e-01, 1.28224501e-01, 2.46034010e-01, 3.34971854e-01, - 7.37216484e-01, 5.62134821e-02, 7.14089724e-01, 9.85549393e-01, 4.66295827e-01, 3.08722434e-03, - 4.70237690e-01, 2.66524167e-01, 7.93875484e-01, 4.54795911e-02, 8.09702944e-01, 1.47709735e-02, - 1.70082405e-01, 6.35905179e-01, 3.75379109e-01, 4.30315011e-01, 3.15788760e-01, 5.58065230e-01, - 2.24643800e-01, 2.42142981e-01, 6.57283636e-01, 3.34921891e-01, 1.26588975e-01, 7.68064155e-01, - 9.43856291e-01, 4.47518596e-01, 5.44453573e-01, 9.95764932e-01, 7.16444391e-01, 8.51019765e-01, - 1.01179183e-01, 4.45473958e-01, 4.60327322e-01, 4.96895844e-02, 4.72907738e-01, 5.58987444e-01, - 3.41027487e-01, 1.56175026e-01, 7.58283148e-01, 6.83600909e-01, 2.14623396e-01, 3.27348880e-01, - 3.92517893e-01, 6.70418431e-01, 5.16440832e-01, 8.63140348e-01, 5.73277464e-01, 3.46608058e-01, - 7.39396341e-01, 7.20852434e-01, 2.35653246e-02, 3.89935659e-01, 7.53783745e-01, 6.34563528e-01, - 8.79339335e-01, 7.41599159e-02, 5.62433904e-01, 6.15553852e-01, 4.56956324e-01, 5.20047447e-01, - 5.26845015e-02, 5.58471266e-01, 1.63632233e-01, 5.38936665e-02, 6.49593683e-01, 2.56838748e-01, - 8.99035326e-01, 7.20847756e-01, 5.68954684e-01, 7.43684755e-01, 5.70924238e-01, 3.82318724e-01, - 4.89328290e-01, 5.62208561e-01, 4.97540804e-02, 4.18011085e-01, 6.88041565e-01, 2.16234653e-01, - 7.89548214e-01, 8.46136387e-01, 8.46816189e-01, 1.73842353e-01, 6.11627842e-02, 8.44440559e-01, - 4.50646654e-01, 3.74785037e-01, 4.87196697e-01, 4.56276448e-01, 9.13284391e-01, 4.15715464e-01, - 7.13597697e-01, 1.23641270e-02, 5.10031271e-01, 4.74601930e-02, 2.55731159e-01, 3.22090006e-01, - 1.91165703e-01, 4.51170940e-01, 7.50843157e-01, 4.42420576e-01, 4.25380660e-01, 4.50667257e-01, - 6.55689206e-01, 9.68257670e-02, 1.96528793e-01, 8.97343028e-01, 4.99940904e-01, 6.65504083e-01, - 9.41828079e-01, 4.54397338e-01, 5.61893331e-01, 5.09839880e-01, 4.53117514e-01, 8.96804127e-02, - 1.74888861e-01, 6.65641378e-01, 2.81668336e-01, 1.89532742e-01, 5.61668382e-01, 8.68330157e-02, - 8.25092797e-01, 5.18106324e-01, 1.71904024e-01, 3.68385523e-01, 1.62005436e-01, 7.48507399e-01, - 9.30274827e-01, 2.38198517e-01, 9.52222901e-01, 5.23587800e-01, 6.94384557e-01, 1.09338652e-01, - 4.83356794e-01, 2.73050402e-01, 3.68027050e-01, 5.92366466e-01, 1.83192289e-01, 8.60376029e-01, - 7.13926203e-01, 8.16750052e-01, 1.57890291e-01, 6.25691951e-01, 5.24831646e-01, 1.73873797e-01, - 1.02429784e-01, 9.17488471e-01, 4.03584434e-01, 9.31170884e-01, 2.79386137e-01, 8.77745206e-01, - 2.45200576e-01, 1.28896951e-01, 3.15713052e-01, 5.27874291e-01, 2.16444335e-01, 7.03883817e-01, - 7.74738919e-02, 8.42422142e-01, 3.75598924e-01, 3.51002411e-01, 6.22752776e-01, 4.82407943e-01, - 7.43107867e-01, 9.46182666e-01, 9.44344819e-01, 3.28124763e-01, 1.06147431e-01, 1.65102684e-01, - 3.84060507e-01, 2.91057722e-01, 7.68173662e-02, 1.03543651e-01, 6.76698940e-01, 1.43141994e-01, - 7.21342202e-01, 6.69471294e-03, 9.07298311e-01, 5.57080171e-01, 8.10954489e-01, 4.11120526e-01, - 2.06407453e-01, 2.59590556e-01, 7.58512718e-01, 5.79873897e-01, 2.92875650e-01, 2.83686529e-01, - 2.42829343e-01, 9.19323719e-01, 3.46832864e-01, 3.58238858e-01, 7.42827585e-01, 2.05760059e-01, - 9.58438860e-01, 5.66326411e-01, 6.60292846e-01, 5.61095078e-02, 6.79465531e-01, 7.05118513e-01, - 4.44713264e-01, 2.09732933e-01, 5.22732436e-01, 1.74396512e-01, 5.29356748e-01, 4.38475687e-01, - 4.94036404e-01, 4.09785794e-01, 6.40025507e-01, 5.79371821e-01, 1.57726118e-01, 6.04572263e-01, - 5.41072639e-01, 5.18847173e-01, 1.97093284e-01, 8.91767002e-01, 4.29050835e-01, 8.25490570e-01, - 3.87699807e-01, 4.50705808e-01, 2.49371643e-01, 3.36074898e-01, 9.29925118e-01, 6.65393649e-01, - 9.07275994e-01, 3.73075859e-01, 4.14044139e-03, 2.37463702e-01, 2.25893784e-01, 2.46900245e-01, - 4.50350196e-01, 3.48618117e-01, 5.07193932e-01, 5.23435142e-01, 8.13611417e-01, 8.92715622e-01, - 1.02623450e-01, 3.06088345e-01, 7.80461650e-01, 2.21453645e-01, 2.01419652e-01, 2.84254457e-01, - 3.68286735e-01, 7.39358243e-01, 8.97879394e-01, 9.81599566e-01, 7.56526442e-01, 7.37645545e-01, - 4.23976657e-02, 8.25922012e-01, 2.60956996e-01, 2.90702065e-01, 8.98388344e-01, 3.03733299e-01, - 8.49071471e-01, 3.45835425e-01, 7.65458276e-01, 5.68094872e-01, 8.93770930e-01, 9.93161641e-01, - 5.63368667e-02, 4.26548945e-01, 5.46745780e-01, 5.75674571e-01, 7.94599487e-01, 7.18935553e-02, - 4.46492976e-01, 6.40240123e-01, 2.73246969e-01, 2.00465968e-01, 1.30718835e-01, 1.92492005e-01, - 1.96617189e-01, 6.61271644e-01, 8.12687657e-01, 8.66342445e-01 - - }, - {0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - 10, - false, - 5}}; - -typedef LinkageTest LinkageTestF_Int; -TEST_P(LinkageTestF_Int, Result) { EXPECT_TRUE(score == 1.0); } - -INSTANTIATE_TEST_CASE_P(LinkageTest, LinkageTestF_Int, ::testing::ValuesIn(linkage_inputsf2)); -} // end namespace raft diff --git a/cpp/test/cluster/spectral.cu b/cpp/test/cluster/spectral.cu deleted file mode 100644 index b8ee611f29..0000000000 --- a/cpp/test/cluster/spectral.cu +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (c) 2020-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" - -#include -#include -#include - -#include - -#include -#include - -namespace raft { -namespace cluster { - -/** - * Warning: There appears to be a CUDA 12.2 bug in cusparse that causes an - * alignment issue. We've fixed the bug in our code through a workaround - * (see raft/sparse/linalg/spmm.hpp for fix). This test is meant to fail - * in the case where the fix is accidentally reverted, so that it doesn't - * break any downstream libraries that depend on RAFT - */ -TEST(Raft, Spectral) -{ - raft::handle_t handle; - - std::vector h_offsets({0, 2, 4, 7, 10, 12, 14}); - std::vector h_indices({1, 2, 0, 2, 0, 1, 3, 2, 4, 5, 3, 5, 3, 4}); - std::vector h_values( - {1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0}); - std::vector expected_clustering({1, 1, 1, 0, 0, 0}); - - int32_t n_clusters{2}; - int32_t n_eigenvectors{2}; - int32_t evs_max_it{100}; - int32_t kmean_max_it{100}; - int32_t restartIter_lanczos = 15 + n_eigenvectors; - float evs_tol{0.001}; - float kmean_tol{0.001}; - unsigned long long seed1{1234567}; - unsigned long long seed2{12345678}; - bool reorthog{false}; - - rmm::device_uvector offsets(h_offsets.size(), handle.get_stream()); - rmm::device_uvector indices(h_indices.size(), handle.get_stream()); - rmm::device_uvector values(h_indices.size(), handle.get_stream()); - rmm::device_uvector clustering(expected_clustering.size(), handle.get_stream()); - rmm::device_uvector eigenvalues(n_eigenvectors, handle.get_stream()); - rmm::device_uvector eigenvectors(n_eigenvectors * expected_clustering.size(), - handle.get_stream()); - - rmm::device_uvector exp_dev(expected_clustering.size(), handle.get_stream()); - - raft::update_device( - exp_dev.data(), expected_clustering.data(), expected_clustering.size(), handle.get_stream()); - - raft::update_device(offsets.data(), h_offsets.data(), h_offsets.size(), handle.get_stream()); - raft::update_device(indices.data(), h_indices.data(), h_indices.size(), handle.get_stream()); - raft::update_device(values.data(), h_values.data(), h_values.size(), handle.get_stream()); - - raft::spectral::matrix::sparse_matrix_t const matrix{ - handle, - offsets.data(), - indices.data(), - values.data(), - static_cast(offsets.size() - 1), - static_cast(indices.size())}; - - raft::spectral::eigen_solver_config_t eig_cfg{ - n_eigenvectors, evs_max_it, restartIter_lanczos, evs_tol, reorthog, seed1}; - raft::spectral::lanczos_solver_t eig_solver{eig_cfg}; - - raft::spectral::cluster_solver_config_t clust_cfg{ - n_clusters, kmean_max_it, kmean_tol, seed2}; - raft::spectral::kmeans_solver_t cluster_solver{clust_cfg}; - - raft::spectral::partition(handle, - matrix, - eig_solver, - cluster_solver, - clustering.data(), - eigenvalues.data(), - eigenvectors.data()); - - ASSERT_TRUE(devArrMatch(expected_clustering.data(), - exp_dev.data(), - exp_dev.size(), - 1, - raft::Compare(), - handle.get_stream())); -} - -} // namespace cluster -} // namespace raft \ No newline at end of file diff --git a/cpp/test/distance/dist_adj.cu b/cpp/test/distance/dist_adj.cu deleted file mode 100644 index a22fb7b1f9..0000000000 --- a/cpp/test/distance/dist_adj.cu +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Copyright (c) 2018-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "dist_adj.cuh" - -#include -#include -#include -#include -#include -#include - -#include - -#include - -namespace raft { -namespace distance { - -template -RAFT_KERNEL naiveDistanceAdjKernel(uint8_t* dist, - const DataType* x, - const DataType* y, - int m, - int n, - int k, - DataType eps, - bool isRowMajor) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) return; - DataType acc = DataType(0); - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto diff = x[xidx] - y[yidx]; - acc += diff * diff; - } - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - dist[outidx] = acc <= eps; -} - -template -void naiveDistanceAdj(uint8_t* dist, - const DataType* x, - const DataType* y, - int m, - int n, - int k, - DataType eps, - bool isRowMajor, - cudaStream_t stream) -{ - static const dim3 TPB(16, 32, 1); - dim3 nblks(raft::ceildiv(m, (int)TPB.x), raft::ceildiv(n, (int)TPB.y), 1); - naiveDistanceAdjKernel<<>>(dist, x, y, m, n, k, eps, isRowMajor); - RAFT_CUDA_TRY(cudaPeekAtLastError()); -} - -template -struct DistanceAdjInputs { - DataType eps; - int m, n, k; - bool isRowMajor; - unsigned long long int seed; -}; - -template -::std::ostream& operator<<(::std::ostream& os, const DistanceAdjInputs& dims) -{ - return os; -} - -template -class DistanceAdjTest : public ::testing::TestWithParam> { - public: - DistanceAdjTest() - : params(::testing::TestWithParam>::GetParam()), - stream(resource::get_cuda_stream(handle)), - dist(params.m * params.n, stream), - dist_ref(params.m * params.n, stream) - { - } - - void SetUp() override - { - raft::random::RngState r(params.seed); - int m = params.m; - int n = params.n; - int k = params.k; - bool isRowMajor = params.isRowMajor; - - rmm::device_uvector x(m * k, stream); - rmm::device_uvector y(n * k, stream); - - uniform(handle, r, x.data(), m * k, DataType(-1.0), DataType(1.0)); - uniform(handle, r, y.data(), n * k, DataType(-1.0), DataType(1.0)); - - DataType threshold = params.eps; - - naiveDistanceAdj(dist_ref.data(), x.data(), y.data(), m, n, k, threshold, isRowMajor, stream); - size_t worksize = raft::distance:: - getWorkspaceSize( - x.data(), y.data(), m, n, k); - rmm::device_uvector workspace(worksize, stream); - - using threshold_final_op_ = threshold_final_op; - threshold_final_op_ threshold_op(threshold); - - raft::distance::distance(handle, - x.data(), - y.data(), - dist.data(), - m, - n, - k, - workspace.data(), - worksize, - threshold_op, - isRowMajor); - resource::sync_stream(handle, stream); - } - - void TearDown() override {} - - protected: - DistanceAdjInputs params; - // We use uint8_t even if the output in this test is a bool because - // cutlass doesn't support bool as output buffer yet. In cuda - // sizeof(bool) is 1 byte hence it doesn't increase - // memory consumption if we use uint8_t instead of bool. - rmm::device_uvector dist_ref; - rmm::device_uvector dist; - raft::resources handle; - cudaStream_t stream; -}; - -const std::vector> inputsf = { - {0.01f, 1024, 1024, 32, true, 1234ULL}, - {0.1f, 1024, 1024, 32, true, 1234ULL}, - {1.0f, 1024, 1024, 32, true, 1234ULL}, - {10.0f, 1024, 1024, 32, true, 1234ULL}, - {0.01f, 1024, 1024, 32, false, 1234ULL}, - {0.1f, 1024, 1024, 32, false, 1234ULL}, - {1.0f, 1024, 1024, 32, false, 1234ULL}, - {10.0f, 1024, 1024, 32, false, 1234ULL}, -}; -typedef DistanceAdjTest DistanceAdjTestF; -TEST_P(DistanceAdjTestF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(devArrMatch(dist_ref.data(), dist.data(), m, n, raft::Compare(), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceAdjTests, DistanceAdjTestF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.01, 1024, 1024, 32, true, 1234ULL}, - {0.1, 1024, 1024, 32, true, 1234ULL}, - {1.0, 1024, 1024, 32, true, 1234ULL}, - {10.0, 1024, 1024, 32, true, 1234ULL}, - {0.01, 1024, 1024, 32, false, 1234ULL}, - {0.1, 1024, 1024, 32, false, 1234ULL}, - {1.0, 1024, 1024, 32, false, 1234ULL}, - {10.0, 1024, 1024, 32, false, 1234ULL}, -}; -typedef DistanceAdjTest DistanceAdjTestD; -TEST_P(DistanceAdjTestD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(devArrMatch(dist_ref.data(), dist.data(), m, n, raft::Compare(), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceAdjTests, DistanceAdjTestD, ::testing::ValuesIn(inputsd)); - -} // namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_adj.cuh b/cpp/test/distance/dist_adj.cuh deleted file mode 100644 index 2861cb33de..0000000000 --- a/cpp/test/distance/dist_adj.cuh +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "dist_adj_threshold.cuh" - -#include - -#define instantiate_raft_distance_distance(DT, DataT, AccT, OutT, FinalLambda, IdxT) \ - extern template void raft::distance::distance( \ - raft::resources const& handle, \ - const DataT* x, \ - const DataT* y, \ - OutT* dist, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - void* workspace, \ - size_t worksize, \ - FinalLambda fin_op, \ - bool isRowMajor, \ - DataT metric_arg) - -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Expanded, - float, - float, - uint8_t, - raft::distance::threshold_float, - int); - -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Expanded, - double, - double, - uint8_t, - raft::distance::threshold_double, - int); - -#undef instantiate_raft_distance_distance - -#define instantiate_raft_distance_getWorkspaceSize(DistT, DataT, AccT, OutT, IdxT) \ - extern template size_t raft::distance::getWorkspaceSize( \ - const DataT* x, const DataT* y, IdxT m, IdxT n, IdxT k) - -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, float, float, uint8_t, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, double, double, uint8_t, int); - -#undef instantiate_raft_distance_getWorkspaceSize - -#define instantiate_raft_distance_getWorkspaceSize(DistT, DataT, AccT, OutT, IdxT) \ - extern template size_t raft::distance::getWorkspaceSize( \ - const DataT* x, const DataT* y, IdxT m, IdxT n, IdxT k) - -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, float, float, uint8_t, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, double, double, uint8_t, int); - -#undef instantiate_raft_distance_getWorkspaceSize diff --git a/cpp/test/distance/dist_adj_distance_instance.cu b/cpp/test/distance/dist_adj_distance_instance.cu deleted file mode 100644 index 158a5986c2..0000000000 --- a/cpp/test/distance/dist_adj_distance_instance.cu +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#undef RAFT_EXPLICIT_INSTANTIATE_ONLY - -#include "dist_adj_threshold.cuh" - -#include - -#include - -#define instantiate_raft_distance_distance(DT, DataT, AccT, OutT, FinalLambda, IdxT) \ - template void raft::distance::distance( \ - raft::resources const& handle, \ - const DataT* x, \ - const DataT* y, \ - OutT* dist, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - void* workspace, \ - size_t worksize, \ - FinalLambda fin_op, \ - bool isRowMajor, \ - DataT metric_arg) - -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Expanded, - float, - float, - uint8_t, - raft::distance::threshold_float, - int); - -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Expanded, - double, - double, - uint8_t, - raft::distance::threshold_double, - int); - -#undef instantiate_raft_distance_distance - -#define instantiate_raft_distance_getWorkspaceSize(DistT, DataT, AccT, OutT, IdxT) \ - template size_t raft::distance::getWorkspaceSize( \ - const DataT* x, const DataT* y, IdxT m, IdxT n, IdxT k) - -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, float, float, uint8_t, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, double, double, uint8_t, int); - -#undef instantiate_raft_distance_getWorkspaceSize diff --git a/cpp/test/distance/dist_adj_threshold.cuh b/cpp/test/distance/dist_adj_threshold.cuh deleted file mode 100644 index 78663b3cd1..0000000000 --- a/cpp/test/distance/dist_adj_threshold.cuh +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include // uint8_t - -namespace raft::distance { - -template -struct threshold_final_op { - DataT threshold_val; - - __device__ __host__ threshold_final_op() noexcept : threshold_val(0.0) {} - __device__ __host__ threshold_final_op(DataT val) noexcept : threshold_val(val) {} - __device__ __host__ OutT operator()(AccT d_val, Index g_idx) const noexcept - { - return d_val <= threshold_val; - } -}; - -using threshold_float = threshold_final_op; -using threshold_double = threshold_final_op; - -} // namespace raft::distance diff --git a/cpp/test/distance/dist_canberra.cu b/cpp/test/distance/dist_canberra.cu deleted file mode 100644 index 9b8b6c016b..0000000000 --- a/cpp/test/distance/dist_canberra.cu +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceCanberra : public DistanceTest {}; - -const std::vector> inputsf = { - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceCanberra DistanceCanberraF; -TEST_P(DistanceCanberraF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceCanberraF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001, 1024, 1024, 32, false, 1234ULL}, - {0.001, 1024, 32, 1024, false, 1234ULL}, - {0.001, 32, 1024, 1024, false, 1234ULL}, - {0.003, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceCanberra DistanceCanberraD; -TEST_P(DistanceCanberraD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceCanberraD, ::testing::ValuesIn(inputsd)); - -class BigMatrixCanberra : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixCanberra, Result) {} - -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_correlation.cu b/cpp/test/distance/dist_correlation.cu deleted file mode 100644 index aa2866483a..0000000000 --- a/cpp/test/distance/dist_correlation.cu +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceCorrelation - : public DistanceTest {}; - -template -class DistanceCorrelationXequalY - : public DistanceTestSameBuffer {}; - -const std::vector> inputsf = { - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceCorrelation DistanceCorrelationF; -TEST_P(DistanceCorrelationF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceCorrelationF, ::testing::ValuesIn(inputsf)); - -typedef DistanceCorrelationXequalY DistanceCorrelationXequalYF; -TEST_P(DistanceCorrelationXequalYF, Result) -{ - int m = params.m; - ASSERT_TRUE(raft::devArrMatch(dist_ref[0].data(), - dist[0].data(), - m, - m, - raft::CompareApprox(params.tolerance), - stream)); - ASSERT_TRUE(raft::devArrMatch(dist_ref[1].data(), - dist[1].data(), - m / 2, - m, - raft::CompareApprox(params.tolerance), - stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceCorrelationXequalYF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001, 1024, 1024, 32, false, 1234ULL}, - {0.001, 1024, 32, 1024, false, 1234ULL}, - {0.001, 32, 1024, 1024, false, 1234ULL}, - {0.003, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceCorrelation DistanceCorrelationD; -TEST_P(DistanceCorrelationD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceCorrelationD, ::testing::ValuesIn(inputsd)); - -class BigMatrixCorrelation - : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixCorrelation, Result) {} -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_cos.cu b/cpp/test/distance/dist_cos.cu deleted file mode 100644 index b792ec4039..0000000000 --- a/cpp/test/distance/dist_cos.cu +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright (c) 2018-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceExpCos : public DistanceTest { -}; - -template -class DistanceExpCosXequalY - : public DistanceTestSameBuffer {}; - -const std::vector> inputsf = { - {0.001f, 128, (65536 + 128) * 128, 8, true, 1234ULL}, - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, (65536 + 128) * 128, 128, 8, false, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; - -const std::vector> inputsXeqYf = { - {0.01f, 1024, 1024, 32, true, 1234ULL}, - {0.01f, 1024, 32, 1024, true, 1234ULL}, - {0.01f, 32, 1024, 1024, true, 1234ULL}, - {0.03f, 1024, 1024, 1024, true, 1234ULL}, - {0.01f, 1024, 1024, 32, false, 1234ULL}, - {0.01f, 1024, 32, 1024, false, 1234ULL}, - {0.01f, 32, 1024, 1024, false, 1234ULL}, - {0.03f, 1024, 1024, 1024, false, 1234ULL}, -}; - -typedef DistanceExpCos DistanceExpCosF; -TEST_P(DistanceExpCosF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceExpCosF, ::testing::ValuesIn(inputsf)); - -typedef DistanceExpCosXequalY DistanceExpCosXequalYF; -TEST_P(DistanceExpCosXequalYF, Result) -{ - int m = params.m; - int n = params.m; - ASSERT_TRUE(raft::devArrMatch(dist_ref[0].data(), - dist[0].data(), - m, - n, - raft::CompareApprox(params.tolerance), - stream)); - n = params.isRowMajor ? m : m / 2; - m = params.isRowMajor ? m / 2 : m; - - ASSERT_TRUE(raft::devArrMatch(dist_ref[1].data(), - dist[1].data(), - m, - n, - raft::CompareApprox(params.tolerance), - stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceExpCosXequalYF, ::testing::ValuesIn(inputsXeqYf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceExpCos DistanceExpCosD; -TEST_P(DistanceExpCosD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceExpCosD, ::testing::ValuesIn(inputsd)); - -class BigMatrixCos : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixCos, Result) {} - -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_dice.cu b/cpp/test/distance/dist_dice.cu deleted file mode 100644 index e127659dc6..0000000000 --- a/cpp/test/distance/dist_dice.cu +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright (c) 2018-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceExpDice : public DistanceTest { -}; - -template -class DistanceExpDiceXequalY - : public DistanceTestSameBuffer {}; - -const std::vector> inputsf = { - {0.001f, 128, (65536 + 128) * 128, 8, true, 1234ULL}, - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, (65536 + 128) * 128, 128, 8, false, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; - -const std::vector> inputsXeqYf = { - {0.01f, 1024, 1024, 32, true, 1234ULL}, - {0.01f, 1024, 32, 1024, true, 1234ULL}, - {0.01f, 32, 1024, 1024, true, 1234ULL}, - {0.03f, 1024, 1024, 1024, true, 1234ULL}, - {0.01f, 1024, 1024, 32, false, 1234ULL}, - {0.01f, 1024, 32, 1024, false, 1234ULL}, - {0.01f, 32, 1024, 1024, false, 1234ULL}, - {0.03f, 1024, 1024, 1024, false, 1234ULL}, -}; - -typedef DistanceExpDice DistanceExpDiceF; -TEST_P(DistanceExpDiceF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApproxNaN(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceExpDiceF, ::testing::ValuesIn(inputsf)); - -typedef DistanceExpDiceXequalY DistanceExpDiceXequalYF; -TEST_P(DistanceExpDiceXequalYF, Result) -{ - int m = params.m; - int n = params.m; - ASSERT_TRUE(raft::devArrMatch(dist_ref[0].data(), - dist[0].data(), - m, - n, - raft::CompareApproxNaN(params.tolerance), - stream)); - n = params.isRowMajor ? m : m / 2; - m = params.isRowMajor ? m / 2 : m; - - ASSERT_TRUE(raft::devArrMatch(dist_ref[1].data(), - dist[1].data(), - m, - n, - raft::CompareApproxNaN(params.tolerance), - stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceExpDiceXequalYF, ::testing::ValuesIn(inputsXeqYf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceExpDice DistanceExpDiceD; -TEST_P(DistanceExpDiceD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApproxNaN(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceExpDiceD, ::testing::ValuesIn(inputsd)); - -class BigMatrixDice : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixDice, Result) {} - -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_hamming.cu b/cpp/test/distance/dist_hamming.cu deleted file mode 100644 index 9529ec2eaa..0000000000 --- a/cpp/test/distance/dist_hamming.cu +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceHamming - : public DistanceTest {}; - -const std::vector> inputsf = { - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceHamming DistanceHammingF; -TEST_P(DistanceHammingF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceHammingF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001, 1024, 1024, 32, false, 1234ULL}, - {0.001, 1024, 32, 1024, false, 1234ULL}, - {0.001, 32, 1024, 1024, false, 1234ULL}, - {0.003, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceHamming DistanceHammingD; -TEST_P(DistanceHammingD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceHammingD, ::testing::ValuesIn(inputsd)); - -class BigMatrixHamming - : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixHamming, Result) {} -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_hellinger.cu b/cpp/test/distance/dist_hellinger.cu deleted file mode 100644 index 93d6101a18..0000000000 --- a/cpp/test/distance/dist_hellinger.cu +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceHellingerExp - : public DistanceTest {}; - -const std::vector> inputsf = { - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceHellingerExp DistanceHellingerExpF; -TEST_P(DistanceHellingerExpF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceHellingerExpF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001, 1024, 1024, 32, false, 1234ULL}, - {0.001, 1024, 32, 1024, false, 1234ULL}, - {0.001, 32, 1024, 1024, false, 1234ULL}, - {0.003, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceHellingerExp DistanceHellingerExpD; -TEST_P(DistanceHellingerExpD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceHellingerExpD, ::testing::ValuesIn(inputsd)); - -class BigMatrixHellingerExp - : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixHellingerExp, Result) {} -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_inner_product.cu b/cpp/test/distance/dist_inner_product.cu deleted file mode 100644 index 8dd7ef0874..0000000000 --- a/cpp/test/distance/dist_inner_product.cu +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceInnerProduct - : public DistanceTest {}; - -const std::vector> inputsf = { - {0.001f, 10, 5, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceInnerProduct DistanceInnerProductF; -TEST_P(DistanceInnerProductF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - - ASSERT_TRUE(devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceInnerProductF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceInnerProduct DistanceInnerProductD; -TEST_P(DistanceInnerProductD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - - ASSERT_TRUE(devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceInnerProductD, ::testing::ValuesIn(inputsd)); - -class BigMatrixInnerProduct - : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixInnerProduct, Result) {} - -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_jensen_shannon.cu b/cpp/test/distance/dist_jensen_shannon.cu deleted file mode 100644 index e0e256c925..0000000000 --- a/cpp/test/distance/dist_jensen_shannon.cu +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceJensenShannon - : public DistanceTest {}; - -const std::vector> inputsf = { - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceJensenShannon DistanceJensenShannonF; -TEST_P(DistanceJensenShannonF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceJensenShannonF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001, 1024, 1024, 32, false, 1234ULL}, - {0.001, 1024, 32, 1024, false, 1234ULL}, - {0.001, 32, 1024, 1024, false, 1234ULL}, - {0.003, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceJensenShannon DistanceJensenShannonD; -TEST_P(DistanceJensenShannonD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceJensenShannonD, ::testing::ValuesIn(inputsd)); - -class BigMatrixJensenShannon - : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixJensenShannon, Result) {} -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_kl_divergence.cu b/cpp/test/distance/dist_kl_divergence.cu deleted file mode 100644 index 1f79ebcad4..0000000000 --- a/cpp/test/distance/dist_kl_divergence.cu +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceKLDivergence - : public DistanceTest {}; - -const std::vector> inputsf = { - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceKLDivergence DistanceKLDivergenceF; -TEST_P(DistanceKLDivergenceF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceKLDivergenceF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001, 1024, 1024, 32, false, 1234ULL}, - {0.001, 1024, 32, 1024, false, 1234ULL}, - {0.001, 32, 1024, 1024, false, 1234ULL}, - {0.003, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceKLDivergence DistanceKLDivergenceD; -TEST_P(DistanceKLDivergenceD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceKLDivergenceD, ::testing::ValuesIn(inputsd)); - -class BigMatrixKLDivergence - : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixKLDivergence, Result) {} -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_l1.cu b/cpp/test/distance/dist_l1.cu deleted file mode 100644 index ce62a4aeec..0000000000 --- a/cpp/test/distance/dist_l1.cu +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceUnexpL1 : public DistanceTest {}; - -const std::vector> inputsf = { - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceUnexpL1 DistanceUnexpL1F; -TEST_P(DistanceUnexpL1F, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceUnexpL1F, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001, 1024, 1024, 32, false, 1234ULL}, - {0.001, 1024, 32, 1024, false, 1234ULL}, - {0.001, 32, 1024, 1024, false, 1234ULL}, - {0.003, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceUnexpL1 DistanceUnexpL1D; -TEST_P(DistanceUnexpL1D, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceUnexpL1D, ::testing::ValuesIn(inputsd)); - -class BigMatrixUnexpL1 : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixUnexpL1, Result) {} - -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_l2_exp.cu b/cpp/test/distance/dist_l2_exp.cu deleted file mode 100644 index 0203d9ed9d..0000000000 --- a/cpp/test/distance/dist_l2_exp.cu +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (c) 2018-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceEucExpTest : public DistanceTest { -}; - -template -class DistanceEucExpTestXequalY - : public DistanceTestSameBuffer {}; - -const std::vector> inputsf = { - {0.001f, 128, (65536 + 128) * 128, 8, true, 1234ULL}, - {0.001f, 2048, 4096, 128, true, 1234ULL}, - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.003f, 1021, 1021, 1021, true, 1234ULL}, - {0.001f, (65536 + 128) * 128, 128, 8, false, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, - {0.003f, 1021, 1021, 1021, false, 1234ULL}, -}; - -const std::vector> inputsXeqYf = { - {0.01f, 2048, 4096, 128, true, 1234ULL}, - {0.01f, 1024, 1024, 32, true, 1234ULL}, - {0.01f, 1024, 32, 1024, true, 1234ULL}, - {0.01f, 32, 1024, 1024, true, 1234ULL}, - {0.03f, 1024, 1024, 1024, true, 1234ULL}, - {0.03f, 1021, 1021, 1021, true, 1234ULL}, - {0.01f, 1024, 1024, 32, false, 1234ULL}, - {0.01f, 1024, 32, 1024, false, 1234ULL}, - {0.01f, 32, 1024, 1024, false, 1234ULL}, - {0.03f, 1024, 1024, 1024, false, 1234ULL}, - {0.03f, 1021, 1021, 1021, false, 1234ULL}, -}; - -typedef DistanceEucExpTest DistanceEucExpTestF; -TEST_P(DistanceEucExpTestF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceEucExpTestF, ::testing::ValuesIn(inputsf)); - -typedef DistanceEucExpTestXequalY DistanceEucExpTestXequalYF; -TEST_P(DistanceEucExpTestXequalYF, Result) -{ - int m = params.m; - ASSERT_TRUE(raft::devArrMatch(dist_ref[0].data(), - dist[0].data(), - m, - m, - raft::CompareApprox(params.tolerance), - stream)); - ASSERT_TRUE(raft::devArrMatch(dist_ref[1].data(), - dist[1].data(), - m / 2, - m, - raft::CompareApprox(params.tolerance), - stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, - DistanceEucExpTestXequalYF, - ::testing::ValuesIn(inputsXeqYf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001, 1024, 1024, 32, false, 1234ULL}, - {0.001, 1024, 32, 1024, false, 1234ULL}, - {0.001, 32, 1024, 1024, false, 1234ULL}, - {0.003, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceEucExpTest DistanceEucExpTestD; -TEST_P(DistanceEucExpTestD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceEucExpTestD, ::testing::ValuesIn(inputsd)); - -class BigMatrixEucExp : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixEucExp, Result) {} -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_l2_sqrt_exp.cu b/cpp/test/distance/dist_l2_sqrt_exp.cu deleted file mode 100644 index 5bccabcc3f..0000000000 --- a/cpp/test/distance/dist_l2_sqrt_exp.cu +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceEucSqrtExpTest - : public DistanceTest {}; - -const std::vector> inputsf = { - {0.001f, 2048, 4096, 128, true, 1234ULL}, - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.003f, 1021, 1021, 1021, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, - {0.003f, 1021, 1021, 1021, false, 1234ULL}, -}; -typedef DistanceEucSqrtExpTest DistanceEucSqrtExpTestF; -TEST_P(DistanceEucSqrtExpTestF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceEucSqrtExpTestF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001, 1024, 1024, 32, false, 1234ULL}, - {0.001, 1024, 32, 1024, false, 1234ULL}, - {0.001, 32, 1024, 1024, false, 1234ULL}, - {0.003, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceEucSqrtExpTest DistanceEucSqrtExpTestD; -TEST_P(DistanceEucSqrtExpTestD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceEucSqrtExpTestD, ::testing::ValuesIn(inputsd)); - -class BigMatrixEucSqrtExp - : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixEucSqrtExp, Result) {} -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_l2_unexp.cu b/cpp/test/distance/dist_l2_unexp.cu deleted file mode 100644 index 19b0ff6dbf..0000000000 --- a/cpp/test/distance/dist_l2_unexp.cu +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceEucUnexpTest - : public DistanceTest {}; - -const std::vector> inputsf = { - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceEucUnexpTest DistanceEucUnexpTestF; -TEST_P(DistanceEucUnexpTestF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceEucUnexpTestF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001, 1024, 1024, 32, false, 1234ULL}, - {0.001, 1024, 32, 1024, false, 1234ULL}, - {0.001, 32, 1024, 1024, false, 1234ULL}, - {0.003, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceEucUnexpTest DistanceEucUnexpTestD; -TEST_P(DistanceEucUnexpTestD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceEucUnexpTestD, ::testing::ValuesIn(inputsd)); - -class BigMatrixEucUnexp : public BigMatrixDistanceTest { -}; -TEST_F(BigMatrixEucUnexp, Result) {} -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_l_inf.cu b/cpp/test/distance/dist_l_inf.cu deleted file mode 100644 index 223d186a8d..0000000000 --- a/cpp/test/distance/dist_l_inf.cu +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceLinf : public DistanceTest {}; - -const std::vector> inputsf = { - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceLinf DistanceLinfF; -TEST_P(DistanceLinfF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceLinfF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001, 1024, 1024, 32, false, 1234ULL}, - {0.001, 1024, 32, 1024, false, 1234ULL}, - {0.001, 32, 1024, 1024, false, 1234ULL}, - {0.003, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceLinf DistanceLinfD; -TEST_P(DistanceLinfD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceLinfD, ::testing::ValuesIn(inputsd)); - -class BigMatrixLinf : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixLinf, Result) {} - -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_lp_unexp.cu b/cpp/test/distance/dist_lp_unexp.cu deleted file mode 100644 index 9d6f5921a7..0000000000 --- a/cpp/test/distance/dist_lp_unexp.cu +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceLpUnexp : public DistanceTest { -}; - -const std::vector> inputsf = { - {0.001f, 1024, 1024, 32, true, 1234ULL, 4.0f}, - {0.001f, 1024, 32, 1024, true, 1234ULL, 3.0f}, - {0.001f, 32, 1024, 1024, true, 1234ULL, 4.0f}, - {0.003f, 1024, 1024, 1024, true, 1234ULL, 3.0f}, - {0.001f, 1024, 1024, 32, false, 1234ULL, 4.0f}, - {0.001f, 1024, 32, 1024, false, 1234ULL, 3.0f}, - {0.001f, 32, 1024, 1024, false, 1234ULL, 4.0f}, - {0.003f, 1024, 1024, 1024, false, 1234ULL, 3.0f}, -}; -typedef DistanceLpUnexp DistanceLpUnexpF; -TEST_P(DistanceLpUnexpF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceLpUnexpF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL, 4.0}, - {0.001, 1024, 32, 1024, true, 1234ULL, 3.0}, - {0.001, 32, 1024, 1024, true, 1234ULL, 4.0}, - {0.003, 1024, 1024, 1024, true, 1234ULL, 3.0}, - {0.001, 1024, 1024, 32, false, 1234ULL, 4.0}, - {0.001, 1024, 32, 1024, false, 1234ULL, 3.0}, - {0.001, 32, 1024, 1024, false, 1234ULL, 4.0}, - {0.003, 1024, 1024, 1024, false, 1234ULL, 3.0}, -}; -typedef DistanceLpUnexp DistanceLpUnexpD; -TEST_P(DistanceLpUnexpD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceLpUnexpD, ::testing::ValuesIn(inputsd)); - -class BigMatrixLpUnexp : public BigMatrixDistanceTest { -}; -TEST_F(BigMatrixLpUnexp, Result) {} -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_russell_rao.cu b/cpp/test/distance/dist_russell_rao.cu deleted file mode 100644 index 73cf4b33a4..0000000000 --- a/cpp/test/distance/dist_russell_rao.cu +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceRussellRao - : public DistanceTest {}; - -const std::vector> inputsf = { - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceRussellRao DistanceRussellRaoF; -TEST_P(DistanceRussellRaoF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceRussellRaoF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001, 1024, 1024, 32, false, 1234ULL}, - {0.001, 1024, 32, 1024, false, 1234ULL}, - {0.001, 32, 1024, 1024, false, 1234ULL}, - {0.003, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceRussellRao DistanceRussellRaoD; -TEST_P(DistanceRussellRaoD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceRussellRaoD, ::testing::ValuesIn(inputsd)); - -class BigMatrixRussellRao - : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixRussellRao, Result) {} -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/distance_base.cuh b/cpp/test/distance/distance_base.cuh deleted file mode 100644 index f44fb18519..0000000000 --- a/cpp/test/distance/distance_base.cuh +++ /dev/null @@ -1,708 +0,0 @@ -/* - * Copyright (c) 2018-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" - -#include // common::nvtx::range -#include // make_device_matrix_view -#include // raft::sqrt -#include -#include // raft::resources -#include -#include // raft::distance::DistanceType -#include - -#include // rmm::device_uvector - -#include - -namespace raft { -namespace distance { - -template -RAFT_KERNEL naiveDistanceKernel(DataType* dist, - const DataType* x, - const DataType* y, - int m, - int n, - int k, - raft::distance::DistanceType type, - bool isRowMajor) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) return; - DataType acc = DataType(0); - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto diff = x[xidx] - y[yidx]; - acc += diff * diff; - } - if (type == raft::distance::DistanceType::L2SqrtExpanded || - type == raft::distance::DistanceType::L2SqrtUnexpanded) - acc = raft::sqrt(acc); - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - dist[outidx] = acc; -} - -template -RAFT_KERNEL naiveL1_Linf_CanberraDistanceKernel(DataType* dist, - const DataType* x, - const DataType* y, - int m, - int n, - int k, - raft::distance::DistanceType type, - bool isRowMajor) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) { return; } - - DataType acc = DataType(0); - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto a = x[xidx]; - auto b = y[yidx]; - auto diff = (a > b) ? (a - b) : (b - a); - if (type == raft::distance::DistanceType::Linf) { - acc = raft::max(acc, diff); - } else if (type == raft::distance::DistanceType::Canberra) { - const auto add = raft::abs(a) + raft::abs(b); - // deal with potential for 0 in denominator by - // forcing 1/0 instead - acc += ((add != 0) * diff / (add + (add == 0))); - } else { - acc += diff; - } - } - - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - dist[outidx] = acc; -} - -template -RAFT_KERNEL naiveDiceDistanceKernel( - DataType* dist, const DataType* x, const DataType* y, int m, int n, int k, bool isRowMajor) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) { return; } - - DataType acc_a = DataType(0); - DataType acc_b = DataType(0); - DataType acc_ab = DataType(0); - - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto a = x[xidx]; - auto b = y[yidx]; - acc_a += a; - acc_b += b; - acc_ab += a * b; - } - - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - - // Use 1.0 - (dice dissimilarity) to calc the distance - dist[outidx] = (DataType)1.0 - (2 * acc_ab / ((acc_a) + (acc_b))); -} - -template -RAFT_KERNEL naiveCosineDistanceKernel( - DataType* dist, const DataType* x, const DataType* y, int m, int n, int k, bool isRowMajor) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) { return; } - - DataType acc_a = DataType(0); - DataType acc_b = DataType(0); - DataType acc_ab = DataType(0); - - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto a = x[xidx]; - auto b = y[yidx]; - acc_a += a * a; - acc_b += b * b; - acc_ab += a * b; - } - - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - - // Use 1.0 - (cosine similarity) to calc the distance - dist[outidx] = (DataType)1.0 - acc_ab / (raft::sqrt(acc_a) * raft::sqrt(acc_b)); -} - -template -RAFT_KERNEL naiveInnerProductKernel( - DataType* dist, const DataType* x, const DataType* y, int m, int n, int k, bool isRowMajor) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) { return; } - - DataType acc_ab = DataType(0); - - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto a = x[xidx]; - auto b = y[yidx]; - acc_ab += a * b; - } - - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - dist[outidx] = acc_ab; -} - -template -RAFT_KERNEL naiveHellingerDistanceKernel( - DataType* dist, const DataType* x, const DataType* y, int m, int n, int k, bool isRowMajor) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) { return; } - - DataType acc_ab = DataType(0); - - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto a = x[xidx]; - auto b = y[yidx]; - acc_ab += raft::sqrt(a) * raft::sqrt(b); - } - - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - - // Adjust to replace NaN in sqrt with 0 if input to sqrt is negative - acc_ab = 1 - acc_ab; - auto rectifier = (!signbit(acc_ab)); - dist[outidx] = raft::sqrt(rectifier * acc_ab); -} - -template -RAFT_KERNEL naiveLpUnexpDistanceKernel(DataType* dist, - const DataType* x, - const DataType* y, - int m, - int n, - int k, - bool isRowMajor, - DataType p) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) return; - DataType acc = DataType(0); - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto a = x[xidx]; - auto b = y[yidx]; - auto diff = raft::abs(a - b); - acc += raft::pow(diff, p); - } - auto one_over_p = 1 / p; - acc = raft::pow(acc, one_over_p); - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - dist[outidx] = acc; -} - -template -RAFT_KERNEL naiveHammingDistanceKernel( - DataType* dist, const DataType* x, const DataType* y, int m, int n, int k, bool isRowMajor) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) return; - DataType acc = DataType(0); - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto a = x[xidx]; - auto b = y[yidx]; - acc += (a != b); - } - acc = acc / k; - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - dist[outidx] = acc; -} - -template -RAFT_KERNEL naiveJensenShannonDistanceKernel( - DataType* dist, const DataType* x, const DataType* y, int m, int n, int k, bool isRowMajor) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) return; - DataType acc = DataType(0); - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto a = x[xidx]; - auto b = y[yidx]; - - DataType m = 0.5f * (a + b); - bool a_zero = a == 0; - bool b_zero = b == 0; - - DataType p = (!a_zero * m) / (a_zero + a); - DataType q = (!b_zero * m) / (b_zero + b); - - bool p_zero = p == 0; - bool q_zero = q == 0; - - acc += (-a * (!p_zero * log(p + p_zero))) + (-b * (!q_zero * log(q + q_zero))); - } - acc = raft::sqrt(0.5f * acc); - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - dist[outidx] = acc; -} - -template -RAFT_KERNEL naiveRussellRaoDistanceKernel( - OutType* dist, const DataType* x, const DataType* y, int m, int n, int k, bool isRowMajor) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) return; - OutType acc = OutType(0); - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto a = x[xidx]; - auto b = y[yidx]; - acc += (a * b); - } - acc = (k - acc) / k; - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - dist[outidx] = acc; -} - -template -RAFT_KERNEL naiveKLDivergenceDistanceKernel( - OutType* dist, const DataType* x, const DataType* y, int m, int n, int k, bool isRowMajor) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) return; - OutType acc = OutType(0); - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto a = x[xidx]; - auto b = y[yidx]; - bool b_zero = (b == 0); - bool a_zero = (a == 0); - acc += a * (log(a + a_zero) - log(b + b_zero)); - } - acc = 0.5f * acc; - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - dist[outidx] = acc; -} - -template -RAFT_KERNEL naiveCorrelationDistanceKernel( - OutType* dist, const DataType* x, const DataType* y, int m, int n, int k, bool isRowMajor) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) return; - OutType acc = OutType(0); - auto a_norm = DataType(0); - auto b_norm = DataType(0); - auto a_sq_norm = DataType(0); - auto b_sq_norm = DataType(0); - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto a = x[xidx]; - auto b = y[yidx]; - a_norm += a; - b_norm += b; - a_sq_norm += (a * a); - b_sq_norm += (b * b); - acc += (a * b); - } - - auto numer = k * acc - (a_norm * b_norm); - auto Q_denom = k * a_sq_norm - (a_norm * a_norm); - auto R_denom = k * b_sq_norm - (b_norm * b_norm); - - acc = 1 - (numer / raft::sqrt(Q_denom * R_denom)); - - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - dist[outidx] = acc; -} - -template -void naiveDistance(DataType* dist, - const DataType* x, - const DataType* y, - int m, - int n, - int k, - raft::distance::DistanceType type, - bool isRowMajor, - DataType metric_arg = 2.0f, - cudaStream_t stream = 0) -{ - static const dim3 TPB(4, 256, 1); - dim3 nblks(raft::ceildiv(m, (int)TPB.x), raft::ceildiv(n, (int)TPB.y), 1); - - switch (type) { - case raft::distance::DistanceType::Canberra: - case raft::distance::DistanceType::Linf: - case raft::distance::DistanceType::L1: - naiveL1_Linf_CanberraDistanceKernel - <<>>(dist, x, y, m, n, k, type, isRowMajor); - break; - case raft::distance::DistanceType::L2SqrtUnexpanded: - case raft::distance::DistanceType::L2Unexpanded: - case raft::distance::DistanceType::L2SqrtExpanded: - case raft::distance::DistanceType::L2Expanded: - naiveDistanceKernel - <<>>(dist, x, y, m, n, k, type, isRowMajor); - break; - case raft::distance::DistanceType::CosineExpanded: - naiveCosineDistanceKernel - <<>>(dist, x, y, m, n, k, isRowMajor); - break; - case raft::distance::DistanceType::HellingerExpanded: - naiveHellingerDistanceKernel - <<>>(dist, x, y, m, n, k, isRowMajor); - break; - case raft::distance::DistanceType::LpUnexpanded: - naiveLpUnexpDistanceKernel - <<>>(dist, x, y, m, n, k, isRowMajor, metric_arg); - break; - case raft::distance::DistanceType::HammingUnexpanded: - naiveHammingDistanceKernel - <<>>(dist, x, y, m, n, k, isRowMajor); - break; - case raft::distance::DistanceType::InnerProduct: - naiveInnerProductKernel<<>>(dist, x, y, m, n, k, isRowMajor); - break; - case raft::distance::DistanceType::JensenShannon: - naiveJensenShannonDistanceKernel - <<>>(dist, x, y, m, n, k, isRowMajor); - break; - case raft::distance::DistanceType::RusselRaoExpanded: - naiveRussellRaoDistanceKernel - <<>>(dist, x, y, m, n, k, isRowMajor); - break; - case raft::distance::DistanceType::KLDivergence: - naiveKLDivergenceDistanceKernel - <<>>(dist, x, y, m, n, k, isRowMajor); - break; - case raft::distance::DistanceType::CorrelationExpanded: - naiveCorrelationDistanceKernel - <<>>(dist, x, y, m, n, k, isRowMajor); - break; - case raft::distance::DistanceType::DiceExpanded: - naiveDiceDistanceKernel<<>>(dist, x, y, m, n, k, isRowMajor); - break; - default: FAIL() << "should be here\n"; - } - RAFT_CUDA_TRY(cudaPeekAtLastError()); -} - -template -struct DistanceInputs { - DataType tolerance; - int m, n, k; - bool isRowMajor; - unsigned long long int seed; - DataType metric_arg = 2.0f; -}; - -template -::std::ostream& operator<<(::std::ostream& os, const DistanceInputs& dims) -{ - return os; -} - -// TODO: Remove when mdspan-based raft::runtime::distance::pairwise_distance is -// implemented. -// -// Context: -// https://github.com/rapidsai/raft/issues/1338 -template -constexpr bool layout_to_row_major(); - -template <> -constexpr bool layout_to_row_major() -{ - return true; -} -template <> -constexpr bool layout_to_row_major() -{ - return false; -} - -template -void distanceLauncher(raft::resources const& handle, - DataType* x, - DataType* y, - DataType* dist, - DataType* dist2, - int m, - int n, - int k, - DistanceInputs& params, - DataType threshold, - DataType metric_arg = 2.0f) -{ - auto x_v = make_device_matrix_view(x, m, k); - auto y_v = make_device_matrix_view(y, n, k); - auto dist_v = make_device_matrix_view(dist, m, n); - - raft::distance::distance( - handle, x_v, y_v, dist_v, metric_arg); -} - -template -class DistanceTest : public ::testing::TestWithParam> { - public: - DistanceTest() - : params(::testing::TestWithParam>::GetParam()), - stream(resource::get_cuda_stream(handle)), - x(params.m * params.k, stream), - y(params.n * params.k, stream), - dist_ref(params.m * params.n, stream), - dist(params.m * params.n, stream), - dist2(params.m * params.n, stream) - { - } - - void SetUp() override - { - auto testInfo = testing::UnitTest::GetInstance()->current_test_info(); - common::nvtx::range fun_scope("test::%s/%s", testInfo->test_suite_name(), testInfo->name()); - - raft::random::RngState r(params.seed); - int m = params.m; - int n = params.n; - int k = params.k; - DataType metric_arg = params.metric_arg; - bool isRowMajor = params.isRowMajor; - if (distanceType == raft::distance::DistanceType::HellingerExpanded || - distanceType == raft::distance::DistanceType::JensenShannon || - distanceType == raft::distance::DistanceType::KLDivergence) { - // Hellinger works only on positive numbers - uniform(handle, r, x.data(), m * k, DataType(0.0), DataType(1.0)); - uniform(handle, r, y.data(), n * k, DataType(0.0), DataType(1.0)); - } else if (distanceType == raft::distance::DistanceType::RusselRaoExpanded || - distanceType == raft::distance::DistanceType::DiceExpanded) { - uniform(handle, r, x.data(), m * k, DataType(0.0), DataType(1.0)); - uniform(handle, r, y.data(), n * k, DataType(0.0), DataType(1.0)); - // Russel rao works on boolean values. - bernoulli(handle, r, x.data(), m * k, 0.5f); - bernoulli(handle, r, y.data(), n * k, 0.5f); - } else { - uniform(handle, r, x.data(), m * k, DataType(-1.0), DataType(1.0)); - uniform(handle, r, y.data(), n * k, DataType(-1.0), DataType(1.0)); - } - naiveDistance( - dist_ref.data(), x.data(), y.data(), m, n, k, distanceType, isRowMajor, metric_arg, stream); - - DataType threshold = -10000.f; - - if (isRowMajor) { - distanceLauncher(handle, - x.data(), - y.data(), - dist.data(), - dist2.data(), - m, - n, - k, - params, - threshold, - metric_arg); - - } else { - distanceLauncher(handle, - x.data(), - y.data(), - dist.data(), - dist2.data(), - m, - n, - k, - params, - threshold, - metric_arg); - } - resource::sync_stream(handle, stream); - } - - protected: - raft::resources handle; - cudaStream_t stream; - - DistanceInputs params; - rmm::device_uvector x, y, dist_ref, dist, dist2; -}; - -/* - * This test suite verifies the path when X and Y are same buffer, - * distance metrics which requires norms like L2 expanded/cosine/correlation - * takes a more optimal path in such case to skip norm calculation for Y buffer. - * It may happen that though both X and Y are same buffer but user passes - * different dimensions for them like in case of tiled_brute_force_knn. - */ -template -class DistanceTestSameBuffer : public ::testing::TestWithParam> { - public: - using dev_vector = rmm::device_uvector; - DistanceTestSameBuffer() - : params(::testing::TestWithParam>::GetParam()), - stream(resource::get_cuda_stream(handle)), - x(params.m * params.k, stream), - dist_ref({dev_vector(params.m * params.m, stream), dev_vector(params.m * params.m, stream)}), - dist({dev_vector(params.m * params.m, stream), dev_vector(params.m * params.m, stream)}), - dist2({dev_vector(params.m * params.m, stream), dev_vector(params.m * params.m, stream)}) - { - } - - void SetUp() override - { - auto testInfo = testing::UnitTest::GetInstance()->current_test_info(); - common::nvtx::range fun_scope("test::%s/%s", testInfo->test_suite_name(), testInfo->name()); - - raft::random::RngState r(params.seed); - int m = params.m; - int n = params.m; - int k = params.k; - DataType metric_arg = params.metric_arg; - bool isRowMajor = params.isRowMajor; - if (distanceType == raft::distance::DistanceType::HellingerExpanded || - distanceType == raft::distance::DistanceType::JensenShannon || - distanceType == raft::distance::DistanceType::KLDivergence) { - // Hellinger works only on positive numbers - uniform(handle, r, x.data(), m * k, DataType(0.0), DataType(1.0)); - } else if (distanceType == raft::distance::DistanceType::RusselRaoExpanded || - distanceType == raft::distance::DistanceType::DiceExpanded) { - uniform(handle, r, x.data(), m * k, DataType(0.0), DataType(1.0)); - // Russel rao works on boolean values. - bernoulli(handle, r, x.data(), m * k, 0.5f); - } else { - uniform(handle, r, x.data(), m * k, DataType(-1.0), DataType(1.0)); - } - - for (int i = 0; i < 2; i++) { - // both X and Y are same buffer but when i = 1 - // different dimensions for x & y is passed. - m = m / (i + 1); - naiveDistance(dist_ref[i].data(), - x.data(), - x.data(), - m, - n, - k, - distanceType, - isRowMajor, - metric_arg, - stream); - - DataType threshold = -10000.f; - - if (isRowMajor) { - distanceLauncher(handle, - x.data(), - x.data(), - dist[i].data(), - dist2[i].data(), - m, - n, - k, - params, - threshold, - metric_arg); - - } else { - distanceLauncher(handle, - x.data(), - x.data(), - dist[i].data(), - dist2[i].data(), - m, - n, - k, - params, - threshold, - metric_arg); - } - } - resource::sync_stream(handle, stream); - } - - protected: - raft::resources handle; - cudaStream_t stream; - - DistanceInputs params; - dev_vector x; - static const int N = 2; - std::array dist_ref, dist, dist2; -}; - -template -class BigMatrixDistanceTest : public ::testing::Test { - public: - BigMatrixDistanceTest() - : x(m * k, resource::get_cuda_stream(handle)), - dist(std::size_t(m) * m, resource::get_cuda_stream(handle)){}; - void SetUp() override - { - auto testInfo = testing::UnitTest::GetInstance()->current_test_info(); - common::nvtx::range fun_scope("test::%s/%s", testInfo->test_suite_name(), testInfo->name()); - - void pairwise_distance(raft::resources const& handle, - float* x, - float* y, - float* dists, - int m, - int n, - int k, - raft::distance::DistanceType metric, - bool isRowMajor, - float metric_arg); - constexpr bool row_major = true; - constexpr float metric_arg = 0.0f; - raft::distance::distance( - handle, x.data(), x.data(), dist.data(), m, n, k, row_major, metric_arg); - RAFT_CUDA_TRY(cudaStreamSynchronize(resource::get_cuda_stream(handle))); - } - - protected: - raft::resources handle; - int m = 48000; - int n = 48000; - int k = 1; - rmm::device_uvector x, dist; -}; -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/fused_cosine_nn.cu b/cpp/test/distance/fused_cosine_nn.cu deleted file mode 100644 index d4d632e1dc..0000000000 --- a/cpp/test/distance/fused_cosine_nn.cu +++ /dev/null @@ -1,420 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#undef RAFT_EXPLICIT_INSTANTIATE_ONLY // Search with filter instantiation - -#include "../test_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace raft { -namespace distance { - -template -struct RaftKVPMinReduce { - typedef raft::KeyValuePair KVP; - - DI KVP operator()(LabelT rit, const KVP& a, const KVP& b) { return b.value < a.value ? b : a; } - - DI KVP operator()(const KVP& a, const KVP& b) { return b.value < a.value ? b : a; } - -}; // KVPMinReduce - -template -__global__ void naiveCosKernel(raft::KeyValuePair* min, - DataT* x, - DataT* y, - int m, - int n, - int k, - int* workspace, - DataT maxVal) -{ - int midx = threadIdx.y + blockIdx.y * blockDim.y; - int nidx = threadIdx.x + blockIdx.x * blockDim.x; - DataT acc_a = DataT(0); - DataT acc_b = DataT(0); - DataT acc_ab = DataT(0); - // if (midx >= m || nidx >= n) { return; } - - for (int i = 0; i < k; ++i) { - int xidx = i + midx * k; - int yidx = i + nidx * k; - auto a = x[xidx]; - auto b = y[yidx]; - acc_a += a * a; - acc_b += b * b; - acc_ab += a * b; - } - - // Use 1.0 - (cosine similarity) to calc the distance - DataT acc = maxVal; - if (midx < m || nidx < n) { acc = (DataT)1.0 - acc_ab / (raft::sqrt(acc_a) * raft::sqrt(acc_b)); } - - ReduceOpT redOp; - typedef cub::WarpReduce> WarpReduce; - __shared__ typename WarpReduce::TempStorage temp[NWARPS]; - int warpId = threadIdx.x / raft::WarpSize; - raft::KeyValuePair tmp; - tmp.key = nidx; - tmp.value = midx >= m || nidx >= n ? maxVal : acc; - tmp = WarpReduce(temp[warpId]).Reduce(tmp, RaftKVPMinReduce()); - if (threadIdx.x % raft::WarpSize == 0 && midx < m) { - while (atomicCAS(workspace + midx, 0, 1) == 1) - ; - __threadfence(); - redOp(midx, min + midx, tmp); - __threadfence(); - atomicCAS(workspace + midx, 1, 0); - } -} - -template -void naive(raft::KeyValuePair* min, - DataT* x, - DataT* y, - int m, - int n, - int k, - int* workspace, - cudaStream_t stream) -{ - static const dim3 TPB(32, 16, 1); - dim3 nblks(raft::ceildiv(n, (int)TPB.x), raft::ceildiv(m, (int)TPB.y), 1); - RAFT_CUDA_TRY(cudaMemsetAsync(workspace, 0, sizeof(int) * m, stream)); - auto blks = raft::ceildiv(m, 256); - MinAndDistanceReduceOp op; - detail::initKernel, int> - <<>>(min, m, std::numeric_limits::max(), op); - RAFT_CUDA_TRY(cudaGetLastError()); - naiveCosKernel, 16> - <<>>(min, x, y, m, n, k, workspace, std::numeric_limits::max()); - RAFT_CUDA_TRY(cudaGetLastError()); -} - -template -struct Inputs { - DataT tolerance; - int m, n, k; - unsigned long long int seed; - - friend std::ostream& operator<<(std::ostream& os, const Inputs& p) - { - return os << "m: " << p.m - << ", " - "n: " - << p.n - << ", " - "k: " - << p.k - << ", " - "seed: " - << p.seed - << ", " - "tol: " - << p.tolerance; - } -}; - -template -class FusedCosineNNTest : public ::testing::TestWithParam> { - public: - FusedCosineNNTest() - : params(::testing::TestWithParam>::GetParam()), - stream(resource::get_cuda_stream(handle)), - x(params.m * params.k, stream), - y(params.n * params.k, stream), - xn(params.m, stream), - yn(params.n, stream), - min(params.m, stream), - min_ref(params.m, stream), - workspace(params.m * sizeof(int), stream) - { - } - - protected: - void SetUp() override - { - raft::random::RngState r(params.seed); - int m = params.m; - int n = params.n; - int k = params.k; - uniform(handle, r, x.data(), m * k, DataT(-1.0), DataT(1.0)); - uniform(handle, r, y.data(), n * k, DataT(-1.0), DataT(1.0)); - generateGoldenResult(); - raft::linalg::rowNorm( - xn.data(), x.data(), k, m, raft::linalg::L2Norm, true, stream, raft::sqrt_op{}); - raft::linalg::rowNorm( - yn.data(), y.data(), k, n, raft::linalg::L2Norm, true, stream, raft::sqrt_op{}); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - } - - protected: - raft::resources handle; - cudaStream_t stream; - Inputs params; - rmm::device_uvector x; - rmm::device_uvector y; - rmm::device_uvector xn; - rmm::device_uvector yn; - rmm::device_uvector> min; - rmm::device_uvector> min_ref; - rmm::device_uvector workspace; - - virtual void generateGoldenResult() - { - int m = params.m; - int n = params.n; - int k = params.k; - naive(min_ref.data(), x.data(), y.data(), m, n, k, (int*)workspace.data(), stream); - } - - void runTest(raft::KeyValuePair* out) - { - int m = params.m; - int n = params.n; - int k = params.k; - raft::distance::DistanceType metric = raft::distance::DistanceType::CosineExpanded; - constexpr bool init_out_buffer = true; - fusedDistanceNNMinReduce, int>(out, - x.data(), - y.data(), - xn.data(), - yn.data(), - m, - n, - k, - (void*)workspace.data(), - false, - init_out_buffer, - true, - metric, - 0.0f, - stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - } -}; - -template -struct CompareApproxAbsKVP { - typedef typename raft::KeyValuePair KVP; - CompareApproxAbsKVP(T eps_) : eps(eps_) {} - bool operator()(const KVP& a, const KVP& b) const - { - T diff = std::abs(std::abs(a.value) - std::abs(b.value)); - T m = std::max(std::abs(a.value), std::abs(b.value)); - T ratio = m >= eps ? diff / m : diff; - return (ratio <= eps); - } - - private: - T eps; -}; - -template -struct CompareExactKVP { - typedef typename raft::KeyValuePair KVP; - bool operator()(const KVP& a, const KVP& b) const - { - if (a.value != b.value) return false; - return true; - } -}; - -template -::testing::AssertionResult devArrMatch(const raft::KeyValuePair* expected, - const raft::KeyValuePair* actual, - size_t size, - L eq_compare, - cudaStream_t stream = 0) -{ - typedef typename raft::KeyValuePair KVP; - std::shared_ptr exp_h(new KVP[size]); - std::shared_ptr act_h(new KVP[size]); - raft::update_host(exp_h.get(), expected, size, stream); - raft::update_host(act_h.get(), actual, size, stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - for (size_t i(0); i < size; ++i) { - auto exp = exp_h.get()[i]; - auto act = act_h.get()[i]; - if (!eq_compare(exp, act)) { - return ::testing::AssertionFailure() - << "actual=" << act.key << "," << act.value << " != expected=" << exp.key << "," - << exp.value << " @" << i; - } - } - return ::testing::AssertionSuccess(); -} - -const std::vector> inputsf = { - {0.001f, 32, 32, 32, 1234ULL}, - {0.001f, 32, 64, 32, 1234ULL}, - {0.001f, 64, 32, 32, 1234ULL}, - {0.001f, 64, 64, 32, 1234ULL}, - {0.001f, 128, 32, 32, 1234ULL}, - {0.001f, 128, 64, 32, 1234ULL}, - {0.001f, 128, 128, 64, 1234ULL}, - {0.001f, 64, 128, 128, 1234ULL}, - - {0.001f, 32, 32, 34, 1234ULL}, - {0.001f, 32, 64, 34, 1234ULL}, - {0.001f, 64, 32, 34, 1234ULL}, - {0.001f, 64, 64, 34, 1234ULL}, - {0.001f, 128, 32, 34, 1234ULL}, - {0.001f, 128, 64, 34, 1234ULL}, - {0.001f, 128, 128, 66, 1234ULL}, - {0.001f, 64, 128, 130, 1234ULL}, - - {0.001f, 32, 32, 33, 1234ULL}, - {0.001f, 32, 64, 33, 1234ULL}, - {0.001f, 64, 32, 33, 1234ULL}, - {0.001f, 64, 64, 33, 1234ULL}, - {0.001f, 128, 32, 33, 1234ULL}, - {0.001f, 128, 64, 33, 1234ULL}, - {0.001f, 128, 128, 65, 1234ULL}, - {0.001f, 64, 128, 129, 1234ULL}, - {0.006f, 1805, 134, 2, 1234ULL}, - {0.006f, 8192, 1024, 64, 1234ULL}, - {0.006f, 8192, 1025, 64, 1234ULL}, - - // Repeat with smaller values of k - {0.006f, 32, 32, 1, 1234ULL}, - {0.001f, 32, 64, 2, 1234ULL}, - {0.001f, 64, 32, 3, 1234ULL}, - {0.001f, 64, 64, 4, 1234ULL}, - {0.001f, 128, 32, 5, 1234ULL}, - {0.001f, 128, 64, 6, 1234ULL}, - {0.001f, 128, 128, 7, 1234ULL}, - {0.001f, 64, 128, 8, 1234ULL}, - - {0.001f, 32, 32, 9, 1234ULL}, - {0.001f, 32, 64, 10, 1234ULL}, - {0.001f, 64, 32, 11, 1234ULL}, - {0.001f, 64, 64, 12, 1234ULL}, - {0.001f, 128, 32, 13, 1234ULL}, - {0.001f, 128, 64, 14, 1234ULL}, - {0.001f, 128, 128, 15, 1234ULL}, - {0.001f, 64, 128, 16, 1234ULL}, - - {0.001f, 32, 32, 17, 1234ULL}, - {0.001f, 32, 64, 18, 1234ULL}, - {0.001f, 64, 32, 19, 1234ULL}, - {0.001f, 64, 64, 20, 1234ULL}, - {0.001f, 128, 32, 21, 1234ULL}, - {0.001f, 128, 64, 22, 1234ULL}, - {0.001f, 128, 128, 23, 1234ULL}, - {0.00001, 64, 128, 24, 1234ULL}, - {0.001f, 1805, 134, 25, 1234ULL}, - {0.006f, 8192, 1024, 25, 1234ULL}, - {0.006f, 8192, 1024, 66, 1234ULL}, -}; -typedef FusedCosineNNTest FusedCosineNNTestF; -TEST_P(FusedCosineNNTestF, Result) -{ - runTest(min.data()); - ASSERT_TRUE(devArrMatch( - min_ref.data(), min.data(), params.m, CompareApproxAbsKVP(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(FusedCosineNNTests, FusedCosineNNTestF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.00001, 32, 32, 32, 1234ULL}, {0.00001, 32, 64, 32, 1234ULL}, - {0.00001, 64, 32, 32, 1234ULL}, {0.00001, 64, 64, 32, 1234ULL}, - {0.00001, 128, 32, 32, 1234ULL}, {0.00001, 128, 64, 32, 1234ULL}, - {0.00001, 128, 128, 64, 1234ULL}, {0.00001, 64, 128, 128, 1234ULL}, - - {0.00001, 32, 32, 34, 1234ULL}, {0.00001, 32, 64, 34, 1234ULL}, - {0.00001, 64, 32, 34, 1234ULL}, {0.00001, 64, 64, 34, 1234ULL}, - {0.00001, 128, 32, 34, 1234ULL}, {0.00001, 128, 64, 34, 1234ULL}, - {0.00001, 128, 128, 66, 1234ULL}, {0.00001, 64, 128, 130, 1234ULL}, - - {0.00001, 32, 32, 33, 1234ULL}, {0.00001, 32, 64, 33, 1234ULL}, - {0.00001, 64, 32, 33, 1234ULL}, {0.00001, 64, 64, 33, 1234ULL}, - {0.00001, 128, 32, 33, 1234ULL}, {0.00001, 128, 64, 33, 1234ULL}, - {0.00001, 128, 128, 65, 1234ULL}, {0.00001, 64, 128, 129, 1234ULL}, - - {0.00001, 1805, 134, 2, 1234ULL}, {0.00001, 8192, 1024, 25, 1234ULL}, -}; -typedef FusedCosineNNTest FusedCosineNNTestD; -TEST_P(FusedCosineNNTestD, Result) -{ - runTest(min.data()); - ASSERT_TRUE(devArrMatch( - min_ref.data(), min.data(), params.m, CompareApproxAbsKVP(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(FusedCosineNNTests, FusedCosineNNTestD, ::testing::ValuesIn(inputsd)); - -/// This is to test output determinism of the prim -template -class FusedCosineNNDetTest : public FusedCosineNNTest { - public: - FusedCosineNNDetTest() : stream(resource::get_cuda_stream(handle)), min1(0, stream) {} - - void SetUp() override - { - FusedCosineNNTest::SetUp(); - int m = this->params.m; - min1.resize(m, stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - } - - void TearDown() override { FusedCosineNNTest::TearDown(); } - - protected: - raft::resources handle; - cudaStream_t stream; - - rmm::device_uvector> min1; - - static const int NumRepeats = 3; - - void generateGoldenResult() override {} -}; - -typedef FusedCosineNNDetTest FusedCosineNNDetTestF; -TEST_P(FusedCosineNNDetTestF, Result) -{ - runTest(min.data()); // assumed to be golden - for (int i = 0; i < NumRepeats; ++i) { - runTest(min1.data()); - ASSERT_TRUE(devArrMatch(min.data(), min1.data(), params.m, CompareExactKVP(), stream)); - cudaMemsetAsync(min1.data(), 0, sizeof(*min.data()) * params.m, stream); - } -} -INSTANTIATE_TEST_CASE_P(FusedCosineNNDetTests, FusedCosineNNDetTestF, ::testing::ValuesIn(inputsf)); - -typedef FusedCosineNNDetTest FusedCosineNNDetTestD; -TEST_P(FusedCosineNNDetTestD, Result) -{ - runTest(min.data()); // assumed to be golden - for (int i = 0; i < NumRepeats; ++i) { - runTest(min1.data()); - ASSERT_TRUE(devArrMatch(min.data(), min1.data(), params.m, CompareExactKVP(), stream)); - } -} -INSTANTIATE_TEST_CASE_P(FusedCosineNNDetTests, FusedCosineNNDetTestD, ::testing::ValuesIn(inputsd)); - -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/fused_l2_nn.cu b/cpp/test/distance/fused_l2_nn.cu deleted file mode 100644 index 6fd8f15808..0000000000 --- a/cpp/test/distance/fused_l2_nn.cu +++ /dev/null @@ -1,437 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace raft { -namespace distance { - -template -struct RaftKVPMinReduce { - typedef raft::KeyValuePair KVP; - - DI KVP operator()(LabelT rit, const KVP& a, const KVP& b) { return b.value < a.value ? b : a; } - - DI KVP operator()(const KVP& a, const KVP& b) { return b.value < a.value ? b : a; } - -}; // KVPMinReduce - -template -RAFT_KERNEL naiveKernel(raft::KeyValuePair* min, - DataT* x, - DataT* y, - int m, - int n, - int k, - int* workspace, - DataT maxVal) -{ - int midx = threadIdx.y + blockIdx.y * blockDim.y; - int nidx = threadIdx.x + blockIdx.x * blockDim.x; - DataT acc = DataT(0); - for (int i = 0; i < k; ++i) { - int xidx = i + midx * k; - int yidx = i + nidx * k; - auto diff = midx >= m || nidx >= n ? DataT(0) : x[xidx] - y[yidx]; - acc += diff * diff; - } - - if (Sqrt) { acc = raft::sqrt(acc); } - ReduceOpT redOp; - typedef cub::WarpReduce> WarpReduce; - __shared__ typename WarpReduce::TempStorage temp[NWARPS]; - int warpId = threadIdx.x / raft::WarpSize; - raft::KeyValuePair tmp; - tmp.key = nidx; - tmp.value = midx >= m || nidx >= n ? maxVal : acc; - tmp = WarpReduce(temp[warpId]).Reduce(tmp, RaftKVPMinReduce()); - if (threadIdx.x % raft::WarpSize == 0 && midx < m) { - while (atomicCAS(workspace + midx, 0, 1) == 1) - ; - __threadfence(); - redOp(midx, min + midx, tmp); - __threadfence(); - atomicCAS(workspace + midx, 1, 0); - } -} - -template -void naive(raft::KeyValuePair* min, - DataT* x, - DataT* y, - int m, - int n, - int k, - int* workspace, - cudaStream_t stream) -{ - static const dim3 TPB(32, 16, 1); - dim3 nblks(raft::ceildiv(n, (int)TPB.x), raft::ceildiv(m, (int)TPB.y), 1); - RAFT_CUDA_TRY(cudaMemsetAsync(workspace, 0, sizeof(int) * m, stream)); - auto blks = raft::ceildiv(m, 256); - MinAndDistanceReduceOp op; - detail::initKernel, int> - <<>>(min, m, std::numeric_limits::max(), op); - RAFT_CUDA_TRY(cudaGetLastError()); - naiveKernel, 16> - <<>>(min, x, y, m, n, k, workspace, std::numeric_limits::max()); - RAFT_CUDA_TRY(cudaGetLastError()); -} - -template -struct Inputs { - DataT tolerance; - int m, n, k; - unsigned long long int seed; - - friend std::ostream& operator<<(std::ostream& os, const Inputs& p) - { - return os << "m: " << p.m - << ", " - "n: " - << p.n - << ", " - "k: " - << p.k - << ", " - "seed: " - << p.seed - << ", " - "tol: " - << p.tolerance; - } -}; - -template -class FusedL2NNTest : public ::testing::TestWithParam> { - public: - FusedL2NNTest() - : params(::testing::TestWithParam>::GetParam()), - stream(resource::get_cuda_stream(handle)), - x(params.m * params.k, stream), - y(params.n * params.k, stream), - xn(params.m, stream), - yn(params.n, stream), - min(params.m, stream), - min_ref(params.m, stream), - workspace(params.m * sizeof(int), stream) - { - } - - protected: - void SetUp() override - { - raft::random::RngState r(params.seed); - int m = params.m; - int n = params.n; - int k = params.k; - uniform(handle, r, x.data(), m * k, DataT(-1.0), DataT(1.0)); - uniform(handle, r, y.data(), n * k, DataT(-1.0), DataT(1.0)); - generateGoldenResult(); - raft::linalg::rowNorm(xn.data(), x.data(), k, m, raft::linalg::L2Norm, true, stream); - raft::linalg::rowNorm(yn.data(), y.data(), k, n, raft::linalg::L2Norm, true, stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - } - - protected: - raft::resources handle; - cudaStream_t stream; - Inputs params; - rmm::device_uvector x; - rmm::device_uvector y; - rmm::device_uvector xn; - rmm::device_uvector yn; - rmm::device_uvector> min; - rmm::device_uvector> min_ref; - rmm::device_uvector workspace; - - virtual void generateGoldenResult() - { - int m = params.m; - int n = params.n; - int k = params.k; - naive(min_ref.data(), x.data(), y.data(), m, n, k, (int*)workspace.data(), stream); - } - - void runTest(raft::KeyValuePair* out) - { - int m = params.m; - int n = params.n; - int k = params.k; - - const bool init_out_buffer = true; - fusedL2NNMinReduce, int>(out, - x.data(), - y.data(), - xn.data(), - yn.data(), - m, - n, - k, - (void*)workspace.data(), - Sqrt, - init_out_buffer, - stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - } -}; - -template -struct CompareApproxAbsKVP { - typedef typename raft::KeyValuePair KVP; - CompareApproxAbsKVP(T eps_) : eps(eps_) {} - bool operator()(const KVP& a, const KVP& b) const - { - T diff = std::abs(std::abs(a.value) - std::abs(b.value)); - T m = std::max(std::abs(a.value), std::abs(b.value)); - T ratio = m >= eps ? diff / m : diff; - return (ratio <= eps); - } - - private: - T eps; -}; - -template -struct CompareExactKVP { - typedef typename raft::KeyValuePair KVP; - bool operator()(const KVP& a, const KVP& b) const - { - if (a.value != b.value) return false; - return true; - } -}; - -template -::testing::AssertionResult devArrMatch(const raft::KeyValuePair* expected, - const raft::KeyValuePair* actual, - size_t size, - L eq_compare, - cudaStream_t stream = 0) -{ - typedef typename raft::KeyValuePair KVP; - std::shared_ptr exp_h(new KVP[size]); - std::shared_ptr act_h(new KVP[size]); - raft::update_host(exp_h.get(), expected, size, stream); - raft::update_host(act_h.get(), actual, size, stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - for (size_t i(0); i < size; ++i) { - auto exp = exp_h.get()[i]; - auto act = act_h.get()[i]; - if (!eq_compare(exp, act)) { - return ::testing::AssertionFailure() - << "actual=" << act.key << "," << act.value << " != expected=" << exp.key << "," - << exp.value << " @" << i; - } - } - return ::testing::AssertionSuccess(); -} - -const std::vector> inputsf = { - {0.001f, 32, 32, 32, 1234ULL}, - {0.001f, 32, 64, 32, 1234ULL}, - {0.001f, 64, 32, 32, 1234ULL}, - {0.001f, 64, 64, 32, 1234ULL}, - {0.001f, 128, 32, 32, 1234ULL}, - {0.001f, 128, 64, 32, 1234ULL}, - {0.001f, 128, 128, 64, 1234ULL}, - {0.001f, 64, 128, 128, 1234ULL}, - - {0.001f, 32, 32, 34, 1234ULL}, - {0.001f, 32, 64, 34, 1234ULL}, - {0.001f, 64, 32, 34, 1234ULL}, - {0.001f, 64, 64, 34, 1234ULL}, - {0.001f, 128, 32, 34, 1234ULL}, - {0.001f, 128, 64, 34, 1234ULL}, - {0.001f, 128, 128, 66, 1234ULL}, - {0.001f, 64, 128, 130, 1234ULL}, - - {0.001f, 32, 32, 33, 1234ULL}, - {0.001f, 32, 64, 33, 1234ULL}, - {0.001f, 64, 32, 33, 1234ULL}, - {0.001f, 64, 64, 33, 1234ULL}, - {0.001f, 128, 32, 33, 1234ULL}, - {0.001f, 128, 64, 33, 1234ULL}, - {0.001f, 128, 128, 65, 1234ULL}, - {0.001f, 64, 128, 129, 1234ULL}, - {0.006f, 1805, 134, 2, 1234ULL}, - {0.006f, 8192, 1024, 64, 1234ULL}, - {0.006f, 8192, 1025, 64, 1234ULL}, - - // Repeat with smaller values of k - {0.006f, 32, 32, 1, 1234ULL}, - {0.001f, 32, 64, 2, 1234ULL}, - {0.001f, 64, 32, 3, 1234ULL}, - {0.001f, 64, 64, 4, 1234ULL}, - {0.001f, 128, 32, 5, 1234ULL}, - {0.001f, 128, 64, 6, 1234ULL}, - {0.001f, 128, 128, 7, 1234ULL}, - {0.001f, 64, 128, 8, 1234ULL}, - - {0.001f, 32, 32, 9, 1234ULL}, - {0.001f, 32, 64, 10, 1234ULL}, - {0.001f, 64, 32, 11, 1234ULL}, - {0.001f, 64, 64, 12, 1234ULL}, - {0.001f, 128, 32, 13, 1234ULL}, - {0.001f, 128, 64, 14, 1234ULL}, - {0.001f, 128, 128, 15, 1234ULL}, - {0.001f, 64, 128, 16, 1234ULL}, - - {0.001f, 32, 32, 17, 1234ULL}, - {0.001f, 32, 64, 18, 1234ULL}, - {0.001f, 64, 32, 19, 1234ULL}, - {0.001f, 64, 64, 20, 1234ULL}, - {0.001f, 128, 32, 21, 1234ULL}, - {0.001f, 128, 64, 22, 1234ULL}, - {0.001f, 128, 128, 23, 1234ULL}, - {0.00001, 64, 128, 24, 1234ULL}, - {0.001f, 1805, 134, 25, 1234ULL}, - {0.006f, 8192, 1024, 25, 1234ULL}, - {0.006f, 8192, 1024, 66, 1234ULL}, -}; -typedef FusedL2NNTest FusedL2NNTestF_Sq; -TEST_P(FusedL2NNTestF_Sq, Result) -{ - runTest(min.data()); - ASSERT_TRUE(devArrMatch( - min_ref.data(), min.data(), params.m, CompareApproxAbsKVP(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(FusedL2NNTests, FusedL2NNTestF_Sq, ::testing::ValuesIn(inputsf)); -typedef FusedL2NNTest FusedL2NNTestF_Sqrt; -TEST_P(FusedL2NNTestF_Sqrt, Result) -{ - runTest(min.data()); - ASSERT_TRUE(devArrMatch( - min_ref.data(), min.data(), params.m, CompareApproxAbsKVP(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(FusedL2NNTests, FusedL2NNTestF_Sqrt, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.00001, 32, 32, 32, 1234ULL}, {0.00001, 32, 64, 32, 1234ULL}, - {0.00001, 64, 32, 32, 1234ULL}, {0.00001, 64, 64, 32, 1234ULL}, - {0.00001, 128, 32, 32, 1234ULL}, {0.00001, 128, 64, 32, 1234ULL}, - {0.00001, 128, 128, 64, 1234ULL}, {0.00001, 64, 128, 128, 1234ULL}, - - {0.00001, 32, 32, 34, 1234ULL}, {0.00001, 32, 64, 34, 1234ULL}, - {0.00001, 64, 32, 34, 1234ULL}, {0.00001, 64, 64, 34, 1234ULL}, - {0.00001, 128, 32, 34, 1234ULL}, {0.00001, 128, 64, 34, 1234ULL}, - {0.00001, 128, 128, 66, 1234ULL}, {0.00001, 64, 128, 130, 1234ULL}, - - {0.00001, 32, 32, 33, 1234ULL}, {0.00001, 32, 64, 33, 1234ULL}, - {0.00001, 64, 32, 33, 1234ULL}, {0.00001, 64, 64, 33, 1234ULL}, - {0.00001, 128, 32, 33, 1234ULL}, {0.00001, 128, 64, 33, 1234ULL}, - {0.00001, 128, 128, 65, 1234ULL}, {0.00001, 64, 128, 129, 1234ULL}, - - {0.00001, 1805, 134, 2, 1234ULL}, //{0.00001, 8192, 1024, 25, 1234ULL}, -}; -typedef FusedL2NNTest FusedL2NNTestD_Sq; -TEST_P(FusedL2NNTestD_Sq, Result) -{ - runTest(min.data()); - ASSERT_TRUE(devArrMatch( - min_ref.data(), min.data(), params.m, CompareApproxAbsKVP(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(FusedL2NNTests, FusedL2NNTestD_Sq, ::testing::ValuesIn(inputsd)); -typedef FusedL2NNTest FusedL2NNTestD_Sqrt; -TEST_P(FusedL2NNTestD_Sqrt, Result) -{ - runTest(min.data()); - ASSERT_TRUE(devArrMatch( - min_ref.data(), min.data(), params.m, CompareApproxAbsKVP(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(FusedL2NNTests, FusedL2NNTestD_Sqrt, ::testing::ValuesIn(inputsd)); - -/// This is to test output determinism of the prim -template -class FusedL2NNDetTest : public FusedL2NNTest { - public: - FusedL2NNDetTest() : stream(resource::get_cuda_stream(handle)), min1(0, stream) {} - - void SetUp() override - { - FusedL2NNTest::SetUp(); - int m = this->params.m; - min1.resize(m, stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - } - - void TearDown() override { FusedL2NNTest::TearDown(); } - - protected: - raft::resources handle; - cudaStream_t stream; - - rmm::device_uvector> min1; - - static const int NumRepeats = 3; - - void generateGoldenResult() override {} -}; - -typedef FusedL2NNDetTest FusedL2NNDetTestF_Sq; -TEST_P(FusedL2NNDetTestF_Sq, Result) -{ - runTest(min.data()); // assumed to be golden - for (int i = 0; i < NumRepeats; ++i) { - runTest(min1.data()); - ASSERT_TRUE(devArrMatch(min.data(), min1.data(), params.m, CompareExactKVP(), stream)); - } -} -INSTANTIATE_TEST_CASE_P(FusedL2NNDetTests, FusedL2NNDetTestF_Sq, ::testing::ValuesIn(inputsf)); -typedef FusedL2NNDetTest FusedL2NNDetTestF_Sqrt; -TEST_P(FusedL2NNDetTestF_Sqrt, Result) -{ - runTest(min.data()); // assumed to be golden - for (int i = 0; i < NumRepeats; ++i) { - runTest(min1.data()); - ASSERT_TRUE(devArrMatch(min.data(), min1.data(), params.m, CompareExactKVP(), stream)); - } -} -INSTANTIATE_TEST_CASE_P(FusedL2NNDetTests, FusedL2NNDetTestF_Sqrt, ::testing::ValuesIn(inputsf)); - -typedef FusedL2NNDetTest FusedL2NNDetTestD_Sq; -TEST_P(FusedL2NNDetTestD_Sq, Result) -{ - runTest(min.data()); // assumed to be golden - for (int i = 0; i < NumRepeats; ++i) { - runTest(min1.data()); - ASSERT_TRUE(devArrMatch(min.data(), min1.data(), params.m, CompareExactKVP(), stream)); - } -} -INSTANTIATE_TEST_CASE_P(FusedL2NNDetTests, FusedL2NNDetTestD_Sq, ::testing::ValuesIn(inputsd)); -typedef FusedL2NNDetTest FusedL2NNDetTestD_Sqrt; -TEST_P(FusedL2NNDetTestD_Sqrt, Result) -{ - runTest(min.data()); // assumed to be golden - for (int i = 0; i < NumRepeats; ++i) { - runTest(min1.data()); - ASSERT_TRUE(devArrMatch(min.data(), min1.data(), params.m, CompareExactKVP(), stream)); - } -} -INSTANTIATE_TEST_CASE_P(FusedL2NNDetTests, FusedL2NNDetTestD_Sqrt, ::testing::ValuesIn(inputsd)); - -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/gram.cu b/cpp/test/distance/gram.cu deleted file mode 100644 index e911a25ff1..0000000000 --- a/cpp/test/distance/gram.cu +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Copyright (c) 2019-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "gram_base.cuh" - -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include - -namespace raft::distance::kernels { - -struct GramMatrixInputs { - int n1; // feature vectors in matrix 1 - int n2; // featuer vectors in matrix 2 - int n_cols; // number of elements in a feature vector - bool is_row_major; - KernelParams kernel; - int ld1; - int ld2; - int ld_out; - // We will generate random input using the dimensions given here. - // The reference output is calculated by a custom kernel. -}; - -std::ostream& operator<<(std::ostream& os, const GramMatrixInputs& p) -{ - std::vector kernel_names{"linear", "poly", "rbf", "tanh"}; - os << "/" << p.n1 << "x" << p.n2 << "x" << p.n_cols << "/" - << (p.is_row_major ? "RowMajor/" : "ColMajor/") << kernel_names[p.kernel.kernel] << "/ld_" - << p.ld1 << "x" << p.ld2 << "x" << p.ld_out; - return os; -} - -const std::vector inputs = { - {42, 137, 2, false, {KernelType::LINEAR}}, - {42, 137, 2, true, {KernelType::LINEAR}}, - {42, 137, 2, false, {KernelType::LINEAR}, 64, 179, 181}, - {42, 137, 2, true, {KernelType::LINEAR}, 64, 179, 181}, - {137, 42, 2, false, {KernelType::POLYNOMIAL, 2, 0.5, 2.4}}, - {137, 42, 2, true, {KernelType::POLYNOMIAL, 2, 0.5, 2.4}}, - {137, 42, 2, false, {KernelType::POLYNOMIAL, 2, 0.5, 2.4}, 159, 73, 144}, - {137, 42, 2, true, {KernelType::POLYNOMIAL, 2, 0.5, 2.4}, 159, 73, 144}, - {42, 137, 2, false, {KernelType::TANH, 0, 0.5, 2.4}}, - {42, 137, 2, true, {KernelType::TANH, 0, 0.5, 2.4}}, - {42, 137, 2, false, {KernelType::TANH, 0, 0.5, 2.4}, 64, 155, 49}, - {42, 137, 2, true, {KernelType::TANH, 0, 0.5, 2.4}, 64, 155, 143}, - {3, 4, 2, false, {KernelType::RBF, 0, 0.5}}, - {42, 137, 2, false, {KernelType::RBF, 0, 0.5}}, - {42, 137, 2, true, {KernelType::RBF, 0, 0.5}}, - // Distance kernel does not support LD parameter yet. - //{42, 137, 2, false, {KernelType::RBF, 0, 0.5}, 64, 155, 49}, - // {42, 137, 2, true, {KernelType::RBF, 0, 0.5}, 64, 155, 143}, -}; - -template -class GramMatrixTest : public ::testing::TestWithParam { - protected: - GramMatrixTest() - : params(GetParam()), - handle(), - x1(0, resource::get_cuda_stream(handle)), - x2(0, resource::get_cuda_stream(handle)), - gram(0, resource::get_cuda_stream(handle)), - gram_host(0) - { - auto stream = resource::get_cuda_stream(handle); - - if (params.ld1 == 0) { params.ld1 = params.is_row_major ? params.n_cols : params.n1; } - if (params.ld2 == 0) { params.ld2 = params.is_row_major ? params.n_cols : params.n2; } - if (params.ld_out == 0) { params.ld_out = params.is_row_major ? params.n2 : params.n1; } - // Derive the size of the output from the offset of the last element. - size_t size = get_offset(params.n1 - 1, params.n_cols - 1, params.ld1, params.is_row_major) + 1; - x1.resize(size, stream); - size = get_offset(params.n2 - 1, params.n_cols - 1, params.ld2, params.is_row_major) + 1; - x2.resize(size, stream); - size = get_offset(params.n1 - 1, params.n2 - 1, params.ld_out, params.is_row_major) + 1; - - gram.resize(size, stream); - RAFT_CUDA_TRY(cudaMemsetAsync(gram.data(), 0, gram.size() * sizeof(math_t), stream)); - gram_host.resize(gram.size()); - std::fill(gram_host.begin(), gram_host.end(), 0); - - raft::random::RngState rng(42137ULL); - raft::random::uniform(handle, rng, x1.data(), x1.size(), math_t(0), math_t(1)); - raft::random::uniform(handle, rng, x2.data(), x2.size(), math_t(0), math_t(1)); - } - - ~GramMatrixTest() override {} - - void runTest() - { - std::unique_ptr> kernel = - std::unique_ptr>(KernelFactory::create(params.kernel)); - - auto x1_span = - params.is_row_major - ? raft::make_device_strided_matrix_view( - x1.data(), params.n1, params.n_cols, params.ld1) - : raft::make_device_strided_matrix_view( - x1.data(), params.n1, params.n_cols, params.ld1); - auto x2_span = - params.is_row_major - ? raft::make_device_strided_matrix_view( - x2.data(), params.n2, params.n_cols, params.ld2) - : raft::make_device_strided_matrix_view( - x2.data(), params.n2, params.n_cols, params.ld2); - auto out_span = - params.is_row_major - ? raft::make_device_strided_matrix_view( - gram.data(), params.n1, params.n2, params.ld_out) - : raft::make_device_strided_matrix_view( - gram.data(), params.n1, params.n2, params.ld_out); - - (*kernel)(handle, x1_span, x2_span, out_span); - - auto stream = resource::get_cuda_stream(handle); - naiveGramMatrixKernel(params.n1, - params.n2, - params.n_cols, - x1, - x2, - gram_host.data(), - params.ld1, - params.ld2, - params.ld_out, - params.is_row_major, - params.kernel, - stream, - handle); - - ASSERT_TRUE(raft::devArrMatchHost( - gram_host.data(), gram.data(), gram.size(), raft::CompareApprox(1e-6f), stream)); - } - - GramMatrixInputs params; - raft::resources handle; - - rmm::device_uvector x1; - rmm::device_uvector x2; - rmm::device_uvector gram; - - std::vector gram_host; -}; - -typedef GramMatrixTest GramMatrixTestFloat; -typedef GramMatrixTest GramMatrixTestDouble; - -TEST_P(GramMatrixTestFloat, Gram) { runTest(); } - -INSTANTIATE_TEST_SUITE_P(GramMatrixTests, GramMatrixTestFloat, ::testing::ValuesIn(inputs)); -}; // end namespace raft::distance::kernels diff --git a/cpp/test/distance/gram_base.cuh b/cpp/test/distance/gram_base.cuh deleted file mode 100644 index 170bcb76c1..0000000000 --- a/cpp/test/distance/gram_base.cuh +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include - -#include -#include - -namespace raft { -namespace distance { -namespace kernels { - -// Get the offset of element [i,k]. -HDI int get_offset(int i, int k, int ld, bool is_row_major) -{ - return is_row_major ? i * ld + k : i + k * ld; -} - -// Calculate the Gram matrix on the host. -template -void naiveGramMatrixKernel(int n1, - int n2, - int n_cols, - const rmm::device_uvector& x1, - const rmm::device_uvector& x2, - math_t* gram_host, - int ld1, - int ld2, - int ld_out, - bool is_row_major, - KernelParams kernel, - cudaStream_t stream, - const raft::resources& handle) -{ - std::vector x1_host(x1.size()); - raft::update_host(x1_host.data(), x1.data(), x1.size(), stream); - std::vector x2_host(x2.size()); - raft::update_host(x2_host.data(), x2.data(), x2.size(), stream); - resource::sync_stream(handle, stream); - - for (int i = 0; i < n1; i++) { - for (int j = 0; j < n2; j++) { - float d = 0; - for (int k = 0; k < n_cols; k++) { - if (kernel.kernel == KernelType::RBF) { - math_t diff = x1_host[get_offset(i, k, ld1, is_row_major)] - - x2_host[get_offset(j, k, ld2, is_row_major)]; - d += diff * diff; - } else { - d += x1_host[get_offset(i, k, ld1, is_row_major)] * - x2_host[get_offset(j, k, ld2, is_row_major)]; - } - } - int idx = get_offset(i, j, ld_out, is_row_major); - math_t v = 0; - switch (kernel.kernel) { - case (KernelType::LINEAR): gram_host[idx] = d; break; - case (KernelType::POLYNOMIAL): - v = kernel.gamma * d + kernel.coef0; - gram_host[idx] = std::pow(v, kernel.degree); - break; - case (KernelType::TANH): gram_host[idx] = std::tanh(kernel.gamma * d + kernel.coef0); break; - case (KernelType::RBF): gram_host[idx] = exp(-kernel.gamma * d); break; - } - } - } -} - -} // namespace kernels -} // namespace distance -} // namespace raft diff --git a/cpp/test/distance/masked_nn.cu b/cpp/test/distance/masked_nn.cu deleted file mode 100644 index 34fa07c45b..0000000000 --- a/cpp/test/distance/masked_nn.cu +++ /dev/null @@ -1,438 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -namespace raft::distance::masked_nn { - -// The adjacency pattern determines what distances get computed. -enum AdjacencyPattern { - checkerboard = 0, // adjacency matrix looks like a checkerboard (half the distances are computed) - checkerboard_4 = 1, // checkerboard with tiles of size 4x4 - checkerboard_64 = 2, // checkerboard with tiles of size 64x64 - all_true = 3, // no distance computations can be skipped - all_false = 4 // all distance computations can be skipped -}; - -// Kernels: -// - init_adj: to initialize the adjacency kernel with a specific adjacency pattern -// - referenceKernel: to produce the ground-truth output - -RAFT_KERNEL init_adj(AdjacencyPattern pattern, - int n, - raft::device_matrix_view adj, - raft::device_vector_view group_idxs) -{ - int m = adj.extent(0); - int num_groups = adj.extent(1); - - for (int idx_m = blockIdx.y * blockDim.y + threadIdx.y; idx_m < m; - idx_m += blockDim.y * gridDim.y) { - for (int idx_g = blockIdx.x * blockDim.x + threadIdx.x; idx_g < num_groups; - idx_g += blockDim.x * gridDim.x) { - switch (pattern) { - case checkerboard: adj(idx_m, idx_g) = (idx_m + idx_g) % 2; break; - case checkerboard_4: adj(idx_m, idx_g) = (idx_m / 4 + idx_g) % 2; break; - case checkerboard_64: adj(idx_m, idx_g) = (idx_m / 64 + idx_g) % 2; break; - case all_true: adj(idx_m, idx_g) = true; break; - case all_false: adj(idx_m, idx_g) = false; break; - default: assert(false && "unknown pattern"); - } - } - } - // Each group is of size n / num_groups. - // - // - group_idxs[j] indicates the start of group j + 1 (i.e. is the inclusive - // scan of the group lengths) - // - // - The first group always starts at index zero, so we do not store it. - // - // - The group_idxs[num_groups - 1] should always equal n. - - if (blockIdx.y == 0 && threadIdx.y == 0) { - const int g_stride = blockDim.x * gridDim.x; - for (int idx_g = blockIdx.x * blockDim.x + threadIdx.x; idx_g < num_groups; idx_g += g_stride) { - group_idxs(idx_g) = (idx_g + 1) * (n / num_groups); - } - group_idxs(num_groups - 1) = n; - } -} - -template -__launch_bounds__(32 * NWARPS, 2) RAFT_KERNEL referenceKernel(raft::KeyValuePair* min, - DataT* x, - DataT* y, - bool* adj, - int* group_idxs, - int m, - int n, - int k, - int num_groups, - bool sqrt, - int* workspace, - DataT maxVal) -{ - const int m_stride = blockDim.y * gridDim.y; - const int m_offset = threadIdx.y + blockIdx.y * blockDim.y; - const int n_stride = blockDim.x * gridDim.x; - const int n_offset = threadIdx.x + blockIdx.x * blockDim.x; - - for (int m_grid = 0; m_grid < m; m_grid += m_stride) { - for (int n_grid = 0; n_grid < n; n_grid += n_stride) { - int midx = m_grid + m_offset; - int nidx = n_grid + n_offset; - - // Do a reverse linear search to determine the group index. - int group_idx = 0; - for (int i = num_groups; 0 <= i; --i) { - if (nidx < group_idxs[i]) { group_idx = i; } - } - const bool include_dist = adj[midx * num_groups + group_idx] && midx < m && nidx < n; - - // Compute L2 metric. - DataT acc = DataT(0); - for (int i = 0; i < k; ++i) { - int xidx = i + midx * k; - int yidx = i + nidx * k; - auto diff = x[xidx] - y[yidx]; - acc += diff * diff; - } - if (sqrt) { acc = raft::sqrt(acc); } - ReduceOpT redOp; - typedef cub::WarpReduce> WarpReduce; - __shared__ typename WarpReduce::TempStorage temp[NWARPS]; - int warpId = threadIdx.x / raft::WarpSize; - raft::KeyValuePair tmp; - tmp.key = include_dist ? nidx : -1; - tmp.value = include_dist ? acc : maxVal; - tmp = WarpReduce(temp[warpId]).Reduce(tmp, raft::distance::KVPMinReduce{}); - if (threadIdx.x % raft::WarpSize == 0 && midx < m) { - while (atomicCAS(workspace + midx, 0, 1) == 1) - ; - __threadfence(); - redOp(midx, min + midx, tmp); - __threadfence(); - atomicCAS(workspace + midx, 1, 0); - } - __syncthreads(); - } - } -} - -// Structs -// - Params: holds parameters for test case -// - Inputs: holds the inputs to the functions under test (x, y, adj, group_idxs). Is generated from -// the inputs. -struct Params { - double tolerance; - int m, n, k, num_groups; - bool sqrt; - unsigned long long int seed; - AdjacencyPattern pattern; -}; - -inline auto operator<<(std::ostream& os, const Params& p) -> std::ostream& -{ - os << "m: " << p.m << ", n: " << p.n << ", k: " << p.k << ", num_groups: " << p.num_groups - << ", sqrt: " << p.sqrt << ", seed: " << p.seed << ", tol: " << p.tolerance; - return os; -} - -template -struct Inputs { - using IdxT = int; - - raft::device_matrix x, y; - raft::device_matrix adj; - raft::device_vector group_idxs; - - Inputs(const raft::handle_t& handle, const Params& p) - : x{raft::make_device_matrix(handle, p.m, p.k)}, - y{raft::make_device_matrix(handle, p.n, p.k)}, - adj{raft::make_device_matrix(handle, p.m, p.num_groups)}, - group_idxs{raft::make_device_vector(handle, p.num_groups)} - { - // Initialize x, y - raft::random::RngState r(p.seed); - uniform(handle, r, x.data_handle(), p.m * p.k, DataT(-1.0), DataT(1.0)); - uniform(handle, r, y.data_handle(), p.n * p.k, DataT(-1.0), DataT(1.0)); - - // Initialize adj, group_idxs. - dim3 block(32, 32); - dim3 grid(10, 10); - init_adj<<>>( - p.pattern, p.n, adj.view(), group_idxs.view()); - RAFT_CUDA_TRY(cudaGetLastError()); - } -}; - -template > -auto reference(const raft::handle_t& handle, Inputs inp, const Params& p) - -> raft::device_vector -{ - int m = inp.x.extent(0); - int n = inp.y.extent(0); - int k = inp.x.extent(1); - int num_groups = inp.group_idxs.extent(0); - - if (m == 0 || n == 0 || k == 0 || num_groups == 0) { - return raft::make_device_vector(handle, 0); - } - - // Initialize workspace - auto stream = resource::get_cuda_stream(handle); - rmm::device_uvector workspace(p.m * sizeof(int), stream); - RAFT_CUDA_TRY(cudaMemsetAsync(workspace.data(), 0, sizeof(int) * m, stream)); - - // Initialize output - auto out = raft::make_device_vector(handle, m); - auto blks = raft::ceildiv(m, 256); - MinAndDistanceReduceOp op; - raft::distance::detail::initKernel, int> - <<>>(out.data_handle(), m, std::numeric_limits::max(), op); - RAFT_CUDA_TRY(cudaGetLastError()); - - // Launch reference kernel - const int nwarps = 16; - static const dim3 TPB(32, nwarps, 1); - dim3 nblks(1, 200, 1); - referenceKernel - <<>>(out.data_handle(), - inp.x.data_handle(), - inp.y.data_handle(), - inp.adj.data_handle(), - inp.group_idxs.data_handle(), - m, - n, - k, - num_groups, - p.sqrt, - (int*)workspace.data(), - std::numeric_limits::max()); - RAFT_CUDA_TRY(cudaGetLastError()); - - return out; -} - -template > -auto run_masked_nn(const raft::handle_t& handle, Inputs inp, const Params& p) - -> raft::device_vector -{ - // Compute norms: - auto x_norm = raft::make_device_vector(handle, p.m); - auto y_norm = raft::make_device_vector(handle, p.n); - - raft::linalg::norm(handle, - std::as_const(inp.x).view(), - x_norm.view(), - raft::linalg::L2Norm, - raft::linalg::Apply::ALONG_ROWS); - raft::linalg::norm(handle, - std::as_const(inp.y).view(), - y_norm.view(), - raft::linalg::L2Norm, - raft::linalg::Apply::ALONG_ROWS); - - // Create parameters for masked_l2_nn - using IdxT = int; - using RedOpT = MinAndDistanceReduceOp; - using PairRedOpT = raft::distance::KVPMinReduce; - using ParamT = raft::distance::masked_l2_nn_params; - - bool init_out = true; - ParamT masked_l2_params{RedOpT{}, PairRedOpT{}, p.sqrt, init_out}; - - // Create output - auto out = raft::make_device_vector(handle, p.m); - - // Launch kernel - raft::distance::masked_l2_nn(handle, - masked_l2_params, - inp.x.view(), - inp.y.view(), - x_norm.view(), - y_norm.view(), - inp.adj.view(), - inp.group_idxs.view(), - out.view()); - - resource::sync_stream(handle); - - return out; -} - -template -struct CompareApproxAbsKVP { - typedef typename raft::KeyValuePair KVP; - CompareApproxAbsKVP(T eps_) : eps(eps_) {} - bool operator()(const KVP& a, const KVP& b) const - { - T diff = raft::abs(raft::abs(a.value) - raft::abs(b.value)); - T m = std::max(raft::abs(a.value), raft::abs(b.value)); - T ratio = m >= eps ? diff / m : diff; - return (ratio <= eps); - } - - private: - T eps; -}; - -template -::testing::AssertionResult devArrMatch(const raft::KeyValuePair* expected, - const raft::KeyValuePair* actual, - size_t size, - L eq_compare, - cudaStream_t stream = 0) -{ - typedef typename raft::KeyValuePair KVP; - std::shared_ptr exp_h(new KVP[size]); - std::shared_ptr act_h(new KVP[size]); - raft::update_host(exp_h.get(), expected, size, stream); - raft::update_host(act_h.get(), actual, size, stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - for (size_t i(0); i < size; ++i) { - auto exp = exp_h.get()[i]; - auto act = act_h.get()[i]; - if (!eq_compare(exp, act)) { - return ::testing::AssertionFailure() - << "actual=" << act.key << "," << act.value << " != expected=" << exp.key << "," - << exp.value << " @" << i; - } - } - return ::testing::AssertionSuccess(); -} - -inline auto gen_params() -> std::vector -{ - // Regular powers of two - auto regular = raft::util::itertools::product({0.001f}, // tolerance - {32, 64, 512}, // m - {32, 64, 512}, // n - {8, 32}, // k - {2, 32}, // num_groups - {true, false}, // sqrt - {1234ULL}, // seed - {AdjacencyPattern::all_true, - AdjacencyPattern::checkerboard, - AdjacencyPattern::checkerboard_64, - AdjacencyPattern::all_false}); - - // Irregular sizes to check tiling and bounds checking - auto irregular = raft::util::itertools::product({0.001f}, // tolerance - {511, 512, 513}, // m - {127, 128, 129}, // n - {5}, // k - {3, 9}, // num_groups - {true, false}, // sqrt - {1234ULL}, // seed - {AdjacencyPattern::all_true, - AdjacencyPattern::checkerboard, - AdjacencyPattern::checkerboard_64}); - - regular.insert(regular.end(), irregular.begin(), irregular.end()); - - return regular; -} - -class MaskedL2NNTest : public ::testing::TestWithParam { - // Empty. -}; - -// -TEST_P(MaskedL2NNTest, ReferenceCheckFloat) -{ - using DataT = float; - - // Get parameters; create handle and input data. - Params p = GetParam(); - raft::handle_t handle{}; - Inputs inputs{handle, p}; - - // Calculate reference and test output - auto out_reference = reference(handle, inputs, p); - auto out_fast = run_masked_nn(handle, inputs, p); - - // Check for differences. - ASSERT_TRUE(devArrMatch(out_reference.data_handle(), - out_fast.data_handle(), - p.m, - CompareApproxAbsKVP(p.tolerance), - resource::get_cuda_stream(handle))); -} - -// This test checks whether running the masked_l2_nn twice returns the same -// output. -TEST_P(MaskedL2NNTest, DeterminismCheck) -{ - using DataT = float; - - // Get parameters; create handle and input data. - Params p = GetParam(); - raft::handle_t handle{}; - Inputs inputs{handle, p}; - - // Calculate reference and test output - auto out1 = run_masked_nn(handle, inputs, p); - auto out2 = run_masked_nn(handle, inputs, p); - - // Check for differences. - ASSERT_TRUE(devArrMatch(out1.data_handle(), - out2.data_handle(), - p.m, - CompareApproxAbsKVP(p.tolerance), - resource::get_cuda_stream(handle))); -} - -TEST_P(MaskedL2NNTest, ReferenceCheckDouble) -{ - using DataT = double; - - // Get parameters; create handle and input data. - Params p = GetParam(); - raft::handle_t handle{}; - Inputs inputs{handle, p}; - - // Calculate reference and test output - auto out_reference = reference(handle, inputs, p); - auto out_fast = run_masked_nn(handle, inputs, p); - - // Check for differences. - ASSERT_TRUE(devArrMatch(out_reference.data_handle(), - out_fast.data_handle(), - p.m, - CompareApproxAbsKVP(p.tolerance), - resource::get_cuda_stream(handle))); -} - -INSTANTIATE_TEST_CASE_P(MaskedL2NNTests, MaskedL2NNTest, ::testing::ValuesIn(gen_params())); - -} // end namespace raft::distance::masked_nn diff --git a/cpp/test/distance/masked_nn_compress_to_bits.cu b/cpp/test/distance/masked_nn_compress_to_bits.cu deleted file mode 100644 index 2512af5e4f..0000000000 --- a/cpp/test/distance/masked_nn_compress_to_bits.cu +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "../test_utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include - -namespace raft::distance::masked_nn::compress_to_bits { - -/** - * @brief Transpose and decompress 2D bitfield to boolean matrix - * - * Inverse operation of compress_to_bits - * - * @tparam T - * - * @parameter[in] in An `m x n` bitfield matrix. Row major. - * @parameter in_rows The number of rows of `in`, i.e. `m`. - * @parameter in_cols The number of cols of `in`, i.e. `n`. - * - * @parameter[out] out An `(m * bits_per_elem) x n` boolean matrix. - */ -template ::value>> -RAFT_KERNEL decompress_bits_kernel(const T* in, int in_rows, int in_cols, bool* out) -{ - constexpr int bits_per_element = 8 * sizeof(T); - - const size_t i = threadIdx.y + blockIdx.y * blockDim.y; - const size_t j = threadIdx.x + blockIdx.x * blockDim.x; - - if (in_rows <= i || in_cols <= j) { return; } - - const size_t out_rows = in_rows * bits_per_element; - const size_t out_cols = in_cols; - const size_t out_i = i * bits_per_element; - const size_t out_j = j; - - if (out_rows <= out_i && out_cols <= out_j) { return; } - - T bitfield = in[i * in_cols + j]; - for (int bitpos = 0; bitpos < bits_per_element; ++bitpos) { - bool bit = ((T(1) << bitpos) & bitfield) != 0; - out[(out_i + bitpos) * out_cols + out_j] = bit; - } -} - -/** - * @brief Transpose and decompress 2D bitfield to boolean matrix - * - * Inverse operation of compress_to_bits - * - * @tparam T - * - * @parameter[in] in An `m x n` bitfield matrix. Row major. - * @parameter in_rows The number of rows of `in`, i.e. `m`. - * @parameter in_cols The number of cols of `in`, i.e. `n`. - * - * @parameter[out] out An `n x (m * bits_per_elem)` boolean matrix. - */ -template ::value>> -void decompress_bits(const raft::handle_t& handle, const T* in, int in_rows, int in_cols, bool* out) -{ - auto stream = resource::get_cuda_stream(handle); - dim3 grid(raft::ceildiv(in_cols, 32), raft::ceildiv(in_rows, 32)); - dim3 block(32, 32); - decompress_bits_kernel<<>>(in, in_rows, in_cols, out); - RAFT_CUDA_TRY(cudaGetLastError()); -} - -// Params holds parameters for test case -struct Params { - int m, n; -}; - -inline auto operator<<(std::ostream& os, const Params& p) -> std::ostream& -{ - return os << "m: " << p.m << ", n: " << p.n; -} - -// Check that the following holds -// -// decompress(compress(x)) == x -// -// for 2D boolean matrices x. -template -void check_invertible(const Params& p) -{ - using raft::distance::detail::compress_to_bits; - constexpr int bits_per_elem = sizeof(T) * 8; - - // Make m and n that are safe to ceildiv. - int m = raft::round_up_safe(p.m, bits_per_elem); - int n = p.n; - - // Generate random input - raft::handle_t handle{}; - raft::random::RngState r(1ULL); - auto in = raft::make_device_matrix(handle, m, n); - raft::random::bernoulli(handle, r, in.data_handle(), m * n, 0.5f); - - int tmp_m = raft::ceildiv(m, bits_per_elem); - int out_m = tmp_m * bits_per_elem; - - auto tmp = raft::make_device_matrix(handle, tmp_m, n); - auto out = raft::make_device_matrix(handle, out_m, n); - - resource::sync_stream(handle); - RAFT_CUDA_TRY(cudaGetLastError()); - - ASSERT_EQ(in.extent(0), out.extent(0)) << "M does not match"; - ASSERT_EQ(in.extent(1), out.extent(1)) << "N does not match"; - - compress_to_bits(handle, in.view(), tmp.view()); - resource::sync_stream(handle); - RAFT_CUDA_TRY(cudaGetLastError()); - - decompress_bits(handle, tmp.data_handle(), tmp.extent(0), tmp.extent(1), out.data_handle()); - resource::sync_stream(handle); - RAFT_CUDA_TRY(cudaGetLastError()); - - // Check for differences. - ASSERT_TRUE(raft::devArrMatch(in.data_handle(), - out.data_handle(), - in.extent(0) * in.extent(1), - raft::Compare(), - resource::get_cuda_stream(handle))); - resource::sync_stream(handle); - RAFT_CUDA_TRY(cudaGetLastError()); -} - -void check_all_true(const Params& p) -{ - using raft::distance::detail::compress_to_bits; - using T = uint64_t; - constexpr int bits_per_elem = sizeof(T) * 8; - - // Make m and n that are safe to ceildiv. - int m = raft::round_up_safe(p.m, bits_per_elem); - int n = p.n; - - raft::handle_t handle{}; - raft::random::RngState r(1ULL); - auto in = raft::make_device_matrix(handle, m, n); - raft::matrix::fill(handle, in.view(), true); - - int tmp_m = raft::ceildiv(m, bits_per_elem); - auto tmp = raft::make_device_matrix(handle, tmp_m, n); - resource::sync_stream(handle); - RAFT_CUDA_TRY(cudaGetLastError()); - - compress_to_bits(handle, in.view(), tmp.view()); - resource::sync_stream(handle); - RAFT_CUDA_TRY(cudaGetLastError()); - - auto expected = raft::make_device_matrix(handle, tmp_m, n); - raft::matrix::fill(handle, expected.view(), ~T(0)); - - // Check for differences. - ASSERT_TRUE(raft::devArrMatch(expected.data_handle(), - tmp.data_handle(), - tmp.extent(0) * tmp.extent(1), - raft::Compare(), - resource::get_cuda_stream(handle))); - resource::sync_stream(handle); - RAFT_CUDA_TRY(cudaGetLastError()); -} - -class CompressToBitsTest : public ::testing::TestWithParam { - // Empty. -}; - -TEST_P(CompressToBitsTest, CheckTrue64) { check_all_true(GetParam()); } - -TEST_P(CompressToBitsTest, CheckInvertible64) -{ - using T = uint64_t; - check_invertible(GetParam()); -} - -TEST_P(CompressToBitsTest, CheckInvertible32) -{ - using T = uint32_t; - check_invertible(GetParam()); -} - -std::vector params = raft::util::itertools::product( - {1, 3, 32, 33, 63, 64, 65, 128, 10013}, {1, 3, 32, 33, 63, 64, 65, 13001}); - -INSTANTIATE_TEST_CASE_P(CompressToBits, CompressToBitsTest, ::testing::ValuesIn(params)); - -} // namespace raft::distance::masked_nn::compress_to_bits \ No newline at end of file diff --git a/cpp/test/neighbors/ann_brute_force.cuh b/cpp/test/neighbors/ann_brute_force.cuh deleted file mode 100644 index 6370c5ee83..0000000000 --- a/cpp/test/neighbors/ann_brute_force.cuh +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "../test_utils.cuh" -#include "ann_utils.cuh" -#include "knn_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include - -#include -#include - -#include - -#include -#include -#include - -namespace raft::neighbors::brute_force { - -template -struct AnnBruteForceInputs { - IdxT num_queries; - IdxT num_db_vecs; - IdxT dim; - IdxT k; - raft::distance::DistanceType metric; - bool host_dataset; -}; - -template -::std::ostream& operator<<(::std::ostream& os, const AnnBruteForceInputs& p) -{ - os << "{ " << p.num_queries << ", " << p.num_db_vecs << ", " << p.dim << ", " << p.k << ", " - << static_cast(p.metric) << ", " << p.host_dataset << '}' << std::endl; - return os; -} - -template -class AnnBruteForceTest : public ::testing::TestWithParam> { - public: - AnnBruteForceTest() - : stream_(resource::get_cuda_stream(handle_)), - ps(::testing::TestWithParam>::GetParam()), - database(0, stream_), - search_queries(0, stream_) - { - } - - void testBruteForce() - { - size_t queries_size = ps.num_queries * ps.k; - - rmm::device_uvector distances_naive_dev(queries_size, stream_); - rmm::device_uvector indices_naive_dev(queries_size, stream_); - naive_knn(handle_, - distances_naive_dev.data(), - indices_naive_dev.data(), - search_queries.data(), - database.data(), - ps.num_queries, - ps.num_db_vecs, - ps.dim, - ps.k, - ps.metric); - resource::sync_stream(handle_); - - { - // Require exact result for brute force - rmm::device_uvector distances_bruteforce_dev(queries_size, stream_); - rmm::device_uvector indices_bruteforce_dev(queries_size, stream_); - brute_force::index_params index_params{}; - brute_force::search_params search_params{}; - index_params.metric = ps.metric; - index_params.metric_arg = 0; - - auto device_dataset = std::optional>{}; - auto idx = [this, &index_params]() { - if (ps.host_dataset) { - auto host_database = raft::make_host_matrix(ps.num_db_vecs, ps.dim); - raft::copy( - host_database.data_handle(), database.data(), ps.num_db_vecs * ps.dim, stream_); - return brute_force::build( - handle_, index_params, raft::make_const_mdspan(host_database.view())); - } else { - auto database_view = raft::make_device_matrix_view( - (const DataT*)database.data(), ps.num_db_vecs, ps.dim); - return brute_force::build(handle_, index_params, database_view); - } - }(); - - auto search_queries_view = raft::make_device_matrix_view( - search_queries.data(), ps.num_queries, ps.dim); - auto indices_out_view = raft::make_device_matrix_view( - indices_bruteforce_dev.data(), ps.num_queries, ps.k); - auto dists_out_view = raft::make_device_matrix_view( - distances_bruteforce_dev.data(), ps.num_queries, ps.k); - brute_force::serialize(handle_, std::string{"brute_force_index"}, idx); - - auto index_loaded = - brute_force::deserialize(handle_, std::string{"brute_force_index"}); - ASSERT_EQ(idx.size(), index_loaded.size()); - - brute_force::search(handle_, - search_params, - index_loaded, - search_queries_view, - indices_out_view, - dists_out_view); - - resource::sync_stream(handle_); - - ASSERT_TRUE(raft::spatial::knn::devArrMatchKnnPair(indices_naive_dev.data(), - indices_bruteforce_dev.data(), - distances_naive_dev.data(), - distances_bruteforce_dev.data(), - ps.num_queries, - ps.k, - 0.001f, - stream_, - true)); - brute_force::serialize(handle_, std::string{"brute_force_index"}, idx, false); - index_loaded = brute_force::deserialize(handle_, std::string{"brute_force_index"}); - index_loaded.update_dataset(handle_, idx.dataset()); - ASSERT_EQ(idx.size(), index_loaded.size()); - - brute_force::search(handle_, - search_params, - index_loaded, - search_queries_view, - indices_out_view, - dists_out_view); - - resource::sync_stream(handle_); - - ASSERT_TRUE(raft::spatial::knn::devArrMatchKnnPair(indices_naive_dev.data(), - indices_bruteforce_dev.data(), - distances_naive_dev.data(), - distances_bruteforce_dev.data(), - ps.num_queries, - ps.k, - 0.001f, - stream_, - true)); - } - } - - void SetUp() override - { - database.resize(ps.num_db_vecs * ps.dim, stream_); - search_queries.resize(ps.num_queries * ps.dim, stream_); - - raft::random::RngState r(1234ULL); - if constexpr (std::is_same{}) { - raft::random::uniform( - handle_, r, database.data(), ps.num_db_vecs * ps.dim, DataT(0.1), DataT(2.0)); - raft::random::uniform( - handle_, r, search_queries.data(), ps.num_queries * ps.dim, DataT(0.1), DataT(2.0)); - } else { - raft::random::uniformInt( - handle_, r, database.data(), ps.num_db_vecs * ps.dim, DataT(1), DataT(20)); - raft::random::uniformInt( - handle_, r, search_queries.data(), ps.num_queries * ps.dim, DataT(1), DataT(20)); - } - resource::sync_stream(handle_); - } - - void TearDown() override - { - resource::sync_stream(handle_); - database.resize(0, stream_); - search_queries.resize(0, stream_); - } - - private: - raft::resources handle_; - rmm::cuda_stream_view stream_; - AnnBruteForceInputs ps; - rmm::device_uvector database; - rmm::device_uvector search_queries; -}; - -const std::vector> inputs = { - // test various dims (aligned and not aligned to vector sizes) - {1000, 10000, 1, 16, raft::distance::DistanceType::L2Expanded, true}, - {1000, 10000, 2, 16, raft::distance::DistanceType::L2Expanded, true}, - {1000, 10000, 3, 16, raft::distance::DistanceType::L2Expanded, true}, - {1000, 10000, 4, 16, raft::distance::DistanceType::L2Expanded, true}, - {1000, 10000, 5, 16, raft::distance::DistanceType::InnerProduct, true}, - {1000, 10000, 8, 16, raft::distance::DistanceType::InnerProduct, true}, - {1000, 10000, 5, 16, raft::distance::DistanceType::L2SqrtExpanded, true}, - {1000, 10000, 8, 16, raft::distance::DistanceType::L2SqrtExpanded, true}, - - // test dims that do not fit into kernel shared memory limits - {1000, 10000, 2048, 16, raft::distance::DistanceType::L2Expanded, true}, - {1000, 10000, 2049, 16, raft::distance::DistanceType::L2Expanded, true}, - {1000, 10000, 2050, 16, raft::distance::DistanceType::InnerProduct, true}, - {1000, 10000, 2051, 16, raft::distance::DistanceType::InnerProduct, true}, - {1000, 10000, 2052, 16, raft::distance::DistanceType::InnerProduct, true}, - {1000, 10000, 2053, 16, raft::distance::DistanceType::L2Expanded, true}, - {1000, 10000, 2056, 16, raft::distance::DistanceType::L2Expanded, true}, - - // host input data - {1000, 10000, 16, 10, raft::distance::DistanceType::L2Expanded, false}, - {1000, 10000, 16, 10, raft::distance::DistanceType::L2Expanded, false}, - {1000, 10000, 16, 10, raft::distance::DistanceType::L2Expanded, false}, - {100, 10000, 16, 10, raft::distance::DistanceType::L2Expanded, false}, - {20, 100000, 16, 10, raft::distance::DistanceType::L2Expanded, false}, - {1000, 100000, 16, 10, raft::distance::DistanceType::L2Expanded, false}, - {10000, 131072, 8, 10, raft::distance::DistanceType::L2Expanded, false}, - - {1000, 10000, 16, 10, raft::distance::DistanceType::InnerProduct, false}}; -} // namespace raft::neighbors::brute_force diff --git a/cpp/test/neighbors/ann_brute_force/test_float.cu b/cpp/test/neighbors/ann_brute_force/test_float.cu deleted file mode 100644 index f157b5f65c..0000000000 --- a/cpp/test/neighbors/ann_brute_force/test_float.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_brute_force.cuh" - -#include - -namespace raft::neighbors::brute_force { - -using AnnBruteForceTest_float = AnnBruteForceTest; -TEST_P(AnnBruteForceTest_float, AnnBruteForce) { this->testBruteForce(); } - -INSTANTIATE_TEST_CASE_P(AnnBruteForceTest, AnnBruteForceTest_float, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::brute_force diff --git a/cpp/test/neighbors/ann_cagra.cuh b/cpp/test/neighbors/ann_cagra.cuh deleted file mode 100644 index cc787d3e57..0000000000 --- a/cpp/test/neighbors/ann_cagra.cuh +++ /dev/null @@ -1,949 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#undef RAFT_EXPLICIT_INSTANTIATE_ONLY // Search with filter instantiation - -#include "../test_utils.cuh" -#include "ann_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include - -#include - -#include -#include -#include -#include - -namespace raft::neighbors::cagra { -namespace { - -/* A filter that excludes all indices below `offset`. */ -struct test_cagra_sample_filter { - static constexpr unsigned offset = 300; - inline _RAFT_HOST_DEVICE auto operator()( - // query index - const uint32_t query_ix, - // the index of the current sample inside the current inverted list - const uint32_t sample_ix) const - { - return sample_ix >= offset; - } -}; - -// For sort_knn_graph test -template -void RandomSuffle(raft::host_matrix_view index) -{ - for (IdxT i = 0; i < index.extent(0); i++) { - uint64_t rand = i; - IdxT* const row_ptr = index.data_handle() + i * index.extent(1); - for (unsigned j = 0; j < index.extent(1); j++) { - // Swap two indices at random - rand = raft::neighbors::cagra::detail::device::xorshift64(rand); - const auto i0 = rand % index.extent(1); - rand = raft::neighbors::cagra::detail::device::xorshift64(rand); - const auto i1 = rand % index.extent(1); - - const auto tmp = row_ptr[i0]; - row_ptr[i0] = row_ptr[i1]; - row_ptr[i1] = tmp; - } - } -} - -template -testing::AssertionResult CheckOrder(raft::host_matrix_view index_test, - raft::host_matrix_view dataset, - raft::distance::DistanceType metric) -{ - for (IdxT i = 0; i < index_test.extent(0); i++) { - const DatatT* const base_vec = dataset.data_handle() + i * dataset.extent(1); - const IdxT* const index_row = index_test.data_handle() + i * index_test.extent(1); - DistanceT prev_distance = metric == raft::distance::DistanceType::L2Expanded - ? 0 - : std::numeric_limits::max(); - for (unsigned j = 0; j < index_test.extent(1) - 1; j++) { - const DatatT* const target_vec = dataset.data_handle() + index_row[j] * dataset.extent(1); - DistanceT distance = 0; - switch (metric) { - case raft::distance::DistanceType::L2Expanded: - for (unsigned l = 0; l < dataset.extent(1); l++) { - const auto diff = - static_cast(target_vec[l]) - static_cast(base_vec[l]); - distance += diff * diff; - } - if (prev_distance > distance) { - return testing::AssertionFailure() - << "Wrong index order (row = " << i << ", neighbor_id = " << j - << "). (distance[neighbor_id-1] = " << prev_distance - << "should be lesser than distance[neighbor_id] = " << distance << ")"; - } - break; - case raft::distance::DistanceType::InnerProduct: - for (unsigned l = 0; l < dataset.extent(1); l++) { - const auto prod = - static_cast(target_vec[l]) * static_cast(base_vec[l]); - distance += prod; - } - if (prev_distance < distance) { - return testing::AssertionFailure() - << "Wrong index order (row = " << i << ", neighbor_id = " << j - << "). (distance[neighbor_id-1] = " << prev_distance - << "should be greater than distance[neighbor_id] = " << distance << ")"; - } - break; - default: - return testing::AssertionFailure() - << "Distance metric " << metric - << " not supported. Only L2Expanded and InnerProduct are supported"; - } - prev_distance = distance; - } - } - return testing::AssertionSuccess(); -} - -template -struct fpi_mapper {}; - -template <> -struct fpi_mapper { - using type = int64_t; - static constexpr int kBitshiftBase = 53; -}; - -template <> -struct fpi_mapper { - using type = int32_t; - static constexpr int kBitshiftBase = 24; -}; - -template <> -struct fpi_mapper { - using type = int16_t; - static constexpr int kBitshiftBase = 11; -}; - -// Generate dataset to ensure no rounding error occurs in the norm computation of any two vectors. -// When testing the CAGRA index sorting function, rounding errors can affect the norm and alter the -// order of the index. To ensure the accuracy of the test, we utilize the dataset. The generation -// method is based on the error-free transformation (EFT) method. -template -RAFT_KERNEL GenerateRoundingErrorFreeDataset_kernel(T* const ptr, - const uint32_t size, - const typename fpi_mapper::type resolution) -{ - const auto tid = threadIdx.x + blockIdx.x * blockDim.x; - if (tid >= size) { return; } - - const float u32 = *reinterpret_cast::type*>(ptr + tid); - ptr[tid] = u32 / resolution; -} - -template -void GenerateRoundingErrorFreeDataset( - const raft::resources& handle, - T* const ptr, - const uint32_t n_row, - const uint32_t dim, - raft::random::RngState& rng, - const bool diff_flag // true if compute the norm between two vectors -) -{ - using mapper_type = fpi_mapper; - using int_type = typename mapper_type::type; - auto cuda_stream = resource::get_cuda_stream(handle); - const uint32_t size = n_row * dim; - const uint32_t block_size = 256; - const uint32_t grid_size = (size + block_size - 1) / block_size; - - const auto bitshift = (mapper_type::kBitshiftBase - std::log2(dim) - (diff_flag ? 1 : 0)) / 2; - // Skip the test when `dim` is too big for type `T` to allow rounding error-free test. - if (bitshift <= 1) { GTEST_SKIP(); } - const int_type resolution = int_type{1} << static_cast(std::floor(bitshift)); - raft::random::uniformInt( - handle, rng, reinterpret_cast(ptr), size, -resolution, resolution - 1); - - GenerateRoundingErrorFreeDataset_kernel - <<>>(ptr, size, resolution); -} - -template -void InitDataset(const raft::resources& handle, - DataT* const datatset_ptr, - std::uint32_t size, - std::uint32_t dim, - raft::distance::DistanceType metric, - raft::random::RngState& r) -{ - if constexpr (std::is_same_v || std::is_same_v) { - GenerateRoundingErrorFreeDataset(handle, datatset_ptr, size, dim, r, true); - - if (metric == raft::distance::InnerProduct) { - auto dataset_view = raft::make_device_matrix_view(datatset_ptr, size, dim); - raft::linalg::row_normalize( - handle, raft::make_const_mdspan(dataset_view), dataset_view, raft::linalg::L2Norm); - } - } else if constexpr (std::is_same_v || std::is_same_v) { - if constexpr (std::is_same_v) { - raft::random::uniformInt(handle, r, datatset_ptr, size * dim, DataT(-10), DataT(10)); - } else { - raft::random::uniformInt(handle, r, datatset_ptr, size * dim, DataT(1), DataT(20)); - } - - if (metric == raft::distance::InnerProduct) { - // TODO (enp1s0): Change this once row_normalize supports (u)int8 matrices. - // https://github.com/rapidsai/raft/issues/2291 - - using ComputeT = float; - auto dataset_view = raft::make_device_matrix_view(datatset_ptr, size, dim); - auto dev_row_norm = raft::make_device_vector(handle, size); - const auto normalized_norm = - (std::is_same_v ? 40 : 20) * std::sqrt(static_cast(dim)); - - raft::linalg::reduce(dev_row_norm.data_handle(), - datatset_ptr, - dim, - size, - 0.f, - true, - true, - resource::get_cuda_stream(handle), - false, - raft::sq_op(), - raft::add_op(), - raft::sqrt_op()); - raft::linalg::matrix_vector_op( - handle, - raft::make_const_mdspan(dataset_view), - raft::make_const_mdspan(dev_row_norm.view()), - dataset_view, - raft::linalg::Apply::ALONG_COLUMNS, - [normalized_norm] __device__(DataT elm, ComputeT norm) { - const ComputeT v = elm / norm * normalized_norm; - const ComputeT max_v_range = std::numeric_limits::max(); - const ComputeT min_v_range = std::numeric_limits::min(); - return static_cast(std::min(max_v_range, std::max(min_v_range, v))); - }); - } - } -} -} // namespace - -struct AnnCagraInputs { - int n_queries; - int n_rows; - int dim; - int k; - graph_build_algo build_algo; - search_algo algo; - int max_queries; - int team_size; - int itopk_size; - int search_width; - raft::distance::DistanceType metric; - bool host_dataset; - bool include_serialized_dataset; - // std::optional - double min_recall; // = std::nullopt; -}; - -inline ::std::ostream& operator<<(::std::ostream& os, const AnnCagraInputs& p) -{ - std::vector algo = {"single-cta", "multi_cta", "multi_kernel", "auto"}; - std::vector build_algo = {"IVF_PQ", "NN_DESCENT"}; - os << "{n_queries=" << p.n_queries << ", dataset shape=" << p.n_rows << "x" << p.dim - << ", k=" << p.k << ", " << algo.at((int)p.algo) << ", max_queries=" << p.max_queries - << ", itopk_size=" << p.itopk_size << ", search_width=" << p.search_width - << ", metric=" << static_cast(p.metric) << (p.host_dataset ? ", host" : ", device") - << ", build_algo=" << build_algo.at((int)p.build_algo) << '}' << std::endl; - return os; -} - -template -class AnnCagraTest : public ::testing::TestWithParam { - public: - AnnCagraTest() - : stream_(resource::get_cuda_stream(handle_)), - ps(::testing::TestWithParam::GetParam()), - database(0, stream_), - search_queries(0, stream_) - { - } - - protected: - void testCagra() - { - // TODO (tarang-jain): remove when NN Descent index building support InnerProduct. Reference - // issue: https://github.com/rapidsai/raft/issues/2276 - if (ps.metric == distance::InnerProduct && ps.build_algo == graph_build_algo::NN_DESCENT) - GTEST_SKIP(); - - size_t queries_size = ps.n_queries * ps.k; - std::vector indices_Cagra(queries_size); - std::vector indices_naive(queries_size); - std::vector distances_Cagra(queries_size); - std::vector distances_naive(queries_size); - - { - rmm::device_uvector distances_naive_dev(queries_size, stream_); - rmm::device_uvector indices_naive_dev(queries_size, stream_); - naive_knn(handle_, - distances_naive_dev.data(), - indices_naive_dev.data(), - search_queries.data(), - database.data(), - ps.n_queries, - ps.n_rows, - ps.dim, - ps.k, - ps.metric); - update_host(distances_naive.data(), distances_naive_dev.data(), queries_size, stream_); - update_host(indices_naive.data(), indices_naive_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - { - rmm::device_uvector distances_dev(queries_size, stream_); - rmm::device_uvector indices_dev(queries_size, stream_); - - { - cagra::index_params index_params; - index_params.metric = ps.metric; // Note: currently ony the cagra::index_params metric is - // not used for knn_graph building. - index_params.build_algo = ps.build_algo; - cagra::search_params search_params; - search_params.algo = ps.algo; - search_params.max_queries = ps.max_queries; - search_params.team_size = ps.team_size; - search_params.itopk_size = ps.itopk_size; - - auto database_view = raft::make_device_matrix_view( - (const DataT*)database.data(), ps.n_rows, ps.dim); - - { - cagra::index index(handle_); - if (ps.host_dataset) { - auto database_host = raft::make_host_matrix(ps.n_rows, ps.dim); - raft::copy(database_host.data_handle(), database.data(), database.size(), stream_); - auto database_host_view = raft::make_host_matrix_view( - (const DataT*)database_host.data_handle(), ps.n_rows, ps.dim); - index = cagra::build(handle_, index_params, database_host_view); - } else { - index = cagra::build(handle_, index_params, database_view); - }; - cagra::serialize(handle_, "cagra_index", index, ps.include_serialized_dataset); - } - - auto index = cagra::deserialize(handle_, "cagra_index"); - if (!ps.include_serialized_dataset) { index.update_dataset(handle_, database_view); } - - auto search_queries_view = raft::make_device_matrix_view( - search_queries.data(), ps.n_queries, ps.dim); - auto indices_out_view = - raft::make_device_matrix_view(indices_dev.data(), ps.n_queries, ps.k); - auto dists_out_view = raft::make_device_matrix_view( - distances_dev.data(), ps.n_queries, ps.k); - - cagra::search( - handle_, search_params, index, search_queries_view, indices_out_view, dists_out_view); - update_host(distances_Cagra.data(), distances_dev.data(), queries_size, stream_); - update_host(indices_Cagra.data(), indices_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - // for (int i = 0; i < min(ps.n_queries, 10); i++) { - // // std::cout << "query " << i << std::end; - // print_vector("T", indices_naive.data() + i * ps.k, ps.k, std::cout); - // print_vector("C", indices_Cagra.data() + i * ps.k, ps.k, std::cout); - // print_vector("T", distances_naive.data() + i * ps.k, ps.k, std::cout); - // print_vector("C", distances_Cagra.data() + i * ps.k, ps.k, std::cout); - // } - - double min_recall = ps.min_recall; - EXPECT_TRUE(eval_neighbours(indices_naive, - indices_Cagra, - distances_naive, - distances_Cagra, - ps.n_queries, - ps.k, - 0.003, - min_recall)); - EXPECT_TRUE(eval_distances(handle_, - database.data(), - search_queries.data(), - indices_dev.data(), - distances_dev.data(), - ps.n_rows, - ps.dim, - ps.n_queries, - ps.k, - ps.metric, - 1.0e-4)); - } - } - - void SetUp() override - { - database.resize(((size_t)ps.n_rows) * ps.dim, stream_); - search_queries.resize(ps.n_queries * ps.dim, stream_); - raft::random::RngState r(1234ULL); - InitDataset(handle_, database.data(), ps.n_rows, ps.dim, ps.metric, r); - InitDataset(handle_, search_queries.data(), ps.n_queries, ps.dim, ps.metric, r); - resource::sync_stream(handle_); - } - - void TearDown() override - { - resource::sync_stream(handle_); - database.resize(0, stream_); - search_queries.resize(0, stream_); - } - - private: - raft::resources handle_; - rmm::cuda_stream_view stream_; - AnnCagraInputs ps; - rmm::device_uvector database; - rmm::device_uvector search_queries; -}; - -template -class AnnCagraSortTest : public ::testing::TestWithParam { - public: - AnnCagraSortTest() - : ps(::testing::TestWithParam::GetParam()), database(0, handle_.get_stream()) - { - } - - protected: - void testCagraSort() - { - if (ps.metric == distance::InnerProduct && ps.build_algo == graph_build_algo::NN_DESCENT) - GTEST_SKIP(); - - { - // Step 1: Build a sorted KNN graph by CAGRA knn build - auto database_view = raft::make_device_matrix_view( - (const DataT*)database.data(), ps.n_rows, ps.dim); - auto database_host = raft::make_host_matrix(ps.n_rows, ps.dim); - raft::copy( - database_host.data_handle(), database.data(), database.size(), handle_.get_stream()); - auto database_host_view = raft::make_host_matrix_view( - (const DataT*)database_host.data_handle(), ps.n_rows, ps.dim); - - cagra::index_params index_params; - auto knn_graph = - raft::make_host_matrix(ps.n_rows, index_params.intermediate_graph_degree); - - if (ps.build_algo == graph_build_algo::IVF_PQ) { - auto build_params = ivf_pq::index_params::from_dataset(database_view, ps.metric); - if (ps.host_dataset) { - cagra::build_knn_graph( - handle_, database_host_view, knn_graph.view(), 2, build_params); - } else { - cagra::build_knn_graph( - handle_, database_view, knn_graph.view(), 2, build_params); - } - } else { - auto nn_descent_idx_params = experimental::nn_descent::index_params{}; - nn_descent_idx_params.graph_degree = index_params.intermediate_graph_degree; - nn_descent_idx_params.intermediate_graph_degree = index_params.intermediate_graph_degree; - - if (ps.host_dataset) { - cagra::build_knn_graph( - handle_, database_host_view, knn_graph.view(), nn_descent_idx_params); - } else { - cagra::build_knn_graph( - handle_, database_host_view, knn_graph.view(), nn_descent_idx_params); - } - } - - handle_.sync_stream(); - ASSERT_TRUE(CheckOrder(knn_graph.view(), database_host.view(), ps.metric)); - - if (ps.metric != raft::distance::DistanceType::InnerProduct) { - RandomSuffle(knn_graph.view()); - - cagra::sort_knn_graph(handle_, database_view, knn_graph.view()); - handle_.sync_stream(); - - ASSERT_TRUE(CheckOrder(knn_graph.view(), database_host.view(), ps.metric)); - } - } - } - - void SetUp() override - { - database.resize(((size_t)ps.n_rows) * ps.dim, handle_.get_stream()); - raft::random::RngState r(1234ULL); - if constexpr (std::is_same_v || std::is_same_v) { - GenerateRoundingErrorFreeDataset(handle_, database.data(), ps.n_rows, ps.dim, r, false); - } else { - raft::random::uniformInt( - handle_, r, database.data(), ps.n_rows * ps.dim, DataT(1), DataT(20)); - } - handle_.sync_stream(); - } - - void TearDown() override - { - handle_.sync_stream(); - database.resize(0, handle_.get_stream()); - } - - private: - raft::device_resources handle_; - AnnCagraInputs ps; - rmm::device_uvector database; -}; - -template -class AnnCagraFilterTest : public ::testing::TestWithParam { - public: - AnnCagraFilterTest() - : stream_(resource::get_cuda_stream(handle_)), - ps(::testing::TestWithParam::GetParam()), - database(0, stream_), - search_queries(0, stream_) - { - } - - protected: - void testCagraFilter() - { - if (ps.metric == distance::InnerProduct && ps.build_algo == graph_build_algo::NN_DESCENT) - GTEST_SKIP(); - - size_t queries_size = ps.n_queries * ps.k; - std::vector indices_Cagra(queries_size); - std::vector indices_naive(queries_size); - std::vector distances_Cagra(queries_size); - std::vector distances_naive(queries_size); - - { - rmm::device_uvector distances_naive_dev(queries_size, stream_); - rmm::device_uvector indices_naive_dev(queries_size, stream_); - auto* database_filtered_ptr = database.data() + test_cagra_sample_filter::offset * ps.dim; - naive_knn(handle_, - distances_naive_dev.data(), - indices_naive_dev.data(), - search_queries.data(), - database_filtered_ptr, - ps.n_queries, - ps.n_rows - test_cagra_sample_filter::offset, - ps.dim, - ps.k, - ps.metric); - raft::linalg::addScalar(indices_naive_dev.data(), - indices_naive_dev.data(), - IdxT(test_cagra_sample_filter::offset), - queries_size, - stream_); - update_host(distances_naive.data(), distances_naive_dev.data(), queries_size, stream_); - update_host(indices_naive.data(), indices_naive_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - { - rmm::device_uvector distances_dev(queries_size, stream_); - rmm::device_uvector indices_dev(queries_size, stream_); - - { - cagra::index_params index_params; - index_params.metric = ps.metric; // Note: currently ony the cagra::index_params metric is - // not used for knn_graph building. - index_params.nn_descent_niter = 50; - cagra::search_params search_params; - search_params.algo = ps.algo; - search_params.max_queries = ps.max_queries; - search_params.team_size = ps.team_size; - search_params.hashmap_mode = cagra::hash_mode::HASH; - - // TODO: setting search_params.itopk_size here breaks the filter tests, but is required for - // k>1024 skip these tests until fixed - if (ps.k >= 1024) { GTEST_SKIP(); } - // search_params.itopk_size = ps.itopk_size; - - auto database_view = raft::make_device_matrix_view( - (const DataT*)database.data(), ps.n_rows, ps.dim); - - cagra::index index(handle_); - if (ps.host_dataset) { - auto database_host = raft::make_host_matrix(ps.n_rows, ps.dim); - raft::copy(database_host.data_handle(), database.data(), database.size(), stream_); - auto database_host_view = raft::make_host_matrix_view( - (const DataT*)database_host.data_handle(), ps.n_rows, ps.dim); - index = cagra::build(handle_, index_params, database_host_view); - } else { - index = cagra::build(handle_, index_params, database_view); - } - - if (!ps.include_serialized_dataset) { index.update_dataset(handle_, database_view); } - - auto search_queries_view = raft::make_device_matrix_view( - search_queries.data(), ps.n_queries, ps.dim); - auto indices_out_view = - raft::make_device_matrix_view(indices_dev.data(), ps.n_queries, ps.k); - auto dists_out_view = raft::make_device_matrix_view( - distances_dev.data(), ps.n_queries, ps.k); - - cagra::search_with_filtering(handle_, - search_params, - index, - search_queries_view, - indices_out_view, - dists_out_view, - test_cagra_sample_filter()); - update_host(distances_Cagra.data(), distances_dev.data(), queries_size, stream_); - update_host(indices_Cagra.data(), indices_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - // Test filter - bool unacceptable_node = false; - for (int q = 0; q < ps.n_queries; q++) { - for (int i = 0; i < ps.k; i++) { - const auto n = indices_Cagra[q * ps.k + i]; - unacceptable_node = unacceptable_node | !test_cagra_sample_filter()(q, n); - } - } - EXPECT_FALSE(unacceptable_node); - - double min_recall = ps.min_recall; - // TODO(mfoerster): re-enable uniquenes test - EXPECT_TRUE(eval_neighbours(indices_naive, - indices_Cagra, - distances_naive, - distances_Cagra, - ps.n_queries, - ps.k, - 0.003, - min_recall, - false)); - EXPECT_TRUE(eval_distances(handle_, - database.data(), - search_queries.data(), - indices_dev.data(), - distances_dev.data(), - ps.n_rows, - ps.dim, - ps.n_queries, - ps.k, - ps.metric, - 1.0e-4)); - } - } - - void testCagraRemoved() - { - if (ps.metric == distance::InnerProduct && ps.build_algo == graph_build_algo::NN_DESCENT) - GTEST_SKIP(); - - size_t queries_size = ps.n_queries * ps.k; - std::vector indices_Cagra(queries_size); - std::vector indices_naive(queries_size); - std::vector distances_Cagra(queries_size); - std::vector distances_naive(queries_size); - - { - rmm::device_uvector distances_naive_dev(queries_size, stream_); - rmm::device_uvector indices_naive_dev(queries_size, stream_); - auto* database_filtered_ptr = database.data() + test_cagra_sample_filter::offset * ps.dim; - naive_knn(handle_, - distances_naive_dev.data(), - indices_naive_dev.data(), - search_queries.data(), - database_filtered_ptr, - ps.n_queries, - ps.n_rows - test_cagra_sample_filter::offset, - ps.dim, - ps.k, - ps.metric); - raft::linalg::addScalar(indices_naive_dev.data(), - indices_naive_dev.data(), - IdxT(test_cagra_sample_filter::offset), - queries_size, - stream_); - update_host(distances_naive.data(), distances_naive_dev.data(), queries_size, stream_); - update_host(indices_naive.data(), indices_naive_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - { - rmm::device_uvector distances_dev(queries_size, stream_); - rmm::device_uvector indices_dev(queries_size, stream_); - - { - cagra::index_params index_params; - index_params.metric = ps.metric; // Note: currently ony the cagra::index_params metric is - // not used for knn_graph building. - index_params.nn_descent_niter = 50; - cagra::search_params search_params; - search_params.algo = ps.algo; - search_params.max_queries = ps.max_queries; - search_params.team_size = ps.team_size; - search_params.hashmap_mode = cagra::hash_mode::HASH; - - // TODO: setting search_params.itopk_size here breaks the filter tests, but is required for - // k>1024 skip these tests until fixed - if (ps.k >= 1024) { GTEST_SKIP(); } - // search_params.itopk_size = ps.itopk_size; - - auto database_view = raft::make_device_matrix_view( - (const DataT*)database.data(), ps.n_rows, ps.dim); - - cagra::index index(handle_); - if (ps.host_dataset) { - auto database_host = raft::make_host_matrix(ps.n_rows, ps.dim); - raft::copy(database_host.data_handle(), database.data(), database.size(), stream_); - auto database_host_view = raft::make_host_matrix_view( - (const DataT*)database_host.data_handle(), ps.n_rows, ps.dim); - index = cagra::build(handle_, index_params, database_host_view); - } else { - index = cagra::build(handle_, index_params, database_view); - } - - if (!ps.include_serialized_dataset) { index.update_dataset(handle_, database_view); } - - auto search_queries_view = raft::make_device_matrix_view( - search_queries.data(), ps.n_queries, ps.dim); - auto indices_out_view = - raft::make_device_matrix_view(indices_dev.data(), ps.n_queries, ps.k); - auto dists_out_view = raft::make_device_matrix_view( - distances_dev.data(), ps.n_queries, ps.k); - auto removed_indices = - raft::make_device_vector(handle_, test_cagra_sample_filter::offset); - thrust::sequence( - resource::get_thrust_policy(handle_), - thrust::device_pointer_cast(removed_indices.data_handle()), - thrust::device_pointer_cast(removed_indices.data_handle() + removed_indices.extent(0))); - resource::sync_stream(handle_); - raft::core::bitset removed_indices_bitset( - handle_, removed_indices.view(), ps.n_rows); - cagra::search_with_filtering( - handle_, - search_params, - index, - search_queries_view, - indices_out_view, - dists_out_view, - raft::neighbors::filtering::bitset_filter(removed_indices_bitset.view())); - update_host(distances_Cagra.data(), distances_dev.data(), queries_size, stream_); - update_host(indices_Cagra.data(), indices_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - double min_recall = ps.min_recall; - // TODO(mfoerster): re-enable uniquenes test - EXPECT_TRUE(eval_neighbours(indices_naive, - indices_Cagra, - distances_naive, - distances_Cagra, - ps.n_queries, - ps.k, - 0.003, - min_recall, - false)); - EXPECT_TRUE(eval_distances(handle_, - database.data(), - search_queries.data(), - indices_dev.data(), - distances_dev.data(), - ps.n_rows, - ps.dim, - ps.n_queries, - ps.k, - ps.metric, - 1.0e-4)); - } - } - - void SetUp() override - { - database.resize(((size_t)ps.n_rows) * ps.dim, stream_); - search_queries.resize(ps.n_queries * ps.dim, stream_); - raft::random::RngState r(1234ULL); - InitDataset(handle_, database.data(), ps.n_rows, ps.dim, ps.metric, r); - InitDataset(handle_, search_queries.data(), ps.n_queries, ps.dim, ps.metric, r); - resource::sync_stream(handle_); - } - - void TearDown() override - { - resource::sync_stream(handle_); - database.resize(0, stream_); - search_queries.resize(0, stream_); - } - - private: - raft::resources handle_; - rmm::cuda_stream_view stream_; - AnnCagraInputs ps; - rmm::device_uvector database; - rmm::device_uvector search_queries; -}; - -inline std::vector generate_inputs() -{ - // TODO(tfeher): test MULTI_CTA kernel with search_width > 1 to allow multiple CTA per queries - std::vector inputs = raft::util::itertools::product( - {100}, - {1000}, - {1, 8, 17, 1599}, - {16}, // k - {graph_build_algo::IVF_PQ, graph_build_algo::NN_DESCENT}, - {search_algo::SINGLE_CTA, search_algo::MULTI_CTA, search_algo::MULTI_KERNEL}, - {0, 1, 10, 100}, // query size - {0}, - {256}, - {1}, - {raft::distance::DistanceType::L2Expanded, raft::distance::DistanceType::InnerProduct}, - {false}, - {true}, - {0.995}); - - auto inputs2 = raft::util::itertools::product( - {100}, - {1000}, - {1, 8, 17, 1599}, - {1}, // k - {graph_build_algo::IVF_PQ, graph_build_algo::NN_DESCENT}, - {search_algo::SINGLE_CTA, search_algo::MULTI_CTA, search_algo::MULTI_KERNEL}, - {0, 1, 10, 100}, // query size - {0}, - {256}, - {1}, - {raft::distance::DistanceType::L2Expanded, raft::distance::DistanceType::InnerProduct}, - {false}, - {true}, - {99. / 100} - // smaller threshould than the other test cases because it is too strict for Top-1 search - ); - inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); - - inputs2 = raft::util::itertools::product( - {100}, - {1000}, - {1, 3, 5, 7, 8, 17, 64, 128, 137, 192, 256, 512, 619, 1024}, // dim - {16}, // k - {graph_build_algo::IVF_PQ, graph_build_algo::NN_DESCENT}, - {search_algo::AUTO}, - {10}, - {0}, - {64}, - {1}, - {raft::distance::DistanceType::L2Expanded, raft::distance::DistanceType::InnerProduct}, - {false}, - {true}, - {0.995}); - inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); - inputs2 = raft::util::itertools::product( - {100}, - {1000}, - {64}, - {16}, - {graph_build_algo::IVF_PQ, graph_build_algo::NN_DESCENT}, - {search_algo::AUTO}, - {10}, - {0, 4, 8, 16, 32}, // team_size - {64}, - {1}, - {raft::distance::DistanceType::L2Expanded, raft::distance::DistanceType::InnerProduct}, - {false}, - {false}, - {0.995}); - inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); - - inputs2 = raft::util::itertools::product( - {100}, - {1000}, - {64}, - {16}, - {graph_build_algo::IVF_PQ, graph_build_algo::NN_DESCENT}, - {search_algo::AUTO}, - {10}, - {0}, // team_size - {32, 64, 128, 256, 512, 768}, - {1}, - {raft::distance::DistanceType::L2Expanded, raft::distance::DistanceType::InnerProduct}, - {false}, - {true}, - {0.995}); - inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); - - inputs2 = raft::util::itertools::product( - {100}, - {10000, 20000}, - {32}, - {10}, - {graph_build_algo::IVF_PQ, graph_build_algo::NN_DESCENT}, - {search_algo::AUTO}, - {10}, - {0}, // team_size - {64}, - {1}, - {raft::distance::DistanceType::L2Expanded, raft::distance::DistanceType::InnerProduct}, - {false, true}, - {false}, - {0.995}); - inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); - - inputs2 = raft::util::itertools::product( - {100}, - {20000}, - {32}, - {2048}, // k - {graph_build_algo::NN_DESCENT}, - {search_algo::AUTO}, - {10}, - {0}, - {4096}, // itopk_size - {1}, - {raft::distance::DistanceType::L2Expanded, raft::distance::DistanceType::InnerProduct}, - {false}, - {false}, - {0.995}); - inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); - - return inputs; -} - -const std::vector inputs = generate_inputs(); - -} // namespace raft::neighbors::cagra diff --git a/cpp/test/neighbors/ann_cagra/search_kernel_uint64_t.cuh b/cpp/test/neighbors/ann_cagra/search_kernel_uint64_t.cuh deleted file mode 100644 index 412e71bff1..0000000000 --- a/cpp/test/neighbors/ann_cagra/search_kernel_uint64_t.cuh +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include // none_cagra_sample_filter -#include // RAFT_EXPLICIT - -namespace raft::neighbors::cagra::detail { - -namespace multi_cta_search { -#define instantiate_kernel_selection( \ - DATASET_DESCRIPTOR, TEAM_SIZE, MAX_DATASET_DIM, DATA_T, INDEX_T, DISTANCE_T, SAMPLE_FILTER_T) \ - extern template void \ - select_and_run, \ - SAMPLE_FILTER_T>( \ - raft::neighbors::cagra::detail::DATASET_DESCRIPTOR dataset_desc, \ - raft::device_matrix_view graph, \ - INDEX_T* const topk_indices_ptr, \ - DISTANCE_T* const topk_distances_ptr, \ - const DATA_T* const queries_ptr, \ - const uint32_t num_queries, \ - const INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ - uint32_t topk, \ - uint32_t block_size, \ - uint32_t result_buffer_size, \ - uint32_t smem_size, \ - int64_t hash_bitlen, \ - INDEX_T* hashmap_ptr, \ - uint32_t num_cta_per_query, \ - uint32_t num_random_samplings, \ - uint64_t rand_xor_mask, \ - uint32_t num_seeds, \ - size_t itopk_size, \ - size_t search_width, \ - size_t min_iterations, \ - size_t max_iterations, \ - SAMPLE_FILTER_T sample_filter, \ - raft::distance::DistanceType metric, \ - cudaStream_t stream); - -instantiate_kernel_selection(standard_dataset_descriptor_t, - 32, - 1024, - float, - uint64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection(standard_dataset_descriptor_t, - 8, - 128, - float, - uint64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection(standard_dataset_descriptor_t, - 16, - 256, - float, - uint64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection(standard_dataset_descriptor_t, - 32, - 512, - float, - uint64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -#undef instantiate_kernel_selection -} // namespace multi_cta_search - -namespace single_cta_search { - -#define instantiate_single_cta_select_and_run( \ - DATASET_DESCRIPTOR, TEAM_SIZE, MAX_DATASET_DIM, DATA_T, INDEX_T, DISTANCE_T, SAMPLE_FILTER_T) \ - extern template void \ - select_and_run, \ - SAMPLE_FILTER_T>( \ - raft::neighbors::cagra::detail::DATASET_DESCRIPTOR dataset_desc, \ - raft::device_matrix_view graph, \ - INDEX_T* const topk_indices_ptr, \ - DISTANCE_T* const topk_distances_ptr, \ - const DATA_T* const queries_ptr, \ - const uint32_t num_queries, \ - const INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ - uint32_t topk, \ - uint32_t num_itopk_candidates, \ - uint32_t block_size, \ - uint32_t smem_size, \ - int64_t hash_bitlen, \ - INDEX_T* hashmap_ptr, \ - size_t small_hash_bitlen, \ - size_t small_hash_reset_interval, \ - uint32_t num_random_samplings, \ - uint64_t rand_xor_mask, \ - uint32_t num_seeds, \ - size_t itopk_size, \ - size_t search_width, \ - size_t min_iterations, \ - size_t max_iterations, \ - SAMPLE_FILTER_T sample_filter, \ - raft::distance::DistanceType metric, \ - cudaStream_t stream); - -instantiate_single_cta_select_and_run(standard_dataset_descriptor_t, - 32, - 1024, - float, - uint64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run(standard_dataset_descriptor_t, - 8, - 128, - float, - uint64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run(standard_dataset_descriptor_t, - 16, - 256, - float, - uint64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run(standard_dataset_descriptor_t, - 32, - 512, - float, - uint64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace single_cta_search -} // namespace raft::neighbors::cagra::detail diff --git a/cpp/test/neighbors/ann_cagra/test_float_int64_t.cu b/cpp/test/neighbors/ann_cagra/test_float_int64_t.cu deleted file mode 100644 index ff7e839abf..0000000000 --- a/cpp/test/neighbors/ann_cagra/test_float_int64_t.cu +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_cagra.cuh" -#include "search_kernel_uint64_t.cuh" - -#include - -namespace raft::neighbors::cagra { - -typedef AnnCagraTest AnnCagraTestF_I64; -TEST_P(AnnCagraTestF_I64, AnnCagra) { this->testCagra(); } - -INSTANTIATE_TEST_CASE_P(AnnCagraTest, AnnCagraTestF_I64, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::cagra diff --git a/cpp/test/neighbors/ann_cagra/test_float_uint32_t.cu b/cpp/test/neighbors/ann_cagra/test_float_uint32_t.cu deleted file mode 100644 index 7d29ce4f99..0000000000 --- a/cpp/test/neighbors/ann_cagra/test_float_uint32_t.cu +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_cagra.cuh" - -#include - -namespace raft::neighbors::cagra { - -typedef AnnCagraTest AnnCagraTestF_U32; -TEST_P(AnnCagraTestF_U32, AnnCagra) { this->testCagra(); } - -typedef AnnCagraSortTest AnnCagraSortTestF_U32; -TEST_P(AnnCagraSortTestF_U32, AnnCagraSort) { this->testCagraSort(); } - -typedef AnnCagraFilterTest AnnCagraFilterTestF_U32; -TEST_P(AnnCagraFilterTestF_U32, AnnCagraFilter) -{ - this->testCagraFilter(); - this->testCagraRemoved(); -} - -INSTANTIATE_TEST_CASE_P(AnnCagraTest, AnnCagraTestF_U32, ::testing::ValuesIn(inputs)); -INSTANTIATE_TEST_CASE_P(AnnCagraSortTest, AnnCagraSortTestF_U32, ::testing::ValuesIn(inputs)); -INSTANTIATE_TEST_CASE_P(AnnCagraFilterTest, AnnCagraFilterTestF_U32, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::cagra diff --git a/cpp/test/neighbors/ann_cagra/test_half_int64_t.cu b/cpp/test/neighbors/ann_cagra/test_half_int64_t.cu deleted file mode 100644 index bcdd95bece..0000000000 --- a/cpp/test/neighbors/ann_cagra/test_half_int64_t.cu +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_cagra.cuh" -#include "search_kernel_uint64_t.cuh" - -#include - -namespace raft::neighbors::cagra { - -typedef AnnCagraTest AnnCagraTestH_I64; -TEST_P(AnnCagraTestH_I64, AnnCagra) { this->testCagra(); } - -INSTANTIATE_TEST_CASE_P(AnnCagraTest, AnnCagraTestH_I64, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::cagra diff --git a/cpp/test/neighbors/ann_cagra/test_half_uint32_t.cu b/cpp/test/neighbors/ann_cagra/test_half_uint32_t.cu deleted file mode 100644 index ec7144f8d0..0000000000 --- a/cpp/test/neighbors/ann_cagra/test_half_uint32_t.cu +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_cagra.cuh" - -#include - -namespace raft::neighbors::cagra { - -typedef AnnCagraTest AnnCagraTestH_U32; -TEST_P(AnnCagraTestH_U32, AnnCagra) { this->testCagra(); } - -typedef AnnCagraSortTest AnnCagraSortTestH_U32; -TEST_P(AnnCagraSortTestH_U32, AnnCagraSort) { this->testCagraSort(); } - -typedef AnnCagraFilterTest AnnCagraFilterTestH_U32; -TEST_P(AnnCagraFilterTestH_U32, AnnCagraFilter) -{ - this->testCagraFilter(); - this->testCagraRemoved(); -} - -INSTANTIATE_TEST_CASE_P(AnnCagraTest, AnnCagraTestH_U32, ::testing::ValuesIn(inputs)); -INSTANTIATE_TEST_CASE_P(AnnCagraSortTest, AnnCagraSortTestH_U32, ::testing::ValuesIn(inputs)); -INSTANTIATE_TEST_CASE_P(AnnCagraFilterTest, AnnCagraFilterTestH_U32, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::cagra diff --git a/cpp/test/neighbors/ann_cagra/test_int8_t_uint32_t.cu b/cpp/test/neighbors/ann_cagra/test_int8_t_uint32_t.cu deleted file mode 100644 index b2242d89b1..0000000000 --- a/cpp/test/neighbors/ann_cagra/test_int8_t_uint32_t.cu +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_cagra.cuh" - -#include - -namespace raft::neighbors::cagra { - -typedef AnnCagraTest AnnCagraTestI8_U32; -TEST_P(AnnCagraTestI8_U32, AnnCagra) { this->testCagra(); } -typedef AnnCagraSortTest AnnCagraSortTestI8_U32; -TEST_P(AnnCagraSortTestI8_U32, AnnCagraSort) { this->testCagraSort(); } -typedef AnnCagraFilterTest AnnCagraFilterTestI8_U32; -TEST_P(AnnCagraFilterTestI8_U32, AnnCagraFilter) -{ - this->testCagraFilter(); - this->testCagraRemoved(); -} - -INSTANTIATE_TEST_CASE_P(AnnCagraTest, AnnCagraTestI8_U32, ::testing::ValuesIn(inputs)); -INSTANTIATE_TEST_CASE_P(AnnCagraSortTest, AnnCagraSortTestI8_U32, ::testing::ValuesIn(inputs)); -INSTANTIATE_TEST_CASE_P(AnnCagraFilterTest, AnnCagraFilterTestI8_U32, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::cagra diff --git a/cpp/test/neighbors/ann_cagra/test_uint8_t_uint32_t.cu b/cpp/test/neighbors/ann_cagra/test_uint8_t_uint32_t.cu deleted file mode 100644 index 302b2bec18..0000000000 --- a/cpp/test/neighbors/ann_cagra/test_uint8_t_uint32_t.cu +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_cagra.cuh" - -#include - -namespace raft::neighbors::cagra { - -typedef AnnCagraTest AnnCagraTestU8_U32; -TEST_P(AnnCagraTestU8_U32, AnnCagra) { this->testCagra(); } - -typedef AnnCagraSortTest AnnCagraSortTestU8_U32; -TEST_P(AnnCagraSortTestU8_U32, AnnCagraSort) { this->testCagraSort(); } - -typedef AnnCagraFilterTest AnnCagraFilterTestU8_U32; -TEST_P(AnnCagraFilterTestU8_U32, AnnCagraSort) -{ - this->testCagraFilter(); - this->testCagraRemoved(); -} - -INSTANTIATE_TEST_CASE_P(AnnCagraTest, AnnCagraTestU8_U32, ::testing::ValuesIn(inputs)); -INSTANTIATE_TEST_CASE_P(AnnCagraSortTest, AnnCagraSortTestU8_U32, ::testing::ValuesIn(inputs)); -INSTANTIATE_TEST_CASE_P(AnnCagraFilterTest, AnnCagraFilterTestU8_U32, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::cagra diff --git a/cpp/test/neighbors/ann_cagra_vpq.cuh b/cpp/test/neighbors/ann_cagra_vpq.cuh deleted file mode 100644 index 6b24bca921..0000000000 --- a/cpp/test/neighbors/ann_cagra_vpq.cuh +++ /dev/null @@ -1,336 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "../test_utils.cuh" -#include "ann_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include - -#include - -#include -#include -#include -#include -#include - -namespace { -template -void GenerateDataset(T* const dataset_ptr, - T* const query_ptr, - const std::size_t dataset_size, - const std::size_t query_size, - const std::size_t dim, - const std::size_t num_centers, - cudaStream_t cuda_stream) -{ - auto center_list = raft::make_host_matrix(num_centers, dim); - auto host_dataset = raft::make_host_matrix(std::max(dataset_size, query_size), dim); - - std::normal_distribution dist(0, 1); - std::mt19937 mt(0); - for (std::size_t i = 0; i < center_list.size(); i++) { - center_list.data_handle()[i] = dist(mt); - } - - std::uniform_int_distribution i_dist(0, num_centers - 1); - for (std::size_t i = 0; i < dataset_size; i++) { - const auto center_index = i_dist(mt); - for (std::size_t j = 0; j < dim; j++) { - host_dataset.data_handle()[i * dim + j] = - center_list.data_handle()[center_index + j] + dist(mt) * 1e-1; - } - } - raft::copy(dataset_ptr, host_dataset.data_handle(), dataset_size * dim, cuda_stream); - - for (std::size_t i = 0; i < query_size; i++) { - const auto center_index = i_dist(mt); - for (std::size_t j = 0; j < dim; j++) { - host_dataset.data_handle()[i * dim + j] = - center_list.data_handle()[center_index + j] + dist(mt) * 1e-1; - } - } - raft::copy(query_ptr, host_dataset.data_handle(), query_size * dim, cuda_stream); -} -} // namespace - -namespace raft::neighbors::cagra { -struct AnnCagraVpqInputs { - int n_queries; - int n_rows; - int dim; - int k; - int pq_len; - int pq_bits; - graph_build_algo build_algo; - search_algo algo; - int max_queries; - int team_size; - int itopk_size; - int search_width; - raft::distance::DistanceType metric; - bool host_dataset; - bool include_serialized_dataset; - // std::optional - double min_recall; // = std::nullopt; -}; - -inline ::std::ostream& operator<<(::std::ostream& os, const AnnCagraVpqInputs& p) -{ - std::vector algo = {"single-cta", "multi_cta", "multi_kernel", "auto"}; - std::vector build_algo = {"IVF_PQ", "NN_DESCENT"}; - os << "{n_queries=" << p.n_queries << ", dataset shape=" << p.n_rows << "x" << p.dim - << ", k=" << p.k << ", pq_bits=" << p.pq_bits << ", pq_len=" << p.pq_len << ", " - << algo.at((int)p.algo) << ", max_queries=" << p.max_queries << ", itopk_size=" << p.itopk_size - << ", search_width=" << p.search_width << ", metric=" << static_cast(p.metric) - << (p.host_dataset ? ", host" : ", device") - << ", build_algo=" << build_algo.at((int)p.build_algo) << '}' << std::endl; - return os; -} - -template -class AnnCagraVpqTest : public ::testing::TestWithParam { - public: - AnnCagraVpqTest() - : stream_(resource::get_cuda_stream(handle_)), - ps(::testing::TestWithParam::GetParam()), - database(0, stream_), - search_queries(0, stream_) - { - } - - protected: - void testCagra() - { - size_t queries_size = ps.n_queries * ps.k; - std::vector indices_Cagra(queries_size); - std::vector indices_naive(queries_size); - std::vector distances_Cagra(queries_size); - std::vector distances_naive(queries_size); - - { - rmm::device_uvector distances_naive_dev(queries_size, stream_); - rmm::device_uvector indices_naive_dev(queries_size, stream_); - naive_knn(handle_, - distances_naive_dev.data(), - indices_naive_dev.data(), - search_queries.data(), - database.data(), - ps.n_queries, - ps.n_rows, - ps.dim, - ps.k, - ps.metric); - update_host(distances_naive.data(), distances_naive_dev.data(), queries_size, stream_); - update_host(indices_naive.data(), indices_naive_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - const auto vpq_k = ps.k * 4; - { - rmm::device_uvector distances_dev(vpq_k * ps.n_queries, stream_); - rmm::device_uvector indices_dev(vpq_k * ps.n_queries, stream_); - - { - if ((ps.dim % ps.pq_len) != 0) { - // TODO: remove this requirement in the algorithm. - GTEST_SKIP() << "(TODO) At the moment dim, (" << ps.dim - << ") must be a multiple of pq_len (" << ps.pq_len << ")"; - } - cagra::index_params index_params; - index_params.compression = vpq_params{.pq_bits = static_cast(ps.pq_bits), - .pq_dim = static_cast(ps.dim / ps.pq_len)}; - index_params.metric = ps.metric; // Note: currently ony the cagra::index_params metric is - // not used for knn_graph building. - index_params.build_algo = ps.build_algo; - cagra::search_params search_params; - search_params.algo = ps.algo; - search_params.max_queries = ps.max_queries; - search_params.team_size = ps.team_size; - search_params.itopk_size = ps.itopk_size; - - auto database_view = - raft::make_device_matrix_view(database.data(), ps.n_rows, ps.dim); - - { - cagra::index index(handle_); - if (ps.host_dataset) { - auto database_host = raft::make_host_matrix(ps.n_rows, ps.dim); - raft::copy(database_host.data_handle(), database.data(), database.size(), stream_); - auto database_host_view = raft::make_host_matrix_view( - database_host.data_handle(), ps.n_rows, ps.dim); - index = cagra::build(handle_, index_params, database_host_view); - } else { - index = cagra::build(handle_, index_params, database_view); - }; - cagra::serialize(handle_, "cagra_index", index, ps.include_serialized_dataset); - } - - auto index = cagra::deserialize(handle_, "cagra_index"); - if (!ps.include_serialized_dataset) { index.update_dataset(handle_, database_view); } - - // CAGRA-Q sanity check: we've built the right index type - auto* vpq_dataset = - dynamic_cast*>(&index.data()); - EXPECT_NE(vpq_dataset, nullptr) - << "Expected VPQ dataset, because we're testing CAGRA-Q here."; - - auto search_queries_view = raft::make_device_matrix_view( - search_queries.data(), ps.n_queries, ps.dim); - auto indices_out_view = - raft::make_device_matrix_view(indices_dev.data(), ps.n_queries, vpq_k); - auto dists_out_view = raft::make_device_matrix_view( - distances_dev.data(), ps.n_queries, vpq_k); - - cagra::search( - handle_, search_params, index, search_queries_view, indices_out_view, dists_out_view); - - { - auto host_dataset = raft::make_host_matrix(ps.n_rows, ps.dim); - raft::copy( - host_dataset.data_handle(), (const DataT*)database.data(), ps.n_rows * ps.dim, stream_); - - auto host_queries = raft::make_host_matrix(ps.n_queries, ps.dim); - raft::copy(host_queries.data_handle(), - (const DataT*)search_queries_view.data_handle(), - ps.n_queries * ps.dim, - stream_); - - auto host_index_candidate = raft::make_host_matrix(ps.n_queries, vpq_k); - raft::copy(host_index_candidate.data_handle(), - indices_out_view.data_handle(), - ps.n_queries * vpq_k, - stream_); - - auto host_indices_Cagra_view = - raft::make_host_matrix_view(indices_Cagra.data(), ps.n_queries, ps.k); - - auto host_dists_Cagra_view = - raft::make_host_matrix_view(distances_Cagra.data(), ps.n_queries, ps.k); - - resource::sync_stream(handle_); - - raft::neighbors::refine(handle_, - raft::make_const_mdspan(host_dataset.view()), - raft::make_const_mdspan(host_queries.view()), - raft::make_const_mdspan(host_index_candidate.view()), - host_indices_Cagra_view, - host_dists_Cagra_view, - ps.metric); - - raft::copy(indices_dev.data(), - host_indices_Cagra_view.data_handle(), - ps.k * ps.n_queries, - stream_); - raft::copy(distances_dev.data(), - host_dists_Cagra_view.data_handle(), - ps.k * ps.n_queries, - stream_); - resource::sync_stream(handle_); - } - } - - double min_recall = ps.min_recall; - EXPECT_TRUE(eval_neighbours(indices_naive, - indices_Cagra, - distances_naive, - distances_Cagra, - ps.n_queries, - ps.k, - 0.003, - min_recall)); - EXPECT_TRUE(eval_distances(handle_, - database.data(), - search_queries.data(), - indices_dev.data(), - distances_dev.data(), - ps.n_rows, - ps.dim, - ps.n_queries, - ps.k, - ps.metric, - 1.0e-4)); - } - } - - void SetUp() override - { - database.resize(((size_t)ps.n_rows) * ps.dim, stream_); - search_queries.resize(ps.n_queries * ps.dim, stream_); - GenerateDataset(database.data(), - search_queries.data(), - ps.n_rows, - ps.n_queries, - ps.dim, - static_cast(std::sqrt(ps.n_rows)), - stream_); - resource::sync_stream(handle_); - } - - void TearDown() override - { - resource::sync_stream(handle_); - database.resize(0, stream_); - search_queries.resize(0, stream_); - } - - private: - raft::resources handle_; - rmm::cuda_stream_view stream_; - AnnCagraVpqInputs ps; - rmm::device_uvector database; - rmm::device_uvector search_queries; -}; - -const std::vector vpq_inputs = raft::util::itertools::product( - {100}, // n_queries - {1000, 10000}, // n_rows - {128, 132, 192, 256, 512, 768}, // dim - {8, 12}, // k - {2, 4}, // pq_len - {8}, // pq_bits - {graph_build_algo::NN_DESCENT}, // build_algo - {search_algo::SINGLE_CTA, search_algo::MULTI_CTA}, // algo - {0}, // max_queries - {0}, // team_size - {512}, // itopk_size - {1}, // search_width - {raft::distance::DistanceType::L2Expanded}, // metric - {false}, // host_dataset - {true}, // include_serialized_dataset - {0.8} // min_recall -); - -} // namespace raft::neighbors::cagra diff --git a/cpp/test/neighbors/ann_cagra_vpq/test_float_int64_t.cu b/cpp/test/neighbors/ann_cagra_vpq/test_float_int64_t.cu deleted file mode 100644 index f60edb5ed6..0000000000 --- a/cpp/test/neighbors/ann_cagra_vpq/test_float_int64_t.cu +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#undef RAFT_EXPLICIT_INSTANTIATE_ONLY -#include "../ann_cagra_vpq.cuh" - -#include - -namespace raft::neighbors::cagra { - -typedef AnnCagraVpqTest AnnCagraVpqTestF_I64; -TEST_P(AnnCagraVpqTestF_I64, AnnCagraVpq) { this->testCagra(); } - -INSTANTIATE_TEST_CASE_P(AnnCagraVpqTest, AnnCagraVpqTestF_I64, ::testing::ValuesIn(vpq_inputs)); - -} // namespace raft::neighbors::cagra diff --git a/cpp/test/neighbors/ann_cagra_vpq/test_float_uint32_t.cu b/cpp/test/neighbors/ann_cagra_vpq/test_float_uint32_t.cu deleted file mode 100644 index 19d3f32250..0000000000 --- a/cpp/test/neighbors/ann_cagra_vpq/test_float_uint32_t.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_cagra_vpq.cuh" - -#include - -namespace raft::neighbors::cagra { - -typedef AnnCagraVpqTest AnnCagraVpqTestF_U32; -TEST_P(AnnCagraVpqTestF_U32, AnnCagraVpq) { this->testCagra(); } - -INSTANTIATE_TEST_CASE_P(AnnCagraVpqTest, AnnCagraVpqTestF_U32, ::testing::ValuesIn(vpq_inputs)); - -} // namespace raft::neighbors::cagra diff --git a/cpp/test/neighbors/ann_ivf_flat.cuh b/cpp/test/neighbors/ann_ivf_flat.cuh deleted file mode 100644 index de6af589fa..0000000000 --- a/cpp/test/neighbors/ann_ivf_flat.cuh +++ /dev/null @@ -1,675 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "../test_utils.cuh" -#include "ann_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include - -#include -#include - -#include - -#include -#include -#include - -namespace raft::neighbors::ivf_flat { - -struct test_ivf_sample_filter { - static constexpr unsigned offset = 300; -}; - -template -struct AnnIvfFlatInputs { - IdxT num_queries; - IdxT num_db_vecs; - IdxT dim; - IdxT k; - IdxT nprobe; - IdxT nlist; - raft::distance::DistanceType metric; - bool adaptive_centers; - bool host_dataset; -}; - -template -::std::ostream& operator<<(::std::ostream& os, const AnnIvfFlatInputs& p) -{ - os << "{ " << p.num_queries << ", " << p.num_db_vecs << ", " << p.dim << ", " << p.k << ", " - << p.nprobe << ", " << p.nlist << ", " << static_cast(p.metric) << ", " - << p.adaptive_centers << ", " << p.host_dataset << '}' << std::endl; - return os; -} - -template -class AnnIVFFlatTest : public ::testing::TestWithParam> { - public: - AnnIVFFlatTest() - : stream_(resource::get_cuda_stream(handle_)), - ps(::testing::TestWithParam>::GetParam()), - database(0, stream_), - search_queries(0, stream_) - { - } - - void testIVFFlat() - { - size_t queries_size = ps.num_queries * ps.k; - std::vector indices_ivfflat(queries_size); - std::vector indices_naive(queries_size); - std::vector distances_ivfflat(queries_size); - std::vector distances_naive(queries_size); - - { - rmm::device_uvector distances_naive_dev(queries_size, stream_); - rmm::device_uvector indices_naive_dev(queries_size, stream_); - naive_knn(handle_, - distances_naive_dev.data(), - indices_naive_dev.data(), - search_queries.data(), - database.data(), - ps.num_queries, - ps.num_db_vecs, - ps.dim, - ps.k, - ps.metric); - update_host(distances_naive.data(), distances_naive_dev.data(), queries_size, stream_); - update_host(indices_naive.data(), indices_naive_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - { - // unless something is really wrong with clustering, this could serve as a lower bound on - // recall - double min_recall = static_cast(ps.nprobe) / static_cast(ps.nlist); - - rmm::device_uvector distances_ivfflat_dev(queries_size, stream_); - rmm::device_uvector indices_ivfflat_dev(queries_size, stream_); - - { - // legacy interface - raft::spatial::knn::IVFFlatParam ivfParams; - ivfParams.nprobe = ps.nprobe; - ivfParams.nlist = ps.nlist; - raft::spatial::knn::knnIndex index; - - approx_knn_build_index(handle_, - &index, - dynamic_cast(&ivfParams), - ps.metric, - (IdxT)0, - database.data(), - ps.num_db_vecs, - ps.dim); - - resource::sync_stream(handle_); - approx_knn_search(handle_, - distances_ivfflat_dev.data(), - indices_ivfflat_dev.data(), - &index, - ps.k, - search_queries.data(), - ps.num_queries); - - update_host(distances_ivfflat.data(), distances_ivfflat_dev.data(), queries_size, stream_); - update_host(indices_ivfflat.data(), indices_ivfflat_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - ASSERT_TRUE(eval_neighbours(indices_naive, - indices_ivfflat, - distances_naive, - distances_ivfflat, - ps.num_queries, - ps.k, - 0.001, - min_recall)); - { - ivf_flat::index_params index_params; - ivf_flat::search_params search_params; - index_params.n_lists = ps.nlist; - index_params.metric = ps.metric; - index_params.adaptive_centers = ps.adaptive_centers; - search_params.n_probes = ps.nprobe; - - index_params.add_data_on_build = false; - index_params.kmeans_trainset_fraction = 0.5; - index_params.metric_arg = 0; - - ivf_flat::index idx(handle_, index_params, ps.dim); - ivf_flat::index index_2(handle_, index_params, ps.dim); - - if (!ps.host_dataset) { - auto database_view = raft::make_device_matrix_view( - (const DataT*)database.data(), ps.num_db_vecs, ps.dim); - idx = ivf_flat::build(handle_, index_params, database_view); - rmm::device_uvector vector_indices(ps.num_db_vecs, stream_); - thrust::sequence(resource::get_thrust_policy(handle_), - thrust::device_pointer_cast(vector_indices.data()), - thrust::device_pointer_cast(vector_indices.data() + ps.num_db_vecs)); - resource::sync_stream(handle_); - - IdxT half_of_data = ps.num_db_vecs / 2; - - auto half_of_data_view = raft::make_device_matrix_view( - (const DataT*)database.data(), half_of_data, ps.dim); - - const std::optional> no_opt = std::nullopt; - index_2 = ivf_flat::extend(handle_, half_of_data_view, no_opt, idx); - - auto new_half_of_data_view = raft::make_device_matrix_view( - database.data() + half_of_data * ps.dim, IdxT(ps.num_db_vecs) - half_of_data, ps.dim); - - auto new_half_of_data_indices_view = raft::make_device_vector_view( - vector_indices.data() + half_of_data, IdxT(ps.num_db_vecs) - half_of_data); - - ivf_flat::extend(handle_, - new_half_of_data_view, - std::make_optional>( - new_half_of_data_indices_view), - &index_2); - - } else { - auto host_database = raft::make_host_matrix(ps.num_db_vecs, ps.dim); - raft::copy( - host_database.data_handle(), database.data(), ps.num_db_vecs * ps.dim, stream_); - idx = - ivf_flat::build(handle_, index_params, raft::make_const_mdspan(host_database.view())); - - auto vector_indices = raft::make_host_vector(handle_, ps.num_db_vecs); - std::iota(vector_indices.data_handle(), vector_indices.data_handle() + ps.num_db_vecs, 0); - - IdxT half_of_data = ps.num_db_vecs / 2; - - auto half_of_data_view = raft::make_host_matrix_view( - (const DataT*)host_database.data_handle(), half_of_data, ps.dim); - - const std::optional> no_opt = std::nullopt; - index_2 = ivf_flat::extend(handle_, half_of_data_view, no_opt, idx); - - auto new_half_of_data_view = raft::make_host_matrix_view( - host_database.data_handle() + half_of_data * ps.dim, - IdxT(ps.num_db_vecs) - half_of_data, - ps.dim); - auto new_half_of_data_indices_view = raft::make_host_vector_view( - vector_indices.data_handle() + half_of_data, IdxT(ps.num_db_vecs) - half_of_data); - ivf_flat::extend(handle_, - new_half_of_data_view, - std::make_optional>( - new_half_of_data_indices_view), - &index_2); - } - - auto search_queries_view = raft::make_device_matrix_view( - search_queries.data(), ps.num_queries, ps.dim); - auto indices_out_view = raft::make_device_matrix_view( - indices_ivfflat_dev.data(), ps.num_queries, ps.k); - auto dists_out_view = raft::make_device_matrix_view( - distances_ivfflat_dev.data(), ps.num_queries, ps.k); - ivf_flat::detail::serialize(handle_, "ivf_flat_index", index_2); - - auto index_loaded = ivf_flat::detail::deserialize(handle_, "ivf_flat_index"); - ASSERT_EQ(index_2.size(), index_loaded.size()); - - ivf_flat::search(handle_, - search_params, - index_loaded, - search_queries_view, - indices_out_view, - dists_out_view); - - update_host(distances_ivfflat.data(), distances_ivfflat_dev.data(), queries_size, stream_); - update_host(indices_ivfflat.data(), indices_ivfflat_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - - // Test the centroid invariants - if (index_2.adaptive_centers()) { - // The centers must be up-to-date with the corresponding data - std::vector list_sizes(index_2.n_lists()); - std::vector list_indices(index_2.n_lists()); - rmm::device_uvector centroid(ps.dim, stream_); - raft::copy( - list_sizes.data(), index_2.list_sizes().data_handle(), index_2.n_lists(), stream_); - raft::copy( - list_indices.data(), index_2.inds_ptrs().data_handle(), index_2.n_lists(), stream_); - resource::sync_stream(handle_); - for (uint32_t l = 0; l < index_2.n_lists(); l++) { - if (list_sizes[l] == 0) continue; - rmm::device_uvector cluster_data(list_sizes[l] * ps.dim, stream_); - raft::spatial::knn::detail::utils::copy_selected((IdxT)list_sizes[l], - (IdxT)ps.dim, - database.data(), - list_indices[l], - (IdxT)ps.dim, - cluster_data.data(), - (IdxT)ps.dim, - stream_); - raft::stats::mean( - centroid.data(), cluster_data.data(), ps.dim, list_sizes[l], false, true, stream_); - ASSERT_TRUE(raft::devArrMatch(index_2.centers().data_handle() + ps.dim * l, - centroid.data(), - ps.dim, - raft::CompareApprox(0.001), - stream_)); - } - } else { - // The centers must be immutable - ASSERT_TRUE(raft::devArrMatch(index_2.centers().data_handle(), - idx.centers().data_handle(), - index_2.centers().size(), - raft::Compare(), - stream_)); - } - } - ASSERT_TRUE(eval_neighbours(indices_naive, - indices_ivfflat, - distances_naive, - distances_ivfflat, - ps.num_queries, - ps.k, - 0.001, - min_recall)); - } - } - - void testPacker() - { - ivf_flat::index_params index_params; - ivf_flat::search_params search_params; - index_params.n_lists = ps.nlist; - index_params.metric = ps.metric; - index_params.adaptive_centers = false; - search_params.n_probes = ps.nprobe; - - index_params.add_data_on_build = false; - index_params.kmeans_trainset_fraction = 1.0; - index_params.metric_arg = 0; - - auto database_view = raft::make_device_matrix_view( - (const DataT*)database.data(), ps.num_db_vecs, ps.dim); - - auto idx = ivf_flat::build(handle_, index_params, database_view); - - const std::optional> no_opt = std::nullopt; - index extend_index = ivf_flat::extend(handle_, database_view, no_opt, idx); - - auto list_sizes = raft::make_host_vector(idx.n_lists()); - update_host(list_sizes.data_handle(), - extend_index.list_sizes().data_handle(), - extend_index.n_lists(), - stream_); - resource::sync_stream(handle_); - - auto& lists = idx.lists(); - - // conservative memory allocation for codepacking - auto list_device_spec = list_spec{idx.dim(), false}; - - for (uint32_t label = 0; label < idx.n_lists(); label++) { - uint32_t list_size = list_sizes.data_handle()[label]; - - ivf::resize_list(handle_, lists[label], list_device_spec, list_size, 0); - } - - helpers::recompute_internal_state(handle_, &idx); - - using interleaved_group = Pow2; - - for (uint32_t label = 0; label < idx.n_lists(); label++) { - uint32_t list_size = list_sizes.data_handle()[label]; - - if (list_size > 0) { - uint32_t padded_list_size = interleaved_group::roundUp(list_size); - uint32_t n_elems = padded_list_size * idx.dim(); - auto list_data = lists[label]->data; - auto list_inds = extend_index.lists()[label]->indices; - - // fetch the flat codes - auto flat_codes = make_device_matrix(handle_, list_size, idx.dim()); - - matrix::gather( - handle_, - make_device_matrix_view( - (const DataT*)database.data(), static_cast(ps.num_db_vecs), idx.dim()), - make_device_vector_view((const IdxT*)list_inds.data_handle(), - list_size), - flat_codes.view()); - - helpers::codepacker::pack( - handle_, make_const_mdspan(flat_codes.view()), idx.veclen(), 0, list_data.view()); - - { - auto mask = make_device_vector(handle_, n_elems); - - linalg::map_offset(handle_, - mask.view(), - [dim = idx.dim(), - list_size, - padded_list_size, - chunk_size = util::FastIntDiv(idx.veclen())] __device__(auto i) { - uint32_t max_group_offset = interleaved_group::roundDown(list_size); - if (i < max_group_offset * dim) { return true; } - uint32_t surplus = (i - max_group_offset * dim); - uint32_t ingroup_id = interleaved_group::mod(surplus / chunk_size); - return ingroup_id < (list_size - max_group_offset); - }); - - // ensure that the correct number of indices are masked out - ASSERT_TRUE(thrust::reduce(resource::get_thrust_policy(handle_), - mask.data_handle(), - mask.data_handle() + n_elems, - 0) == list_size * ps.dim); - - auto packed_list_data = make_device_vector(handle_, n_elems); - - linalg::map_offset(handle_, - packed_list_data.view(), - [mask = mask.data_handle(), - list_data = list_data.data_handle()] __device__(uint32_t i) { - if (mask[i]) return list_data[i]; - return DataT{0}; - }); - - auto extend_data = extend_index.lists()[label]->data; - auto extend_data_filtered = make_device_vector(handle_, n_elems); - linalg::map_offset(handle_, - extend_data_filtered.view(), - [mask = mask.data_handle(), - extend_data = extend_data.data_handle()] __device__(uint32_t i) { - if (mask[i]) return extend_data[i]; - return DataT{0}; - }); - - ASSERT_TRUE(raft::devArrMatch(packed_list_data.data_handle(), - extend_data_filtered.data_handle(), - n_elems, - raft::Compare(), - stream_)); - } - - auto unpacked_flat_codes = - make_device_matrix(handle_, list_size, idx.dim()); - - helpers::codepacker::unpack( - handle_, list_data.view(), idx.veclen(), 0, unpacked_flat_codes.view()); - - ASSERT_TRUE(raft::devArrMatch(flat_codes.data_handle(), - unpacked_flat_codes.data_handle(), - list_size * ps.dim, - raft::Compare(), - stream_)); - } - } - } - - void testFilter() - { - size_t queries_size = ps.num_queries * ps.k; - std::vector indices_ivfflat(queries_size); - std::vector indices_naive(queries_size); - std::vector distances_ivfflat(queries_size); - std::vector distances_naive(queries_size); - - { - rmm::device_uvector distances_naive_dev(queries_size, stream_); - rmm::device_uvector indices_naive_dev(queries_size, stream_); - auto* database_filtered_ptr = database.data() + test_ivf_sample_filter::offset * ps.dim; - naive_knn(handle_, - distances_naive_dev.data(), - indices_naive_dev.data(), - search_queries.data(), - database_filtered_ptr, - ps.num_queries, - ps.num_db_vecs - test_ivf_sample_filter::offset, - ps.dim, - ps.k, - ps.metric); - raft::linalg::addScalar(indices_naive_dev.data(), - indices_naive_dev.data(), - IdxT(test_ivf_sample_filter::offset), - queries_size, - stream_); - update_host(distances_naive.data(), distances_naive_dev.data(), queries_size, stream_); - update_host(indices_naive.data(), indices_naive_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - { - // unless something is really wrong with clustering, this could serve as a lower bound on - // recall - double min_recall = static_cast(ps.nprobe) / static_cast(ps.nlist); - - auto distances_ivfflat_dev = raft::make_device_matrix(handle_, ps.num_queries, ps.k); - auto indices_ivfflat_dev = - raft::make_device_matrix(handle_, ps.num_queries, ps.k); - - { - ivf_flat::index_params index_params; - ivf_flat::search_params search_params; - index_params.n_lists = ps.nlist; - index_params.metric = ps.metric; - index_params.adaptive_centers = ps.adaptive_centers; - search_params.n_probes = ps.nprobe; - - index_params.add_data_on_build = true; - index_params.kmeans_trainset_fraction = 0.5; - index_params.metric_arg = 0; - - // Create IVF Flat index - auto database_view = raft::make_device_matrix_view( - (const DataT*)database.data(), ps.num_db_vecs, ps.dim); - auto index = ivf_flat::build(handle_, index_params, database_view); - - // Create Bitset filter - auto removed_indices = - raft::make_device_vector(handle_, test_ivf_sample_filter::offset); - thrust::sequence(resource::get_thrust_policy(handle_), - thrust::device_pointer_cast(removed_indices.data_handle()), - thrust::device_pointer_cast(removed_indices.data_handle() + - test_ivf_sample_filter::offset)); - resource::sync_stream(handle_); - - raft::core::bitset removed_indices_bitset( - handle_, removed_indices.view(), ps.num_db_vecs); - - // Search with the filter - auto search_queries_view = raft::make_device_matrix_view( - search_queries.data(), ps.num_queries, ps.dim); - ivf_flat::search_with_filtering( - handle_, - search_params, - index, - search_queries_view, - indices_ivfflat_dev.view(), - distances_ivfflat_dev.view(), - raft::neighbors::filtering::bitset_filter(removed_indices_bitset.view())); - - update_host( - distances_ivfflat.data(), distances_ivfflat_dev.data_handle(), queries_size, stream_); - update_host( - indices_ivfflat.data(), indices_ivfflat_dev.data_handle(), queries_size, stream_); - resource::sync_stream(handle_); - } - ASSERT_TRUE(eval_neighbours(indices_naive, - indices_ivfflat, - distances_naive, - distances_ivfflat, - ps.num_queries, - ps.k, - 0.001, - min_recall)); - } - } - - void SetUp() override - { - database.resize(ps.num_db_vecs * ps.dim, stream_); - search_queries.resize(ps.num_queries * ps.dim, stream_); - - raft::random::RngState r(1234ULL); - if constexpr (std::is_same{}) { - raft::random::uniform( - handle_, r, database.data(), ps.num_db_vecs * ps.dim, DataT(0.1), DataT(2.0)); - raft::random::uniform( - handle_, r, search_queries.data(), ps.num_queries * ps.dim, DataT(0.1), DataT(2.0)); - } else { - raft::random::uniformInt( - handle_, r, database.data(), ps.num_db_vecs * ps.dim, DataT(1), DataT(20)); - raft::random::uniformInt( - handle_, r, search_queries.data(), ps.num_queries * ps.dim, DataT(1), DataT(20)); - } - resource::sync_stream(handle_); - } - - void TearDown() override - { - resource::sync_stream(handle_); - database.resize(0, stream_); - search_queries.resize(0, stream_); - } - - private: - raft::resources handle_; - rmm::cuda_stream_view stream_; - AnnIvfFlatInputs ps; - rmm::device_uvector database; - rmm::device_uvector search_queries; -}; - -const std::vector> inputs = { - // test various dims (aligned and not aligned to vector sizes) - {1000, 10000, 1, 16, 40, 1024, raft::distance::DistanceType::L2Expanded, true}, - {1000, 10000, 2, 16, 40, 1024, raft::distance::DistanceType::L2Expanded, false}, - {1000, 10000, 3, 16, 40, 1024, raft::distance::DistanceType::L2Expanded, true}, - {1000, 10000, 4, 16, 40, 1024, raft::distance::DistanceType::L2Expanded, false}, - {1000, 10000, 5, 16, 40, 1024, raft::distance::DistanceType::InnerProduct, false}, - {1000, 10000, 8, 16, 40, 1024, raft::distance::DistanceType::InnerProduct, true}, - {1000, 10000, 5, 16, 40, 1024, raft::distance::DistanceType::L2SqrtExpanded, false}, - {1000, 10000, 8, 16, 40, 1024, raft::distance::DistanceType::L2SqrtExpanded, true}, - - // test dims that do not fit into kernel shared memory limits - {1000, 10000, 2048, 16, 40, 1024, raft::distance::DistanceType::L2Expanded, false}, - {1000, 10000, 2049, 16, 40, 1024, raft::distance::DistanceType::L2Expanded, false}, - {1000, 10000, 2050, 16, 40, 1024, raft::distance::DistanceType::InnerProduct, false}, - {1000, 10000, 2051, 16, 40, 1024, raft::distance::DistanceType::InnerProduct, true}, - {1000, 10000, 2052, 16, 40, 1024, raft::distance::DistanceType::InnerProduct, false}, - {1000, 10000, 2053, 16, 40, 1024, raft::distance::DistanceType::L2Expanded, true}, - {1000, 10000, 2056, 16, 40, 1024, raft::distance::DistanceType::L2Expanded, true}, - - // various random combinations - {1000, 10000, 16, 10, 40, 1024, raft::distance::DistanceType::L2Expanded, false}, - {1000, 10000, 16, 10, 50, 1024, raft::distance::DistanceType::L2Expanded, false}, - {1000, 10000, 16, 10, 70, 1024, raft::distance::DistanceType::L2Expanded, false}, - {100, 10000, 16, 10, 20, 512, raft::distance::DistanceType::L2Expanded, false}, - {20, 100000, 16, 10, 20, 1024, raft::distance::DistanceType::L2Expanded, true}, - {1000, 100000, 16, 10, 20, 1024, raft::distance::DistanceType::L2Expanded, true}, - {10000, 131072, 8, 10, 20, 1024, raft::distance::DistanceType::L2Expanded, false}, - - // various combinations with k>raft::matrix::detail::select::warpsort::kMaxCapacity - {1000, 10000, 16, 1024, 40, 1024, raft::distance::DistanceType::L2SqrtExpanded, true}, - {1000, 10000, 2053, 512, 50, 1024, raft::distance::DistanceType::L2SqrtExpanded, false}, - {1000, 10000, 2049, 2048, 70, 1024, raft::distance::DistanceType::L2SqrtExpanded, false}, - {1000, 10000, 16, 4000, 100, 2048, raft::distance::DistanceType::L2SqrtExpanded, false}, - {10, 10000, 16, 4000, 100, 2048, raft::distance::DistanceType::L2SqrtExpanded, false}, - {10, 10000, 16, 4000, 120, 2048, raft::distance::DistanceType::L2SqrtExpanded, true}, - {20, 100000, 16, 257, 20, 1024, raft::distance::DistanceType::L2SqrtExpanded, true}, - {1000, 100000, 16, 259, 20, 1024, raft::distance::DistanceType::L2Expanded, true, true}, - {10000, 131072, 8, 280, 20, 1024, raft::distance::DistanceType::InnerProduct, false}, - {100000, 1024, 32, 257, 64, 64, raft::distance::DistanceType::L2Expanded, false}, - {100000, 1024, 32, 257, 64, 64, raft::distance::DistanceType::L2SqrtExpanded, false}, - {100000, 1024, 32, 257, 64, 64, raft::distance::DistanceType::InnerProduct, false}, - {100000, 1024, 16, 300, 20, 60, raft::distance::DistanceType::L2Expanded, false}, - {100000, 1024, 16, 500, 20, 60, raft::distance::DistanceType::L2SqrtExpanded, false}, - {100000, 1024, 16, 700, 20, 60, raft::distance::DistanceType::InnerProduct, false}, - - // host input data - {1000, 10000, 16, 10, 40, 1024, raft::distance::DistanceType::L2Expanded, false, true}, - {1000, 10000, 16, 10, 50, 1024, raft::distance::DistanceType::L2Expanded, false, true}, - {1000, 10000, 16, 10, 70, 1024, raft::distance::DistanceType::L2Expanded, false, true}, - {100, 10000, 16, 10, 20, 512, raft::distance::DistanceType::L2Expanded, false, true}, - {20, 100000, 16, 10, 20, 1024, raft::distance::DistanceType::L2Expanded, false, true}, - {1000, 100000, 16, 10, 20, 1024, raft::distance::DistanceType::L2Expanded, false, true}, - {10000, 131072, 8, 10, 20, 1024, raft::distance::DistanceType::L2Expanded, false, true}, - - {1000, 10000, 16, 10, 40, 1024, raft::distance::DistanceType::InnerProduct, true}, - {1000, 10000, 16, 10, 50, 1024, raft::distance::DistanceType::InnerProduct, true}, - {1000, 10000, 16, 10, 70, 1024, raft::distance::DistanceType::InnerProduct, false}, - {100, 10000, 16, 10, 20, 512, raft::distance::DistanceType::InnerProduct, true}, - {20, 100000, 16, 10, 20, 1024, raft::distance::DistanceType::InnerProduct, true}, - {1000, 100000, 16, 10, 20, 1024, raft::distance::DistanceType::InnerProduct, false}, - {10000, 131072, 8, 10, 50, 1024, raft::distance::DistanceType::InnerProduct, true}, - - {1000, 10000, 4096, 20, 50, 1024, raft::distance::DistanceType::InnerProduct, false}, - - // test splitting the big query batches (> max gridDim.y) into smaller batches - {100000, 1024, 32, 10, 64, 64, raft::distance::DistanceType::InnerProduct, false}, - {1000000, 1024, 32, 10, 256, 256, raft::distance::DistanceType::InnerProduct, false}, - {98306, 1024, 32, 10, 64, 64, raft::distance::DistanceType::InnerProduct, true}, - - // test radix_sort for getting the cluster selection - {1000, - 10000, - 16, - 10, - raft::matrix::detail::select::warpsort::kMaxCapacity * 2, - raft::matrix::detail::select::warpsort::kMaxCapacity * 4, - raft::distance::DistanceType::L2Expanded, - false}, - {1000, - 10000, - 16, - 10, - raft::matrix::detail::select::warpsort::kMaxCapacity * 4, - raft::matrix::detail::select::warpsort::kMaxCapacity * 4, - raft::distance::DistanceType::InnerProduct, - false}, - - // The following two test cases should show very similar recall. - // num_queries, num_db_vecs, dim, k, nprobe, nlist, metric, adaptive_centers - {20000, 8712, 3, 10, 51, 66, raft::distance::DistanceType::L2Expanded, false}, - {100000, 8712, 3, 10, 51, 66, raft::distance::DistanceType::L2Expanded, false}}; - -} // namespace raft::neighbors::ivf_flat diff --git a/cpp/test/neighbors/ann_ivf_flat/test_filter_float_int64_t.cu b/cpp/test/neighbors/ann_ivf_flat/test_filter_float_int64_t.cu deleted file mode 100644 index 0e1036e566..0000000000 --- a/cpp/test/neighbors/ann_ivf_flat/test_filter_float_int64_t.cu +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#undef RAFT_EXPLICIT_INSTANTIATE_ONLY // Enable instantiation of search with filter -#include "../ann_ivf_flat.cuh" - -namespace raft::neighbors::ivf_flat { - -typedef AnnIVFFlatTest AnnIVFFlatFilterTestF; -TEST_P(AnnIVFFlatFilterTestF, AnnIVFFlatFilter) { this->testFilter(); } - -INSTANTIATE_TEST_CASE_P(AnnIVFFlatTest, AnnIVFFlatFilterTestF, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::ivf_flat diff --git a/cpp/test/neighbors/ann_ivf_flat/test_float_int64_t.cu b/cpp/test/neighbors/ann_ivf_flat/test_float_int64_t.cu deleted file mode 100644 index 2ff17b8536..0000000000 --- a/cpp/test/neighbors/ann_ivf_flat/test_float_int64_t.cu +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_ivf_flat.cuh" - -#include - -namespace raft::neighbors::ivf_flat { - -typedef AnnIVFFlatTest AnnIVFFlatTestF; -TEST_P(AnnIVFFlatTestF, AnnIVFFlat) -{ - this->testIVFFlat(); - this->testPacker(); -} - -INSTANTIATE_TEST_CASE_P(AnnIVFFlatTest, AnnIVFFlatTestF, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::ivf_flat diff --git a/cpp/test/neighbors/ann_ivf_flat/test_int8_t_int64_t.cu b/cpp/test/neighbors/ann_ivf_flat/test_int8_t_int64_t.cu deleted file mode 100644 index 6fe12506aa..0000000000 --- a/cpp/test/neighbors/ann_ivf_flat/test_int8_t_int64_t.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_ivf_flat.cuh" - -#include - -namespace raft::neighbors::ivf_flat { - -typedef AnnIVFFlatTest AnnIVFFlatTestF_int8; -TEST_P(AnnIVFFlatTestF_int8, AnnIVFFlat) { this->testIVFFlat(); } - -INSTANTIATE_TEST_CASE_P(AnnIVFFlatTest, AnnIVFFlatTestF_int8, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::ivf_flat diff --git a/cpp/test/neighbors/ann_ivf_flat/test_uint8_t_int64_t.cu b/cpp/test/neighbors/ann_ivf_flat/test_uint8_t_int64_t.cu deleted file mode 100644 index ab6001c71b..0000000000 --- a/cpp/test/neighbors/ann_ivf_flat/test_uint8_t_int64_t.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_ivf_flat.cuh" - -#include - -namespace raft::neighbors::ivf_flat { - -typedef AnnIVFFlatTest AnnIVFFlatTestF_uint8; -TEST_P(AnnIVFFlatTestF_uint8, AnnIVFFlat) { this->testIVFFlat(); } - -INSTANTIATE_TEST_CASE_P(AnnIVFFlatTest, AnnIVFFlatTestF_uint8, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::ivf_flat diff --git a/cpp/test/neighbors/ann_ivf_pq.cuh b/cpp/test/neighbors/ann_ivf_pq.cuh deleted file mode 100644 index 4ebe02027f..0000000000 --- a/cpp/test/neighbors/ann_ivf_pq.cuh +++ /dev/null @@ -1,1095 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "../test_utils.cuh" -#include "ann_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include - -#include -#include - -#include - -#include -#include -#include -#include -#include - -namespace raft::neighbors::ivf_pq { - -struct test_ivf_sample_filter { - static constexpr unsigned offset = 1500; -}; - -struct ivf_pq_inputs { - uint32_t num_db_vecs = 4096; - uint32_t num_queries = 1024; - uint32_t dim = 64; - uint32_t k = 32; - std::optional min_recall = std::nullopt; - - ivf_pq::index_params index_params; - ivf_pq::search_params search_params; - - // Set some default parameters for tests - ivf_pq_inputs() - { - index_params.n_lists = max(32u, min(1024u, num_db_vecs / 128u)); - index_params.kmeans_trainset_fraction = 1.0; - } -}; - -inline auto operator<<(std::ostream& os, const ivf_pq::codebook_gen& p) -> std::ostream& -{ - switch (p) { - case ivf_pq::codebook_gen::PER_CLUSTER: os << "codebook_gen::PER_CLUSTER"; break; - case ivf_pq::codebook_gen::PER_SUBSPACE: os << "codebook_gen::PER_SUBSPACE"; break; - default: RAFT_FAIL("unreachable code"); - } - return os; -} - -inline auto operator<<(std::ostream& os, const ivf_pq_inputs& p) -> std::ostream& -{ - ivf_pq_inputs dflt; - bool need_comma = false; -#define PRINT_DIFF_V(spec, val) \ - do { \ - if (dflt spec != p spec) { \ - if (need_comma) { os << ", "; } \ - os << #spec << " = " << val; \ - need_comma = true; \ - } \ - } while (0) -#define PRINT_DIFF(spec) PRINT_DIFF_V(spec, p spec) - - os << "ivf_pq_inputs {"; - PRINT_DIFF(.num_db_vecs); - PRINT_DIFF(.num_queries); - PRINT_DIFF(.dim); - PRINT_DIFF(.k); - PRINT_DIFF_V(.min_recall, p.min_recall.value_or(0)); - PRINT_DIFF_V(.index_params.metric, print_metric{p.index_params.metric}); - PRINT_DIFF(.index_params.metric_arg); - PRINT_DIFF(.index_params.add_data_on_build); - PRINT_DIFF(.index_params.n_lists); - PRINT_DIFF(.index_params.kmeans_n_iters); - PRINT_DIFF(.index_params.kmeans_trainset_fraction); - PRINT_DIFF(.index_params.pq_bits); - PRINT_DIFF(.index_params.pq_dim); - PRINT_DIFF(.index_params.codebook_kind); - PRINT_DIFF(.index_params.force_random_rotation); - PRINT_DIFF(.search_params.n_probes); - PRINT_DIFF_V(.search_params.lut_dtype, print_dtype{p.search_params.lut_dtype}); - PRINT_DIFF_V(.search_params.internal_distance_dtype, - print_dtype{p.search_params.internal_distance_dtype}); - os << "}"; - return os; -} - -template -void compare_vectors_l2( - const raft::resources& res, T a, T b, uint32_t label, double compression_ratio, double eps) -{ - auto n_rows = a.extent(0); - auto dim = a.extent(1); - rmm::mr::managed_memory_resource managed_memory; - auto dist = make_device_mdarray(res, &managed_memory, make_extents(n_rows)); - linalg::map_offset(res, dist.view(), [a, b, dim] __device__(uint32_t i) { - spatial::knn::detail::utils::mapping f{}; - double d = 0.0f; - for (uint32_t j = 0; j < dim; j++) { - double t = f(a(i, j)) - f(b(i, j)); - d += t * t; - } - return sqrt(d / double(dim)); - }); - resource::sync_stream(res); - for (uint32_t i = 0; i < n_rows; i++) { - double d = dist(i); - // The theoretical estimate of the error is hard to come up with, - // the estimate below is based on experimentation + curse of dimensionality - ASSERT_LE(d, 1.2 * eps * std::pow(2.0, compression_ratio)) - << " (label = " << label << ", ix = " << i << ", eps = " << eps << ")"; - } -} - -template -auto min_output_size(const raft::resources& handle, - const ivf_pq::index& index, - uint32_t n_probes) -> IdxT -{ - auto acc_sizes = index.accum_sorted_sizes(); - uint32_t last_nonzero = index.n_lists(); - while (last_nonzero > 0 && acc_sizes(last_nonzero - 1) == acc_sizes(last_nonzero)) { - last_nonzero--; - } - return acc_sizes(last_nonzero) - acc_sizes(last_nonzero - std::min(last_nonzero, n_probes)); -} - -template -class ivf_pq_test : public ::testing::TestWithParam { - public: - ivf_pq_test() - : stream_(resource::get_cuda_stream(handle_)), - ps(::testing::TestWithParam::GetParam()), - database(0, stream_), - search_queries(0, stream_) - { - } - - void gen_data() - { - database.resize(size_t{ps.num_db_vecs} * size_t{ps.dim}, stream_); - search_queries.resize(size_t{ps.num_queries} * size_t{ps.dim}, stream_); - - raft::random::RngState r(1234ULL); - if constexpr (std::is_same{}) { - raft::random::uniform( - handle_, r, database.data(), ps.num_db_vecs * ps.dim, DataT(0.1), DataT(2.0)); - raft::random::uniform( - handle_, r, search_queries.data(), ps.num_queries * ps.dim, DataT(0.1), DataT(2.0)); - } else { - raft::random::uniformInt( - handle_, r, database.data(), ps.num_db_vecs * ps.dim, DataT(1), DataT(20)); - raft::random::uniformInt( - handle_, r, search_queries.data(), ps.num_queries * ps.dim, DataT(1), DataT(20)); - } - resource::sync_stream(handle_); - } - - void calc_ref() - { - size_t queries_size = size_t{ps.num_queries} * size_t{ps.k}; - rmm::device_uvector distances_naive_dev(queries_size, stream_); - rmm::device_uvector indices_naive_dev(queries_size, stream_); - naive_knn(handle_, - distances_naive_dev.data(), - indices_naive_dev.data(), - search_queries.data(), - database.data(), - ps.num_queries, - ps.num_db_vecs, - ps.dim, - ps.k, - ps.index_params.metric); - distances_ref.resize(queries_size); - update_host(distances_ref.data(), distances_naive_dev.data(), queries_size, stream_); - indices_ref.resize(queries_size); - update_host(indices_ref.data(), indices_naive_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - auto build_only() - { - auto ipams = ps.index_params; - ipams.add_data_on_build = true; - - auto index_view = - raft::make_device_matrix_view(database.data(), ps.num_db_vecs, ps.dim); - return ivf_pq::build(handle_, ipams, index_view); - } - - auto build_2_extends() - { - auto db_indices = make_device_vector(handle_, ps.num_db_vecs); - linalg::map_offset(handle_, db_indices.view(), identity_op{}); - resource::sync_stream(handle_); - auto size_1 = IdxT(ps.num_db_vecs) / 2; - auto size_2 = IdxT(ps.num_db_vecs) - size_1; - auto vecs_1 = database.data(); - auto vecs_2 = database.data() + size_t(size_1) * size_t(ps.dim); - auto inds_1 = db_indices.data_handle(); - auto inds_2 = db_indices.data_handle() + size_t(size_1); - - auto ipams = ps.index_params; - ipams.add_data_on_build = false; - - auto database_view = - raft::make_device_matrix_view(database.data(), ps.num_db_vecs, ps.dim); - auto idx = ivf_pq::build(handle_, ipams, database_view); - - auto vecs_2_view = raft::make_device_matrix_view(vecs_2, size_2, ps.dim); - auto inds_2_view = raft::make_device_vector_view(inds_2, size_2); - ivf_pq::extend(handle_, vecs_2_view, inds_2_view, &idx); - - auto vecs_1_view = - raft::make_device_matrix_view(vecs_1, size_1, ps.dim); - auto inds_1_view = raft::make_device_vector_view(inds_1, size_1); - ivf_pq::extend(handle_, vecs_1_view, inds_1_view, &idx); - return idx; - } - - auto build_serialize() - { - ivf_pq::serialize(handle_, "ivf_pq_index", build_only()); - return ivf_pq::deserialize(handle_, "ivf_pq_index"); - } - - void check_reconstruction(const index& index, - double compression_ratio, - uint32_t label, - uint32_t n_take, - uint32_t n_skip) - { - auto& rec_list = index.lists()[label]; - auto dim = index.dim(); - n_take = std::min(n_take, rec_list->size.load()); - n_skip = std::min(n_skip, rec_list->size.load() - n_take); - - if (n_take == 0) { return; } - - auto rec_data = make_device_matrix(handle_, n_take, dim); - auto orig_data = make_device_matrix(handle_, n_take, dim); - - ivf_pq::helpers::reconstruct_list_data(handle_, index, rec_data.view(), label, n_skip); - - matrix::gather(database.data(), - IdxT{dim}, - IdxT{n_take}, - rec_list->indices.data_handle() + n_skip, - IdxT{n_take}, - orig_data.data_handle(), - stream_); - - compare_vectors_l2(handle_, rec_data.view(), orig_data.view(), label, compression_ratio, 0.06); - } - - void check_reconstruct_extend(index* index, double compression_ratio, uint32_t label) - { - // NB: this is not reference, the list is retained; the index will have to create a new list on - // `erase_list` op. - auto old_list = index->lists()[label]; - auto n_rows = old_list->size.load(); - if (n_rows == 0) { return; } - - auto vectors_1 = make_device_matrix(handle_, n_rows, index->dim()); - auto indices = make_device_vector(handle_, n_rows); - copy(indices.data_handle(), old_list->indices.data_handle(), n_rows, stream_); - - ivf_pq::helpers::reconstruct_list_data(handle_, *index, vectors_1.view(), label, 0); - ivf_pq::helpers::erase_list(handle_, index, label); - // NB: passing the type parameter because const->non-const implicit conversion of the mdspans - // breaks type inference - ivf_pq::helpers::extend_list( - handle_, index, vectors_1.view(), indices.view(), label); - - auto& new_list = index->lists()[label]; - ASSERT_NE(old_list.get(), new_list.get()) - << "The old list should have been shared and retained after ivf_pq index has erased the " - "corresponding cluster."; - - auto vectors_2 = make_device_matrix(handle_, n_rows, index->dim()); - ivf_pq::helpers::reconstruct_list_data(handle_, *index, vectors_2.view(), label, 0); - // The code search is unstable, and there's high chance of repeating values of the lvl-2 codes. - // Hence, encoding-decoding chain often leads to altering both the PQ codes and the - // reconstructed data. - compare_vectors_l2( - handle_, vectors_1.view(), vectors_2.view(), label, compression_ratio, 0.04); // 0.025); - } - - void check_packing(index* index, uint32_t label) - { - auto old_list = index->lists()[label]; - auto n_rows = old_list->size.load(); - - if (n_rows == 0) { return; } - - auto codes = make_device_matrix(handle_, n_rows, index->pq_dim()); - auto indices = make_device_vector(handle_, n_rows); - copy(indices.data_handle(), old_list->indices.data_handle(), n_rows, stream_); - - ivf_pq::helpers::unpack_list_data(handle_, *index, codes.view(), label, 0); - ivf_pq::helpers::erase_list(handle_, index, label); - ivf_pq::helpers::extend_list_with_codes( - handle_, index, codes.view(), indices.view(), label); - - auto& new_list = index->lists()[label]; - ASSERT_NE(old_list.get(), new_list.get()) - << "The old list should have been shared and retained after ivf_pq index has erased the " - "corresponding cluster."; - auto list_data_size = (n_rows / ivf_pq::kIndexGroupSize) * new_list->data.extent(1) * - new_list->data.extent(2) * new_list->data.extent(3); - - ASSERT_TRUE(old_list->data.size() >= list_data_size); - ASSERT_TRUE(new_list->data.size() >= list_data_size); - ASSERT_TRUE(devArrMatch(old_list->data.data_handle(), - new_list->data.data_handle(), - list_data_size, - Compare{})); - - // Pack a few vectors back to the list. - int row_offset = 9; - int n_vec = 3; - ASSERT_TRUE(row_offset + n_vec < n_rows); - size_t offset = row_offset * index->pq_dim(); - auto codes_to_pack = make_device_matrix_view( - codes.data_handle() + offset, n_vec, index->pq_dim()); - ivf_pq::helpers::pack_list_data(handle_, index, codes_to_pack, label, row_offset); - ASSERT_TRUE(devArrMatch(old_list->data.data_handle(), - new_list->data.data_handle(), - list_data_size, - Compare{})); - - // Another test with the API that take list_data directly - auto list_data = index->lists()[label]->data.view(); - uint32_t n_take = 4; - ASSERT_TRUE(row_offset + n_take < n_rows); - auto codes2 = raft::make_device_matrix(handle_, n_take, index->pq_dim()); - ivf_pq::helpers::codepacker::unpack( - handle_, list_data, index->pq_bits(), row_offset, codes2.view()); - - // Write it back - ivf_pq::helpers::codepacker::pack( - handle_, make_const_mdspan(codes2.view()), index->pq_bits(), row_offset, list_data); - ASSERT_TRUE(devArrMatch(old_list->data.data_handle(), - new_list->data.data_handle(), - list_data_size, - Compare{})); - } - void check_packing_contiguous(index* index, uint32_t label) - { - auto old_list = index->lists()[label]; - auto n_rows = old_list->size.load(); - - if (n_rows == 0) { return; } - - auto codes = make_device_matrix(handle_, n_rows, index->pq_dim()); - auto indices = make_device_vector(handle_, n_rows); - copy(indices.data_handle(), old_list->indices.data_handle(), n_rows, stream_); - - uint32_t code_size = ceildiv(index->pq_dim() * index->pq_bits(), 8); - - auto codes_compressed = make_device_matrix(handle_, n_rows, code_size); - - ivf_pq::helpers::unpack_contiguous_list_data( - handle_, *index, codes_compressed.data_handle(), n_rows, label, 0); - ivf_pq::helpers::erase_list(handle_, index, label); - ivf_pq::detail::extend_list_prepare(handle_, index, make_const_mdspan(indices.view()), label); - ivf_pq::helpers::pack_contiguous_list_data( - handle_, index, codes_compressed.data_handle(), n_rows, label, 0); - ivf_pq::helpers::recompute_internal_state(handle_, index); - - auto& new_list = index->lists()[label]; - ASSERT_NE(old_list.get(), new_list.get()) - << "The old list should have been shared and retained after ivf_pq index has erased the " - "corresponding cluster."; - auto list_data_size = (n_rows / ivf_pq::kIndexGroupSize) * new_list->data.extent(1) * - new_list->data.extent(2) * new_list->data.extent(3); - - ASSERT_TRUE(old_list->data.size() >= list_data_size); - ASSERT_TRUE(new_list->data.size() >= list_data_size); - ASSERT_TRUE(devArrMatch(old_list->data.data_handle(), - new_list->data.data_handle(), - list_data_size, - Compare{})); - - // Pack a few vectors back to the list. - uint32_t row_offset = 9; - uint32_t n_vec = 3; - ASSERT_TRUE(row_offset + n_vec < n_rows); - size_t offset = row_offset * code_size; - auto codes_to_pack = make_device_matrix_view( - codes_compressed.data_handle() + offset, n_vec, index->pq_dim()); - ivf_pq::helpers::pack_contiguous_list_data( - handle_, index, codes_to_pack.data_handle(), n_vec, label, row_offset); - ASSERT_TRUE(devArrMatch(old_list->data.data_handle(), - new_list->data.data_handle(), - list_data_size, - Compare{})); - - // // Another test with the API that take list_data directly - auto list_data = index->lists()[label]->data.view(); - uint32_t n_take = 4; - ASSERT_TRUE(row_offset + n_take < n_rows); - auto codes2 = raft::make_device_matrix(handle_, n_take, code_size); - ivf_pq::helpers::codepacker::unpack_contiguous(handle_, - list_data, - index->pq_bits(), - row_offset, - n_take, - index->pq_dim(), - codes2.data_handle()); - - // Write it back - ivf_pq::helpers::codepacker::pack_contiguous(handle_, - codes2.data_handle(), - n_vec, - index->pq_dim(), - index->pq_bits(), - row_offset, - list_data); - ASSERT_TRUE(devArrMatch(old_list->data.data_handle(), - new_list->data.data_handle(), - list_data_size, - Compare{})); - } - - template - void run(BuildIndex build_index) - { - index index = build_index(); - - double compression_ratio = - static_cast(ps.dim * 8) / static_cast(index.pq_dim() * index.pq_bits()); - - for (uint32_t label = 0; label < index.n_lists(); label++) { - switch (label % 3) { - case 0: { - // Reconstruct and re-write vectors for one label - check_reconstruct_extend(&index, compression_ratio, label); - } break; - case 1: { - // Dump and re-write codes for one label - check_packing(&index, label); - check_packing_contiguous(&index, label); - } break; - default: { - // check a small subset of data in a randomly chosen cluster to see if the data - // reconstruction works well. - check_reconstruction(index, compression_ratio, label, 100, 7); - } - } - } - - size_t queries_size = ps.num_queries * ps.k; - std::vector indices_ivf_pq(queries_size); - std::vector distances_ivf_pq(queries_size); - - rmm::device_uvector distances_ivf_pq_dev(queries_size, stream_); - rmm::device_uvector indices_ivf_pq_dev(queries_size, stream_); - - auto query_view = - raft::make_device_matrix_view(search_queries.data(), ps.num_queries, ps.dim); - auto inds_view = raft::make_device_matrix_view( - indices_ivf_pq_dev.data(), ps.num_queries, ps.k); - auto dists_view = raft::make_device_matrix_view( - distances_ivf_pq_dev.data(), ps.num_queries, ps.k); - - ivf_pq::search( - handle_, ps.search_params, index, query_view, inds_view, dists_view); - - update_host(distances_ivf_pq.data(), distances_ivf_pq_dev.data(), queries_size, stream_); - update_host(indices_ivf_pq.data(), indices_ivf_pq_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - - // A very conservative lower bound on recall - double min_recall = - static_cast(ps.search_params.n_probes) / static_cast(ps.index_params.n_lists); - // Using a heuristic to lower the required recall due to code-packing errors - min_recall = - std::min(std::erfc(0.05 * compression_ratio / std::max(min_recall, 0.5)), min_recall); - // Use explicit per-test min recall value if provided. - min_recall = ps.min_recall.value_or(min_recall); - - ASSERT_TRUE(eval_neighbours(indices_ref, - indices_ivf_pq, - distances_ref, - distances_ivf_pq, - ps.num_queries, - ps.k, - 0.0001 * compression_ratio, - min_recall)) - << ps; - - // Test a few extra invariants - IdxT min_results = min_output_size(handle_, index, ps.search_params.n_probes); - IdxT max_oob = ps.k <= min_results ? 0 : ps.k - min_results; - IdxT found_oob = 0; - for (uint32_t query_ix = 0; query_ix < ps.num_queries; query_ix++) { - for (uint32_t k = 0; k < ps.k; k++) { - auto flat_i = query_ix * ps.k + k; - auto found_ix = indices_ivf_pq[flat_i]; - if (found_ix == ivf_pq::kOutOfBoundsRecord) { - found_oob++; - continue; - } - ASSERT_NE(found_ix, ivf::kInvalidRecord) - << "got an invalid record at query_ix = " << query_ix << ", k = " << k - << " (distance = " << distances_ivf_pq[flat_i] << ")"; - ASSERT_LT(found_ix, ps.num_db_vecs) - << "got an impossible index = " << found_ix << " at query_ix = " << query_ix - << ", k = " << k << " (distance = " << distances_ivf_pq[flat_i] << ")"; - } - } - ASSERT_LE(found_oob, max_oob) - << "got too many records out-of-bounds (see ivf_pq::kOutOfBoundsRecord)."; - if (found_oob > 0) { - RAFT_LOG_WARN( - "Got %zu results out-of-bounds because of large top-k (%zu) and small n_probes (%u) and " - "small DB size/n_lists ratio (%zu / %u)", - size_t(found_oob), - size_t(ps.k), - ps.search_params.n_probes, - size_t(ps.num_db_vecs), - ps.index_params.n_lists); - } - } - - void SetUp() override // NOLINT - { - gen_data(); - calc_ref(); - } - - void TearDown() override // NOLINT - { - cudaGetLastError(); - resource::sync_stream(handle_); - database.resize(0, stream_); - search_queries.resize(0, stream_); - } - - private: - raft::resources handle_; - rmm::cuda_stream_view stream_; - ivf_pq_inputs ps; // NOLINT - rmm::device_uvector database; // NOLINT - rmm::device_uvector search_queries; // NOLINT - std::vector indices_ref; // NOLINT - std::vector distances_ref; // NOLINT -}; - -template -class ivf_pq_filter_test : public ::testing::TestWithParam { - public: - ivf_pq_filter_test() - : stream_(resource::get_cuda_stream(handle_)), - ps(::testing::TestWithParam::GetParam()), - database(0, stream_), - search_queries(0, stream_) - { - } - - void gen_data() - { - database.resize(size_t{ps.num_db_vecs} * size_t{ps.dim}, stream_); - search_queries.resize(size_t{ps.num_queries} * size_t{ps.dim}, stream_); - - raft::random::RngState r(1234ULL); - if constexpr (std::is_same{}) { - raft::random::uniform( - handle_, r, database.data(), ps.num_db_vecs * ps.dim, DataT(0.1), DataT(2.0)); - raft::random::uniform( - handle_, r, search_queries.data(), ps.num_queries * ps.dim, DataT(0.1), DataT(2.0)); - } else { - raft::random::uniformInt( - handle_, r, database.data(), ps.num_db_vecs * ps.dim, DataT(1), DataT(20)); - raft::random::uniformInt( - handle_, r, search_queries.data(), ps.num_queries * ps.dim, DataT(1), DataT(20)); - } - resource::sync_stream(handle_); - } - - void calc_ref() - { - size_t queries_size = size_t{ps.num_queries} * size_t{ps.k}; - rmm::device_uvector distances_naive_dev(queries_size, stream_); - rmm::device_uvector indices_naive_dev(queries_size, stream_); - naive_knn(handle_, - distances_naive_dev.data(), - indices_naive_dev.data(), - search_queries.data(), - database.data() + test_ivf_sample_filter::offset * ps.dim, - ps.num_queries, - ps.num_db_vecs - test_ivf_sample_filter::offset, - ps.dim, - ps.k, - ps.index_params.metric); - raft::linalg::addScalar(indices_naive_dev.data(), - indices_naive_dev.data(), - IdxT(test_ivf_sample_filter::offset), - queries_size, - stream_); - distances_ref.resize(queries_size); - update_host(distances_ref.data(), distances_naive_dev.data(), queries_size, stream_); - indices_ref.resize(queries_size); - update_host(indices_ref.data(), indices_naive_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - auto build_only() - { - auto ipams = ps.index_params; - ipams.add_data_on_build = true; - - auto index_view = - raft::make_device_matrix_view(database.data(), ps.num_db_vecs, ps.dim); - return ivf_pq::build(handle_, ipams, index_view); - } - - template - void run(BuildIndex build_index) - { - index index = build_index(); - - double compression_ratio = - static_cast(ps.dim * 8) / static_cast(index.pq_dim() * index.pq_bits()); - size_t queries_size = ps.num_queries * ps.k; - std::vector indices_ivf_pq(queries_size); - std::vector distances_ivf_pq(queries_size); - - rmm::device_uvector distances_ivf_pq_dev(queries_size, stream_); - rmm::device_uvector indices_ivf_pq_dev(queries_size, stream_); - - auto query_view = - raft::make_device_matrix_view(search_queries.data(), ps.num_queries, ps.dim); - auto inds_view = raft::make_device_matrix_view( - indices_ivf_pq_dev.data(), ps.num_queries, ps.k); - auto dists_view = raft::make_device_matrix_view( - distances_ivf_pq_dev.data(), ps.num_queries, ps.k); - - // Create Bitset filter - auto removed_indices = - raft::make_device_vector(handle_, test_ivf_sample_filter::offset); - thrust::sequence( - resource::get_thrust_policy(handle_), - thrust::device_pointer_cast(removed_indices.data_handle()), - thrust::device_pointer_cast(removed_indices.data_handle() + test_ivf_sample_filter::offset)); - resource::sync_stream(handle_); - - raft::core::bitset removed_indices_bitset( - handle_, removed_indices.view(), ps.num_db_vecs); - ivf_pq::search_with_filtering( - handle_, - ps.search_params, - index, - query_view, - inds_view, - dists_view, - raft::neighbors::filtering::bitset_filter(removed_indices_bitset.view())); - - update_host(distances_ivf_pq.data(), distances_ivf_pq_dev.data(), queries_size, stream_); - update_host(indices_ivf_pq.data(), indices_ivf_pq_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - - // A very conservative lower bound on recall - double min_recall = - static_cast(ps.search_params.n_probes) / static_cast(ps.index_params.n_lists); - // Using a heuristic to lower the required recall due to code-packing errors - min_recall = - std::min(std::erfc(0.05 * compression_ratio / std::max(min_recall, 0.5)), min_recall); - // Use explicit per-test min recall value if provided. - min_recall = ps.min_recall.value_or(min_recall); - - ASSERT_TRUE(eval_neighbours(indices_ref, - indices_ivf_pq, - distances_ref, - distances_ivf_pq, - ps.num_queries, - ps.k, - 0.0001 * compression_ratio, - min_recall)) - << ps; - } - - void SetUp() override // NOLINT - { - gen_data(); - calc_ref(); - } - - void TearDown() override // NOLINT - { - cudaGetLastError(); - resource::sync_stream(handle_); - database.resize(0, stream_); - search_queries.resize(0, stream_); - } - - private: - raft::resources handle_; - rmm::cuda_stream_view stream_; - ivf_pq_inputs ps; // NOLINT - rmm::device_uvector database; // NOLINT - rmm::device_uvector search_queries; // NOLINT - std::vector indices_ref; // NOLINT - std::vector distances_ref; // NOLINT -}; - -/* Test cases */ -using test_cases_t = std::vector; - -// concatenate parameter sets for different type -template -auto operator+(const std::vector& a, const std::vector& b) -> std::vector -{ - std::vector res = a; - res.insert(res.end(), b.begin(), b.end()); - return res; -} - -inline auto defaults() -> test_cases_t { return {ivf_pq_inputs{}}; } - -template -auto map(const std::vector& xs, F f) -> std::vector -{ - std::vector ys(xs.size()); - std::transform(xs.begin(), xs.end(), ys.begin(), f); - return ys; -} - -inline auto with_dims(const std::vector& dims) -> test_cases_t -{ - return map(dims, [](uint32_t d) { - ivf_pq_inputs x; - x.dim = d; - return x; - }); -} - -/** These will surely trigger the fastest kernel available. */ -inline auto small_dims() -> test_cases_t { return with_dims({1, 2, 3, 4, 5, 8, 15, 16, 17}); } - -inline auto small_dims_per_cluster() -> test_cases_t -{ - return map(small_dims(), [](const ivf_pq_inputs& x) { - ivf_pq_inputs y(x); - y.index_params.codebook_kind = ivf_pq::codebook_gen::PER_CLUSTER; - return y; - }); -} - -inline auto big_dims() -> test_cases_t -{ - // with_dims({512, 513, 1023, 1024, 1025, 2048, 2049, 2050, 2053, 6144, 8192, 12288, 16384}); - auto xs = with_dims({512, 513, 1023, 1024, 1025, 2048, 2049, 2050, 2053, 6144}); - return map(xs, [](const ivf_pq_inputs& x) { - ivf_pq_inputs y(x); - uint32_t pq_len = 2; - y.index_params.pq_dim = div_rounding_up_safe(x.dim, pq_len); - // This comes from pure experimentation, also the recall depens a lot on pq_len. - y.min_recall = 0.48 + 0.028 * std::log2(x.dim); - return y; - }); -} - -/** These will surely trigger no-smem-lut kernel. */ -inline auto big_dims_moderate_lut() -> test_cases_t -{ - return map(big_dims(), [](const ivf_pq_inputs& x) { - ivf_pq_inputs y(x); - uint32_t pq_len = 2; - y.index_params.pq_dim = round_up_safe(div_rounding_up_safe(x.dim, pq_len), 4u); - y.index_params.pq_bits = 6; - y.search_params.lut_dtype = CUDA_R_16F; - y.min_recall = 0.69; - return y; - }); -} - -/** Some of these should trigger no-basediff kernel. */ -inline auto big_dims_small_lut() -> test_cases_t -{ - return map(big_dims(), [](const ivf_pq_inputs& x) { - ivf_pq_inputs y(x); - uint32_t pq_len = 8; - y.index_params.pq_dim = round_up_safe(div_rounding_up_safe(x.dim, pq_len), 4u); - y.index_params.pq_bits = 6; - y.search_params.lut_dtype = CUDA_R_8U; - y.min_recall = 0.21; - return y; - }); -} - -/** - * A minimal set of tests to check various enum-like parameters. - */ -inline auto enum_variety() -> test_cases_t -{ - test_cases_t xs; -#define ADD_CASE(f) \ - do { \ - xs.push_back({}); \ - ([](ivf_pq_inputs & x) f)(xs[xs.size() - 1]); \ - } while (0); - - ADD_CASE({ - x.index_params.codebook_kind = ivf_pq::codebook_gen::PER_CLUSTER; - x.min_recall = 0.86; - }); - ADD_CASE({ - x.index_params.codebook_kind = ivf_pq::codebook_gen::PER_SUBSPACE; - x.min_recall = 0.86; - }); - ADD_CASE({ - x.index_params.codebook_kind = ivf_pq::codebook_gen::PER_CLUSTER; - x.index_params.pq_bits = 4; - x.min_recall = 0.79; - }); - ADD_CASE({ - x.index_params.codebook_kind = ivf_pq::codebook_gen::PER_CLUSTER; - x.index_params.pq_bits = 5; - x.min_recall = 0.83; - }); - - ADD_CASE({ - x.index_params.pq_bits = 6; - x.min_recall = 0.84; - }); - ADD_CASE({ - x.index_params.pq_bits = 7; - x.min_recall = 0.85; - }); - ADD_CASE({ - x.index_params.pq_bits = 8; - x.min_recall = 0.86; - }); - - ADD_CASE({ - x.index_params.force_random_rotation = true; - x.min_recall = 0.86; - }); - ADD_CASE({ - x.index_params.force_random_rotation = false; - x.min_recall = 0.86; - }); - - ADD_CASE({ - x.search_params.lut_dtype = CUDA_R_32F; - x.min_recall = 0.86; - }); - ADD_CASE({ - x.search_params.lut_dtype = CUDA_R_16F; - x.min_recall = 0.86; - }); - ADD_CASE({ - x.search_params.lut_dtype = CUDA_R_8U; - x.min_recall = 0.84; - }); - - ADD_CASE({ - x.search_params.internal_distance_dtype = CUDA_R_32F; - x.min_recall = 0.86; - }); - ADD_CASE({ - x.search_params.internal_distance_dtype = CUDA_R_16F; - x.search_params.lut_dtype = CUDA_R_16F; - x.min_recall = 0.86; - }); - - return xs; -} - -inline auto enum_variety_l2() -> test_cases_t -{ - return map(enum_variety(), [](const ivf_pq_inputs& x) { - ivf_pq_inputs y(x); - y.index_params.metric = distance::DistanceType::L2Expanded; - return y; - }); -} - -inline auto enum_variety_ip() -> test_cases_t -{ - return map(enum_variety(), [](const ivf_pq_inputs& x) { - ivf_pq_inputs y(x); - if (y.min_recall.has_value()) { - if (y.search_params.lut_dtype == CUDA_R_8U) { - // InnerProduct score is signed, - // thus we're forced to used signed 8-bit representation, - // thus we have one bit less precision - y.min_recall = y.min_recall.value() * 0.90; - } else { - // In other cases it seems to perform a little bit better, still worse than L2 - y.min_recall = y.min_recall.value() * 0.94; - } - } - y.index_params.metric = distance::DistanceType::InnerProduct; - return y; - }); -} - -inline auto enum_variety_l2sqrt() -> test_cases_t -{ - return map(enum_variety(), [](const ivf_pq_inputs& x) { - ivf_pq_inputs y(x); - y.index_params.metric = distance::DistanceType::L2SqrtExpanded; - return y; - }); -} - -/** - * Try different number of n_probes, some of which may trigger the non-fused version of the search - * kernel. - */ -inline auto var_n_probes() -> test_cases_t -{ - ivf_pq_inputs dflt; - std::vector xs; - for (auto x = dflt.index_params.n_lists; x >= 1; x /= 2) { - xs.push_back(x); - } - return map(xs, [](uint32_t n_probes) { - ivf_pq_inputs x; - x.search_params.n_probes = n_probes; - return x; - }); -} - -/** - * Try different number of nearest neighbours. - * Values smaller than 32 test if the code behaves well when Capacity (== 32) does not change, - * but `k <= Capacity` changes. - * - * Values between `32 and ivf_pq::detail::kMaxCapacity` test various instantiations of the - * main kernel (Capacity-templated) - * - * Values above ivf_pq::detail::kMaxCapacity should trigger the non-fused version of the kernel - * (manage_local_topk = false). - * - * Also we test here various values that are close-but-not-power-of-two to catch any problems - * related to rounding/alignment. - * - * Note, we cannot control explicitly which instance of the search kernel to choose, hence it's - * important to try a variety of different values of `k` to make sure all paths are triggered. - * - * Set the log level to DEBUG (5) or above to inspect the selected kernel instances. - */ -inline auto var_k() -> test_cases_t -{ - return map( - {1, 2, 3, 5, 8, 15, 16, 32, 63, 65, 127, 128, 256, 257, 1023, 2048, 2049}, [](uint32_t k) { - ivf_pq_inputs x; - x.k = k; - // when there's not enough data, try more cluster probes - x.search_params.n_probes = max(x.search_params.n_probes, min(x.index_params.n_lists, k)); - return x; - }); -} - -/** - * Cases brought up from downstream projects. - */ -inline auto special_cases() -> test_cases_t -{ - test_cases_t xs; - -#define ADD_CASE(f) \ - do { \ - xs.push_back({}); \ - ([](ivf_pq_inputs & x) f)(xs[xs.size() - 1]); \ - } while (0); - - ADD_CASE({ - x.num_db_vecs = 1183514; - x.dim = 100; - x.num_queries = 10000; - x.k = 10; - x.index_params.codebook_kind = ivf_pq::codebook_gen::PER_SUBSPACE; - x.index_params.pq_dim = 10; - x.index_params.pq_bits = 8; - x.index_params.n_lists = 1024; - x.search_params.n_probes = 50; - }); - - ADD_CASE({ - x.num_db_vecs = 10000; - x.dim = 16; - x.num_queries = 500; - x.k = 128; - x.index_params.metric = distance::DistanceType::L2Expanded; - x.index_params.codebook_kind = ivf_pq::codebook_gen::PER_SUBSPACE; - x.index_params.pq_bits = 8; - x.index_params.n_lists = 100; - x.search_params.n_probes = 100; - }); - - ADD_CASE({ - x.num_db_vecs = 10000; - x.dim = 16; - x.num_queries = 500; - x.k = 129; - x.index_params.metric = distance::DistanceType::L2Expanded; - x.index_params.codebook_kind = ivf_pq::codebook_gen::PER_SUBSPACE; - x.index_params.pq_bits = 8; - x.index_params.n_lists = 100; - x.search_params.n_probes = 100; - }); - - ADD_CASE({ - x.num_db_vecs = 4335; - x.dim = 4; - x.num_queries = 100000; - x.k = 12; - x.index_params.metric = distance::DistanceType::L2Expanded; - x.index_params.codebook_kind = ivf_pq::codebook_gen::PER_SUBSPACE; - x.index_params.pq_dim = 2; - x.index_params.pq_bits = 8; - x.index_params.n_lists = 69; - x.search_params.n_probes = 69; - }); - - ADD_CASE({ - x.num_db_vecs = 4335; - x.dim = 4; - x.num_queries = 100000; - x.k = 12; - x.index_params.metric = distance::DistanceType::L2Expanded; - x.index_params.codebook_kind = ivf_pq::codebook_gen::PER_CLUSTER; - x.index_params.pq_dim = 2; - x.index_params.pq_bits = 8; - x.index_params.n_lists = 69; - x.search_params.n_probes = 69; - }); - - return xs; -} - -/* Test instantiations */ - -#define TEST_BUILD_SEARCH(type) \ - TEST_P(type, build_search) /* NOLINT */ \ - { \ - this->run([this]() { return this->build_only(); }); \ - } - -#define TEST_BUILD_EXTEND_SEARCH(type) \ - TEST_P(type, build_extend_search) /* NOLINT */ \ - { \ - this->run([this]() { return this->build_2_extends(); }); \ - } - -#define TEST_BUILD_SERIALIZE_SEARCH(type) \ - TEST_P(type, build_serialize_search) /* NOLINT */ \ - { \ - this->run([this]() { return this->build_serialize(); }); \ - } - -#define INSTANTIATE(type, vals) \ - INSTANTIATE_TEST_SUITE_P(IvfPq, type, ::testing::ValuesIn(vals)); /* NOLINT */ - -} // namespace raft::neighbors::ivf_pq diff --git a/cpp/test/neighbors/ann_ivf_pq/ivf_pq_build_float_uint32_t.cu b/cpp/test/neighbors/ann_ivf_pq/ivf_pq_build_float_uint32_t.cu deleted file mode 100644 index 5ba21c3c2f..0000000000 --- a/cpp/test/neighbors/ann_ivf_pq/ivf_pq_build_float_uint32_t.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index -#include - -#define instantiate_raft_neighbors_ivf_pq_build(T, IdxT) \ - template raft::neighbors::ivf_pq::index raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - raft::device_matrix_view dataset); \ - \ - template auto raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - const T* dataset, \ - IdxT n_rows, \ - uint32_t dim) \ - ->raft::neighbors::ivf_pq::index; - -instantiate_raft_neighbors_ivf_pq_build(float, uint32_t); - -#undef instantiate_raft_neighbors_ivf_pq_build diff --git a/cpp/test/neighbors/ann_ivf_pq/ivf_pq_build_test-ext.cuh b/cpp/test/neighbors/ann_ivf_pq/ivf_pq_build_test-ext.cuh deleted file mode 100644 index cd5435ab2e..0000000000 --- a/cpp/test/neighbors/ann_ivf_pq/ivf_pq_build_test-ext.cuh +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include -#include // raft::neighbors::ivf_pq::index -#include - -#define instantiate_raft_neighbors_ivf_pq_build(T, IdxT) \ - extern template raft::neighbors::ivf_pq::index raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - raft::device_matrix_view dataset); \ - \ - extern template auto raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - const T* dataset, \ - IdxT n_rows, \ - uint32_t dim) \ - ->raft::neighbors::ivf_pq::index; - -instantiate_raft_neighbors_ivf_pq_build(float, uint32_t); - -#undef instantiate_raft_neighbors_ivf_pq_build diff --git a/cpp/test/neighbors/ann_ivf_pq/ivf_pq_search_float_uint32_t.cu b/cpp/test/neighbors/ann_ivf_pq/ivf_pq_search_float_uint32_t.cu deleted file mode 100644 index 00baa59f58..0000000000 --- a/cpp/test/neighbors/ann_ivf_pq/ivf_pq_search_float_uint32_t.cu +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index -#include - -#include - -#define instantiate_raft_neighbors_ivf_pq_search(T, IdxT) \ - template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances); \ - \ - template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances) - -instantiate_raft_neighbors_ivf_pq_search(float, uint32_t); - -#undef instantiate_raft_neighbors_ivf_pq_search - -#define instantiate_raft_neighbors_ivf_pq_search_with_filtering(T, IdxT, FilterT) \ - template void raft::neighbors::ivf_pq::search_with_filtering( \ - raft::resources const& handle, \ - const search_params& params, \ - const index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances, \ - FilterT sample_filter) - -#define COMMA , -instantiate_raft_neighbors_ivf_pq_search_with_filtering( - float, uint32_t, raft::neighbors::filtering::bitset_filter); - -instantiate_raft_neighbors_ivf_pq_search_with_filtering( - int8_t, int64_t, raft::neighbors::filtering::bitset_filter); - -instantiate_raft_neighbors_ivf_pq_search_with_filtering( - float, uint32_t, raft::neighbors::filtering::none_ivf_sample_filter); - -#undef COMMA -#undef instantiate_raft_neighbors_ivf_pq_search_with_filtering diff --git a/cpp/test/neighbors/ann_ivf_pq/test_filter_float_int64_t.cu b/cpp/test/neighbors/ann_ivf_pq/test_filter_float_int64_t.cu deleted file mode 100644 index 70d5d8761f..0000000000 --- a/cpp/test/neighbors/ann_ivf_pq/test_filter_float_int64_t.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_ivf_pq.cuh" - -#include -#include - -namespace raft::neighbors::ivf_pq { - -using f32_f32_i64_filter = ivf_pq_filter_test; - -TEST_BUILD_SEARCH(f32_f32_i64_filter) -INSTANTIATE(f32_f32_i64_filter, defaults() + big_dims_moderate_lut()); -} // namespace raft::neighbors::ivf_pq diff --git a/cpp/test/neighbors/ann_ivf_pq/test_filter_int8_t_int64_t.cu b/cpp/test/neighbors/ann_ivf_pq/test_filter_int8_t_int64_t.cu deleted file mode 100644 index ba96a8db0b..0000000000 --- a/cpp/test/neighbors/ann_ivf_pq/test_filter_int8_t_int64_t.cu +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_ivf_pq.cuh" - -#include -#include - -namespace raft::neighbors::ivf_pq { - -using f32_i08_i64_filter = ivf_pq_filter_test; - -TEST_BUILD_SEARCH(f32_i08_i64_filter) -INSTANTIATE(f32_i08_i64_filter, big_dims()); - -} // namespace raft::neighbors::ivf_pq diff --git a/cpp/test/neighbors/ann_ivf_pq/test_float_int64_t.cu b/cpp/test/neighbors/ann_ivf_pq/test_float_int64_t.cu deleted file mode 100644 index 9859061d70..0000000000 --- a/cpp/test/neighbors/ann_ivf_pq/test_float_int64_t.cu +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_ivf_pq.cuh" - -namespace raft::neighbors::ivf_pq { - -using f32_f32_i64 = ivf_pq_test; - -TEST_BUILD_EXTEND_SEARCH(f32_f32_i64) -TEST_BUILD_SERIALIZE_SEARCH(f32_f32_i64) -INSTANTIATE(f32_f32_i64, defaults() + small_dims() + big_dims_moderate_lut()); - -} // namespace raft::neighbors::ivf_pq diff --git a/cpp/test/neighbors/ann_ivf_pq/test_float_uint32_t.cu b/cpp/test/neighbors/ann_ivf_pq/test_float_uint32_t.cu deleted file mode 100644 index b8ada2249a..0000000000 --- a/cpp/test/neighbors/ann_ivf_pq/test_float_uint32_t.cu +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_ivf_pq.cuh" -#include "ivf_pq_build_test-ext.cuh" - -#include -#include - -namespace raft::neighbors::ivf_pq { - -using f32_f32_u32 = ivf_pq_test; -using f32_f32_u32_filter = ivf_pq_filter_test; - -TEST_BUILD_SEARCH(f32_f32_u32) -TEST_BUILD_SERIALIZE_SEARCH(f32_f32_u32) -INSTANTIATE(f32_f32_u32, defaults() + var_n_probes() + var_k() + special_cases()); - -TEST_BUILD_SEARCH(f32_f32_u32_filter) -INSTANTIATE(f32_f32_u32_filter, defaults()); -} // namespace raft::neighbors::ivf_pq diff --git a/cpp/test/neighbors/ann_ivf_pq/test_int8_t_int64_t.cu b/cpp/test/neighbors/ann_ivf_pq/test_int8_t_int64_t.cu deleted file mode 100644 index 970bdd6a12..0000000000 --- a/cpp/test/neighbors/ann_ivf_pq/test_int8_t_int64_t.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_ivf_pq.cuh" - -#include -namespace raft::neighbors::ivf_pq { - -using f32_i08_i64 = ivf_pq_test; - -TEST_BUILD_SEARCH(f32_i08_i64) -TEST_BUILD_SERIALIZE_SEARCH(f32_i08_i64) -INSTANTIATE(f32_i08_i64, defaults() + big_dims() + var_k()); - -} // namespace raft::neighbors::ivf_pq diff --git a/cpp/test/neighbors/ann_ivf_pq/test_uint8_t_int64_t.cu b/cpp/test/neighbors/ann_ivf_pq/test_uint8_t_int64_t.cu deleted file mode 100644 index e949c2f7ed..0000000000 --- a/cpp/test/neighbors/ann_ivf_pq/test_uint8_t_int64_t.cu +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_ivf_pq.cuh" - -namespace raft::neighbors::ivf_pq { - -using f32_u08_i64 = ivf_pq_test; - -TEST_BUILD_SEARCH(f32_u08_i64) -TEST_BUILD_EXTEND_SEARCH(f32_u08_i64) -INSTANTIATE(f32_u08_i64, small_dims_per_cluster() + enum_variety()); - -} // namespace raft::neighbors::ivf_pq diff --git a/cpp/test/neighbors/ann_nn_descent.cuh b/cpp/test/neighbors/ann_nn_descent.cuh deleted file mode 100644 index 5070d83b15..0000000000 --- a/cpp/test/neighbors/ann_nn_descent.cuh +++ /dev/null @@ -1,332 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "ann_utils.cuh" - -#include -#include -#include -#include - -#include - -#include - -#include -#include -#include -#include - -namespace raft::neighbors::experimental::nn_descent { - -struct AnnNNDescentInputs { - int n_rows; - int dim; - int graph_degree; - raft::distance::DistanceType metric; - bool host_dataset; - double min_recall; -}; - -struct AnnNNDescentBatchInputs { - std::pair recall_cluster; - int n_rows; - int dim; - int graph_degree; - raft::distance::DistanceType metric; - bool host_dataset; -}; - -inline ::std::ostream& operator<<(::std::ostream& os, const AnnNNDescentInputs& p) -{ - os << "dataset shape=" << p.n_rows << "x" << p.dim << ", graph_degree=" << p.graph_degree - << ", metric=" << static_cast(p.metric) << (p.host_dataset ? ", host" : ", device") - << std::endl; - return os; -} - -inline ::std::ostream& operator<<(::std::ostream& os, const AnnNNDescentBatchInputs& p) -{ - os << "dataset shape=" << p.n_rows << "x" << p.dim << ", graph_degree=" << p.graph_degree - << ", metric=" << static_cast(p.metric) << (p.host_dataset ? ", host" : ", device") - << ", clusters=" << p.recall_cluster.second << std::endl; - return os; -} - -template -class AnnNNDescentTest : public ::testing::TestWithParam { - public: - AnnNNDescentTest() - : stream_(resource::get_cuda_stream(handle_)), - ps(::testing::TestWithParam::GetParam()), - database(0, stream_) - { - } - - protected: - void testNNDescent() - { - size_t queries_size = ps.n_rows * ps.graph_degree; - std::vector indices_NNDescent(queries_size); - std::vector distances_NNDescent(queries_size); - std::vector indices_naive(queries_size); - std::vector distances_naive(queries_size); - - { - rmm::device_uvector distances_naive_dev(queries_size, stream_); - rmm::device_uvector indices_naive_dev(queries_size, stream_); - naive_knn(handle_, - distances_naive_dev.data(), - indices_naive_dev.data(), - database.data(), - database.data(), - ps.n_rows, - ps.n_rows, - ps.dim, - ps.graph_degree, - ps.metric); - update_host(indices_naive.data(), indices_naive_dev.data(), queries_size, stream_); - update_host(distances_naive.data(), distances_naive_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - { - { - nn_descent::index_params index_params; - index_params.metric = ps.metric; - index_params.graph_degree = ps.graph_degree; - index_params.intermediate_graph_degree = 2 * ps.graph_degree; - index_params.max_iterations = 100; - index_params.return_distances = true; - - auto database_view = raft::make_device_matrix_view( - (const DataT*)database.data(), ps.n_rows, ps.dim); - - { - if (ps.host_dataset) { - auto database_host = raft::make_host_matrix(ps.n_rows, ps.dim); - raft::copy(database_host.data_handle(), database.data(), database.size(), stream_); - auto database_host_view = raft::make_host_matrix_view( - (const DataT*)database_host.data_handle(), ps.n_rows, ps.dim); - index index{handle_, ps.n_rows, static_cast(ps.graph_degree), true}; - nn_descent::build( - handle_, index_params, database_host_view, index, DistEpilogue()); - raft::copy( - indices_NNDescent.data(), index.graph().data_handle(), queries_size, stream_); - if (index.distances().has_value()) { - raft::copy(distances_NNDescent.data(), - index.distances().value().data_handle(), - queries_size, - stream_); - } - - } else { - index index{handle_, ps.n_rows, static_cast(ps.graph_degree), true}; - nn_descent::build( - handle_, index_params, database_view, index, DistEpilogue()); - raft::copy( - indices_NNDescent.data(), index.graph().data_handle(), queries_size, stream_); - if (index.distances().has_value()) { - raft::copy(distances_NNDescent.data(), - index.distances().value().data_handle(), - queries_size, - stream_); - } - }; - } - resource::sync_stream(handle_); - } - - double min_recall = ps.min_recall; - EXPECT_TRUE(eval_neighbours(indices_naive, - indices_NNDescent, - distances_naive, - distances_NNDescent, - ps.n_rows, - ps.graph_degree, - 0.001, - min_recall)); - } - } - - void SetUp() override - { - database.resize(((size_t)ps.n_rows) * ps.dim, stream_); - raft::random::RngState r(1234ULL); - if constexpr (std::is_same{}) { - raft::random::normal(handle_, r, database.data(), ps.n_rows * ps.dim, DataT(0.1), DataT(2.0)); - } else { - raft::random::uniformInt( - handle_, r, database.data(), ps.n_rows * ps.dim, DataT(1), DataT(20)); - } - resource::sync_stream(handle_); - } - - void TearDown() override - { - resource::sync_stream(handle_); - database.resize(0, stream_); - } - - private: - raft::resources handle_; - rmm::cuda_stream_view stream_; - AnnNNDescentInputs ps; - rmm::device_uvector database; -}; - -template -class AnnNNDescentBatchTest : public ::testing::TestWithParam { - public: - AnnNNDescentBatchTest() - : stream_(resource::get_cuda_stream(handle_)), - ps(::testing::TestWithParam::GetParam()), - database(0, stream_) - { - } - - void testNNDescentBatch() - { - size_t queries_size = ps.n_rows * ps.graph_degree; - std::vector indices_NNDescent(queries_size); - std::vector distances_NNDescent(queries_size); - std::vector indices_naive(queries_size); - std::vector distances_naive(queries_size); - - { - rmm::device_uvector distances_naive_dev(queries_size, stream_); - rmm::device_uvector indices_naive_dev(queries_size, stream_); - naive_knn(handle_, - distances_naive_dev.data(), - indices_naive_dev.data(), - database.data(), - database.data(), - ps.n_rows, - ps.n_rows, - ps.dim, - ps.graph_degree, - ps.metric); - update_host(indices_naive.data(), indices_naive_dev.data(), queries_size, stream_); - update_host(distances_naive.data(), distances_naive_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - { - { - nn_descent::index_params index_params; - index_params.metric = ps.metric; - index_params.graph_degree = ps.graph_degree; - index_params.intermediate_graph_degree = 2 * ps.graph_degree; - index_params.max_iterations = 10; - index_params.return_distances = true; - index_params.n_clusters = ps.recall_cluster.second; - - auto database_view = raft::make_device_matrix_view( - (const DataT*)database.data(), ps.n_rows, ps.dim); - - { - if (ps.host_dataset) { - auto database_host = raft::make_host_matrix(ps.n_rows, ps.dim); - raft::copy(database_host.data_handle(), database.data(), database.size(), stream_); - auto database_host_view = raft::make_host_matrix_view( - (const DataT*)database_host.data_handle(), ps.n_rows, ps.dim); - auto index = nn_descent::build( - handle_, index_params, database_host_view, DistEpilogue()); - raft::copy( - indices_NNDescent.data(), index.graph().data_handle(), queries_size, stream_); - if (index.distances().has_value()) { - raft::copy(distances_NNDescent.data(), - index.distances().value().data_handle(), - queries_size, - stream_); - } - - } else { - auto index = nn_descent::build( - handle_, index_params, database_view, DistEpilogue()); - raft::copy( - indices_NNDescent.data(), index.graph().data_handle(), queries_size, stream_); - if (index.distances().has_value()) { - raft::copy(distances_NNDescent.data(), - index.distances().value().data_handle(), - queries_size, - stream_); - } - }; - } - resource::sync_stream(handle_); - } - double min_recall = ps.recall_cluster.first; - EXPECT_TRUE(eval_neighbours(indices_naive, - indices_NNDescent, - distances_naive, - distances_NNDescent, - ps.n_rows, - ps.graph_degree, - 0.01, - min_recall, - true, - static_cast(ps.graph_degree * 0.1))); - } - } - - void SetUp() override - { - database.resize(((size_t)ps.n_rows) * ps.dim, stream_); - raft::random::RngState r(1234ULL); - if constexpr (std::is_same{}) { - raft::random::normal(handle_, r, database.data(), ps.n_rows * ps.dim, DataT(0.1), DataT(2.0)); - } else { - raft::random::uniformInt( - handle_, r, database.data(), ps.n_rows * ps.dim, DataT(1), DataT(20)); - } - resource::sync_stream(handle_); - } - - void TearDown() override - { - resource::sync_stream(handle_); - database.resize(0, stream_); - } - - private: - raft::resources handle_; - rmm::cuda_stream_view stream_; - AnnNNDescentBatchInputs ps; - rmm::device_uvector database; -}; - -const std::vector inputs = raft::util::itertools::product( - {1000, 2000}, // n_rows - {3, 5, 7, 8, 17, 64, 128, 137, 192, 256, 512, 619, 1024}, // dim - {32, 64}, // graph_degree - {raft::distance::DistanceType::L2Expanded}, - {false, true}, - {0.90}); - -// TODO: Investigate why this test is failing -// Reference issue https://github.com/rapidsai/raft/issues/2450 -// const std::vector inputsBatch = -// raft::util::itertools::product( -// {std::make_pair(0.9, 3lu), std::make_pair(0.9, 2lu)}, // min_recall, n_clusters -// {4000, 5000}, // n_rows -// {192, 512}, // dim -// {32, 64}, // graph_degree -// {raft::distance::DistanceType::L2Expanded}, -// {false, true}); - -} // namespace raft::neighbors::experimental::nn_descent diff --git a/cpp/test/neighbors/ann_nn_descent/test_batch_float_uint32_t.cu b/cpp/test/neighbors/ann_nn_descent/test_batch_float_uint32_t.cu deleted file mode 100644 index c6f56e8c39..0000000000 --- a/cpp/test/neighbors/ann_nn_descent/test_batch_float_uint32_t.cu +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_nn_descent.cuh" - -#include - -namespace raft::neighbors::experimental::nn_descent { - -typedef AnnNNDescentBatchTest AnnNNDescentBatchTestF_U32; -TEST_P(AnnNNDescentBatchTestF_U32, AnnNNDescentBatch) { this->testNNDescentBatch(); } - -INSTANTIATE_TEST_CASE_P(AnnNNDescentBatchTest, - AnnNNDescentBatchTestF_U32, - ::testing::ValuesIn(inputsBatch)); - -} // namespace raft::neighbors::experimental::nn_descent diff --git a/cpp/test/neighbors/ann_nn_descent/test_float_uint32_t.cu b/cpp/test/neighbors/ann_nn_descent/test_float_uint32_t.cu deleted file mode 100644 index ec6d04ad12..0000000000 --- a/cpp/test/neighbors/ann_nn_descent/test_float_uint32_t.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_nn_descent.cuh" - -#include - -namespace raft::neighbors::experimental::nn_descent { - -typedef AnnNNDescentTest AnnNNDescentTestF_U32; -TEST_P(AnnNNDescentTestF_U32, AnnNNDescent) { this->testNNDescent(); } - -INSTANTIATE_TEST_CASE_P(AnnNNDescentTest, AnnNNDescentTestF_U32, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::experimental::nn_descent diff --git a/cpp/test/neighbors/ann_nn_descent/test_int8_t_uint32_t.cu b/cpp/test/neighbors/ann_nn_descent/test_int8_t_uint32_t.cu deleted file mode 100644 index 27fa42d636..0000000000 --- a/cpp/test/neighbors/ann_nn_descent/test_int8_t_uint32_t.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_nn_descent.cuh" - -#include - -namespace raft::neighbors::experimental::nn_descent { - -typedef AnnNNDescentTest AnnNNDescentTestI8_U32; -TEST_P(AnnNNDescentTestI8_U32, AnnNNDescent) { this->testNNDescent(); } - -INSTANTIATE_TEST_CASE_P(AnnNNDescentTest, AnnNNDescentTestI8_U32, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::experimental::nn_descent diff --git a/cpp/test/neighbors/ann_nn_descent/test_uint8_t_uint32_t.cu b/cpp/test/neighbors/ann_nn_descent/test_uint8_t_uint32_t.cu deleted file mode 100644 index 3afe79dcc4..0000000000 --- a/cpp/test/neighbors/ann_nn_descent/test_uint8_t_uint32_t.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_nn_descent.cuh" - -#include - -namespace raft::neighbors::experimental::nn_descent { - -typedef AnnNNDescentTest AnnNNDescentTestUI8_U32; -TEST_P(AnnNNDescentTestUI8_U32, AnnNNDescent) { this->testNNDescent(); } - -INSTANTIATE_TEST_CASE_P(AnnNNDescentTest, AnnNNDescentTestUI8_U32, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::experimental::nn_descent diff --git a/cpp/test/neighbors/ann_utils.cuh b/cpp/test/neighbors/ann_utils.cuh deleted file mode 100644 index 82e3ace9da..0000000000 --- a/cpp/test/neighbors/ann_utils.cuh +++ /dev/null @@ -1,335 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "../test_utils.cuh" - -#include // raft::make_device_matrix -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include - -namespace raft::neighbors { - -struct print_dtype { - cudaDataType_t value; -}; - -inline auto operator<<(std::ostream& os, const print_dtype& p) -> std::ostream& -{ - switch (p.value) { - case CUDA_R_16F: os << "CUDA_R_16F"; break; - case CUDA_C_16F: os << "CUDA_C_16F"; break; - case CUDA_R_16BF: os << "CUDA_R_16BF"; break; - case CUDA_C_16BF: os << "CUDA_C_16BF"; break; - case CUDA_R_32F: os << "CUDA_R_32F"; break; - case CUDA_C_32F: os << "CUDA_C_32F"; break; - case CUDA_R_64F: os << "CUDA_R_64F"; break; - case CUDA_C_64F: os << "CUDA_C_64F"; break; - case CUDA_R_4I: os << "CUDA_R_4I"; break; - case CUDA_C_4I: os << "CUDA_C_4I"; break; - case CUDA_R_4U: os << "CUDA_R_4U"; break; - case CUDA_C_4U: os << "CUDA_C_4U"; break; - case CUDA_R_8I: os << "CUDA_R_8I"; break; - case CUDA_C_8I: os << "CUDA_C_8I"; break; - case CUDA_R_8U: os << "CUDA_R_8U"; break; - case CUDA_C_8U: os << "CUDA_C_8U"; break; - case CUDA_R_16I: os << "CUDA_R_16I"; break; - case CUDA_C_16I: os << "CUDA_C_16I"; break; - case CUDA_R_16U: os << "CUDA_R_16U"; break; - case CUDA_C_16U: os << "CUDA_C_16U"; break; - case CUDA_R_32I: os << "CUDA_R_32I"; break; - case CUDA_C_32I: os << "CUDA_C_32I"; break; - case CUDA_R_32U: os << "CUDA_R_32U"; break; - case CUDA_C_32U: os << "CUDA_C_32U"; break; - case CUDA_R_64I: os << "CUDA_R_64I"; break; - case CUDA_C_64I: os << "CUDA_C_64I"; break; - case CUDA_R_64U: os << "CUDA_R_64U"; break; - case CUDA_C_64U: os << "CUDA_C_64U"; break; - default: RAFT_FAIL("unreachable code"); - } - return os; -} - -struct print_metric { - raft::distance::DistanceType value; -}; - -inline auto operator<<(std::ostream& os, const print_metric& p) -> std::ostream& -{ - switch (p.value) { - case raft::distance::L2Expanded: os << "distance::L2Expanded"; break; - case raft::distance::L2SqrtExpanded: os << "distance::L2SqrtExpanded"; break; - case raft::distance::CosineExpanded: os << "distance::CosineExpanded"; break; - case raft::distance::L1: os << "distance::L1"; break; - case raft::distance::L2Unexpanded: os << "distance::L2Unexpanded"; break; - case raft::distance::L2SqrtUnexpanded: os << "distance::L2SqrtUnexpanded"; break; - case raft::distance::InnerProduct: os << "distance::InnerProduct"; break; - case raft::distance::Linf: os << "distance::Linf"; break; - case raft::distance::Canberra: os << "distance::Canberra"; break; - case raft::distance::LpUnexpanded: os << "distance::LpUnexpanded"; break; - case raft::distance::CorrelationExpanded: os << "distance::CorrelationExpanded"; break; - case raft::distance::JaccardExpanded: os << "distance::JaccardExpanded"; break; - case raft::distance::HellingerExpanded: os << "distance::HellingerExpanded"; break; - case raft::distance::Haversine: os << "distance::Haversine"; break; - case raft::distance::BrayCurtis: os << "distance::BrayCurtis"; break; - case raft::distance::JensenShannon: os << "distance::JensenShannon"; break; - case raft::distance::HammingUnexpanded: os << "distance::HammingUnexpanded"; break; - case raft::distance::KLDivergence: os << "distance::KLDivergence"; break; - case raft::distance::RusselRaoExpanded: os << "distance::RusselRaoExpanded"; break; - case raft::distance::DiceExpanded: os << "distance::DiceExpanded"; break; - case raft::distance::Precomputed: os << "distance::Precomputed"; break; - default: RAFT_FAIL("unreachable code"); - } - return os; -} - -template -struct idx_dist_pair { - IdxT idx; - DistT dist; - CompareDist eq_compare; - auto operator==(const idx_dist_pair& a) const -> bool - { - if (idx == a.idx) return true; - if (eq_compare(dist, a.dist)) return true; - return false; - } - idx_dist_pair(IdxT x, DistT y, CompareDist op) : idx(x), dist(y), eq_compare(op) {} -}; - -/** Calculate recall value using only neighbor indices - */ -template -auto calc_recall(const std::vector& expected_idx, - const std::vector& actual_idx, - size_t rows, - size_t cols) -{ - size_t match_count = 0; - size_t total_count = static_cast(rows) * static_cast(cols); - for (size_t i = 0; i < rows; ++i) { - for (size_t k = 0; k < cols; ++k) { - size_t idx_k = i * cols + k; // row major assumption! - auto act_idx = actual_idx[idx_k]; - for (size_t j = 0; j < cols; ++j) { - size_t idx = i * cols + j; // row major assumption! - auto exp_idx = expected_idx[idx]; - if (act_idx == exp_idx) { - match_count++; - break; - } - } - } - } - return std::make_tuple( - static_cast(match_count) / static_cast(total_count), match_count, total_count); -} - -/** check uniqueness of indices - */ -template -auto check_unique_indices(const std::vector& actual_idx, - size_t rows, - size_t cols, - size_t max_duplicates) -{ - size_t max_count; - size_t dup_count = 0lu; - std::set unique_indices; - for (size_t i = 0; i < rows; ++i) { - unique_indices.clear(); - max_count = 0; - for (size_t k = 0; k < cols; ++k) { - size_t idx_k = i * cols + k; // row major assumption! - auto act_idx = actual_idx[idx_k]; - if (act_idx == std::numeric_limits::max()) { - max_count++; - } else if (unique_indices.find(act_idx) == unique_indices.end()) { - unique_indices.insert(act_idx); - } else { - dup_count++; - if (dup_count > max_duplicates) { - return testing::AssertionFailure() - << "Duplicated index " << act_idx << " at k " << k << " for query " << i << "! "; - } - } - } - } - return testing::AssertionSuccess(); -} - -template -auto eval_recall(const std::vector& expected_idx, - const std::vector& actual_idx, - size_t rows, - size_t cols, - double eps, - double min_recall, - bool test_unique = true) -> testing::AssertionResult -{ - auto [actual_recall, match_count, total_count] = - calc_recall(expected_idx, actual_idx, rows, cols); - double error_margin = (actual_recall - min_recall) / std::max(1.0 - min_recall, eps); - RAFT_LOG_INFO("Recall = %f (%zu/%zu), the error is %2.1f%% %s the threshold (eps = %f).", - actual_recall, - match_count, - total_count, - std::abs(error_margin * 100.0), - error_margin < 0 ? "above" : "below", - eps); - if (actual_recall < min_recall - eps) { - return testing::AssertionFailure() - << "actual recall (" << actual_recall << ") is lower than the minimum expected recall (" - << min_recall << "); eps = " << eps << ". "; - } - if (test_unique) - return check_unique_indices(actual_idx, rows, cols); - else - return testing::AssertionSuccess(); -} - -/** Overload of calc_recall to account for distances - */ -template -auto calc_recall(const std::vector& expected_idx, - const std::vector& actual_idx, - const std::vector& expected_dist, - const std::vector& actual_dist, - size_t rows, - size_t cols, - double eps) -{ - size_t match_count = 0; - size_t total_count = static_cast(rows) * static_cast(cols); - for (size_t i = 0; i < rows; ++i) { - for (size_t k = 0; k < cols; ++k) { - size_t idx_k = i * cols + k; // row major assumption! - auto act_idx = actual_idx[idx_k]; - auto act_dist = actual_dist[idx_k]; - for (size_t j = 0; j < cols; ++j) { - size_t idx = i * cols + j; // row major assumption! - auto exp_idx = expected_idx[idx]; - auto exp_dist = expected_dist[idx]; - idx_dist_pair exp_kvp(exp_idx, exp_dist, raft::CompareApprox(eps)); - idx_dist_pair act_kvp(act_idx, act_dist, raft::CompareApprox(eps)); - if (exp_kvp == act_kvp) { - match_count++; - break; - } - } - } - } - return std::make_tuple( - static_cast(match_count) / static_cast(total_count), match_count, total_count); -} - -/** same as eval_recall, but in case indices do not match, - * then check distances as well, and accept match if actual dist is equal to expected_dist */ -template -auto eval_neighbours(const std::vector& expected_idx, - const std::vector& actual_idx, - const std::vector& expected_dist, - const std::vector& actual_dist, - size_t rows, - size_t cols, - double eps, - double min_recall, - bool test_unique = true, - size_t max_duplicates = 0) -> testing::AssertionResult -{ - auto [actual_recall, match_count, total_count] = - calc_recall(expected_idx, actual_idx, expected_dist, actual_dist, rows, cols, eps); - double error_margin = (actual_recall - min_recall) / std::max(1.0 - min_recall, eps); - RAFT_LOG_INFO("Recall = %f (%zu/%zu), the error is %2.1f%% %s the threshold (eps = %f).", - actual_recall, - match_count, - total_count, - std::abs(error_margin * 100.0), - error_margin < 0 ? "above" : "below", - eps); - if (actual_recall < min_recall - eps) { - return testing::AssertionFailure() - << "actual recall (" << actual_recall << ") is lower than the minimum expected recall (" - << min_recall << "); eps = " << eps << ". "; - } - if (test_unique) - return check_unique_indices(actual_idx, rows, cols, max_duplicates); - else - return testing::AssertionSuccess(); -} - -template -auto eval_distances(raft::resources const& handle, - const T* x, // dataset, n_rows * n_cols - const T* queries, // n_queries * n_cols - const IdxT* neighbors, // n_queries * k - const DistT* distances, // n_queries *k - size_t n_rows, - size_t n_cols, - size_t n_queries, - uint32_t k, - raft::distance::DistanceType metric, - double eps) -> testing::AssertionResult -{ - // for each vector, we calculate the actual distance to the k neighbors - - for (size_t i = 0; i < n_queries; i++) { - auto y = raft::make_device_matrix(handle, k, n_cols); - auto naive_dist = raft::make_device_matrix(handle, 1, k); - - raft::matrix::copy_rows( - handle, - make_device_matrix_view(x, n_rows, n_cols), - y.view(), - make_device_vector_view(neighbors + i * k, k)); - - dim3 block_dim(16, 32, 1); - auto grid_y = - static_cast(std::min(raft::ceildiv(k, block_dim.y), 32768)); - dim3 grid_dim(raft::ceildiv(n_rows, block_dim.x), grid_y, 1); - - naive_distance_kernel - <<>>( - naive_dist.data_handle(), queries + i * n_cols, y.data_handle(), 1, k, n_cols, metric); - - if (!devArrMatch(distances + i * k, - naive_dist.data_handle(), - naive_dist.size(), - CompareApprox(eps))) { - std::cout << n_rows << "x" << n_cols << ", " << k << std::endl; - std::cout << "query " << i << std::endl; - print_vector(" indices", neighbors + i * k, k, std::cout); - print_vector("n dist", distances + i * k, k, std::cout); - print_vector("c dist", naive_dist.data_handle(), naive_dist.size(), std::cout); - - return testing::AssertionFailure(); - } - } - return testing::AssertionSuccess(); -} -} // namespace raft::neighbors diff --git a/cpp/test/neighbors/fused_l2_knn.cu b/cpp/test/neighbors/fused_l2_knn.cu deleted file mode 100644 index e4d018aff0..0000000000 --- a/cpp/test/neighbors/fused_l2_knn.cu +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "./knn_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include -#include - -namespace raft { -namespace spatial { -namespace knn { -struct FusedL2KNNInputs { - int num_queries; - int num_db_vecs; - int dim; - int k; - raft::distance::DistanceType metric_; -}; - -template -class FusedL2KNNTest : public ::testing::TestWithParam { - public: - FusedL2KNNTest() - : stream_(resource::get_cuda_stream(handle_)), - params_(::testing::TestWithParam::GetParam()), - database(params_.num_db_vecs * params_.dim, stream_), - search_queries(params_.num_queries * params_.dim, stream_), - raft_indices_(params_.num_queries * params_.k, stream_), - raft_distances_(params_.num_queries * params_.k, stream_), - ref_indices_(params_.num_queries * params_.k, stream_), - ref_distances_(params_.num_queries * params_.k, stream_) - { - RAFT_CUDA_TRY(cudaMemsetAsync(database.data(), 0, database.size() * sizeof(T), stream_)); - RAFT_CUDA_TRY( - cudaMemsetAsync(search_queries.data(), 0, search_queries.size() * sizeof(T), stream_)); - RAFT_CUDA_TRY( - cudaMemsetAsync(raft_indices_.data(), 0, raft_indices_.size() * sizeof(int64_t), stream_)); - RAFT_CUDA_TRY( - cudaMemsetAsync(raft_distances_.data(), 0, raft_distances_.size() * sizeof(T), stream_)); - RAFT_CUDA_TRY( - cudaMemsetAsync(ref_indices_.data(), 0, ref_indices_.size() * sizeof(int64_t), stream_)); - RAFT_CUDA_TRY( - cudaMemsetAsync(ref_distances_.data(), 0, ref_distances_.size() * sizeof(T), stream_)); - } - - protected: - void testBruteForce() - { - // calculate the naive knn, by calculating the full pairwise distances and doing a k-select - rmm::device_uvector temp_distances(num_db_vecs * num_queries, stream_); - distance::pairwise_distance( - handle_, - raft::make_device_matrix_view(search_queries.data(), num_queries, dim), - raft::make_device_matrix_view(database.data(), num_db_vecs, dim), - raft::make_device_matrix_view(temp_distances.data(), num_queries, num_db_vecs), - metric); - - matrix::select_k( - handle_, - make_device_matrix_view(temp_distances.data(), num_queries, num_db_vecs), - std::nullopt, - make_device_matrix_view(ref_distances_.data(), num_queries, k_), - make_device_matrix_view(ref_indices_.data(), num_queries, k_), - true, - true); - - auto index_view = - raft::make_device_matrix_view(database.data(), num_db_vecs, dim); - auto query_view = - raft::make_device_matrix_view(search_queries.data(), num_queries, dim); - auto out_indices_view = - raft::make_device_matrix_view(raft_indices_.data(), num_queries, k_); - auto out_dists_view = - raft::make_device_matrix_view(raft_distances_.data(), num_queries, k_); - raft::neighbors::brute_force::fused_l2_knn( - handle_, index_view, query_view, out_indices_view, out_dists_view, metric); - - // verify. - ASSERT_TRUE(devArrMatchKnnPair(ref_indices_.data(), - raft_indices_.data(), - ref_distances_.data(), - raft_distances_.data(), - num_queries, - k_, - float(0.001), - stream_)); - } - - void SetUp() override - { - num_queries = params_.num_queries; - num_db_vecs = params_.num_db_vecs; - dim = params_.dim; - k_ = params_.k; - metric = params_.metric_; - - unsigned long long int seed = 1234ULL; - raft::random::RngState r(seed); - uniform(handle_, r, database.data(), num_db_vecs * dim, T(-1.0), T(1.0)); - uniform(handle_, r, search_queries.data(), num_queries * dim, T(-1.0), T(1.0)); - } - - private: - raft::resources handle_; - cudaStream_t stream_ = 0; - FusedL2KNNInputs params_; - int num_queries; - int num_db_vecs; - int dim; - rmm::device_uvector database; - rmm::device_uvector search_queries; - rmm::device_uvector raft_indices_; - rmm::device_uvector raft_distances_; - rmm::device_uvector ref_indices_; - rmm::device_uvector ref_distances_; - int k_; - raft::distance::DistanceType metric; -}; - -const std::vector inputs = { - {100, 1000, 16, 10, raft::distance::DistanceType::L2Expanded}, - {256, 256, 30, 10, raft::distance::DistanceType::L2Expanded}, - {1000, 10000, 16, 10, raft::distance::DistanceType::L2Expanded}, - {100, 1000, 16, 50, raft::distance::DistanceType::L2Expanded}, - {20, 10000, 16, 10, raft::distance::DistanceType::L2Expanded}, - {1000, 10000, 16, 50, raft::distance::DistanceType::L2Expanded}, - {1000, 10000, 32, 50, raft::distance::DistanceType::L2Expanded}, - {10000, 40000, 32, 30, raft::distance::DistanceType::L2Expanded}, - // L2 unexpanded - {100, 1000, 16, 10, raft::distance::DistanceType::L2Unexpanded}, - {1000, 10000, 16, 10, raft::distance::DistanceType::L2Unexpanded}, - {100, 1000, 16, 50, raft::distance::DistanceType::L2Unexpanded}, - {20, 10000, 16, 50, raft::distance::DistanceType::L2Unexpanded}, - {1000, 10000, 16, 50, raft::distance::DistanceType::L2Unexpanded}, - {1000, 10000, 32, 50, raft::distance::DistanceType::L2Unexpanded}, - {10000, 40000, 32, 30, raft::distance::DistanceType::L2Unexpanded}, -}; - -typedef FusedL2KNNTest FusedL2KNNTestF; -TEST_P(FusedL2KNNTestF, FusedBruteForce) { this->testBruteForce(); } - -INSTANTIATE_TEST_CASE_P(FusedL2KNNTest, FusedL2KNNTestF, ::testing::ValuesIn(inputs)); - -} // namespace knn -} // namespace spatial -} // namespace raft diff --git a/cpp/test/neighbors/knn.cu b/cpp/test/neighbors/knn.cu deleted file mode 100644 index a98de85bda..0000000000 --- a/cpp/test/neighbors/knn.cu +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" - -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include -#include - -namespace raft::neighbors::brute_force { -struct KNNInputs { - std::vector> input; - int k; - std::vector labels; -}; - -template -RAFT_KERNEL build_actual_output( - int* output, int n_rows, int k, const int* idx_labels, const IdxT* indices) -{ - int element = threadIdx.x + blockDim.x * blockIdx.x; - if (element >= n_rows * k) return; - - output[element] = idx_labels[indices[element]]; -} - -RAFT_KERNEL build_expected_output(int* output, int n_rows, int k, const int* labels) -{ - int row = threadIdx.x + blockDim.x * blockIdx.x; - if (row >= n_rows) return; - - int cur_label = labels[row]; - for (int i = 0; i < k; i++) { - output[row * k + i] = cur_label; - } -} - -template -class KNNTest : public ::testing::TestWithParam { - public: - KNNTest() - : params_(::testing::TestWithParam::GetParam()), - stream(resource::get_cuda_stream(handle)), - actual_labels_(0, stream), - expected_labels_(0, stream), - input_(0, stream), - search_data_(0, stream), - indices_(0, stream), - distances_(0, stream), - search_labels_(0, stream) - { - } - - protected: - void testBruteForce() - { - // #if (RAFT_ACTIVE_LEVEL >= RAFT_LEVEL_DEBUG) - raft::print_device_vector("Input array: ", input_.data(), rows_ * cols_, std::cout); - std::cout << "K: " << k_ << std::endl; - raft::print_device_vector("Labels array: ", search_labels_.data(), rows_, std::cout); - // #endif - - std::vector> index = { - make_device_matrix_view((const T*)(input_.data()), rows_, cols_)}; - auto search = raft::make_device_matrix_view( - (const T*)(search_data_.data()), rows_, cols_); - - auto indices = raft::make_device_matrix_view(indices_.data(), rows_, k_); - auto distances = - raft::make_device_matrix_view(distances_.data(), rows_, k_); - - auto metric = raft::distance::DistanceType::L2Unexpanded; - knn(handle, index, search, indices, distances, metric, std::make_optional(0)); - - build_actual_output<<>>( - actual_labels_.data(), rows_, k_, search_labels_.data(), indices_.data()); - - build_expected_output<<>>( - expected_labels_.data(), rows_, k_, search_labels_.data()); - - ASSERT_TRUE(devArrMatch( - expected_labels_.data(), actual_labels_.data(), rows_ * k_, raft::Compare(), stream)); - } - - void SetUp() override - { - rows_ = params_.input.size(); - cols_ = params_.input[0].size(); - k_ = params_.k; - - actual_labels_.resize(rows_ * k_, stream); - expected_labels_.resize(rows_ * k_, stream); - input_.resize(rows_ * cols_, stream); - search_data_.resize(rows_ * cols_, stream); - indices_.resize(rows_ * k_, stream); - distances_.resize(rows_ * k_, stream); - search_labels_.resize(rows_, stream); - - RAFT_CUDA_TRY( - cudaMemsetAsync(actual_labels_.data(), 0, actual_labels_.size() * sizeof(int), stream)); - RAFT_CUDA_TRY( - cudaMemsetAsync(expected_labels_.data(), 0, expected_labels_.size() * sizeof(int), stream)); - RAFT_CUDA_TRY(cudaMemsetAsync(input_.data(), 0, input_.size() * sizeof(float), stream)); - RAFT_CUDA_TRY( - cudaMemsetAsync(search_data_.data(), 0, search_data_.size() * sizeof(float), stream)); - RAFT_CUDA_TRY(cudaMemsetAsync(indices_.data(), 0, indices_.size() * sizeof(IdxT), stream)); - RAFT_CUDA_TRY(cudaMemsetAsync(distances_.data(), 0, distances_.size() * sizeof(float), stream)); - RAFT_CUDA_TRY( - cudaMemsetAsync(search_labels_.data(), 0, search_labels_.size() * sizeof(int), stream)); - - std::vector row_major_input; - for (std::size_t i = 0; i < params_.input.size(); ++i) { - for (std::size_t j = 0; j < params_.input[i].size(); ++j) { - row_major_input.push_back(params_.input[i][j]); - } - } - rmm::device_buffer input_d = - rmm::device_buffer(row_major_input.data(), row_major_input.size() * sizeof(float), stream); - float* input_ptr = static_cast(input_d.data()); - - rmm::device_buffer labels_d = - rmm::device_buffer(params_.labels.data(), params_.labels.size() * sizeof(int), stream); - int* labels_ptr = static_cast(labels_d.data()); - - raft::copy(input_.data(), input_ptr, rows_ * cols_, stream); - raft::copy(search_data_.data(), input_ptr, rows_ * cols_, stream); - raft::copy(search_labels_.data(), labels_ptr, rows_, stream); - resource::sync_stream(handle, stream); - } - - private: - raft::resources handle; - cudaStream_t stream; - - KNNInputs params_; - int rows_; - int cols_; - rmm::device_uvector input_; - rmm::device_uvector search_data_; - rmm::device_uvector indices_; - rmm::device_uvector distances_; - int k_; - - rmm::device_uvector search_labels_; - rmm::device_uvector actual_labels_; - rmm::device_uvector expected_labels_; -}; - -const std::vector inputs = { - // 2D - {{ - {2.7810836, 2.550537003}, - {1.465489372, 2.362125076}, - {3.396561688, 4.400293529}, - {1.38807019, 1.850220317}, - {3.06407232, 3.005305973}, - {7.627531214, 2.759262235}, - {5.332441248, 2.088626775}, - {6.922596716, 1.77106367}, - {8.675418651, -0.242068655}, - {7.673756466, 3.508563011}, - }, - 2, - {0, 0, 0, 0, 0, 1, 1, 1, 1, 1}}}; - -typedef KNNTest KNNTestFint32_t; -TEST_P(KNNTestFint32_t, BruteForce) { this->testBruteForce(); } -typedef KNNTest KNNTestFuint32_t; -TEST_P(KNNTestFuint32_t, BruteForce) { this->testBruteForce(); } - -INSTANTIATE_TEST_CASE_P(KNNTest, KNNTestFint32_t, ::testing::ValuesIn(inputs)); -INSTANTIATE_TEST_CASE_P(KNNTest, KNNTestFuint32_t, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::brute_force diff --git a/cpp/test/neighbors/refine.cu b/cpp/test/neighbors/refine.cu deleted file mode 100644 index 05e6048e56..0000000000 --- a/cpp/test/neighbors/refine.cu +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "ann_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include - -#include - -namespace raft::neighbors { - -template -class RefineTest : public ::testing::TestWithParam> { - public: - RefineTest() - : stream_(resource::get_cuda_stream(handle_)), - data(handle_, ::testing::TestWithParam>::GetParam()) - { - } - - protected: - public: // tamas remove - void testRefine() - { - std::vector indices(data.p.n_queries * data.p.k); - std::vector distances(data.p.n_queries * data.p.k); - - if (data.p.host_data) { - raft::neighbors::refine(handle_, - data.dataset_host.view(), - data.queries_host.view(), - data.candidates_host.view(), - data.refined_indices_host.view(), - data.refined_distances_host.view(), - data.p.metric); - raft::copy(indices.data(), - data.refined_indices_host.data_handle(), - data.refined_indices_host.size(), - stream_); - raft::copy(distances.data(), - data.refined_distances_host.data_handle(), - data.refined_distances_host.size(), - stream_); - - } else { - raft::neighbors::refine(handle_, - data.dataset.view(), - data.queries.view(), - data.candidates.view(), - data.refined_indices.view(), - data.refined_distances.view(), - data.p.metric); - update_host(distances.data(), - data.refined_distances.data_handle(), - data.refined_distances.size(), - stream_); - update_host( - indices.data(), data.refined_indices.data_handle(), data.refined_indices.size(), stream_); - } - resource::sync_stream(handle_); - - double min_recall = 1; - - ASSERT_TRUE(raft::neighbors::eval_neighbours(data.true_refined_indices_host, - indices, - data.true_refined_distances_host, - distances, - data.p.n_queries, - data.p.k, - 0.001, - min_recall)); - } - - public: - raft::resources handle_; - rmm::cuda_stream_view stream_; - RefineHelper data; -}; - -const std::vector> inputs = - raft::util::itertools::product>( - {static_cast(137)}, - {static_cast(1000)}, - {static_cast(16)}, - {static_cast(1), static_cast(10), static_cast(33)}, - {static_cast(33)}, - {raft::distance::DistanceType::L2Expanded, raft::distance::DistanceType::InnerProduct}, - {false, true}); - -typedef RefineTest RefineTestF; -TEST_P(RefineTestF, AnnRefine) { this->testRefine(); } - -INSTANTIATE_TEST_CASE_P(RefineTest, RefineTestF, ::testing::ValuesIn(inputs)); - -typedef RefineTest RefineTestF_uint8; -TEST_P(RefineTestF_uint8, AnnRefine) { this->testRefine(); } -INSTANTIATE_TEST_CASE_P(RefineTest, RefineTestF_uint8, ::testing::ValuesIn(inputs)); - -typedef RefineTest RefineTestF_int8; -TEST_P(RefineTestF_int8, AnnRefine) { this->testRefine(); } -INSTANTIATE_TEST_CASE_P(RefineTest, RefineTestF_int8, ::testing::ValuesIn(inputs)); -} // namespace raft::neighbors diff --git a/cpp/test/neighbors/tiled_knn.cu b/cpp/test/neighbors/tiled_knn.cu deleted file mode 100644 index d378100711..0000000000 --- a/cpp/test/neighbors/tiled_knn.cu +++ /dev/null @@ -1,352 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "./ann_utils.cuh" -#include "./knn_utils.cuh" - -#include -#include -#include -#include // raft::distance::pairwise_distance -#include -#include -#include -#include -#include // raft::neighbors::detail::brute_force_knn_impl - -#include - -#include - -#include -#include -#include - -namespace raft::neighbors::brute_force { - -struct TiledKNNInputs { - int num_queries; - int num_db_vecs; - int dim; - int k; - int row_tiles; - int col_tiles; - raft::distance::DistanceType metric; - bool row_major; -}; - -std::ostream& operator<<(std::ostream& os, const TiledKNNInputs& input) -{ - return os << "num_queries:" << input.num_queries << " num_vecs:" << input.num_db_vecs - << " dim:" << input.dim << " k:" << input.k << " row_tiles:" << input.row_tiles - << " col_tiles:" << input.col_tiles << " metric:" << print_metric{input.metric} - << " row_major:" << input.row_major; -} - -template -class TiledKNNTest : public ::testing::TestWithParam { - public: - TiledKNNTest() - : stream_(resource::get_cuda_stream(handle_)), - params_(::testing::TestWithParam::GetParam()), - database(params_.num_db_vecs * params_.dim, stream_), - search_queries(params_.num_queries * params_.dim, stream_), - raft_indices_(params_.num_queries * params_.k, stream_), - raft_distances_(params_.num_queries * params_.k, stream_), - ref_indices_(params_.num_queries * params_.k, stream_), - ref_distances_(params_.num_queries * params_.k, stream_) - { - raft::matrix::fill( - handle_, - raft::make_device_matrix_view(database.data(), params_.num_db_vecs, params_.dim), - T{0.0}); - raft::matrix::fill( - handle_, - raft::make_device_matrix_view(search_queries.data(), params_.num_queries, params_.dim), - T{0.0}); - raft::matrix::fill( - handle_, - raft::make_device_matrix_view(raft_indices_.data(), params_.num_queries, params_.k), - 0); - raft::matrix::fill( - handle_, - raft::make_device_matrix_view(raft_distances_.data(), params_.num_queries, params_.k), - T{0.0}); - raft::matrix::fill( - handle_, - raft::make_device_matrix_view(ref_indices_.data(), params_.num_queries, params_.k), - 0); - raft::matrix::fill( - handle_, - raft::make_device_matrix_view(ref_distances_.data(), params_.num_queries, params_.k), - T{0.0}); - } - - protected: - void testBruteForce() - { - float metric_arg = 3.0; - - // calculate the naive knn, by calculating the full pairwise distances and doing a k-select - rmm::device_uvector temp_distances(num_db_vecs * num_queries, stream_); - rmm::device_uvector workspace(0, stream_); - distance::pairwise_distance(handle_, - search_queries.data(), - database.data(), - temp_distances.data(), - num_queries, - num_db_vecs, - dim, - workspace, - metric, - params_.row_major, - metric_arg); - - // setting the 'isRowMajor' flag in the pairwise distances api, not only sets - // the inputs as colmajor - but also the output. this means we have to transpose in this - // case - auto temp_dist = temp_distances.data(); - rmm::device_uvector temp_row_major_dist(num_db_vecs * num_queries, stream_); - if (!params_.row_major) { - raft::linalg::transpose( - handle_, temp_dist, temp_row_major_dist.data(), num_queries, num_db_vecs, stream_); - temp_dist = temp_row_major_dist.data(); - } - - matrix::select_k( - handle_, - raft::make_device_matrix_view(temp_dist, num_queries, num_db_vecs), - std::nullopt, - raft::make_device_matrix_view(ref_distances_.data(), params_.num_queries, params_.k), - raft::make_device_matrix_view(ref_indices_.data(), params_.num_queries, params_.k), - raft::distance::is_min_close(metric), - true); - - if ((params_.row_tiles == 0) && (params_.col_tiles == 0)) { - std::vector input{database.data()}; - std::vector sizes{static_cast(num_db_vecs)}; - neighbors::detail::brute_force_knn_impl(handle_, - input, - sizes, - dim, - const_cast(search_queries.data()), - num_queries, - raft_indices_.data(), - raft_distances_.data(), - k_, - params_.row_major, - params_.row_major, - nullptr, - metric, - metric_arg); - } else { - neighbors::detail::tiled_brute_force_knn(handle_, - search_queries.data(), - database.data(), - num_queries, - num_db_vecs, - dim, - k_, - raft_distances_.data(), - raft_indices_.data(), - metric, - metric_arg, - params_.row_tiles, - params_.col_tiles); - } - - // verify. - ASSERT_TRUE(raft::spatial::knn::devArrMatchKnnPair(ref_indices_.data(), - raft_indices_.data(), - ref_distances_.data(), - raft_distances_.data(), - num_queries, - k_, - float(0.001), - stream_, - true)); - - // Also test out the 'index' api - where we can use precomputed norms - if (params_.row_major) { - auto idx = - raft::neighbors::brute_force::build(handle_, - raft::make_device_matrix_view( - database.data(), params_.num_db_vecs, params_.dim), - metric, - metric_arg); - - auto query_view = raft::make_device_matrix_view( - search_queries.data(), params_.num_queries, params_.dim); - - raft::neighbors::brute_force::search( - handle_, - idx, - query_view, - raft::make_device_matrix_view( - raft_indices_.data(), params_.num_queries, params_.k), - raft::make_device_matrix_view( - raft_distances_.data(), params_.num_queries, params_.k)); - - ASSERT_TRUE(raft::spatial::knn::devArrMatchKnnPair(ref_indices_.data(), - raft_indices_.data(), - ref_distances_.data(), - raft_distances_.data(), - num_queries, - k_, - float(0.001), - stream_, - true)); - // also test out the batch api. First get new reference results (all k, up to a certain - // max size) - auto all_size = std::min(params_.num_db_vecs, 1024); - auto all_indices = raft::make_device_matrix(handle_, num_queries, all_size); - auto all_distances = raft::make_device_matrix(handle_, num_queries, all_size); - raft::neighbors::brute_force::search( - handle_, idx, query_view, all_indices.view(), all_distances.view()); - - int64_t offset = 0; - auto query = make_batch_k_query(handle_, idx, query_view, k_); - for (auto batch : *query) { - auto batch_size = batch.batch_size(); - auto indices = raft::make_device_matrix(handle_, num_queries, batch_size); - auto distances = raft::make_device_matrix(handle_, num_queries, batch_size); - - matrix::slice_coordinates coords{0, offset, num_queries, offset + batch_size}; - - matrix::slice(handle_, raft::make_const_mdspan(all_indices.view()), indices.view(), coords); - matrix::slice( - handle_, raft::make_const_mdspan(all_distances.view()), distances.view(), coords); - - ASSERT_TRUE(raft::spatial::knn::devArrMatchKnnPair(indices.data_handle(), - batch.indices().data_handle(), - distances.data_handle(), - batch.distances().data_handle(), - num_queries, - batch_size, - float(0.001), - stream_, - true)); - - offset += batch_size; - if (offset + batch_size > all_size) break; - } - - // also test out with variable batch sizes - offset = 0; - int64_t batch_size = k_; - query = make_batch_k_query(handle_, idx, query_view, batch_size); - for (auto it = query->begin(); it != query->end(); it.advance(batch_size)) { - // batch_size could be less than requested (in the case of final batch). handle. - ASSERT_TRUE(it->indices().extent(1) <= batch_size); - batch_size = it->indices().extent(1); - - auto indices = raft::make_device_matrix(handle_, num_queries, batch_size); - auto distances = raft::make_device_matrix(handle_, num_queries, batch_size); - - matrix::slice_coordinates coords{0, offset, num_queries, offset + batch_size}; - matrix::slice(handle_, raft::make_const_mdspan(all_indices.view()), indices.view(), coords); - matrix::slice( - handle_, raft::make_const_mdspan(all_distances.view()), distances.view(), coords); - - ASSERT_TRUE(raft::spatial::knn::devArrMatchKnnPair(indices.data_handle(), - it->indices().data_handle(), - distances.data_handle(), - it->distances().data_handle(), - num_queries, - batch_size, - float(0.001), - stream_, - true)); - - offset += batch_size; - if (offset + batch_size > all_size) break; - - batch_size += 2; - } - } - } - - void SetUp() override - { - num_queries = params_.num_queries; - num_db_vecs = params_.num_db_vecs; - dim = params_.dim; - k_ = params_.k; - metric = params_.metric; - - unsigned long long int seed = 1234ULL; - raft::random::RngState r(seed); - - // JensenShannon distance requires positive values - T min_val = metric == raft::distance::DistanceType::JensenShannon ? T(0.0) : T(-1.0); - uniform(handle_, r, database.data(), num_db_vecs * dim, min_val, T(1.0)); - uniform(handle_, r, search_queries.data(), num_queries * dim, min_val, T(1.0)); - } - - private: - raft::resources handle_; - cudaStream_t stream_ = 0; - TiledKNNInputs params_; - int num_queries; - int num_db_vecs; - int dim; - rmm::device_uvector database; - rmm::device_uvector search_queries; - rmm::device_uvector raft_indices_; - rmm::device_uvector raft_distances_; - rmm::device_uvector ref_indices_; - rmm::device_uvector ref_distances_; - int k_; - raft::distance::DistanceType metric; -}; - -const std::vector random_inputs = { - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::L2Expanded, true}, - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::L2Unexpanded, true}, - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::L2SqrtExpanded, true}, - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::L2SqrtUnexpanded, true}, - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::L1, true}, - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::Linf, true}, - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::InnerProduct, true}, - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::CorrelationExpanded, true}, - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::CosineExpanded, true}, - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::LpUnexpanded, true}, - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::JensenShannon, true}, - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::L2SqrtExpanded, true}, - // BrayCurtis isn't currently supported by pairwise_distance api - // {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::BrayCurtis}, - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::Canberra, true}, - {10000, 40000, 32, 30, 512, 1024, raft::distance::DistanceType::L2Expanded, true}, - {345, 1023, 16, 128, 512, 1024, raft::distance::DistanceType::CosineExpanded, true}, - {789, 20516, 64, 256, 512, 4096, raft::distance::DistanceType::L2SqrtExpanded, true}, - // Test where the final column tile has < K items: - {4, 12, 32, 6, 4, 8, raft::distance::DistanceType::L2Expanded, true}, - // Test where passing column_tiles < K - {1, 40, 32, 30, 1, 8, raft::distance::DistanceType::L2Expanded, true}, - // Passing tile sizes of 0 means to use brute_force_knn_impl (instead of the - // tiled_brute_force_knn api). - {1000, 500000, 128, 128, 0, 0, raft::distance::DistanceType::L2Expanded, true}, - {1000, 500000, 128, 128, 0, 0, raft::distance::DistanceType::L2Expanded, false}, - {1000, 5000, 128, 128, 0, 0, raft::distance::DistanceType::LpUnexpanded, true}, - {1000, 5000, 128, 128, 0, 0, raft::distance::DistanceType::L2SqrtExpanded, false}, - {1000, 5000, 128, 128, 0, 0, raft::distance::DistanceType::InnerProduct, false}}; - -typedef TiledKNNTest TiledKNNTestF; -TEST_P(TiledKNNTestF, BruteForce) { this->testBruteForce(); } - -INSTANTIATE_TEST_CASE_P(TiledKNNTest, TiledKNNTestF, ::testing::ValuesIn(random_inputs)); -} // namespace raft::neighbors::brute_force diff --git a/cpp/test/sparse/gram.cu b/cpp/test/sparse/gram.cu deleted file mode 100644 index 3505a3ddf5..0000000000 --- a/cpp/test/sparse/gram.cu +++ /dev/null @@ -1,332 +0,0 @@ -/* - * Copyright (c) 2019-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#if defined RAFT_DISTANCE_COMPILED -#include -#include -#endif - -#include "../distance/gram_base.cuh" -#include "../test_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include - -namespace raft::distance::kernels { - -/** - * Structure to describe structure of the input matrices: - * - DENSE: dense, dense - * - MIX: CSR, dense - * - CSR: CSR, CSR - */ -enum SparseType { DENSE, MIX, CSR }; - -struct GramMatrixInputs { - int n1; // feature vectors in matrix 1 - int n2; // featuer vectors in matrix 2 - int n_cols; // number of elements in a feature vector - bool is_row_major; - SparseType sparse_input; - KernelParams kernel; - int ld1; - int ld2; - int ld_out; - // We will generate random input using the dimensions given here. - // The reference output is calculated by a custom kernel. -}; - -std::ostream& operator<<(std::ostream& os, const GramMatrixInputs& p) -{ - std::vector kernel_names{"linear", "poly", "rbf", "tanh"}; - os << "/" << p.n1 << "x" << p.n2 << "x" << p.n_cols << "/" - << (p.is_row_major ? "RowMajor/" : "ColMajor/") - << (p.sparse_input == SparseType::DENSE - ? "DenseDense/" - : (p.sparse_input == SparseType::MIX ? "CsrDense/" : "CsrCsr/")) - << kernel_names[p.kernel.kernel] << "/ld_" << p.ld1 << "x" << p.ld2 << "x" << p.ld_out; - return os; -} - -/*struct KernelParams { - // Kernel function parameters - KernelType kernel; //!< Type of the kernel function - int degree; //!< Degree of polynomial kernel (ignored by others) - double gamma; //!< multiplier in the - double coef0; //!< additive constant in poly and tanh kernels -};*/ - -// const KernelParams linear_kernel_params{.kernel=KernelType::LINEAR}; - -// {KernelType::POLYNOMIAL, 2, 0.5, 2.4}, {KernelType::TANH, 0, 0.5, 2.4}, {KernelType::RBF, 0, 0.5} -const std::vector inputs = raft::util::itertools::product( - {42}, - {137}, - {2}, - {true, false}, - {SparseType::DENSE, SparseType::MIX, SparseType::CSR}, - {KernelParams{KernelType::LINEAR}, - KernelParams{KernelType::POLYNOMIAL, 2, 0.5, 2.4}, - KernelParams{KernelType::TANH, 0, 0.5, 2.4}, - KernelParams{KernelType::RBF, 0, 0.5}}); - -// (ld_1, ld_2, ld_out) not supported by RBF and CSR -const std::vector inputs_ld = raft::util::itertools::product( - {137}, - {42}, - {2}, - {true, false}, - {SparseType::DENSE, SparseType::MIX}, - {KernelParams{KernelType::LINEAR}, - KernelParams{KernelType::POLYNOMIAL, 2, 0.5, 2.4}, - KernelParams{KernelType::TANH, 0, 0.5, 2.4}}, - {159}, - {73}, - {144}); - -// (ld_1, ld_2) are supported by CSR -const std::vector inputs_ld_csr = - raft::util::itertools::product( - {42}, - {137}, - {2}, - {true, false}, - {SparseType::CSR, SparseType::MIX}, - {KernelParams{KernelType::LINEAR}, - KernelParams{KernelType::POLYNOMIAL, 2, 0.5, 2.4}, - KernelParams{KernelType::TANH, 0, 0.5, 2.4}}, - {64}, - {155}, - {0}); - -template -class GramMatrixTest : public ::testing::TestWithParam { - protected: - GramMatrixTest() - : params(GetParam()), - stream(resource::get_cuda_stream(handle)), - x1(0, stream), - x2(0, stream), - x1_csr_indptr(0, stream), - x1_csr_indices(0, stream), - x1_csr_data(0, stream), - x2_csr_indptr(0, stream), - x2_csr_indices(0, stream), - x2_csr_data(0, stream), - gram(0, stream), - gram_host(0) - { - if (params.ld1 == 0) { params.ld1 = params.is_row_major ? params.n_cols : params.n1; } - if (params.ld2 == 0) { params.ld2 = params.is_row_major ? params.n_cols : params.n2; } - if (params.ld_out == 0) { params.ld_out = params.is_row_major ? params.n2 : params.n1; } - // Derive the size of the output from the offset of the last element. - size_t size = get_offset(params.n1 - 1, params.n_cols - 1, params.ld1, params.is_row_major) + 1; - x1.resize(size, stream); - size = get_offset(params.n2 - 1, params.n_cols - 1, params.ld2, params.is_row_major) + 1; - x2.resize(size, stream); - size = get_offset(params.n1 - 1, params.n2 - 1, params.ld_out, params.is_row_major) + 1; - - gram.resize(size, stream); - RAFT_CUDA_TRY(cudaMemsetAsync(gram.data(), 0, gram.size() * sizeof(math_t), stream)); - gram_host.resize(gram.size()); - std::fill(gram_host.begin(), gram_host.end(), 0); - - raft::random::RngState r(42137ULL); - raft::random::uniform(handle, r, x1.data(), x1.size(), math_t(0), math_t(1)); - raft::random::uniform(handle, r, x2.data(), x2.size(), math_t(0), math_t(1)); - - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - } - - ~GramMatrixTest() override {} - - int prepareCsr(math_t* dense, int n_rows, int ld, int* indptr, int* indices, math_t* data) - { - int nnz = 0; - double eps = 1e-6; - int n_cols = params.n_cols; - bool is_row_major = params.is_row_major; - size_t dense_size = get_offset(n_rows - 1, n_cols - 1, ld, is_row_major) + 1; - - std::vector dense_host(dense_size); - raft::update_host(dense_host.data(), dense, dense_size, stream); - resource::sync_stream(handle, stream); - - std::vector indptr_host(n_rows + 1); - std::vector indices_host(n_rows * n_cols); - std::vector data_host(n_rows * n_cols); - - // create csr matrix from dense (with threshold) - for (int i = 0; i < n_rows; ++i) { - indptr_host[i] = nnz; - for (int j = 0; j < n_cols; ++j) { - math_t value = dense_host[get_offset(i, j, ld, is_row_major)]; - if (value > eps) { - indices_host[nnz] = j; - data_host[nnz] = value; - nnz++; - } - } - } - indptr_host[n_rows] = nnz; - - // fill back dense matrix from CSR - std::fill(dense_host.data(), dense_host.data() + dense_size, 0); - for (int i = 0; i < n_rows; ++i) { - for (int idx = indptr_host[i]; idx < indptr_host[i + 1]; ++idx) { - dense_host[get_offset(i, indices_host[idx], ld, is_row_major)] = data_host[idx]; - } - } - - raft::update_device(dense, dense_host.data(), dense_size, stream); - raft::update_device(indptr, indptr_host.data(), n_rows + 1, stream); - raft::update_device(indices, indices_host.data(), nnz, stream); - raft::update_device(data, data_host.data(), nnz, stream); - resource::sync_stream(handle, stream); - return nnz; - } - - void runTest() - { - std::unique_ptr> kernel = - std::unique_ptr>(KernelFactory::create(params.kernel)); - - auto x1_span = - params.is_row_major - ? raft::make_device_strided_matrix_view( - x1.data(), params.n1, params.n_cols, params.ld1) - : raft::make_device_strided_matrix_view( - x1.data(), params.n1, params.n_cols, params.ld1); - auto x2_span = - params.is_row_major - ? raft::make_device_strided_matrix_view( - x2.data(), params.n2, params.n_cols, params.ld2) - : raft::make_device_strided_matrix_view( - x2.data(), params.n2, params.n_cols, params.ld2); - auto out_span = - params.is_row_major - ? raft::make_device_strided_matrix_view( - gram.data(), params.n1, params.n2, params.ld_out) - : raft::make_device_strided_matrix_view( - gram.data(), params.n1, params.n2, params.ld_out); - - if (params.sparse_input == SparseType::DENSE) { - (*kernel)(handle, x1_span, x2_span, out_span); - } else { - x1_csr_indptr.reserve(params.n1 + 1, stream); - x1_csr_indices.reserve(params.n1 * params.n_cols, stream); - x1_csr_data.reserve(params.n1 * params.n_cols, stream); - int x1_nnz = prepareCsr(x1.data(), - params.n1, - params.ld1, - x1_csr_indptr.data(), - x1_csr_indices.data(), - x1_csr_data.data()); - - auto x1_csr_structure = raft::make_device_compressed_structure_view( - x1_csr_indptr.data(), x1_csr_indices.data(), params.n1, params.n_cols, x1_nnz); - auto x1_csr = raft::device_csr_matrix_view( - raft::device_span(x1_csr_data.data(), x1_csr_structure.get_nnz()), - x1_csr_structure); - - if (params.sparse_input == SparseType::MIX) { - (*kernel)(handle, x1_csr, x2_span, out_span); - } else { - x2_csr_indptr.reserve(params.n2 + 1, stream); - x2_csr_indices.reserve(params.n2 * params.n_cols, stream); - x2_csr_data.reserve(params.n2 * params.n_cols, stream); - int x2_nnz = prepareCsr(x2.data(), - params.n2, - params.ld2, - x2_csr_indptr.data(), - x2_csr_indices.data(), - x2_csr_data.data()); - - auto x2_csr_structure = raft::make_device_compressed_structure_view( - x2_csr_indptr.data(), x2_csr_indices.data(), params.n2, params.n_cols, x2_nnz); - auto x2_csr = raft::device_csr_matrix_view( - raft::device_span(x2_csr_data.data(), x2_csr_structure.get_nnz()), - x2_csr_structure); - - (*kernel)(handle, x1_csr, x2_csr, out_span); - } - } - // Something in gram is executing not on the 'stream' and therefore - // a full device sync is required - RAFT_CUDA_TRY(cudaDeviceSynchronize()); - naiveGramMatrixKernel(params.n1, - params.n2, - params.n_cols, - x1, - x2, - gram_host.data(), - params.ld1, - params.ld2, - params.ld_out, - params.is_row_major, - params.kernel, - stream, - handle); - resource::sync_stream(handle, stream); - - ASSERT_TRUE(raft::devArrMatchHost( - gram_host.data(), gram.data(), gram.size(), raft::CompareApprox(1e-6f), stream)); - } - - raft::resources handle; - cudaStream_t stream = 0; - GramMatrixInputs params; - - rmm::device_uvector x1; - rmm::device_uvector x2; - - rmm::device_uvector x1_csr_indptr; - rmm::device_uvector x1_csr_indices; - rmm::device_uvector x1_csr_data; - rmm::device_uvector x2_csr_indptr; - rmm::device_uvector x2_csr_indices; - rmm::device_uvector x2_csr_data; - - rmm::device_uvector gram; - std::vector gram_host; -}; - -typedef GramMatrixTest GramMatrixTestFloatStandard; -typedef GramMatrixTest GramMatrixTestFloatLd; -typedef GramMatrixTest GramMatrixTestFloatLdCsr; - -TEST_P(GramMatrixTestFloatStandard, Gram) { runTest(); } -TEST_P(GramMatrixTestFloatLd, Gram) { runTest(); } -TEST_P(GramMatrixTestFloatLdCsr, Gram) { runTest(); } - -INSTANTIATE_TEST_SUITE_P(GramMatrixTests, GramMatrixTestFloatStandard, ::testing::ValuesIn(inputs)); -INSTANTIATE_TEST_SUITE_P(GramMatrixTests, GramMatrixTestFloatLd, ::testing::ValuesIn(inputs_ld)); -INSTANTIATE_TEST_SUITE_P(GramMatrixTests, - GramMatrixTestFloatLdCsr, - ::testing::ValuesIn(inputs_ld_csr)); -}; // end namespace raft::distance::kernels diff --git a/cpp/test/sparse/neighbors/brute_force.cu b/cpp/test/sparse/neighbors/brute_force.cu deleted file mode 100644 index ed2705a253..0000000000 --- a/cpp/test/sparse/neighbors/brute_force.cu +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Copyright (c) 2018-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../../test_utils.cuh" - -#include -#include -#include -#include - -#include -#include - -namespace raft { -namespace sparse { -namespace selection { - -using namespace raft; -using namespace raft::sparse; - -template -struct SparseKNNInputs { - value_idx n_cols; - - std::vector indptr_h; - std::vector indices_h; - std::vector data_h; - - std::vector out_dists_ref_h; - std::vector out_indices_ref_h; - - int k; - - int batch_size_index = 2; - int batch_size_query = 2; - - raft::distance::DistanceType metric = raft::distance::DistanceType::L2SqrtExpanded; -}; - -template -::std::ostream& operator<<(::std::ostream& os, const SparseKNNInputs& dims) -{ - return os; -} - -template -class SparseKNNTest : public ::testing::TestWithParam> { - public: - SparseKNNTest() - : params(::testing::TestWithParam>::GetParam()), - indptr(0, resource::get_cuda_stream(handle)), - indices(0, resource::get_cuda_stream(handle)), - data(0, resource::get_cuda_stream(handle)), - out_indices(0, resource::get_cuda_stream(handle)), - out_dists(0, resource::get_cuda_stream(handle)), - out_indices_ref(0, resource::get_cuda_stream(handle)), - out_dists_ref(0, resource::get_cuda_stream(handle)) - { - } - - protected: - void SetUp() override - { - n_rows = params.indptr_h.size() - 1; - nnz = params.indices_h.size(); - k = params.k; - - make_data(); - - raft::sparse::neighbors::brute_force_knn(indptr.data(), - indices.data(), - data.data(), - nnz, - n_rows, - params.n_cols, - indptr.data(), - indices.data(), - data.data(), - nnz, - n_rows, - params.n_cols, - out_indices.data(), - out_dists.data(), - k, - handle, - params.batch_size_index, - params.batch_size_query, - params.metric); - - RAFT_CUDA_TRY(cudaStreamSynchronize(resource::get_cuda_stream(handle))); - } - - void compare() - { - ASSERT_TRUE(devArrMatch( - out_dists_ref.data(), out_dists.data(), n_rows * k, CompareApprox(1e-4))); - ASSERT_TRUE( - devArrMatch(out_indices_ref.data(), out_indices.data(), n_rows * k, Compare())); - } - - protected: - void make_data() - { - std::vector indptr_h = params.indptr_h; - std::vector indices_h = params.indices_h; - std::vector data_h = params.data_h; - - auto stream = resource::get_cuda_stream(handle); - indptr.resize(indptr_h.size(), stream); - indices.resize(indices_h.size(), stream); - data.resize(data_h.size(), stream); - - update_device(indptr.data(), indptr_h.data(), indptr_h.size(), stream); - update_device(indices.data(), indices_h.data(), indices_h.size(), stream); - update_device(data.data(), data_h.data(), data_h.size(), stream); - - std::vector out_dists_ref_h = params.out_dists_ref_h; - std::vector out_indices_ref_h = params.out_indices_ref_h; - - out_indices_ref.resize(out_indices_ref_h.size(), stream); - out_dists_ref.resize(out_dists_ref_h.size(), stream); - - update_device( - out_indices_ref.data(), out_indices_ref_h.data(), out_indices_ref_h.size(), stream); - update_device(out_dists_ref.data(), out_dists_ref_h.data(), out_dists_ref_h.size(), stream); - - out_dists.resize(n_rows * k, stream); - out_indices.resize(n_rows * k, stream); - } - - raft::resources handle; - - int n_rows, nnz, k; - - // input data - rmm::device_uvector indptr, indices; - rmm::device_uvector data; - - // output data - rmm::device_uvector out_indices; - rmm::device_uvector out_dists; - - rmm::device_uvector out_indices_ref; - rmm::device_uvector out_dists_ref; - - SparseKNNInputs params; -}; - -const std::vector> inputs_i32_f = { - {9, // ncols - {0, 2, 4, 6, 8}, // indptr - {0, 4, 0, 3, 0, 2, 0, 8}, // indices - {0.0f, 1.0f, 5.0f, 6.0f, 5.0f, 6.0f, 0.0f, 1.0f}, // data - {0, 1.41421, 0, 7.87401, 0, 7.87401, 0, 1.41421}, // dists - {0, 3, 1, 0, 2, 0, 3, 0}, // inds - 2, - 2, - 2, - raft::distance::DistanceType::L2SqrtExpanded}}; -typedef SparseKNNTest SparseKNNTestF; -TEST_P(SparseKNNTestF, Result) { compare(); } -INSTANTIATE_TEST_CASE_P(SparseKNNTest, SparseKNNTestF, ::testing::ValuesIn(inputs_i32_f)); - -}; // end namespace selection -}; // end namespace sparse -}; // end namespace raft diff --git a/cpp/test/sparse/neighbors/cross_component_nn.cu b/cpp/test/sparse/neighbors/cross_component_nn.cu deleted file mode 100644 index 13f7ae9de4..0000000000 --- a/cpp/test/sparse/neighbors/cross_component_nn.cu +++ /dev/null @@ -1,1036 +0,0 @@ -/* - * Copyright (c) 2018-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// XXX: We allow the instantiation of masked_l2_nn here: -// raft::linkage::FixConnectivitiesRedOp red_op(params.n_row); -// raft::linkage::cross_component_nn( -// handle, out_edges, data.data(), colors.data(), params.n_row, params.n_col, red_op); -// -// TODO: consider adding this to libraft.so or creating an instance in a -// separate translation unit for this test. -// -// TODO: edge case testing. Reference: https://github.com/rapidsai/raft/issues/1669 - -#include "../../test_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include - -#include - -namespace raft { -namespace sparse { - -using namespace std; - -template -struct ConnectComponentsInputs { - value_idx n_row; - value_idx n_col; - std::vector data; - - int c; -}; - -template -class ConnectComponentsTest - : public ::testing::TestWithParam> { - protected: - void basicTest() - { - raft::resources handle; - - auto stream = resource::get_cuda_stream(handle); - - params = ::testing::TestWithParam>::GetParam(); - - raft::sparse::COO out_edges(resource::get_cuda_stream(handle)); - raft::sparse::COO out_edges_batched(resource::get_cuda_stream(handle)); - - rmm::device_uvector data(params.n_row * params.n_col, - resource::get_cuda_stream(handle)); - - raft::copy(data.data(), params.data.data(), data.size(), resource::get_cuda_stream(handle)); - - rmm::device_uvector indptr(params.n_row + 1, stream); - - /** - * 1. Construct knn graph - */ - raft::sparse::COO knn_graph_coo(stream); - - raft::sparse::neighbors::knn_graph(handle, - data.data(), - params.n_row, - params.n_col, - raft::distance::DistanceType::L2SqrtExpanded, - knn_graph_coo, - params.c); - - raft::sparse::convert::sorted_coo_to_csr( - knn_graph_coo.rows(), knn_graph_coo.nnz, indptr.data(), params.n_row + 1, stream); - - /** - * 2. Construct MST, sorted by weights - */ - rmm::device_uvector colors(params.n_row, stream); - - auto mst_coo = raft::mst::mst(handle, - indptr.data(), - knn_graph_coo.cols(), - knn_graph_coo.vals(), - params.n_row, - knn_graph_coo.nnz, - colors.data(), - stream, - false, - true); - - /** - * 3. cross_component_nn to fix connectivities - */ - raft::linkage::FixConnectivitiesRedOp red_op(params.n_row); - raft::linkage::cross_component_nn(handle, - out_edges, - data.data(), - colors.data(), - params.n_row, - params.n_col, - red_op, - params.n_row, - params.n_col); - - raft::linkage::cross_component_nn(handle, - out_edges_batched, - data.data(), - colors.data(), - params.n_row, - params.n_col, - red_op, - params.n_row / 2, - params.n_col / 2); - - ASSERT_TRUE(out_edges.nnz == out_edges_batched.nnz); - - ASSERT_TRUE( - devArrMatch(out_edges.rows(), out_edges_batched.rows(), out_edges.nnz, Compare())); - - ASSERT_TRUE( - devArrMatch(out_edges.cols(), out_edges_batched.cols(), out_edges.nnz, Compare())); - - ASSERT_TRUE(devArrMatch( - out_edges.vals(), out_edges_batched.vals(), out_edges.nnz, CompareApprox(1e-4))); - - /** - * Construct final edge list - */ - rmm::device_uvector indptr2(params.n_row + 1, stream); - - raft::sparse::convert::sorted_coo_to_csr( - out_edges.rows(), out_edges.nnz, indptr2.data(), params.n_row + 1, stream); - - auto output_mst = raft::mst::mst(handle, - indptr2.data(), - out_edges.cols(), - out_edges.vals(), - params.n_row, - out_edges.nnz, - colors.data(), - stream, - false, - false); - - resource::sync_stream(handle, stream); - - // The sum of edges for both MST runs should be n_rows - 1 - final_edges = output_mst.n_edges + mst_coo.n_edges; - } - - void SetUp() override { basicTest(); } - - void TearDown() override {} - - protected: - ConnectComponentsInputs params; - - value_idx final_edges; -}; - -const std::vector> fix_conn_inputsf2 = { - // Test n_clusters == n_points - {10, - 5, - {0.21390334, 0.50261639, 0.91036676, 0.59166485, 0.71162682, 0.10248392, 0.77782677, 0.43772379, - 0.4035871, 0.3282796, 0.47544681, 0.59862974, 0.12319357, 0.06239463, 0.28200272, 0.1345717, - 0.50498218, 0.5113505, 0.16233086, 0.62165332, 0.42281548, 0.933117, 0.41386077, 0.23264562, - 0.73325968, 0.37537541, 0.70719873, 0.14522645, 0.73279625, 0.9126674, 0.84854131, 0.28890216, - 0.85267903, 0.74703138, 0.83842071, 0.34942792, 0.27864171, 0.70911132, 0.21338564, 0.32035554, - 0.73788331, 0.46926692, 0.57570162, 0.42559178, 0.87120209, 0.22734951, 0.01847905, 0.75549396, - 0.76166195, 0.66613745}, - -1}, - // Test n_points == 100 - {100, - 10, - {6.26168372e-01, 9.30437651e-01, 6.02450208e-01, 2.73025296e-01, 9.53050619e-01, 3.32164396e-01, - 6.88942598e-01, 5.79163537e-01, 6.70341547e-01, 2.70140602e-02, 9.30429671e-01, 7.17721157e-01, - 9.89948537e-01, 7.75253347e-01, 1.34491522e-02, 2.48522428e-02, 3.51413378e-01, 7.64405834e-01, - 7.86373507e-01, 7.18748577e-01, 8.66998621e-01, 6.80316582e-01, 2.51288712e-01, 4.91078420e-01, - 3.76246281e-01, 4.86828710e-01, 5.67464772e-01, 5.30734742e-01, 8.99478296e-01, 7.66699088e-01, - 9.49339111e-01, 3.55248484e-01, 9.06046929e-01, 4.48407772e-01, 6.96395305e-01, 2.44277335e-01, - 7.74840000e-01, 5.21046603e-01, 4.66423971e-02, 5.12019638e-02, 8.95019614e-01, 5.28956953e-01, - 4.31536306e-01, 5.83857744e-01, 4.41787364e-01, 4.68656523e-01, 5.73971433e-01, 6.79989654e-01, - 3.19650588e-01, 6.12579596e-01, 6.49126442e-02, 8.39131142e-01, 2.85252117e-01, 5.84848929e-01, - 9.46507115e-01, 8.58440748e-01, 3.61528940e-01, 2.44215959e-01, 3.80101125e-01, 4.57128957e-02, - 8.82216988e-01, 8.31498633e-01, 7.23474381e-01, 7.75788607e-01, 1.40864146e-01, 6.62092382e-01, - 5.13985168e-01, 3.00686418e-01, 8.70109949e-01, 2.43187753e-01, 2.89391938e-01, 2.84214238e-01, - 8.70985521e-01, 8.77491176e-01, 6.72537226e-01, 3.30929686e-01, 1.85934324e-01, 9.16222614e-01, - 6.18239142e-01, 2.64768597e-01, 5.76145451e-01, 8.62961369e-01, 6.84757925e-01, 7.60549082e-01, - 1.27645356e-01, 4.51004673e-01, 3.92292980e-01, 4.63170803e-01, 4.35449330e-02, 2.17583404e-01, - 5.71832605e-02, 2.06763039e-01, 3.70116249e-01, 2.09750028e-01, 6.17283019e-01, 8.62549231e-01, - 9.84156240e-02, 2.66249156e-01, 3.87635103e-01, 2.85591012e-02, 4.24826068e-01, 4.45795088e-01, - 6.86227676e-01, 1.08848960e-01, 5.96731841e-02, 3.71770228e-01, 1.91548833e-01, 6.95136078e-01, - 9.00700636e-01, 8.76363105e-01, 2.67334632e-01, 1.80619709e-01, 7.94060419e-01, 1.42854171e-02, - 1.09372387e-01, 8.74028108e-01, 6.46403232e-01, 4.86588834e-01, 5.93446175e-02, 6.11886291e-01, - 8.83865057e-01, 3.15879821e-01, 2.27043992e-01, 9.76764951e-01, 6.15620336e-01, 9.76199360e-01, - 2.40548962e-01, 3.21795663e-01, 8.75087904e-02, 8.11234663e-01, 6.96070480e-01, 8.12062321e-01, - 1.21958818e-01, 3.44348628e-02, 8.72630414e-01, 3.06162776e-01, 1.76043529e-02, 9.45894971e-01, - 5.33896401e-01, 6.21642973e-01, 4.93062535e-01, 4.48984262e-01, 2.24560379e-01, 4.24052195e-02, - 4.43447610e-01, 8.95646149e-01, 6.05220676e-01, 1.81840491e-01, 9.70831206e-01, 2.12563586e-02, - 6.92582693e-01, 7.55946922e-01, 7.95086143e-01, 6.05328941e-01, 3.99350764e-01, 4.32846636e-01, - 9.81114529e-01, 4.98266428e-01, 6.37127930e-03, 1.59085889e-01, 6.34682067e-05, 5.59429440e-01, - 7.38827633e-01, 8.93214770e-01, 2.16494306e-01, 9.35430573e-02, 4.75665868e-02, 7.80503518e-01, - 7.86240041e-01, 7.06854594e-01, 2.13725879e-02, 7.68246091e-01, 4.50234808e-01, 5.21231104e-01, - 5.01989826e-03, 4.22081572e-02, 1.65337732e-01, 8.54134740e-01, 4.99430262e-01, 8.94525601e-01, - 1.14028379e-01, 3.69739861e-01, 1.32955599e-01, 2.65563824e-01, 2.52811151e-01, 1.44792843e-01, - 6.88449594e-01, 4.44921417e-01, 8.23296587e-01, 1.93266317e-01, 1.19033309e-01, 1.36368966e-01, - 3.42600285e-01, 5.64505195e-01, 5.57594559e-01, 7.44257892e-01, 8.38231569e-02, 4.11548847e-01, - 3.21010077e-01, 8.55081359e-01, 4.30105779e-01, 1.16229135e-01, 9.87731964e-02, 3.14712335e-01, - 4.50880592e-01, 2.72289598e-01, 6.31615256e-01, 8.97432958e-01, 4.44764250e-01, 8.03776440e-01, - 2.68767748e-02, 2.43374608e-01, 4.02141103e-01, 4.98881209e-01, 5.33173003e-01, 8.82890436e-01, - 7.16149148e-01, 4.19664401e-01, 2.29335357e-01, 2.88637806e-01, 3.44696803e-01, 6.78171906e-01, - 5.69849716e-01, 5.86454477e-01, 3.54474989e-01, 9.03876540e-01, 6.45980000e-01, 6.34887593e-01, - 7.88039746e-02, 2.04814126e-01, 7.82251754e-01, 2.43147074e-01, 7.50951808e-01, 1.72799092e-02, - 2.95349590e-01, 6.57991826e-01, 8.81214312e-01, 5.73970708e-01, 2.77610881e-01, 1.82155097e-01, - 7.69797417e-02, 6.44792402e-01, 9.46950998e-01, 7.73064845e-01, 6.04733624e-01, 5.80094567e-01, - 1.67498426e-01, 2.66514296e-01, 6.50140368e-01, 1.91170299e-01, 2.08752199e-01, 3.01664091e-01, - 9.85033484e-01, 2.92909152e-01, 8.65816607e-01, 1.85222119e-01, 2.28814559e-01, 1.34286382e-02, - 2.89234322e-01, 8.18668708e-01, 4.71706924e-01, 9.23199803e-01, 2.80879188e-01, 1.47319284e-01, - 4.13915748e-01, 9.31274932e-02, 6.66322195e-01, 9.66953974e-01, 3.19405786e-01, 6.69486551e-01, - 5.03096313e-02, 6.95225201e-01, 5.78469859e-01, 6.29481655e-01, 1.39252534e-01, 1.22564968e-01, - 6.80663678e-01, 6.34607157e-01, 6.42765834e-01, 1.57127410e-02, 2.92132086e-01, 5.24423878e-01, - 4.68676824e-01, 2.86003928e-01, 7.18608322e-01, 8.95617933e-01, 5.48844309e-01, 1.74517278e-01, - 5.24379196e-01, 2.13526524e-01, 5.88375435e-01, 9.88560185e-01, 4.17435771e-01, 6.14438688e-01, - 9.53760881e-01, 5.27151288e-01, 7.03017278e-01, 3.44448559e-01, 4.47059676e-01, 2.83414901e-01, - 1.98979011e-01, 4.24917361e-01, 5.73172761e-01, 2.32398853e-02, 1.65887230e-01, 4.05552785e-01, - 9.29665524e-01, 2.26135696e-01, 9.20563384e-01, 7.65259963e-01, 4.54820075e-01, 8.97710267e-01, - 3.78559302e-03, 9.15219382e-01, 3.55705698e-01, 6.94905124e-01, 8.58540202e-01, 3.89790666e-01, - 2.49478206e-01, 7.93679304e-01, 4.75830027e-01, 4.40425353e-01, 3.70579459e-01, 1.40578049e-01, - 1.70386675e-01, 7.04056121e-01, 4.85963102e-01, 9.68450060e-01, 6.77178001e-01, 2.65934654e-01, - 2.58915007e-01, 6.70052890e-01, 2.61945109e-01, 8.46207759e-01, 1.01928951e-01, 2.85611334e-01, - 2.45776933e-01, 2.66658783e-01, 3.71724077e-01, 4.34319025e-01, 4.24407347e-01, 7.15417683e-01, - 8.07997684e-01, 1.64296275e-01, 6.01638065e-01, 8.60606804e-02, 2.68719187e-01, 5.11764101e-01, - 9.75844338e-01, 7.81226782e-01, 2.20925515e-01, 7.18135040e-01, 9.82395577e-01, 8.39160243e-01, - 9.08058083e-01, 6.88010677e-01, 8.14271847e-01, 5.12460821e-01, 1.17311345e-01, 5.96075228e-01, - 9.17455497e-01, 2.12052706e-01, 7.04074603e-01, 8.72872565e-02, 8.76047818e-01, 6.96235046e-01, - 8.54801557e-01, 2.49729159e-01, 9.76594604e-01, 2.87386363e-01, 2.36461559e-02, 9.94075254e-01, - 4.25193986e-01, 7.61869994e-01, 5.13334255e-01, 6.44711165e-02, 8.92156689e-01, 3.55235167e-01, - 1.08154647e-01, 8.78446825e-01, 2.43833016e-01, 9.23071293e-01, 2.72724115e-01, 9.46631338e-01, - 3.74510294e-01, 4.08451278e-02, 9.78392777e-01, 3.65079221e-01, 6.37199516e-01, 5.51144906e-01, - 5.25978080e-01, 1.42803678e-01, 4.05451674e-01, 7.79788219e-01, 6.26009784e-01, 3.35249497e-01, - 1.43159543e-02, 1.80363779e-01, 5.05096904e-01, 2.82619947e-01, 5.83561392e-01, 3.10951324e-01, - 8.73223968e-01, 4.38545619e-01, 4.81348800e-01, 6.68497085e-01, 3.79345401e-01, 9.58832501e-01, - 1.89869550e-01, 2.34083070e-01, 2.94066207e-01, 5.74892667e-02, 6.92106828e-02, 9.61127686e-02, - 6.72650672e-02, 8.47345378e-01, 2.80916761e-01, 7.32177357e-03, 9.80785961e-01, 5.73192225e-02, - 8.48781331e-01, 8.83225408e-01, 7.34398275e-01, 7.70381941e-01, 6.20778343e-01, 8.96822048e-01, - 5.40732486e-01, 3.69704071e-01, 5.77305837e-01, 2.08221827e-01, 7.34275341e-01, 1.06110900e-01, - 3.49496706e-01, 8.34948910e-01, 1.56403291e-02, 6.78576376e-01, 8.96141268e-01, 5.94835119e-01, - 1.43943153e-01, 3.49618530e-01, 2.10440392e-01, 3.46585620e-01, 1.05153093e-01, 3.45446174e-01, - 2.72177079e-01, 7.07946300e-01, 4.33717726e-02, 3.31232203e-01, 3.91874320e-01, 4.76338141e-01, - 6.22777789e-01, 2.95989228e-02, 4.32855769e-01, 7.61049310e-01, 3.63279149e-01, 9.47210350e-01, - 6.43721247e-01, 6.58025802e-01, 1.05247633e-02, 5.29974442e-01, 7.30675767e-01, 4.30041079e-01, - 6.62634841e-01, 8.25936616e-01, 9.91253704e-01, 6.79399281e-01, 5.44177006e-01, 7.52876048e-01, - 3.32139049e-01, 7.98732398e-01, 7.38865223e-01, 9.16055132e-01, 6.11736493e-01, 9.63672879e-01, - 1.83778839e-01, 7.27558919e-02, 5.91602822e-01, 3.25235484e-01, 2.34741217e-01, 9.52346277e-01, - 9.18556407e-01, 9.35373324e-01, 6.89209070e-01, 2.56049054e-01, 6.17975395e-01, 7.82285691e-01, - 9.84983432e-01, 6.62322741e-01, 2.04144457e-01, 3.98446577e-01, 1.38918297e-01, 3.05919921e-01, - 3.14043787e-01, 5.91072666e-01, 7.44703771e-01, 8.92272567e-01, 9.78017873e-01, 9.01203161e-01, - 1.41526372e-01, 4.14878484e-01, 6.80683651e-01, 5.01733152e-02, 8.14635389e-01, 2.27926375e-01, - 9.03269815e-01, 8.68443745e-01, 9.86939190e-01, 7.40779486e-01, 2.61005311e-01, 3.19276232e-01, - 9.69509248e-01, 1.11908818e-01, 4.49198556e-01, 1.27056715e-01, 3.84064823e-01, 5.14591811e-01, - 2.10747488e-01, 9.53884090e-01, 8.43167950e-01, 4.51187972e-01, 3.75331782e-01, 6.23566461e-01, - 3.55290379e-01, 2.95705968e-01, 1.69622690e-01, 1.42981830e-01, 2.72180991e-01, 9.46468040e-01, - 3.70932500e-01, 9.94292830e-01, 4.62587505e-01, 7.14817405e-01, 2.45370540e-02, 3.00906377e-01, - 5.75768304e-01, 9.71448393e-01, 6.95574827e-02, 3.93693854e-01, 5.29306116e-01, 5.04694554e-01, - 6.73797120e-02, 6.76596969e-01, 5.50948898e-01, 3.24909641e-01, 7.70337719e-01, 6.51842631e-03, - 3.03264879e-01, 7.61037886e-03, 2.72289601e-01, 1.50502041e-01, 6.71103888e-02, 7.41503703e-01, - 1.92088941e-01, 2.19043977e-01, 9.09320161e-01, 2.37993569e-01, 6.18107973e-02, 8.31447852e-01, - 2.23355609e-01, 1.84789435e-01, 4.16104518e-01, 4.21573859e-01, 8.72446305e-02, 2.97294197e-01, - 4.50328256e-01, 8.72199917e-01, 2.51279916e-01, 4.86219272e-01, 7.57071329e-01, 4.85655942e-01, - 1.06187277e-01, 4.92341327e-01, 1.46017513e-01, 5.25421017e-01, 4.22637906e-01, 2.24685018e-01, - 8.72648431e-01, 5.54051490e-01, 1.80745062e-01, 2.12756336e-01, 5.20883169e-01, 7.60363654e-01, - 8.30254678e-01, 5.00003328e-01, 4.69017439e-01, 6.38105527e-01, 3.50638261e-02, 5.22217353e-02, - 9.06516882e-02, 8.52975842e-01, 1.19985883e-01, 3.74926753e-01, 6.50302066e-01, 1.98875727e-01, - 6.28362507e-02, 4.32693501e-01, 3.10500685e-01, 6.20732833e-01, 4.58503272e-01, 3.20790034e-01, - 7.91284868e-01, 7.93054570e-01, 2.93406765e-01, 8.95399023e-01, 1.06441034e-01, 7.53085241e-02, - 8.67523104e-01, 1.47963482e-01, 1.25584706e-01, 3.81545040e-02, 6.34338619e-01, 1.76368938e-02, - 5.75553531e-02, 5.31607516e-01, 2.63869588e-01, 9.41945823e-01, 9.24028838e-02, 5.21496463e-01, - 7.74866558e-01, 5.65210610e-01, 7.28015327e-02, 6.51963790e-01, 8.94727453e-01, 4.49571590e-01, - 1.29932405e-01, 8.64026259e-01, 9.92599934e-01, 7.43721560e-01, 8.87300215e-01, 1.06369925e-01, - 8.11335531e-01, 7.87734900e-01, 9.87344678e-01, 5.32502820e-01, 4.42612382e-01, 9.64041183e-01, - 1.66085871e-01, 1.12937664e-01, 5.24423470e-01, 6.54689333e-01, 4.59119726e-01, 5.22774091e-01, - 3.08722276e-02, 6.26979315e-01, 4.49754105e-01, 8.07495757e-01, 2.34199499e-01, 1.67765675e-01, - 9.22168418e-01, 3.73210378e-01, 8.04432575e-01, 5.61890354e-01, 4.47025593e-01, 6.43155678e-01, - 2.40407640e-01, 5.91631279e-01, 1.59369206e-01, 7.75799090e-01, 8.32067212e-01, 5.59791576e-02, - 6.39105224e-01, 4.85274738e-01, 2.12630838e-01, 2.81431312e-02, 7.16205363e-01, 6.83885011e-01, - 5.23869697e-01, 9.99418314e-01, 8.35331599e-01, 4.69877463e-02, 6.74712562e-01, 7.99273684e-01, - 2.77001890e-02, 5.75809742e-01, 2.78513031e-01, 8.36209905e-01, 7.25472379e-01, 4.87173943e-01, - 7.88311357e-01, 9.64676177e-01, 1.75752651e-01, 4.98112580e-01, 8.08850418e-02, 6.40981131e-01, - 4.06647450e-01, 8.46539387e-01, 2.12620694e-01, 9.11012851e-01, 8.25041445e-01, 8.90065575e-01, - 9.63626055e-01, 5.96689242e-01, 1.63372670e-01, 4.51640148e-01, 3.43026542e-01, 5.80658851e-01, - 2.82327625e-01, 4.75535418e-01, 6.27760926e-01, 8.46314115e-01, 9.61961932e-01, 3.19806094e-01, - 5.05508062e-01, 5.28102944e-01, 6.13045057e-01, 7.44714938e-01, 1.50586073e-01, 7.91878033e-01, - 4.89839179e-01, 3.10496849e-01, 8.82309038e-01, 2.86922314e-01, 4.84687559e-01, 5.20838630e-01, - 4.62955493e-01, 2.38185305e-01, 5.47259907e-02, 7.10916137e-01, 7.31887202e-01, 6.25602317e-01, - 8.77741168e-01, 4.19881322e-01, 4.81222328e-01, 1.28224501e-01, 2.46034010e-01, 3.34971854e-01, - 7.37216484e-01, 5.62134821e-02, 7.14089724e-01, 9.85549393e-01, 4.66295827e-01, 3.08722434e-03, - 4.70237690e-01, 2.66524167e-01, 7.93875484e-01, 4.54795911e-02, 8.09702944e-01, 1.47709735e-02, - 1.70082405e-01, 6.35905179e-01, 3.75379109e-01, 4.30315011e-01, 3.15788760e-01, 5.58065230e-01, - 2.24643800e-01, 2.42142981e-01, 6.57283636e-01, 3.34921891e-01, 1.26588975e-01, 7.68064155e-01, - 9.43856291e-01, 4.47518596e-01, 5.44453573e-01, 9.95764932e-01, 7.16444391e-01, 8.51019765e-01, - 1.01179183e-01, 4.45473958e-01, 4.60327322e-01, 4.96895844e-02, 4.72907738e-01, 5.58987444e-01, - 3.41027487e-01, 1.56175026e-01, 7.58283148e-01, 6.83600909e-01, 2.14623396e-01, 3.27348880e-01, - 3.92517893e-01, 6.70418431e-01, 5.16440832e-01, 8.63140348e-01, 5.73277464e-01, 3.46608058e-01, - 7.39396341e-01, 7.20852434e-01, 2.35653246e-02, 3.89935659e-01, 7.53783745e-01, 6.34563528e-01, - 8.79339335e-01, 7.41599159e-02, 5.62433904e-01, 6.15553852e-01, 4.56956324e-01, 5.20047447e-01, - 5.26845015e-02, 5.58471266e-01, 1.63632233e-01, 5.38936665e-02, 6.49593683e-01, 2.56838748e-01, - 8.99035326e-01, 7.20847756e-01, 5.68954684e-01, 7.43684755e-01, 5.70924238e-01, 3.82318724e-01, - 4.89328290e-01, 5.62208561e-01, 4.97540804e-02, 4.18011085e-01, 6.88041565e-01, 2.16234653e-01, - 7.89548214e-01, 8.46136387e-01, 8.46816189e-01, 1.73842353e-01, 6.11627842e-02, 8.44440559e-01, - 4.50646654e-01, 3.74785037e-01, 4.87196697e-01, 4.56276448e-01, 9.13284391e-01, 4.15715464e-01, - 7.13597697e-01, 1.23641270e-02, 5.10031271e-01, 4.74601930e-02, 2.55731159e-01, 3.22090006e-01, - 1.91165703e-01, 4.51170940e-01, 7.50843157e-01, 4.42420576e-01, 4.25380660e-01, 4.50667257e-01, - 6.55689206e-01, 9.68257670e-02, 1.96528793e-01, 8.97343028e-01, 4.99940904e-01, 6.65504083e-01, - 9.41828079e-01, 4.54397338e-01, 5.61893331e-01, 5.09839880e-01, 4.53117514e-01, 8.96804127e-02, - 1.74888861e-01, 6.65641378e-01, 2.81668336e-01, 1.89532742e-01, 5.61668382e-01, 8.68330157e-02, - 8.25092797e-01, 5.18106324e-01, 1.71904024e-01, 3.68385523e-01, 1.62005436e-01, 7.48507399e-01, - 9.30274827e-01, 2.38198517e-01, 9.52222901e-01, 5.23587800e-01, 6.94384557e-01, 1.09338652e-01, - 4.83356794e-01, 2.73050402e-01, 3.68027050e-01, 5.92366466e-01, 1.83192289e-01, 8.60376029e-01, - 7.13926203e-01, 8.16750052e-01, 1.57890291e-01, 6.25691951e-01, 5.24831646e-01, 1.73873797e-01, - 1.02429784e-01, 9.17488471e-01, 4.03584434e-01, 9.31170884e-01, 2.79386137e-01, 8.77745206e-01, - 2.45200576e-01, 1.28896951e-01, 3.15713052e-01, 5.27874291e-01, 2.16444335e-01, 7.03883817e-01, - 7.74738919e-02, 8.42422142e-01, 3.75598924e-01, 3.51002411e-01, 6.22752776e-01, 4.82407943e-01, - 7.43107867e-01, 9.46182666e-01, 9.44344819e-01, 3.28124763e-01, 1.06147431e-01, 1.65102684e-01, - 3.84060507e-01, 2.91057722e-01, 7.68173662e-02, 1.03543651e-01, 6.76698940e-01, 1.43141994e-01, - 7.21342202e-01, 6.69471294e-03, 9.07298311e-01, 5.57080171e-01, 8.10954489e-01, 4.11120526e-01, - 2.06407453e-01, 2.59590556e-01, 7.58512718e-01, 5.79873897e-01, 2.92875650e-01, 2.83686529e-01, - 2.42829343e-01, 9.19323719e-01, 3.46832864e-01, 3.58238858e-01, 7.42827585e-01, 2.05760059e-01, - 9.58438860e-01, 5.66326411e-01, 6.60292846e-01, 5.61095078e-02, 6.79465531e-01, 7.05118513e-01, - 4.44713264e-01, 2.09732933e-01, 5.22732436e-01, 1.74396512e-01, 5.29356748e-01, 4.38475687e-01, - 4.94036404e-01, 4.09785794e-01, 6.40025507e-01, 5.79371821e-01, 1.57726118e-01, 6.04572263e-01, - 5.41072639e-01, 5.18847173e-01, 1.97093284e-01, 8.91767002e-01, 4.29050835e-01, 8.25490570e-01, - 3.87699807e-01, 4.50705808e-01, 2.49371643e-01, 3.36074898e-01, 9.29925118e-01, 6.65393649e-01, - 9.07275994e-01, 3.73075859e-01, 4.14044139e-03, 2.37463702e-01, 2.25893784e-01, 2.46900245e-01, - 4.50350196e-01, 3.48618117e-01, 5.07193932e-01, 5.23435142e-01, 8.13611417e-01, 8.92715622e-01, - 1.02623450e-01, 3.06088345e-01, 7.80461650e-01, 2.21453645e-01, 2.01419652e-01, 2.84254457e-01, - 3.68286735e-01, 7.39358243e-01, 8.97879394e-01, 9.81599566e-01, 7.56526442e-01, 7.37645545e-01, - 4.23976657e-02, 8.25922012e-01, 2.60956996e-01, 2.90702065e-01, 8.98388344e-01, 3.03733299e-01, - 8.49071471e-01, 3.45835425e-01, 7.65458276e-01, 5.68094872e-01, 8.93770930e-01, 9.93161641e-01, - 5.63368667e-02, 4.26548945e-01, 5.46745780e-01, 5.75674571e-01, 7.94599487e-01, 7.18935553e-02, - 4.46492976e-01, 6.40240123e-01, 2.73246969e-01, 2.00465968e-01, 1.30718835e-01, 1.92492005e-01, - 1.96617189e-01, 6.61271644e-01, 8.12687657e-01, 8.66342445e-01 - - }, - -4}}; - -typedef ConnectComponentsTest ConnectComponentsTestF_Int; -TEST_P(ConnectComponentsTestF_Int, Result) -{ - /** - * Verify the src & dst vertices on each edge have different colors - */ - EXPECT_TRUE(final_edges == params.n_row - 1); -} - -INSTANTIATE_TEST_CASE_P(ConnectComponentsTest, - ConnectComponentsTestF_Int, - ::testing::ValuesIn(fix_conn_inputsf2)); - -template -struct MutualReachabilityFixConnectivitiesRedOp { - value_t* core_dists; - value_idx m; - - DI MutualReachabilityFixConnectivitiesRedOp() : m(0) {} - - MutualReachabilityFixConnectivitiesRedOp(value_t* core_dists_, value_idx m_) - : core_dists(core_dists_), m(m_){}; - - typedef typename raft::KeyValuePair KVP; - DI void operator()(value_idx rit, KVP* out, const KVP& other) const - { - if (rit < m && other.value < std::numeric_limits::max()) { - value_t core_dist_rit = core_dists[rit]; - value_t core_dist_other = max(core_dist_rit, max(core_dists[other.key], other.value)); - - value_t core_dist_out; - if (out->key > -1) { - core_dist_out = max(core_dist_rit, max(core_dists[out->key], out->value)); - } else { - core_dist_out = out->value; - } - - bool smaller = core_dist_other < core_dist_out; - out->key = smaller ? other.key : out->key; - out->value = smaller ? core_dist_other : core_dist_out; - } - } - - DI KVP operator()(value_idx rit, const KVP& a, const KVP& b) const - { - if (rit < m && a.key > -1) { - value_t core_dist_rit = core_dists[rit]; - value_t core_dist_a = max(core_dist_rit, max(core_dists[a.key], a.value)); - - value_t core_dist_b; - if (b.key > -1) { - core_dist_b = max(core_dist_rit, max(core_dists[b.key], b.value)); - } else { - core_dist_b = b.value; - } - - return core_dist_a < core_dist_b ? KVP(a.key, core_dist_a) : KVP(b.key, core_dist_b); - } - - return b; - } - - DI void init(value_t* out, value_t maxVal) const { *out = maxVal; } - DI void init(KVP* out, value_t maxVal) const - { - out->key = -1; - out->value = maxVal; - } - - DI void init_key(value_t& out, value_idx idx) const { return; } - DI void init_key(KVP& out, value_idx idx) const { out.key = idx; } - - DI value_t get_value(KVP& out) const { return out.value; } - DI value_t get_value(value_t& out) const { return out; } - - void gather(const raft::resources& handle, value_idx* map) - { - auto tmp_core_dists = raft::make_device_vector(handle, m); - thrust::gather(raft::resource::get_thrust_policy(handle), - map, - map + m, - core_dists, - tmp_core_dists.data_handle()); - raft::copy_async( - core_dists, tmp_core_dists.data_handle(), m, raft::resource::get_cuda_stream(handle)); - } - - void scatter(const raft::resources& handle, value_idx* map) - { - auto tmp_core_dists = raft::make_device_vector(handle, m); - thrust::scatter(raft::resource::get_thrust_policy(handle), - core_dists, - core_dists + m, - map, - tmp_core_dists.data_handle()); - raft::copy_async( - core_dists, tmp_core_dists.data_handle(), m, raft::resource::get_cuda_stream(handle)); - } -}; - -template -struct ConnectComponentsMutualReachabilityInputs { - value_idx n_row; - value_idx n_col; - std::vector data; - std::vector core_dists; - std::vector colors; - std::vector expected_rows; - std::vector expected_cols; - std::vector expected_vals; -}; - -template -class ConnectComponentsEdgesTest - : public ::testing::TestWithParam> { - protected: - void basicTest() - { - raft::resources handle; - - auto stream = resource::get_cuda_stream(handle); - - params = ::testing::TestWithParam< - ConnectComponentsMutualReachabilityInputs>::GetParam(); - - raft::sparse::COO out_edges_unbatched(resource::get_cuda_stream(handle)); - raft::sparse::COO out_edges_batched(resource::get_cuda_stream(handle)); - - rmm::device_uvector data(params.n_row * params.n_col, - resource::get_cuda_stream(handle)); - rmm::device_uvector core_dists(params.n_row, resource::get_cuda_stream(handle)); - rmm::device_uvector colors(params.n_row, resource::get_cuda_stream(handle)); - - raft::copy(data.data(), params.data.data(), data.size(), resource::get_cuda_stream(handle)); - raft::copy(core_dists.data(), - params.core_dists.data(), - core_dists.size(), - resource::get_cuda_stream(handle)); - raft::copy( - colors.data(), params.colors.data(), colors.size(), resource::get_cuda_stream(handle)); - - /** - * 3. cross_component_nn to fix connectivities - */ - MutualReachabilityFixConnectivitiesRedOp red_op(core_dists.data(), - params.n_row); - - raft::linkage::cross_component_nn(handle, - out_edges_unbatched, - data.data(), - colors.data(), - params.n_row, - params.n_col, - red_op, - params.n_row, - params.n_col); - - raft::linkage::cross_component_nn(handle, - out_edges_batched, - data.data(), - colors.data(), - params.n_row, - params.n_col, - red_op, - 11, - 1); - - ASSERT_TRUE(out_edges_unbatched.nnz == out_edges_batched.nnz && - out_edges_unbatched.nnz == params.expected_rows.size()); - - ASSERT_TRUE(devArrMatch(out_edges_unbatched.rows(), - params.expected_rows.data(), - out_edges_unbatched.nnz, - Compare())); - - ASSERT_TRUE(devArrMatch(out_edges_unbatched.cols(), - params.expected_cols.data(), - out_edges_unbatched.nnz, - Compare())); - - ASSERT_TRUE(devArrMatch(out_edges_unbatched.vals(), - params.expected_vals.data(), - out_edges_unbatched.nnz, - CompareApprox(1e-4))); - - ASSERT_TRUE(devArrMatch(out_edges_batched.rows(), - params.expected_rows.data(), - out_edges_batched.nnz, - Compare())); - - ASSERT_TRUE(devArrMatch(out_edges_batched.cols(), - params.expected_cols.data(), - out_edges_batched.nnz, - Compare())); - - ASSERT_TRUE(devArrMatch(out_edges_batched.vals(), - params.expected_vals.data(), - out_edges_batched.nnz, - CompareApprox(1e-4))); - } - - void SetUp() override { basicTest(); } - - void TearDown() override {} - - protected: - ConnectComponentsMutualReachabilityInputs params; -}; - -const std::vector> mr_fix_conn_inputsf2 = { - {100, - 2, - {-7.72642, -8.39496, 5.4534, 0.742305, -2.97867, 9.55685, 6.04267, 0.571319, -6.52184, - -6.31932, 3.64934, 1.40687, -2.17793, 9.98983, 4.42021, 2.33028, 4.73696, 2.94181, - -3.66019, 9.38998, -3.05358, 9.12521, -6.65217, -5.57297, -6.35769, -6.58313, -3.61553, - 7.81808, -1.77073, 9.18565, -7.95052, -6.39764, -6.60294, -6.05293, -2.58121, 10.0178, - -7.76348, -6.72638, -6.40639, -6.95294, -2.97262, 8.54856, -6.95673, -6.53896, -7.32614, - -6.02371, -2.1478, 10.5523, -2.54502, 10.5789, -2.96984, 10.0714, 3.22451, 1.55252, - -6.25396, -7.73727, -7.85431, -6.09303, -8.11658, -8.20057, -7.55965, -6.64786, 4.936, - 2.23423, 4.44752, 2.27472, -5.72103, -7.70079, -0.929985, 9.78172, -3.10984, 8.72259, - -2.44167, 7.58954, -2.18511, 8.6292, 5.55528, 2.30192, 4.73164, -0.0143992, -8.2573, - -7.81793, -2.98837, 8.82863, 4.60517, 0.804492, -3.83738, 9.21115, -2.62485, 8.71318, - 3.57758, 2.44676, -8.48711, -6.69548, -6.70645, -6.49479, -6.86663, -5.42658, 3.83139, - 1.47141, 2.02013, 2.79507, 4.64499, 1.73858, -1.69667, 10.3705, -6.61974, -6.09829, - -6.05757, -4.98332, -7.10309, -6.16611, -3.52203, 9.32853, -2.26724, 7.10101, 6.11777, - 1.4549, -4.23412, 8.452, -6.58655, -7.59446, 3.93783, 1.64551, -7.12502, -7.63385, - 2.72111, 1.94666, -7.14428, -4.15994, -6.66553, -8.12585, 4.70011, 4.43641, -7.76914, - -7.69592, 4.11012, 2.48644, 4.89743, 1.89872, 4.29716, 1.17089, -6.62913, -6.53366, - -8.07093, -6.22356, -2.16558, 7.25125, 4.73953, 1.46969, -5.91625, -6.46733, 5.43091, - 1.06378, -6.82142, -8.02308, 6.52606, 2.14775, 3.08922, 2.04173, -2.14756, 8.36917, - 3.85663, 1.65111, -1.68665, 7.79344, -5.01385, -6.40628, -2.52269, 7.95658, -2.30033, - 7.05462, -1.04355, 8.78851, 3.72045, 3.5231, -3.98772, 8.29444, 4.24777, 0.509655, - 4.72693, 1.67416, 5.7827, 2.7251, -3.41722, 7.60198, 5.22674, 4.16363, -3.1109, - 10.8666, -3.18612, 9.62596, -1.4782, 9.94557, 4.47859, 2.37722, -5.79658, -5.82631, - -3.34842, 8.70507}, - {0.978428, 1.01917, 0.608673, 1.45629, 0.310713, 0.689461, 0.701126, 0.63296, 0.774788, - 0.701648, 0.513282, 0.757651, 0.45638, 0.973111, 0.901396, 0.613692, 0.482497, 0.688143, - 0.72428, 0.666345, 0.58232, 0.554756, 0.710315, 0.903611, 0.694115, 0.796099, 0.639759, - 0.798998, 0.639839, 1.30727, 0.663729, 0.57476, 0.571348, 1.14662, 1.26518, 0.485068, - 0.78207, 0.791621, 1.01678, 1.28509, 1.14715, 0.381395, 0.850507, 0.788511, 0.588341, - 0.878516, 0.928669, 0.405874, 0.776421, 0.612274, 1.84963, 0.57476, 0.95226, 0.488078, - 1.24868, 0.515136, 0.589378, 0.903632, 1.01678, 1.09964, 0.666345, 0.713265, 0.877168, - 1.10053, 1.96887, 1.03574, 2.03728, 0.969553, 0.774788, 0.586338, 0.65168, 0.435472, - 0.664396, 0.790584, 0.678637, 0.715964, 0.865494, 0.978428, 1.59242, 0.861109, 0.833259, - 0.65168, 0.903632, 1.49599, 0.76347, 0.960453, 1.1848, 1.37398, 0.928957, 1.07848, - 0.661798, 1.21104, 1.04579, 1.89047, 1.24288, 0.529553, 0.903611, 0.620897, 0.882467, - 0.647189}, - {0, 1, 2, 1, 0, 1, 2, 1, 1, 2, 2, 0, 0, 2, 2, 0, 0, 2, 0, 0, 2, 0, 0, 2, 2, - 2, 1, 0, 0, 0, 0, 1, 1, 0, 2, 2, 2, 2, 1, 1, 0, 2, 1, 2, 2, 1, 0, 0, 0, 1, - 1, 1, 2, 0, 0, 0, 2, 2, 1, 2, 0, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 0, 2, 1, - 0, 1, 0, 1, 1, 2, 1, 2, 0, 2, 2, 2, 1, 2, 1, 1, 1, 2, 1, 2, 2, 2, 1, 0, 2}, - {50, 54, 57, 63, 82, 87}, - {57, 63, 50, 54, 87, 82}, - {6.0764, 11.1843, 6.0764, 11.1843, 6.89004, 6.89004}}, - {1000, - 2, - {-6.59634, -7.13901, -6.13753, -6.58082, 5.19821, 2.04918, -2.96856, 8.16444, - -2.76879, 7.51114, -6.82261, -6.61152, 5.02008, 2.58376, 5.55621, 2.31966, - 4.86379, 3.33731, 5.84639, 1.15623, -2.17159, 8.60241, -4.97844, -6.94077, - -2.31014, 8.41407, 5.5582, 0.402669, 5.25265, 0.919754, 5.85298, 2.11489, - -3.29245, 8.69222, -1.9621, 8.81209, -1.53408, 8.86723, -2.18227, 8.79519, - 4.60519, 2.20738, -6.4759, -6.9043, -7.18766, -6.10045, -9.00148, -7.48793, - 4.01674, 1.41769, -2.45347, 10.1085, -3.20892, 9.22827, -3.18612, 9.62596, - 4.81977, 3.36517, 4.90693, 2.8628, -6.44269, -5.68946, -8.30144, -5.37878, - 4.61485, 2.79094, -1.98726, 9.31127, -3.66019, 9.38998, -6.58607, -8.23669, - -7.46015, -6.29153, 4.08468, 3.85433, -6.36842, -5.50645, -6.83602, -5.18506, - -0.627173, 10.3597, 3.98846, 1.48928, -2.9968, 8.58173, -7.2144, -7.28376, - -0.660242, 10.1409, -4.23528, -8.38308, -3.15984, 8.52716, -2.40987, 9.76567, - -8.7548, -6.76508, 4.56971, 0.312209, -7.5487, -5.8402, -1.6096, 9.32159, - 5.04813, 0.270586, -7.6525, -6.47306, -1.79758, 7.88964, -9.0153, -3.74236, - -3.5715, 9.48788, -1.65154, 8.85435, -3.47412, 9.70034, 6.31245, 2.39219, - 4.03851, 2.29295, -3.17098, 9.86672, -6.90693, -7.81338, -6.22373, -6.68537, - -3.22204, 9.12072, -0.365254, 9.6482, -7.76712, -7.31757, 4.15669, 3.54716, - 4.1937, 0.083629, -3.03896, 9.52755, -6.29293, -7.35501, -2.95926, 9.63714, - 4.02709, 1.58547, 4.56828, 1.93595, 5.6242, 1.75918, -7.36237, -7.83344, - 5.32177, 3.81988, -2.43183, 8.153, -1.97939, 10.4559, -3.49492, 9.51833, - 3.39602, 1.28026, -2.42215, 8.71528, -3.57682, 8.87191, -2.77385, 11.7345, - 5.71351, 0.946654, -6.50253, -6.90937, 4.08239, 0.603367, -5.64134, -6.85884, - -2.76177, 7.7665, -2.25165, 8.93984, -3.49071, 9.47639, -1.06792, 7.57842, - 5.15754, 1.24743, 3.63574, 1.20537, -6.07969, -8.49642, 4.12227, 2.19696, - -7.17144, -8.4433, -1.92234, 11.2047, 3.23237, 1.19535, 3.85389, 0.641937, - 4.82665, 1.21779, -7.68923, -6.45605, -7.00816, -8.76196, -5.12894, 9.83619, - -5.66247, -5.35879, 3.05598, 2.73358, 6.06038, 1.40242, -1.69568, 7.78342, - 5.13391, 2.23384, -2.96984, 10.0714, -5.36618, -6.2493, 5.55896, 1.6829, - 3.55882, 2.58911, 5.36155, 0.844118, -0.0634456, 9.14351, 4.88368, 1.40909, - -7.04675, -6.59753, -7.78333, -6.55575, 5.39881, 2.25436, -2.85189, 8.64285, - -2.22821, 8.39159, 3.88591, 1.69249, -7.55481, -7.02463, 4.60032, 2.65467, - -6.90615, -7.76198, -6.76005, -7.85318, 4.15044, 3.01733, -7.18884, -7.63227, - 4.68874, 2.01376, 3.51716, 2.35558, -3.81367, 9.68396, 4.42644, 3.4639, - 4.81758, 0.637825, -6.20705, -4.98023, -1.68603, 9.0876, -4.99504, -5.33687, - -1.77073, 9.18565, 4.86433, 3.02027, 4.20538, 1.664, 4.59042, 2.64799, - -3.09856, 9.86389, -3.02306, 7.95507, -6.32402, -6.79053, -7.67205, -7.18807, - -8.10918, -6.38341, -1.67979, 6.80315, 4.00249, 3.16219, -2.54391, 7.84561, - -3.22764, 8.80084, -2.63712, 8.05875, -2.41744, 7.02672, -6.71117, -5.56251, - 5.18348, 1.60256, -7.40824, -6.29375, -4.22233, 10.3682, 4.8509, 1.87646, - -2.99456, 9.09616, 5.1332, 2.15801, -2.27358, 9.78515, -6.73874, -8.64855, - 4.96124, 2.39509, -3.70949, 8.67978, -4.13674, 9.06237, 2.80367, 2.48116, - -0.876786, 7.58414, -3.7005, 9.67084, 6.48652, 0.903085, 6.28189, 2.98299, - -6.07922, -6.12582, -5.67921, -7.537, 4.55014, 3.41329, -1.63688, 9.19763, - -4.02439, 10.3812, 5.23053, 3.08187, -2.2951, 7.76855, -6.24491, -5.77041, - 6.02415, 2.53708, -6.91286, -7.08823, 4.83193, 1.66405, -7.07454, -5.74634, - -2.09576, 10.8911, 3.29543, 1.05452, -3.49973, 8.44799, 5.2922, 0.396778, - -2.54502, 10.5789, -6.38865, -6.14523, -1.75221, 8.09212, -9.30387, -5.99606, - -2.98113, 10.1032, -6.2017, -7.36802, 4.63628, 0.814805, -1.81905, 8.61307, - 4.88926, 3.55062, 3.08325, 2.57918, -2.51717, 10.4942, -5.75358, -6.9315, - 6.36742, 2.40949, 5.74806, 0.933264, 4.74408, 1.91058, -7.41496, -6.97064, - -2.98414, 8.36096, 6.72825, 1.83358, -2.95349, 9.39159, -3.35599, 7.49944, - 6.18738, 3.76905, -3.17182, 9.58488, 5.17863, 1.0525, -3.0397, 8.43847, - -2.23874, 8.96405, 3.04689, 2.41364, 6.14064, 2.82339, -6.33334, -6.87369, - -7.92444, -8.84647, 3.65129, 0.86958, 5.29842, 3.98337, -2.06538, 9.78892, - -6.89494, -6.30082, -2.52144, 8.11703, -8.11398, -7.47257, 5.3381, 2.36666, - -6.93452, -6.59456, -7.50634, -6.01772, 6.23438, 1.12621, -2.15218, 8.32138, - -7.04777, -7.3522, -2.52771, 8.72563, -2.77907, 8.03552, 4.29123, 1.62391, - -8.07551, -6.43551, -3.28202, 8.77747, -2.21308, 9.27534, -8.25153, -8.49367, - -3.54644, 8.82395, -8.05867, -5.69243, 4.46681, 1.98875, 3.8362, 3.61229, - -6.96231, -7.00186, 5.18993, 1.00483, -5.35116, -6.37227, 5.23298, 1.66362, - -5.68306, -7.03864, -9.03144, -7.59926, -6.10127, -7.4313, 4.83572, 0.994797, - -7.32695, -5.59909, 0.569683, 10.1339, 3.35957, 2.84563, -2.4122, 9.60944, - 5.00855, 1.57983, -2.57528, 7.80327, 3.96349, 3.77411, 4.59429, 2.21651, - -6.54765, -6.68961, 4.76798, 1.29212, -1.67351, 7.88458, 5.63615, 1.47941, - -2.5301, 9.13161, 4.26075, 1.76959, 4.67788, 2.0932, 4.39955, 1.59835, - 3.91274, 1.72565, -4.1786, 9.55765, -7.34566, -8.47481, 4.8364, 2.68217, - -7.36848, -7.99973, -5.84708, -5.7534, 5.37252, 1.89245, -2.1707, 8.599, - -1.3299, 9.0818, -6.79122, -5.40258, 5.56391, 1.78827, -0.194539, 7.14702, - 4.60489, 3.74397, 5.50995, 2.46885, -3.98772, 8.29444, -5.21837, -7.33721, - -1.63959, 10.3699, -5.92932, -5.1695, -5.88358, -7.6369, 4.11716, 3.02218, - -6.54114, -7.17551, 3.97179, 2.96521, -6.75325, -4.94118, 5.26169, 0.402945, - 3.25031, 0.327771, -0.44845, 10.7696, -2.15141, 9.57507, 7.04329, 1.91555, - -3.74615, 7.69383, -7.52318, -5.85015, -6.80419, -8.48208, -4.57664, 8.92517, - 4.57574, 2.30193, 4.84098, 3.02382, -9.43355, -5.94579, -3.52203, 9.32853, - 3.43018, 2.5731, -6.15725, -7.25294, -6.69861, -8.17694, -2.40955, 8.51081, - -4.82342, -7.98332, -7.10611, -6.51274, 5.86755, 0.763529, -6.56045, -5.53966, - -3.61553, 7.81808, 4.3825, 0.304586, -6.52818, -5.80996, 4.59972, 0.542395, - -6.90603, -6.59995, -6.3585, -6.23489, -6.01915, -7.46319, -5.38694, -7.15123, - -7.83475, -6.45651, 5.89564, 1.07856, -5.15266, -7.27975, -6.97978, -7.08378, - 5.83493, 0.449983, -2.62374, 10.2521, -7.34494, -6.98606, -6.79719, -8.33766, - 3.54757, 1.65676, -8.40528, -5.61753, -5.85556, -6.28758, 4.66862, 3.25162, - -6.26047, -4.82261, 4.61552, 4.11544, -1.36637, 9.76622, 4.2517, 2.14359, - -2.45099, 7.87132, -0.376164, 7.0622, 4.34493, 3.22091, 6.95921, 2.36649, - -6.70319, -7.24714, -5.56932, -5.48443, -7.43149, -4.32191, -3.23956, 9.23074, - -5.77255, -7.00049, 4.96601, 0.722056, -7.88617, -5.74023, 4.18757, -0.45071, - -7.12569, -7.72336, 5.27366, 2.38697, 3.93487, 1.9174, 3.19186, -0.225636, - -3.41722, 7.60198, -3.08286, 8.46743, -5.87905, -7.55073, -5.26425, -7.20243, - -2.97867, 9.55685, -1.23153, 8.42272, -2.33602, 9.3996, -3.33819, 8.45411, - -3.58009, 9.49676, 3.78152, 2.67348, -1.54582, 9.42707, -4.04331, 10.292, - 3.3452, 3.134, -2.75494, 8.74156, -3.26555, 7.59203, -7.27139, -7.80252, - 3.5293, 3.72544, 6.11642, 3.35326, 4.01611, 3.8872, 4.89591, 2.95586, - -7.06677, -5.89438, 4.19438, 3.42655, -6.11355, -5.65318, -7.59645, -8.74665, - -5.80362, -6.8588, 3.80453, 4.11832, 5.70655, 3.14247, -4.98084, 8.21739, - -1.87642, 11.285, 4.39864, 2.32523, -3.48388, 9.80137, 4.02836, 0.566509, - -2.41212, 9.98293, -5.40846, -7.08943, 4.01506, 1.99926, -3.43613, 8.95476, - -7.24458, -7.71932, 6.02204, 2.62188, -6.29999, -6.55431, 6.19038, 0.974816, - 3.55882, 3.02632, -7.06011, -3.687, -1.55877, 8.43738, -5.14711, -4.64881, - 4.7167, 0.690177, -7.90381, -5.02602, 4.17218, 2.31967, -0.643423, 9.48812, - -7.95237, -6.64086, -4.05986, 9.08285, -6.24158, -6.37927, -6.6105, -7.2233, - -6.21675, -5.70664, -3.29967, 9.48575, 3.41775, 2.68617, -2.24948, 8.10997, - -2.24931, 9.79611, -9.0523, -6.03269, -2.2587, 9.36073, 5.20965, 2.42088, - -3.10159, 8.1503, -6.67906, -5.73147, 4.0687, 2.54575, -1.24229, 8.30662, - -2.09627, 8.45056, -7.87801, -6.57832, 4.72216, 3.03865, -0.929985, 9.78172, - -8.56307, -7.68598, -7.05257, -5.1684, -7.09076, -7.86729, 4.61432, 3.1459, - -6.34133, -5.8076, -3.82943, 10.8457, -8.46082, -5.98507, 5.34763, 1.4107, - -1.68714, 10.9111, -1.67886, 8.1582, -0.623012, 9.18886, -4.21258, 8.95874, - -2.16744, 10.8905, -6.57158, -7.27176, 2.14047, 4.26411, -8.44217, -7.40916, - 5.29008, 1.87399, 4.31824, 4.04992, -3.77008, 9.93215, -2.72688, 10.1131, - -6.14278, -7.16144, -3.92457, 8.59364, -5.92649, -6.59299, 4.68369, 1.82617, - -6.89905, -7.18329, 3.95173, 4.22561, -7.66453, -6.23183, -2.44167, 7.58954, - -6.36603, -7.41281, -6.45081, -6.187, -6.6125, -6.37138, 5.46036, 2.48044, - -2.14756, 8.36917, -2.3889, 9.52872, 3.80752, 2.44459, -3.98778, 10.158, - -6.63887, -4.27843, -8.65266, -5.61819, -7.97003, -5.46918, -5.9604, -7.54825, - -0.916011, 8.50307, -3.69246, 6.97505, -7.98533, -7.09503, -2.30033, 7.05462, - 4.76218, 2.51647, -7.04981, -7.33334, 3.66401, 3.02681, -2.50408, 8.7797, - 7.19996, 1.87711, 4.01291, 3.78562, -0.356015, 8.24694, -0.958046, 9.12996, - 4.60675, 3.76773, 6.21945, 1.45031, 4.27744, 0.8535, -4.72232, -7.48582, - 6.03923, 2.8978, -3.26833, 9.16468, -7.97059, -7.29092, -2.3998, 9.74005, - -2.66721, 8.58741, -7.36269, -6.73332, -7.87893, -7.38488, 4.65023, 0.661333, - -4.8171, -7.94764, -4.11564, 9.21775, 4.80633, 2.46562, -2.72887, 9.3714, - -5.26735, -5.5652, 4.9826, 2.42992, -6.17018, -7.3156, 4.38084, 1.77682, - 5.35084, 2.41743, -2.61796, 9.416, 5.27229, 2.94572, -7.52315, -5.95227, - -1.45077, 7.25555, -3.79916, 7.71921, -2.23251, 9.84147, 3.70054, 1.82908, - -1.93831, 10.1499, -6.18324, -5.9248, -3.33142, 9.25797, -6.08536, -8.1344, - 5.95727, 2.17077, 4.87366, 0.417274, -6.529, -6.39092, -9.24256, -7.88984, - -6.36652, -7.13966, -3.90777, 9.57726, -7.06252, -5.50523, -2.26423, 8.50734, - -2.84498, 10.6833, 5.0391, 2.62037, -2.74815, 8.10672, 3.35945, 3.72796, - -4.11668, 9.19892, 5.66903, 2.44577, -1.63807, 8.68826, -7.42587, -6.48831, - 6.17063, 3.19193, -2.28511, 9.02688, -7.10088, -7.15692, 4.46293, 1.17487, - -5.91017, -6.45292, -2.26724, 7.10101, -2.43339, 8.33712, -4.63309, 8.48853, - -3.31769, 8.51253, -2.49078, 10.6907, -1.30798, 8.60621, 6.30535, 2.98754, - -5.79384, -6.78213, -1.93213, 8.81124, 4.55773, 3.09047, 6.37584, 2.17108, - 4.3927, 1.29119, -3.2245, 9.69388, -1.69634, 9.64392, 2.799, 0.693593, - -2.1426, 8.07441, -8.4505, -8.00688, 4.736, 1.51089, -2.5863, 9.35544, - -2.94924, 9.14503, 6.2054, 1.90742, 5.67172, 0.487609, -5.69071, -6.17181, - -8.24651, -7.10488, -7.34424, -6.67895, -6.71977, -7.90778, -1.82294, 7.40157, - -9.40991, -7.16611, -4.37999, 8.66277, -1.42615, 10.0681, -2.00828, 8.03673, - -7.50228, -6.6855, -5.65859, -6.29801, -8.02335, -6.77155, -3.40761, 9.50621, - -2.82447, 9.77326, -1.5938, 9.34304, -3.5213, 7.35943, -3.36961, 8.62973, - -7.01708, -5.92724, 5.20886, 3.60157, -1.71817, 8.1049, -2.46363, 8.36269, - -2.77809, 7.90776, -2.75459, 8.26055, -2.03596, 8.94146, -4.53434, 9.20074, - -7.44387, -6.69556, -6.90099, -7.62732, 3.29169, 2.71643, 6.08686, 2.16972, - -2.31111, 8.86993, -5.75046, 7.9899, 4.69951, 1.32623, 4.71851, -0.025031, - -6.42374, -4.71511, -8.04974, -8.68209, -3.16103, 9.06168, -6.18267, -7.21393, - -7.94202, -6.4518, -7.07697, -7.03138, 3.93554, 0.564708, -1.20372, 9.03529, - -7.10611, -7.83955, -7.47529, -5.50567, -6.15453, -6.36393, -2.98024, 9.24634, - -7.75761, -7.70699, -3.08597, 9.76968, -8.04954, -9.75237, 5.2534, 0.950377, - 5.63789, -0.923086, -5.7065, -6.51047, -8.02132, -7.07377, -8.28594, -6.96322, - -7.70722, -6.79397, -2.4962, 10.4678, 5.02846, 4.46617, 4.02648, 1.6707, - -0.319395, 8.20599, 4.74525, 0.639144, -1.0313, 8.49602, 4.08766, 2.6061, - 3.63826, 1.69207, 2.55795, 3.66963, 5.2826, 3.30232, -1.04355, 8.78851, - -6.84762, -7.63353, -4.70868, -7.056, 3.53651, -0.179721, -3.38482, 7.63149, - -5.9265, -6.36702, -0.986074, 9.5532, -2.42261, 8.85861, -7.42835, -6.78726, - -4.02857, 8.53005, -8.22675, -7.85172, -5.57529, -8.5426, 6.03009, 2.53098, - -7.10448, -7.53011, -3.4988, 8.8885, -2.62485, 8.71318, -6.39489, -7.72647, - 3.93789, 1.31027, 4.27627, 1.91622, -0.923181, 7.77647, -5.16017, 10.1058, - -6.44307, -5.97617, -7.24495, -6.69543, 6.27331, 0.826824, -6.55655, -7.13246, - 5.66245, 4.41292, -2.13805, 8.4103, 5.23463, 2.82659, -4.86624, -6.74357, - -6.14082, -6.26474, -2.67048, 9.41834, -1.26311, 6.9409, -7.20231, -7.13094, - -1.35109, 9.80595, 3.9906, 0.749229, -6.75696, -5.25543, 4.84826, -0.0685652, - -7.4914, -6.91715, 4.46725, 2.85683, -2.95571, 9.87068, 6.32381, 1.51429, - -6.81177, -6.02734, -2.57188, 9.96943, -4.28792, 10.5103, 3.65025, 2.91394, - -7.11856, -7.24693, -6.98693, -6.43239, 4.7651, 1.54376, 4.00092, 0.65008, - -7.14816, -7.7713, -7.58803, -8.39382, 4.3321, 2.19232, -7.89545, -6.81843, - -2.11475, 8.5933, -0.743743, 9.41927, 3.64849, -0.18022, -1.68665, 7.79344, - 4.00214, 1.44217, -6.96799, -7.25012, -1.58302, 10.9237, -6.68524, -7.23328, - 4.65831, 2.32075, 4.62024, 2.52566, -4.23412, 8.452, -0.822056, 9.89593, - -7.19868, -7.67614, -3.32742, 11.1067, 5.27861, 0.830165, 4.48982, 2.09875, - -6.58087, -7.6319, -0.880582, 7.63418, -7.01088, -6.80326, -7.31601, -6.98972, - -6.85883, -7.60811, 6.14328, 2.85053, -7.49206, -6.51861, -2.28174, 10.3214, - 4.81074, 1.78919, -5.58987, -6.20693, 4.08096, 2.35038, -1.5029, 8.43739, - 4.11536, 2.46254, -3.28299, 7.76963, 4.31953, 2.39734, 4.91146, 0.696421, - -1.4782, 9.94557, -3.34842, 8.70507, -6.97822, -6.86126, 4.10012, 1.19486, - -2.50395, 9.06127, 4.41891, 2.00006, -2.73266, 9.72829, 3.5436, 0.533119, - 5.78864, 0.233456, -6.62589, -6.41242, -2.21942, 11.0897, -6.76636, -8.31839, - -2.71732, 8.52129, -5.20972, -6.48544, 3.26056, 1.24224, 3.45228, 2.28299, - 4.72171, 1.87428, -7.52585, -5.1048, 5.0695, 2.18086, -6.55646, -7.02771, - 3.23727, 3.72275, 3.41411, 0.508795, -7.80698, -6.64174, -5.90443, -6.37902, - -0.387041, 10.0468, -1.3506, 8.1936, -6.08614, -8.62864, -5.91478, -5.26453, - -2.61623, 7.97904, 4.45459, 1.84335, -6.66643, -7.63208, 3.6729, 1.92546, - -1.32976, 8.54511, 6.31758, 1.41958, 4.63381, 2.81166, -7.01394, -6.0693, - -2.7786, 9.73183, -2.90131, 7.55077, -7.13842, -5.28146, 6.71514, 1.28398, - -6.98408, -7.04893, -3.03946, 8.22141, -2.76417, 10.5183, -7.35347, -6.89456, - 4.19345, 2.16726, -2.02819, 9.23817, 4.97076, 2.8067, -0.544473, 9.04955, - 4.90727, 2.29487, -6.31871, -7.17559, 3.71665, 0.621485, 4.7903, 2.33813, - -6.47994, -7.53147, -6.80958, -5.71823, -8.07326, -5.96096, 4.77342, 1.8207, - 5.71856, 1.93466, -2.70156, 9.31583, -2.1478, 10.5523, 4.78855, 1.63608, - 5.53507, 2.60834, -7.00058, -6.46058, 5.4738, 2.43235, -1.34603, 9.02452, - -7.5337, -8.71074, -7.30893, -7.57253, -5.33752, -4.87402, -7.01364, -6.86542, - -7.93331, -7.94791, -5.69392, -6.16116, -7.32291, -7.76491, -6.41965, -7.55783, - -7.87996, -7.55785, -6.69005, -5.87906, 3.92147, 2.86809, -1.5552, 9.66568, - 5.07989, 1.47112, -7.48524, -5.0541, -1.82724, 8.70402, -2.00421, 9.88004, - -2.62153, 8.79332, -7.52111, -6.44819, 4.06424, 2.09518, -6.65494, -5.94752, - 6.93878, 1.61033, -3.95728, 7.60682, 5.67016, 2.21196, -7.81507, -5.79413, - -2.41152, 8.24128, -3.83738, 9.21115, 4.5516, 4.55288, -5.75551, -5.93258, - 4.56545, 2.59384, -7.45614, -9.47115, -2.39568, 9.67642, 5.57816, 1.45712, - -7.48184, -6.41134, -1.99415, 12.867, -8.35854, -6.69675, -7.52559, -7.6793, - 5.7454, 3.1602, 2.94692, 1.87483, -8.77324, -6.66682, -3.21125, 8.68662, - -6.25806, -7.24972, 5.17639, 1.0747, -2.44897, 11.4775, -3.30172, 8.89955, - -2.85191, 8.21201, -8.85893, -6.1322, 4.08957, 1.30155, -5.88132, -7.31173, - -7.10309, -7.22943, -2.46068, 8.18334, -7.01226, -7.85464, 4.75411, 2.12347, - -3.42862, 10.5642, 7.16681, 1.4423, 5.42568, 2.39863, -6.00833, -8.22609, - -1.7619, 9.62466, -2.49527, 8.99016, -2.98837, 8.82863, -2.97262, 8.54856, - -1.34142, 9.26871, -5.99652, -6.95795, -1.87061, 7.35277, -8.68277, -8.46425, - -7.01808, -8.10441, -7.04269, -7.62501, -7.69783, -6.88348, -2.19829, 10.4896, - 4.67396, 1.2032, -5.58263, -6.90298, -5.69224, -4.29055, 4.77285, 1.27305, - -3.33469, 8.6929, -2.54195, 8.47086, 4.46492, 1.21742, 5.41158, -0.875373, - -8.68069, -7.42278, -3.88687, 8.07646, 4.6682, 2.00293, -8.29799, -8.64092, - -1.86382, 10.3829, -6.51234, -5.04193, 4.54458, 2.25219, -1.93264, 9.32554, - -3.06285, 7.81641, -6.90714, -5.10786, 4.69653, 2.50286, 6.43757, 2.61401, - -1.85483, 8.9587, 4.60224, 3.07647, 4.4492, 2.1906, 5.02181, 2.40321, - -2.22923, 7.8888, 5.68943, 1.43793, -6.71097, -6.43817, -5.00633, -5.80006, - -2.43763, 8.53663, 5.72577, 2.44787, -6.57079, -5.17789, -5.77867, -4.92176, - -6.57222, -6.06437, 3.96639, 2.25216, -7.95177, -9.80146, 4.92574, 2.30763, - -7.6221, -8.20013, -6.4132, -6.91575, 4.01432, 2.36897, 3.0833, 1.54505, - -1.99416, 9.52807, -7.85128, -8.25973, -0.86423, 8.76525, -6.31412, -8.64087, - -8.07355, -6.73717, -2.52821, 8.01176, -5.82357, -6.65687, -7.08865, -7.73063, - -5.56251, -6.99818, -2.12513, 8.98159, -6.89834, -7.26863, -7.92654, -6.34346, - 4.86201, 1.49442, 4.92905, 4.42847, -5.57789, -5.3186, 4.34232, 3.34888, - 2.64614, 2.34723, -4.10363, 8.41491, -2.18648, 8.18706, -3.39871, 8.19848, - -2.66098, 9.6026, -6.95927, -6.42774, -5.61392, -7.74628, 5.60376, 4.18369, - 5.28536, 4.13642, 4.8428, 0.457426, -6.33816, -6.12095, -2.4394, 8.62897, - 4.56938, 2.45967, 4.0582, 0.958413, 5.62164, 1.64834, 5.73119, 2.58231, - 4.66806, 1.96405, -6.71905, -6.87706, -2.18503, 8.88414, -6.03901, -6.33338, - -8.38435, -6.12005, 0.0641622, 9.0735, 5.19967, 3.05395, -5.48716, -7.13016, - -6.85541, -5.46789, -1.88353, 8.15713, 4.27891, 3.1325, -2.75816, 9.98586, - -2.03022, 9.34795, -7.66741, -7.50096, -3.39305, 9.16801, -8.49476, -5.71537, - -1.68378, 9.8278, -7.41559, -6.07205, -3.15577, 7.93274, 5.22381, 1.61388, - 3.65739, 1.74854, 4.94251, 1.21889, -7.12832, -5.27276, -9.58286, -6.20223, - -2.21613, 8.29993, 5.34799, 2.92987, 4.09496, 2.37231, -7.25183, -5.79136, - -6.46981, -7.12137, -6.28607, -9.8205, 4.52865, 1.06926, -3.10984, 8.72259, - 3.61865, 2.68153, -5.96604, -7.68329, 3.11435, 1.28126, -1.1064, 7.61243, - -2.17688, 8.2658, -3.27246, 7.2094, -5.55143, -6.32388, -1.69667, 10.3705, - -2.16558, 7.25125, -6.36572, -6.70053, 4.12259, 3.38252, -4.80554, -7.79949, - -5.23966, -6.13798, 4.21969, 1.69139, -1.98985, 10.547, -2.52269, 7.95658, - -6.75642, -6.32862, -3.51521, 7.8001, 4.70435, -0.00229688, 6.25359, 2.4267, - 5.82935, 0.745562, 5.24778, 2.15978, 5.48052, 1.32055, -3.05358, 9.12521, - -3.18922, 9.24654, 4.47276, 2.11988, 5.36751, 2.02512, -2.18511, 8.6292, - -2.48469, 9.51228, 5.57556, 3.24472, -2.58121, 10.0178, -6.12629, -6.49895, - -4.54732, 8.0062, -4.20166, 10.5438, -7.61422, -7.69036, -4.42797, 8.98777, - 4.45301, 1.53344, 4.59296, 2.45021, -6.81264, -6.36417, 4.62346, 3.16156, - -5.93007, -8.36501, -2.78425, 6.71237, -6.17141, -6.64689, -5.20608, 8.95999, - -7.30598, -5.73166, 4.39572, 2.93726, -1.89503, 9.77179, -5.683, -7.48989, - 4.80924, 0.559455, -2.17793, 9.98983, 5.23728, 2.67434, -7.03976, -6.20877, - 3.90435, 3.20926, -7.78536, -7.53388, -1.00684, 9.08838, -5.26741, -5.98327, - 3.28002, 2.71942, -1.47166, 8.50427, -2.32733, 9.26251, 5.16271, 1.39947, - -6.59093, -6.61979, -2.44492, 7.93654, -1.05805, 9.97356, -3.1109, 10.8666, - 3.38834, 3.41693, 4.83098, 2.01961, -2.74013, 9.71049, -3.34892, 8.41489, - 4.94768, 0.263001, 3.57477, 1.66795, 5.78915, 1.26999, -4.81812, -5.67174, - -1.88508, 9.64263, 3.69048, 4.60555, 4.03037, 1.7862, -7.4418, -7.08933}, - {0.127717, 0.211407, 0.195547, 0.21633, 0.39671, 0.229008, 0.20839, 0.169236, 0.314314, - 0.322473, 0.169506, 0.45499, 0.147819, 0.296502, 0.15198, 0.356444, 0.0992833, 0.220833, - 0.296206, 0.178067, 0.135359, 0.189725, 0.243099, 0.519986, 0.168105, 0.273465, 0.126033, - 0.18045, 0.282832, 0.193901, 0.213704, 0.425046, 0.203191, 0.228674, 0.209267, 0.355039, - 0.212918, 0.315495, 0.294112, 0.257576, 0.5786, 0.186019, 0.171919, 0.171919, 0.449151, - 1.34947, 0.171919, 0.16341, 0.641387, 0.342115, 0.267343, 0.246125, 0.277612, 0.181462, - 0.22944, 1.95598, 0.164897, 0.235803, 0.228273, 0.314629, 0.127403, 0.241241, 0.189362, - 0.151691, 0.130085, 0.526707, 0.217069, 0.282306, 0.531523, 0.177035, 0.169776, 0.20395, - 0.177165, 0.146628, 0.280013, 0.223033, 0.50947, 0.184133, 0.295329, 0.183219, 0.28166, - 0.179348, 0.276462, 1.00283, 0.248147, 0.214453, 0.231732, 0.170672, 0.256893, 0.133271, - 0.151137, 0.500823, 0.23678, 0.376983, 0.362061, 0.140013, 0.388863, 0.398552, 0.38015, - 0.190081, 0.167115, 0.206884, 0.473849, 1.05117, 0.435665, 0.323618, 0.326201, 0.32226, - 0.201787, 0.246496, 0.28325, 0.226596, 0.238153, 0.277268, 0.674629, 0.179433, 0.175651, - 0.154778, 0.178195, 0.192796, 0.103571, 0.227621, 0.201124, 0.160525, 0.160964, 0.240099, - 0.258027, 0.134127, 0.127717, 0.341378, 0.311595, 0.282306, 0.168988, 0.40775, 0.246125, - 0.583131, 0.236804, 0.238633, 0.194824, 0.169315, 0.244227, 0.249511, 0.189725, 0.305662, - 0.301415, 0.658641, 0.250944, 0.151792, 0.141383, 0.143843, 0.563347, 0.184216, 0.204155, - 0.221764, 0.314908, 0.144518, 0.228808, 0.255785, 0.163457, 0.424705, 0.170202, 0.312598, - 0.300629, 0.532614, 0.661392, 0.228273, 0.543432, 0.257175, 0.258994, 0.281413, 0.273897, - 0.246837, 0.293489, 0.25533, 0.260492, 0.213704, 0.3091, 0.17103, 0.172285, 0.241399, - 0.35999, 0.372243, 0.269191, 0.390239, 0.31761, 0.200593, 0.22197, 0.752914, 0.266571, - 0.13102, 0.268659, 0.293723, 0.356294, 0.296258, 0.264531, 0.15468, 0.358535, 0.243711, - 0.112147, 0.121659, 0.197101, 0.515292, 0.245628, 0.279863, 0.789807, 0.195156, 0.196073, - 0.149564, 0.118675, 0.389373, 0.233821, 0.176128, 0.481088, 0.360027, 0.553152, 0.208207, - 0.171608, 0.160489, 0.334298, 0.139426, 0.168603, 0.266199, 0.326458, 0.103571, 0.171208, - 0.130961, 0.190887, 0.177229, 0.241651, 0.115152, 0.196753, 0.481088, 0.230965, 0.354631, - 0.14591, 0.328543, 0.141544, 0.195888, 0.290379, 0.245954, 0.184547, 0.575214, 0.186929, - 0.28527, 0.292213, 1.20033, 0.281528, 0.15625, 0.211524, 0.186398, 0.298061, 0.147393, - 0.245349, 0.164527, 0.224771, 0.222382, 0.251643, 0.148835, 0.135359, 0.204967, 0.193024, - 0.486309, 0.389686, 0.211921, 0.307405, 0.38666, 0.26802, 0.16605, 0.323134, 0.268397, - 0.217894, 0.974118, 0.371618, 0.156201, 0.305787, 0.339305, 0.371032, 0.381765, 0.22747, - 0.24906, 0.100884, 0.253192, 0.314253, 0.388289, 0.580947, 1.00267, 0.241998, 0.489101, - 0.341501, 0.247423, 0.328311, 0.440281, 0.14927, 0.244469, 0.846828, 0.191725, 0.217429, - 0.123403, 0.322875, 0.145373, 0.757259, 0.190086, 0.316286, 0.268397, 0.296721, 0.440472, - 0.186848, 0.232134, 0.180239, 0.219724, 0.205886, 0.250975, 0.145636, 0.312476, 0.366418, - 0.128135, 0.315235, 0.264531, 0.161815, 0.31631, 0.296489, 0.37171, 0.197217, 0.195625, - 0.479579, 0.443037, 0.323347, 0.193616, 0.160251, 0.8952, 0.256291, 0.593345, 0.177165, - 0.409514, 0.847863, 0.111448, 0.210031, 0.251347, 0.351953, 0.705204, 0.117901, 0.182343, - 0.230179, 0.83632, 0.22104, 0.145163, 0.200326, 0.23431, 0.21868, 0.253575, 0.186562, - 0.192757, 0.172716, 0.27396, 0.258581, 0.327892, 0.376138, 0.223477, 0.302375, 0.145845, - 0.436902, 0.421794, 0.328543, 0.19246, 0.238889, 0.254866, 0.284674, 0.457849, 0.202937, - 0.392568, 0.453083, 0.782713, 0.465401, 0.178623, 0.304863, 0.190081, 0.228641, 0.255135, - 0.245037, 0.217526, 0.109584, 0.276462, 0.182301, 0.38582, 0.349942, 1.3889, 0.30235, - 0.796353, 0.160168, 0.643204, 0.153752, 0.410268, 0.186439, 0.256834, 0.185783, 0.0957629, - 0.226596, 0.197951, 0.17123, 0.192836, 0.18405, 0.575784, 0.228874, 0.201787, 0.241209, - 0.217386, 0.195751, 0.291585, 0.144531, 0.14176, 0.157635, 0.410268, 0.476338, 0.308148, - 0.148077, 0.152093, 0.196791, 0.568087, 0.414026, 0.250587, 0.473463, 0.293645, 0.396768, - 0.2766, 0.38664, 0.135034, 1.50827, 0.472527, 0.268418, 0.40383, 0.375914, 0.246496, - 0.176474, 0.340405, 0.220833, 0.138782, 0.159009, 0.444219, 0.259582, 0.33638, 0.195586, - 0.210974, 0.200288, 0.148129, 0.0974216, 0.211588, 0.280081, 0.44113, 0.773921, 0.553848, - 0.448079, 0.183136, 0.380854, 0.685021, 0.308767, 0.553276, 0.181578, 0.164759, 0.313889, - 0.137886, 0.545387, 0.278449, 0.736895, 0.360054, 0.358929, 0.457315, 0.343278, 0.507662, - 0.280829, 0.113886, 0.23146, 0.160584, 0.192796, 0.147561, 0.241272, 0.168988, 0.730511, - 0.27836, 0.179847, 0.22555, 0.418069, 0.158348, 0.128965, 0.179454, 0.126366, 0.164434, - 0.273633, 0.309556, 0.500823, 0.367852, 0.192875, 0.230262, 0.32724, 0.249969, 0.142618, - 0.494229, 0.36108, 0.227931, 0.23113, 0.742825, 0.190126, 0.33741, 0.280598, 0.145268, - 0.378423, 0.211921, 0.183594, 0.59201, 0.279563, 0.195683, 0.248101, 0.199754, 0.342494, - 0.174343, 0.14149, 0.28085, 0.175781, 0.518738, 0.17223, 0.489904, 0.181167, 0.354286, - 0.297824, 0.280829, 0.219412, 0.22814, 0.195625, 0.313949, 0.294708, 0.211551, 0.236255, - 0.666933, 0.204808, 0.52591, 0.180725, 0.186889, 0.246589, 0.410575, 0.338348, 0.206219, - 0.361766, 0.158143, 0.280816, 0.4149, 0.773082, 0.340046, 0.369672, 0.256923, 0.167195, - 0.197217, 0.252339, 0.172716, 0.191526, 0.263085, 0.345698, 0.168286, 0.243099, 0.434631, - 0.22944, 0.161862, 0.206589, 0.23457, 0.181924, 0.419063, 0.183427, 0.186152, 0.236352, - 0.306336, 0.149002, 1.50086, 0.188231, 0.442757, 0.485602, 0.466662, 0.17329, 0.141329, - 0.180619, 0.160061, 0.192569, 0.270999, 0.117901, 0.362693, 0.217561, 0.208975, 0.233658, - 0.175173, 1.10307, 0.14625, 1.31124, 0.237608, 0.286784, 0.325112, 0.2485, 0.259641, - 0.553152, 0.179039, 0.780781, 0.174758, 0.297824, 0.2558, 0.235949, 0.952186, 0.356744, - 0.312646, 0.189362, 0.574524, 0.705204, 0.213168, 0.225956, 0.424165, 0.169506, 0.137109, - 0.352451, 0.454554, 0.653302, 0.31261, 0.194412, 0.23719, 0.137886, 0.31498, 0.199085, - 0.203875, 0.597248, 1.10036, 0.196869, 0.22104, 0.451345, 0.105613, 0.683928, 0.135204, - 0.25533, 0.607871, 0.219724, 0.184464, 0.725001, 0.160061, 0.333407, 0.192569, 0.234147, - 0.47178, 0.161815, 0.242455, 0.215305, 0.410575, 0.242376, 0.211335, 0.462804, 0.275065, - 0.126878, 0.170404, 0.179433, 0.147244, 0.109584, 0.352905, 0.158215, 0.197604, 0.172407, - 0.407506, 0.645446, 0.313061, 0.165602, 0.136663, 0.55444, 0.15527, 0.133128, 0.125912, - 0.340405, 0.44521, 0.122783, 0.814526, 0.243773, 0.15743, 0.266743, 0.684458, 0.22221, - 0.181294, 0.193901, 0.258802, 0.167195, 0.292056, 0.132309, 0.227671, 0.117334, 0.271758, - 0.146185, 0.225042, 0.225964, 0.194863, 0.290274, 0.138438, 0.196714, 0.266012, 0.267771, - 0.162544, 0.244258, 0.358038, 0.522617, 0.192875, 0.45066, 0.330396, 0.223477, 0.42967, - 0.350884, 0.404655, 0.123155, 0.431583, 0.191675, 0.147354, 0.609034, 0.459487, 0.187337, - 0.215128, 0.604169, 0.330165, 0.494229, 0.40775, 0.167377, 0.192648, 0.234635, 0.275578, - 0.253094, 0.420063, 0.228299, 0.206478, 0.20395, 0.377656, 0.317393, 0.478623, 0.159009, - 0.217034, 0.300933, 0.139754, 0.153901, 0.261077, 0.22834, 0.449609, 0.157672, 0.176474, - 0.285704, 0.180186, 0.212738, 0.266428, 0.388313, 0.0954637, 0.298093, 0.251643, 0.330696, - 0.159572, 0.210666, 0.149411, 0.139618, 0.338472, 0.450304, 0.208793, 0.583609, 0.185865, - 0.400576, 0.21626, 0.174867, 0.239144, 0.249113, 0.200402, 0.275065, 0.238793, 0.205784, - 0.4475, 0.231262, 0.259082, 0.20934, 0.16806, 0.193616, 0.213811, 0.395632, 0.482465, - 0.274649, 0.307405, 0.165866, 0.334275, 0.683337, 0.368825, 0.14625, 0.780742, 0.163457, - 0.226596, 0.138713, 1.79155, 0.400443, 0.233658, 0.426399, 0.623024, 0.670955, 0.123588, - 0.110899, 0.173751, 0.651068, 0.199983, 0.190887, 0.541435, 0.21324, 0.266571, 0.134638, - 0.179348, 0.145636, 0.170929, 0.623252, 0.587738, 0.109688, 0.515314, 0.217666, 0.213311, - 0.249144, 0.187947, 0.270999, 0.268311, 0.469782, 0.763609, 0.32124, 0.146315, 0.265223, - 0.298694, 0.197623, 0.21349, 0.845778, 0.175466, 0.123588, 0.17223, 0.258603, 1.17119, - 0.538142, 0.407675, 0.120288, 0.587238, 0.244664, 0.333956, 0.132812, 0.21399, 0.302375, - 0.275882, 0.134284, 0.377555, 0.228541, 0.187307, 0.143804, 0.180545, 0.222451, 0.239638, - 0.188028, 0.46334, 0.175868, 0.242392, 0.314762, 0.44473, 0.21962, 0.175966, 1.12364, - 0.138837, 0.400576, 0.18184, 0.137706, 0.409763, 0.216894, 0.466662, 0.376604, 0.487155, - 0.283143, 0.118547, 0.221591, 0.122783, 0.179007, 0.16628, 0.180999, 0.239845, 0.169607, - 0.578402, 0.396537, 0.222288, 0.563237, 0.371238, 0.138658, 0.324336, 0.191526, 0.168603, - 0.357715, 0.640905, 0.460706, 0.220902, 0.240797, 0.164062, 0.157853, 0.34457, 0.196092, - 0.289353, 0.104597, 0.259641, 0.126878, 0.175781, 0.441458, 0.820108, 0.261864, 0.23431, - 0.254506, 0.271955, 0.227529, 0.22834, 0.196753, 0.224906, 0.193783, 0.419481, 0.236933, - 0.229706, 0.29785, 0.222947, 0.177606, 0.216911, 0.305188, 0.933438, 0.116666, 0.278483, - 0.0973824, 0.271224, 0.127717, 1.28139, 0.276283, 0.180704, 0.234554, 0.285984, 0.290172, - 0.49594, 0.135879, 0.436784, 0.206219, 0.342215, 0.374165, 0.182217, 0.274864, 0.625, - 0.356925, 0.194324, 0.342215, 0.113012, 0.155123, 0.254207, 0.438919, 0.262548, 0.302299, - 0.179528, 0.312744, 0.168513, 0.142618, 0.150543, 0.231361, 0.166004, 0.186725, 0.38848, - 0.179857, 0.182301, 0.629476, 0.44113, 0.289669, 0.328543, 0.279938, 0.14625, 0.187174, - 0.157635, 0.396749, 0.798931, 0.201541, 0.778619, 0.265883, 0.258027, 0.218576, 0.266571, - 0.160168, 0.230303, 0.273633, 0.233298, 0.30175, 0.217069, 0.345145, 0.397901, 0.224499, - 0.248101, 0.241335, 0.222947, 0.237094, 0.176518, 0.380032, 0.634775, 0.426193, 0.16362, - 0.231097, 0.219898, 0.343789, 0.275578, 0.282022, 0.628542, 0.232184, 0.848367, 0.200754, - 0.179177}, - {0, 0, 2, 3, 3, 0, 2, 2, 2, 2, 3, 0, 3, 2, 2, 2, 3, 3, 3, 3, 2, 0, 0, 0, 2, 3, 3, 3, 2, 2, 0, 0, - 2, 3, 3, 0, 0, 2, 0, 0, 3, 2, 3, 0, 3, 0, 3, 3, 0, 2, 0, 3, 2, 0, 3, 0, 3, 3, 3, 2, 2, 3, 0, 0, - 3, 3, 0, 2, 2, 3, 0, 3, 2, 2, 2, 0, 2, 3, 3, 3, 2, 3, 3, 3, 2, 0, 2, 0, 3, 3, 3, 3, 2, 2, 0, 2, - 0, 3, 2, 2, 2, 0, 0, 3, 0, 2, 2, 3, 2, 3, 0, 2, 2, 2, 3, 2, 0, 0, 2, 3, 3, 2, 0, 2, 0, 0, 2, 0, - 2, 2, 3, 2, 2, 0, 3, 0, 3, 2, 2, 2, 3, 3, 0, 0, 0, 3, 2, 3, 3, 3, 3, 0, 2, 0, 3, 2, 3, 2, 3, 0, - 2, 3, 3, 2, 3, 3, 2, 2, 0, 0, 2, 3, 3, 2, 3, 0, 2, 0, 2, 0, 3, 2, 3, 2, 3, 0, 3, 0, 3, 0, 2, 3, - 2, 2, 3, 0, 2, 2, 2, 0, 3, 2, 3, 3, 2, 3, 2, 3, 3, 2, 2, 0, 0, 2, 2, 3, 0, 3, 0, 2, 0, 0, 2, 3, - 0, 3, 3, 2, 0, 3, 3, 0, 3, 0, 2, 2, 0, 2, 0, 2, 0, 0, 0, 2, 0, 3, 2, 3, 2, 3, 2, 2, 0, 2, 3, 2, - 3, 2, 2, 2, 2, 3, 0, 2, 0, 0, 2, 3, 3, 0, 2, 3, 2, 2, 3, 0, 3, 0, 0, 2, 0, 2, 0, 2, 2, 3, 3, 2, - 3, 0, 0, 3, 2, 2, 0, 3, 2, 0, 0, 3, 0, 0, 2, 0, 3, 2, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 2, 3, 0, 0, - 2, 0, 0, 2, 0, 2, 3, 2, 3, 3, 2, 2, 0, 0, 0, 3, 0, 2, 0, 2, 0, 2, 2, 2, 3, 3, 0, 0, 3, 3, 3, 3, - 3, 2, 3, 3, 2, 3, 3, 0, 2, 2, 2, 2, 0, 2, 0, 0, 0, 2, 2, 3, 3, 2, 3, 2, 3, 0, 2, 3, 0, 2, 0, 2, - 2, 0, 3, 0, 2, 0, 2, 3, 0, 3, 0, 0, 0, 3, 2, 3, 3, 0, 3, 2, 3, 0, 2, 3, 3, 0, 2, 3, 0, 0, 0, 2, - 0, 3, 0, 2, 3, 3, 3, 3, 3, 0, 2, 0, 2, 2, 3, 3, 0, 3, 0, 2, 0, 2, 0, 3, 0, 0, 0, 2, 3, 3, 2, 3, - 0, 0, 0, 0, 3, 3, 0, 3, 2, 0, 2, 3, 2, 2, 3, 3, 2, 2, 2, 0, 2, 3, 0, 3, 3, 0, 0, 2, 0, 3, 2, 3, - 0, 2, 0, 2, 2, 3, 2, 0, 3, 3, 3, 2, 3, 0, 3, 0, 2, 2, 0, 0, 0, 3, 0, 3, 3, 2, 3, 2, 3, 2, 3, 0, - 2, 3, 0, 2, 0, 3, 3, 3, 3, 3, 3, 2, 0, 3, 2, 2, 2, 3, 3, 2, 3, 0, 2, 3, 3, 2, 2, 0, 0, 0, 0, 3, - 0, 3, 3, 3, 0, 0, 0, 3, 3, 3, 3, 3, 0, 2, 3, 3, 3, 3, 3, 3, 0, 0, 2, 2, 3, 3, 2, 2, 0, 0, 3, 0, - 0, 0, 2, 3, 0, 0, 0, 3, 0, 3, 0, 2, 2, 0, 0, 0, 0, 3, 2, 2, 3, 2, 3, 2, 2, 2, 2, 3, 0, 0, 2, 3, - 0, 3, 3, 0, 3, 0, 0, 2, 0, 3, 3, 0, 2, 2, 3, 3, 0, 0, 2, 0, 2, 3, 2, 0, 0, 3, 3, 0, 3, 2, 0, 2, - 0, 2, 3, 2, 0, 3, 3, 2, 0, 0, 2, 2, 0, 0, 2, 0, 3, 3, 2, 3, 2, 0, 3, 0, 2, 2, 3, 3, 0, 3, 2, 2, - 0, 3, 0, 0, 0, 2, 0, 3, 2, 0, 2, 3, 2, 3, 2, 2, 3, 3, 0, 2, 3, 2, 3, 2, 2, 0, 3, 0, 3, 0, 2, 2, - 2, 0, 2, 0, 2, 2, 0, 0, 3, 3, 0, 0, 3, 2, 0, 2, 3, 2, 2, 0, 3, 3, 0, 2, 0, 3, 3, 0, 2, 3, 2, 3, - 2, 0, 2, 2, 0, 0, 0, 2, 2, 3, 3, 2, 2, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 2, 0, 3, 3, - 3, 0, 2, 0, 2, 3, 2, 0, 3, 3, 2, 0, 2, 0, 3, 2, 0, 3, 0, 0, 2, 2, 0, 3, 0, 2, 3, 3, 3, 0, 2, 0, - 0, 3, 0, 2, 3, 2, 2, 0, 3, 3, 3, 3, 3, 0, 3, 0, 0, 0, 0, 3, 2, 0, 0, 2, 3, 3, 2, 2, 0, 3, 2, 0, - 3, 0, 2, 3, 3, 0, 2, 2, 3, 2, 2, 2, 3, 2, 0, 0, 3, 2, 0, 0, 0, 2, 0, 2, 0, 0, 2, 2, 3, 0, 3, 0, - 0, 3, 0, 0, 0, 3, 0, 0, 2, 2, 0, 2, 2, 3, 3, 3, 3, 0, 0, 2, 2, 2, 0, 3, 2, 2, 2, 2, 2, 0, 3, 0, - 0, 3, 2, 0, 0, 3, 2, 3, 3, 0, 3, 0, 3, 0, 3, 2, 2, 2, 0, 0, 3, 2, 2, 0, 0, 0, 2, 3, 2, 0, 2, 3, - 3, 3, 0, 3, 3, 0, 2, 0, 0, 2, 3, 3, 0, 3, 2, 2, 2, 2, 2, 3, 3, 2, 2, 3, 3, 2, 3, 0, 3, 3, 0, 3, - 2, 2, 0, 2, 0, 3, 0, 3, 0, 2, 3, 0, 2, 3, 2, 0, 2, 0, 3, 0, 2, 3, 3, 2, 0, 3, 3, 3, 2, 2, 3, 3, - 2, 2, 2, 0, 3, 2, 2, 0}, - {271, 271, 329, 343, 387, 426, 426, 601}, - {426, 601, 426, 387, 343, 271, 329, 271}, - {3.70991, 4.43491, 3.76334, 9.43944, 9.43944, 3.70991, 3.76334, 4.43491}}}; - -typedef ConnectComponentsEdgesTest ConnectComponentsEdgesTestF_Int; -TEST_P(ConnectComponentsEdgesTestF_Int, Result) { EXPECT_TRUE(true); } - -INSTANTIATE_TEST_CASE_P(ConnectComponentsEdgesTest, - ConnectComponentsEdgesTestF_Int, - ::testing::ValuesIn(mr_fix_conn_inputsf2)); - -}; // namespace sparse -}; // end namespace raft diff --git a/cpp/test/sparse/neighbors/knn_graph.cu b/cpp/test/sparse/neighbors/knn_graph.cu deleted file mode 100644 index db610099e9..0000000000 --- a/cpp/test/sparse/neighbors/knn_graph.cu +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../../test_utils.cuh" - -#include -#include -#include -#include - -#include -#include - -#include - -#include - -namespace raft { -namespace sparse { - -template -RAFT_KERNEL assert_symmetry( - value_idx* rows, value_idx* cols, value_t* vals, value_idx nnz, value_idx* sum) -{ - int tid = blockDim.x * blockIdx.x + threadIdx.x; - - if (tid >= nnz) return; - - atomicAdd(sum, rows[tid]); - atomicAdd(sum, -1 * cols[tid]); -} - -template -struct KNNGraphInputs { - value_idx m; - value_idx n; - - std::vector X; - - int k = 2; -}; - -template -::std::ostream& operator<<(::std::ostream& os, const KNNGraphInputs& dims) -{ - return os; -} - -template -class KNNGraphTest : public ::testing::TestWithParam> { - public: - KNNGraphTest() - : params(::testing::TestWithParam>::GetParam()), - stream(resource::get_cuda_stream(handle)), - X(0, stream) - { - X.resize(params.X.size(), stream); - } - - protected: - void SetUp() override - { - out = new raft::sparse::COO(stream); - - update_device(X.data(), params.X.data(), params.X.size(), stream); - - raft::sparse::neighbors::knn_graph( - handle, X.data(), params.m, params.n, raft::distance::DistanceType::L2Unexpanded, *out); - - rmm::device_scalar sum(stream); - sum.set_value_to_zero_async(stream); - - /** - * Assert the knn graph is symmetric - */ - assert_symmetry<<nnz, 256), 256, 0, stream>>>( - out->rows(), out->cols(), out->vals(), out->nnz, sum.data()); - - sum_h = sum.value(stream); - resource::sync_stream(handle, stream); - } - - void TearDown() override { delete out; } - - protected: - raft::resources handle; - cudaStream_t stream; - - // input data - raft::sparse::COO* out; - - rmm::device_uvector X; - - value_idx sum_h; - - KNNGraphInputs params; -}; - -const std::vector> knn_graph_inputs_fint = { - // Test n_clusters == n_points - {4, 2, {0, 100, 0.01, 0.02, 5000, 10000, -5, -2}, 2}}; - -typedef KNNGraphTest KNNGraphTestF_int; -TEST_P(KNNGraphTestF_int, Result) -{ - // nnz should not be larger than twice m * k - ASSERT_TRUE(out->nnz <= (params.m * params.k * 2)); - ASSERT_TRUE(sum_h == 0); -} - -INSTANTIATE_TEST_CASE_P(KNNGraphTest, - KNNGraphTestF_int, - ::testing::ValuesIn(knn_graph_inputs_fint)); - -} // namespace sparse -} // namespace raft diff --git a/cpp/test/stats/neighborhood_recall.cu b/cpp/test/stats/neighborhood_recall.cu deleted file mode 100644 index 1a76154e2e..0000000000 --- a/cpp/test/stats/neighborhood_recall.cu +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../neighbors/ann_utils.cuh" -#include "../test_utils.h" - -#include -#include -#include -#include -#include -#include - -#include - -#include - -namespace raft::stats { - -struct NeighborhoodRecallInputs { - int n_rows; - int n_cols; - int k; -}; - -template -class NeighborhoodRecallTest : public ::testing::TestWithParam { - public: - NeighborhoodRecallTest() - : ps{::testing::TestWithParam::GetParam()}, - data_1{raft::make_device_matrix(res, ps.n_rows, ps.n_cols)}, - data_2{raft::make_device_matrix(res, ps.n_rows, ps.n_cols)} - { - } - - protected: - void test_recall() - { - size_t queries_size = ps.n_rows * ps.k; - - // calculate nn for dataset 1 - auto distances_1 = raft::make_device_matrix(res, ps.n_rows, ps.k); - auto indices_1 = raft::make_device_matrix(res, ps.n_rows, ps.k); - raft::neighbors::naive_knn( - res, - distances_1.data_handle(), - indices_1.data_handle(), - data_1.data_handle(), - data_1.data_handle(), - ps.n_rows, - ps.n_rows, - ps.n_cols, - ps.k, - raft::distance::DistanceType::L2Expanded); - std::vector distances_1_h(queries_size); - std::vector indices_1_h(queries_size); - raft::copy(distances_1_h.data(), - distances_1.data_handle(), - ps.n_rows * ps.k, - raft::resource::get_cuda_stream(res)); - raft::copy(indices_1_h.data(), - indices_1.data_handle(), - ps.n_rows * ps.k, - raft::resource::get_cuda_stream(res)); - - // calculate nn for dataset 2 - auto distances_2 = raft::make_device_matrix(res, ps.n_rows, ps.k); - auto indices_2 = raft::make_device_matrix(res, ps.n_rows, ps.k); - raft::neighbors::naive_knn( - res, - distances_2.data_handle(), - indices_2.data_handle(), - data_2.data_handle(), - data_2.data_handle(), - ps.n_rows, - ps.n_rows, - ps.n_cols, - ps.k, - raft::distance::DistanceType::L2Expanded); - std::vector distances_2_h(queries_size); - std::vector indices_2_h(queries_size); - raft::copy(distances_2_h.data(), - distances_2.data_handle(), - ps.n_rows * ps.k, - raft::resource::get_cuda_stream(res)); - raft::copy(indices_2_h.data(), - indices_2.data_handle(), - ps.n_rows * ps.k, - raft::resource::get_cuda_stream(res)); - - raft::resource::sync_stream(res); - - // find CPU recall scores - [[maybe_unused]] auto [indices_only_recall_h, mc1, tc1] = - raft::neighbors::calc_recall(indices_1_h, indices_2_h, ps.n_rows, ps.k); - [[maybe_unused]] auto [recall_h, mc2, tc2] = raft::neighbors::calc_recall( - indices_1_h, indices_2_h, distances_1_h, distances_2_h, ps.n_rows, ps.k, 0.001); - - // find GPU recall scores - auto s1 = 0; - auto indices_only_recall_scalar = raft::make_host_scalar(s1); - neighborhood_recall(res, - raft::make_const_mdspan(indices_1.view()), - raft::make_const_mdspan(indices_2.view()), - indices_only_recall_scalar.view()); - - auto s2 = 0; - auto recall_scalar = raft::make_host_scalar(s2); - DistanceT s3 = 0.001; - auto eps_mda = raft::make_host_scalar(s3); - - neighborhood_recall(res, - raft::make_const_mdspan(indices_1.view()), - raft::make_const_mdspan(indices_2.view()), - recall_scalar.view(), - raft::make_const_mdspan(distances_1.view()), - raft::make_const_mdspan(distances_2.view())); - - // assert correctness - ASSERT_TRUE(raft::match(indices_only_recall_h, - *indices_only_recall_scalar.data_handle(), - raft::CompareApprox(0.01))); - ASSERT_TRUE( - raft::match(recall_h, *recall_scalar.data_handle(), raft::CompareApprox(0.01))); - } - - void SetUp() override - { - // form two random datasets - raft::random::Rng r1(1234ULL); - r1.normal(data_1.data_handle(), - ps.n_rows * ps.n_cols, - DistanceT(0.1), - DistanceT(2.0), - raft::resource::get_cuda_stream(res)); - raft::random::Rng r2(21111ULL); - r2.normal(data_2.data_handle(), - ps.n_rows * ps.n_cols, - DistanceT(0.1), - DistanceT(2.0), - raft::resource::get_cuda_stream(res)); - resource::sync_stream(res); - } - - private: - raft::resources res; - NeighborhoodRecallInputs ps; - raft::device_matrix data_1; - raft::device_matrix data_2; -}; - -const std::vector inputs = - raft::util::itertools::product({10, 50, 100}, // n_rows - {80, 100}, // n_cols - {32, 64}); // k - -using NeighborhoodRecallTestF_U32 = NeighborhoodRecallTest; -TEST_P(NeighborhoodRecallTestF_U32, AnnCagra) { this->test_recall(); } - -INSTANTIATE_TEST_CASE_P(NeighborhoodRecallTest, - NeighborhoodRecallTestF_U32, - ::testing::ValuesIn(inputs)); - -} // end namespace raft::stats diff --git a/cpp/test/stats/silhouette_score.cu b/cpp/test/stats/silhouette_score.cu deleted file mode 100644 index ad080f5894..0000000000 --- a/cpp/test/stats/silhouette_score.cu +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "../test_utils.cuh" - -#include -#include -#include -#include - -#include - -#include - -#include -#include -#include - -namespace raft { -namespace stats { - -// parameter structure definition -struct silhouetteScoreParam { - int nRows; - int nCols; - int nLabels; - raft::distance::DistanceType metric; - int chunk; - double tolerance; -}; - -// test fixture class -template -class silhouetteScoreTest : public ::testing::TestWithParam { - protected: - silhouetteScoreTest() - : d_X(0, resource::get_cuda_stream(handle)), - sampleSilScore(0, resource::get_cuda_stream(handle)), - d_labels(0, resource::get_cuda_stream(handle)) - { - } - - void host_silhouette_score() - { - // generating random value test input - std::vector h_X(nElements, 0.0); - std::vector h_labels(nRows, 0); - std::random_device rd; - std::default_random_engine dre(nElements * nLabels); - std::uniform_int_distribution intGenerator(0, nLabels - 1); - std::uniform_real_distribution realGenerator(0, 100); - - std::generate(h_X.begin(), h_X.end(), [&]() { return realGenerator(dre); }); - std::generate(h_labels.begin(), h_labels.end(), [&]() { return intGenerator(dre); }); - - // allocating and initializing memory to the GPU - auto stream = resource::get_cuda_stream(handle); - d_X.resize(nElements, stream); - d_labels.resize(nElements, stream); - RAFT_CUDA_TRY(cudaMemsetAsync(d_X.data(), 0, d_X.size() * sizeof(DataT), stream)); - RAFT_CUDA_TRY(cudaMemsetAsync(d_labels.data(), 0, d_labels.size() * sizeof(LabelT), stream)); - sampleSilScore.resize(nElements, stream); - - raft::update_device(d_X.data(), &h_X[0], (int)nElements, stream); - raft::update_device(d_labels.data(), &h_labels[0], (int)nElements, stream); - - // finding the distance matrix - - rmm::device_uvector d_distanceMatrix(nRows * nRows, stream); - double* h_distanceMatrix = (double*)malloc(nRows * nRows * sizeof(double*)); - - raft::distance::pairwise_distance( - handle, d_X.data(), d_X.data(), d_distanceMatrix.data(), nRows, nRows, nCols, params.metric); - - resource::sync_stream(handle, stream); - - raft::update_host(h_distanceMatrix, d_distanceMatrix.data(), nRows * nRows, stream); - - // finding the bincount array - - double* binCountArray = (double*)malloc(nLabels * sizeof(double*)); - memset(binCountArray, 0, nLabels * sizeof(double)); - - for (int i = 0; i < nRows; ++i) { - binCountArray[h_labels[i]] += 1; - } - - // finding the average intra cluster distance for every element - - double* a = (double*)malloc(nRows * sizeof(double*)); - - for (int i = 0; i < nRows; ++i) { - int myLabel = h_labels[i]; - double sumOfIntraClusterD = 0; - - for (int j = 0; j < nRows; ++j) { - if (h_labels[j] == myLabel) { sumOfIntraClusterD += h_distanceMatrix[i * nRows + j]; } - } - - if (binCountArray[myLabel] <= 1) - a[i] = -1; - else - a[i] = sumOfIntraClusterD / (binCountArray[myLabel] - 1); - } - - // finding the average inter cluster distance for every element - - double* b = (double*)malloc(nRows * sizeof(double*)); - - for (int i = 0; i < nRows; ++i) { - int myLabel = h_labels[i]; - double minAvgInterCD = ULLONG_MAX; - - for (int j = 0; j < nLabels; ++j) { - int curClLabel = j; - if (curClLabel == myLabel) continue; - double avgInterCD = 0; - - for (int k = 0; k < nRows; ++k) { - if (h_labels[k] == curClLabel) { avgInterCD += h_distanceMatrix[i * nRows + k]; } - } - - if (binCountArray[curClLabel]) - avgInterCD /= binCountArray[curClLabel]; - else - avgInterCD = ULLONG_MAX; - minAvgInterCD = min(minAvgInterCD, avgInterCD); - } - - b[i] = minAvgInterCD; - } - - // finding the silhouette score for every element - - double* truthSampleSilScore = (double*)malloc(nRows * sizeof(double*)); - for (int i = 0; i < nRows; ++i) { - if (a[i] == -1) - truthSampleSilScore[i] = 0; - else if (a[i] == 0 && b[i] == 0) - truthSampleSilScore[i] = 0; - else - truthSampleSilScore[i] = (b[i] - a[i]) / max(a[i], b[i]); - truthSilhouetteScore += truthSampleSilScore[i]; - } - - truthSilhouetteScore /= nRows; - } - - // the constructor - void SetUp() override - { - // getting the parameters - params = ::testing::TestWithParam::GetParam(); - - nRows = params.nRows; - nCols = params.nCols; - nLabels = params.nLabels; - chunk = params.chunk; - nElements = nRows * nCols; - - host_silhouette_score(); - - // calling the silhouette_score CUDA implementation - computedSilhouetteScore = raft::stats::silhouette_score( - handle, - raft::make_device_matrix_view(d_X.data(), nRows, nCols), - raft::make_device_vector_view(d_labels.data(), nRows), - std::make_optional(raft::make_device_vector_view(sampleSilScore.data(), nRows)), - nLabels, - params.metric); - - batchedSilhouetteScore = raft::stats::silhouette_score_batched( - handle, - raft::make_device_matrix_view(d_X.data(), nRows, nCols), - raft::make_device_vector_view(d_labels.data(), nRows), - std::make_optional(raft::make_device_vector_view(sampleSilScore.data(), nRows)), - nLabels, - chunk, - params.metric); - } - - // declaring the data values - raft::resources handle; - silhouetteScoreParam params; - int nLabels; - rmm::device_uvector d_X; - rmm::device_uvector sampleSilScore; - rmm::device_uvector d_labels; - int nRows; - int nCols; - int nElements; - double truthSilhouetteScore = 0; - double computedSilhouetteScore = 0; - double batchedSilhouetteScore = 0; - int chunk; -}; - -// setting test parameter values -const std::vector inputs = { - {4, 2, 3, raft::distance::DistanceType::L2Expanded, 4, 0.00001}, - {4, 2, 2, raft::distance::DistanceType::L2SqrtUnexpanded, 2, 0.00001}, - {8, 8, 3, raft::distance::DistanceType::L2Unexpanded, 4, 0.00001}, - {11, 2, 5, raft::distance::DistanceType::L2Expanded, 3, 0.00001}, - {40, 2, 8, raft::distance::DistanceType::L2Expanded, 10, 0.00001}, - {12, 7, 3, raft::distance::DistanceType::CosineExpanded, 8, 0.00001}, - {7, 5, 5, raft::distance::DistanceType::L1, 2, 0.00001}}; - -// writing the test suite -typedef silhouetteScoreTest silhouetteScoreTestClass; -TEST_P(silhouetteScoreTestClass, Result) -{ - ASSERT_NEAR(computedSilhouetteScore, truthSilhouetteScore, params.tolerance); - ASSERT_NEAR(batchedSilhouetteScore, truthSilhouetteScore, params.tolerance); -} -INSTANTIATE_TEST_CASE_P(silhouetteScore, silhouetteScoreTestClass, ::testing::ValuesIn(inputs)); - -} // end namespace stats -} // end namespace raft diff --git a/cpp/test/stats/trustworthiness.cu b/cpp/test/stats/trustworthiness.cu deleted file mode 100644 index 846c192022..0000000000 --- a/cpp/test/stats/trustworthiness.cu +++ /dev/null @@ -1,354 +0,0 @@ -/* - * Copyright (c) 2018-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" - -#include -#include -#include -#include - -#include - -#include -#include - -namespace raft { -namespace stats { - -class TrustworthinessScoreTest : public ::testing::Test { - public: - TrustworthinessScoreTest() - : d_X(0, resource::get_cuda_stream(handle)), d_X_embedded(0, resource::get_cuda_stream(handle)) - { - } - - protected: - void basicTest() - { - std::vector X = { - 5.6142087, 8.59787, -4.382763, -3.6452143, -5.8816037, -0.6330313, 4.6920023, - -0.79210913, 0.6106314, 2.1210914, 5.919943, -8.43784, -6.4819884, 0.41001374, - -6.1052523, -4.0825715, -5.314755, -2.834671, 5.751696, -6.5012555, -0.4719201, - -7.53353, 7.6789393, -1.4959852, -5.5977287, -9.564147, 1.2902534, 3.559834, - -6.7659483, 8.265964, 4.595404, 9.133477, -6.1553917, -6.319754, -2.9039452, - 4.4150834, -3.094395, -4.426273, 9.584571, -5.64133, 6.6209483, 7.4044604, - 3.9620576, 5.639907, 10.33007, -0.8792053, 5.143776, -7.464049, 1.2448754, - -5.6300974, 5.4518576, 4.119535, 6.749645, 7.627064, -7.2298336, 1.9681473, - -6.9083176, 6.404673, 0.07186685, 9.0994835, 8.51037, -8.986389, 0.40534487, - 2.115397, 4.086756, 1.2284287, -2.6272132, 0.06527536, -9.587425, -7.206078, - 7.864875, 7.4397306, -6.9233336, -2.6643622, 3.3466153, 7.0408177, -3.6069896, - -9.971769, 4.4075623, 7.9063697, 2.559074, 4.323717, 1.6867131, -1.1576937, - -9.893141, -3.251416, -7.4889135, -4.0588717, -2.73338, -7.4852257, 3.4460473, - 9.759119, -5.4680476, -4.722435, -8.032619, -1.4598992, 4.227361, 3.135568, - 1.1950601, 1.1982028, 6.998856, -6.131138, -6.6921015, 0.5361224, -7.1213965, - -5.6104236, -7.2212887, -2.2710054, 8.544764, -6.0254574, 1.4582269, -5.5587835, - 8.031556, -0.26328218, -5.2591386, -9.262641, 2.8691363, 5.299787, -9.209455, - 8.523085, 5.180329, 10.655528, -5.7171874, -6.7739563, -3.6306462, 4.067106, - -1.5912259, -3.2345476, 8.042973, -3.6364832, 4.1242137, 9.886953, 5.4743724, - 6.3058076, 9.369645, -0.5175337, 4.9859877, -7.879498, 1.358422, -4.147944, - 3.8984218, 5.894656, 6.4903927, 8.702036, -8.023722, 2.802145, -7.748032, - 5.8461113, -0.34215945, 11.298865, 1.4107164, -9.949621, -1.6257563, -10.655836, - 2.4528909, 1.1570255, 5.170669, 2.8398793, 7.1838694, 9.088459, 2.631155, - 3.964414, 2.8769252, 0.04198391, -0.16993195, 3.6747139, -2.8377378, 6.1782537, - 10.759618, -4.5642614, -8.522967, 0.8614642, 6.623416, -1.029324, 5.5488334, - -7.804511, 2.128833, 7.9042315, 7.789576, -2.7944536, 0.72271067, -10.511495, - -0.78634536, -10.661714, 2.9376361, 1.9148129, 6.22859, 0.26264945, 8.028384, - 6.8743043, 0.9351067, 7.0690722, 4.2846055, 1.4134506, -0.18144785, 5.2778087, - -1.7140163, 9.217541, 8.602799, -2.6537218, -7.8377395, 1.1244944, 5.4540544, - -0.38506773, 3.9885726, -10.76455, 1.4440702, 9.136163, 6.664117, -5.7046547, - 8.038592, -9.229767, -0.2799413, 3.6064725, 4.187257, 1.0516582, -2.0707326, - -0.7615968, -8.561018, -3.7831352, 10.300297, 5.332594, -6.5880876, -4.2508664, - 1.7985519, 5.7226253, -4.1223383, -9.6697855, 1.4885283, 7.524974, 1.7206005, - 4.890457, 3.7264557, 0.4428284, -9.922455, -4.250455, -6.4410596, -2.107994, - -1.4109765, -6.1325397, 0.32883006, 6.0489736, 7.7257385, -8.281174, 1.0129383, - -10.792166, 8.378851, 10.802716, 9.848448, -9.188757, 1.3151443, 1.9971865, - -2.521849, 4.3268294, -7.775683, -2.2902298, 3.0824065, -7.17559, 9.6100855, - 7.3965735, -10.476525, 5.895973, -3.6974669, -7.6688933, 1.7354839, -7.4045196, - -1.7992063, -4.0394845, 5.2471714, -2.250571, 2.528036, -8.343515, -2.2374575, - -10.019771, 0.73371273, 3.1853926, 2.7994921, 2.6637669, 7.620401, 7.515571, - 0.68636256, 5.834537, 4.650282, -1.0362619, 0.4461701, 3.7870514, -4.1340904, - 7.202998, 9.736904, -3.005512, -8.920467, 1.1228397, 6.2598724, 1.2812365, - 4.5442104, -8.791537, 0.92113096, 8.464749, 8.359035, -4.3923397, 1.2252625, - -10.1986475, -1.4409319, -10.013967, 3.9071581, 1.683064, 4.877419, 1.6570637, - 9.559105, 7.3546534, 0.36635467, 5.220211, 4.6303267, 0.6601065, 0.16149978, - 3.8818731, -3.4438233, 8.42085, 8.659159, -3.0935583, -8.039611, 2.3060374, - 5.134666, 1.0458113, 6.0190983, -9.143728, 0.99048865, 9.210842, 6.670241, - -5.9614363, 0.8747396, 7.078824, 8.067469, -10.314754, 0.45977542, -9.28306, - 9.1838665, 9.318644, 7.189082, -11.092555, 1.0320464, 3.882163, 0.10953151, - 7.9029684, -6.9068265, -1.3526366, 5.3996363, -8.430931, 11.452577, 6.39663, - -11.090514, 4.6662245, -3.1268113, -8.357452, 2.2276728, -10.357126, -0.9291848, - -3.4193344, 3.1289792, -2.5030103, 6.772719, 11.457757, -4.2125936, -6.684548, - -4.7611327, 3.6960156, -2.3030636, -3.0591488, 10.452471, -4.1267314, 5.66614, - 7.501461, 5.072407, 6.636537, 8.990381, -0.2559256, 4.737867, -6.2149944, - 2.535682, -5.5484023, 5.7113924, 3.4742818, 7.9915137, 7.0052586, -7.156467, - 1.4354781, -8.286235, 5.7523417, -2.4175215, 9.678009, 0.05066403, -9.645226, - -2.2658763, -9.518178, 4.493372, 2.3232365, 2.1659086, 0.42507997, 8.360246, - 8.23535, 2.6878164, 5.236947, 3.4924245, -0.6089895, 0.8884741, 4.359464, - -4.6073823, 7.83441, 8.958755, -3.4690795, -9.182282, 1.2478025, 5.6311107, - -1.2408862, 3.6316886, -8.684654, 2.1078515, 7.2813864, 7.9265943, -3.6135032, - 0.4571511, 8.493568, 10.496853, -7.432897, 0.8625995, -9.607528, 7.2899456, - 8.83158, 8.908199, -10.300263, 1.1451302, 3.7871468, -0.97040755, 5.7664757, - -8.9688, -2.146672, 5.9641485, -6.2908535, 10.126465, 6.1553903, -12.066902, - 6.301596, -5.0419583, -8.228695, 2.4879954, -8.918582, -3.7434099, -4.1593685, - 3.7431836, -1.1704745, 0.5524103, 9.109399, 9.571567, -11.209955, 1.2462777, - -9.554555, 9.091726, 11.477966, 7.630937, -10.450911, 1.9205878, 5.358983, - -0.44546837, 6.7611346, -9.74753, -0.5939732, 3.8892255, -6.437991, 10.294727, - 5.6723895, -10.7883, 6.192348, -5.293862, -10.811491, 1.0194173, -7.074576, - -3.192368, -2.5231771, 4.2791643, -0.53309685, 0.501366, 9.636625, 7.710316, - -6.4219728, 1.0975566, -8.218886, 6.9011984, 9.873679, 8.903804, -9.316832, - 1.2404599, 4.9039655, 1.2272617, 4.541515, -5.2753224, -3.2196746, 3.1303136, - -7.285681, 9.041425, 5.6417427, -9.93667, 5.7548947, -5.113397, -8.544622, - 4.182665, -7.7709813, -3.2810235, -3.312072, 3.8900535, -2.0604856, 6.709082, - -8.461194, 1.2666026, 4.8770437, 2.6955879, 3.0340345, -1.1614609, -3.536341, - -7.090382, -5.36146, 9.072544, 6.4554095, -4.4728956, -1.88395, 3.1095037, - 8.782348, -3.316743, -8.65248, 1.6802986, 8.186188, 2.1783829, 4.931278, - 4.158475, 1.4033595, -11.320101, -3.7084908, -6.740436, -2.5555193, -1.0451177, - -6.5569925, 0.82810307, 8.505919, 8.332857, -9.488569, -0.21588463, -8.056692, - 8.493993, 7.6401625, 8.812983, -9.377281, 2.4369764, 3.1766508, 0.6300803, - 5.6666765, -7.913654, -0.42301777, 4.506412, -7.8954244, 10.904591, 5.042256, - -9.626183, 8.347351, -3.605006, -7.923387, 1.1024277, -8.705793, -2.5151258, - -2.5066147, 4.0515003, -2.060757, 6.2635093, 8.286584, -6.0509276, -6.76452, - -3.1158175, 1.6578803, -1.4608748, -1.24211, 8.151246, -4.2970877, 6.093071, - 7.4911637, 4.51018, 4.8425875, 9.211085, -2.4386222, 4.5830803, -5.6079445, - 2.3713675, -4.0707507, 3.1787417, 5.462342, 6.915912, 6.3928423, -7.2970796, - 5.0112796, -9.140893, 4.9990606, 0.38391754, 7.7088532, 1.9340848, 8.18833, - 8.16617, -9.42086, -0.3388326, -9.659727, 8.243045, 8.099073, 8.439428, - -7.038694, 2.1077902, 3.3866816, -1.9975324, 7.4972878, -7.2525196, -1.553731, - 4.08758, -6.6922374, 9.50525, 4.026735, -9.243538, 7.2740564, -3.9319072, - -6.3228955, 1.6693478, -7.923119, -3.7423058, -2.2813146, 5.3469067, -1.8285407, - 3.3118162, 8.826356, -4.4641976, -6.4751124, -9.200089, -2.519147, 4.225298, - 2.4105988, -0.4344186, 0.53441775, 5.2836394, -8.2816105, -4.996147, -1.6870759, - -7.8543897, -3.9788852, -7.0346904, -3.1289773, 7.4567637, -5.6227813, 1.0709786, - -8.866012, 8.427324, -1.1755563, -5.789216, -8.197835, 5.3342214, 6.0646234, - -6.8975716, 7.717031, 3.480355, 8.312151, -3.6645212, -3.0976524, -8.090359, - -1.9176173, 2.4257212, 1.9700835, 0.4098958, 2.1341088, 7.652741, -9.9595585, - -5.989757, 0.10119354, -7.935407, -5.792786, -5.22783, -4.318978, 5.414037, - -6.4621663, 1.670883, -6.9224787, 8.696932, -2.0214002, -6.6681314, -8.326418, - 4.9049683, 5.4442496, -6.403739, 7.5822453, 7.0972915, -9.072851, -0.23897195, - 1.7662339, 5.3096304, 1.983179, -2.222645, -0.34700772, -9.094717, -6.107907, - 9.525174, 8.1550665, -5.6940084, -4.1636486, 1.7360662, 8.528821, -3.7299833, - -9.341266, 2.608542, 9.108706, 0.7978509, 4.2488184, 2.454484, 0.9446999, - -10.106636, -3.8973773, -6.6566644, -4.5647273, -0.99837756, -6.568582, 9.324853, - -7.9020953, 2.0910501, 2.2896829, 1.6790711, 1.3159255, -3.5258796, 1.8898442, - -8.105812, -4.924962, 8.771129, 7.1202874, -5.991957, -3.4106019, 2.4450088, - 7.796387, -3.055946, -7.8971434, 1.9856719, 9.001636, 1.8511922, 3.019749, - 3.1227696, 0.4822102, -10.021213, -3.530504, -6.225959, -3.0029628, -1.7881511, - -7.3879776, 1.3925704, 9.499782, -3.7318087, -3.7074296, -7.7466836, -1.5284524, - 4.0535855, 3.112011, 0.10340207, -0.5429599, 6.67026, -9.155924, -4.924038, - 0.64248866, -10.0103655, -3.2742946, -4.850029, -3.6707063, 8.586258, -5.855605, - 4.906918, -6.7813993, 7.9938135, -2.5473144, -5.688948, -7.822478, 2.1421318, - 4.66659, -9.701272, 9.549149, 0.8998125, -8.651497, -0.56899565, -8.639817, - 2.3088377, 2.1264515, 3.2764478, 2.341989, 8.594338, 8.630639, 2.8440373, - 6.2043204, 4.433932, 0.6320018, -1.8179281, 5.09452, -1.5741565, 8.153934, - 8.744339, -3.6945698, -8.883078, 1.5329908, 5.2745943, 0.44716078, 4.8809066, - -7.9594903, 1.134374, 9.233994, 6.5528665, -4.520542, 9.477355, -8.622195, - -0.23191702, 2.0485356, 3.9379985, 1.5916302, -1.4516805, -0.0843819, -7.8554378, - -5.88308, 7.999766, 6.2572145, -5.585321, -4.0097756, 0.42382592, 6.160884, - -3.631315, -8.333449, 2.770595, 7.8495173, 3.3331623, 4.940415, 3.6207345, - -0.037517, -11.034698, -3.185103, -6.614664, -3.2177854, -2.0792234, -6.8879867, - 7.821685, -8.455084, 1.0784642, 4.0033927, 2.7343264, 2.6052725, -4.1224284, - -0.89305353, -6.8267674, -4.9715133, 8.880253, 5.6994023, -5.9695024, -4.9181266, - 1.3017995, 7.972617, -3.9452884, -10.424556, 2.4504194, 6.21529, 0.93840516, - 4.2070026, 6.159839, 0.91979957, -8.706724, -4.317946, -6.6823545, -3.0388, - -2.464262, -7.3716645, 1.3926703, 6.544412, -5.6251183, -5.122411, -8.622049, - -2.3905911, 3.9138813, 1.9779967, -0.05011125, 0.13310997, 7.229751, -9.742043, - -8.08724, 1.2426697, -7.9230795, -3.3162494, -7.129571, -3.5488048, 7.4701195, - -5.2357526, 0.5917681, -6.272206, 6.342328, -2.909731, -4.991607, -8.845513, - 3.3228495, 7.033246, -7.8180246, 8.214469, 6.3910093, 9.185153, -6.20472, - -7.713809, -3.8481297, 3.5579286, 0.7078448, -3.2893546, 7.384514, -4.448121, - 3.0104196, 9.492943, 8.024847, 4.9114385, 9.965594, -3.014036, 5.182494, - -5.8806014, 2.5312455, -5.9926524, 4.474469, 6.3717875, 6.993105, 6.493093, - -8.935534, 3.004074, -8.055647, 8.315765, -1.3026813, 8.250377, 0.02606229, - 6.8508425, 9.655665, -7.0116496, -0.41060972, -10.049198, 7.897801, 6.7791023, - 8.3362, -9.821014, 2.491157, 3.5160472, -1.6228812, 7.398063, -8.769123, - -3.1743705, 3.2827861, -6.497855, 10.831924, 5.2761307, -9.704417, 4.3817043, - -3.9841619, -8.111647, 1.1883026, -8.115312, -2.9240117, -5.8879666, 4.20928, - -0.3587938, 6.935672, -10.177582, 0.48819053, 3.1250648, 2.9306343, 3.082544, - -3.477687, -1.3768549, -7.4922366, -3.756631, 10.039836, 3.6670392, -5.9761434, - -4.4728765, 3.244255, 7.027899, -2.3806512, -10.4100685, 1.605716, 7.7953773, - 0.5408159, 1.7156523, 3.824097, -1.0604783, -10.142124, -5.246805, -6.5283823, - -4.579547, -2.42714, -6.709197, 2.7782338, 7.33353, -6.454507, -2.9929368, - -7.8362985, -2.695445, 2.4900775, 1.6682367, 0.4641757, -1.0495365, 6.9631333, - -9.291356, -8.23837, -0.34263706, -8.275113, -2.8454232, -5.0864096, -2.681942, - 7.5450225, -6.2517986, 0.06810654, -6.470652, 4.9042645, -1.8369255, -6.6937943, - -7.9625087, 2.8510258, 6.180508, -8.282598, 7.919079, 1.4897474, 6.7217417, - -4.2459426, -4.114431, -8.375707, -2.143264, 5.6972933, 1.5574739, 0.39375135, - 1.7930849, 5.1737595, -7.826241, -5.160268, -0.80433255, -7.839536, -5.2620406, - -5.4643164, -3.185536, 6.620315, -7.065227, 1.0524757, -6.125088, 5.7126627, - -1.6161644, -3.852159, -9.164279, 2.7005782, 5.946544, -8.468236, 8.2145405, - 1.1035942, 6.590157, -4.0461283, -4.8090615, -7.6702685, -2.1121511, 5.1147075, - 1.6128504, 2.0064135, 1.0544407, 6.0038295, -7.8282537, -4.801278, 0.32349443, - -8.0649805, -4.372714, -5.61336, -5.21394, 8.176595, -5.4753284, 1.7800134, - -8.267283, 7.2133374, -0.16594432, -6.317046, -9.490406, 4.1261597, 5.473317, - -7.7551675, 7.007468, 7.478628, -8.801905, 0.10975724, 3.5478222, 4.797803, - 1.3825226, -3.357369, 0.99262005, -6.94877, -5.4781394, 9.632604, 5.7492557, - -5.9014316, -3.1632116, 2.340859, 8.708098, -3.1255999, -8.848661, 4.5612836, - 8.455157, 0.73460823, 4.112301, 4.392744, -0.30759293, -6.8036823, -3.0331545, - -8.269506, -2.82415, -0.9411246, -5.993506, 2.1618164, -8.716055, -0.7432543, - -10.255819, 3.095418, 2.5131428, 4.752442, 0.9907621, 7.8279433, 7.85814, - 0.50430876, 5.2840405, 4.457291, 0.03330028, -0.40692952, 3.9244103, -2.117118, - 7.6977615, 8.759009, -4.2157164, -9.136053, 3.247858, 4.668686, 0.76162136, - 5.3833632, -9.231471, 0.44309422, 8.380872, 6.7211227, -3.091507, 2.173508, - -9.038242, -1.3666698, -9.819077, 0.37825826, 2.3898845, 4.2440815, 1.9161536, - 7.24787, 6.9124637, 1.6238527, 5.1140285, 3.1935842, 1.02845, -1.1273454, - 5.638998, -2.497932, 8.342559, 8.586319, -2.9069402, -7.6387944, 3.5975037, - 4.4115705, 0.41506064, 4.9078383, -9.68327, 1.8159529, 9.744613, 8.40622, - -4.495336, 9.244892, -8.789869, 1.3158468, 4.018167, 3.3922846, 2.652022, - -2.7495477, 0.2528986, -8.268324, -6.004913, 10.428784, 6.6580734, -5.537176, - -1.7177434, 2.7504628, 6.7735, -2.4454272, -9.998361, 2.9483433, 6.8266654, - 2.3787718, 4.472637, 2.5871701, 0.7355365, -7.7027745, -4.1879907, -7.172832, - -4.1843605, -0.03646783, -5.419406, 6.958486, 11.011111, -7.1821184, -7.956423, - -3.408451, 4.6850276, -2.348787, -4.398289, 6.9787564, -3.8324208, 5.967827, - 8.433518, 4.660108, 5.5657144, 9.964243, -1.3515275, 6.404833, -6.4805903, - 2.4379845, -6.0816774, 1.752272, 5.3771873, 6.9613523, 6.9788294, -6.3894596, - 3.7521114, -6.8034263, 6.4458385, -0.7233525, 10.512529, 4.362273, 9.231461, - -6.3382263, -7.659, -3.461823, 4.71463, 0.17817476, -3.685746, 7.2962036, - -4.6489477, 5.218017, 11.546999, 4.7218375, 6.8498397, 9.281103, -3.900459, - 6.844054, -7.0886965, -0.05019227, -8.233724, 5.5808983, 6.374517, 8.321048, - 7.969449, -7.3478637, 1.4917561, -8.003144, 4.780668, -1.1981848, 7.753739, - 2.0260844, -8.880096, -3.4258451, -7.141975, 1.9637157, 1.814725, 5.311151, - 1.4831505, 7.8483663, 7.257948, 1.395786, 6.417756, 5.376912, 0.59505713, - 0.00062552, 3.6634305, -4.159713, 7.3571978, 10.966816, -2.5419605, -8.466229, - 1.904205, 5.6338267, -0.52567476, 5.59736, -8.361799, 0.5009981, 8.460681, - 7.3891273, -3.5272243, 5.0552278, 9.921456, -7.69693, -7.286378, -1.9198836, - 3.1666567, -2.5832257, -2.2445817, 9.888111, -5.076563, 5.677401, 7.497946, - 5.662994, 5.414262, 8.566503, -2.5530663, 7.1032815, -6.0612082, 1.3419591, - -4.9595256, 4.3377542, 4.3790717, 6.793512, 8.383502, -7.1278043, 3.3240774, - -9.379446, 6.838661, -0.81241214, 8.694813, 0.79141915, 7.632467, 8.575382, - -8.533798, 0.28954387, -7.5675836, 5.8653326, 8.97235, 7.1649346, -10.575289, - 0.9359381, 5.02381, -0.5609511, 5.543464, -7.69131, -2.1792977, 2.4729247, - -6.1917787, 10.373678, 7.6549597, -8.809486, 5.5657206, -3.3169382, -8.042887, - 2.0874746, -7.079005, -3.33398, -3.6843317, 4.0172358, -2.0754814, 1.1726758, - 7.4618697, 6.9483604, -8.469206, 0.7401797, -10.318176, 8.384557, 10.5476265, - 9.146971, -9.250223, 0.6290606, 4.4941425, -0.7514017, 7.2271705, -8.309598, - -1.4761636, 4.0140634, -6.021102, 9.132852, 5.6610966, -11.249811, 8.359293, - -1.9445792, -7.7393436, -0.3931331, -8.824441, -2.5995944, -2.5714035, 4.140213, - -3.6863053, 5.517265, 9.020411, -4.9286127, -7.871219, -3.7446704, 2.5179656, - -1.4543481, -2.2703636, 7.010597, -3.6436229, 6.753862, 7.4129915, 7.1406755, - 5.653706, 9.5445175, 0.15698843, 4.761813, -7.698002, 1.6870106, -4.5410123, - 4.171763, 5.3747005, 6.341021, 7.456738, -8.231657, 2.763487, -9.208167, - 6.676799, -1.1957736, 10.062605, 4.0975976, 7.312957, -2.4981596, -2.9658387, - -8.150425, -2.1075552, 2.64375, 1.6636052, 1.1483809, 0.09276015, 5.8556347, - -7.8481026, -5.9913163, -0.02840613, -9.937289, -1.0486673, -5.2340155, -3.83912, - 7.7165728, -8.409944, 0.80863273, -6.9119215, 7.5712357, 0.36031485, -6.056131, - -8.470033, 1.8678337, 3.0121377, -7.3096333, 8.205484, 5.262654, 8.774514, - -4.7603083, -7.2096143, -4.437014, 3.6080024, -1.624254, -4.2787876, 8.880863, - -4.8984556, 5.1782074, 9.944454, 3.911282, 3.5396595, 8.867042, -1.2006199, - 5.393288, -5.6455317, 0.7829499, -4.0338907, 2.479272, 6.5080743, 8.582535, - 7.0097537, -6.9823785, 3.984318, -7.225381, 5.3135114, -1.0391048, 8.951443, - -0.70119005, -8.510742, -0.42949116, -10.9224825, 2.8176029, 1.6800792, 5.778404, - 1.7269998, 7.1975236, 7.7258267, 2.7632928, 5.3399253, 3.4650044, 0.01971426, - -1.6468811, 4.114996, -1.5110453, 6.8689218, 8.269899, -3.1568048, -7.0344677, - 1.2911975, 5.950357, 0.19028673, 4.657226, -8.199647, 2.246055, 8.989509, - 5.3101015, -4.2400866}; - - std::vector X_embedded = { - -0.41849962, -0.53906363, 0.46958843, -0.35832694, -0.23779503, -0.29751351, -0.01072748, - -0.21353109, -0.54769957, -0.55086273, 0.37093949, -0.12714292, -0.06639574, -0.36098689, - -0.13060696, -0.07362658, -1.01205945, -0.39285606, 0.2864089, -0.32031146, -0.19595343, - 0.08900568, -0.04813879, -0.06563424, -0.42655188, -0.69014251, 0.51459783, -0.1942696, - -0.07767916, -0.6119386, 0.04813685, -0.22557008, -0.56890118, -0.60293794, 0.43429622, - -0.09240723, -0.00624062, -0.25800395, -0.1886092, 0.01655941, -0.01961523, -0.14147359, - 0.41414487, -0.8512944, -0.61199242, -0.18586016, 0.14024924, -0.41635606, -0.02890144, - 0.1065347, 0.39700791, -1.14060664, -0.95313865, 0.14416681, 0.17306046, -0.53189689, - -0.98987544, -0.67918193, 0.41787854, -0.20878236, -0.06612862, 0.03502904, -0.03765266, - -0.0980606, -0.00971657, 0.29432917, 0.36575687, -1.1645509, -0.89094597, 0.03718805, - 0.2310573, -0.38345811, -0.10401925, -0.10653082, 0.38469055, -0.88302094, -0.80197543, - 0.03548668, 0.02775662, -0.54374295, 0.03379983, 0.00923623, 0.29320273, -1.05263519, - -0.93360096, 0.03778313, 0.12360487, -0.56437284, 0.0644429, 0.33432651, 0.36450726, - -1.22978747, -0.83822101, -0.18796451, 0.34888434, -0.3801491, -0.45327303, -0.59747899, - 0.39697698, -0.15616602, -0.06159166, -0.40301991, -0.11725303, -0.11913263, -0.12406619, - -0.11227967, 0.43083835, -0.90535849, -0.81646025, 0.10012121, -0.0141237, -0.63747931, - 0.04805023, 0.34190539, 0.50725192, -1.17861414, -0.74641538, -0.09333111, 0.27992678, - -0.56214809, 0.04970971, 0.36249384, 0.57705611, -1.16913795, -0.69849908, 0.10957897, - 0.27983218, -0.62088525, 0.0410459, 0.23973398, 0.40960434, -1.14183664, -0.83321381, - 0.02149482, 0.21720445, -0.49869928, -0.95655465, -0.51680422, 0.45761383, -0.08351214, - -0.12151554, 0.00819737, -0.20813803, -0.01055793, 0.25319234, 0.36154974, 0.1822421, - -1.15837133, -0.92209691, -0.0501582, 0.08535917, -0.54003763, -1.08675635, -1.04009593, - 0.09408128, 0.07009826, -0.01762833, -0.19180447, -0.18029785, -0.20342001, 0.04034991, - 0.1814747, 0.36906669, -1.13532007, -0.8852452, 0.0782818, 0.16825101, -0.50301319, - -0.29128098, -0.65341312, 0.51484352, -0.38758236, -0.22531103, -0.55021971, 0.10804344, - -0.3521522, -0.38849035, -0.74110794, 0.53761131, -0.25142813, -0.1118066, -0.47453368, - 0.06347904, -0.23796193, -1.02682328, -0.47594091, 0.39515916, -0.2782529, -0.16566519, - 0.08063579, 0.00810116, -0.06213913, -1.059654, -0.62496334, 0.53698546, -0.11806234, - 0.00356161, 0.11513405, -0.14213292, 0.04102662, -0.36622161, -0.73686272, 0.48323864, - -0.27338892, -0.14203401, -0.41736352, 0.03332564, -0.21907479, -0.06396769, 0.01831361, - 0.46263444, -1.01878166, -0.86486858, 0.17622118, -0.01249686, -0.74530888, -0.9354887, - -0.5027945, 0.38170099, -0.15547098, 0.00677824, -0.04677663, -0.13541745, 0.07253501, - -0.97933143, -0.58001202, 0.48235369, -0.18836913, -0.02430783, 0.07572441, -0.08101331, - 0.00630076, -0.16881248, -0.67989182, 0.46083611, -0.43910736, -0.29321918, -0.38735861, - 0.07669903, -0.29749861, -0.40047669, -0.56722462, 0.33168188, -0.13118173, -0.06672747, - -0.56856316, -0.26269144, -0.14236671, 0.10651901, 0.4962585, 0.38848072, -1.06653547, - -0.64079332, -0.47378591, 0.43195483, -0.04856951, -0.9840439, -0.70610428, 0.34028092, - -0.2089237, -0.05382041, 0.01625874, -0.02080803, -0.12535211, -0.04146428, -1.24533033, - 0.48944879, 0.0578458, 0.26708388, -0.90321028, 0.35377088, -0.36791429, -0.35382384, - -0.52748734, 0.42854419, -0.31744713, -0.19174226, -0.39073724, -0.03258846, -0.19978228, - -0.36185205, -0.57412046, 0.43681973, -0.25414538, -0.12904905, -0.46334973, -0.03123853, - -0.11303604, -0.87073672, -0.45441297, 0.41825858, -0.25303507, -0.21845073, 0.10248682, - -0.11045569, -0.10002795, -0.00572806, 0.16519061, 0.42651513, -1.11417019, -0.83789682, - 0.02995787, 0.16843079, -0.53874511, 0.03056994, 0.17877036, 0.49632853, -1.03276777, - -0.74778616, -0.03971953, 0.10907949, -0.67385727, -0.9523471, -0.56550741, 0.40409449, - -0.2703723, -0.10175014, 0.13605487, -0.06306008, -0.01768126, -0.4749442, -0.56964815, - 0.39389887, -0.19248079, -0.04161081, -0.38728487, -0.20341556, -0.12656988, -0.35949609, - -0.46137866, 0.28798422, -0.06603147, -0.04363992, -0.60343552, -0.23565227, -0.10242701, - -0.06792886, 0.09689897, 0.33259571, -0.98854214, -0.84444433, 0.00673901, 0.13457057, - -0.43145794, -0.51500046, -0.50821936, 0.38000089, 0.0132636, 0.0580942, -0.40157595, - -0.11967677, 0.02549113, -0.10350953, 0.22918226, 0.40411913, -1.05619383, -0.71218503, - -0.02197581, 0.26422262, -0.34765676, 0.06601537, 0.21712676, 0.34723559, -1.20982027, - -0.95646334, 0.00793948, 0.27620381, -0.43475035, -0.67326003, -0.6137197, 0.43724492, - -0.17666136, -0.06591748, -0.18937394, -0.07400128, -0.06881691, -0.5201112, -0.61088628, - 0.4225319, -0.18969463, -0.06921366, -0.33993208, -0.06990873, -0.10288513, -0.70659858, - -0.56003648, 0.46628812, -0.16090363, -0.0185108, -0.1431348, -0.1128775, -0.0078648, - -0.02323332, 0.04292452, 0.39291084, -0.94897962, -0.63863206, -0.16546988, 0.23698957, - -0.30633628}; - - auto stream = resource::get_cuda_stream(handle); - - d_X.resize(X.size(), stream); - d_X_embedded.resize(X_embedded.size(), stream); - raft::update_device(d_X.data(), X.data(), X.size(), stream); - raft::update_device(d_X_embedded.data(), X_embedded.data(), X_embedded.size(), stream); - auto n_sample = 50; - auto n_features_origin = 30; - auto n_features_embedded = 8; - - // euclidean test - score = trustworthiness_score( - handle, - raft::make_device_matrix_view(d_X.data(), n_sample, n_features_origin), - raft::make_device_matrix_view( - d_X_embedded.data(), n_sample, n_features_embedded), - 5); - } - - void SetUp() override { basicTest(); } - - void TearDown() override {} - - protected: - raft::resources handle; - - rmm::device_uvector d_X; - rmm::device_uvector d_X_embedded; - - double score; -}; - -typedef TrustworthinessScoreTest TrustworthinessScoreTestF; -TEST_F(TrustworthinessScoreTestF, Result) { ASSERT_TRUE(0.9375 < score && score < 0.9379); } -}; // namespace stats -}; // namespace raft diff --git a/docs/source/build.md b/docs/source/build.md index 3d059d5a69..5a0dbf7e11 100644 --- a/docs/source/build.md +++ b/docs/source/build.md @@ -31,7 +31,6 @@ Both the C++ and Python APIs require CMake to build from source. The easiest way to install RAFT is through conda and several packages are provided. - `libraft-headers` C++ headers -- `libraft` (optional) C++ shared library containing pre-compiled template instantiations and runtime API. - `pylibraft` (optional) Python library - `raft-dask` (optional) Python library for deployment of multi-node multi-GPU algorithms that use the RAFT `raft::comms` abstraction layer in Dask clusters. @@ -48,14 +47,12 @@ mamba install -c rapidsai -c conda-forge -c nvidia raft-dask pylibraft cuda-vers Note that the above commands will also install `libraft-headers` and `libraft`. -You can also install the conda packages individually using the `mamba` command above. For example, if you'd like to install RAFT's headers and pre-compiled shared library to use in your project: +You can also install the conda packages individually using the `mamba` command above. For example, if you'd like to install RAFT's headers to use in your project: ```bash # for CUDA 12.0 -mamba install -c rapidsai -c conda-forge -c nvidia libraft libraft-headers cuda-version=12.0 +mamba install -c rapidsai -c conda-forge -c nvidia libraft-headers cuda-version=12.0 ``` -If installing the C++ APIs Please see [using libraft](https://docs.rapids.ai/api/raft/nightly/using_libraft/) for more information on using the pre-compiled shared library. You can also refer to the [example C++ template project](https://github.com/rapidsai/raft/tree/branch-24.12/cpp/template) for a ready-to-go CMake configuration that you can drop into your project and build against installed RAFT development artifacts above. - ## Installing Python through Pip `pylibraft` and `raft-dask` both have packages that can be [installed through pip](https://rapids.ai/pip.html#install). @@ -72,20 +69,18 @@ pip install pylibraft-cu12 --extra-index-url=https://pypi.nvidia.com pip install raft-dask-cu12 --extra-index-url=https://pypi.nvidia.com ``` -These packages statically build RAFT's pre-compiled instantiations, so the C++ headers and pre-compiled shared library won't be readily available to use in your code. - ## Building C++ and Python from source ### CUDA/GPU Requirements - cmake 3.26.4+ - GCC 9.3+ (9.5.0+ recommended) -- CUDA Toolkit 11.2+ +- CUDA Toolkit 11.8+ - NVIDIA driver 450.80.02+ -- Pascal architecture or better (compute capability >= 6.0) +- Volta architecture or better (compute capability >= 7.0) ### Build Dependencies -In addition to the libraries included with cudatoolkit 11.0+, there are some other dependencies below for building RAFT from source. Many of the dependencies are optional and depend only on the primitives being used. All of these can be installed with cmake or [rapids-cpm](https://github.com/rapidsai/rapids-cmake#cpm) and many of them can be installed with [conda](https://anaconda.org). +In addition to the libraries included with cudatoolkit 11.8+, there are some other dependencies below for building RAFT from source. Many of the dependencies are optional and depend only on the primitives being used. All of these can be installed with cmake or [rapids-cpm](https://github.com/rapidsai/rapids-cmake#cpm) and many of them can be installed with [conda](https://anaconda.org). #### Required - [RMM](https://github.com/rapidsai/rmm) corresponding to RAFT version. @@ -108,9 +103,9 @@ mamba env create --name rapids_raft -f conda/environments/all_cuda-125_arch-x86_ mamba activate rapids_raft ``` -All of RAFT's C++ APIs can be used header-only and optional pre-compiled shared libraries provide some host-accessible runtime APIs and template instantiations to accelerate compile times. +All of RAFT's C++ APIs can be used header-only. -The process for building from source with CUDA 11 differs slightly in that your host system will also need to have CUDA toolkit installed which is greater than, or equal to, the version you install into you conda environment. Installing CUDA toolkit into your host system is necessary because `nvcc` is not provided with Conda's cudatoolkit dependencies for CUDA 11. The following example will install create and install dependencies for a CUDA 11.8 conda environment +The process for building from source with CUDA 11 differs slightly in that your host system will also need to have CUDA toolkit installed which is greater than, or equal to, the version you install into you conda environment. Installing CUDA toolkit into your host system is necessary because `nvcc` is not provided with Conda's cudatoolkit dependencies for CUDA 11. The following example will install create and install dependencies for a CUDA 11.8 conda environment: ```bash mamba env create --name rapids_raft -f conda/environments/all_cuda-118_arch-x86_64.yaml mamba activate rapids_raft @@ -138,7 +133,7 @@ Once installed, `libraft` headers (and dependencies which were downloaded and in ### C++ Shared Library (optional) -A shared library can be built for speeding up compile times. The shared library also contains a runtime API that allows you to invoke RAFT APIs directly from C++ source files (without `nvcc`). The shared library can also significantly improve re-compile times both while developing RAFT and using its APIs to develop applications. Pass the `--compile-lib` flag to `build.sh` to build the library: +A shared library must be built in order to build `pylibraft`. Pass the `--compile-lib` flag to `build.sh` to build the library: ```bash ./build.sh libraft --compile-lib ``` @@ -167,23 +162,17 @@ Compile the tests using the `tests` target in `build.sh`. ./build.sh libraft tests ``` -Test compile times can be improved significantly by using the optional shared libraries. If installed, they will be used automatically when building the tests but `--compile-libs` can be used to add additional compilation units and compile them with the tests. - -```bash -./build.sh libraft tests --compile-lib -``` - The tests are broken apart by algorithm category, so you will find several binaries in `cpp/build/` named `*_TEST`. For example, to run the distance tests: ```bash -./cpp/build/DISTANCE_TEST +./cpp/build/MATRIX_TEST ``` It can take sometime to compile all of the tests. You can build individual tests by providing a semicolon-separated list to the `--limit-tests` option in `build.sh`: ```bash -./build.sh libraft tests -n --limit-tests=NEIGHBORS_TEST;DISTANCE_TEST;MATRIX_TEST +./build.sh libraft tests -n --limit-tests=CORE_TEST;MATRIX_TEST ``` ### C++ Primitives Microbenchmarks @@ -196,7 +185,7 @@ The benchmarks are broken apart by algorithm category, so you will find several It can take sometime to compile all of the benchmarks. You can build individual benchmarks by providing a semicolon-separated list to the `--limit-bench-prims` option in `build.sh`: ```bash -./build.sh libraft bench-prims -n --limit-bench=NEIGHBORS_PRIMS_BENCH;DISTANCE_PRIMS_BENCH;LINALG_PRIMS_BENCH +./build.sh libraft bench-prims -n --limit-bench=NEIGHBORS_PRIMS_BENCH;MATRIX_PRIMS_BENCH;LINALG_PRIMS_BENCH ``` ### Python libraries @@ -301,8 +290,6 @@ PROPERTIES CXX_STANDARD 17 INTERFACE_POSITION_INDEPENDENT_CODE ON) ``` -The [C++ example template project](https://github.com/rapidsai/raft/tree/HEAD/cpp/template) provides an end-to-end buildable example of what a `CMakeLists.txt` that uses RAFT should look like. The items below point out some of the needed details. - #### CMake Targets The `raft::raft` CMake target is made available when including RAFT into your CMake project but additional CMake targets can be made available by adding to the `COMPONENTS` option in CMake's `find_package(raft)` (refer to [CMake docs](https://cmake.org/cmake/help/latest/command/find_package.html#basic-signature) to learn more). The components should be separated by spaces. The `raft::raft` target will always be available. Note that the `distributed` component also exports additional dependencies. diff --git a/docs/source/cpp_api.rst b/docs/source/cpp_api.rst index e60ef4e697..74f706bf46 100644 --- a/docs/source/cpp_api.rst +++ b/docs/source/cpp_api.rst @@ -8,13 +8,10 @@ C++ API :maxdepth: 4 cpp_api/core.rst - cpp_api/cluster.rst - cpp_api/distance.rst cpp_api/linalg.rst cpp_api/matrix.rst cpp_api/mdspan.rst cpp_api/mnmg.rst - cpp_api/neighbors.rst cpp_api/random.rst cpp_api/solver.rst cpp_api/sparse.rst diff --git a/docs/source/cpp_api/cluster.rst b/docs/source/cpp_api/cluster.rst deleted file mode 100644 index b0485992b3..0000000000 --- a/docs/source/cpp_api/cluster.rst +++ /dev/null @@ -1,18 +0,0 @@ -Cluster -======= - -This page provides C++ API references for the publicly-exposed elements of the `raft/cluster` headers. RAFT provides -fundamental clustering algorithms which are, themselves, considered reusable building blocks for other algorithms. - -.. role:: py(code) - :language: c++ - :class: highlight - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - cluster_kmeans.rst - cluster_kmeans_balanced.rst - cluster_slhc.rst - cluster_spectral.rst \ No newline at end of file diff --git a/docs/source/cpp_api/cluster_kmeans.rst b/docs/source/cpp_api/cluster_kmeans.rst deleted file mode 100644 index fa040ddc18..0000000000 --- a/docs/source/cpp_api/cluster_kmeans.rst +++ /dev/null @@ -1,13 +0,0 @@ -K-Means -======= - -.. role:: py(code) - :language: c++ - :class: highlight - -``#include `` - -.. doxygennamespace:: raft::cluster::kmeans - :project: RAFT - :members: - :content-only: diff --git a/docs/source/cpp_api/cluster_kmeans_balanced.rst b/docs/source/cpp_api/cluster_kmeans_balanced.rst deleted file mode 100644 index 5d07fcc1e3..0000000000 --- a/docs/source/cpp_api/cluster_kmeans_balanced.rst +++ /dev/null @@ -1,13 +0,0 @@ -K-Means -======= - -.. role:: py(code) - :language: c++ - :class: highlight - -``#include `` - -.. doxygennamespace:: raft::cluster::kmeans_balanced - :project: RAFT - :members: - :content-only: diff --git a/docs/source/cpp_api/cluster_slhc.rst b/docs/source/cpp_api/cluster_slhc.rst deleted file mode 100644 index fc45ae699a..0000000000 --- a/docs/source/cpp_api/cluster_slhc.rst +++ /dev/null @@ -1,13 +0,0 @@ -Hierarchical Clustering -======================= - -.. role:: py(code) - :language: c++ - :class: highlight - -``#include `` - -.. doxygennamespace:: raft::cluster::hierarchy - :project: RAFT - :members: - :content-only: diff --git a/docs/source/cpp_api/cluster_spectral.rst b/docs/source/cpp_api/cluster_spectral.rst deleted file mode 100644 index a71f431ab8..0000000000 --- a/docs/source/cpp_api/cluster_spectral.rst +++ /dev/null @@ -1,13 +0,0 @@ -Spectral Clustering -=================== - -.. role:: py(code) - :language: c++ - :class: highlight - -``#include `` - -.. doxygennamespace:: raft::spectral - :project: RAFT - :members: - :content-only: diff --git a/docs/source/cpp_api/distance.rst b/docs/source/cpp_api/distance.rst deleted file mode 100644 index fd81295def..0000000000 --- a/docs/source/cpp_api/distance.rst +++ /dev/null @@ -1,28 +0,0 @@ -Distance -======== - -This page provides C++ class references for the publicly-exposed elements of the `raft/distance` package. RAFT's -distances have been highly optimized and support a wide assortment of different distance measures. - -.. role:: py(code) - :language: c++ - :class: highlight - -Distance Types --------------- - -``#include `` - -namespace *raft::distance* - -.. doxygenenum:: raft::distance::DistanceType - :project: RAFT - - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - distance_pairwise.rst - distance_1nn.rst - diff --git a/docs/source/cpp_api/distance_1nn.rst b/docs/source/cpp_api/distance_1nn.rst deleted file mode 100644 index 8c1c00d6c9..0000000000 --- a/docs/source/cpp_api/distance_1nn.rst +++ /dev/null @@ -1,24 +0,0 @@ -1-Nearest Neighbors -=================== - -.. role:: py(code) - :language: c++ - :class: highlight - -``#include `` - -namespace *raft::distance* - -.. doxygengroup:: fused_l2_nn - :project: RAFT - :members: - :content-only: - -``#include `` -namespace *raft::distance* - -.. doxygengroup:: masked_nn - :project: RAFT - :members: - :content-only: - diff --git a/docs/source/cpp_api/distance_pairwise.rst b/docs/source/cpp_api/distance_pairwise.rst deleted file mode 100644 index 2a9c9a92f5..0000000000 --- a/docs/source/cpp_api/distance_pairwise.rst +++ /dev/null @@ -1,17 +0,0 @@ -Pairwise Distance -================= - -.. role:: py(code) - :language: c++ - :class: highlight - -``#include `` - -namespace *raft::distance* - -.. doxygengroup:: distance_mdspan - :project: RAFT - :members: - :content-only: - - diff --git a/docs/source/cpp_api/matrix.rst b/docs/source/cpp_api/matrix.rst index 17953bc128..c983faa9db 100644 --- a/docs/source/cpp_api/matrix.rst +++ b/docs/source/cpp_api/matrix.rst @@ -17,4 +17,3 @@ headers cover many operations on matrices that are otherwise not covered by `raf matrix_manipulation.rst matrix_ordering.rst matrix_reduction.rst - matrix_selection.rst \ No newline at end of file diff --git a/docs/source/cpp_api/matrix_selection.rst b/docs/source/cpp_api/matrix_selection.rst deleted file mode 100644 index 4842a75e0e..0000000000 --- a/docs/source/cpp_api/matrix_selection.rst +++ /dev/null @@ -1,69 +0,0 @@ -Matrix Selection -================ - -.. role:: py(code) - :language: c++ - :class: highlight - - -Copy ----- - -``#include `` - -namespace *raft::matrix* - -.. doxygengroup:: matrix_copy - :project: RAFT - :members: - :content-only: - -Diagonal --------- - -``#include `` - -namespace *raft::matrix* - -.. doxygengroup:: matrix_diagonal - :project: RAFT - :members: - :content-only: - - -Gather ------- - -``#include `` - -namespace *raft::matrix* - -.. doxygengroup:: matrix_gather - :project: RAFT - :members: - :content-only: - - -Slicing -------- - -``#include `` - -namespace *raft::matrix* - -.. doxygengroup:: matrix_slice - :project: RAFT - :members: - :content-only: - -Triangular ----------- - -``#include `` - -namespace *raft::matrix* - -.. doxygengroup:: matrix_triangular - :project: RAFT - :members: - :content-only: diff --git a/docs/source/cpp_api/neighbors.rst b/docs/source/cpp_api/neighbors.rst deleted file mode 100644 index 876f68b1bf..0000000000 --- a/docs/source/cpp_api/neighbors.rst +++ /dev/null @@ -1,19 +0,0 @@ -Neighbors -========= - -This page provides C++ class references for the publicly-exposed elements of the neighbors package. - -.. role:: py(code) - :language: c++ - :class: highlight - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - neighbors_brute_force.rst - neighbors_ivf_flat.rst - neighbors_ivf_pq.rst - neighbors_epsilon_neighborhood.rst - neighbors_ball_cover.rst - neighbors_cagra.rst \ No newline at end of file diff --git a/docs/source/cpp_api/neighbors_ball_cover.rst b/docs/source/cpp_api/neighbors_ball_cover.rst deleted file mode 100644 index 85bd6b2d8e..0000000000 --- a/docs/source/cpp_api/neighbors_ball_cover.rst +++ /dev/null @@ -1,17 +0,0 @@ -Random Ball Cover -================= - -.. role:: py(code) - :language: c++ - :class: highlight - -``#include `` - -namespace *raft::neighbors::ball_cover* - -.. doxygengroup:: random_ball_cover - :project: RAFT - :members: - :content-only: - - diff --git a/docs/source/cpp_api/neighbors_brute_force.rst b/docs/source/cpp_api/neighbors_brute_force.rst deleted file mode 100644 index 525addf428..0000000000 --- a/docs/source/cpp_api/neighbors_brute_force.rst +++ /dev/null @@ -1,18 +0,0 @@ -Brute-Force -=========== - -.. role:: py(code) - :language: c++ - :class: highlight - - -``#include `` - -namespace *raft::neighbors::brute_force* - -.. doxygengroup:: brute_force_knn - :project: RAFT - :members: - :content-only: - - diff --git a/docs/source/cpp_api/neighbors_cagra.rst b/docs/source/cpp_api/neighbors_cagra.rst deleted file mode 100644 index 99ecd3a985..0000000000 --- a/docs/source/cpp_api/neighbors_cagra.rst +++ /dev/null @@ -1,31 +0,0 @@ -CAGRA -===== - -CAGRA is a graph-based nearest neighbors implementation with state-of-the art query performance for both small- and large-batch sized search. - -Please note that the CAGRA implementation is currently experimental and the API is subject to change from release to release. We are currently working on promoting CAGRA to a top-level stable API within RAFT. - -.. role:: py(code) - :language: c++ - :class: highlight - -``#include `` - -namespace *raft::neighbors::cagra* - -.. doxygengroup:: cagra - :project: RAFT - :members: - :content-only: - - -Serializer Methods ------------------- -``#include `` - -namespace *raft::neighbors::cagra* - -.. doxygengroup:: cagra_serialize - :project: RAFT - :members: - :content-only: diff --git a/docs/source/cpp_api/neighbors_epsilon_neighborhood.rst b/docs/source/cpp_api/neighbors_epsilon_neighborhood.rst deleted file mode 100644 index f291a7605f..0000000000 --- a/docs/source/cpp_api/neighbors_epsilon_neighborhood.rst +++ /dev/null @@ -1,15 +0,0 @@ -Epsilon Neighborhood -==================== - -.. role:: py(code) - :language: c++ - :class: highlight - -``#include `` - -namespace *raft::neighbors::epsilon_neighborhood* - -.. doxygengroup:: epsilon_neighbors - :project: RAFT - :members: - :content-only: diff --git a/docs/source/cpp_api/neighbors_hnsw.rst b/docs/source/cpp_api/neighbors_hnsw.rst deleted file mode 100644 index 86f9544c35..0000000000 --- a/docs/source/cpp_api/neighbors_hnsw.rst +++ /dev/null @@ -1,29 +0,0 @@ -HNSW -===== - -HNSW is a graph-based nearest neighbors implementation for the CPU. -This implementation provides the ability to serialize a CAGRA graph and read it as a base-layer-only hnswlib graph. - -.. role:: py(code) - :language: c++ - :class: highlight - -``#include `` - -namespace *raft::neighbors::hnsw* - -.. doxygengroup:: hnsw - :project: RAFT - :members: - :content-only: - -Serializer Methods ------------------- -``#include `` - -namespace *raft::neighbors::hnsw* - -.. doxygengroup:: hnsw_serialize - :project: RAFT - :members: - :content-only: diff --git a/docs/source/cpp_api/neighbors_ivf_flat.rst b/docs/source/cpp_api/neighbors_ivf_flat.rst deleted file mode 100644 index aa03fa8a80..0000000000 --- a/docs/source/cpp_api/neighbors_ivf_flat.rst +++ /dev/null @@ -1,37 +0,0 @@ -IVF-Flat -======== - -.. role:: py(code) - :language: c++ - :class: highlight - -``#include `` - -namespace *raft::neighbors::ivf_flat* - -.. doxygengroup:: ivf_flat - :project: RAFT - :members: - :content-only: - -Serializer Methods ------------------- -``#include `` - -namespace *raft::neighbors::ivf_flat* - -.. doxygengroup:: ivf_flat_serialize - :project: RAFT - :members: - :content-only: - -Helper Methods --------------- -``#include `` - -namespace *raft::neighbors::ivf_flat::helpers* - -.. doxygengroup:: ivf_flat_helpers - :project: RAFT - :members: - :content-only: \ No newline at end of file diff --git a/docs/source/cpp_api/neighbors_ivf_pq.rst b/docs/source/cpp_api/neighbors_ivf_pq.rst deleted file mode 100644 index 5301105c56..0000000000 --- a/docs/source/cpp_api/neighbors_ivf_pq.rst +++ /dev/null @@ -1,48 +0,0 @@ -IVF-PQ -====== - -.. role:: py(code) - :language: c++ - :class: highlight - -``#include `` - -namespace *raft::neighbors::ivf_pq* - -.. doxygengroup:: ivf_pq - :project: RAFT - :members: - :content-only: - -Serializer Methods ------------------- -``#include `` - -namespace *raft::neighbors::ivf_pq* - -.. doxygengroup:: ivf_pq_serialize - :project: RAFT - :members: - :content-only: - -Candidate Refinement --------------------- -``#include `` - -namespace *raft::neighbors* - -.. doxygengroup:: ann_refine - :project: RAFT - :members: - :content-only: - -Helper Methods --------------- -``#include `` - -namespace *raft::neighbors::ivf_pq::helpers* - -.. doxygengroup:: ivf_pq_helpers - :project: RAFT - :members: - :content-only: \ No newline at end of file diff --git a/docs/source/cpp_api/sparse.rst b/docs/source/cpp_api/sparse.rst index 9ad7b6aeda..64197accaf 100644 --- a/docs/source/cpp_api/sparse.rst +++ b/docs/source/cpp_api/sparse.rst @@ -13,9 +13,7 @@ Core to RAFT's computational patterns for sparse data is its vocabulary of spars :caption: Contents: sparse_types.rst - sparse_distance.rst sparse_linalg.rst sparse_matrix.rst - sparse_neighbors.rst sparse_solver.rst diff --git a/docs/source/cpp_api/sparse_distance.rst b/docs/source/cpp_api/sparse_distance.rst deleted file mode 100644 index e85e43695d..0000000000 --- a/docs/source/cpp_api/sparse_distance.rst +++ /dev/null @@ -1,7 +0,0 @@ -Sparse Distance -=============== - -.. doxygennamespace:: raft::sparse::distance - :project: RAFT - :members: - :content-only: diff --git a/docs/source/cpp_api/sparse_neighbors.rst b/docs/source/cpp_api/sparse_neighbors.rst deleted file mode 100644 index 9610913da6..0000000000 --- a/docs/source/cpp_api/sparse_neighbors.rst +++ /dev/null @@ -1,7 +0,0 @@ -Sparse Neighbors -================ - -.. doxygennamespace:: raft::sparse::neighbors - :project: RAFT - :members: - :content-only: diff --git a/docs/source/cpp_api/stats.rst b/docs/source/cpp_api/stats.rst index fd23ce2149..b1eca53e0b 100644 --- a/docs/source/cpp_api/stats.rst +++ b/docs/source/cpp_api/stats.rst @@ -16,4 +16,3 @@ This page provides C++ class references for the publicly-exposed elements of the stats_regression.rst stats_classification.rst stats_clustering.rst - stats_neighborhood.rst diff --git a/docs/source/cpp_api/stats_neighborhood.rst b/docs/source/cpp_api/stats_neighborhood.rst deleted file mode 100644 index 7c7ad90a49..0000000000 --- a/docs/source/cpp_api/stats_neighborhood.rst +++ /dev/null @@ -1,30 +0,0 @@ -Neighborhood Model Scoring -========================== - -.. role:: py(code) - :language: c++ - :class: highlight - -Trustworthiness ---------------- - -``#include `` - -namespace *raft::stats* - -.. doxygengroup:: stats_trustworthiness - :project: RAFT - :members: - :content-only: - -Neighborhood Recall -------------------- - -``#include `` - -namespace *raft::stats* - -.. doxygengroup:: stats_neighborhood_recall - :project: RAFT - :members: - :content-only: diff --git a/docs/source/index.rst b/docs/source/index.rst index 46ebd1b737..21be1bafbd 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -11,7 +11,6 @@ Useful Resources .. _raft_reference: https://docs.rapids.ai/api/raft/stable/ -- `Example Notebooks `_: Example Jupyter notebooks - `RAPIDS Community `_: Get help, contribute, and collaborate. - `GitHub repository `_: Download the RAFT source code. - `Issue tracker `_: Report issues or request features. @@ -36,16 +35,12 @@ While not exhaustive, the following general categories help summarize the accele * - Category - Examples - * - Nearest Neighbors - - pairwise distances, vector search, epsilon neighborhoods, neighborhood graph construction * - Data Formats - sparse & dense, conversions, data generation * - Dense Operations - linear algebra, matrix and vector operations, slicing, norms, factorization, least squares, svd & eigenvalue problems * - Sparse Operations - linear algebra, eigenvalue problems, slicing, norms, reductions, factorization, symmetrization, components & labeling - * - Basic Clustering - - spectral clustering, hierarchical clustering, k-means * - Solvers - combinatorial optimization, iterative solvers * - Statistics @@ -61,8 +56,6 @@ While not exhaustive, the following general categories help summarize the accele build.md cpp_api.rst pylibraft_api.rst - using_libraft.md - vector_search_tutorial.md raft_dask_api.rst using_raft_comms.rst developer_guide.md diff --git a/docs/source/pylibraft_api.rst b/docs/source/pylibraft_api.rst index df25b76985..aaa359e646 100644 --- a/docs/source/pylibraft_api.rst +++ b/docs/source/pylibraft_api.rst @@ -7,9 +7,5 @@ Python API .. toctree:: :maxdepth: 4 - pylibraft_api/cluster.rst pylibraft_api/common.rst - pylibraft_api/distance.rst - pylibraft_api/matrix.rst - pylibraft_api/neighbors.rst pylibraft_api/random.rst diff --git a/docs/source/pylibraft_api/cluster.rst b/docs/source/pylibraft_api/cluster.rst deleted file mode 100644 index 085297fe34..0000000000 --- a/docs/source/pylibraft_api/cluster.rst +++ /dev/null @@ -1,20 +0,0 @@ -Cluster -======= - -This page provides pylibraft class references for the publicly-exposed elements of the `pylibraft.cluster` package. - -.. role:: py(code) - :language: python - :class: highlight - -KMeans -###### - -.. autoclass:: pylibraft.cluster.kmeans.KMeansParams - :members: - -.. autofunction:: pylibraft.cluster.kmeans.fit - -.. autofunction:: pylibraft.cluster.kmeans.cluster_cost - -.. autofunction:: pylibraft.cluster.kmeans.compute_new_centroids diff --git a/docs/source/pylibraft_api/distance.rst b/docs/source/pylibraft_api/distance.rst deleted file mode 100644 index d14ed6fc08..0000000000 --- a/docs/source/pylibraft_api/distance.rst +++ /dev/null @@ -1,15 +0,0 @@ -Distance -======== - -This page provides `pylibraft` class references for the publicly-exposed elements of the `pylibraft.distance` package. RAFT's -distances have been highly optimized and support a wide assortment of different distance measures. - - -.. role:: py(code) - :language: python - :class: highlight - -.. autofunction:: pylibraft.distance.pairwise_distance - -.. autofunction:: pylibraft.distance.fused_l2_nn_argmin - diff --git a/docs/source/pylibraft_api/matrix.rst b/docs/source/pylibraft_api/matrix.rst deleted file mode 100644 index 884a466ec1..0000000000 --- a/docs/source/pylibraft_api/matrix.rst +++ /dev/null @@ -1,11 +0,0 @@ -Matrix -====== - -This page provides `pylibraft` class references for the publicly-exposed elements of the `pylibraft.matrix` package. - - -.. role:: py(code) - :language: python - :class: highlight - -.. autofunction:: pylibraft.matrix.select_k diff --git a/docs/source/pylibraft_api/neighbors.rst b/docs/source/pylibraft_api/neighbors.rst deleted file mode 100644 index e9e890fccb..0000000000 --- a/docs/source/pylibraft_api/neighbors.rst +++ /dev/null @@ -1,99 +0,0 @@ -Neighbors -========= - -This page provides pylibraft class references for the publicly-exposed elements of the neighbors package. - -.. role:: py(code) - :language: python - :class: highlight - - -Brute Force -########### - -.. autofunction:: pylibraft.neighbors.brute_force.knn - - -CAGRA -##### - -.. autoclass:: pylibraft.neighbors.cagra.IndexParams - :members: - -.. autofunction:: pylibraft.neighbors.cagra.build - -.. autoclass:: pylibraft.neighbors.cagra.SearchParams - :members: - -.. autofunction:: pylibraft.neighbors.cagra.search - -Serializer Methods ------------------- -.. autofunction:: pylibraft.neighbors.cagra.save - -.. autofunction:: pylibraft.neighbors.cagra.load - -HNSW -#### - -.. autoclass:: pylibraft.neighbors.hnsw.SearchParams - :members: - -.. autofunction:: pylibraft.neighbors.hnsw.from_cagra - -.. autofunction:: pylibraft.neighbors.hnsw.search - -Serializer Methods ------------------- -.. autofunction:: pylibraft.neighbors.hnsw.save - -.. autofunction:: pylibraft.neighbors.hnsw.load - -IVF-Flat -######## - -.. autoclass:: pylibraft.neighbors.ivf_flat.IndexParams - :members: - -.. autofunction:: pylibraft.neighbors.ivf_flat.build - -.. autofunction:: pylibraft.neighbors.ivf_flat.extend - -.. autoclass:: pylibraft.neighbors.ivf_flat.SearchParams - :members: - -.. autofunction:: pylibraft.neighbors.ivf_flat.search - -Serializer Methods ------------------- - -.. autofunction:: pylibraft.neighbors.ivf_flat.save - -.. autofunction:: pylibraft.neighbors.ivf_flat.load - -IVF-PQ -###### - -.. autoclass:: pylibraft.neighbors.ivf_pq.IndexParams - :members: - -.. autofunction:: pylibraft.neighbors.ivf_pq.build - -.. autofunction:: pylibraft.neighbors.ivf_pq.extend - -.. autoclass:: pylibraft.neighbors.ivf_pq.SearchParams - :members: - -.. autofunction:: pylibraft.neighbors.ivf_pq.search - -Serializer Methods ------------------- - -.. autofunction:: pylibraft.neighbors.ivf_pq.save - -.. autofunction:: pylibraft.neighbors.ivf_pq.load - -Candidate Refinement --------------------- - -.. autofunction:: pylibraft.neighbors.refine diff --git a/docs/source/quick_start.md b/docs/source/quick_start.md index 3909b40f20..309e801597 100644 --- a/docs/source/quick_start.md +++ b/docs/source/quick_start.md @@ -23,7 +23,7 @@ auto vector = raft::make_device_vector(handle, n_cols); auto matrix = raft::make_device_matrix(handle, n_rows, n_cols); ``` -The `mdspan` is a lightweight non-owning view that can wrap around any pointer, maintaining shape, layout, and indexing information for accessing elements. +The `mdspan` is a lightweight non-owning view that can wrap around any pointer, maintaining shape, layout, and indexing information for accessing elements. We can construct `mdspan` instances directly from the above `mdarray` instances: @@ -95,13 +95,13 @@ auto vector = raft::make_device_vector_view(vector_ptr, raft::make_vector_stride Most of the primitives in RAFT accept a `raft::handle_t` object for the management of resources which are expensive to create, such CUDA streams, stream pools, and handles to other CUDA libraries like `cublas` and `cusolver`. The example below demonstrates creating a RAFT handle and using it with `device_matrix` and `device_vector` to allocate memory, generating random clusters, and computing -pairwise Euclidean distances: +pairwise Euclidean distances with [NVIDIA cuVS](https://github.com/rapidsai/cuvs): ```c++ #include #include #include -#include +#include raft::handle_t handle; @@ -114,20 +114,20 @@ auto output = raft::make_device_matrix(handle, n_samples, n_samples); raft::random::make_blobs(handle, input.view(), labels.view()); -auto metric = raft::distance::DistanceType::L2SqrtExpanded; -raft::distance::pairwise_distance(handle, input.view(), input.view(), output.view(), metric); +auto metric = cuvs::distance::DistanceType::L2SqrtExpanded; +cuvs::distance::pairwise_distance(handle, input.view(), input.view(), output.view(), metric); ``` ## Python Example The `pylibraft` package contains a Python API for RAFT algorithms and primitives. `pylibraft` integrates nicely into other libraries by being very lightweight with minimal dependencies and accepting any object that supports the `__cuda_array_interface__`, such as [CuPy's ndarray](https://docs.cupy.dev/en/stable/user_guide/interoperability.html#rmm). The number of RAFT algorithms exposed in this package is continuing to grow from release to release. -The example below demonstrates computing the pairwise Euclidean distances between CuPy arrays. Note that CuPy is not a required dependency for `pylibraft`. +The example below demonstrates computing the pairwise Euclidean distances between CuPy arrays with the [NVIDIA cuVS](https://github.com/rapidsai/cuvs) library. Note that CuPy is not a required dependency for `pylibraft`. ```python import cupy as cp -from pylibraft.distance import pairwise_distance +from cuvs.distance import pairwise_distance n_samples = 5000 n_features = 50 @@ -170,7 +170,7 @@ pylibraft.config.set_output_as(lambda device_ndarray: return device_ndarray.copy ```python import cupy as cp -from pylibraft.distance import pairwise_distance +from cuvs.distance import pairwise_distance n_samples = 5000 n_features = 50 diff --git a/docs/source/using_libraft.md b/docs/source/using_libraft.md deleted file mode 100644 index 70a17e289b..0000000000 --- a/docs/source/using_libraft.md +++ /dev/null @@ -1,64 +0,0 @@ -# Using The Pre-Compiled Binary - -At its core, RAFT is a header-only template library, which makes it very powerful in that APIs can be called with various different combinations of data types and only the templates which are actually used will be compiled into your binaries. This increased flexibility comes with a drawback that all the APIs need to be declared inline and thus calls which are made frequently in your code could be compiled again in each source file for which they are invoked. - -For most functions, compile-time overhead is minimal but some of RAFT's APIs take a substantial time to compile. As a rule of thumb, most functionality in `raft::distance`, `raft::neighbors`, and `raft::cluster` is expensive to compile and most functionality in other namespaces has little compile-time overhead. - -There are three ways to speed up compile times: - -1. Continue to use RAFT as a header-only library and create a CUDA source file - in your project to explicitly instantiate the templates which are slow to - compile. This can be tedious and will still require compiling the slow code - at least once, but it's the most flexible option if you are using types that - aren't already compiled into `libraft` - -2. If you are able to use one of the template types that are already being - compiled into `libraft`, you can use the pre-compiled template - instantiations, which are described in more detail in the following section. - -3. If you would like to use RAFT but either cannot or would prefer not to - compile any CUDA code yourself, you can simply add `libraft` to your link - libraries and use the growing set of `raft::runtime` APIs. - -### How do I verify template instantiations didn't compile into my binary? - -To verify that you are not accidentally instantiating templates that have not been pre-compiled in RAFT, set the `RAFT_EXPLICIT_INSTANTIATE_ONLY` macro. This only works if you are linking with the pre-compiled libraft (i.e., when `RAFT_COMPILED` has been defined). To check if, for instance, `raft::distance::distance` has been precompiled with specific template arguments, you can set `RAFT_EXPLICIT_INSTANTIATE_ONLY` at the top of the file you are compiling, as in the following example: - -```c++ - -#ifdef RAFT_COMPILED -#define RAFT_EXPLICIT_INSTANTIATE_ONLY -#endif - -#include -#include -#include - -int main() -{ - raft::resources handle{}; - - // Change IdxT to uint64_t and you will get an error because you are - // instantiating a template that has not been pre-compiled. - using IdxT = int; - - const float* x = nullptr; - const float* y = nullptr; - float* out = nullptr; - int m = 1024; - int n = 1024; - int k = 1024; - bool row_major = true; - raft::distance::distance( - handle, x, y, out, m, n, k, row_major, 2.0f); -} -``` - -## Runtime APIs - -RAFT contains a growing list of runtime APIs that, unlike the pre-compiled -template instantiations, allow you to link against `libraft` and invoke RAFT -directly from `cpp` files. The benefit to RAFT's runtime APIs is that they can -be used from code that is compiled with a `c++` compiler (rather than the CUDA -compiler `nvcc`). This enables the `runtime` APIs to power `pylibraft`. - diff --git a/docs/source/vector_search_tutorial.md b/docs/source/vector_search_tutorial.md deleted file mode 100644 index 8f7b2d1bfd..0000000000 --- a/docs/source/vector_search_tutorial.md +++ /dev/null @@ -1,409 +0,0 @@ -# Vector Search in C++ Tutorial - -## Table of Contents - -- [Step 1: Starting off with RAFT](#step-1-starting-off-with-raft) -- [Step 2: Generate some data](#step-2-generate-some-data) -- [Step 3: Using brute-force indexes](#step-3-using-brute-force-indexes) -- [Step 4: Using the ANN indexes](#step-4-using-the-ann-indexes) -- [Step 5: Evaluate neighborhood quality](#step-5-evaluate-neighborhood-quality) -- [Advanced Features](#advanced-features) - - [Serialization](#serialization) - - [Filtering](#filtering) - - [Stream Pools](#stream-pools) - - [Device Resources Manager](#device-resources-manager) - - [Device Memory Resources](#device-memory-resources) - - [Workspace Memory Resource](#workspace-memory-resource) - -RAFT has several important algorithms for performing vector search on the GPU and this tutorial walks through the primary vector search APIs from start to finish to provide a reference for quick setup and C++ API usage. - -This tutorial assumes RAFT has been installed and/or added to your build so that you are able to compile and run RAFT code. If not done already, please follow the [build and install instructions](build.md) and consider taking a look at the [example c++ template project](https://github.com/rapidsai/raft/tree/HEAD/cpp/template) for ready-to-go examples that you can immediately build and start playing with. - -For more information about the various APIs demonstrated in this tutorial, along with comprehensive usage examples of all the APIs offered by RAFT, please refer to the [RAFT's C++ API Documentation](https://docs.rapids.ai/api/raft/nightly/cpp_api/). - -## Step 1: Starting off with RAFT - -### CUDA Development? - -If you are reading this tuturial then you probably know about CUDA and its relationship to general-purpose GPU computing (GPGPU). You probably also know about Nvidia GPUs but might not necessarily be familiar with the programming model nor GPU computing. The good news is that extensive knowledge of CUDA and GPUs are not needed in order to get started with or build applications with RAFT. RAFT hides away most of the complexities behind simple single-threaded stateless functions that are inherently asynchronous, meaning the result of a computation isn't necessarily read to be used when the function executes and control is given back to the user. The functions are, however, allowed to be chained together in a sequence of calls that don't need to wait for subsequent computations to complete in order to continue execution. In fact, the only time you need to wait for the computation to complete is when you are ready to use the result. - -A common structure you will encounter when using RAFT is a `raft::device_resources` object. This object is a container for important resources for a single GPU that might be needed during computation. If communicating with multiple GPUs, multiple `device_resources` might be needed, one for each GPU. `device_resources` contains several methods for managing its state but most commonly, you'll call the `sync_stream()` to guarantee all recently submitted computation has completed (as mentioned above.) - -A simple example of using `raft::device_resources` in RAFT: - -```c++ -#include - -raft::device_resources res; -// Call a bunch of RAFT functions in sequence... -res.sync_stream() -``` - -### Host vs Device Memory - -We differentiate between two different types of memory. `host` memory is your traditional RAM memory that is primarily accessible by applications on the CPU. `device` memory, on the other hand, is what we call the special memory on the GPU, which is not accessible from the CPU. In order to access host memory from the GPU, it needs to be explicitly copied to the GPU and in order to access device memory by the CPU, it needs to be explicitly copied there. We have several mechanisms available for allocating and managing the lifetime of device memory on the stack so that we don't need to explicitly allocate and free pointers on the heap. For example, instead of a `std::vector` for host memory, we can use `rmm::device_uvector` on the device. The following function will copy an array from host memory to device memory: - -```c++ -#include -#include -#include - -raft::device_resources res; - -std::vector my_host_vector = {0, 1, 2, 3, 4}; -rmm::device_uvector my_device_vector(my_host_vector.size(), res.get_stream()); - -raft::copy(my_device_vector.data(), my_host_vector.data(), my_host_vector.size(), res.get_stream()); -``` - -Since a stream is involved in the copy operation above, RAFT functions can be invoked immediately so long as the same `device_resources` instances is used (or, more specifically, the same main stream from the `devices_resources`.) As you might notice in the example above, `res.get_stream()` can be used to extract the main stream from a `device_resources` instance. - -### Multi-dimensional data representation - -`rmm::device_uvector` is a great mechanism for allocating and managing a chunk of device memory. While it's possible to use a single array to represent objects in higher dimensions like matrices, it lacks the means to pass that information along. For example, in addition to knowing that we have a 2d structure, we would need to know the number of rows, the number of columns, and even whether we read the columns or rows first (referred to as column- or row-major respectively). - -For this reason, RAFT relies on the `mdspan` standard, which was composed specifically for this purpose. To be even more, `mdspan` itself doesn't actually allocate or own any data on host or device because it's just a view over an existing memory on host device. The `mdspan` simply gives us a way to represent multi-dimensional data so we can pass along the needed metadata to our APIs. Even more powerful is that we can design functions that only accept a matrix of `float` in device memory that is laid out in row-major format. - -The memory-owning counterpart to the `mdspan` is the `mdarray` and the `mdarray` can allocate memory on device or host and carry along with it the metadata about its shape and layout. An `mdspan` can be produced from an `mdarray` for invoking RAFT APIs with `mdarray.view()`. They also follow similar paradigms to the STL, where we represent an immutable `mdspan` of `int` using `mdspan` instead of `const mdspan` to ensure it's the type carried along by the `mdspan` that's not allowed to change. - -Many RAFT functions require `mdspan` to represent immutable input data and there's no implicit conversion between `mdspan` and `mdspan` we use `raft::make_const_mdspan()` to alleviate the pain of constructing a new `mdspan` to invoke these functions. - -The following example demonstrates how to create `mdarray` matrices in both device and host memory, copy one to the other, and create mdspans out of them: - -```c++ -#include -#include -#include - -raft::device_resources res; - -int n_rows = 10; -int n_cols = 10; - -auto device_matrix = raft::make_device_matrix(res, n_rows, n_cols); -auto host_matrix = raft::make_host_matrix(res, n_rows, n_cols); - -// Set the diagonal to 1 -for(int i = 0; i < n_rows; i++) { - host_matrix(i, i) = 1; -} - -raft::copy(res, device_matrix.view(), host_matrix.view()); -``` - -## Step 2: Generate some data - -Let's build upon the fundamentals from the prior section and actually invoke some of RAFT's computational APIs on the device. A good starting point is data generation. - -```c++ -#include -#include - -raft::device_resources res; - -int n_rows = 10000; -int n_cols = 10000; - -auto dataset = raft::make_device_matrix(res, n_rows, n_cols); -auto labels = raft::make_device_vector(res, n_rows); - -raft::random::make_blobs(res, dataset.view(), labels.view()); -``` - -That's it. We've now generated a random 10kx10k matrix with points that cleanly separate into Gaussian clusters, along with a vector of cluster labels for each of the data points. Notice the `cuh` extension in the header file include for `make_blobs`. This signifies to us that this file contains CUDA device functions like kernel code so the CUDA compiler, `nvcc` is needed in order to compile any code that uses it. Generally, any source files that include headers with a `cuh` extension use the `.cu` extension instead of `.cpp`. The rule here is that `cpp` source files contain code which can be compiled with a C++ compiler like `g++` while `cu` files require the CUDA compiler. - -Since the `make_blobs` code generates the random dataset on the GPU device, we didn't need to do any host to device copies in this one. `make_blobs` is also asynchronous, so if we don't need to copy and use the data in host memory right away, we can continue calling RAFT functions with the `device_resources` instance and the data transformations will all be scheduled on the same stream. - -## Step 3: Using brute-force indexes - -### Build brute-force index - -Consider the `(10k, 10k)` shaped random matrix we generated in the previous step. We want to be able to find the k-nearest neighbors for all points of the matrix, or what we refer to as the all-neighbors graph, which means finding the neighbors of all data points within the same matrix. -```c++ -#include - -raft::device_resources res; - -// set number of neighbors to search for -int const k = 64; - -auto bfknn_index = raft::neighbors::brute_force::build(res, - raft::make_const_mdspan(dataset.view())); -``` - -### Query brute-force index - -```c++ - -// using matrix `dataset` from previous example -auto search = raft::make_const_mdspan(dataset.view()); - -// Indices and Distances are of dimensions (n, k) -// where n is number of rows in the search matrix -auto reference_indices = raft::make_device_matrix(res, search.extent(0), k); // stores index of neighbors -auto reference_distances = raft::make_device_matrix(res, search.extent(0), k); // stores distance to neighbors - -raft::neighbors::brute_force::search(res, - bfknn_index, - search, - reference_indices.view(), - reference_distances.view()); -``` - -We have established several things here by building a flat index. Now we know the exact 64 neighbors of all points in the matrix, and this algorithm can be generally useful in several ways: -1. Creating a baseline to compare against when building an approximate nearest neighbors index. -2. Directly using the brute-force algorithm when accuracy is more important than speed of computation. Don't worry, our implementation is still the best in-class and will provide not only significant speedups over other brute force methods, but also be quick relatively when the matrices are small! - - -## Step 4: Using the ANN indexes - -### Build a CAGRA index - -Next we'll train an ANN index. We'll use our graph-based CAGRA algorithm for this example but the other index types use a very similar pattern. - -```c++ -#include - -raft::device_resources res; - -// use default index parameters -raft::neighbors::cagra::index_params index_params; - -auto index = raft::neighbors::cagra::build(res, index_params, raft::make_const_mdspan(dataset.view())); -``` - -### Query the CAGRA index - -Now that we've trained a CAGRA index, we can query it by first allocating our output `mdarray` objects and passing the trained index model into the search function. - -```c++ -// create output arrays -auto indices = raft::make_device_matrix(res, n_rows, k); -auto distances = raft::make_device_matrix(res, n_rows, k); - -// use default search parameters -raft::neighbors::cagra::search_params search_params; - -// search K nearest neighbors -raft::neighbors::cagra::search( -res, search_params, index, search, indices.view(), distances.view()); -``` - -## Step 5: Evaluate neighborhood quality - -In step 3 we built a flat index and queried for exact neighbors while in step 4 we build an ANN index and queried for approximate neighbors. How do you quickly figure out the quality of our approximate neighbors and whether it's in an acceptable range based on your needs? Just compute the `neighborhood_recall` which gives a single value in the range [0, 1]. Closer the value to 1, higher the quality of the approximation. - -```c++ -#include - -raft::device_resources res; - -// Assuming matrices as type raft::device_matrix_view and variables as -// indices : approximate neighbor indices -// reference_indices : exact neighbor indices -// distances : approximate neighbor distances -// reference_distances : exact neighbor distances - -// We want our `neighborhood_recall` value in host memory -float const recall_scalar = 0.0; -auto recall_value = raft::make_host_scalar(recall_scalar); - -raft::stats::neighborhood_recall(res, - raft::make_const_mdspan(indices.view()), - raft::make_const_mdspan(reference_indices.view()), - recall_value.view(), - raft::make_const_mdspan(distances.view()), - raft::make_const_mdspan(reference_distances.view())); - -res.sync_stream(); -``` - -Notice we can run invoke the functions for index build and search for both algorithms, one right after the other, because we don't need to access any outputs from the algorithms in host memory. We will need to synchronize the stream on the `raft::device_resources` instance before we can read the result of the `neighborhood_recall` computation, though. - -Similar to a Numpy array, when we use a `host_scalar`, we are really using a multi-dimensional structure that contains only a single dimension, and further a single element. We can use element indexing to access the resulting element directly. -```c++ -std::cout << recall_value(0) << std::endl; -``` - -While it may seem like unnecessary additional work to wrap the result in a `host_scalar` mdspan, this API choice is made intentionally to support the possibility of also receiving the result as a `device_scalar` so that it can be used directly on the device for follow-on computations without having to incur the synchronization or transfer cost of bringing the result to host. This pattern becomes even more important when the result is being computed in a loop, such as an iterative solver, and the cost of synchronization and device-to-host (d2h) transfer becomes very expensive. - -## Advanced features - -The following sections present some advanced features that we have found can be useful for squeezing more utilization out of GPU hardware. As you've seen in this tutorial, RAFT provides several very useful tools and building blocks for developing accelerated applications beyond vector search capabilities. - -### Serialization - -Most of the indexes in `raft::neighbors` can be serialized to/from streams and files on disk. The index types that support this feature have include files with the naming convention `_serialize.cuh`. The serialization functions are similar across the different index types, with the primary difference being that some index types require a pointer to all the training data for search. Since the original training dataset can be quite large, the `serialize()` function for these index types includes an argument `include_dataset`, which allows the user to specify whether the dataset should be included in the serialized form. The index types that allow for this also include a method `update_datasets()` to allow for the dataset to be re-attached to the index after it is deserialized. - -The following example demonstrates serializing and deserializing a CAGRA index to and from a file. For index types that don't require the training data, you can remove the `include_dataset` and `update_dataset()` parts. We will assume the CAGRA index has been built using the code from [Step 4](#build-a-cagra-index) above: - -```c++ -#include -#include - -using namespace raft::neighbors; - -raft::neighbors::cagra::serialize(res, "cagra_serialized.dat", index, false); - -auto index_deser = raft::neighbors::cagra::deserialize(res, "cagra_serialized.dat"); -index_deser.update_dataset(dataset); -``` - -### Filtering - -As of RAFT 23.10, support for pre-filtering of neighbors has been added to ANN index. This search feature can enable multiple use-cases, such as filtering a vector based on it's attributes (hybrid searches), the removal of vectors already added to the index, or the control of access in searches for security purposes. -The filtering is available through the `search_with_filtering()` function of the ANN index, and is done by applying a predicate function on the GPU, which usually have the signature `(uint32_t query_ix, uint32_t sample_ix) -> bool`. - -One of the most commonly used mechanism for filtering is the bitset: the bitset is a data structure that allows to test the presence of a value in a set through a fast lookup, and is implemented as a bit array so that every element contains a `0` or a `1` (respectively `false` and `true` in boolean logic). RAFT provides a `raft::core::bitset` class that can be used to create and manipulate bitsets on the GPU, and a `raft::core::bitset_view` class that can be used to pass bitsets to filtering functions. - -The following example demonstrates how to use the filtering API (assume the CAGRA index is built using the code from [Step 4](#build-a-cagra-index) above: - -```c++ -#include -#include - -using namespace raft::neighbors; - -cagra::search_params search_params; - -// create a bitset to filter the search -auto removed_indices = raft::make_device_vector(res, n_removed_indices); -raft::core::bitset removed_indices_bitset( - res, removed_indices.view(), dataset.extent(0)); - -// ... Populate the bitset ... - -// search K nearest neighbours according to a bitset filter -auto neighbors = raft::make_device_matrix(res, n_queries, k); -auto distances = raft::make_device_matrix(res, n_queries, k); -cagra::search_with_filtering(res, search_params, index, queries, neighbors, distances, - filtering::bitset_filter(removed_indices_bitset.view())); -``` - -### Stream pools - -Within each CPU thread, CUDA uses `streams` to submit asynchronous work. You can think of a stream as a queue. Each stream can submit work to the GPU independently of other streams but work submitted within each stream is queued and executed in the order in which it was submitted. Similar to how we can use thread pools to bound the parallelism of CPU threads, we can use CUDA stream pools to bound the amount of concurrent asynchronous work that can be scheduled on a GPU. Each instance of `device_resources` has a main stream, but can also create a stream pool. For a single CPU thread, multiple different instances of `device_resources` can be created with different main streams and used to invoke a series of RAFT functions concurrently on the same or different GPU devices, so long as the target devices have available resources to perform the work. Once a device is saturated, queued work on streams will be scheduled and wait for a chance to do more work. During this time the streams are waiting, the CPU thread will still continue its own execution asynchronously unless `sync_stream_pool()` is called, causing the thread to block and wait for the thread pools to complete. - -Also, beware that before splitting GPU work onto multiple different concurrent streams, it can often be important to wait for the main stream in the `device_resources`. This can be done with `wait_stream_pool_on_stream()`. - -To summarize, if wanting to execute multiple different streams in parallel, we would often use a stream pool like this: -```c++ -#include - -#include -#include - -int n_streams = 5; - -rmm::cuda_stream stream; -std::shared_ptr stream_pool(5) -raft::device_resources res(stream.view(), stream_pool); - -// Submit some work on the main stream... - -res.wait_stream_pool_on_stream() -for(int i = 0; i < n_streams; ++i) { - rmm::cuda_stream_view stream_from_pool = res.get_next_usable_stream(); - raft::device_resources pool_res(stream_from_pool); - // Submit some work with pool_res... -} - -res.sync_stream_pool(); -``` - -### Device resources manager - -In multi-threaded applications, it is often useful to create a set of -`raft::device_resources` objects on startup to avoid the overhead of -re-initializing underlying resources every time a `raft::device_resources` object -is needed. To help simplify this common initialization logic, RAFT -provides a `raft::device_resources_manager` to handle this for downstream -applications. On startup, the application can specify certain limits on the -total resource consumption of the `raft::device_resources` objects that will be -generated: -```c++ -#include - -void initialize_application() { - // Set the total number of CUDA streams to use on each GPU across all CPU - // threads. If this method is not called, the default stream per thread - // will be used. - raft::device_resources_manager::set_streams_per_device(16); - - // Create a memory pool with given max size in bytes. Passing std::nullopt will allow - // the pool to grow to the available memory of the device. - raft::device_resources_manager::set_max_mem_pool_size(std::nullopt); - - // Set the initial size of the memory pool in bytes. - raft::device_resources_manager::set_init_mem_pool_size(16000000); - - // If neither of the above methods are called, no memory pool will be used -} -``` -While this example shows some commonly used settings, -`raft::device_resources_manager` provides support for several other -resource options and constraints, including options to initialize entire -stream pools that can be used by an individual `raft::device_resources` object. After -this initialization method is called, the following function could be called -from any CPU thread: -```c++ -void foo() { - raft::device_resources const& res = raft::device_resources_manager::get_device_resources(); - // Submit some work with res - res.sync_stream(); -} -``` - -If any `raft::device_resources_manager` setters are called _after_ the first -call to `raft::device_resources_manager::get_device_resources()`, these new -settings are ignored, and a warning will be logged. If a thread calls -`raft::device_resources_manager::get_device_resources()` multiple times, it is -guaranteed to access the same underlying `raft::device_resources` object every -time. This can be useful for chaining work in different calls on the same -thread without keeping a persistent reference to the resources object. - -### Device memory resources - -The RAPIDS software ecosystem makes heavy use of the [RAPIDS Memory Manager](https://github.com/rapidsai/rmm) (RMM) to enable zero-copy sharing of device memory across various GPU-enabled libraries such as PyTorch, Jax, Tensorflow, and FAISS. A really powerful feature of RMM is the ability to set a memory resource, such as a pooled memory resource that allocates a block of memory up front to speed up subsequent smaller allocations, and have all the libraries in the GPU ecosystem recognize and use that same memory resource for all of their memory allocations. - -As an example, the following code snippet creates a `pool_memory_resource` and sets it as the default memory resource, which means all other libraries that use RMM will now allocate their device memory from this same pool: -```c++ -#include - -rmm::mr::cuda_memory_resource cuda_mr; -// Construct a resource that uses a coalescing best-fit pool allocator -// set the initial size to half of the free device memory -auto init_size = rmm::percent_of_free_device_memory(50); -rmm::mr::pool_memory_resource pool_mr{&cuda_mr, init_size}; -rmm::mr::set_current_device_resource(&pool_mr); // Updates the current device resource pointer to `pool_mr` -``` - -The `raft::device_resources` object will now also use the `rmm::current_device_resource`. This isn't limited to C++, however. Often a user will be interacting with PyTorch, RAPIDS, or Tensorflow through Python and so they can set and use RMM's `current_device_resource` [right in Python](https://github.com/rapidsai/rmm#using-rmm-in-python-code). - -### Workspace memory resource - -As mentioned above, `raft::device_resources` will use `rmm::current_device_resource` by default for all memory allocations. However, there are times when a particular algorithm might benefit from using a different memory resource such as a `managed_memory_resource`, which creates a unified memory space between device and host memory, paging memory in and out of device as needed. Most of RAFT's algorithms allocate temporary memory as needed to perform their computations and we can control the memory resource used for these temporary allocations through the `workspace_resource` in the `raft::device_resources` instance. - -For some applications, the `managed_memory_resource`, can enable a memory space that is larger than the GPU, thus allowing a natural spilling to host memory when needed. This isn't always the best way to use managed memory, though, as it can quickly lead to thrashing and severely impact performance. Still, when it can be used, it provides a very powerful tool that can also avoid out of memory errors when enough host memory is available. - -The following creates a managed memory allocator and set it as the `workspace_resource` of the `raft::device_resources` instance: -```c++ -#include -#include - -std::shared_ptr managed_resource; -raft::device_resource res(managed_resource); -``` - -The `workspace_resource` uses an `rmm::mr::limiting_resource_adaptor`, which limits the total amount of allocation possible. This allows RAFT algorithms to work within the confines of the memory constraints imposed by the user so that things like batch sizes can be automatically set to reasonable values without exceeding the allotted memory. By default, this limit restricts the memory allocation space for temporary workspace buffers to the memory available on the device. - -The below example specifies the total number of bytes that RAFT can use for temporary workspace allocations to 3GB: -```c++ -#include -#include - -#include - -std::shared_ptr managed_resource; -raft::device_resource res(managed_resource, std::make_optional(3 * 1024^3)); -``` diff --git a/notebooks/VectorSearch_QuestionRetrieval.ipynb b/notebooks/VectorSearch_QuestionRetrieval.ipynb deleted file mode 100644 index 33a2f60228..0000000000 --- a/notebooks/VectorSearch_QuestionRetrieval.ipynb +++ /dev/null @@ -1,632 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "f5499b54", - "metadata": {}, - "source": [ - "\n", - "# Similar Questions Retrieval\n", - "\n", - "This notebook is inspired by the [similar search example of Sentence-Transformers](https://www.sbert.net/examples/applications/semantic-search/README.html#similar-questions-retrieval), and adapted to support [RAFT ANN](https://github.com/rapidsai/raft) algorithm.\n", - "\n", - "The model was pre-trained on the [Natural Questions dataset](https://ai.google.com/research/NaturalQuestions). It consists of about 100k real Google search queries, together with an annotated passage from Wikipedia that provides the answer. It is an example of an asymmetric search task. As corpus, we use the smaller [Simple English Wikipedia](http://sbert.net/datasets/simplewiki-2020-11-01.jsonl.gz) so that it fits easily into memory.\n", - "\n", - "The steps to install the latest stable `pylibraft` package are available in the [documentation](https://docs.rapids.ai/api/raft/stable/build)." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "e8d55ede", - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: sentence_transformers in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (2.2.2)\n", - "Requirement already satisfied: torch in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (2.0.1)\n", - "Requirement already satisfied: transformers<5.0.0,>=4.6.0 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from sentence_transformers) (4.31.0)\n", - "Requirement already satisfied: tqdm in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from sentence_transformers) (4.65.0)\n", - "Requirement already satisfied: torchvision in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from sentence_transformers) (0.15.2)\n", - "Requirement already satisfied: numpy in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from sentence_transformers) (1.24.4)\n", - "Requirement already satisfied: scikit-learn in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from sentence_transformers) (1.3.0)\n", - "Requirement already satisfied: scipy in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from sentence_transformers) (1.11.1)\n", - "Requirement already satisfied: nltk in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from sentence_transformers) (3.8.1)\n", - "Requirement already satisfied: sentencepiece in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from sentence_transformers) (0.1.99)\n", - "Requirement already satisfied: huggingface-hub>=0.4.0 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from sentence_transformers) (0.16.4)\n", - "Requirement already satisfied: filelock in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (3.12.2)\n", - "Requirement already satisfied: typing-extensions in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (4.7.1)\n", - "Requirement already satisfied: sympy in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (1.12)\n", - "Requirement already satisfied: networkx in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (3.1)\n", - "Requirement already satisfied: jinja2 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (3.1.2)\n", - "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (11.7.99)\n", - "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (11.7.99)\n", - "Requirement already satisfied: nvidia-cuda-cupti-cu11==11.7.101 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (11.7.101)\n", - "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (8.5.0.96)\n", - "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (11.10.3.66)\n", - "Requirement already satisfied: nvidia-cufft-cu11==10.9.0.58 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (10.9.0.58)\n", - "Requirement already satisfied: nvidia-curand-cu11==10.2.10.91 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (10.2.10.91)\n", - "Requirement already satisfied: nvidia-cusolver-cu11==11.4.0.1 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (11.4.0.1)\n", - "Requirement already satisfied: nvidia-cusparse-cu11==11.7.4.91 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (11.7.4.91)\n", - "Requirement already satisfied: nvidia-nccl-cu11==2.14.3 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (2.14.3)\n", - "Requirement already satisfied: nvidia-nvtx-cu11==11.7.91 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (11.7.91)\n", - "Requirement already satisfied: triton==2.0.0 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (2.0.0)\n", - "Requirement already satisfied: setuptools in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch) (68.0.0)\n", - "Requirement already satisfied: wheel in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch) (0.41.0)\n", - "Requirement already satisfied: cmake in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from triton==2.0.0->torch) (3.27.0)\n", - "Requirement already satisfied: lit in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from triton==2.0.0->torch) (16.0.6)\n", - "Requirement already satisfied: fsspec in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence_transformers) (2023.6.0)\n", - "Requirement already satisfied: requests in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence_transformers) (2.31.0)\n", - "Requirement already satisfied: pyyaml>=5.1 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence_transformers) (6.0)\n", - "Requirement already satisfied: packaging>=20.9 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence_transformers) (23.1)\n", - "Requirement already satisfied: regex!=2019.12.17 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from transformers<5.0.0,>=4.6.0->sentence_transformers) (2023.6.3)\n", - "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from transformers<5.0.0,>=4.6.0->sentence_transformers) (0.13.3)\n", - "Requirement already satisfied: safetensors>=0.3.1 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from transformers<5.0.0,>=4.6.0->sentence_transformers) (0.3.1)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from jinja2->torch) (2.1.3)\n", - "Requirement already satisfied: click in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from nltk->sentence_transformers) (8.1.6)\n", - "Requirement already satisfied: joblib in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from nltk->sentence_transformers) (1.3.0)\n", - "Requirement already satisfied: threadpoolctl>=2.0.0 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from scikit-learn->sentence_transformers) (3.2.0)\n", - "Requirement already satisfied: mpmath>=0.19 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from sympy->torch) (1.3.0)\n", - "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torchvision->sentence_transformers) (10.0.0)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from requests->huggingface-hub>=0.4.0->sentence_transformers) (3.2.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from requests->huggingface-hub>=0.4.0->sentence_transformers) (3.4)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from requests->huggingface-hub>=0.4.0->sentence_transformers) (2.0.4)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from requests->huggingface-hub>=0.4.0->sentence_transformers) (2023.7.22)\n" - ] - } - ], - "source": [ - "!pip install sentence_transformers torch\n", - "\n", - "# Note: if you have a Hopper based GPU, like an H100, use these to install:\n", - "# pip install torch --index-url https://download.pytorch.org/whl/cu118\n", - "# pip install sentence_transformers" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "eb1e81c3", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Jul 31 14:35:31 2023 \n", - "+-----------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 525.105.17 Driver Version: 525.105.17 CUDA Version: 12.0 |\n", - "|-------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|===============================+======================+======================|\n", - "| 0 NVIDIA H100 80G... On | 00000000:1B:00.0 Off | 0 |\n", - "| N/A 30C P0 75W / 700W | 0MiB / 81559MiB | 0% Default |\n", - "| | | Disabled |\n", - "+-------------------------------+----------------------+----------------------+\n", - "| 1 NVIDIA H100 80G... On | 00000000:43:00.0 Off | 0 |\n", - "| N/A 31C P0 72W / 700W | 0MiB / 81559MiB | 0% Default |\n", - "| | | Disabled |\n", - "+-------------------------------+----------------------+----------------------+\n", - "| 2 NVIDIA H100 80G... On | 00000000:52:00.0 Off | 0 |\n", - "| N/A 34C P0 70W / 700W | 0MiB / 81559MiB | 0% Default |\n", - "| | | Disabled |\n", - "+-------------------------------+----------------------+----------------------+\n", - "| 3 NVIDIA H100 80G... On | 00000000:61:00.0 Off | 0 |\n", - "| N/A 33C P0 70W / 700W | 0MiB / 81559MiB | 0% Default |\n", - "| | | Disabled |\n", - "+-------------------------------+----------------------+----------------------+\n", - "| 4 NVIDIA H100 80G... On | 00000000:9D:00.0 Off | 0 |\n", - "| N/A 32C P0 74W / 700W | 0MiB / 81559MiB | 0% Default |\n", - "| | | Disabled |\n", - "+-------------------------------+----------------------+----------------------+\n", - "| 5 NVIDIA H100 80G... On | 00000000:C3:00.0 Off | 0 |\n", - "| N/A 30C P0 73W / 700W | 0MiB / 81559MiB | 0% Default |\n", - "| | | Disabled |\n", - "+-------------------------------+----------------------+----------------------+\n", - "| 6 NVIDIA H100 80G... On | 00000000:D1:00.0 Off | 0 |\n", - "| N/A 33C P0 73W / 700W | 0MiB / 81559MiB | 0% Default |\n", - "| | | Disabled |\n", - "+-------------------------------+----------------------+----------------------+\n", - "| 7 NVIDIA H100 80G... On | 00000000:DF:00.0 Off | 0 |\n", - "| N/A 35C P0 73W / 700W | 0MiB / 81559MiB | 0% Default |\n", - "| | | Disabled |\n", - "+-------------------------------+----------------------+----------------------+\n", - " \n", - "+-----------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=============================================================================|\n", - "| No running processes found |\n", - "+-----------------------------------------------------------------------------+\n" - ] - } - ], - "source": [ - "!nvidia-smi" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "ee4c5cc0", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/raid/danteg/miniconda3/envs/raftann/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [ - "import json\n", - "from sentence_transformers import SentenceTransformer, CrossEncoder, util\n", - "import time\n", - "import gzip\n", - "import os\n", - "import torch\n", - "import pylibraft\n", - "from pylibraft.neighbors import ivf_flat, ivf_pq\n", - "pylibraft.config.set_output_as(lambda device_ndarray: device_ndarray.copy_to_host())\n", - "\n", - "if not torch.cuda.is_available():\n", - " print(\"Warning: No GPU found. Please add GPU to your notebook\")" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "0a1a6307", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Passages: 509663\n" - ] - } - ], - "source": [ - "# We use the Bi-Encoder to encode all passages, so that we can use it with semantic search\n", - "model_name = 'nq-distilbert-base-v1'\n", - "bi_encoder = SentenceTransformer(model_name)\n", - "\n", - "# As dataset, we use Simple English Wikipedia. Compared to the full English wikipedia, it has only\n", - "# about 170k articles. We split these articles into paragraphs and encode them with the bi-encoder\n", - "\n", - "wikipedia_filepath = 'data/simplewiki-2020-11-01.jsonl.gz'\n", - "\n", - "if not os.path.exists(wikipedia_filepath):\n", - " util.http_get('http://sbert.net/datasets/simplewiki-2020-11-01.jsonl.gz', wikipedia_filepath)\n", - "\n", - "passages = []\n", - "with gzip.open(wikipedia_filepath, 'rt', encoding='utf8') as fIn:\n", - " for line in fIn:\n", - " data = json.loads(line.strip())\n", - " for paragraph in data['paragraphs']:\n", - " # We encode the passages as [title, text]\n", - " passages.append([data['title'], paragraph])\n", - "\n", - "# If you like, you can also limit the number of passages you want to use\n", - "print(\"Passages:\", len(passages))\n", - "\n", - "# To speed things up, pre-computed embeddings are downloaded.\n", - "# The provided file encoded the passages with the model 'nq-distilbert-base-v1'\n", - "if model_name == 'nq-distilbert-base-v1':\n", - " embeddings_filepath = 'simplewiki-2020-11-01-nq-distilbert-base-v1.pt'\n", - " if not os.path.exists(embeddings_filepath):\n", - " util.http_get('http://sbert.net/datasets/simplewiki-2020-11-01-nq-distilbert-base-v1.pt', embeddings_filepath)\n", - "\n", - " corpus_embeddings = torch.load(embeddings_filepath)\n", - " corpus_embeddings = corpus_embeddings.float() # Convert embedding file to float\n", - " if torch.cuda.is_available():\n", - " corpus_embeddings = corpus_embeddings.to('cuda')\n", - "else: # Here, we compute the corpus_embeddings from scratch (which can take a while depending on the GPU)\n", - " corpus_embeddings = bi_encoder.encode(passages, convert_to_tensor=True, show_progress_bar=True)" - ] - }, - { - "cell_type": "markdown", - "id": "1f4e9b9d", - "metadata": {}, - "source": [ - "# Vector Search using RAPIDS RAFT\n", - "Now that our embeddings are ready to be indexed and that the model has been loaded, we can use RAPIDS RAFT to do our vector search.\n", - "\n", - "This is done in two step: First we build the index, then we search it.\n", - "With `pylibraft` all you need is those four Python lines:" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "ad90b4be", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[W] [14:35:48.810785] [raft::ivf_pq::build] the default cuda resource is used for the raft workspace allocations. This may lead to a significant slowdown for this algorithm. Consider using the default pool resource (`raft::resource::set_workspace_to_pool_resource`) or set your own resource explicitly (`raft::resource::set_workspace_resource`).\n", - "[W] [14:35:53.831753] [raft::ivf_pq::extend] the default cuda resource is used for the raft workspace allocations. This may lead to a significant slowdown for this algorithm. Consider using the default pool resource (`raft::resource::set_workspace_to_pool_resource`) or set your own resource explicitly (`raft::resource::set_workspace_resource`).\n", - "CPU times: user 2.21 s, sys: 2.49 s, total: 4.7 s\n", - "Wall time: 5.13 s\n" - ] - } - ], - "source": [ - "%%time\n", - "params = ivf_pq.IndexParams(n_lists=150, pq_dim=96)\n", - "pq_index = ivf_pq.build(params, corpus_embeddings)\n", - "search_params = ivf_pq.SearchParams()\n", - "\n", - "def search_raft_pq(query, top_k = 5):\n", - " # Encode the query using the bi-encoder and find potentially relevant passages\n", - " question_embedding = bi_encoder.encode(query, convert_to_tensor=True)\n", - "\n", - " hits = ivf_pq.search(search_params, pq_index, question_embedding[None], top_k)\n", - "\n", - " # Output of top-k hits\n", - " print(\"Input question:\", query)\n", - " for k in range(top_k):\n", - " print(\"\\t{:.3f}\\t{}\".format(hits[0][0, k], passages[hits[1][0, k]]))" - ] - }, - { - "cell_type": "markdown", - "id": "07935bca", - "metadata": {}, - "source": [ - "For IVF-PQ we want to reduce the memory footprint while keeping a good accuracy." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "724dcacb", - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "IVF-PQ memory footprint: 373.3 MB\n", - "Original dataset: 1493.2 MB\n", - "Memory saved: 75.0%\n" - ] - } - ], - "source": [ - "pq_index_mem = pq_index.pq_dim * pq_index.size * pq_index.pq_bits\n", - "print(\"IVF-PQ memory footprint: {:.1f} MB\".format(pq_index_mem / 2**20))\n", - "\n", - "original_mem = corpus_embeddings.shape[0] * corpus_embeddings.shape[1] * 4\n", - "print(\"Original dataset: {:.1f} MB\".format(original_mem / 2**20))\n", - "\n", - "print(\"Memory saved: {:.1f}%\".format(100 * (1 - pq_index_mem / original_mem)))" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "c27d4715", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[W] [14:36:07.640223] [raft::ivf_pq::search] the default cuda resource is used for the raft workspace allocations. This may lead to a significant slowdown for this algorithm. Consider using the default pool resource (`raft::resource::set_workspace_to_pool_resource`) or set your own resource explicitly (`raft::resource::set_workspace_resource`).\n", - "Input question: Who was Grace Hopper?\n", - "\t190.855\t['Leona Helmsley', 'Leona Helmsley (July 4, 1920 – August 20, 2007) was an American businesswoman. She was known for having a flamboyant personality. She had a reputation for tyrannical behavior; she was nicknamed the Queen of Mean.']\n", - "\t195.364\t['Grace Hopper', 'Hopper was born in New York, USA. Hopper graduated from Vassar College in 1928 and Yale University in 1934 with a Ph.D degree in mathematics. She joined the US Navy during the World War II in 1943. She worked on computers in the Navy for 43 years. She then worked in other private industry companies after 1949. She retired from the Navy in 1986 and died on January 1, 1992.']\n", - "\t202.536\t['Anita Borg', 'Anita Borg (January 17, 1949 – April 6, 2003) was an American computer scientist. She founded the Institute for Women and Technology and the Grace Hopper Celebration of Women in Computing.']\n", - "\t203.717\t['Brett Butler', 'Brett Butler (born January 30, 1958) is an American actress and stand-up comedian. She is best known for playing Grace in the sitcom \"Grace Under Fire\". She has also done other television programs and comedy acts.']\n", - "\t203.991\t['Nellie Bly', 'Elizabeth Cochrane Seaman (born Elizabeth Jane Cochran; May 5, 1864 – January 27, 1922), better known by her pen name Nellie Bly, was an American journalist, novelist and inventor. She was a newspaper reporter, who worked at various jobs for exposing poor working conditions. Nellie Bly, also, fought for women\\'s right and was known for investigative reporting. She best known for her record-breaking trip around the world in 72 days, inspired by the adventure novel \"Around the World in Eighty Days\" by Jules Verne. In the 1880s, she went undercover as a mentally ill patient in a psychiatric hospital for ten days, with the report being made public in a book called \"\"Ten Days in a Mad-House\"\". She was added to the National Women\\'s Hall of Fame in 1998.']\n", - "CPU times: user 98.3 ms, sys: 81.2 ms, total: 180 ms\n", - "Wall time: 120 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "search_raft_pq(query=\"Who was Grace Hopper?\")" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "bc375518", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Input question: Who was Alan Turing?\n", - "\t139.827\t['Alan Turing', 'Alan Mathison Turing OBE FRS (London, 23 June 1912 – Wilmslow, Cheshire, 7 June 1954) was an English mathematician and computer scientist. He was born in Maida Vale, London.']\n", - "\t169.849\t['William Kahan', 'William Morton Kahan (born June 5, 1933) is a Canadian mathematician and computer scientist. He received the Turing Award in 1989 for \"\"his fundamental contributions to numerical analysis\".\" He was named an ACM Fellow in 1994, and added to the National Academy of Engineering in 2005.']\n", - "\t177.520\t['Rolf Noskwith', 'Rolf Noskwith (19 June 1919 – 3 January 2017) was a British businessman. During the Second World War, he worked under Alan Turing as a cryptographer at the British military base Bletchley Park in Milton Keynes, Buckinghamshire.']\n", - "\t179.202\t['Marvin Minsky', \"Marvin Lee Minsky (August 9, 1927 – January 24, 2016) was an American cognitive scientist in the field of artificial intelligence (AI). He was the co-founder of the Massachusetts Institute of Technology's AI laboratory, and author of several texts on AI and philosophy. He won the Turing Award in 1969.\"]\n", - "\t179.819\t['Edsger W. Dijkstra', 'Edsger Wybe Dijkstra (May 11, 1930 – August 6, 2002; ) was a Dutch computer scientist. He received the 1972 Turing Award for fundamental contributions to developing programming languages, and was the Schlumberger Centennial Chair of Computer Sciences at The University of Texas at Austin from 1984 until 2000.']\n", - "CPU times: user 4.89 ms, sys: 7.52 ms, total: 12.4 ms\n", - "Wall time: 12 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "search_raft_pq(query=\"Who was Alan Turing?\")" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "ab154181", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Input question: What is creating tides?\n", - "\t125.037\t['Tide', \"A tide is the periodic rising and falling of Earth's ocean surface caused mainly by the gravitational pull of the Moon acting on the oceans. Tides cause changes in the depth of marine and estuarine (river mouth) waters. Tides also make oscillating currents known as tidal streams (~'rip tides'). This means that being able to predict the tide is important for coastal navigation. The strip of seashore that is under water at high tide and exposed at low tide, called the intertidal zone, is an important ecological product of ocean tides.\"]\n", - "\t163.835\t['Tidal energy', \"Many things affect tides. The pull of the Moon is the largest effect, and most of the energy comes from the slowing of the Earth's spin.\"]\n", - "\t167.368\t['Storm surge', 'A storm surge is a sudden rise of water hitting areas close to the coast. Storm surges are usually created by a hurricane or other tropical cyclone. The surge happens because a storm has fast winds and low atmospheric pressure. Water is pushed on shore, and the water level rises. Strong storm surges can flood coastal towns and destroy homes. A storm surge is considered the deadliest part of a hurricane. They kill many people each year.']\n", - "\t177.143\t['Tidal force', 'Tidal force is caused by gravity and makes tides happen. This is because the gravitational field changes across the middle of a body (the diameter).']\n", - "\t186.108\t['Tsunami', \"A tsunami is a natural disaster which is a series of fast-moving waves in the ocean caused by powerful earthquakes, volcanic eruptions, landslides, or simply an asteroid or a meteor crash inside the ocean. A tsunami has a very long wavelength. It can be hundreds of kilometers long. Usually, a tsunami starts suddenly. The waves travel at a great speed across an ocean with little energy loss. They can remove sand from beaches, destroy trees, toss and drag vehicles, houses and even destroy whole towns. Tsunamis can even be caused when a meteorite strikes the earth's surface, though it is very rare. A tsunami normally occurs in the Pacific Ocean, especially in what is called the ring of fire, but can occur in any large body of water.\"]\n", - "CPU times: user 4.44 ms, sys: 4.65 ms, total: 9.09 ms\n", - "Wall time: 12.4 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "search_raft_pq(query = \"What is creating tides?\")" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "2d6017ed", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 208 ms, sys: 63.8 ms, total: 271 ms\n", - "Wall time: 286 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "params = ivf_flat.IndexParams(n_lists=150)\n", - "flat_index = ivf_flat.build(params, corpus_embeddings)\n", - "search_params = ivf_flat.SearchParams()\n", - "\n", - "def search_raft_flat(query, top_k = 5):\n", - " # Encode the query using the bi-encoder and find potentially relevant passages\n", - " question_embedding = bi_encoder.encode(query, convert_to_tensor=True)\n", - " \n", - " start_time = time.time()\n", - " hits = ivf_flat.search(search_params, flat_index, question_embedding[None], top_k)\n", - " end_time = time.time()\n", - "\n", - " # Output of top-k hits\n", - " print(\"Input question:\", query)\n", - " print(\"Results (after {:.3f} seconds):\".format(end_time - start_time))\n", - " for k in range(top_k):\n", - " print(\"\\t{:.3f}\\t{}\".format(hits[0][0, k], passages[hits[1][0, k]]))" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "f5cfb644", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Input question: Who was Grace Hopper?\n", - "Results (after 0.002 seconds):\n", - "\t181.650\t['Grace Hopper', 'Hopper was born in New York, USA. Hopper graduated from Vassar College in 1928 and Yale University in 1934 with a Ph.D degree in mathematics. She joined the US Navy during the World War II in 1943. She worked on computers in the Navy for 43 years. She then worked in other private industry companies after 1949. She retired from the Navy in 1986 and died on January 1, 1992.']\n", - "\t192.946\t['Leona Helmsley', 'Leona Helmsley (July 4, 1920 – August 20, 2007) was an American businesswoman. She was known for having a flamboyant personality. She had a reputation for tyrannical behavior; she was nicknamed the Queen of Mean.']\n", - "\t194.951\t['Grace Hopper', 'Grace Murray Hopper (December 9 1906 – January 1 1992) was an American computer scientist and United States Navy officer.']\n", - "\t202.192\t['Nellie Bly', 'Elizabeth Cochrane Seaman (born Elizabeth Jane Cochran; May 5, 1864 – January 27, 1922), better known by her pen name Nellie Bly, was an American journalist, novelist and inventor. She was a newspaper reporter, who worked at various jobs for exposing poor working conditions. Nellie Bly, also, fought for women\\'s right and was known for investigative reporting. She best known for her record-breaking trip around the world in 72 days, inspired by the adventure novel \"Around the World in Eighty Days\" by Jules Verne. In the 1880s, she went undercover as a mentally ill patient in a psychiatric hospital for ten days, with the report being made public in a book called \"\"Ten Days in a Mad-House\"\". She was added to the National Women\\'s Hall of Fame in 1998.']\n", - "\t205.038\t['Abbie Hoffman', 'Abbot Howard \"Abbie\" Hoffman (November 30, 1936 – April 12, 1989) was an American social and political activist.']\n", - "CPU times: user 6.48 ms, sys: 0 ns, total: 6.48 ms\n", - "Wall time: 6.22 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "search_raft_flat(query=\"Who was Grace Hopper?\")" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "b5694d00", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Input question: Who was Alan Turing?\n", - "Results (after 0.002 seconds):\n", - "\t106.131\t['Alan Turing', 'Alan Mathison Turing OBE FRS (London, 23 June 1912 – Wilmslow, Cheshire, 7 June 1954) was an English mathematician and computer scientist. He was born in Maida Vale, London.']\n", - "\t158.646\t['William Kahan', 'William Morton Kahan (born June 5, 1933) is a Canadian mathematician and computer scientist. He received the Turing Award in 1989 for \"\"his fundamental contributions to numerical analysis\".\" He was named an ACM Fellow in 1994, and added to the National Academy of Engineering in 2005.']\n", - "\t165.094\t['Alan Turing', 'A brilliant mathematician and cryptographer Alan was to become the founder of modern-day computer science and artificial intelligence; designing a machine at Bletchley Park to break secret Enigma encrypted messages used by the Nazi German war machine to protect sensitive commercial, diplomatic and military communications during World War 2. Thus, Turing made the single biggest contribution to the Allied victory in the war against Nazi Germany, possibly saving the lives of an estimated 2 million people, through his effort in shortening World War II.']\n", - "\t167.321\t['Rolf Noskwith', 'Rolf Noskwith (19 June 1919 – 3 January 2017) was a British businessman. During the Second World War, he worked under Alan Turing as a cryptographer at the British military base Bletchley Park in Milton Keynes, Buckinghamshire.']\n", - "\t176.480\t['Marvin Minsky', \"Marvin Lee Minsky (August 9, 1927 – January 24, 2016) was an American cognitive scientist in the field of artificial intelligence (AI). He was the co-founder of the Massachusetts Institute of Technology's AI laboratory, and author of several texts on AI and philosophy. He won the Turing Award in 1969.\"]\n", - "CPU times: user 4.81 ms, sys: 1.19 ms, total: 6 ms\n", - "Wall time: 6.06 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "search_raft_flat(query=\"Who was Alan Turing?\")" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "fcfc3c5b", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Input question: What is creating tides?\n", - "Results (after 0.002 seconds):\n", - "\t94.909\t['Tide', \"A tide is the periodic rising and falling of Earth's ocean surface caused mainly by the gravitational pull of the Moon acting on the oceans. Tides cause changes in the depth of marine and estuarine (river mouth) waters. Tides also make oscillating currents known as tidal streams (~'rip tides'). This means that being able to predict the tide is important for coastal navigation. The strip of seashore that is under water at high tide and exposed at low tide, called the intertidal zone, is an important ecological product of ocean tides.\"]\n", - "\t159.539\t['Tidal energy', \"Many things affect tides. The pull of the Moon is the largest effect, and most of the energy comes from the slowing of the Earth's spin.\"]\n", - "\t159.740\t['Storm surge', 'A storm surge is a sudden rise of water hitting areas close to the coast. Storm surges are usually created by a hurricane or other tropical cyclone. The surge happens because a storm has fast winds and low atmospheric pressure. Water is pushed on shore, and the water level rises. Strong storm surges can flood coastal towns and destroy homes. A storm surge is considered the deadliest part of a hurricane. They kill many people each year.']\n", - "\t178.283\t['Sea', 'Wind blowing over the surface of a body of water forms waves. The friction between air and water caused by a gentle breeze on a pond causes ripples to form. A strong blow over the ocean causes larger waves as the moving air pushes against the raised ridges of water. The waves reach their greatest height when the rate at which they travel nearly matches the speed of the wind. The waves form at right angles to the direction from which the wind blows. In open water, if the wind continues to blow, as happens in the Roaring Forties in the southern hemisphere, long, organized masses of water called swell roll across the ocean. If the wind dies down, the wave formation is reduced but waves already formed continue to travel in their original direction until they meet land. Small waves form in small areas of water with islands and other landmasses but large waves form in open stretches of sea where the wind blows steadily and strongly. When waves meet other waves coming from different directions, interference between the two can produce broken, irregular seas.']\n", - "\t181.498\t['Tidal force', 'Tidal force is caused by gravity and makes tides happen. This is because the gravitational field changes across the middle of a body (the diameter).']\n", - "CPU times: user 5.91 ms, sys: 0 ns, total: 5.91 ms\n", - "Wall time: 5.65 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "search_raft_flat(query = \"What is creating tides?\")" - ] - }, - { - "cell_type": "markdown", - "id": "a59d7b32-0832-4c3a-864e-aeb2e6e7fe1f", - "metadata": {}, - "source": [ - "## Using CAGRA: GPU graph-based Vector Search\n", - "\n", - "CAGRA is a graph-based nearest neighbors implementation with state-of-the art query performance for both small- and large-batch sized vector searches. \n", - "\n", - "CAGRA follows the same two-step APIs as IVF-FLAT and IVF-PQ in RAFT. First we build the index:" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "50df1f43-c580-4019-949a-06bdc7185536", - "metadata": {}, - "outputs": [], - "source": [ - "from pylibraft.neighbors import cagra" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "091cde52-4652-4230-af2b-75c35357f833", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 35.3 s, sys: 4.5 s, total: 39.8 s\n", - "Wall time: 2.16 s\n" - ] - } - ], - "source": [ - "%%time\n", - "params = cagra.IndexParams(intermediate_graph_degree=32, graph_degree=16, build_algo=\"nn_descent\")\n", - "cagra_index = cagra.build(params, corpus_embeddings)\n", - "search_params = cagra.SearchParams(algo=\"multi_cta\")" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "df229e21-f6b6-4d6c-ad54-2724f8738934", - "metadata": {}, - "outputs": [], - "source": [ - "def search_raft_cagra(query, top_k = 5):\n", - " # Encode the query using the bi-encoder and find potentially relevant passages\n", - " question_embedding = bi_encoder.encode(query, convert_to_tensor=True)\n", - "\n", - " start_time = time.time()\n", - " hits = cagra.search(search_params, cagra_index, question_embedding[None], top_k)\n", - " end_time = time.time()\n", - "\n", - " # Output of top-k hits\n", - " print(\"Results (after {:.3f} seconds):\".format(end_time - start_time))\n", - " print(\"Input question:\", query)\n", - " for k in range(top_k):\n", - " print(\"\\t{:.3f}\\t{}\".format(hits[0][0, k], passages[hits[1][0, k]]))" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "b5e862fd-b7e5-4423-8fbf-36918f02c8f3", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Results (after 0.005 seconds):\n", - "Input question: Who was Grace Hopper?\n", - "\t181.649\t['Grace Hopper', 'Hopper was born in New York, USA. Hopper graduated from Vassar College in 1928 and Yale University in 1934 with a Ph.D degree in mathematics. She joined the US Navy during the World War II in 1943. She worked on computers in the Navy for 43 years. She then worked in other private industry companies after 1949. She retired from the Navy in 1986 and died on January 1, 1992.']\n", - "\t192.946\t['Leona Helmsley', 'Leona Helmsley (July 4, 1920 – August 20, 2007) was an American businesswoman. She was known for having a flamboyant personality. She had a reputation for tyrannical behavior; she was nicknamed the Queen of Mean.']\n", - "\t194.951\t['Grace Hopper', 'Grace Murray Hopper (December 9 1906 – January 1 1992) was an American computer scientist and United States Navy officer.']\n", - "\t202.192\t['Nellie Bly', 'Elizabeth Cochrane Seaman (born Elizabeth Jane Cochran; May 5, 1864 – January 27, 1922), better known by her pen name Nellie Bly, was an American journalist, novelist and inventor. She was a newspaper reporter, who worked at various jobs for exposing poor working conditions. Nellie Bly, also, fought for women\\'s right and was known for investigative reporting. She best known for her record-breaking trip around the world in 72 days, inspired by the adventure novel \"Around the World in Eighty Days\" by Jules Verne. In the 1880s, she went undercover as a mentally ill patient in a psychiatric hospital for ten days, with the report being made public in a book called \"\"Ten Days in a Mad-House\"\". She was added to the National Women\\'s Hall of Fame in 1998.']\n", - "\t205.038\t['Abbie Hoffman', 'Abbot Howard \"Abbie\" Hoffman (November 30, 1936 – April 12, 1989) was an American social and political activist.']\n", - "CPU times: user 4.18 ms, sys: 3.88 ms, total: 8.07 ms\n", - "Wall time: 9.97 ms\n" - ] - } - ], - "source": [ - "%%time \n", - "search_raft_cagra(query=\"Who was Grace Hopper?\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/ivf_flat_example.ipynb b/notebooks/ivf_flat_example.ipynb deleted file mode 100644 index 08b9d78169..0000000000 --- a/notebooks/ivf_flat_example.ipynb +++ /dev/null @@ -1,674 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "4f49c5c4-1170-42a7-9d6a-b90acd00c3c3", - "metadata": {}, - "source": [ - "# RAFT IVF Flat Example Notebook" - ] - }, - { - "cell_type": "markdown", - "id": "4bcfe810-f120-422c-b2bb-72cc43d0c4ca", - "metadata": {}, - "source": [ - "## Introduction\n", - "\n", - "This notebook demonstrates how to run approximate nearest neighbor search using RAFT IVF-Flat algorithm.\n", - "It builds and searches an index using a dataset from the ann-benchmarks million-scale datasets, saves/loads the index to disk, and explores important parameters for fine-tuning the search performance and accuracy of the index." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "fe73ada7-7b7f-4005-9440-85428194311b", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import cupy as cp\n", - "import numpy as np\n", - "from pylibraft.common import DeviceResources\n", - "from pylibraft.neighbors import ivf_flat\n", - "import matplotlib.pyplot as plt\n", - "import tempfile\n", - "from utils import BenchmarkTimer, calc_recall, load_dataset" - ] - }, - { - "cell_type": "markdown", - "id": "da9e8615-ea9f-4735-b70f-15ccab36c0d9", - "metadata": {}, - "source": [ - "For best performance it is recommended to use an RMM pooling allocator, to minimize the overheads of repeated CUDA allocations." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "5350e4d9-0993-406a-80af-29538b5677c2", - "metadata": {}, - "outputs": [], - "source": [ - "import rmm\n", - "from rmm.allocators.cupy import rmm_cupy_allocator\n", - "mr = rmm.mr.PoolMemoryResource(\n", - " rmm.mr.CudaMemoryResource(),\n", - " initial_pool_size=2**30\n", - ")\n", - "rmm.mr.set_current_device_resource(mr)\n", - "cp.cuda.set_allocator(rmm_cupy_allocator)" - ] - }, - { - "cell_type": "markdown", - "id": "b0d935f2-ba24-44fc-bdfe-a769b7fcd8e6", - "metadata": {}, - "source": [ - "The following GPU is used for this notebook" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "a5daa4b4-96de-4e74-bfd6-505b13595f62", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Thu Sep 21 02:30:53 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 NVIDIA H100 PCIe On | 00000000:41:00.0 Off | 0 |\n", - "| N/A 35C P0 69W / 350W | 1487MiB / 81559MiB | 0% Default |\n", - "| | | Disabled |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "| 0 N/A N/A 3940 C /opt/conda/envs/rapids/bin/python 1474MiB |\n", - "+---------------------------------------------------------------------------------------+\n" - ] - } - ], - "source": [ - "# Report the GPU in use\n", - "!nvidia-smi" - ] - }, - { - "cell_type": "markdown", - "id": "88a654cc-6389-4526-a3e6-826de5606a09", - "metadata": {}, - "source": [ - "## Load dataset\n", - "\n", - "The ANN benchmarks website provides the datasets in HDF5 format.\n", - "\n", - "The list of prepared datasets can be found at https://github.com/erikbern/ann-benchmarks/#data-sets" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "5f529ad6-b0bd-495c-bf7c-43f10fb6aa14", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The index and data will be saved in /tmp/raft_example\n" - ] - } - ], - "source": [ - "WORK_FOLDER = os.path.join(tempfile.gettempdir(), \"raft_example\")\n", - "f = load_dataset(\"http://ann-benchmarks.com/sift-128-euclidean.hdf5\", work_folder=WORK_FOLDER)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "3d68a7db-bcf4-449c-96c3-1e8ab146c84d", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loaded dataset of size (1000000, 128), 0.5 GiB; metric: 'euclidean'.\n", - "Number of test queries: 10000\n" - ] - } - ], - "source": [ - "metric = f.attrs['distance']\n", - "\n", - "dataset = cp.array(f['train'])\n", - "queries = cp.array(f['test'])\n", - "gt_neighbors = cp.array(f['neighbors'])\n", - "gt_distances = cp.array(f['distances'])\n", - "\n", - "itemsize = dataset.dtype.itemsize \n", - "\n", - "print(f\"Loaded dataset of size {dataset.shape}, {dataset.size*itemsize/(1<<30):4.1f} GiB; metric: '{metric}'.\")\n", - "print(f\"Number of test queries: {queries.shape[0]}\")" - ] - }, - { - "cell_type": "markdown", - "id": "9f463c50-d1d3-49be-bcfe-952602efa603", - "metadata": {}, - "source": [ - "## Build index\n", - "We set [IndexParams](https://docs.rapids.ai/api/raft/nightly/pylibraft_api/neighbors/#pylibraft.neighbors.ivf_flat.IndexParams) and build the index. The index parameters will be discussed in more detail in later sections of this notebook." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "737f8841-93f9-4c8e-b2e1-787d4474ef94", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 120 ms, sys: 5.33 ms, total: 125 ms\n", - "Wall time: 124 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "build_params = ivf_flat.IndexParams(\n", - " n_lists=1024,\n", - " metric=\"euclidean\",\n", - " kmeans_trainset_fraction=0.1,\n", - " kmeans_n_iters=20,\n", - " add_data_on_build=True\n", - " )\n", - "\n", - "index = ivf_flat.build(build_params, dataset)" - ] - }, - { - "cell_type": "markdown", - "id": "a16a0cf6-3b05-4afd-9bb8-54431e0d7439", - "metadata": {}, - "source": [ - "The index is built. We can print some basic information of the index" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "1aec7024-6e5d-4d2c-82e6-7b5734aec958", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Index(type=IVF-FLAT, metric=euclidean, size=1000000, dim=128, n_lists=1024, adaptive_centers=False)\n" - ] - } - ], - "source": [ - "print(index)" - ] - }, - { - "cell_type": "markdown", - "id": "df7d4958-56a3-48ea-bd64-3486fdb57fb7", - "metadata": {}, - "source": [ - "## Search neighbors" - ] - }, - { - "cell_type": "markdown", - "id": "89ba2eaa-4c85-4e1c-b07c-920394e55dce", - "metadata": {}, - "source": [ - "It is recommended to reuse [device recosources](https://docs.rapids.ai/api/raft/nightly/pylibraft_api/common/#pylibraft.common.DeviceResources) across multiple invocations of search, since constructing these can be time consuming. We will reuse the resources by passing the same handle to each RAFT API call." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "46e0421b-9335-47a2-8451-a91f56c2f086", - "metadata": {}, - "outputs": [], - "source": [ - "handle = DeviceResources()" - ] - }, - { - "cell_type": "markdown", - "id": "a6365229-18fd-468f-af30-e24b950cbd6e", - "metadata": {}, - "source": [ - "After setting [SearchParams](https://docs.rapids.ai/api/raft/nightly/pylibraft_api/neighbors/#pylibraft.neighbors.ivf_flat.SearchParams) we search for for `k=10` neighbors." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "595454e1-7240-4b43-9a73-963d5670b00c", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 171 ms, sys: 52.6 ms, total: 224 ms\n", - "Wall time: 236 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "n_queries=10000\n", - "# n_probes is the number of clusters we select in the first (coarse) search step. This is the only hyper parameter for search.\n", - "search_params = ivf_flat.SearchParams(n_probes=30)\n", - "\n", - "# Search 10 nearest neighbors.\n", - "distances, indices = ivf_flat.search(search_params, index, cp.asarray(queries[:n_queries,:]), k=10, handle=handle)\n", - " \n", - "# RAFT calls are asynchronous (when handle arg is provided), we need to sync before accessing the results.\n", - "handle.sync()\n", - "distances, neighbors = cp.asnumpy(distances), cp.asnumpy(indices)" - ] - }, - { - "cell_type": "markdown", - "id": "43d20ca7-7b9e-4046-bb52-640a2744db75", - "metadata": {}, - "source": [ - "The returned arrays have shape {n_queries x 10] and store the distance values and the indices of the searched vectors. We check how accurate the search is. The accuracy of the search is quantified as `recall`, which is a value between 0 and 1 and tells us what fraction of the returned neighbors are actual k nearest neighbors. " - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "8cd9cd20-ca00-4a35-a0a0-86636521b31a", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.97406" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "calc_recall(neighbors, gt_neighbors)" - ] - }, - { - "cell_type": "markdown", - "id": "cde5079c-9777-45a1-9545-cffbcc59988f", - "metadata": {}, - "source": [ - "## Save and load the index\n", - "You can serialize the index to file using [save](https://docs.rapids.ai/api/raft/nightly/pylibraft_api/neighbors/#pylibraft.neighbors.ivf_flat.save), and [load](https://docs.rapids.ai/api/raft/nightly/pylibraft_api/neighbors/#pylibraft.neighbors.ivf_flat.load) it later." - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "bf94e45c-e7fb-4aa3-a611-ddaee7ac41ae", - "metadata": {}, - "outputs": [], - "source": [ - "index_file = os.path.join(WORK_FOLDER, \"my_ivf_flat_index.bin\")\n", - "ivf_flat.save(index_file, index)" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "1622d9be-be41-4d25-be99-d348c5e54957", - "metadata": {}, - "outputs": [], - "source": [ - "index = ivf_flat.load(index_file)" - ] - }, - { - "cell_type": "markdown", - "id": "15d503e5-05e8-47ce-8501-e13fc512099c", - "metadata": {}, - "source": [ - "## Tune search parameters\n", - "Search has a single hyper parameter: `n_probes`, which describes how many neighboring cluster is searched (probed) for each query. Within a probed cluster, the distance is computed between all the vectors in the cluster and the query point, and the top-k neighbors are selected. Finally, the top-k neighbors are selected from all the neighbor candidates from the probed clusters.\n", - "\n", - "Let's see how search accuracy and latency changes when we change the `n_probes` parameter." - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "ace0c31f-af75-4352-a438-123a9a03612c", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Benchmarking search with n_probes = 10\n", - "recall 0.86625\n", - "Average search time: 0.026 +/- 0.000259 s\n", - "Queries per second (QPS): 384968\n", - "\n", - "Benchmarking search with n_probes = 20\n", - "recall 0.94705\n", - "Average search time: 0.050 +/- 5.43e-05 s\n", - "Queries per second (QPS): 198880\n", - "\n", - "Benchmarking search with n_probes = 30\n", - "recall 0.97406\n", - "Average search time: 0.075 +/- 8.59e-05 s\n", - "Queries per second (QPS): 133954\n", - "\n", - "Benchmarking search with n_probes = 50\n", - "recall 0.99169\n", - "Average search time: 0.123 +/- 4.78e-05 s\n", - "Queries per second (QPS): 80997\n", - "\n", - "Benchmarking search with n_probes = 100\n", - "recall 0.99844\n", - "Average search time: 0.244 +/- 0.000249 s\n", - "Queries per second (QPS): 40934\n", - "\n", - "Benchmarking search with n_probes = 200\n", - "recall 0.99932\n", - "Average search time: 0.468 +/- 0.000367 s\n", - "Queries per second (QPS): 21382\n", - "\n", - "Benchmarking search with n_probes = 500\n", - "recall 0.99933\n", - "Average search time: 1.039 +/- 0.000209 s\n", - "Queries per second (QPS): 9625\n", - "\n", - "Benchmarking search with n_probes = 1024\n", - "recall 0.99935\n", - "Average search time: 0.701 +/- 0.00579 s\n", - "Queries per second (QPS): 14273\n" - ] - } - ], - "source": [ - "n_probes = np.asarray([10, 20, 30, 50, 100, 200, 500, 1024]);\n", - "qps = np.zeros(n_probes.shape);\n", - "recall = np.zeros(n_probes.shape);\n", - "\n", - "for i in range(len(n_probes)):\n", - " print(\"\\nBenchmarking search with n_probes =\", n_probes[i])\n", - " timer = BenchmarkTimer(reps=1, warmup=1)\n", - " for rep in timer.benchmark_runs():\n", - " distances, neighbors = ivf_flat.search(\n", - " ivf_flat.SearchParams(n_probes=n_probes[i]),\n", - " index,\n", - " cp.asarray(queries),\n", - " k=10,\n", - " handle=handle,\n", - " )\n", - " handle.sync()\n", - " \n", - " recall[i] = calc_recall(cp.asnumpy(neighbors), gt_neighbors)\n", - " print(\"recall\", recall[i])\n", - "\n", - " timings = np.asarray(timer.timings)\n", - " avg_time = timings.mean()\n", - " std_time = timings.std()\n", - " qps[i] = queries.shape[0] / avg_time\n", - " print(\"Average search time: {0:7.3f} +/- {1:7.3} s\".format(avg_time, std_time))\n", - " print(\"Queries per second (QPS): {0:8.0f}\".format(qps[i]))" - ] - }, - { - "cell_type": "markdown", - "id": "20b2498c-7231-4211-990e-600d5c26a9a1", - "metadata": {}, - "source": [ - "The plots below illustrate how the accuracy (recall) and the throughput (queries per second) depends on the `n_probes` parameter." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e1ac370f-91c8-4054-95c7-a749df5f16d2", - "metadata": {}, - "outputs": [], - "source": [ - "fig = plt.figure(figsize=(12,3))\n", - "ax = fig.add_subplot(131)\n", - "ax.plot(n_probes, recall,'o-')\n", - "#ax.set_xticks(bench_k, bench_k)\n", - "ax.set_xlabel('n_probes')\n", - "ax.grid()\n", - "ax.set_ylabel('recall (@k=10)')\n", - "\n", - "ax = fig.add_subplot(132)\n", - "ax.plot(n_probes, qps,'o-')\n", - "#ax.set_xticks(bench_k, bench_k)\n", - "ax.set_xlabel('n_probes')\n", - "ax.grid()\n", - "ax.set_ylabel('queries per second');\n", - "\n", - "ax = fig.add_subplot(133)\n", - "ax.plot(recall, qps,'o-')\n", - "#ax.set_xticks(bench_k, bench_k)\n", - "ax.set_xlabel('recall')\n", - "ax.grid()\n", - "ax.set_ylabel('queries per second');\n", - "#ax.set_yscale('log')" - ] - }, - { - "cell_type": "markdown", - "id": "81e7ad6a-bddc-45de-9cce-0fb913f91efe", - "metadata": {}, - "source": [ - "## Adjust build parameters\n", - "### n_lists\n", - "The number of clusters (or lists) is set by the n_list parameter. Let's change it to 100 clusters." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "addbfff3-7773-4290-9608-5489edf4886d", - "metadata": {}, - "outputs": [], - "source": [ - "%%time\n", - "build_params = ivf_flat.IndexParams(\n", - " n_lists=100,\n", - " metric=\"euclidean\",\n", - " kmeans_trainset_fraction=1,\n", - " kmeans_n_iters=20,\n", - " add_data_on_build=True\n", - " )\n", - "\n", - "index = ivf_flat.build(build_params, dataset, handle=handle)" - ] - }, - { - "cell_type": "markdown", - "id": "48db27f9-54c8-4dac-839b-af94ada8885f", - "metadata": {}, - "source": [ - "The ratio of n_probes / n_list will determine how large fraction of the dataset is searched for each query. The right combination depends on the use case. Here we will search 10 of the clusters for each query." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8a0149ad-de38-4195-97a5-ce5d5d877036", - "metadata": {}, - "outputs": [], - "source": [ - "%%time\n", - "n_queries=10000\n", - "\n", - "search_params = ivf_flat.SearchParams(n_probes=10)\n", - "\n", - "# Search 10 nearest neighbors.\n", - "distances, indices = ivf_flat.search(search_params, index, cp.asarray(queries[:n_queries,:]), k=10, handle=handle)\n", - " \n", - "handle.sync()\n", - "distances, neighbors = cp.asnumpy(distances), cp.asnumpy(indices)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "eedc3ec4-06af-42c5-8cdf-490a5c2bc49a", - "metadata": {}, - "outputs": [], - "source": [ - "calc_recall(neighbors, gt_neighbors)" - ] - }, - { - "cell_type": "markdown", - "id": "0c44800f-1e9e-4f7b-87fe-0f25e6590faa", - "metadata": {}, - "source": [ - "### trainset_fraction\n", - "During clustering we can sub-sample the dataset. The parameter `trainset_fraction` determines what fraction to use. Often we get good results by using only 1/10th of the dataset for clustering. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5a54d190-64d4-4cd4-a497-365cbffda871", - "metadata": {}, - "outputs": [], - "source": [ - "%%time\n", - "build_params = ivf_flat.IndexParams( \n", - " n_lists=100, \n", - " metric=\"sqeuclidean\", \n", - " kmeans_trainset_fraction=0.1, \n", - " kmeans_n_iters=20 \n", - " ) \n", - "index = ivf_flat.build(build_params, dataset, handle=handle)" - ] - }, - { - "cell_type": "markdown", - "id": "9d86a213-d6ae-4fca-9082-cb5a4d1dab36", - "metadata": {}, - "source": [ - "We see only a minimal change in the recall" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4cc992e8-a5e5-4508-b790-0e934160b660", - "metadata": {}, - "outputs": [], - "source": [ - "search_params = ivf_flat.SearchParams(n_probes=10)\n", - "\n", - "distances, indices = ivf_flat.search(search_params, index, cp.asarray(queries[:n_queries,:]), k=10, handle=handle)\n", - " \n", - "handle.sync()\n", - "distances, neighbors = cp.asnumpy(distances), cp.asnumpy(indices)\n", - "calc_recall(neighbors, gt_neighbors)" - ] - }, - { - "cell_type": "markdown", - "id": "25289ebc-7d89-4fa6-bc62-e25b6e77750c", - "metadata": {}, - "source": [ - "### Add vectors on build\n", - "Currently you cannot configure how RAFT sub-samples the input. If you want to have a fine control on how the training set is selected, then create the index in two steps:\n", - "1. Define cluster centers on a training set, but do not add any vector to the index\n", - "2. Add vectors to the index (extend)\n", - "\n", - "This workflow shall be familiar to FAISS users. Note that raft does not require adding the data in batches, internal batching is used when necessary.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7ebcf970-94ed-4825-9885-277bd984b90c", - "metadata": {}, - "outputs": [], - "source": [ - "# subsample the dataset\n", - "n_train = 10000\n", - "train_set = dataset[cp.random.choice(dataset.shape[0], n_train, replace=False),:]\n", - "\n", - "# build using training set\n", - "build_params = ivf_flat.IndexParams(\n", - " n_lists=1024,\n", - " metric=\"sqeuclidean\",\n", - " kmeans_trainset_fraction=1,\n", - " kmeans_n_iters=20,\n", - " add_data_on_build=False\n", - " )\n", - "index = ivf_flat.build(build_params, train_set)\n", - "\n", - "print(\"Index before adding vectors\", index)\n", - "\n", - "ivf_flat.extend(index, dataset, cp.arange(dataset.shape[0], dtype=cp.int64))\n", - "\n", - "print(\"Index after adding vectors\", index)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "029d48a9-baf7-4263-af43-9e500ef3cce4", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.11" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/tutorial_ivf_pq.ipynb b/notebooks/tutorial_ivf_pq.ipynb deleted file mode 100644 index 397e39bfba..0000000000 --- a/notebooks/tutorial_ivf_pq.ipynb +++ /dev/null @@ -1,1365 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# RAFT IVF-PQ tutorial\n", - "In this tutorial you will learn to build IVF-PQ index and use it to search approximate nearest neighbors (ANN).\n", - "We will start with a brief overview of the functionality, but then dive into details to gain the understanding of the model parameters.\n", - "Along the way, we will benchmark the model and give some practical recommendations on how to maximize its performance for various use cases.\n", - "\n", - "This tutorial uses the data from [ANN benchmarks website](https://ann-benchmarks.com)." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collecting adjustText\n", - " Downloading adjustText-0.8-py3-none-any.whl (9.1 kB)\n", - "Collecting h5py\n", - " Downloading h5py-3.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.8 MB)\n", - "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.8/4.8 MB\u001b[0m \u001b[31m46.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n", - "\u001b[?25hCollecting matplotlib\n", - " Downloading matplotlib-3.7.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.6 MB)\n", - "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.6/11.6 MB\u001b[0m \u001b[31m97.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: numpy in /opt/conda/envs/cuml_dev/lib/python3.9/site-packages (from adjustText) (1.24.4)\n", - "Collecting contourpy>=1.0.1 (from matplotlib)\n", - " Downloading contourpy-1.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (300 kB)\n", - "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m300.4/300.4 kB\u001b[0m \u001b[31m86.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting cycler>=0.10 (from matplotlib)\n", - " Downloading cycler-0.11.0-py3-none-any.whl (6.4 kB)\n", - "Collecting fonttools>=4.22.0 (from matplotlib)\n", - " Downloading fonttools-4.41.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.5 MB)\n", - "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.5/4.5 MB\u001b[0m \u001b[31m115.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m\n", - "\u001b[?25hCollecting kiwisolver>=1.0.1 (from matplotlib)\n", - " Downloading kiwisolver-1.4.4-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.6 MB)\n", - "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m119.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /opt/conda/envs/cuml_dev/lib/python3.9/site-packages (from matplotlib) (23.1)\n", - "Requirement already satisfied: pillow>=6.2.0 in /opt/conda/envs/cuml_dev/lib/python3.9/site-packages (from matplotlib) (10.0.0)\n", - "Collecting pyparsing<3.1,>=2.3.1 (from matplotlib)\n", - " Downloading pyparsing-3.0.9-py3-none-any.whl (98 kB)\n", - "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m98.3/98.3 kB\u001b[0m \u001b[31m43.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: python-dateutil>=2.7 in /opt/conda/envs/cuml_dev/lib/python3.9/site-packages (from matplotlib) (2.8.2)\n", - "Collecting importlib-resources>=3.2.0 (from matplotlib)\n", - " Downloading importlib_resources-6.0.0-py3-none-any.whl (31 kB)\n", - "Requirement already satisfied: zipp>=3.1.0 in /opt/conda/envs/cuml_dev/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib) (3.15.0)\n", - "Requirement already satisfied: six>=1.5 in /opt/conda/envs/cuml_dev/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n", - "Installing collected packages: pyparsing, kiwisolver, importlib-resources, h5py, fonttools, cycler, contourpy, matplotlib, adjustText\n", - "Successfully installed adjustText-0.8 contourpy-1.1.0 cycler-0.11.0 fonttools-4.41.1 h5py-3.9.0 importlib-resources-6.0.0 kiwisolver-1.4.4 matplotlib-3.7.2 pyparsing-3.0.9\n" - ] - } - ], - "source": [ - "!pip install adjustText h5py matplotlib" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import tempfile\n", - "import cupy as cp\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "import rmm\n", - "import urllib.request\n", - "import h5py\n", - "\n", - "from rmm.allocators.cupy import rmm_cupy_allocator\n", - "from pylibraft.common import DeviceResources\n", - "from pylibraft.neighbors import ivf_pq, refine\n", - "from adjustText import adjust_text\n", - "from utils import calc_recall, load_dataset\n", - "\n", - "%matplotlib inline" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "# A clumsy helper for inspecting properties of an object\n", - "def show_properties(obj):\n", - " return {\n", - " attr: getattr(obj, attr)\n", - " for attr in dir(obj)\n", - " if type(getattr(type(obj), attr)).__name__ == 'getset_descriptor'\n", - " }" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The index and data will be saved in /tmp/raft_ivf_pq_tutorial\n" - ] - } - ], - "source": [ - "# We'll need to load store some data in this tutorial\n", - "WORK_FOLDER = os.path.join(tempfile.gettempdir(), 'raft_ivf_pq_tutorial')\n", - "\n", - "if not os.path.exists(WORK_FOLDER):\n", - " os.makedirs(WORK_FOLDER)\n", - "print(\"The index and data will be saved in\", WORK_FOLDER)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fri Jul 28 08:21:25 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.49 Driver Version: 535.49 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 NVIDIA H100 PCIe On | 00000000:41:00.0 Off | 0 |\n", - "| N/A 34C P0 46W / 350W | 4MiB / 81559MiB | 0% Default |\n", - "| | | Disabled |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "| No running processes found |\n", - "+---------------------------------------------------------------------------------------+\n" - ] - } - ], - "source": [ - "# Report the GPU in use to put the measurements into perspective\n", - "!nvidia-smi" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Use the pool memory resource\n", - "RAFT uses RMM allocator widely across its algorithms, including the performance-sensitive parts like IVF-PQ search.\n", - "It's strongly advised to set up the RMM pool memory resource to minimize the overheads of repeated CUDA allocations.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "pool = rmm.mr.PoolMemoryResource(\n", - " rmm.mr.CudaMemoryResource(),\n", - " initial_pool_size=2**30\n", - ")\n", - "rmm.mr.set_current_device_resource(pool)\n", - "cp.cuda.set_allocator(rmm_cupy_allocator)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Get the data\n", - "The [ANN benchmarks website](https://ann-benchmarks.com) provides the datasets in [HDF5 format](https://www.hdfgroup.org/solutions/hdf5/).\n", - "\n", - "The list of prepared datasets can be found at https://github.com/erikbern/ann-benchmarks/#data-sets" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The index and data will be saved in /tmp/raft_example\n" - ] - } - ], - "source": [ - "DATASET_URL = \"http://ann-benchmarks.com/sift-128-euclidean.hdf5\"\n", - "f = load_dataset(DATASET_URL)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Load the dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loaded dataset of size (1000000, 128); metric: 'euclidean'.\n", - "Number of test queries: 10000\n" - ] - } - ], - "source": [ - "metric = f.attrs['distance']\n", - "\n", - "dataset = cp.array(f['train'])\n", - "queries = cp.array(f['test'])\n", - "gt_neighbors = cp.array(f['neighbors'])\n", - "gt_distances = cp.array(f['distances'])\n", - "\n", - "print(f\"Loaded dataset of size {dataset.shape}; metric: '{metric}'.\")\n", - "print(f\"Number of test queries: {queries.shape[0]}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Build the index\n", - "Construction of the index generally consists of two phases: training (building the clusters) and filling-in (extending the index with data).\n", - "In the first phase, a balanced hierarchical k-means algorithm clusters the training data.\n", - "In the second phase, the new data is classified and added into the appropriate clusters in the index.\n", - "Hence, a user should call `ivf_pq.build` once and then possibly `ivf_pq.extend` several times.\n", - "Though for user convenience `ivf_pq.build` by default adds the whole training set into the index." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "# RAFT's DeviceResources controls the GPU, cuda stream, memory policies etc.\n", - "# For now, we just create a default instance.\n", - "resources = DeviceResources()" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'add_data_on_build': True,\n", - " 'codebook_kind': 0,\n", - " 'conservative_memory_allocation': False,\n", - " 'force_random_rotation': False,\n", - " 'kmeans_n_iters': 20,\n", - " 'kmeans_trainset_fraction': 0.5,\n", - " 'metric': 1,\n", - " 'n_lists': 1024,\n", - " 'pq_bits': 8,\n", - " 'pq_dim': 64}" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# First, we need to initialize the build/indexing parameters.\n", - "# One of the more important parameters is the product quantisation (PQ) dim.\n", - "# Effectively, this parameter says\n", - "# \"shrink the dataset to this dimensionality to reduce the index size\".\n", - "# It must be not bigger than the dataset dim,\n", - "# and it should be divisible by 32 for better GPU performance.\n", - "pq_dim = 1\n", - "while pq_dim * 2 < dataset.shape[1]:\n", - " pq_dim = pq_dim * 2\n", - "# We'll use the ANN-benchmarks-provided metric and sensible defaults for the rest of parameters.\n", - "index_params = ivf_pq.IndexParams(n_lists=1024, metric=metric, pq_dim=pq_dim)\n", - "\n", - "show_properties(index_params)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 1.71 s, sys: 16.2 ms, total: 1.72 s\n", - "Wall time: 1.71 s\n" - ] - }, - { - "data": { - "text/plain": [ - "Index(type=IVF-PQ, metric=euclidean, codebook=subspace, size=1000000, dim=128, pq_dim=64, pq_bits=8, n_lists=1024, rot_dim=128)" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "## Build the index\n", - "# This function takes a row-major either numpy or cupy (GPU) array.\n", - "# Generally, it's a bit faster with GPU inputs, but the CPU version may come in handy\n", - "# if the whole dataset cannot fit into GPU memory.\n", - "index = ivf_pq.build(index_params, dataset, handle=resources)\n", - "# This function is asynchronous so we need to explicitly synchronize the GPU before we can measure the execution time\n", - "resources.sync()\n", - "index" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Index serialization\n", - "For bigger datasets, building an index can take some time. To avoid building the index from scratch every time you need it, you can save it to a file. Here is how this works:" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 89.7 ms, sys: 56 ms, total: 146 ms\n", - "Wall time: 145 ms\n" - ] - }, - { - "data": { - "text/plain": [ - "Index(type=IVF-PQ, metric=euclidean, codebook=subspace, size=1000000, dim=128, pq_dim=64, pq_bits=8, n_lists=1024, rot_dim=128)" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "index_filepath = os.path.join(WORK_FOLDER, \"ivf_pq.bin\")\n", - "ivf_pq.save(index_filepath, index) \n", - "loaded_index = ivf_pq.load(index_filepath)\n", - "resources.sync()\n", - "index" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Search\n", - "The search function returns the requested number `k` of (approximate) nearest neighbor in no particular order.\n", - "Besides the queries and `k`, the function can take a few more parameters to tweak the performance of the algorithm.\n", - "Again, these are passed via the struct with some sensible defaults." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'internal_distance_dtype': 0, 'lut_dtype': 0, 'n_probes': 20}" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "k = 10\n", - "search_params = ivf_pq.SearchParams()\n", - "show_properties(search_params)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 19.9 ms, sys: 12.3 ms, total: 32.2 ms\n", - "Wall time: 31.5 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "distances, neighbors = ivf_pq.search(search_params, index, queries, k, handle=resources)\n", - "# Sync the GPU to make sure we've got the timing right\n", - "resources.sync()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Measuring the quality of the predictions\n", - "We use [recall](https://en.wikipedia.org/wiki/Precision_and_recall) to measure the quality of the prediction." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Got recall = 0.85409 with the default parameters (k = 10).\n" - ] - } - ], - "source": [ - "recall_first_try = calc_recall(neighbors, gt_neighbors)\n", - "print(f\"Got recall = {recall_first_try} with the default parameters (k = {k}).\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Refine\n", - "Let's improve our results a little bit!\n", - "The refinement operation follows an approximate NN search.\n", - "It recomputes the exact distances for the already selected candidates and selects a subset of them thus improving the recall." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 193 ms, sys: 142 µs, total: 193 ms\n", - "Wall time: 191 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "candidates = ivf_pq.search(search_params, index, queries, k * 2, handle=resources)[1]\n", - "distances, neighbors = refine(dataset, queries, candidates, k, handle=resources)\n", - "resources.sync()" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Got recall = 0.94953 with 2x refinement (k = 10).\n" - ] - } - ], - "source": [ - "recall_refine2x = calc_recall(neighbors, gt_neighbors)\n", - "print(f\"Got recall = {recall_refine2x} with 2x refinement (k = {k}).\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Tweaking search parameters\n", - "Before diving deep into tweaking the model, let's quickly define the performance metrics.\n", - "As we've mentioned earlier, we use the recall to measure the quality of prediction.\n", - "The other important metric is the speed of the search.\n", - "We measure the speed in terms of queries per second (QPS).\n", - "\n", - "Most of the time, by changing the model parameters we balance the trade-off between the QPS and the recall." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Number of neighbors\n", - "Let's see how QPS depens on `k`. " - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "16.5 ms ± 13.3 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "17 ms ± 2.12 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "17.5 ms ± 2.92 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "18 ms ± 3.05 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "18.7 ms ± 4.25 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "23.4 ms ± 45.5 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "25.9 ms ± 5.49 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "40.2 ms ± 12.7 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "23.6 ms ± 26.6 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "28.7 ms ± 18.5 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA00AAAGwCAYAAAB1kI7CAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAACENklEQVR4nOzdeViVdf7/8ec5h305ICAisoi7uIsbZqalUpHVZGXlmGOmo1+tUWZabEzLFsvfpDVl2ao1ZYsz7ZaKlpqJqSju+4aKgAubIHCA8/sDPRPjdizgZnk9rotLz7k/575fvD0gb+7787lNdrvdjoiIiIiIiFyU2egAIiIiIiIiNZmaJhERERERkctQ0yQiIiIiInIZappEREREREQuQ02TiIiIiIjIZahpEhERERERuQw1TSIiIiIiIpfhYnSA+qSsrIy0tDR8fX0xmUxGxxERERERqdfsdjt5eXmEhoZiNl/6fJKapmqUlpZGeHi40TFERERERORXjhw5QlhY2CW3q2mqRr6+vkD5P4rVajUsh81mY+nSpQwaNAhXV1fDctQGqpXzVCvnqVbOU62cp1pdHdXLeaqV81Qr59WUWuXm5hIeHu74Of1S1DRVo/OX5FmtVsObJi8vL6xWq76gr0C1cp5q5TzVynmqlfNUq6ujejlPtXKeauW8mlarK02d0UIQIiIiIiIil6GmSURERERE5DLUNImIiIiIiFyGmiYREREREZHLUNMkIiIiIiJyGWqaRERERERELkNNk4iIiIiIyGWoaRIREREREbkMNU0iIiIiIiKXoaZJRERERETkMtQ0iYiIiIiIXIaaJhERERERkctQ0yQiIiIiInIZLkYHkOp16GQ+j/9nM9ZiM757T9K9WRC+Hq5GxxIRERERqbHUNNUzq/edZO3BLMDM0g82YjZBmxAr3Zs2oFvTALo1bUBjP0+jY4qIiIiI1BhqmuqZ69sE8+xt0Xy1ZhvpJd4cyTrLjuO57Diey/tJhwFo4u9ZoYlqFeyL2WwyOLmIiIiIiDEMn9N07Ngx/vjHPxIYGIinpycdOnRgw4YNju12u52pU6fSuHFjPD09GTBgAHv37q2wj9OnTzNs2DCsViv+/v6MGjWKM2fOVBizZcsWrr32Wjw8PAgPD2fmzJkXZFm4cCFt2rTBw8ODDh068N1331XY7kyWmi7U35Oh3cL4Y4syfki4ll+euIE593Vl5DVN6dDED7MJjmWf5cuUNKZ8uY0bX/6JztOXMnLeOub8uI9fDpyi0FZq9KchIiIiIlJtDD3TlJWVxTXXXEP//v35/vvvadiwIXv37qVBgwaOMTNnzuSf//wn77//PlFRUTz55JPExcWxY8cOPDw8ABg2bBjHjx8nMTERm83GyJEjGTNmDAsWLAAgNzeXQYMGMWDAAObOncvWrVt54IEH8Pf3Z8yYMQCsWbOGe++9lxkzZnDLLbewYMECbr/9djZu3Ej79u2dzlLbNLJ6EN+xMfEdGwNwpqiElNRs1h86TfLhLDamZpFbWMKPu0/w4+4TALhaTHRo4kf3pgF0axpATGQDArzdjPw0RERERESqjKFN04svvkh4eDjz5s1zPBcVFeX4u91u5+WXX2bKlCncdtttAHzwwQc0atSIL7/8knvuuYedO3eyePFi1q9fT7du3QB49dVXufnmm/nHP/5BaGgoH330EcXFxbz33nu4ubnRrl07UlJSmDVrlqNpeuWVV7jxxht55JFHAHjmmWdITEzktddeY+7cuU5lqQt83F3o0zKIPi2DACgpLWPn8TzWHzrNhsOnWX8oixN5RWxMzWZjajZvrjoAQPOG3o4mqnvTBkQEeGEy6ZI+EREREan9DG2avv76a+Li4rjrrrtYuXIlTZo04f/+7/8YPXo0AAcPHiQ9PZ0BAwY4XuPn50fPnj1JSkrinnvuISkpCX9/f0fDBDBgwADMZjO//PILf/jDH0hKSqJv3764uf33bEhcXBwvvvgiWVlZNGjQgKSkJBISEirki4uL48svv3Q6y/8qKiqiqKjI8Tg3NxcAm82GzWb7HZX7fc4f29kMbRp50aaRF8N7hmG32zmSdZbkw9kkp2ax4XA2+0/kOz4+WX8EgCAfN2Ii/ImJbEC3SH/ahvjiYjH8atCrdrW1qs9UK+epVs5TrZynWl0d1ct5qpXzVCvn1ZRaOXt8Q5umAwcO8MYbb5CQkMATTzzB+vXrefjhh3Fzc2PEiBGkp6cD0KhRowqva9SokWNbeno6wcHBFba7uLgQEBBQYcyvz2D9ep/p6ek0aNCA9PT0Kx7nSln+14wZM3j66acveH7p0qV4eXldoirVJzEx8Te/1h3o7Qq9W0B+JBzMM3Hg3EfqGTh5ppglOzJZsiMTADeznUgfO82s0MzXTlNfOx6WSvpEqsHvqVV9o1o5T7VynmrlPNXq6qhezlOtnKdaOc/oWhUUFDg1ztCmqaysjG7duvH8888D0KVLF7Zt28bcuXMZMWKEkdEqxeTJkyucvcrNzSU8PJxBgwZhtVoNy2Wz2UhMTGTgwIG4ulb+PZqKbKVsTcsl+XA2Gw5nsTE1m9zCEvbmmthbfrINswnaNvala0QDukX4ExPpTyNrzZsXVtW1qktUK+epVs5TrZynWl0d1ct5qpXzVCvn1ZRanb8S7EoMbZoaN25MdHR0hefatm3Lf/7zHwBCQkIAyMjIoHHjxo4xGRkZdO7c2TEmMzOzwj5KSko4ffq04/UhISFkZGRUGHP+8ZXG/Hr7lbL8L3d3d9zd3S943tXVtUZ8IVVVDldXV2JbeBDbovwMYFmZnb2ZZ9hw+DQbDmWx/tBpjmadZXtaHtvT8vjX2lQAwgM86RZZvsx596YBtGjoU2OWOq8p/2a1gWrlPNXKeaqV81Srq6N6OU+1cp5q5Tyja+XssQ1tmq655hp2795d4bk9e/YQGRkJlC8KERISwvLlyx2NSW5uLr/88gvjxo0DIDY2luzsbJKTk4mJiQHghx9+oKysjJ49ezrG/P3vf8dmszkKk5iYSOvWrR0r9cXGxrJ8+XImTpzoyJKYmEhsbKzTWeTizGYTrUN8aR3iy7Ce5f+26TmFFZqoncdzOXL6LEdOH+OLTccA8PN0LZ8Tda6J6tDEDw/XWnRNn4iIiIjUCYY2TZMmTaJ37948//zz3H333axbt4633nqLt956CwCTycTEiRN59tlnadmypWOZ79DQUG6//Xag/MzUjTfeyOjRo5k7dy42m40JEyZwzz33EBoaCsB9993H008/zahRo3jsscfYtm0br7zyCrNnz3Zk+ctf/sJ1113HSy+9RHx8PJ988gkbNmy4qizivBA/D27pGMotHcv/jfIKbWxKLb+cb8Oh02xKzSbnrI0fdmXyw65z86IsZjqG+RHTtAHdI8uXOm+gpc5FREREpIoZ2jR1796dL774gsmTJzN9+nSioqJ4+eWXGTZsmGPMo48+Sn5+PmPGjCE7O5s+ffqwePHiCvdF+uijj5gwYQI33HADZrOZIUOG8M9//tOx3c/Pj6VLlzJ+/HhiYmIICgpi6tSpjuXGAXr37s2CBQuYMmUKTzzxBC1btuTLL7903KPJ2Szy2/h6uNK3VUP6tmoIgK20jB1puY77Ra0/lMXJM0XlTdXhLN6kfKnzlsE+dGsaQLfI8rNR4QGeWupcRERERCqVoU0TwC233MItt9xyye0mk4np06czffr0S44JCAhw3Mj2Ujp27MhPP/102TF33XUXd9111+/KIpXD1WKmU7g/ncL9efDa8nt2HT5VUH6/qENZbDh8mv0n8tmbeYa9mWf4eF35vKhgX/dz94tqQLfIANo2rp1LnYuIiIhIzWF40yTiDJPJRNMgb5oGeXNXt3AATp0pIvncmaf1h06z7VgOmXlFLNp6nEVbjwPg5WYpX6Hv3LyozuH+eLvrbS8iIiIiztNPj1JrBfq4M6hdCIPala9sWGgrZfORbEcTlXw4i7zCElbvO8nqfScBsJhNRDe2OpqobpENCK6BS52LiIiISM2hpknqDA9XCz2bBdKzWSBQvtT5nsw81h8qX1xiw6EsjmWfZeuxHLYey2Hez4cAiAjwcjRR3Zs2oFlQzVnqXERERESMp6ZJ6iyz2USbECttQqwM71W+1Hla9lnHCn3rD2WxKz2X1NMFpJ4u4PON5Uud+3u50i2yAd2aBtAlzEpJmZGfhYiIiIgYTU2T1Cuh/p7c6u/JrZ3KlzrPLbSx8XDWuRX6TpNyJJvsAhvLdmaybGf5UucWk4X5R9fSObwBnc8tTtEsyFtno0RERETqCTVNUq9ZPVzp1zqYfq2DASguKWN7Wo6jiVp/6DSn821sPZbL1mO5/GvtYQB83V3oGO5Hp7DyJqpzuD+NNDdKREREpE5S0yTyK24uZrpENKBLRAMevLYZxcXFfPjF9wS07Mq2tDw2H81m67Ec8opK+HnfKX7ed8rx2hCrh+NMVKdwPzo08cPXw9XAz0ZEREREKoOaJpHLMJlMBHrAzR1CuL1r+VLnJaVl7Mk4Q8qRbDYfyWbz0Wz2ZOSRnlvI4u3pLN6efu610KKhj+N+U53D/Gkd4oubi+4bJSIiIlKbqGkSuUouFjPRoVaiQ63c1zMCgPyiErYdy2Hz0Ww2H8kh5Ug2x7LPOm6+++/ko0D5maz2oVbHJX2dwvyJDPTCZNL8KBEREZGaSk2TSCXwdnepsNw5wIm8IseZqPNnpXILS9iYms3G1GzHOD9P13NnovwcZ6WCfNwN+CxERERE5GLUNIlUkYa+7gyIbsSA6EYA2O12Dp0qYPORc03U0Wy2p+WSc9bGqj0nWLXnhOO1YQ08HZf0dQr3p30TK15u+nIVERERMYJ+ChOpJiaTiaggb6KCvLm9SxOgfLW+3el5pBzNJiW1vJHaf+IMR7POcjTrLIu2HAfAbIJWjXwdC010DvenZbAPLhbNjxIRERGpamqaRAzk5mKmQ5gfHcL8HDfgzS20se1oDilHzy00cSSH9NxCdqXnsSs9j0/WHwHA09VChyZ+dAo/d1lfmD9hDTw1P0pERESkkqlpEqlhrB6u9G4RRO8WQY7n0nMKK8yN2nI0hzNFJaw7dJp1h047xgV6u/13kYlwfzqF+eHv5WbEpyEiIiJSZ6hpEqkFQvw8CPELIa5dCABlZXYOnDxDypEcx2ITO4/nciq/mB92ZfLDrkzHa5sGejnORHUK96ddqBUPV4tRn4qIiIhIraOmSaQWMptNtAj2pUWwL3fGhAFQaCtl5/HcXy00kcPBk/kcOlXAoVMFfJWSBoCL2USbxr6OJc87h/vTrKEPFrMu6xMRERG5GDVNInWEh6uFLhEN6BLRwPFcdkExW47mVFj6/OSZYrYdy2XbsVw+JBUAH3eXc/Oj/Okc7kfn8AaE+HkY9amIiIiI1ChqmkTqMH8vN/q2akjfVg2B8mXP03IKzy0wkc2mI9lsPTc/KunAKZIOnHK8tpHV3XFJX+dwfzqE+WH1cDXqUxERERExjJomkXrEZDLRxN+TJv6e3NyhMQAlpWXsO3Hm3GV95WeldmfkkZFbxNIdGSzdkeF4ffOG3nQK96fLuYUm2oRYcXPRsuciIiJSt6lpEqnnXCxm2oRYaRNiZWj38ufOFpeyPS3HMTcq5UgWR06fZf+JfPafyOfzjccAcLOYiQ610jncn/aNfTh9tnyRChEREZG6RE2TiFzA081Ct6YBdGsa4Hju1Jkithw930iVX96XVWAj5dzCE+VceHnHD7RtbKV9qJV2oX60a2KlZbCvzkiJiIhIraWmSUScEujjTv82wfRvEwyUz486cvqs4ya8m1Kz2Ho0i4LiUpIPZ5F8OMvxWleLiVaNfGl/rolqF2qlbWMrXm76FiQiIiI1n35iEZHfxGQyERHoRUSgF7d2CsVms/Htou9o070ve04UsO1YDtvTctl2LIfcwhK2p+WyPS0XNpx/PTQL8qZ9Ez/ahVrLG6pQP/y8tNiEiIiI1CxqmkSk0phN0CLYh7ZNGnBb5yZA+Rmpo1lnzzVN/22kMvOKHHOkzt9DCqCJvyftm5Rf2nf+z2Bfd0wm3UdKREREjKGmSUSqlMlkIjzAi/AAL25sH+J4PjOvkO1puew410xtO5ZL6ukCjmWf5Vj2WZZs/++qfUE+buXzo0L/20xFBHipkRIREZFqoaZJRAwR7OtBcGsP+rcOdjyXc9bmaKJ2pOWyLS2HfZlnOHmmmJV7TrByzwnHWF93F6LPLzYRaqV9Ez+aN/TGxaIFJ0RERKRyqWkSkRrDz9OV2OaBxDYPdDx3triUXem5jjlR29Ny2JWeR15RCb8cPM0vB087xrq7mGnT2HrujFT5PKnWIb54uFqM+HRERESkjlDTJCI1mqebhS4RDegS0cDxnK20jH2ZZ/47T+pYLjuO53KmqITNR8pX8zvPYjbRoqHPuVX7/GgfaiU61IqvhxacEBEREeeoaRKRWsfVYqZt4/Jly++MCQPKb6p7+HSBY37U+UUnTucXszsjj90ZeY6b8gJEBnrRPtSP6HOX9rULtRLk427UpyQiIiI1mJomEakTzGYTUUHeRAV5c0vHUKB85b703EK2HyufH3V+4Ylj2Wc5fKqAw6cKWLT1uGMfIVYPx6V97c41Uk38PbXghIiISD2npklE6iyTyURjP08a+3kyILqR4/nT+cWOhSbOX+J38GQ+6bmFpOcWsnxXpmOsv5erY9W+839GBXljMauREhERqS/UNIlIvRPg7UaflkH0aRnkeC6/qISdx3P/e1PetFz2ZuSRXWDj532n+HnfKcdYLzcLbRv/d7GJ6FArrRr54uailftERETqIjVNIiKAt7sL3ZoG0K1pgOO5opJS9mac+VUjlcPO47kUFJeSfDiL5MNZjrGuFhOtGvk6lj9vF1o+58rLTd9mRUREajv9by4icgnuLhbaN/GjfRM/x3OlZXYOnjzjWGzi/J+5hSWOZdE/23AUAJMJmgV5O27Ie/4SP29XXdonIiJSm6hpEhG5ChaziRbBvrQI9uX2Lk2A8gUnjmaddazYd/7MVGZeEftP5LP/RD5fb05z7KOJvwdBZjOnA1Pp0zKYFsE+WmxCRESkBlPTJCLyO5lMJsIDvAgP8OLG9o0dz2fmFTpW7DvfSKWeLuBYdiHHMLP5213ALoJ83Ol97qa+vZsHEhHgpSZKRESkBjF01vJTTz2FyWSq8NGmTRvH9n79+l2wfezYsRX2kZqaSnx8PF5eXgQHB/PII49QUlJSYcyKFSvo2rUr7u7utGjRgvnz51+QZc6cOTRt2hQPDw969uzJunXrKmwvLCxk/PjxBAYG4uPjw5AhQ8jIyKi8YohInRPs60H/1sGM79+CN/4Yw6pH+7N52iA+fKAb8eGl9G4WgLuLmZNnivh6cxqTP9/Kdf9vBX1e/JG/fraZ/yQfJS37rNGfhoiISL1n+Jmmdu3asWzZMsdjF5eKkUaPHs306dMdj728vBx/Ly0tJT4+npCQENasWcPx48e5//77cXV15fnnnwfg4MGDxMfHM3bsWD766COWL1/Ogw8+SOPGjYmLiwPg008/JSEhgblz59KzZ09efvll4uLi2L17N8HBwQBMmjSJRYsWsXDhQvz8/JgwYQJ33HEHP//8c5XVRkTqHj9PV3pGBXAqzM7NN3ejFDMpR7JZs/8USftPknIkm2PZZ/nPxqP8Z2P53KimgV7ENg+id/NAejULpKGvbsIrIiJSnQxvmlxcXAgJCbnkdi8vr0tuX7p0KTt27GDZsmU0atSIzp0788wzz/DYY4/x1FNP4ebmxty5c4mKiuKll14CoG3btqxevZrZs2c7mqZZs2YxevRoRo4cCcDcuXNZtGgR7733Ho8//jg5OTm8++67LFiwgOuvvx6AefPm0bZtW9auXUuvXr0qsyQiUo94uFro1ay8GWJgKwqKS9hwKKu8iTpwiq1Hszl0qoBDp1L5eF0qAK0a+dC7eRCxzQPpFRWIn5erwZ+FiIhI3WZ407R3715CQ0Px8PAgNjaWGTNmEBER4dj+0Ucf8eGHHxISEsLgwYN58sknHWebkpKS6NChA40a/femlXFxcYwbN47t27fTpUsXkpKSGDBgQIVjxsXFMXHiRACKi4tJTk5m8uTJju1ms5kBAwaQlJQEQHJyMjabrcJ+2rRpQ0REBElJSZdsmoqKiigqKnI8zs3NBcBms2Gz2X5LuSrF+WMbmaG2UK2cp1o573K1cjVBbJQ/sVH+QHPyCm2sO5TFLwezSDpwml3peezJOMOejDPMX3MIkwmiG/vSKyqAXs0C6BbZAB93w7+1Vxq9r5ynWl0d1ct5qpXzVCvn1ZRaOXt8Q/9n7dmzJ/Pnz6d169YcP36cp59+mmuvvZZt27bh6+vLfffdR2RkJKGhoWzZsoXHHnuM3bt38/nnnwOQnp5eoWECHI/T09MvOyY3N5ezZ8+SlZVFaWnpRcfs2rXLsQ83Nzf8/f0vGHP+OBczY8YMnn766QueX7p0aYXLDI2SmJhodIRaQ7VynmrlvKupVWegcxScCYN9uSb25pjYm2si46yJ7Wl5bE/L492fD2PGToQPtPSz09LPTpSPHTdLlX0K1UbvK+epVldH9XKeauU81cp5RteqoKDAqXGGNk033XST4+8dO3akZ8+eREZG8tlnnzFq1CjGjBnj2N6hQwcaN27MDTfcwP79+2nevLkRka/K5MmTSUhIcDzOzc0lPDycQYMGYbVaDctls9lITExk4MCBuLrqsp7LUa2cp1o5rzJrlZFbyC8Hs1h78DRJB05zNOssh87AoTMmEo+V33S3S7g/vZoFENssgI5N/HBzMXQNoKui95XzVKuro3o5T7VynmrlvJpSq/NXgl1JjbqGw9/fn1atWrFv376Lbu/ZsycA+/bto3nz5oSEhFywyt35Fe3Oz4MKCQm5YJW7jIwMrFYrnp6eWCwWLBbLRcf8eh/FxcVkZ2dXONv06zEX4+7ujrv7hRO2XV1da8QXUk3JURuoVs5TrZxXGbUKC3QlLNCXId3KL2s+crqApAOnSNp/ijX7T5KRW8S6Q1msO5TFP3/Yj6erhW5NGzjmRLUPteJiqflNlN5XzlOtro7q5TzVynmqlfOMrpWzx65RTdOZM2fYv38/w4cPv+j2lJQUABo3Lr8PSmxsLM899xyZmZmOVe4SExOxWq1ER0c7xnz33XcV9pOYmEhsbCwAbm5uxMTEsHz5cm6//XYAysrKWL58ORMmTAAgJiYGV1dXli9fzpAhQwDYvXs3qampjv2IiNQE5+8XdXe3cOx2OwdP5jsWlVi7/xSn8ov5ae9Jftp7EgBfdxd6NgugV7NAejcPok2IL2az7hElIiLya4Y2TX/7298YPHgwkZGRpKWlMW3aNCwWC/feey/79+9nwYIF3HzzzQQGBrJlyxYmTZpE37596dixIwCDBg0iOjqa4cOHM3PmTNLT05kyZQrjx493nOEZO3Ysr732Go8++igPPPAAP/zwA5999hmLFi1y5EhISGDEiBF069aNHj168PLLL5Ofn+9YTc/Pz49Ro0aRkJBAQEAAVquVhx56iNjYWK2cJyI1lslkollDH5o19OGPvSIpK7OzJzPv3FmoU6w9cIq8whKW7cxk2c5MABp4uZ5roAKJbR5E84beutGuiIjUe4Y2TUePHuXee+/l1KlTNGzYkD59+rB27VoaNmxIYWEhy5YtczQw4eHhDBkyhClTpjheb7FY+Pbbbxk3bhyxsbF4e3szYsSICvd1ioqKYtGiRUyaNIlXXnmFsLAw3nnnHcdy4wBDhw7lxIkTTJ06lfT0dDp37szixYsrLA4xe/ZszGYzQ4YMoaioiLi4OF5//fXqKZSISCUwm020CbHSJsTKyGuiKC2zsyMtlzX7T7Jm/ynWHzpNVoGN77el8/228kVugn3diW1e3kT1bh5EeIDxi9iIiIhUN0Obpk8++eSS28LDw1m5cuUV9xEZGXnB5Xf/q1+/fmzatOmyYyZMmOC4HO9iPDw8mDNnDnPmzLliJhGR2sBiNtEhzI8OYX78+brm2ErL2HI0mzX7yi/n23A4i8y8Ir5KSeOrlDQAmvh7ljdQLQKJbRZEiJ+HwZ+FiIhI1atRc5pERMQ4rhYzMZEBxEQG8NANLSm0lbIxNYu15y7nSzmSzbHssyxMPsrC5KMANAvyJrZ5YPmNdpsFEuRz4eI3IiIitZ2aJhERuSgPVwu9mwfRu3kQCUB+UQnrD512rM637VgOB07mc+BkPh/9kgpAmxBfx5yons0C8fPU6lEiIlL7qWkSERGneLu70K91MP1al69WmnPWxrqDp1mz/yRJ+0+xKz3P8TF/zSHMJmgX6nduUYlAujcNwNtd/+2IiEjto/+9RETkN/HzdGVgdCMGRpcvmnPqTBFrD5xrog6c4sCJfLYey2HrsRzeXHUAF7OJTuH+5U1Us0C6RjbAw9Vi8GchIiJyZWqaRESkUgT6uBPfsTHxHcvvpZeeU0jSgfKzUD/vO8Wx7LMkH84i+XAWr/6wDzcXMzERDRyr83UM88fNpebfaFdEROofNU0iIlIlQvw8+EOXMP7QJQyAI6cLzt0jqnyJ88y8ovL5UQdOMSsRvNwsdGsacG5580DahfoZ/BmIiIiUU9MkIiLVIjzAi/AAL+7uHo7dbufAyXzW7D9F0rk5UVkFNlbtOcGqPScA8PVwoUfTBgQVmxhUWoar1pQQERGDqGkSEZFqZzKZaN7Qh+YNfRjeK5KyMju7M/IcTdQvB06TV1jC8l0nAAuu3+/m2T90NDq2iIjUU2qaRETEcGazibaNrbRtbGVUnyhKSsvYnpbLkm3HeX3lAT785QjXtgomrl2I0VFFRKQe0oxbERGpcVwsZjqF+zNpQAv6NS4D4G8LN3PkdIHByUREpD5S0yQiIjXarRFldA73I6+whPELNlJUUmp0JBERqWfUNImISI1mMcMrd3fEz9OVLUdzmPHdLqMjiYhIPaOmSUREarxQf09m3d0JgPlrDvH91uMGJxIRkfpETZOIiNQKN7RtxJ/7NgPg0X9v4fCpfIMTiYhIfaGmSUREao2/xbUmJrIBeUXl85sKbZrfJCIiVU9Nk4iI1BquFjOv3deFBl6ubDuWy3OLdhodSURE6gE1TSIiUqs09vNk1tDOAPxr7WG+2ZxmbCAREanz1DSJiEit0791MOP6NQdg8udbOXhS85tERKTqqGkSEZFa6a8DW9GjaQBnikoY/5HmN4mISNVR0yQiIrWSi8XMP+/tQqC3GzuO5zL92x1GRxIRkTpKTZOIiNRaIX4ezB7aGZMJFvySylcpx4yOJCIidZCaJhERqdX6tmrIhP4tAHji863sP3HG4EQiIlLXqGkSEZFa7y83tKRnVAD5xaWa3yQiIpVOTZOIiNR6LhYzr97bhSAfN3al5/HU19uNjiQiInWImiYREakTgq0evHJPF0wm+GT9Eb7YdNToSCIiUkeoaRIRkTrjmhZBPHx9SwCe+Hwb+zLzDE4kIiJ1gZomERGpUx6+oSW9mwdy1lbK/320kbPFmt8kIiK/j5omERGpUyxmEy/f05kgH3f2ZJxh6lfbjI4kIiK1nJomERGpc4J9PfjnvZ0xm2Bh8lH+naz5TSIi8tupaRIRkTqpd/MgJg5oBcCUL7eyJ0Pzm0RE5LdR0yQiInXW+P4tuLZlEIW2Mv7vo43kF5UYHUlERGohNU0iIlJnWcwmZg/tTLCvO/syz/Dkl9uw2+1GxxIRkVpGTZOIiNRpQT7u/PPeLphN8PmmYyzcoPlNIiJyddQ0iYhInderWSB/HdQagCe/2sau9FyDE4mISG2ipklEROqFcdc157pWDSkqKZ/fdEbzm0RExElqmkREpF4wm03MursTIVYPDpzI5+9fbNX8JhERcYqhTdNTTz2FyWSq8NGmTRvH9sLCQsaPH09gYCA+Pj4MGTKEjIyMCvtITU0lPj4eLy8vgoODeeSRRygpqfjbwxUrVtC1a1fc3d1p0aIF8+fPvyDLnDlzaNq0KR4eHvTs2ZN169ZV2O5MFhERqdkCfdx59b4uWMwmvkpJ45P1R4yOJCIitYDhZ5ratWvH8ePHHR+rV692bJs0aRLffPMNCxcuZOXKlaSlpXHHHXc4tpeWlhIfH09xcTFr1qzh/fffZ/78+UydOtUx5uDBg8THx9O/f39SUlKYOHEiDz74IEuWLHGM+fTTT0lISGDatGls3LiRTp06ERcXR2ZmptNZRESkdujeNIC/nZvfNO3r7exI0/wmERG5PMObJhcXF0JCQhwfQUFBAOTk5PDuu+8ya9Ysrr/+emJiYpg3bx5r1qxh7dq1ACxdupQdO3bw4Ycf0rlzZ2666SaeeeYZ5syZQ3FxMQBz584lKiqKl156ibZt2zJhwgTuvPNOZs+e7cgwa9YsRo8ezciRI4mOjmbu3Ll4eXnx3nvvOZ1FRERqjz/3bUb/1g0pLilj/IKN5BXajI4kIiI1mIvRAfbu3UtoaCgeHh7ExsYyY8YMIiIiSE5OxmazMWDAAMfYNm3aEBERQVJSEr169SIpKYkOHTrQqFEjx5i4uDjGjRvH9u3b6dKlC0lJSRX2cX7MxIkTASguLiY5OZnJkyc7tpvNZgYMGEBSUhKAU1kupqioiKKiIsfj3Nzy32babDZsNuP+gz5/bCMz1BaqlfNUK+epVs6rylq9eEc7bp2TxMGT+Tz+7y3MvrsDJpOp0o9TXfS+ujqql/NUK+epVs6rKbVy9viGNk09e/Zk/vz5tG7dmuPHj/P0009z7bXXsm3bNtLT03Fzc8Pf37/Caxo1akR6ejoA6enpFRqm89vPb7vcmNzcXM6ePUtWVhalpaUXHbNr1y7HPq6U5WJmzJjB008/fcHzS5cuxcvL65Kvqy6JiYlGR6g1VCvnqVbOU62cV1W1uicC/rndwqJt6XjlH6NPSO1fGELvq6ujejlPtXKeauU8o2tVUFDg1DhDm6abbrrJ8feOHTvSs2dPIiMj+eyzz/D09DQwWeWYPHkyCQkJjse5ubmEh4czaNAgrFarYblsNhuJiYkMHDgQV1dXw3LUBqqV81Qr56lWzquOWrmvPsSLS/bw1RFXht3Yg3ahxn1//j30vro6qpfzVCvnqVbOqym1On8l2JUYfnner/n7+9OqVSv27dvHwIEDKS4uJjs7u8IZnoyMDEJCQgAICQm5YJW78yva/XrM/65yl5GRgdVqxdPTE4vFgsViueiYX+/jSlkuxt3dHXd39wued3V1rRFfSDUlR22gWjlPtXKeauW8qqzV2H4tSE7NZtnOTP7y2Ra+eagPVo/a+++i99XVUb2cp1o5T7VyntG1cvbYhi8E8Wtnzpxh//79NG7cmJiYGFxdXVm+fLlj++7du0lNTSU2NhaA2NhYtm7dWmGVu8TERKxWK9HR0Y4xv97H+THn9+Hm5kZMTEyFMWVlZSxfvtwxxpksIiJSO5lMJv5xVyea+Hty+FQBj/9ni+7fJCIiFRjaNP3tb39j5cqVHDp0iDVr1vCHP/wBi8XCvffei5+fH6NGjSIhIYEff/yR5ORkRo4cSWxsrGPhhUGDBhEdHc3w4cPZvHkzS5YsYcqUKYwfP95xhmfs2LEcOHCARx99lF27dvH666/z2WefMWnSJEeOhIQE3n77bd5//3127tzJuHHjyM/PZ+TIkQBOZRERkdrL38uN1+7rgovZxHdb0/nX2sNGRxIRkRrE0Mvzjh49yr333supU6do2LAhffr0Ye3atTRs2BCA2bNnYzabGTJkCEVFRcTFxfH66687Xm+xWPj2228ZN24csbGxeHt7M2LECKZPn+4YExUVxaJFi5g0aRKvvPIKYWFhvPPOO8TFxTnGDB06lBMnTjB16lTS09Pp3LkzixcvrrA4xJWyiIhI7dYlogGP39SGZxft5Nlvd9IlvAEdwvyMjiUiIjWAoU3TJ598ctntHh4ezJkzhzlz5lxyTGRkJN99991l99OvXz82bdp02TETJkxgwoQJvyuLiIjUbqP6RLHu4GmW7sjg/xYk8+1D1+LnqXkJIiL1XY2a0yQiImIkk8nE/7uzE2ENPDly+iyP/nuz5jeJiIiaJhERkV/z83Jlzn1dcbWYWLI9g3k/HzI6koiIGExNk4iIyP/oFO7PEze3BWDG9ztJOZJtbCARETGUmiYREZGL+FPvptzUPgRbqZ3xH20kp8BmdCQRETGImiYREZGLMJlMvHhnRyICvDiWfZa/aX6TiEi9paZJRETkEqwerrw+rCtuFjOJOzJ4d/VBoyOJiIgB1DSJiIhcRvsmfjx5S/n8phe+38XG1CyDE4mISHVT0yQiInIFf+wVSXyHxpSU2XlowSayC4qNjiQiItVITZOIiMgVmEwmXhjSgaaB5fOb/vrZZsrKNL9JRKS+UNMkIiLiBF8PV+YM64qbi5nluzJ5+6cDRkcSEZFqoqZJRETESe1C/Zg2OBqAmUt2s+HQaYMTiYhIdVDTJCIichXu6xHBrZ1CKS2z89DHmzidr/lNIiJ1nZomERGRq2AymXj+jg40C/LmeE4hCZ+laH6TiEgdp6ZJRETkKvm4uzBnWFfcXcys2H2Cuav2Gx1JRESqkJomERGR36BtYytP39oOgJeW7mHdQc1vEhGpq9Q0iYiI/EZDu4fzhy5Nzs1v2sipM0VGRxIRkSqgpklEROQ3MplMPHt7e5o39CYjt4hJun+TiEidpKZJRETkd/B2d+H1YTF4uJpZtecEr6/YZ3QkERGpZGqaREREfqfWIb5Mv609ALMS95C0/5TBiUREpDKpaRIREakEd3cLZ0jXMMrs8PAnmziRp/lNIiJ1hZomERGRSvLM7e1oGezDibwiJn2aQqnmN4mI1AlqmkRERCqJl5sLrw/riqerhdX7TvLaD5rfJCJSF6hpEhERqUQtG/ny7O3l85teXr6HNftOGpxIRER+LzVNIiIilWxITBh3dwvDboeHP0khM6/Q6EgiIvI7qGkSERGpAk/f2p7WjXw5eaaIv3ys+U0iIrWZmiYREZEq4OlmYc6wrni5WUg6cIpXlu81OpKIiPxGappERESqSItgH57/QwcAXv1hLz/tPWFwIhER+S3UNImIiFSh27s04d4e4djtMPGTFDJyNb9JRKS2UdMkIiJSxaYNbkfbxlZO5Rfz0MebKCktMzqSiIhcBTVNIiIiVczD1cKc+7rg7WZh3cHTvLxM85tERGoTNU0iIiLVoFlDH2YM6QjAnBX7WLlH85tERGoLNU0iIiLV5NZOoQzrGYHdDpM+TeF4zlmjI4mIiBPUNImIiFSjJ2+Jpl2oldP5xTys+U0iIrWCmiYREZFqVD6/qSs+7i6sP5TFP5buMTqSiIhcgZomERGRatY0yJsXz81vmrtyPz/uyjQ4kYiIXI6aJhEREQPEd2zM/bGRAEz6LIW0bM1vEhGpqWpM0/TCCy9gMpmYOHGi47l+/fphMpkqfIwdO7bC61JTU4mPj8fLy4vg4GAeeeQRSkpKKoxZsWIFXbt2xd3dnRYtWjB//vwLjj9nzhyaNm2Kh4cHPXv2ZN26dRW2FxYWMn78eAIDA/Hx8WHIkCFkZGRU2ucvIiL1z9/j29KhiR/ZBTYmLNiITfObRERqpBrRNK1fv54333yTjh07XrBt9OjRHD9+3PExc+ZMx7bS0lLi4+MpLi5mzZo1vP/++8yfP5+pU6c6xhw8eJD4+Hj69+9PSkoKEydO5MEHH2TJkiWOMZ9++ikJCQlMmzaNjRs30qlTJ+Li4sjM/O/lEpMmTeKbb75h4cKFrFy5krS0NO64444qqoiIiNQH7i7l85t8PVzYmJrN/1uy2+hIIiJyEYY3TWfOnGHYsGG8/fbbNGjQ4ILtXl5ehISEOD6sVqtj29KlS9mxYwcffvghnTt35qabbuKZZ55hzpw5FBcXAzB37lyioqJ46aWXaNu2LRMmTODOO+9k9uzZjv3MmjWL0aNHM3LkSKKjo5k7dy5eXl689957AOTk5PDuu+8ya9Ysrr/+emJiYpg3bx5r1qxh7dq1VVwhERGpyyICvfh/d5b/0vCtVQdYtkNXMYiI1DQuRgcYP3488fHxDBgwgGefffaC7R999BEffvghISEhDB48mCeffBIvLy8AkpKS6NChA40aNXKMj4uLY9y4cWzfvp0uXbqQlJTEgAEDKuwzLi7OcRlgcXExycnJTJ482bHdbDYzYMAAkpKSAEhOTsZms1XYT5s2bYiIiCApKYlevXpd9HMrKiqiqKjI8Tg3NxcAm82GzWa7mjJVqvPHNjJDbaFaOU+1cp5q5bz6UqsbWgdxf68IPlibyl8XpvDV/8XSxN/zqvZRX2pVWVQv56lWzlOtnFdTauXs8Q1tmj755BM2btzI+vXrL7r9vvvuIzIyktDQULZs2cJjjz3G7t27+fzzzwFIT0+v0DABjsfp6emXHZObm8vZs2fJysqitLT0omN27drl2Iebmxv+/v4XjDl/nIuZMWMGTz/99AXPL1261NH4GSkxMdHoCLWGauU81cp5qpXz6kOtOtkhwttCan4JI95cxcPtSnH5DdeD1IdaVSbVy3mqlfNUK+cZXauCggKnxhnWNB05coS//OUvJCYm4uHhcdExY8aMcfy9Q4cONG7cmBtuuIH9+/fTvHnz6or6m02ePJmEhATH49zcXMLDwxk0aFCFywyrm81mIzExkYEDB+Lq6mpYjtpAtXKeauU81cp59a1WXa85y22vJ3H4TAnbLM154qbWTr+2vtXq91K9nKdaOU+1cl5NqdX5K8GuxLCmKTk5mczMTLp27ep4rrS0lFWrVvHaa69RVFSExWKp8JqePXsCsG/fPpo3b05ISMgFq9ydX9EuJCTE8ef/rnKXkZGB1WrF09MTi8WCxWK56Jhf76O4uJjs7OwKZ5t+PeZi3N3dcXd3v+B5V1fXGvGFVFNy1AaqlfNUK+epVs6rL7WKCnblH3d1Ysy/kpm35jC9mgcR1+7S/89cTH2pVWVRvZynWjlPtXKe0bVy9tiGLQRxww03sHXrVlJSUhwf3bp1Y9iwYaSkpFzQMAGkpKQA0LhxYwBiY2PZunVrhVXuEhMTsVqtREdHO8YsX768wn4SExOJjY0FwM3NjZiYmApjysrKWL58uWNMTEwMrq6uFcbs3r2b1NRUxxgREZHKMKhdCA/2iQLgkYWbOXLauUtHRESk6hh2psnX15f27dtXeM7b25vAwEDat2/P/v37WbBgATfffDOBgYFs2bKFSZMm0bdvX8fS5IMGDSI6Oprhw4czc+ZM0tPTmTJlCuPHj3ec4Rk7diyvvfYajz76KA888AA//PADn332GYsWLXIcNyEhgREjRtCtWzd69OjByy+/TH5+PiNHjgTAz8+PUaNGkZCQQEBAAFarlYceeojY2NhLLgIhIiLyWz16Yxs2HM4i5Ug2ExZsZOHY3rj9lglOIiJSKQxfPe9S3NzcWLZsmaOBCQ8PZ8iQIUyZMsUxxmKx8O233zJu3DhiY2Px9vZmxIgRTJ8+3TEmKiqKRYsWMWnSJF555RXCwsJ45513iIuLc4wZOnQoJ06cYOrUqaSnp9O5c2cWL15cYXGI2bNnYzabGTJkCEVFRcTFxfH6669XTzFERKRecXMx89p9XYj/52o2H83h+e928tSt7YyOJSJSb9WopmnFihWOv4eHh7Ny5corviYyMpLvvvvusmP69evHpk2bLjtmwoQJTJgw4ZLbPTw8mDNnDnPmzLliJhERkd8rrIEXs+7uxKj3NzB/zSF6RgVwU4fGRscSEamXdK5fRESkhrqhbSP+3LcZAI/+ewuppzS/SUTECGqaREREarC/xbUmJrIBeUUljF+wkaKSUqMjiYjUO2qaREREajBXi5lX7+1CAy9Xth7L4blFO42OJCJS71RK03T48GF27NhBWVlZZexOREREfiXU35NZQzsD8EHSYb7dkmZsIBGReuaqmqb33nuPWbNmVXhuzJgxNGvWjA4dOtC+fXuOHDlSqQFFREQE+rcOZly/5gA8/p+tHDyZb3AiEZH646qaprfeeosGDRo4Hi9evJh58+bxwQcfsH79evz9/Xn66acrPaSIiIjAXwe2onvTBpwpKmH8RxsptGl+k4hIdbiqpmnv3r1069bN8firr77itttuY9iwYXTt2pXnn3+e5cuXV3pIERERAReLmVfv7UqAtxs7jufyzLc7jI4kIlIvXFXTdPbsWaxWq+PxmjVr6Nu3r+Nxs2bNSE9Pr7x0IiIiUkGInwezh3bGZIKPfknlq5RjRkcSEanzrqppioyMJDk5GYCTJ0+yfft2rrnmGsf29PR0/Pz8KjehiIiIVHBdq4aM79cCgCc+38r+E2cMTiQiUrddVdM0YsQIxo8fzzPPPMNdd91FmzZtiImJcWxfs2YN7du3r/SQIiIiUtHEAS3pGRVAfnGp5jeJiFSxq2qaHn30UUaPHs3nn3+Oh4cHCxcurLD9559/5t57763UgCIiInIhl3P3bwrycWNXeh7PfrfL6EgiInWWy9UMNpvNTJ8+nenTp190+/82USIiIlJ1gq0evDy0C8Pf+4VPNxzDrYWJm40OJSJSB131zW0//fRThg0bxl133cXcuXOrIpOIiIg4qU/LIB66viUAnx4w8/P+UwYnEhGpe66qaXrjjTe499572bBhA3v37mX8+PE88sgjVZVNREREnDD62igAistM/Gl+MsPeWUvy4SyDU4mI1B1X1TS99tprTJs2jd27d5OSksL777/P66+/XlXZRERExAm+Hq4kP9GfaxuV4Wox8fO+Uwx5Yw0PzF/PtmM5RscTEan1rqppOnDgACNGjHA8vu+++ygpKeH48eOVHkxEREScZ/V05c5mZSRO7MPQbuFYzCZ+2JXJLa+uZtyHyezJyDM6okidt+1YLukFRqeQqnBVTVNRURHe3t7/fbHZjJubG2fPnq30YCIiInL1mvh78uKdHVmWcB23dw7FZILvt6UT9/IqJn6yiYMn842OKFIn7UjL4Q9z1zJjswvfbUs3Oo5UsqtaPQ/gySefxMvLy/G4uLiY5557rsJNbWfNmlU56UREROQ3iQry5uV7uvB//VswO3EP329L58uUNL7Zcpw7u4bx0A0tCGvgdeUdiYhTPvwl1fH3vy7cisVi4ZaOoQYmksp0VU1T37592b17d4XnevfuzYEDBxyPTSZT5SQTERGR361VI1/e+GMM247lMCtxDz/syuTTDUf4fNNR7u0Rwfj+LWhk9TA6pkitdvJMEf9OPgpAM187B/Lg4Y83cba4lLu6hRucTirDVTVNK1asqPD45MmTuLm5YbVaKzOTiIiIVLL2Tfx470/dST6cxazE3fy87xQfJB3m0/VHuD82krHXNSfQx93omCK10gdJhykuKaNjEysjw0+TZIvks+RjPPLvLZy1lXJ/bFOjI8rvdNX3acrOzmb8+PEEBQXRqFEjGjRoQEhICJMnT6agQDPfREREarKYyAZ89GAvFozuSUxkA4pKynj7p4NcO/NH/rFkNzkFNqMjitQqZ4tL+VfSIQAe7NMUswmevS2akdc0BWDqV9uZu3K/cQGlUlzVmabTp08TGxvLsWPHGDZsGG3btgVgx44dvPrqqyQmJrJ69Wq2bNnC2rVrefjhh6sktIiIiPw+vZsHETs2kBV7TvDS0t1sO5bLaz/u4/2kQ4y5thkj+0Th437VU59F6p1/Jx8hq8BGeIAnA9sGszS1fLrK1Fui8XZz4bUf9/HC97soKCph0sBWmspSS13Vd8Pp06fj5ubG/v37adSo0QXbBg0axPDhw1m6dCn//Oc/KzWoiIiIVC6TyUT/1sH0a9WQJdszmJW4mz0ZZ3gpcQ/v/XyQcf2aM7xXUzzdLEZHFamRSsvsvLP6IAAP9mmGi+W/F3GZTCb+FtcaL3cLMxfv5p8/7CO/uJQp8W3VONVCV3V53pdffsk//vGPCxomgJCQEGbOnMl//vMfEhISKtzPSURERGouk8nEje1D+P4vfXnlns5EBXmTVWDj+e92cd3/+5EPkg5RVFJqdEyRGmfp9nQOnyrA38uVu7qFXXTM//VrwVODowF4d/VBnvhiG2Vl9uqMKZXgqpqm48eP065du0tub9++PWazmWnTpv3uYCIiIlK9LGYTt3VuQuKkvsy8syNN/D3JzCti6lfbuf4fK/l0fSq20jKjY4rUCHa7nTdXla8gPbxXJF5ul76A60/XRDFzSEdMJvh4XSp/XbiZEn0t1SpX1TQFBQVx6NChS24/ePAgwcHBvzeTiIiIGMjFYububuH8+Ld+PHN7expZ3TmWfZbH/rOVgbNW8uWmY5TqN+VSz204nEXKkWzcXMxOrY53d/dwXrmnCxaziS82HWPCgk0Ul6hxqi2uqmmKi4vj73//O8XFxRdsKyoq4sknn+TGG2+stHAiIiJiHDcXM8N7RbLykf5MiW9LoLcbh04VMPHTFG58eRXfbz2uy4yk3nrr3FmmIV2b0NDXueX6b+0UyhvDuuJmMbN4ezpj/rWBQpsufa0Nrqppmj59Ort376Zly5bMnDmTr7/+mq+++ooXXniBli1bsnPnTp566qkqiioiIiJG8HC18OC1zVj1aH8eiWuN1cOFvZlnGPfRRga/tpofdmVgt6t5kvpj/4kzLNuZAcCoPs2u6rWD2oXw7p+64eFqZsXuE/xp3jrOFJVURUypRFfVNIWFhZGUlER0dDSTJ0/m9ttv5w9/+AN///vfiY6O5ueffyYiIqKqsoqIiIiBvN1dGN+/BT89dj0PX98CbzcL29NyeWD+Bu54Yw0/7zup5knqhXd+OojdDgPaNqJFsM9Vv/7alg354IGe+Li7sPbAaYa/+4vukVbDXfXNbaOiovj+++85efIka9euZe3atZw4cYLFixfTokWLqsgoIiIiNYifpysJg1rz02PX8+e+zfBwNbMpNZth7/zCvW+vZcOh00ZHFKkyJ/KK+M/GowCM6Xt1Z5l+rUdUAB892BN/L1c2pWZz79trOXWmqLJiSiW76qbpvAYNGtCjRw969OhBQEBAZWYSERGRWiDA243JN7dl1SP9+VPvprhZzKw9cJo75yYx4r11bDmabXREkUr3r6RDFJeU0Tncn+5NG/yufXUK9+eTMb0I8nFjx/Fc7n4zifScwkpKKpXpNzdNIiIiIgDBVg+eurUdPz7Sj3t7hGMxm1i55wS3vvYzYz7YwK70XKMjilSKguISPlh7GCg/y1QZN6ltE2Llsz/H0tjPg/0n8rn7zSSOnC743fuVyqWmSURERCpFE39PZtzRkeUJ13FHlyaYTLB0RwY3vfITD328if0nzhgdUeR3+XfyUbILbEQEeBHXLqTS9tusoQ+f/TmWiAAvUk8XcPebSRzQ10uNoqZJREREKlXTIG9mDe3M0ol9ie/QGLsdvtmcxsBZK3lk4Wb9Fl1qpdIyO+/8dBCAB6+NwmL+/WeZfi08wIvP/hxL84beHM8p5O431+osbQ2ipklERESqRMtGvswZ1pVFD/dhQNtgyuywMPko17+0gilfbtXcDalVlmxPJ/V0Af5ertwZE1Ylxwjx8+CzP8cS3djKyTNFDH1zLZuPZFfJseTqqGkSERGRKtUu1I93RnTni//rzbUtg7CV2vlwbSp9/9+PPPPtDk5qxTCp4ex2O2+eu5nt/b0i8XJzqbJjBfq48/HoXnQO9yfnrI1h7/zCeq1Iabga0zS98MILmEwmJk6c6HiusLCQ8ePHExgYiI+PD0OGDCEjI6PC61JTU4mPj8fLy4vg4GAeeeQRSkoq3iBsxYoVdO3aFXd3d1q0aMH8+fMvOP6cOXNo2rQpHh4e9OzZk3Xr1lXY7kwWERERubQuEQ3416iefDKmF92bNqC4pIx3Vx/k2hd/ZObiXWQXFBsdUeSi1h/KYvORbNxczAyPbVrlx/PzcuXDB3vSMyqAM0UlDH/3F37ae6LKjyuXViOapvXr1/Pmm2/SsWPHCs9PmjSJb775hoULF7Jy5UrS0tK44447HNtLS0uJj4+nuLiYNWvW8P777zN//nymTp3qGHPw4EHi4+Pp378/KSkpTJw4kQcffJAlS5Y4xnz66ackJCQwbdo0Nm7cSKdOnYiLiyMzM9PpLCIiIuKcXs0C+ezPsXzwQA86hflx1lbK6yv2c+2LP/LKsr3kFeomn1KzvHXuLNOQrmE09HWvlmP6uLswf2QPrmvVkEJbGaPmbyBxh35hbxTDm6YzZ84wbNgw3n77bRo0+O9a9zk5Obz77rvMmjWL66+/npiYGObNm8eaNWtYu3YtAEuXLmXHjh18+OGHdO7cmZtuuolnnnmGOXPmUFxc/tuquXPnEhUVxUsvvUTbtm2ZMGECd955J7Nnz3Yca9asWYwePZqRI0cSHR3N3Llz8fLy4r333nM6i4iIiDjPZDLRt1VDvhx/DW8Nj6FNiC95RSXMXraHa2f+yNyV+ykoLrnyjkSq2L7MMyzbmYHJVL4ARHXydLPw1v0x3NguhOLSMsZ+mMzXm9OqNYOUq7oLMp00fvx44uPjGTBgAM8++6zj+eTkZGw2GwMGDHA816ZNGyIiIkhKSqJXr14kJSXRoUMHGjVq5BgTFxfHuHHj2L59O126dCEpKanCPs6POX8ZYHFxMcnJyUyePNmx3Ww2M2DAAJKSkpzOcjFFRUUUFf33Ou3c3PIVUGw2Gzabcb9FO39sIzPUFqqV81Qr56lWzlOtnFeba9W/VSDXtejF99sz+OcP+zhwsoAXvt/FOz8dYGzfKO7pFoa7q6VSj1mb61Xd6nut3l61D4AbWjckwt/9snWoilqZgdl3tcfdxcRXm4/zl082ceZsMXfFNKm0YxihpryvnD2+oU3TJ598wsaNG1m/fv0F29LT03Fzc8Pf37/C840aNSI9Pd0x5tcN0/nt57ddbkxubi5nz54lKyuL0tLSi47ZtWuX01kuZsaMGTz99NMXPL906VK8vLwu+brqkpiYaHSEWkO1cp5q5TzVynmqlfNqc61MwIQWkOxnYvFRMyfPFPPsd7t5bdku4sLK6NnQjqWSr5GpzfWqbvWxVrnF8J+NFsBEtOU433133KnXVUWt+nnCiWAzazLNPPHldpJTttC3sb3Sj1PdjH5fFRQ4dwsEw5qmI0eO8Je//IXExEQ8PDyMilGlJk+eTEJCguNxbm4u4eHhDBo0CKvValgum81GYmIiAwcOxNXV1bActYFq5TzVynmqlfNUK+fVpVoNBp4oKeM/m44xZ8UBMnKL+PSAhTVZnjzUvzm3dmr8u++RU5fqVdXqc61mL9tHif0AncP9mDC0BybT5d93VV2reLudGYv3MG/NYf5zyEJUy5b8uW/1XjJYWWrK++r8lWBXYljTlJycTGZmJl27dnU8V1payqpVq3jttddYsmQJxcXFZGdnVzjDk5GRQUhI+R2YQ0JCLljl7vyKdr8e87+r3GVkZGC1WvH09MRisWCxWC465tf7uFKWi3F3d8fd/cLJgq6urjXim05NyVEbqFbOU62cp1o5T7VyXl2plasr3N+7GXd3j2TBL6m8vmIfR7LO8ujn23jzp4NMGtiKm9s3xvw7m6e6Uq/qUN9qVVBcwoL1RwD4c9/muLm5Of3aqqzV1MHt8PVw5Z8/7OMfiXspLLHz10GtrtjQ1VRGv6+cPbZhC0HccMMNbN26lZSUFMdHt27dGDZsmOPvrq6uLF++3PGa3bt3k5qaSmxsLACxsbFs3bq1wip3iYmJWK1WoqOjHWN+vY/zY87vw83NjZiYmApjysrKWL58uWNMTEzMFbOIiIhI5fNwtfBAnyhWPdqfx25sg5+nK/tP5DNhwSbiX13Nsh0Z2O21/xIlqXkWbjhKdoGNyEAvBrW79C/Jq5vJZCJhUGseu7ENAK/9uI9nvt2pr4MqZtiZJl9fX9q3b1/hOW9vbwIDAx3Pjxo1ioSEBAICArBarTz00EPExsY6Fl4YNGgQ0dHRDB8+nJkzZ5Kens6UKVMYP3684wzP2LFjee2113j00Ud54IEH+OGHH/jss89YtGiR47gJCQmMGDGCbt260aNHD15++WXy8/MZOXIkAH5+flfMIiIiIlXHy82Fcf2aM6xXBO+tPsg7Px1k5/FcHvxgA53C/fnboFb0aRFUa3/bLjVLSWkZ76wuX2b8wT5Rv/ty0Kowrl9zvN0tTP1qO+/9fJCC4hKe+0OHGpm1LjB89bzLmT17NmazmSFDhlBUVERcXByvv/66Y7vFYuHbb79l3LhxxMbG4u3tzYgRI5g+fbpjTFRUFIsWLWLSpEm88sorhIWF8c477xAXF+cYM3ToUE6cOMHUqVNJT0+nc+fOLF68uMLiEFfKIiIiIlXP6uHKxAGtGBHblLd+OsD8nw+x+Ug2w99dR4+oAP46sBU9mwUaHVNquSXbMzhy+iwNvFy5Mybc6DiXdH9sUzxdLTz2ny18sv4IZ22lvHRXJ1wqe8UUqVlN04oVKyo89vDwYM6cOcyZM+eSr4mMjOS777677H779evHpk2bLjtmwoQJTJgw4ZLbnckiIiIi1aOBtxuP3diGB66J4vUV+/hobSrrDp5m6FtrubZlEH8d1JrO4f5Gx5RayG6389aq/QAMj22Kp1vlLndf2e7qFo6Hq4VJn6bwVUoahbZS/nlvF9xdanbu2kZtqIiIiNRaDX3dmTa4HSse6cd9PSNwMZv4ae9Jbp/zMw++v4Edac6tjCVy3rqDp9l8NAd3FzP3x0YaHccpgzuFMvePMbi5mFmyPYPRHyRztrjU6Fh1ipomERERqfVC/T15/g8d+OGv/RjSNQyzCZbtzODmf/7E+AUb2ZeZZ3REqSXeWlU+l2lITBhBPheuglxTDYhuxHsjuuPpamHVnhOMmLeOM0UlRseqM9Q0iYiISJ0REejFS3d3Yumk67ilY2MAFm05zqDZq0j4LIXUU87dyFLqp32ZeSzflYnJVL4ARG3Tp2UQH4zqga+7C+sOnmbYO7+QU2AzOladoKZJRERE6pwWwT68dl9Xvv/LtQyMbkSZHT7feIzrX1rB5M+3cjyn0OiIUgO9veogAAPbNqJZQx+D0/w23ZsG8NHonvh7ubL5SDb3vL2Wk2eKjI5V66lpEhERkTqrbWMrb9/fjS/HX0PfVg0pKbPz8bpUbpj9E+tPaGlm+a/MvEK+2HQMgDF9mxmc5vfpGObPp2NiCfJxZ+fxXO5+M4l0/aLgd1HTJCIiInVe53B/PnigB5/9OZYeUQHYSu38+6BZv4EXhw/WHKa4tIyuEf50axpgdJzfrXWILwvHxhLq58GBE/nc9eYajpzW5am/lZomERERqTd6RAXwyehetA+1UlhqYtayfUZHkhogv6iEf609DNT+s0y/FhXkzWdjY4kM9OLI6bPcNTeJ/SfOGB2rVlLTJCIiIvWK2Wziyfg2APx74zG2Hs0xOJEYbeGGI+SctdE00IuB0SFGx6lUYQ28+OzPsbQM9iE9t5Chbyax87iW4r9aappERESk3uka4U9MUBl2Ozz9zXbsdrvRkcQgJaVlvLO6fAGIUdc2w2Kue3PdGlk9+GRML9qFWjl5pph73lpLypFso2PVKmqaREREpF66NaIMT1czGw5n8fXmNKPjiEEWb0/naNZZArzduLNrmNFxqkygjzsLRveia4Q/OWdt/PGdX/jlwCmjY9UaappERESkXvJ3hz+fm7/ywve7KCjWjUDrG7vd7riZ7fBekXi6WQxOVLX8PF3516iexDYL5ExRCSPmrWPlnhNGx6oV1DSJiIhIvTXqmkjCGnhyPKeQuSsPGB1HqtkvB0+z5WgO7i5m7o+NNDpOtfB2d2HeyO70b92QQlsZo9/fwJLt6UbHqvHUNImIiEi95eFq4e83twXgzZX7OZqlJZnrk/Nnme6MCSPQx93gNNXHw9XCm8O7cVP7EIpLy/i/jzbyVcoxo2PVaGqaREREpF67sX0IvZoFUFRSxozvdhkdR6rJ3ow8ftiVickED15bd5YZd5abi5lX7+3CHV2aUFpmZ+KnKXyyLtXoWDWWmiYRERGp10wmE9MGt8NsgkVbj7NWk+Prhbd/Kj/LNCi6EVFB3ganMYaLxcw/7urEsJ4R2O3w+Odbee/cSoJSkZomERERqffaNrZyX88IAJ7+ZgelZVqCvC7LzC3ky03lKyaO6dvc4DTGMptNPHt7e0ZfGwXA9G93MOdH3fT5f6lpEhEREQESBrbG6uHCzuO5fLJelynVZfPXHKK4tIyYyAbERDYwOo7hTCYTT9zclr/c0BKA/7dkNzMX79L9y35FTZOIiIgIEODtRsLAVgD8Y8lucgpsBieSqpBfVMKHaw8DMKZv/ZvLdCkmk4lJA1sx+aY2ALy+Yj9Pf7ODMp11BdQ0iYiIiDgM6xVJy2AfsgpsvLx8j9FxpAp8uv4IuYUlRAV5M6BtI6Pj1Dh/vq45z9zWDig/Izf58626XBU1TSIiIiIOrhYz0waX/8D4QdJh9mbkGZxIKlNJaRnvnlvoYFSfKCxmk8GJaqbhsU35x12dMJvg0w1HmPRpCrbSMqNjGUpNk4iIiMiv9GkZxMDoRpSW2Zn+7Q7N66hDvt+WzrHsswR4u3FnTJjRcWq0O2PCePXerriYTXy9OY3/+2gjhbZSo2MZRk2TiIiIyP+YEt8WN4uZn/aeZPnOTKPjSCWw2+2Om9neHxuJh6vF4EQ1X3zHxrw5PAY3FzOJOzIY/cEGzhbXz8ZJTZOIiIjI/4gM9GbUuSWYn120g6KS+vmDYl2y9sBpth7Lwd3FzP2xTY2OU2vc0LYR8/7UHS83Cz/tPcmI99aRV1j/FklR0yQiIiJyEeP7t6ChrzuHThUw7+dDRseR3+mtVfsBuKtbGAHebganqV2uaRHEv0b1wNfdhXWHTvPHd34hu6DY6FjVSk2TiIiIyEX4uLvw2I3lyy+/unwvmXmFBieS32pPRh4/7j6ByQQP9tEy479FTGQAH4/pRQMvVzYfzeGet9ZyIq/I6FjVRk2TiIiIyCXc0aUJncL9yS8u5f8t3m10HPmN3j43lykuOoSmQd4Gp6m92jfx49M/x9LQ151d6XkMfTOJ4zlnjY5VLdQ0iYiIiFyC2Wxi2uBoABYmH2XzkWxjA8lVy8wt5MuUYwCMuU5nmX6vVo18WfjnWJr4e3LgZD53zU0i9VSB0bGqnJomERERkcvoGtGAO7o0AeCpb7ZrCfJaZt6aQ9hK7XSLbEDXiAZGx6kTmgZ58+mfe9E00IujWWe568017Mus2/c0U9MkIiIicgWP3dQGLzcLm1Kz+Solzeg44qQzRSV8tPYwAGP66ixTZQpr4MVnf46lZbAPGblFDH1zLdvTcoyOVWXUNImIiIhcQSOrB+P7twBgxvc7yS8qMTiROOPT9UfILSyhWZA3A9o2MjpOnRNs9eDTP8fSvomVU/nF3PvWWjalZhkdq0qoaRIRERFxwqg+UUQEeJGRW8QbK/YbHUeuoKS0jPdWHwTgwWubYTabDE5UNwV4u7FgdC9iIhuQW1jCH9/5hbUHThkdq9KpaRIRERFxgoerhb/HtwXgrZ8OcOR03Z/8Xpst2nqcY9lnCfR2446uTYyOU6dZPVz54IEe9G4eSH5xKSPeW8eK3ZlGx6pUappEREREnDQouhHXtAikuKSM5xbtNDqOXILdbuftn8qXGR/RuykerhaDE9V93u4uvPen7lzfJpiikjJGf7CBxdvSjY5VadQ0iYiIiDjJZDIx9ZZ2WMwmFm9PZ82+k0ZHkotIOnCKbcdy8XA188dekUbHqTc8XC3M/WMM8R0aYyu1M37BRr7cdMzoWJVCTZOIiIjIVWgd4ssfe0YAMP3bHZSUlhmcSP7XW+duZntXTDgB3m4Gp6lf3FzMvHJPZ4Z0DaO0zM6kz1JY8Euq0bF+NzVNIiIiIldp0sBW+Hu5sis9j4/X1f4fCOuS3el5rNh9ApMJHrw2yug49ZKLxcz/u7Mjf+wVgd0OT3yxlXfOXS5ZWxnaNL3xxht07NgRq9WK1WolNjaW77//3rG9X79+mEymCh9jx46tsI/U1FTi4+Px8vIiODiYRx55hJKSisuArlixgq5du+Lu7k6LFi2YP3/+BVnmzJlD06ZN8fDwoGfPnqxbt67C9sLCQsaPH09gYCA+Pj4MGTKEjIyMyiuGiIiI1Br+Xm4kDGwFwEuJe8guKDY4kZx3fi7Tje1CiAz0NjhN/WU2m3jmtvb8+dz9sZ5dtJNXl++ttTeHNrRpCgsL44UXXiA5OZkNGzZw/fXXc9ttt7F9+3bHmNGjR3P8+HHHx8yZMx3bSktLiY+Pp7i4mDVr1vD+++8zf/58pk6d6hhz8OBB4uPj6d+/PykpKUycOJEHH3yQJUuWOMZ8+umnJCQkMG3aNDZu3EinTp2Ii4sjM/O/q35MmjSJb775hoULF7Jy5UrS0tK44447qrhCIiIiUlPd1yOC1o18yS6w8fKyvUbHESAjt5CvUsrn0OhmtsYzmUw8flMbJg347y8YZi7ZXSsbJ0ObpsGDB3PzzTfTsmVLWrVqxXPPPYePjw9r1651jPHy8iIkJMTxYbVaHduWLl3Kjh07+PDDD+ncuTM33XQTzzzzDHPmzKG4uPw3PnPnziUqKoqXXnqJtm3bMmHCBO68805mz57t2M+sWbMYPXo0I0eOJDo6mrlz5+Ll5cV7770HQE5ODu+++y6zZs3i+uuvJyYmhnnz5rFmzZoKWUVERKT+cLGYmTo4GoB/rT3M7vQ8gxPJvJ8PYSu1071pA7pENDA6jlDeOP1lQEv+fnP5cv1vrNjPU19vp6ysdjVOLkYHOK+0tJSFCxeSn59PbGys4/mPPvqIDz/8kJCQEAYPHsyTTz6Jl5cXAElJSXTo0IFGjf57h+e4uDjGjRvH9u3b6dKlC0lJSQwYMKDCseLi4pg4cSIAxcXFJCcnM3nyZMd2s9nMgAEDSEpKAiA5ORmbzVZhP23atCEiIoKkpCR69ep10c+pqKiIoqIix+Pc3FwAbDYbNpvtt5SpUpw/tpEZagvVynmqlfNUK+epVs5Tra5OZdWrR6QfA9sGk7gzk6e/3sb8P8VgMtWtm6jWlvfWmaISPvrlMACjekcakre21MoIf4oNx80C077ZyftJh8k9W0RfD+Nr5ezxDW+atm7dSmxsLIWFhfj4+PDFF18QHV3+W5v77ruPyMhIQkND2bJlC4899hi7d+/m888/ByA9Pb1CwwQ4Hqenp192TG5uLmfPniUrK4vS0tKLjtm1a5djH25ubvj7+18w5vxxLmbGjBk8/fTTFzy/dOlSR+NnpMTERKMj1BqqlfNUK+epVs5TrZynWl2dyqhXLw/40WRhzYHTvPjRYjoG1K7foDurpr+3fkwzkVdoIdjDztkDG/juoHFZanqtjOIPDGthYsE+M1+kpHMo0IzdnojFwGvfCgqcu0m14U1T69atSUlJIScnh3//+9+MGDGClStXEh0dzZgxYxzjOnToQOPGjbnhhhvYv38/zZs3NzC1cyZPnkxCQoLjcW5uLuHh4QwaNKjCZYbVzWazkZiYyMCBA3F1dTUsR22gWjlPtXKeauU81cp5qtXVqex6nfTdyxurDrI004dJQ3vjXoduplob3lu20jJenL0aKOThuHbc0i3MmBy1oFZGuxnoujmNv/57G5tOmRnRvwM3d2piWJ7zV4JdieFNk5ubGy1atAAgJiaG9evX88orr/Dmm29eMLZnz54A7Nu3j+bNmxMSEnLBKnfnV7QLCQlx/Pm/q9xlZGRgtVrx9PTEYrFgsVguOubX+yguLiY7O7vC2aZfj7kYd3d33N3dL3je1dW1Rnwh1ZQctYFq5TzVynmqlfNUK+epVlensuo14YZWfJ6SxpGss3yw7ij/169FJaSrWWrye+u77cdIyykkyMeNO7tF4Gpw01qTa1UTDOkWidXTlUWrNhDfqYmhtXL22DXuPk1lZWUV5gH9WkpKCgCNGzcGIDY2lq1bt1ZY5S4xMRGr1eq4xC82Npbly5dX2E9iYqJj3pSbmxsxMTEVxpSVlbF8+XLHmJiYGFxdXSuM2b17N6mpqRXmX4mIiEj95O3uwuM3tQHgtR/2kZFbaHCi+sNutztuZjsitikedegsX13Wr1VD+ofWnktZDW2aJk+ezKpVqzh06BBbt25l8uTJrFixgmHDhrF//36eeeYZkpOTOXToEF9//TX3338/ffv2pWPHjgAMGjSI6Ohohg8fzubNm1myZAlTpkxh/PjxjjM8Y8eO5cCBAzz66KPs2rWL119/nc8++4xJkyY5ciQkJPD222/z/vvvs3PnTsaNG0d+fj4jR44EwM/Pj1GjRpGQkMCPP/5IcnIyI0eOJDY29pKLQIiIiEj9clunJnSJ8KeguJQXF+8yOk69sWb/Kban5eLpauGPvSKNjiN1lKGX52VmZnL//fdz/Phx/Pz86NixI0uWLGHgwIEcOXKEZcuW8fLLL5Ofn094eDhDhgxhypQpjtdbLBa+/fZbxo0bR2xsLN7e3owYMYLp06c7xkRFRbFo0SImTZrEK6+8QlhYGO+88w5xcXGOMUOHDuXEiRNMnTqV9PR0OnfuzOLFiyssDjF79mzMZjNDhgyhqKiIuLg4Xn/99eoplIiIiNR4ZrOJaYPbcfucn/l84zGG94rUstfV4PxZpru7hdHA283gNFJXGdo0vfvuu5fcFh4ezsqVK6+4j8jISL777rvLjunXrx+bNm267JgJEyYwYcKES2738PBgzpw5zJkz54qZREREpH7qHO7PnTFh/Dv5KE99s4MvxvXGbK5bS5DXJLvSc1m55wRmE4zqo5vZStWpcXOaRERERGqzR+Na4+1mYfORbL7YdMzoOHXa26vK1xW/qX1jIgKNv52L1F1qmkREREQqUbDVgwnXtwTgxcW7OFNUYnCiuik9p5CvN5c3paP76iyTVC01TSIiIiKV7IE+TYkM9CIzr4g5P+4zOk6dNG/NQWyldnpEBdA53N/oOFLHqWkSERERqWTuLhamxJff/uTdnw5y+FS+wYnqlrxCGwvWpgIw5lqdZZKqp6ZJREREpAoMaBvMtS2DKC4t47lFO42OU6d8uv4IeUUlNG/ozfVtgo2OI/WAmiYRERGRKmAymZh6SzQWs4mlOzJYvfek0ZHqBFtpGe+tLl8AYvS1zbQ6oVQLNU0iIiIiVaRlI1+Gn7vh6tPfbKektMzgRLXfoi3HScspJMjHndu7NDE6jtQTappEREREqtCkAa1o4OXK3swzfPRLqtFxajW73e64me2fekfi4WoxOJHUF2qaRERERKqQn5crfx3UGoBZiXvIyi82OFHt9fO+U+w4nounq4VhPSONjiP1iJomERERkSp2b48I2oT4knPWxqzEPUbHqbXe+qn8LNPQ7uE08HYzOI3UJ2qaRERERKqYxWxi2uB2AHz0y2F2Hs81OFHts/N4Lqv2nMBsglF9ooyOI/WMmiYRERGRahDbPJCbO4RQZofp3+zAbrcbHalWefvcWaabOjQmPMDL4DRS36hpEhEREakmk29qi7uLmaQDp1iyPd3oOLXG8ZyzfJ2SBuhmtmIMNU0iIiIi1SQ8wIs/9y3/of/ZRTsptJUanKh2mP/zIUrK7PSMCqBTuL/RcaQeUtMkIiIiUo3G9mtOiNWDo1lneefcJWdyaXmFNhacW6p9TF+dZRJjqGkSERERqUZebi5MvrkNAHN+3E96TqHBiWq2T9YdIa+ohBbBPvRvHWx0HKmn1DSJiIiIVLNbO4XSLbIBZ22lvPD9TqPj1Fi20jLe+/kgAKOvjcJsNhmcSOorNU0iIiIi1cxkKl+C3GSCL1PSSD582uhINdK3W9I4nlNIkI87t3dpYnQcqcfUNImIiIgYoEOYH3fFhAHw9Dc7KCvTEuS/ZrfbeWtV+Vmmkdc0xd3FYnAiqc/UNImIiIgY5JG4Nvi4u7DlaA7/3njU6Dg1yup9J9l5PBcvNwvDekYYHUfqOTVNIiIiIgZp6OvOwze0AGDm4t3kFdoMTlRzvLWqfGXBu7uF4+/lZnAaqe/UNImIiIgY6E+9o4gK8ubkmSJe+3Gf0XFqhB1pufy09yRmE4zqE2V0HBE1TSIiIiJGcnMx8+QtbQF4b/VBDp7MNziR8c7fv+rmDo0JD/AyOI2ImiYRERERw/VvHcx1rRpiK7Xz3KIdRscxVFr2Wb7enAboZrZSc6hpEhERETGYyWTiyVuicTGbWLYzk5V7ThgdyTDz1xyipMxOr2YBdAzzNzqOCKCmSURERKRGaBHsw4jeTQGY/s12bKVlxgYyQG6hjQW/pAI6yyQ1i5omERERkRri4RtaEuDtxv4T+fwr6bDRcardJ+tSOVNUQstgH/q1CjY6joiDmiYRERGRGsLP05W/DWoNwOxlezh1psjgRNWnuKSM91YfAmD0tc0wm03GBhL5FTVNIiIiIjXI0O7hRDe2kldYwkuJe4yOU22+3ZJGem4hDX3dua1LqNFxRCpQ0yQiIiJSg1jMJqYNjgbg43WpbE/LMThR1bPb7Y6b2f6pd1PcXSwGJxKpSE2TiIiISA3Ts1kg8R0bY7fD09/swG63Gx2pSv209yS70vPwcrPwx56RRscRuYCaJhEREZEa6Imb2+LuYmbdwdN8tzXd6DhV6u1zN7Md2j0cPy9Xg9OIXEhNk4iIiEgN1MTfk7HXNQfg+e92UmgrNThR1dielsNPe09iMZt44Jooo+OIXJSaJhEREZEaaux1zQn18+BY9lneXHnA6DhV4p2fDgJwc4fGhAd4GZxG5OLUNImIiIjUUJ5uFibf3BaAN1buIy37rMGJKlda9lm+2ZwGwJhrdTNbqbnUNImIiIjUYLd0bEyPpgEU2sp44ftdRsepVPN+PkhJmZ3YZoF0CPMzOo7IJRnaNL3xxht07NgRq9WK1WolNjaW77//3rG9sLCQ8ePHExgYiI+PD0OGDCEjI6PCPlJTU4mPj8fLy4vg4GAeeeQRSkpKKoxZsWIFXbt2xd3dnRYtWjB//vwLssyZM4emTZvi4eFBz549WbduXYXtzmQRERERqWwmk4mpg6MxmeDrzWmsP3Ta6EiVIrfQxsfrjgAwpq/OMknNZmjTFBYWxgsvvEBycjIbNmzg+uuv57bbbmP79u0ATJo0iW+++YaFCxeycuVK0tLSuOOOOxyvLy0tJT4+nuLiYtasWcP777/P/PnzmTp1qmPMwYMHiY+Pp3///qSkpDBx4kQefPBBlixZ4hjz6aefkpCQwLRp09i4cSOdOnUiLi6OzMxMx5grZRERERGpKu2b+HFP93AAnvp6O6VltX8J8o9/SeVMUQktg33o17qh0XFELsvQpmnw4MHcfPPNtGzZklatWvHcc8/h4+PD2rVrycnJ4d1332XWrFlcf/31xMTEMG/ePNasWcPatWsBWLp0KTt27ODDDz+kc+fO3HTTTTzzzDPMmTOH4uJiAObOnUtUVBQvvfQSbdu2ZcKECdx5553Mnj3bkWPWrFmMHj2akSNHEh0dzdy5c/Hy8uK9994DcCqLiIiISFX666DW+Hq4sD0tl38nHzE6zu9SXFLGvJ8PATC6bzNMJpOxgUSuwMXoAOeVlpaycOFC8vPziY2NJTk5GZvNxoABAxxj2rRpQ0REBElJSfTq1YukpCQ6dOhAo0aNHGPi4uIYN24c27dvp0uXLiQlJVXYx/kxEydOBKC4uJjk5GQmT57s2G42mxkwYABJSUkATmW5mKKiIoqKihyPc3NzAbDZbNhstt9Yqd/v/LGNzFBbqFbOU62cp1o5T7Vynmp1dWpjvfzczTzUvznPf7+bmYt3M7BNEL4eVX9Po6qo1Zeb0kjPLSTY152b2wXXqn+Hy6mN7yuj1JRaOXt8w5umrVu3EhsbS2FhIT4+PnzxxRdER0eTkpKCm5sb/v7+FcY3atSI9PTyG7ylp6dXaJjObz+/7XJjcnNzOXv2LFlZWZSWll50zK5duxz7uFKWi5kxYwZPP/30Bc8vXboULy/jl9RMTEw0OkKtoVo5T7VynmrlPNXKearV1alt9Qosg2APC5n5xfz1veXc3rSs2o5dWbWy22H2FgtgokeDApYvXVwp+61Jatv7ykhG16qgoMCpcYY3Ta1btyYlJYWcnBz+/e9/M2LECFauXGl0rEoxefJkEhISHI9zc3MJDw9n0KBBWK1Ww3LZbDYSExMZOHAgrq666/blqFbOU62cp1o5T7Vynmp1dWpzvfxaneDBf23ipwwLj991Lc0aelfp8Sq7Vj/tPcnxtRvxdrPw9B/7Y/WsXfW/nNr8vqpuNaVW568EuxLDmyY3NzdatGgBQExMDOvXr+eVV15h6NChFBcXk52dXeEMT0ZGBiEhIQCEhIRcsMrd+RXtfj3mf1e5y8jIwGq14unpicViwWKxXHTMr/dxpSwX4+7ujru7+wXPu7q61ogvpJqSozZQrZynWjlPtXKeauU81erq1MZ6DWgXyvVtjvHDrkxeWLKHeSN7VMtxK6tW7645DMDQ7hEEWo2/8qYq1Mb3lVGMrpWzx65x92kqKyujqKiImJgYXF1dWb58uWPb7t27SU1NJTY2FoDY2Fi2bt1aYZW7xMRErFYr0dHRjjG/3sf5Mef34ebmRkxMTIUxZWVlLF++3DHGmSwiIiIi1WVKfFtczCZ+3H2CH3dlXvkFNcS2Yzn8vO8UFrOJB/o0NTqOiNMMPdM0efJkbrrpJiIiIsjLy2PBggWsWLGCJUuW4Ofnx6hRo0hISCAgIACr1cpDDz1EbGysY+GFQYMGER0dzfDhw5k5cybp6elMmTKF8ePHO87wjB07ltdee41HH32UBx54gB9++IHPPvuMRYsWOXIkJCQwYsQIunXrRo8ePXj55ZfJz89n5MiRAE5lEREREakuzRr6MPKaprz900Ge+XYH17QIws2lxv0u/AJv/3QAgPgOjQlrUDfPMkndZGjTlJmZyf3338/x48fx8/OjY8eOLFmyhIEDBwIwe/ZszGYzQ4YMoaioiLi4OF5//XXH6y0WC99++y3jxo0jNjYWb29vRowYwfTp0x1joqKiWLRoEZMmTeKVV14hLCyMd955h7i4OMeYoUOHcuLECaZOnUp6ejqdO3dm8eLFFRaHuFIWERERker00A0t+WLTMQ6czOeDpEM8eG3NvkHsseyzfLvlOKCb2UrtY2jT9O677152u4eHB3PmzGHOnDmXHBMZGcl333132f3069ePTZs2XXbMhAkTmDBhwu/KIiIiIlJdrB6uPBLXmsf+s5VXlu3l9i5NCPK5cC51TTFv9UFKy+z0bh5I+yZ+RscRuSo1/zyuiIiIiFzUnTHhtG9iJa+ohH8s2W10nEvKOWvj43WpgM4ySe2kpklERESklrKYTTw1uB0An244wrZjOQYnuriP16WSX1xK60a+XNeqodFxRK6amiYRERGRWqxb0wBu7RSK3Q5Pf7Mdu91udKQKikvKmPfzQQAevDYKk8lkcCKRq6emSURERKSWe/ymNni4mll/KItvzi22UFN8vTmNjNwiGlndua1zE6PjiPwmappEREREarlQf0/+r18LAGZ8t5OzxaUGJypnt9t5e1X5MuN/6h1VK5ZFF7kYvXNFRERE6oAxfZvRxN+T4zmFzF253+g4AKzcc4LdGXl4u1m4r2eE0XFEfjM1TSIiIiJ1gIerhSdubgvA3JX7OZpVYHAieOvcWaZ7ekTg5+lqcBqR305Nk4iIiEgdcXOHEHpGBVBUUsaM73cZmmXbsRzW7D+FxWzigT5RhmYR+b3UNImIiIjUESaTiamDozGbYNGW4/xy4JRhWc6fZbqlY2Oa+HsalkOkMqhpEhEREalD2oX6cU+P8vlDT32zg9Ky6l+C/GhWAYu2lq/iN/pa3cxWaj81TSIiIiJ1zF8HtsLq4cLO47l8uv5ItR//vdWHKC2zc02LQNo38av244tUNjVNIiIiInVMoI87Ewe0AuAfS3eTU2CrtmPnFNj4ZH0qAGP6Nq+244pUJTVNIiIiInXQ8NhIWgT7cDq/mFeW762243607jAFxaW0CfGlb8ugajuuSFVS0yQiIiJSB7lazDx5SzQAHyQdYl9mXpUfs6iklPk/HwLK5zKZTKYqP6ZIdVDTJCIiIlJHXdeqIQPaBlNSZmf6tzux26t2UYivU9LIzCsixOrB4E6hVXoskeqkpklERESkDvt7fDSuFhOr9pzgh12ZVXYcu93O2z+VLzM+8pqmuLnox0ypO/RuFhEREanDooK8HTeXfebbHRSXlFXJcVbsOcGejDP4uLtwb8+IKjmGiFHUNImIiIjUcRP6tyDIx51DpwqY9/PBKjnGWyvLzzLd2yMcq4drlRxDxChqmkRERETqOF8PVx67sTUAr/6wj8y8wkrd/9ajOSQdOIWL2cTIa6Iqdd8iNYGaJhEREZF6YEjXMDqG+XGmqIR/LNldqft+69xcpls6NibU37NS9y1SE6hpEhEREakHzGYT0wa3A2Bh8lG2HM2ulP0eOV3Ad1uPAzC6b7NK2adITaOmSURERKSeiIlswB+6NMFuh6e+3l4pS5C/9/NBSsvs9GkRRLtQv0pIKVLzqGkSERERqUceu7ENnq4WNqZm8/XmtN+1r5wCG5+uPwLAGJ1lkjpMTZOIiIhIPRLi58H4/s0BmPHdLgqKS37zvj785TAFxaW0CfHl2pZBlRVRpMZR0yQiIiJSzzx4bTPCGniSnlvIGyv2/6Z9FJWUMn/NIaD8LJPJZKrEhCI1i5omERERkXrGw9XClPi2ALy56gBHThdc9T6+2pTGibwiQqwe3NIxtLIjitQoappERERE6qG4diHENgukuKSM57/beVWvLSuzO5YZf6BPU9xc9COl1G16h4uIiIjUQyaTiWm3RmM2wffb0lmz/6TTr12xJ5N9mWfwcXfhnh4RVZhSpGZQ0yQiIiJST7UJsTKsZyQA07/ZQUlpmVOve2tV+Vmm+3pGYPVwrbJ8IjWFmiYRERGReixhYCv8PF3ZlZ7Hx+eWD7+cLUezWXvgNC5mE3/q3bTqA4rUAGqaREREROqxBt5uJAxsBcCspbvJLii+7PjzZ5lu7RRKqL9nlecTqQnUNImIiIjUc8N6RtCqkQ9ZBTZeXrb3kuOOnC7gu63HgfJly0XqCzVNIiIiIvWci8XM1FvaAfCvtYfZk5F30XHvrj5ImR2ubRlEdKi1OiOKGEpNk4iIiIjQp2UQg6IbUVpm55lvd2C32ytszy6w8dmG8jlPY/rqLJPUL2qaRERERASAv8e3xc1i5qe9J0nckVFh28frj1BQXErbxlb6tAgyKKGIMdQ0iYiIiAgAkYHejLo2CoBnF+2kqKQUgJIy+GBtKgBj+kZhMpkMyyhiBEObphkzZtC9e3d8fX0JDg7m9ttvZ/fu3RXG9OvXD5PJVOFj7NixFcakpqYSHx+Pl5cXwcHBPPLII5SUlFQYs2LFCrp27Yq7uzstWrRg/vz5F+SZM2cOTZs2xcPDg549e7Ju3boK2wsLCxk/fjyBgYH4+PgwZMgQMjIyLtiPiIiISG01vn8Lgn3dST1dwHurDwGw/oSJk2eKaeznwS0dQ40NKGIAQ5umlStXMn78eNauXUtiYiI2m41BgwaRn59fYdzo0aM5fvy442PmzJmObaWlpcTHx1NcXMyaNWt4//33mT9/PlOnTnWMOXjwIPHx8fTv35+UlBQmTpzIgw8+yJIlSxxjPv30UxISEpg2bRobN26kU6dOxMXFkZmZ6RgzadIkvvnmGxYuXMjKlStJS0vjjjvuqMIKiYiIiFQvH3cXHruxDQCv/bCX9NxCfjxe/iPjA9dE4WrRhUpS/7gYefDFixdXeDx//nyCg4NJTk6mb9++jue9vLwICQm56D6WLl3Kjh07WLZsGY0aNaJz584888wzPPbYYzz11FO4ubkxd+5coqKieOmllwBo27Ytq1evZvbs2cTFxQEwa9YsRo8ezciRIwGYO3cuixYt4r333uPxxx8nJyeHd999lwULFnD99dcDMG/ePNq2bcvatWvp1atXpddHRERExAh/6NKED9YeZvORbB54P5mMsyZ83F24p0e40dFEDGFo0/S/cnJyAAgICKjw/EcffcSHH35ISEgIgwcP5sknn8TLywuApKQkOnToQKNGjRzj4+LiGDduHNu3b6dLly4kJSUxYMCACvuMi4tj4sSJABQXF5OcnMzkyZMd281mMwMGDCApKQmA5ORkbDZbhf20adOGiIgIkpKSLto0FRUVUVRU5Hicm5sLgM1mw2azXXV9Ksv5YxuZobZQrZynWjlPtXKeauU81erqqF5XNuWmVtz11jr2ZpZfAXR318Z4WFSzy9H7ynk1pVbOHr/GNE1lZWVMnDiRa665hvbt2zuev++++4iMjCQ0NJQtW7bw2GOPsXv3bj7//HMA0tPTKzRMgONxenr6Zcfk5uZy9uxZsrKyKC0tveiYXbt2Ofbh5uaGv7//BWPOH+d/zZgxg6effvqC55cuXepo+oyUmJhodIRaQ7VynmrlPNXKeaqV81Srq6N6XV73hmbWnzBjNtmJLDrId98dNDpSraD3lfOMrlVBQYFT42pM0zR+/Hi2bdvG6tWrKzw/ZswYx987dOhA48aNueGGG9i/fz/Nmzev7phXZfLkySQkJDge5+bmEh4ezqBBg7BajbshnM1mIzExkYEDB+Lq6mpYjtpAtXKeauU81cp5qpXzVKuro3o5p3teEX/5dDON7Ke46xbV6kr0vnJeTanV+SvBrqRGNE0TJkzg22+/ZdWqVYSFhV12bM+ePQHYt28fzZs3JyQk5IJV7s6vaHd+HlRISMgFq9xlZGRgtVrx9PTEYrFgsVguOubX+yguLiY7O7vC2aZfj/lf7u7uuLu7X/C8q6trjfhCqik5agPVynmqlfNUK+epVs5Tra6O6nV5oQGuLHiwB999951qdRVUK+cZXStnj23o8id2u50JEybwxRdf8MMPPxAVFXXF16SkpADQuHFjAGJjY9m6dWuFVe4SExOxWq1ER0c7xixfvrzCfhITE4mNjQXAzc2NmJiYCmPKyspYvny5Y0xMTAyurq4VxuzevZvU1FTHGBERERERqXsMPdM0fvx4FixYwFdffYWvr69jbpCfnx+enp7s37+fBQsWcPPNNxMYGMiWLVuYNGkSffv2pWPHjgAMGjSI6Ohohg8fzsyZM0lPT2fKlCmMHz/ecZZn7NixvPbaazz66KM88MAD/PDDD3z22WcsWrTIkSUhIYERI0bQrVs3evTowcsvv0x+fr5jNT0/Pz9GjRpFQkICAQEBWK1WHnroIWJjY7VynoiIiIhIHWZo0/TGG28A5Tew/bV58+bxpz/9CTc3N5YtW+ZoYMLDwxkyZAhTpkxxjLVYLHz77beMGzeO2NhYvL29GTFiBNOnT3eMiYqKYtGiRUyaNIlXXnmFsLAw3nnnHcdy4wBDhw7lxIkTTJ06lfT0dDp37szixYsrLA4xe/ZszGYzQ4YMoaioiLi4OF5//fUqqo6IiIiIiNQEhjZNdrv9stvDw8NZuXLlFfcTGRnJd999d9kx/fr1Y9OmTZcdM2HCBCZMmHDJ7R4eHsyZM4c5c+ZcMZOIiIiIiNQNuqWziIiIiIjIZahpEhERERERuQw1TSIiIiIiIpehpklEREREROQy1DSJiIiIiIhchpomERERERGRy1DTJCIiIiIichlqmkRERERERC5DTZOIiIiIiMhlqGkSERERERG5DBejA9QndrsdgNzcXENz2Gw2CgoKyM3NxdXV1dAsNZ1q5TzVynmqlfNUK+epVldH9XKeauU81cp5NaVW538uP/9z+qWoaapGeXl5AISHhxucREREREREzsvLy8PPz++S2032K7VVUmnKyspIS0vD19cXk8l02bHdu3dn/fr1Tu/7asbn5uYSHh7OkSNHsFqtTh+jPqqNtbra905lqY5aVfbnVhn7+y37+K21Murf1kjV9TVYm2p7qax1pVbV9XXubL1+T57f8tqa+F6sjf8X/q/qqmtl1crI90F1HLt79+4sX768RtTKbreTl5dHaGgoZvOlZy7pTFM1MpvNhIWFOTXWYrFc1RvoascDWK3WWvvNr7rVplr9lvdCZarKWlX251YZ+/s9+7jaWhn9b2ukqv4arE21vVLW2l6r6v46v1K9fk+e3/LamvxerE3/F/6v6q7r762Vke+D6jj2r49RE2p1uTNM52khiBpq/PjxVTpe6q66/F6o7M+tMvZXnfWuy/+2RqtNtTU6a1Ufv6Z9nf+e1/+W1xr971tX1ba6Gpm3Oo5dmceorlrp8rx6KDc3Fz8/P3Jycmrtb4yqi2rlPNXKeaqV81Qr56lWV0f1cp5q5TzVynm1rVY601QPubu7M23aNNzd3Y2OUuOpVs5TrZynWjlPtXKeanV1VC/nqVbOU62cV9tqpTNNIiIiIiIil6EzTSIiIiIiIpehpklEREREROQy1DSJiIiIiIhchpomERERERGRy1DTVI+sWrWKwYMHExoaislk4ssvvzQ6Uo00Y8YMunfvjq+vL8HBwdx+++3s3r3b6Fi1wgsvvIDJZGLixIlGR6mRSktLefLJJ4mKisLT05PmzZvzzDPPoPV4nPv+tHPnTm699Vb8/Pzw9vame/fupKamVn9Yg73xxht07NjRcUPI2NhYvv/+ewBOnz7NQw89ROvWrfH09CQiIoKHH36YnJwcg1Mb59ixY/zxj38kMDAQT09POnTowIYNGy46duzYsZhMJl5++eXqDWmAy33N2Ww2HnvsMTp06IC3tzehoaHcf//9pKWlVdjHnj17uO222wgKCsJqtdKnTx9+/PHHav5Mqp4zPxf069cPk8lU4WPs2LEX7Gv+/Pl07NgRDw8PgoODa939o67kqaeeuqAObdq0cWx/66236NevH1arFZPJRHZ2doXXHzp0iFGjRlX4f3LatGkUFxdX82dyITVN9Uh+fj6dOnVizpw5Rkep0VauXMn48eNZu3YtiYmJ2Gw2Bg0aRH5+vtHRarT169fz5ptv0rFjR6Oj1Fgvvvgib7zxBq+99ho7d+7kxRdfZObMmbz66qtGRzPclb4/7d+/nz59+tCmTRtWrFjBli1bePLJJ/Hw8KjmpMYLCwvjhRdeIDk5mQ0bNnD99ddz2223sX37dtLS0khLS+Mf//gH27ZtY/78+SxevJhRo0YZHdsQWVlZXHPNNbi6uvL999+zY8cOXnrpJRo0aHDB2C+++IK1a9cSGhpqQNLqd7mvuYKCAjZu3MiTTz7Jxo0b+fzzz9m9eze33nprhXG33HILJSUl/PDDDyQnJ9OpUyduueUW0tPTq+vTqBbO/lwwevRojh8/7viYOXNmhe2zZs3i73//O48//jjbt29n2bJlxMXFVeenUi3atWtXoQ6rV692bCsoKODGG2/kiSeeuOhrd+3aRVlZGW+++Sbbt29n9uzZzJ0795Ljq5Vd6iXA/sUXXxgdo1bIzMy0A/aVK1caHaXGysvLs7ds2dKemJhov+666+x/+ctfjI5UI8XHx9sfeOCBCs/dcccd9mHDhhmUqGa62PenoUOH2v/4xz8aE6gWaNCggf2dd9656LbPPvvM7ubmZrfZbNWcyniPPfaYvU+fPlccd/ToUXuTJk3s27Zts0dGRtpnz55d9eFqEGd+Jli3bp0dsB8+fNhut9vtJ06csAP2VatWOcbk5ubaAXtiYmJVxjXcxX4uuNL/fadPn7Z7enraly1bVg0JjTNt2jR7p06drjjuxx9/tAP2rKysK46dOXOmPSoq6veH+510pknkCs5f1hIQEGBwkppr/PjxxMfHM2DAAKOj1Gi9e/dm+fLl7NmzB4DNmzezevVqbrrpJoOT1WxlZWUsWrSIVq1aERcXR3BwMD179tQlxpRf8vnJJ5+Qn59PbGzsRcfk5ORgtVpxcXGp5nTG+/rrr+nWrRt33XUXwcHBdOnShbfffrvCmLKyMoYPH84jjzxCu3btDEpa8+Xk5GAymfD39wcgMDCQ1q1b88EHH5Cfn09JSQlvvvkmwcHBxMTEGBu2il3q54KPPvqIoKAg2rdvz+TJkykoKHBsS0xMpKysjGPHjtG2bVvCwsK4++67OXLkSLVmrw579+4lNDSUZs2aMWzYsN99GXVOTk6N+Bms/n0HFbkKZWVlTJw4kWuuuYb27dsbHadG+uSTT9i4cSPr1683OkqN9/jjj5Obm0ubNm2wWCyUlpby3HPPMWzYMKOj1WiZmZmcOXOGF154gWeffZYXX3yRxYsXc8cdd/Djjz9y3XXXGR2x2m3dupXY2FgKCwvx8fHhiy++IDo6+oJxJ0+e5JlnnmHMmDEGpDTegQMHeOONN0hISOCJJ55g/fr1PPzww7i5uTFixAig/LJZFxcXHn74YYPT1lyFhYU89thj3HvvvVitVgBMJhPLli3j9ttvx9fXF7PZTHBwMIsXL77o5Y91xaV+LrjvvvuIjIwkNDSULVu28Nhjj7F7924+//xzoPy9WFZWxvPPP88rr7yCn58fU6ZMYeDAgWzZsgU3NzejPqVK1bNnT+bPn0/r1q05fvw4Tz/9NNdeey3btm3D19f3qve3b98+Xn31Vf7xj39UQdqrZPSpLjEGujzPKWPHjrVHRkbajxw5YnSUGik1NdUeHBxs37x5s+M5XZ53aR9//LE9LCzM/vHHH9u3bNli/+CDD+wBAQH2+fPnGx2tRvnf70/Hjh2zA/Z77723wrjBgwfb77nnnmpOVzMUFRXZ9+7da9+wYYP98ccftwcFBdm3b99eYUxOTo69R48e9htvvNFeXFxsUFJjubq62mNjYys899BDD9l79eplt9vt9g0bNtgbNWpkP3bsmGO7Ls+rqLi42D548GB7ly5d7Dk5OY7ny8rK7Lfeeqv9pptusq9evdqenJxsHzdunL1Jkyb2tLS0akpe/Zz9uWD58uV2wL5v3z673W63P/fcc3bAvmTJEseYzMxMu9lsti9evLhKMxspKyvLbrVaL7h82JnL844ePWpv3ry5fdSoUVWc0jm6PE/kEiZMmMC3337Ljz/+SFhYmNFxaqTk5GQyMzPp2rUrLi4uuLi4sHLlSv75z3/i4uJCaWmp0RFrlEceeYTHH3+ce+65hw4dOjB8+HAmTZrEjBkzjI5WowUFBeHi4nLBmZS2bdvWy9XzANzc3GjRogUxMTHMmDGDTp068corrzi25+XlceONN+Lr68sXX3yBq6urgWmN07hx48u+b3766ScyMzOJiIhwfA87fPgwf/3rX2natKkBiWsWm83G3XffzeHDh0lMTHScZQL44Ycf+Pbbb/nkk0+45ppr6Nq1K6+//jqenp68//77BqauOlfzc0HPnj2B8jMlUP5eBCq8Hxs2bEhQUFCd/j7m7+9Pq1atHHVwVlpaGv3796d379689dZbVZTu6ujyPJH/Ybfbeeihh/jiiy9YsWIFUVFRRkeqsW644Qa2bt1a4bmRI0fSpk0bHnvsMSwWi0HJaqaCggLM5oq/q7JYLJSVlRmUqHZwc3Oje/fuFyzxu2fPHiIjIw1KVbOUlZVRVFQEQG5uLnFxcbi7u/P111/XyxUGz7vmmmsu+74ZPnz4BXMx4+LiGD58OCNHjqy2nDXR+YZp7969/PjjjwQGBlbYfn6+zv9+TzObzXXue9pv+bkgJSUF+G+zdM011wCwe/duR8N1+vRpTp48Wae/j505c4b9+/czfPhwp19z7Ngx+vfvT0xMDPPmzbvgPWYUNU31yJkzZyp0+gcPHiQlJYWAgAAiIiIMTFazjB8/ngULFvDVV1/h6+vrWDrVz88PT09Pg9PVLL6+vhfM9fL29iYwMFBzwC5i8ODBPPfcc0RERNCuXTs2bdrErFmzeOCBB4yOZrgrfX965JFHGDp0KH379qV///4sXryYb775hhUrVhgX2iCTJ0/mpptuIiIigry8PBYsWMCKFStYsmQJubm5DBo0iIKCAj788ENyc3PJzc0Fyn+rXd9+kTFp0iR69+7N888/z9133/3/27t71ijWMAzAjyRuYhE/JgQShCxoSFIIYiFoE1YECz8QQQhWgVQiSBS0EEH/gAEbmxVN5VYWStTCwq1sFCy0loWAErSzs5DHQs7C+cho5GxG8bpgu2W432Fmd+6d992JFy9eRLPZ7P5yPTw8/K8ysHnz5hgdHY2pqakqIm+YsnNubGwsTp8+Ha9evYpHjx7Fly9fut+FRVFErVaLgwcPxo4dO2Jubi6uXbsWW7Zsidu3b0en04ljx45VNaye+N51wdu3b6PVasXRo0djeHg4Xr9+HRcvXoyZmZnuYzgmJyfj5MmTsbCwEM1mM7Zu3RpXrlyJ6enpOHToUJXD+19dunQpTpw4EfV6Pd6/fx/Xr1+Pvr6+OHPmTERErK6uxurqavfYe/PmTQwNDcX4+HgURRHv3r2LRqMR9Xo9bty4ER8/fuxue3R0tJIxdVU9P5CN89f80X++5ubmqo72S/mvfRQRubS0VHW034I1TWv79OlTLiws5Pj4eA4ODuauXbvy6tWr+fnz56qjVe5HPp/u3LmTExMTOTg4mHv37s0HDx5UF7hC8/PzWa/Xs1ar5cjISB4+fDifPn2amWvvx4jITqdTbfCKLC8v5549e3JgYCCnp6ez2WyWvv9PWdNUds51Op01j6N2u93dxsuXL/PIkSNZFEUODQ3lgQMH8smTJ9UNqke+d12wsrKSMzMzWRRFDgwM5MTERF6+fPlva8Ayv60znJ+fz+3bt2dRFHnq1KlcWVmpYES9Mzs7m2NjY1mr1XLnzp05OzvbXdeV+e0vycv25dLS0pr7u2qbMj2KHgAAYC2/xiRBAACAX5TSBAAAUEJpAgAAKKE0AQAAlFCaAAAASihNAAAAJZQmAACAEkoTAABACaUJAH5Qo9GICxcuVB0DgA2mNAEAAJRQmgAAAEooTQDwkx4/fhzbtm2Le/fuVR0FgB7qrzoAAPyOWq1WnD17NlqtVhw/frzqOAD0kDtNALBOt27dinPnzsXy8rLCBPAHcKcJANbh/v378eHDh3j+/Hns37+/6jgAbAB3mgBgHfbt2xcjIyNx9+7dyMyq4wCwAZQmAFiH3bt3R7vdjocPH8b58+erjgPABjA9DwDWaXJyMtrtdjQajejv74+bN29WHQmAHlKaAOAnTE1NxbNnz6LRaERfX18sLi5WHQmAHtmUJmQDAACsyZomAACAEkoTAABACaUJAACghNIEAABQQmkCAAAooTQBAACUUJoAAABKKE0AAAAllCYAAIASShMAAEAJpQkAAKDEVyQwkXs6kWTSAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "bench_k = np.exp2(np.arange(10)).astype(np.int32)\n", - "bench_avg = np.zeros_like(bench_k, dtype=np.float32)\n", - "bench_std = np.zeros_like(bench_k, dtype=np.float32)\n", - "for i, k in enumerate(bench_k):\n", - " r = %timeit -o ivf_pq.search(search_params, index, queries, k, handle=resources); resources.sync()\n", - " bench_avg[i] = (queries.shape[0] * r.loops / np.array(r.all_runs)).mean()\n", - " bench_std[i] = (queries.shape[0] * r.loops / np.array(r.all_runs)).std()\n", - "\n", - "fig, ax = plt.subplots(1, 1, figsize=plt.figaspect(1/2))\n", - "ax.errorbar(bench_k, bench_avg, bench_std)\n", - "ax.set_xscale('log')\n", - "ax.set_xticks(bench_k, bench_k)\n", - "ax.set_xlabel('k')\n", - "ax.grid()\n", - "ax.set_ylabel('QPS');" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Number of probes\n", - "IVF-PQ search runs in two phases; first it looks for nearest clusters,\n", - "then it searches for the neighbors in every selected cluster.\n", - "\n", - "We can set how many clusters we want to inspect.\n", - "For this, `ivf_pq.SearchParams` has a parameter `n_probes`.\n", - "This is the core parameter to control the QPS/recall trade-off." - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3.67 ms ± 3.91 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "4.78 ms ± 1.74 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "6.65 ms ± 3.72 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "10.2 ms ± 4.86 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "17.2 ms ± 14.2 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "60.2 ms ± 16.6 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "115 ms ± 41.2 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "222 ms ± 184 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "430 ms ± 143 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "829 ms ± 162 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "1.6 s ± 354 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "bench_probes = np.exp2(np.arange(11)).astype(np.int32)\n", - "bench_qps = np.zeros_like(bench_probes, dtype=np.float32)\n", - "bench_recall = np.zeros_like(bench_probes, dtype=np.float32)\n", - "k = 100\n", - "for i, n_probes in enumerate(bench_probes):\n", - " sp = ivf_pq.SearchParams(n_probes=n_probes)\n", - " r = %timeit -o ivf_pq.search(sp, index, queries, k, handle=resources); resources.sync()\n", - " bench_qps[i] = (queries.shape[0] * r.loops / np.array(r.all_runs)).mean()\n", - " bench_recall[i] = calc_recall(ivf_pq.search(sp, index, queries, k, handle=resources)[1], gt_neighbors)\n", - " " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "It's clear that the search time scales almost linearly with the number of probes.\n", - "This is due to the algorithm spending most of the time in the second phase scanning through individual clusters.\n", - "Thanks to the balanced nature of the clustering k-means algorithm, the sizes of the clusters are roughly similar;\n", - "hence the linear relation `n_probes` ~ query time.\n", - "\n", - "Let's draw some plots to illustrate how the number of probes affects QPS and recall." - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABR8AAAFzCAYAAAC3uH7uAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAACbY0lEQVR4nOzdeVxVdf7H8de9l8u+KCCgiCK44r7gntmiZpPtZatmZctg08Q0lTOTTjWT09Q4/WaiLMtsnWyblslMs0zNfU9xR0VREFBZFS7c+/sDJUlUhMs9917ez8eDR91z7znnfbjgFz58zvdrcjgcDkRERERERERERESczGx0ABEREREREREREfFOKj6KiIiIiIiIiIhIo1DxUURERERERERERBqFio8iIiIiIiIiIiLSKFR8FBERERERERERkUah4qOIiIiIiIiIiIg0ChUfRUREREREREREpFGo+CgiIiIiIiIiIiKNwsfoAK5mt9s5ePAgISEhmEwmo+OIiMg5OBwOioqKaNWqFWaz/l5WG41rIiKeQ+Pa+WlcExHxDBcypjW54uPBgweJi4szOoaIiFyA/fv307p1a6NjuCWNayIinkfj2tlpXBMR8Sx1GdOaXPExJCQEqPrkhIaG1usYNpuN+fPnM3LkSKxWqzPjKYMyeHQOZVAGZ2coLCwkLi6u+t9uOZPGNWVQBu/N4C45lMF5GTSunZ8zxrXauMPXkLN54zWBrsuTeOM1ga6rri5kTGtyxcdTrfuhoaEN+iUtMDCQ0NBQQ3/4UQZlcLccyqAMjZVBt12dncY1ZVAG783gLjmUwfkZNK6dnTPGtdq4w9eQs3njNYGuy5N44zWBrutC1WVM00QjIiIiIiIiIiIi0ihUfBQREREREREREZFGoeKjiIiIiIiIiIiINAoVH0VERERERERERKRRqPgoIiIiIiIiIiIijULFRxEREREREREREWkUKj6KiIh4oLS0NJKSkkhOTjY6ioiISINpXBMR8V4qPoqIiHiglJQU0tPTWb16tdFRREREGkzjmoiI91LxUURERERERERERBqFj9EBRETqq6LSTkl5JSVlFZSUVVBcVkFBaRlbjpoI2J6LxWLB7gCHw4GDk/91gAOwn/b/P293YLeftq0u+zgcVec4bVtFZSVbs0xkLd2DxWyp17U5Gvi5qaysZHuWiQNL9mCx1C9DQ53KEHuggH7tIg3JIOe3OauAz9cfILLU6CQiIiIN9+aPezCbTCS2CCYxKoiYUH9MJpPRsUREmjTDi49paWk8//zzZGdn07NnT/7973/Tv3//Wl9rs9mYNm0ab731FllZWXTq1InnnnuOK664wsWpRaQ+7HYHJeUVlJRVUnyyYHiqaFha/sttVUXF4vKa20rLf97nhM1+ljNZeG3bepdeW20ZvsjcaXgG3CBD0r6jKj66sTmr9/POin2AD1/mruCGvq25umcrIoL9jI4mIiJywWb8sJucwrLqx4G+FhJaBNEuIpDKoyZMm7PpEBNGu8gg/K3G/IFWRKSpMbT4OGfOHFJTU5kxYwYDBgzgxRdfZNSoUWzfvp2oqKgzXv+nP/2Jd999l5kzZ9K5c2e++eYbrrvuOpYtW0bv3r0NuAIRATh47Dhr9h1l7d58ftph5r/56yi12Sn5RVGxtLyyUc5vtZgI8vMhyNeHIF8Lx0uKaNYsDLPZjAkwmcBsMlX/vwlT1X9P/r/ZfPo2U+37/HK76cx9zCf/32G3k3Uwi9atW2M2GTO7hd1hJ+vAAWLdIEOHqGBDzi91c2nnKA4eK+X77YfZfLCQzQfT+etXWxneKYob+sRyaZco/Hz0y5mIiLg/u93Bdb1bszu3mN25xWTml1JaXsnmrEI2ZxUCFubO2QRU/UzXunlAVYdki2ASWgRV/39ksK+6JUVEnMjQ4uP06dOZOHEiEyZMAGDGjBl89dVXzJo1iyeeeOKM17/zzjv88Y9/5MorrwTgwQcf5Ntvv+Uf//gH7777rkuzizRVlXYHWw8Vsnbf0ZMFxyMcLDhx2ivMkJ93zmNYzCaCfC0E+/kQ5OdDoJ8PwX4Wgnx9qrcFndpW/f+/2OZ76nWWGoURm83G3LlzufLKgVit1kb6LJxbVYb9XHllN4MzZLpFhmEd1PXozi7pHMXQxOZ8+PlcTkR347MNh/gpq4Bvt+bw7dYcwgKsjOnZkuv7tKZ3XDP9MiYiIm7LbDbxxOjO1Y9tlXYyj5Sy+3AxO7ILWbxhO+X+zcnILaHwRAX7jxxn/5HjLNqeW+M4of4+JEYFkxBZdev2qaJk24hArBYtmyAicqEMKz6Wl5ezdu1aJk+eXL3NbDZz+eWXs3z58lr3KSsrw9/fv8a2gIAAli5detbzlJWVUVb2c9t9YWEhUPVLsc1mq1f2U/vVd39nUAZlcFWO4rIKNuwvYF3mUdZmHmPj/gJKftHBaDGb6BITQs/WIZTkZNKnexKhgX4EnSwoBp0sGAb7Vv3Xz8fsvAKGw47ttNuv3eH9UAbnZTD6+6opCbbCzQPbcM9FiezIKeLTdVn8d/0BcgrLeHdFJu+uyCQhMojr+8Rybe9YWjcPNDqyiIjIOVkt5urC4SUdI4gr3sqVVw7Ax8eH/JJydh8uZnduCbtzi8nIrfr//UdLKTxRwfrMY6zPPFbjeD5mE23CA0loUbMomdgiiGaBvsZcpIiIBzCs+JiXl0dlZSXR0dE1tkdHR7Nt27Za9xk1ahTTp09n2LBhJCYmsnDhQj799FMqK89+K+e0adN46qmnztg+f/58AgMb9ovTggULGrS/MyiDMvxSQ3McKYM9RSb2FJrIKDJxsBQc1CwU+lkctAt2kBDqoF0ItA124Gc5AhyBVkD+FsiHUqo+cms5T2Nzh/dDGRqeobRUq6AYoWN0CE+M7szvR3Vi2e48Pl2XxbzN2WTklfDC/B28MH8HgxIiuL5PLKO7tyTYz/AppEVEROrMZDIRGexHZLAfAxIiajx3wlbJ3vwSdh8uOVmQ/LlAWVpeSUZeCRl5JXy7teYxI4J8qxe5Of027tbNA7GYddeAiDRtHvXbwv/93/8xceJEOnfujMlkIjExkQkTJjBr1qyz7jN58mRSU1OrHxcWFhIXF8fIkSMJDQ2tVw6bzcaCBQsYMWKEobczKoMyNDRHRaWd7TnFrMs8xtrMY6zdd5Ts0yboPiW2mT992jSjb5tm9GnTnI7RwbX+EOUOnwtl8K4Mp7rVxRgWs4mLOrTgog4teObaCr7+6RCfrstieUZ+9ceUz7dwRbcYru8Ty+DESP2CJSIiHs3faqFzTCidY2r+ruhwOMguPEHGyULk6V2ThwpOkF9STn7JEVbtPVJjP18fM+0igkiMCqpxG3dCi2D98U5EmgzD/rWLjIzEYrGQk5NTY3tOTg4xMTG17tOiRQs+++wzTpw4QX5+Pq1ateKJJ54gISHhrOfx8/PDz+/MFTutVmuDfyF3xjEaShmU4UJyFJdVsD7zKGv2HmXtvqOszzxa6y3USS1D6du2Of3im9OvbTgxYf61Hq8+GVxFGbwjg9HZ5WfBfj7c1C+Om/rFceBoKZ+tz+LTdVlk5JXw3/VZ/Hd9FjGh/lzbO5Yb+sTSITrE6MgiIiJOYzKZaBkWQMuwAIa0rzmfdUlZBXvyzixKZuSVUF5hZ3tOEdtzis44Zkyo/2kL3QSRGFV1G3dMqD9m/TFPRLyIYcVHX19f+vbty8KFC7n22msBsNvtLFy4kEmTJp1zX39/f2JjY7HZbHzyySfcfPPNLkgs4nmyjh1nzd4jVYvD7D3KtuxC7I6arwnx86F32+b0O/nRM64ZQforrIicQ+vmgUy6tAMpl7Rnw/5jfLLuAF9uPER24Qlm/LCbGT/spntsGDf0iWVMz1ZEBJ/5R0ARERFvEeTnQ7fYMLrFhtXYXml3cPDYcXadLEpm5JVUFyfzisvILjxBduEJlu3Or7FfgNVSY/XtU92S7SKD8LdaEBHxNIZWGFJTUxk/fjz9+vWjf//+vPjii5SUlFSvfj1u3DhiY2OZNm0aACtXriQrK4tevXqRlZXFn//8Z+x2O4899piRlyHiFioq7ewvhrdXZLJ+fwFr9x3lUI1VqKu0bh5Av7bN6RsfTr+2zekYHaLbJEWkXkwmE73bNKd3m+Y8eVUS3287zCfrsvh+22F+yirgp6wC/vLVVoZ3iuLGvrFc0jmqxur0IiIi3sxiNhEXHkhceCCXdIqq8VxBqY3decW/uI27mH35pRy3VbLlYCFbDtacfsZkgthmAT+vvh3uT26BidyiMlo293Hego4iIk5maPFx7Nix5ObmMmXKFLKzs+nVqxfz5s2rXoQmMzMTs9lc/foTJ07wpz/9iYyMDIKDg7nyyit55513aNasmUFXIGI8u93B28v3Mn3BDgpP+MBPPy/YZDGb6Nrq5C3UbcPpF9+c6NALu4VaRKQu/HwsXNGtJVd0a0l+cRlfbjzIp+uz2HSggG+35vDt1hzCAqyM6dmS6/u0pndcM/2SJCIiTVZYoJU+bZrTp03zGtttlXb2HymtvnX7VFFyd24JBcdtHDh6nANHj/PDjlNLOlp4Kf0HQvx9qouSp7om20cF0SY8CF8f85kBRERcyPB7KydNmnTW26wXLVpU4/HFF19Menq6C1KJeIbM/FJ+//FGVu6pmtg6wOIgOSGS5PgI+sY3p1dcMwJ9Df82F5EmJiLYj7uGtOOuIe3YkVPEp+uy+Gx9FtmFJ3h3RSbvrsgkITKI6/vEcl2f1sQ2CzA6soiIiFuwWswknFyQZgTR1dsdDgdHSsprFCV3HS5i875cjpSbKDpRwYb9x9iw/1iN41nMJtqGB55xG3dCZDDNg3xdfHUi0lSpKiHigex2B++s2Mffvt7GcVslAVYLj43qQLO8zVz1q75apENE3EbH6BCeGN2Z34/qxLLdeXy6Lot5m7PJyCvhhfk7eGH+DgYlRHBNzxjMlec/noiISFNkMpmICPYjItiP/u3CAbDZbMydO5fLRowiq9BWtchNbnGNAmVJeSUZeSVk5JXw7dbDNY4ZHuRbtdDNaUXJDlEhtG4eoLsTRMSpVHwU8TC/7HYc0C6c52/sSctQK3PnbjY4nYhI7SxmExd1aMFFHVrwzLUVzNuczSdrD7A8I7/6I9DHwr7AXUwYmkCkFqkRERGpEz+rhU4x/nSKCamx3eFwkFNYdrIgWbMoebDgBEdKyjlSUs7qvUdr7Bfs50OnmBA6xYTQJSaETjGhdIoJISxADQ4iUj8qPop4CLvdwbsrq7odS8uruh2fGN2ZOwe2xWw2YbPZjI4oIlInwX4+3Ni3NTf2bc2Bo6V8vuEgH6zKZP/R46QtyuD1pXu5sW9rJl6UQHxkkNFxRUREPJLJZCImzJ+YMH8Gt4+s8VxpecXPi938Yn7J4rIK1u47ytp9NYuSrcL86dyyqhDZOSaEzjGhJLQIwmrRnJIicm4qPop4gMz8Uh77ZCMrMn7udvz7jT1oG6FfykXEs7VuHkjKJe25Z3AbnntvHmtLmrMpq5D3Vmby/qpMRneL4b5hifSKa2Z0VBEREa8R6OtDt9gwusWG1dhuq7SzJ6+ErYcK2Z5dxLbsIrZnF5F17DgHC05wsOAE3237+fZtq8VEYotguvyiKBkd6qdbt0WkmoqPIm7sfN2OItJ0paWlkZaWRmWld0yUaDGb6BXhYPIdA1h3oIhXf9jN99tzmftTNnN/ymZAu3DuvziB4R2j9O+fiIgX8rZxzVNZLWY6RofQMbrmLdwFx23syCli26FCtp1WlCwuq6h+fLpmgVY6RYfUKEp2jA4hyE8lCJGmSN/5Im5q/5FSHvt4E8sz8gHo3y6c59XtKCInpaSkkJKSQmFhIWFhYeffwUOYTCYGJkQwMCGC7dlFvLY4gy82ZrFyzxFW7jlCx+hg7huWyNU9W+Hro9u8RES8hbeOa94iLMBKcnw4yfHh1dscDgcHjh4/2SH5c1FyT14Jx0pt1WP36dpGBNIpOoTOLUNPdkmG0DYiCIv+sCji1VR8FHEzdruD91buY9pp3Y6PX9GJcYPi1e0jIk1Kp5gQ/nFzTx4d1ZE3f9zL+ysz2ZFTzKMfbeSFb7Zz99B4bu3fhhB/TYAvIiLiaiaTibjwQOLCA7k8Kbp6+wlbJbsOF5/sjvy5KJlbVMa+/FL25ZcyPz2n+vX+1qpuy1NFyfaRARRrOnsRr6Lio4gbOaPbMT6c529St6OING0twwL4w5VdmHRpe95fmcmspXvILjzBs3O38e+Fu7htYBvuHtKO6FB/o6OKiIg0ef5WS63zSeYXl7E9u4itJ4uS27OL2J5TxAmbnU0HCth0oOC0V/vwz22Lqm7bPq1Tsn1UMP5Wi2svSEQaTMVHETdgtzt4b1Um0+ZupbS8En+rmcev6Mx4dTuKiFQL9bfywMWJTBgSz+cbDjJzcQY7Dxfz6g8ZzFq6h2t7xXLfsAQ6/GKeKhERETFeRLAfg9v71Vh5u9LuIPNI6WlzSRay7VARmUdKyCsuZ8nOPJbszKt+vcVsIj4isKoYeVpRMrZZgH5vEnFjKj6KGGz/kVIe/2QTy3b/3O349xt7EB+pbkcRkdr4+Vi4uV8cN/ZpzffbD/Pq4gxW7TnCR2sP8NHaA1zWOYr7L04kOb65VtoUERFxYxaziXaRQbSLDGJ095YA2Gw2/vvlXNr1GszuvOM/FyWzizhWamN3bgm7c0v4ikPVxwn286FjdPBpc0lWLXQTFqCpWUTcgYqPIgZxOBy8t7Kq27HkZLfjY6M6c9dgdTuKiNSF2Wzisi7RXNYlmvWZR3ltcQbztmSzcNthFm47TK+4ZjxwcQIjkmI0kb2IiIgH8bNAr7hmJCe0qN7mcDg4XFTG1kOFJxe5qfrYdbhq1e11mcdYl3msxnFahflXrbZ9WlEyoUUQVosWrRNxJRUfRQxw4GhVt+OPu6q6HZPjm/P8jT3V7SgiUk+92zTnlTv6sievhJlLMvh47QE27D/GA++uIz4ikInDErihT2vNEyUiIuKhTCYT0aH+RIf6M7xTVPV2W6WdPXklNYqS27OLyDp2nIMFJzhYcILvt+dWv95qMZHYIriqGNmyqkOyS0wo0aF+umNCpJGo+CjiQg6Hg/dXZfLsVz93O/5+VGcmqNtRRMQp2kUG8ex13Xnk8o68vXwvby/fx978Uv74381Mn7+DuwbHc+egtjQL9DU6qoiIiDiB1VK1WnbHX8z5XHDcxo6cotPmk6wqShaXVVQ/ZsPB6teHBVhPdkf+XJTsFB1CkJ/KJiINpe8iERc5cLSUJz75iaW7qiZM7te2Oc/f1JN26nYUEXG6FiF+/G5kJx64OJEP1+zn9SV7yDp2nH8s2MHLi3YzNjmOe4a2Iy480OioIiIi0gjCAqwkx4eTHB9evc3hcHDg6PGTHZI/FyX35JVQcNzGyj1HWLnnSI3jtAkPPKMoGR8RpCldRC6Aio8ijexs3Y53DY7XgCUi0siC/HyYMKQddw5sy1c/HeK1xRlsOVjI7GV7eWfFPn7VvSX3DUugW2yY0VFFRESkkZlMJuLCA4kLD+TypOjq7Sdslew6XHxGUTK3qIzMI6VkHillfnpO9ev9fKq6LTvHhFTdtn2yKBkZ7GfEZYm4PRUfRRqRuh1FRNyDj8XMNb1iubpnK37clc+ri3ezZGceX2w8yBcbDzK0fST3DGmLw2F0UhEREXE1f6uFbrFhZ/wxMr+47LTFbarmlNyeU8QJm52fsgr4Kaugxusjg/2quyQ7nVzgpkN0sOacliZPxUeRRuBwOPjPqv08O3crxWUV+PmY+f2oTkwY0k7djiIiBjKZTAztEMnQDpFsOVjAzMUZfLnpEEt35bF0Vx7xwRba9iqkd3yE0VFFRETEYBHBfgxu78fg9pHV2yrtDjKPlJ42l2RVUXLfkVLyistYuqusuvkEwMdsoldcMwa0a46pwERZhR2r1YirETGOio8iTpZ17DhPfLKJJTurBpy+bZvz/I09SGgRbHAyERE5XddWYbx4S28eHdWJWUv38sHqTPYWV3L9qyu4rX8bfj+qkxamERERkRosZhPtIoNoFxnE6O4tq7eXllewI6e4RlFyW3YRx0ptrNl3lDX7jgIWZv71O5LjwxncPoLBiZF0axWKj8Vs3AWJuICKjyJOUtXtmMlfv1K3o4iIJ2ndPJApY5K4e3AcD7+5iLV5Zt5bmcncnw7x+BWdublfHGb9Oy4iIiLnEOjrQ6+4ZvSKa1a9zeFwsP/IcZZn5LFkRy4/bDtEkc1efccFbCfE34cB7SIYnBjBkPaRdIwOxmTSzx3iXVR8FHGCI2Vw99vrWLorH4A+bZrx/E09SVS3o4iIx4gO9WdcBzuPXN2fZ77azvacIp749Cf+s3o/z1zTlR6tmxkdUURERDyIyWSiTUQgbSLacH2vlnz11QE6Jg9j1d5jLNudz4qMfApPVPDt1hy+3Vq1oE1ksC+DEiOripGJkcSFB6gYKR5PxUeRBlq6K5+/bbRQVpmPn4+ZR0d24u6h6nYUEfFUA9qF87/fDOXt5fv454IdbNx/jGvSfuSW5DY8NqoTzYN0K7aIiIhcOJMJOkQFkxTbnLuGtKPS7mDLwQJ+3JXPst15rN57hLzicr7ceJAvNx4EILZZQHVX5KDECKJD/Q2+CpELp+KjSAMcKjhO6kebKKs00SsujH/c3EvdjiIiXsBqMXPP0HaM6dGSaV9v47/rs/jPqky+3nyI34/qxC3JbfRHJhEREWkQi9lEj9bN6NG6GQ8OT6SsopINmVVdkct257E+8xhZx47z0doDfLT2AADto4IZnFh1m/bAhAjNTy0eQcVHkXqqqLTz8AcbOFpqo3WQg3fvTiY4wM/oWCIi4kRRof78c2wvbu3fhimfb2ZbdhF//O9m5qzez9PXdKsxr5OIiIhIQ/j5WBiQEMGAhAgeGdGRkrIKVu89wvLd+Szbnc/mgwXsOlzMrsPFvL18HyYTdG0VypDEqq7I/u3CCfRVmUfcj74qRerp39/tYtWeIwT5WhjfoQw/H61QJiLirfq3C+d/D/18K/amAwVc9/KPjO0Xx2NXdCZct2KLiIiIkwX5+TC8UxTDO0UBcKy0nBUZR1i2O49lu/PZdbiYzVmFbM4q5NXFGVgtJnrFNWNQYiRDEiPo1aYZfj4Wg69CRMVHkXpZvjuff3+3E4Cnr07CJ2u9wYlERKSx+VjM3D20HVf1bMnfvt7Gp+uy+GD1fr7enM3vR3Xi1v66FVtEREQaT7NAX67oFsMV3WIAOFx4ovoW7R935ZN17Dir9x5l9d6j/GvhTvytZpLjwxl8cgGbbrFh+llFDKHio8gFOlJSzm/nrMfugJv6tubqni2Zq+KjiEiTERXiz/Sbe3Fb/zY8+fkWth4q5E+fbeaD1Zk8fU03+rRpbnREERERaQKiQv25tncs1/aOxeFwsP/I8apC5O58lu/OI6+4nCU781iyMw+AEH8fBiZEMCQxgsHtI+kQFayVtMUlVHwUuQAOh4NHP9pITmEZCS2CeOqaroDD6FgiImKAfvHhfDlpCO+tzOSF+dvZnFXI9S8v4+Z+rXn8is5EBGseYBEREXENk8lEm4hA2kS04Zb+bXA4HOzIKa7uilyZkU/RiQoWpOewID0HgMhgv+rFa4a0jyQuPNDgqxBvpeKjyAV4Y+kevtt2GF8fMy/d2odAXx9sNpvRsURExCA+FjPjB8dzZfeWPDdvGx+vPcCHaw4wb3M2j47qxO0D2ur2JhEREXE5k8lEp5gQOsWEMGFIOyoq7Ww5WMiPu/NYvjuf1XuPkFdcxhcbD/LFxoMAtG4ewJDESAa3j2BQQgRRof4GX4V4CxUfRepo04FjPDdvGwBP/qoLSa1CDU4kIiLuokWIHy/c1JNb+8fx5GdbSD9UyJTPt/DBqv08c21X+rYNNzqiiIiINGE+FjM945rRM64Zvx7enrKKStZnHquaM3JXHhv2H+PA0ePMWbOfOWv2A9AhKriqM7J9JAPbRRAWaDX4KsRTqfgoUgdFJ2w89J/12CodXNE1hjsGtjU6koiIuKG+bcP58qGhvL9yH89/s530Q4Xc8MpybuzbmidGdyZSt2KLiIiIG/DzsTAwIYKBCRGkjuhISVkFq/YeYfnJBWy2HCxk5+Fidh4u5q3l+zCboFtsGIMSIxiSGEm/+OZYdXOH1JGKjyLn4XA4+ON/N7Mvv5TYZgE8d0MPTcorIk61Z88e7r77bnJycrBYLKxYsYKgoCCjY0k9Wcwm7hz0863YH645wMdrD/DNlmx+N6Ijdwxsi4/FbHRMEZFGo3FNxPME+flwSacoLukUBcDRknJW7snnx11VxcjduSVsOlDApgMFvPpDBlaLiZ6tw4isNNFi71H6tYvE10c/30jtVHwUOY+P1hzgi40HsZhN/OvWXmo1FxGnu+uuu/jLX/7CRRddxJEjR/DzU3ecN4gI9uPvN/bklv5tmPL5ZjZnFfLnL9OZs+YAz1zTlX7xuhVbRLyTxjURz9c8yJcrurXkim4tAcgpPMGy3Xks25XPst35ZB07zpp9xwAL895YTYDVQnK78KrFaxIjSWoVqnmvpZqKjyLnsOtwEVO+2AxA6oiOmrNLRJxuy5YtWK1WLrroIgDCw/XvjLfp06Y5n6cM5f1VmbzwzXa2HirkxhnLub5PLJNHd6FFiH4pFxHvoXFNxDtFh/pzXe/WXNe7NQ6Hg8wjpSzZcZhPlm5m3wk/jpTYWLwjl8U7cgEIC7AyMCGcwYmRDGkfQWKLYN1B2ISpJ1bkLE7YKpn0/npO2OwMbR/JgxcnGh1JRNzQ4sWLGTNmDK1atcJkMvHZZ5+d8Zq0tDTi4+Px9/dnwIABrFq1qvq5nTt3EhwczJgxY+jTpw/PPvusC9OLq1jMJu4c2JbvHx3OLclxmEzw6bosLn1hEbOW7qGi0m50RBERQOOaiJyfyWSibUQQY/u15q6OdlY8Ppx5v72IKVclcXmXKEL8fCg4buObLTlM/WILl09fTP9nF/LwB+v5cPV+9h8pNfoSxMUMLz6ea+CqzYsvvkinTp0ICAggLi6ORx55hBMnTrgorTQlf/kqnW3ZRUQG+zJ9bE/MahkXkVqUlJTQs2dP0tLSan1+zpw5pKamMnXqVNatW0fPnj0ZNWoUhw8fBqCiooIlS5bw8ssvs3z5chYsWMCCBQtceQniQuFBvvzthh7899dD6NE6jKKyCp7+XzpX/XspKzPyjY4nIqJxTUQumMlkonNMKHcPbcfr45NZP2UEn6UM4fejOjG0fSR+PmZyi8r4fMNBHvtkExf9/XuG/f17nvhkE19sPEhuUZnRlyCNzNDbrk8NXDNmzGDAgAG8+OKLjBo1iu3btxMVFXXG699//32eeOIJZs2axeDBg9mxYwd33XUXJpOJ6dOnG3AF4q2+/ukQ767IBGD6zb2ICvE3OJGIuKvRo0czevTosz4/ffp0Jk6cyIQJEwCYMWMGX331FbNmzeKJJ54gNjaWfv36ERcXB8CVV17Jhg0bGDFiRK3HKysro6zs5x/QCgsLAbDZbNhstnpdw6n96ru/MzS1DF1jgvhwYn8+WpvFPxbsZFt2EWNfW8FV3aPp79t0Pg/K4Bk5lMF5GYz+eqoLbxjXauMOX0PO5o3XBLouT3Kua+oaE0TXmCDuG9qWMlslGw4UsDzjCMszjrDpQAGZR0rJPFLKB6v3A9AhKohBCREMSghnUEI4QX7Glau88b0C51/XhRzH0OLj+QauX1q2bBlDhgzhtttuAyA+Pp5bb72VlStXujS3eLf9R0p57JNNADxwcSLDOrYwOJGIeKry8nLWrl3L5MmTq7eZzWYuv/xyli9fDkBycjKHDx/m6NGjhIWFsXjxYu6///6zHnPatGk89dRTZ2yfP38+gYGBDcrrDp0pTS1DKPBYV/hfppnlh03876ccFlos7Cn6ll4RDpflqE1Tey/cOQO4Rw5laHiG0lLPvtXQ08a12rjD15CzeeM1ga7Lk9T1mjoCHWPhRAzsLjSxs8DEzkITWSWw83AJOw+X8PaKTPwtDgZHO7g4xk4zA6fG9sb3Cpx3XRcyphlWfKzLwPVLgwcP5t1332XVqlX079+fjIwM5s6dy5133nnW86hDRBku6HiVdh76zzqKTlTQKy6M31zS7rzHdofPg7vkUAZlcHYGo7+vGiovL4/Kykqio6NrbI+Ojmbbtm0A+Pj48OyzzzJs2DAcDgcjR47kqquuOusxJ0+eTGpqavXjwsJC4uLiGDlyJKGhofXKabPZWLBgASNGjMBqtdbrGA3V1DPcBGw6UMCUL9LZcqiIN3dYuCW5NX+4ohMBvhaXZmnq74U7ZXCXHMrgvAynfhfxVJ4yrtXGHb6GnM0brwl0XZ7EWdd0tLSclXuOsjwjn8U78zlw9DjfHTSxONvCld1iuHtIW7q2ct6/B+fjje8VOP+6LmRMM6z4WJeB65duu+028vLyGDp0KA6Hg4qKCh544AH+8Ic/nPU86hBRhgvx5T4zGw6aCbA4uDoynwXfzHN5hoZyhxzKoAzOyuDpHSJ1db5b3E7n5+eHn9+ZfwK2Wq0N/iHCGcdoqKacoW+7SD68bwC/mTmfhQfNfLD6AGv3HePft/Wmc4zrfuA+pSm/F+6WwV1yKEPDMxid3VXcZVyrjTt8DTmbN14T6Lo8SUOvKSrMypheQYzp1Rq73cH32w8zc0kGKzKO8MWmQ3yx6RCDEyOYeFECF3ds4bL1GLzxvQLnXdeFHMPQ264v1KJFi3j22Wd5+eWXGTBgALt27eLhhx/mmWee4cknn6x1H3WIKENdLdmVx7fL1wHw3I09Gd0txuUZGsIdciiDMjg7g6d3iERGRmKxWMjJyamxPScnh5iYuv0bI02Lr4+Zq9vauXNkMr//ZDM7Dxdz9Us/8qdfdeHOgW0xmbT4mYgYR+OaiDQ2s9nEZV2iuaxLND8dKGDmkgy++ukQy3bns2x3Ph2igrn3onZc0ysWf6tr7w6R+jOs+FifgevJJ5/kzjvv5N577wWge/fulJSUcN999/HHP/4Rs/nMxbvVIaIMdXG46ASPfbIZgNsHtOHq3nEuz+As7pBDGZTBWRmMzt5Qvr6+9O3bl4ULF3LttdcCYLfbWbhwIZMmTTI2nLi1IYkRzHv4Ih79aCPfb89lyudbWLwjj+dv7EHzIF+j44lIE6VxTURcqXvrMP51a28eH92Z2T/u4T+r9rPzcDGPf/ITz3+znfGD4rljYFv9bOQBzqzWucjpA9cppwauQYMG1bpPaWnpGQVGi6Wq0u1wGDspu3guu91B6pyN5BWX0zkmhCevSjI6koh4kOLiYjZs2MCGDRsA2LNnDxs2bCAzMxOA1NRUZs6cyVtvvcXWrVt58MEHKSkpqV5srb7S0tJISkoiOTm5oZcgbioi2I9ZdyUz5aokfC1mvt2aw+j/W8Ly3flGRxMRL6ZxTUTcTWyzAP74qySWTb6UP17ZhVZh/uQVl/OPBTsY9LeF/Omzn9iTV2J0TDkHQ2+7Tk1NZfz48fTr14/+/fvz4osv1hi4xo0bR2xsLNOmTQNgzJgxTJ8+nd69e1ffdv3kk08yZsyY6iKkyIWasXg3S3flEWC18NJtvdW6LSIXZM2aNVxyySXVj09N9TF+/Hhmz57N2LFjyc3NZcqUKWRnZ9OrVy/mzZt3xpzHFyolJYWUlBQKCwsJCwtr0LHEfZlMJu4e2o7+7cL5zQfrycgt4bbXV5AyvD0PX94Bq8WwvyOLiJfSuCYi7irU38rEYQncNSSeuT8dYuaSDDZnFfLuikzeW5nJiC7RTByWQL+2zTVVjZsxtPh4voErMzOzRqfjn/70J0wmE3/605/IysqiRYsWjBkzhr/+9a9GXYJ4uLX7jvKP+TsAeOrqrrSPCjE4kYh4muHDh5+3+37SpEm6HU0apFtsGP97aChPfZHOnDX7een7XSzbncf/3dKbuPCGLaAnInI6jWsi4u6sFjPX9Irl6p6tWJFxhJlLMvhu22Hmp+cwPz2HnnHNuO+iBEZ1jcZHf6h1C4YvOHOugWvRokU1Hvv4+DB16lSmTp3qgmTi7QpKbfzmP+uptDu4umcrburX2uhIIiIiZxXo68NzN/ZgaIdI/vDpT6zLPMaV/7eEZ6/vzpierYyOJyIiIuJSJpOJQYkRDEqMYNfhIt5YuodP1mWxcf8xUt5fR+vmAdw9pB03J8cR7Gd4+atJUwlYmiSHw8Hjn2wi69hx2oQH8tfruqktW0REPMKYnq2Y+/BF9GnTjKKyCh76z3oe+3gjpeUVRkcTERERMUT7qBCmXd+DZU9cym8u60B4kC8Hjh7n6f+lM3jaQv729TayC04YHbPJUvFRmqR3V2Yyb0s2VouJl27rTYi/Z6+oKyJNjybmb9riwgP58P5BPHRpe0wm+HDNAa7691I2ZxUYHU1EpF40romIM0QG+5E6oiPLnriUv17XjXaRQRSeqGDGD7u56O/fkTpnA+kHC42O2eSo+ChNztZDhTzzv3QAHr+iMz1aNzM2kIhIPaSkpJCens7q1auNjiIG8bGY+d3ITrx/70BiQv3JyC3h+peX8cbSPeedr01ExN1oXBMRZ/K3Wrh9QFsWpl7MzHH96N8uHFulg0/XZ3Hlv5Zwx+srWbT9sH5mchEVH6VJKS2vYNL76yivsHNp5yjuGdrO6EgiIiINMigxgq8fvogRSdGUV9p55n/p3D17NXnFZUZHExERETGU2WxiRFI0H94/iM9ThnBVj5ZYzCaW7srjrjdXc8WLS/hwzX7KKiqNjurVVHyUJuXPX2xhd24J0aF+PH9jD83zKCIiXqF5kC+v3dmXZ67piq+Pme+35zL6/5awZGeu0dFERERE3ELPuGa8dFsfFj06nLuHtCPI18L2nCIe+3gTQ5/7nrTvd3GstNzomF5JxUdpMj7fkMWHaw5gMsGLY3sTEexndCQRERGnMZlM3Dkoni8mDaFjdDC5RWXc+cYqpn29lfIKu9HxRERERNxCXHggU8YksWzyZUwe3ZmYUH9yi8p4/pvtDJr2HU/9byt5WpvGqVR8lCZhb14Jf/j0JwAeurQDgxIjDE4kIiLSODrHhPJ5ylBuH9AGgFd/yOCmGcvYl19icDIRERER9xEWYOX+ixNZ/Ngl/HNsT5JahnLcVsm7K/fzl/UWUv6zgbX7jhgd0yuo+Cher6yikkn/WUdJeSX924Xzm0vbGx1JRKTBtCqonEuAr4W/XtedGXf0JSzAysYDBVz5f0v47/oDRkcTEamVxjURMYqvj5nrerfmq98M5b17B3Bxh0gcmJiffpgbXlnOdS//yNc/HaLSrsVp6kvFR/F6f5+3nc1ZhTQLtPJ/t/TCx6IvexHxfFoVVOriim4xfP3wRfRvF05JeSWPzNlI6pwNFJdVGB1NRKQGjWsiYjSTycSQ9pG8Pq4PT/Ss4Ka+sfhazKzPPMaD763jkhcWMfvHPZTo56gLpiqMeLWFW3N4Y+keAF64sSctwwIMTiQiIuJarZoF8J+JA0kd0RGzCT5dn8Wv/rWEjfuPGR1NRERExC21DIRnr+3K0icu4aFL29Ms0ErmkVL+/GU6g//2HX+ft43DhZoYsq5UfBSvdajgOI9+tBGACUPiuTwp2uBEIiIixrCYTfzmsg58eP8gYpsFsC+/lBteWcarP+zGrluIRERERGoVFeLP70Z2YtkTl/LMNV2Jjwik4LiNlxftZshz3/HoRxvZll1odEy3p+KjeKVKu4OHP9jA0VIb3WJDeWJ0Z6MjiYiIGK5ffDhzf3MRv+rekgq7g2lfb2P8m6s4XKS/3IuIiIicTaCvD3cOimfh74bz6p196de2ObZKBx+vPcAVLy7hzjdWsnhHLg6H/qhbGxUfxSv9+7udrNpzhCBfC/++tQ9+PhajI4mIiLiFsEArL93Wm79d3x1/q5klO/MY/eISvt9+2OhoIiIiIm7NYjYxqmsMHz84mP/+ejC/6t4SswmW7Mxj3KxVjP6/JXy89gDlFXajo7oVFR/F66zIyOdfC3cC8NfrutMuMsjgRCIiIu7FZDJxS/82/O+hoXSOCSG/pJwJb67mmf+lU6YflkVERETOq3eb5qTd3ocffn8Jdw2OJ9DXwrbsIh79aCNDn/uOlxftoqDUZnRMt6Dio3iVIyXlPPzBeuwOuLFva67tHWt0JBGRRpGWlkZSUhLJyclGRxEP1j4qhM9ShnDX4HgA3li6h5tfW0nOcWNziUjTo3FNRDxVXHggf766K8ufuIzHruhEdKgfh4vK+Pu87Qz620L+/MUWMvNLjY5pKBUfxWs4HA4e/WgjOYVlJLQI4qmruxodSUSk0aSkpJCens7q1auNjiIezt9q4c9Xd+X1cf1oHmgl/VAR03+ykHVMFUgRcR2NayLi6cICrfx6eHuWPHYp/7ipJ51jQigtr2T2sr2MfPEH0g823YVpVHwUrzHrx718t+0wvj5mXrq1D0F+PkZHEhER8RiXJ0Uz77fD6NYqlBOVJv7+zQ6jI4mIiIh4HF8fMzf0bc3XD1/EO/f0p3tsGCdsdl5etMvoaIZR8VG8wuasQv729VYAnvxVF5JahRqcSERExPNEh/oz7bqumHAwd3MOKzPyjY4kIiIi4pFMJhMXdWjBczf0AGDuT4fYf6Rp3n6t4qN4vBMV8PCHG7FVOhjVNZo7BrY1OpKIiIjH6hwTwuBoBwBP/y+dSrvD4EQiIiIiniupVSgXdYjE7qiaX7spUvFRPJrD4eDDPWYyjxwntlkAf7+hJyaTyehYIiIiHu3KODuh/j5sOVjIR2v2Gx1HRERExKPdNywBgDmr93OstNzgNK6n4qN4tP9uOMjaPDMWs4l/3dqLsECr0ZFEREQ8XrAVJl2SCMAL87dTeMJmcCIRERERzzW0fSRdWoZy3FbJeyszjY7jcio+iscqLa/g79/sBODhSxPp2zbc4EQiIiLe444BcSS2CCKvuJyXvmu6E6SLiIiINJTJZOK+Ye0AePPHvZywVRqcyLVUfBSP9dayfeSXlBPh5+DeofFGxxEREfEqVouZP12VBMCbP+5hT16JwYlEREREPNdVPVrRMsyfvOIyPt+QZXQcl1LxUTxS0Qkbry7eDcAVcXasFn0pi0jTkpaWRlJSEsnJyUZHES92Sacohndqga3SwV+/Sjc6joh4MY1rIuLtrBYzdw+p6n58bXEG9ia0qJ8qNuKR3vxxL8dKbSREBtIvsul8w4qInJKSkkJ6ejqrV682Oop4uT/9Kgkfs4lvtx5myc5co+OIiJfSuCYiTcEt/eMI8fNhd24J328/bHQcl1HxUTxOQamNmUsyAPjNpe0xa3FrERGRRtM+Kphxg+IBePrLdCoq7cYGEhEREfFQIf5WbhvQBoBXF2cYnMZ1VHwUjzNzSQZFJyroHBPC6K7RRscRERHxeg9f1oHwIF92Hi5ukis0ioiIiDjLhCHt8DGbWLXnCBv2HzM6jkuo+CgeJb+4jFk/7gHgkREdMavtUUREpNGFBVpJHdERgOkLdnC0pNzgRCIiIiKeKSbMn6t7tQJgZhPpflTxUTzKq4szKC2vpHtsGCOT1PUoIiLiKrf2b0PnmBAKjtt48dsdRscRERER8Vj3DUsA4OvNh8jMLzU4TeNT8VE8xuHCE7y1bC8AqSM6YjKp61FERMRVLGYTU8YkAfDuykx25BQZnEhERETEM3WOCWVYxxbYHfDGUu/vflTxUTzGy4t2U1Zhp0+bZgzv1MLoOCIiIk3O4MRIRnWNptLu4Jn/peNwOIyOJCIiIuKR7j/Z/fjhmgNeP6WNio/iEQ4eO877Jye4/93ITup6FBERMcgfr0zC12Jmyc48Fm49bHQcEREREY80ODGCpJahHLdV8u6KfUbHaVQqPopH+Pd3uyivtDMwIZzBiRFGxxEREWmy2kQEcs9F7QD4y1fplFVUGpxIRERExPOYTCbuv7iq+/Gt5Xs5YfPen6lUfBS3l5lfykdr9gPqehQROSUtLY2kpCSSk5ONjiJNUMol7WkR4sfe/NLq+ZhFRBpC45qINEVXdm9JqzB/8orL+e/6LKPjNBoVH8Xt/d/CnVTYHQzr2ILk+HCj44iIuIWUlBTS09NZvXq10VGkCQr28+GxUZ0A+PfCXeQWlRmcSEQ8ncY1EWmKrBYzdw+tuqNk5pIM7HbvnE/bLYqPaWlpxMfH4+/vz4ABA1i1atVZXzt8+HBMJtMZH7/61a9cmFhcZXduMf9dfwCoWuFaRERE3MMNfVrTo3UYRWUV/GP+dqPjiIiIiHikW/q3IcTfh4zcEr7dmmN0nEZhePFxzpw5pKamMnXqVNatW0fPnj0ZNWoUhw/XPoH5p59+yqFDh6o/Nm/ejMVi4aabbnJxcnGFF7/did0Bl3eJpldcM6PjiIiIyElms4mpY5IAmLNmP5uzCgxOJCIiIuJ5gv18uH1AW6Cq+9EbGV58nD59OhMnTmTChAkkJSUxY8YMAgMDmTVrVq2vDw8PJyYmpvpjwYIFBAYGqvjohbZnF/G/TQcBdT2KiIi4o75tw7m6ZyscDnj6y3QcDu+8VUhERESkMU0YEo/VYmL13qOsyzxqdByn8zHy5OXl5axdu5bJkydXbzObzVx++eUsX768Tsd44403uOWWWwgKCqr1+bKyMsrKfp6HqLCwEACbzYbNZqtX7lP71Xd/Z2gKGf4xfxsOB1zRNZoOLQJqPU9T+Dx4Ug5lUAZnZzD6+0pEzu+J0Z2Zn57Nqr1HmPtTNr/q0dLoSCIiIiIeJTrUn2t6xfLx2gPMXJzBK3f0NTqSUxlafMzLy6OyspLo6Oga26Ojo9m2bdt591+1ahWbN2/mjTfeOOtrpk2bxlNPPXXG9vnz5xMYGHjhoU+zYMGCBu3vDN6aYX8xzE/3wYSD3j5ZzJ177lWfvPXzUB/ukEMZlMFZGUpLS52YREQaQ6tmATxwcSIvfruTZ+du5bIuUfhbLUbHEhEREfEo9w1L4OO1B5i3JZu9eSXER9beZOeJDC0+NtQbb7xB9+7d6d+//1lfM3nyZFJTU6sfFxYWEhcXx8iRIwkNDa3XeW02GwsWLGDEiBFYrdZ6HaOhvD3DxHfWAXmM6dGKu2/sbkiGunKHDO6SQxmUwdkZTnWri4h7u39YIh+u3k/WsePMXJzBQ5d1MDqSiIiIiEfpGB3C8E4tWLQ9lzeW7uGZa7sZHclpDC0+RkZGYrFYyMmpuZpPTk4OMTEx59y3pKSEDz74gKeffvqcr/Pz88PPz++M7VartcG/kDvjGA3ljRnW7jvKoh15WMwmHhnZqU7H9sbPgyfnUAZlcFYGo7OLSN0E+Fp44sou/OY/63l50W5u6hdHTJi/0bFEREREPMp9wxJYtD2Xj9bu55ERHQkP8jU6klMYuuCMr68vffv2ZeHChdXb7HY7CxcuZNCgQefc96OPPqKsrIw77rijsWOKi/1zwQ4AbugTSzsvajMWERHxZmN6tKRf2+Yct1Xy3LzzT58jIiIiIjUNSoigW2woJ2x23lm+z+g4TmP4atepqanMnDmTt956i61bt/Lggw9SUlLChAkTABg3blyNBWlOeeONN7j22muJiIhwdWRpRCsy8lm6Kw+rxcRDl+qWLREREU9hMpmYOqYrJhP8d32WV67UKCIiItKYTCYT9w1LBODt5Xs5Yas0OJFzGF58HDt2LC+88AJTpkyhV69ebNiwgXnz5lUvQpOZmcmhQ4dq7LN9+3aWLl3KPffcY0RkaSQOh4Pp86u6HscmxxEX3rAFgURERMS1urcO46a+rQF46st07HaHwYlEREREPMuV3WKIbRZAfkk5H689YHQcpzC8+AgwadIk9u3bR1lZGStXrmTAgAHVzy1atIjZs2fXeH2nTp1wOByMGDHCxUmlMS3ZmceqvUfw9TEz6RJ1PYqIiHiiR0d1ItjPh437j/HZhiyj44iIiIh4FB+LmXuGtgPgjaV7qPSCP+a6RfFRxOFw8I+Tcz3eMaCtJqkXETmPtLQ0kpKSSE5ONjqKSA1RIf6kXNIegL99vY2SsgqDE4mIJ9C4JiLys7HJcYT6+7Anr4QF6Tnn38HNqfgobuG7bYfZuP8YAVYLDw5PNDqOiIjbS0lJIT09ndWrVxsdReQMdw+Np21EIIeLynhl0W6j44iIB9C4JiLysyA/H+4Y2BaAmUsyDE7TcCo+iuHsdgf/ODnX4/jB8bQI8TM4kYiIiDSEn4+FP1zZBYDXlmSw/0ipwYlEREREPMtdg+PxtZhZu+8oa/cdMTpOg6j4KIb7Zks26YcKCfbz4f5hCUbHEREREScYmRTNkPYRlFfYmfb1VqPjiIiIiHiUqFB/ru3dCoDXFnt296OKj2KoSruD6Sfnerx7aDuaB/kanEhEREScwWQy8eRVSZhNMPenbFZk5BsdSURERMSjTLyoqkFrfnoOGbnFBqepPxUfxVD/23SQnYeLCfX3qV7NSURERLxD55hQbh9QNV/RU1+me8VqjSIiIiKu0iE6hEs7R+FwVK187alUfBTDVFTaefHbnQDcNyyBsACrwYlERETE2R4Z0ZFQfx+2Hipkzur9RscRERER8Sj3nZye7uO1B8grLjM4Tf2o+CiG+XR9FnvySggP8uWuIep6FBER8UbhQb789vKOAPxj/nYKjtsMTiQiIiLiOQa0C6dn6zDKKuy8vXyf0XHqRcVHMUR5hZ1/Lazqenzg4gSC/XwMTiQiIiKN5c5BbUlsEUR+STn/Pjn+i4iIiMj5mUwmJp7sfnxn+V6Ol1canOjCqfgohvhwzX4OHD1OixA/7hwYb3QcERERaURWi5knr0oCYPayvR49YbqIiIiIq13RNYa48ACOltr4eK3nTWOj4qO43AlbJS99twuAlOGJBPhaDE4kIiIijW14pygu7RxFhd3BX7/aanQcEREREY/hYzFzz8np6l5fusfjFvFT8VFc7v2VmWQXnqBVmD+3DmhjdBwRERFxkT/+qgs+ZhMLtx3mhx25RscRERER8Rg3J8cRFmBlX34p87dkGx3ngqj4KC5VWl7By4t2AzDp0g74+ajrUUREpKlIbBHMXYPjAXjmf+nYKu3GBhIRERHxEIG+Ptw5sC0Ary7OwOHwnO5HFR/Fpd5evo+84jLahAdyU7/WRscRERERF3vosg6EB/my63Ax767wzBUbRURERIwwfnA8vhYzG/YfY82+o0bHqTMVH8Vlik7YePWHqq7H31zWAatFX34iIiJNTViAld+N7AjAPxfs4EhJucGJRERERDxDixA/ru8TC8BrizMMTlN3qv6Iy7z5416OltpIiAzi2l6tjI4jIiIiBrkluQ2dY0IoPFHBPxfsMDqOiIiIiMe496IEAL7dmsPu3GKD09SNio/iEgWlNmYuqarK/3ZER3zU9SgiItJkWcwmpo7pCsB7K/exLbvQ4EQiIiIinqF9VDCXd4nG4YDXl3hG96MqQOISry/NoOhEBZ2iQ7iqe0uj44iIiIjBBiVGMLpbDHZH1eIznjRpuoiIiIiR7htW1f34yboscovKDE5zfio+SqM7UlLOrKV7AHhkREfMZpPBiURERMQd/OHKLvj6mPlxVz4L0nOMjiMiIiLiEZLjm9MrrhnlFXbeXr7X6DjnpeKjNLpXf9hNSXkl3WJDGdU12ug4IiJeIS0tjaSkJJKTk42OIlJvceGBTLyoHQB/nbuVsopKgxOJiFE0romI1J3JZKrufnxnxT5KyysMTnRuKj5KozpcdIK3TlbhfzeiEyaTuh5FRJwhJSWF9PR0Vq9ebXQUkQb59fD2RIX4sS+/lDd/3Gt0HBExiMY1EZELM6prDG3CAzlWauOjNQeMjnNOKj5Ko3r5+92csNnp3aYZwzu1MDqOiIiIuJkgPx8eu6IzAC99t4vDRScMTiQiIiLi/ixmE/eevIPk9aUZVNrdd/5sFR+l0Rw8dpz3V2YC6noUERGRs7u+dyw9W4dRXFbBC99sNzqOiIiIiEe4qW8czQOt7D9ynHmbs42Oc1YqPkqjeen7XZRX2hnQLpwh7SOMjiMiIiJuymw2MWVMVwA+WnuAzVmFBicSERERcX8BvhbuHNgWgNcW78bhcM/uR5+6vvD666+v80E//fTTeoUR77H/SCkfrt4PwO9GqutRREREzq1v2+Zc26sVn204yF/mbuPOVkYnEhEREXF/4wbH8+riDDYeKGDVniMMSHC/5q86dz6GhYXV+UPk/xbupMLu4KIOkfRvF250HBEREfEAj4/uTIDVwtrMY6zP1x8uRURERM4nMtiPG/q2BmDmkgyD09Suzp2Pb775ZmPmEC+SkVvMp+uqVlr63chOBqcRERERT9EyLIAHhycyfcEOPt9n5pGyCppZrUbHEhEREXFr9w5tx39WZfLt1sPsOlxE+6gQoyPVoDkfxele/HYndgdc3iWKXnHNjI4jIlIn+/btIz09HbvdbnQUkSbtvmEJtArz51i5iXGz15BfXGZ0JBGPozFNRKRpSWgRzOVdogGYc3IKPHdS5+Jj79696dOnT50+pOnanl3El5sOAvDIiI4GpxEROdOsWbOYPn16jW333XcfCQkJdO/enW7durF/v/sN2CJNhb/Vwr9u6Umgj4NNBwq5ccZy9h8pNTqWiFvSmCYiIqdc1zsWgG+3HjY4yZnqfNv1tdde24gxxFv8c8EOHA4Y3S2Grq00/6eIuJ/XXnuN+++/v/rxvHnzePPNN3n77bfp0qULkyZN4qmnnuL11183MKVI09azdRi/7VbJ7D3B7Mkr4fpXljF7QrJ+thD5BY1pIiJyyrCOLfC1mNmTV8Lu3GISWwQbHalanYuPU6dObcwc4gU2ZxUwb0s2JpO6HkXEfe3cuZN+/fpVP/7888+55ppruP322wF49tlnmTBhglHxROSk6AD48L7+3PvOerZlFzH21RW8dmdfBrePNDqaiNvQmCYiIqcE+/kwMDGCxTty+TY9h8SL3af4qDkfxWmmL9gBwNU9W9Ex2r0mNxUROeX48eOEhoZWP162bBnDhg2rfpyQkEB2drYR0UTkF6JD/Zlz/yAGtAunuKyC8W+u4suNB42OJeI2NKaJiMjpRnSJAuDbrTkGJ6mpXsXHyspKXnjhBfr3709MTAzh4eE1PqTpWZd5lO+2HcZiNvHwZR2MjiMiclZt27Zl7dq1AOTl5bFlyxaGDBlS/Xx2djZhYbq1U8RdhAVYeevu/lzZPQZbpYOH/rOeWUv3GB1LxC1oTBMRkdNdenLRmbX7jrrVon31Kj4+9dRTTJ8+nbFjx1JQUEBqairXX389ZrOZP//5z06OKJ7gnye7Hq/vHUuCG80rICLyS+PHjyclJYVnnnmGm266ic6dO9O3b9/q55ctW0a3bt0MTCgiv+RvtfDvW/swflBbAJ7+XzrTvt6K3e4wOJmIsTSmiYjI6WKbBZDUMhS7A77fnmt0nGr1Kj6+9957zJw5k9/97nf4+Phw66238vrrrzNlyhRWrFhxQcdKS0sjPj4ef39/BgwYwKpVq875+mPHjpGSkkLLli3x8/OjY8eOzJ07tz6XIU6yau8RluzMw2ox8Rt1PYqIm3vssceYOHEin376Kf7+/nz00Uc1nv/xxx+59dZbDUonImdjMZv489Vd+f2oTgC8+kMGj360EVul3eBkIsbRmCYiIr90eVJV9+NCN7r1us4LzpwuOzub7t27AxAcHExBQQEAV111FU8++WSdjzNnzhxSU1OZMWMGAwYM4MUXX2TUqFFs376dqKioM15fXl7OiBEjiIqK4uOPPyY2NpZ9+/bRrFmz+lyGOIHDAf/8dhcAN/eLIy480OBEIiLnZjabefrpp3n66adrff6Xv7iJiPswmUykXNKeqBA/nvj0Jz5dn0VeSTmv3N6HIL96/Vgr4tE0pomIyC+N6BLNvxbu5IcduZywVeJvtRgdqX6dj61bt+bQoUMAJCYmMn/+fABWr16Nn59fnY8zffp0Jk6cyIQJE0hKSmLGjBkEBgYya9asWl8/a9Ysjhw5wmeffcaQIUOIj4/n4osvpmfPnvW5DHGC7QUm1uw7hq+PmUmXtjc6johIncyZM4fbb7+dm266iRkzZhgdR0Qu0E394nh9XD8CrBYW78jl1pkryHOjeY1EXEljmoiInK5bbCjRoX6UlleyIiPf6DhAPTsfr7vuOhYuXMiAAQN46KGHuOOOO3jjjTfIzMzkkUceqdMxysvLWbt2LZMnT67eZjabufzyy1m+fHmt+3zxxRcMGjSIlJQUPv/8c1q0aMFtt93G448/jsVSeyW3rKyMsrKffxgtLCwEwGazYbPZ6nrJNZzar777O4M7ZCgvL2fu/qr69a3JrYkM9HF5Hnf4PLhDBnfJoQzK4OwMjZH/lVdeISUlhQ4dOhAQEMCnn37K7t27ef75551+LhFpPJd0juL9iQO4e/ZqNh0o4MZXlvH23QNoE6G7MKTp0JgmIiK/ZDKZuKxLNO+vzOTbrTkM73TmncWuVq/i49/+9rfq/x87dixt27Zl2bJldOjQgTFjxtTpGHl5eVRWVhIdHV1je3R0NNu2bat1n4yMDL777jtuv/125s6dy65du/j1r3+NzWZj6tSpte4zbdo0nnrqqTO2z58/n8DAhv1wumDBggbt7wxGZth81MS+YgtWs4P25RnMnZthWJam/l6czh1yKIMyOCtDaWmpE5NUeemll5g6dWr1uPHuu+9y//336xc1EQ/Uu01zPnlwMONmrWJvfinXv/Ijsyf0p1usVveVpkFjmoiI1GbEyeLjwq2HeeYaByaTydA8TpkcZ+DAgQwcONAZhzonu91OVFQUr732GhaLhb59+5KVlcXzzz9/1uLj5MmTSU1NrX5cWFhIXFwcI0eOJDQ0tF45bDYbCxYsYMSIEVit1nodo6GMzuBwOHjtlRVAEeMGtuWW0Z1dngGM/zy4SwZ3yaEMyuDsDKe61Z0pIyOD8ePHVz++7bbbuOeeezh06BAtW7Z0+vlEpHEltAjm0wcHc9ebq0k/VMjYV5cz486+XNShhdHRRBqdxjQREanNoMQIAqwWDhWcYMvBQsP/MFuv4uO0adOIjo7m7rvvrrF91qxZ5Obm8vjjj5/3GJGRkVgsFnJyaq6+k5OTQ0xMTK37tGzZEqvVWuMW6y5dupCdnU15eTm+vr5n7OPn51frPJRWq7XBv5A74xgNZVSGH3flseVQEb5mB/cNS2iynwd3y+AuOZRBGZyVoTGyl5WVERQUVP3YbDbj6+vL8ePHnX4uEXGNqFB/5tw/kPvfWcuy3flMeHM1L9zUk2t7xxodTaRRaUwTEZHa+FstDOsYyTdbcvh2a45nFh9fffVV3n///TO2d+3alVtuuaVOxUdfX1/69u3LwoULufbaa4GqzsaFCxcyadKkWvcZMmQI77//Pna7HbO5aq7BHTt20LJly1oLj9J4ZvywG4ABUQ7Cg/S5FxHP8uSTT9aYeqO8vJy//vWvhIX9PChPnz7diGgiUk8h/lbenJDM7z7cyP82HeK3czaQV1zGvRclGB1NpFFpTBMRkdpc3iW6uvj428s7GpqlXsXH7OzsWtv4W7RoUb0Kdl2kpqYyfvx4+vXrR//+/XnxxRcpKSlhwoQJAIwbN47Y2FimTZsGwIMPPshLL73Eww8/zEMPPcTOnTt59tln+c1vflOfy5B6Sj9YyJKdeZhNcElLu9FxREQuyLBhw9i+fXuNbYMHDyYj4+d5a42eE0VE6sfPx8K/bulNixA/3vxxL3/5ais5hSeYPLoLZrO+r8X7aEwTEZGzuaRzFCYTbM4q5FDBcSIDnTLzYr3U68xxcXH8+OOPtGvXrsb2H3/8kVatWtX5OGPHjiU3N5cpU6aQnZ1Nr169mDdvXvUiNJmZmdUdjqfO+8033/DII4/Qo0cPYmNjefjhh+vUaSnOM3NJ1Q8zo7vGEOF/wOA0IiIXZtGiRTUe5+Xl4evrW+95gEXEvZjNJqZclURMqD/Tvt7GzCV7OFxUxvM39sTXx3z+A4h4EI1pIiJyNpHBfvRp05y1+47y7dbD3NK37vU6Z6vXT2ATJ07kt7/9LW+++Sb79u1j3759zJo1i0ceeYSJEyde0LEmTZrEvn37KCsrY+XKlQwYMKD6uUWLFjF79uwarx80aBArVqzgxIkT7N69mz/84Q815oCUxpV17DhfbDwIwL1D440NIyJST8eOHSMlJYXIyEiio6Np3rw5MTExTJ48uVFW2BYR1zKZTNx/cSLTb+6Jj9nE5xsOcvfs1RSXVRgdTcTpNKaJiMjZXN6lqrlv4dac87yycdWr8/H3v/89+fn5/PrXv6a8vBwAf39/Hn/8cSZPnuzUgOJeZi3dQ6XdweDECLrFhpK50ehEIiIX5siRIwwaNIisrCxuv/12unTpAkB6ejr//ve/WbBgAUuXLmXTpk2sWLHCJVN7xMfHExoaitlspnnz5nz//feNfk6RpuD6Pq2JCPbjwXfXsnRXHre8tpw37+pPi5AzFyMU8UTuOKaBxjUREXcxIimK5+ZtY9mufEoM/CNsvYqPJpOJ5557jieffJKtW7cSEBBAhw4dal1VWrxHQamND1ZlAnDfME3eLiKe6emnn8bX15fdu3dXT/Nx+nMjR47kzjvvZP78+fzrX/9yWa5ly5YRHBzssvOJNBUXd2zBB/cNZMKbq9mcVcgNryzjrbv70y4y6Pw7i7g5dx3TQOOaiIg7SGwRTHxEIHvzS1m6K9+wHA2a+CY7O5sjR46QmJiIn58fDofDWbnEDb27ch8l5ZV0jgnh4o4tjI4jIlIvn332GS+88MIZv6QBxMTE8Pe//51PPvmkelE0EfF8PVo345MHB9MmPJDMI6Xc+MoyNh04ZnQskQbTmCYiIudiMpm47NSt19tzDctRr+Jjfn4+l112GR07duTKK6+sXuH6nnvu4Xe/+51TA4p7KKuoZPayvUBV16NWzRMRT3Xo0CG6du161ue7deuG2Wxm6tSpdTre4sWLGTNmDK1atcJkMvHZZ5+d8Zq0tDTi4+Px9/dnwIABrFq1qsbzJpOJiy++mOTkZN57770Luh4RqZv4yCA+fnAQXVuFkl9Szi2vreCHHcb9EC7iDM4e00DjmoiItzk17+Oi7bnYDeoZrFfx8ZFHHsFqtZKZmUlgYGD19rFjxzJv3jynhRP38dn6LHKLymgZ5s+YnsatkCQi0lCRkZHs3bv3rM/v2bOHqKioOh+vpKSEnj17kpaWVuvzc+bMITU1lalTp7Ju3Tp69uzJqFGjOHz4cPVrli5dytq1a/niiy949tln2bRpU53PLyJ1FxXiz5z7BzG0fSSl5ZXcM3s1n647YHQskXpz9pgGGtdERLxNv/jmhAVYOVpqY2+RMRnqNefj/Pnz+eabb2jdunWN7R06dGDfvn1OCSbuw2538OriDADuHtIOq6VBd+uLiBhq1KhR/PGPf2TBggX4+vrWeK6srIwnn3ySK664os7HGz16NKNHjz7r89OnT2fixIlMmDABgBkzZvDVV18xa9YsnnjiCQBiY2MBaNmyJVdeeSXr1q2jR48etR6vrKyMsrKy6seFhYUA2Gw2bDZbnXOf7tR+9d3fGZRBGVyVwc8Mr97eiyf+u5kvN2WT+uFGsgtKuXdIfI07O9zh8+AuOZTBeRmcnd/ZYxp4x7hWG3f4GnI2b7wm0HV5Em+8JvDO67q4QyRfbDrE5qNmp13XhRynXsXHkpKSGh2Ppxw5ckSLznihhdsOk5FbQoifD7f0jzM6johIgzz99NP069ePDh06kJKSQufOnXE4HGzdupWXX36ZsrIy3n77baecq7y8nLVr1zJ58uTqbWazmcsvv5zly5cDVWOq3W4nJCSE4uJivvvuO26++eazHnPatGk89dRTZ2yfP39+rWPzhViwYEGD9ncGZVAGV2W4NBCKW5r5/pCZv3+zk5WbtnNtWzvmX8ws4w6fB3CPHMrQ8AylpaVOTOLaMQ08b1yrjTt8DTmbN14T6Lo8iTdeE3jXdYUfNwEWNh81Oe26LmRMq1fx8aKLLuLtt9/mmWeeAarm9LDb7fz973/nkksuqc8hxY29tng3ALcPbEuIv9XgNCIiDdO6dWuWL1/Or3/9ayZPnly9WJrJZGLEiBG89NJLtGnTxinnysvLo7Ky8oyFAKKjo9m2bRsAOTk5XHfddQBUVlYyceJEkpOTz3rMyZMnk5qaWv24sLCQuLg4Ro4cSWhoaL1y2mw2FixYwIgRI7Bajfl3XhmUwYgMVwFv/LiXv83bwQ+HzARHtOK5G7rh52N2i88DNK33oylkONXV5yyuHNPAc8a12rjD15CzeeM1ga7Lk3jjNYF3XtdFJ2y8O20ROcehQ58hdIgJa/AxL2RMq1fx8fnnn+fSSy9lzZo1lJeX89hjj7FlyxaOHDnCjz/+WJ9Diptau+8oq/cexWoxMWFIvNFxREScol27dnz99dccPXqUnTt3AtC+fXvCw8NdniUhIYGNGzfW+fV+fn613mVgtVob/MORM47RUMqgDK7O8MDwDrRsFsijH23kq83ZHD1u49U7++J/8rzu8HlwlxzK0PAMjZHdncY0cK9xrTbu8DXkbN54TaDr8iTeeE3gXdcVbrXSv11zlu0+wpLdR0mKi2zwMS/kc3PBk/fZbDZ+85vf8OWXXzJ06FCuueYaSkpKuP7661m/fj2JiYkXekhxY6e6Hq/tFUt0qL/BaUREnKt58+b079+f/v37N8ovaZGRkVgsFnJycmpsz8nJISYmxunnE5H6uaZXLG/e1Z8gXwvLducz9tUVHC4qO/+OIm6kscc00LgmIuLJLutctQDZwm25Lj/3BRcfrVYrmzZtonnz5vzxj3/kww8/ZO7cufzlL3+hZcuWjZFRDJKRW8z89KofLO4blmBwGhERz+Pr60vfvn1ZuHBh9Ta73c7ChQsZNGiQgclE5JeGdohkzv2DiAz2Jf1QIWNfW8nh40anEnEvGtdERDzXpZ1aALA28xhHS8pdeu56LVt8xx138MYbbzg7i7iZmUv24HBUVcc7RIcYHUdExC0VFxezYcMGNmzYAMCePXvYsGEDmZmZAKSmpjJz5kzeeusttm7dyoMPPkhJSUn1KqH1lZaWRlJS0jnn0RKRC9MtNoxPHhxMfEQgB46d4MXNFjYeKDA6lohLaVwTEfFOrZsH0CrQQaXdwffbD7v03PWa87GiooJZs2bx7bff0rdvX4KCgmo8P336dKeEE+PkFpXxyboDgLoeRUTOZc2aNTUWWzs1af748eOZPXs2Y8eOJTc3lylTppCdnU2vXr2YN2/eGZP1X6iUlBRSUlIoLCwkLKzhE0aLSJW2EUF8/OBgJry5ip+yChn35hpm3NGXYR1bGB1NxCU0romIeK9OYQ4OlprYdKCA6/u0dtl561V83Lx5M3369AFgx44dNZ4zmUwNTyWGe3v5Xsor7PSKa0b/dsZMVi0i4gmGDx9evbro2UyaNIlJkya5KJGINFRksB/vTOjH2H9/y/YCuOet1bxwU0+u6RVrdDSRRqdxTUTEe4VYq/59Lzxuc+l561V8/P77752dQ9xISVkFby/fB8D9wxJUUBYREZEmJ8jPh/s62/mupBVfbc7mt3M2cLSknLuGtDM6moiIiEi9BJysAhaecG3xsV5zPop3+3DNfgqO24iPCGRkV61aJyIiIk2Tjxmm39Sd8YPa4nDAn79M5x/zt5+3K0xERETEHQWeLD4WuLjzUcVHqaGi0s4bS/cAcO9FCVjM6noUEXFHmphfxDXMZhN/vrorvxvREYB/f7eLP/x3M5V2FSBFnEnjmohI4wtQ8VHcwdzN2Rw4epyIIF9u7Ou6yUdFROTCpKSkkJ6ezurVq42OIuL1TCYTD13WgWev647ZBP9ZlUnKe+s4Yas0OpqI19C4JiLS+AItVX88VfFRDONwOHj1h90AjBsUj7/VYnAiEREREfdx24A2vHx7H3wtZuZtyeauN1e5fM4kERERkfqqnvPxeIVLz6vio1RbtjufLQcLCbBaGDeordFxRERERNzOFd1aMvvuZIL9fFiRcYRbXl1BblGZ0bFEREREzuvUnI/HbZWUV9hddl4VH6Xaq4szALi5X2uaB/kanEZERETEPQ1OjOSD+wYSGexL+qFCbpyxjMz8UqNjiYiIiJyTvwVMJ5f2cOWt1yo+CgDpBwtZvCMXs6lqoRkRERERObtusWF8/MBg4sID2JdfyvWvLGPLwQKjY4mIiIicldkEIX5V7Y8qPorLzVxS1fV4ZfeWxIUHGpxGRETOR6uCihgvPjKITx4YTJeWoeQVl3HLqytYkZFvdCwRj6RxTUTENUIDrICKj+JiWceO8+XGgwDcPyzR4DQiIlIXWhVUxD1Ehfoz5/6B9G8XTlFZBeNmrWLe5myjY4l4HI1rIiKuEepf1fnoykXzVHwUZi3dQ4XdwaCECLq3DjM6joiIiIhHCfW38vbd/RmZFE15hZ1fv7eWD1ZlGh1LRERE5AxhJzsfC9X5KK5ScNxW/cPxfRdrrkcRERGR+vC3Wnj59j6M7ReH3QFPfPoTad/vwuFwGB1NREREpNqpzkfddi0u897KfZSUV9IpOoThHVsYHUdERETEY/lYzPzthu6kXFI1jc3z32znqS/TsdtVgBQRERH3cKrzsaBUxUdxgbKKSt78cS8A9w1LwHRqvXURERERqReTycTvR3VmylVJAMxetpffztlAeYXd4GQiIiIiEKI5H8WVPlufRW5RGTGh/ozp2croOCIiIiJe4+6h7XhxbC98zCa+2HiQe99eQ0lZhdGxREREpIkL02rX4ip2u4PXFmcAcM/Qdvj66EtBRMSTpKWlkZSURHJystFRROQsru0dy+vj+xFgtbB4Ry63vb6SIyXlRscScUsa10REXCNUxUdxlYXbDrM7t4QQPx9u6R9ndBwREblAKSkppKens3r1aqOjiMg5DO8UxfsTB9As0MrG/ce4acYyso4dNzqWiNvRuCYi4hphWnBGXOW1xbsBuG1gG0L8rQanEREREfFevds05+MHBtEyzJ/duSXc+MoyduYUGR1LREREmqCfb7t23XQwKj42QWv3HWX13qNYLSbuHtLO6DgiIiIiXq99VAifPDiY9lHBHCo4wY0zlrN231GjY4mIiEgTU73gjDofpTGd6nq8tlcs0aH+BqcRERERaRpaNQvgo/sH0SuuGQXHbdz++gq+337Y6FgiIiLShJzqfGxyxce0tDTi4+Px9/dnwIABrFq16qyvnT17NiaTqcaHv78KaHWVkVvM/PQcAO4blmBwGhEREZGmpXmQL+9PHMDFHVtwwmZn4ltr+O/6A0bHEhERkSbi1IIzRWUVVNodLjmn4cXHOXPmkJqaytSpU1m3bh09e/Zk1KhRHD589r8Ch4aGcujQoeqPffv2uTCxZ3t96R4cDriscxQdokOMjiMiIiLS5AT6+vD6+H5c26sVFXYHj8zZyOtLMoyOJSIiIk1A6MnbrsF13Y+GFx+nT5/OxIkTmTBhAklJScyYMYPAwEBmzZp11n1MJhMxMTHVH9HR0S5M7Llyi8r4eG3VX9bV9SgiIiJiHKvFzPSbe1XPv/2Xr7by3LxtOByu6UAQERGRpslqMRPoawGg8IRrio8+539J4ykvL2ft2rVMnjy5epvZbObyyy9n+fLlZ92vuLiYtm3bYrfb6dOnD88++yxdu3at9bVlZWWUlZVVPy4sLATAZrNhs9Xvk3xqv/ru7wz1yfDm0gzKK+z0aB1K79YhDc7vqZ8Hb8zgLjmUQRmcncHo7ysRkcZkNpt48qouRIb48vd523ll0W7yi8t49rru+FgM7xEQERERLxUWYKW0vJICF3U+Glp8zMvLo7Ky8ozOxejoaLZt21brPp06dWLWrFn06NGDgoICXnjhBQYPHsyWLVto3br1Ga+fNm0aTz311Bnb58+fT2BgYIPyL1iwoEH7O0NdM5RVwux1FsBE38CjfP311y7P0JiU4WfukEMZlMFZGUpLS52YxLukpaWRlpZGZWWl0VFEpAFMJhO/Ht6e8EBf/vDfn/hwzQGOlNh46bbe+FstRscTcRmNayIirhMWYOVQwYmmUXysj0GDBjFo0KDqx4MHD6ZLly68+uqrPPPMM2e8fvLkyaSmplY/LiwsJC4ujpEjRxIaGlqvDDabjQULFjBixAisVmu9jtFQF5rh7RWZlFZso214II/fPgSL2eTyDI1BGdwrhzIog7MznOpWlzOlpKSQkpJCYWEhYWFhRscRkQa6pX8bmgf58tB/1vPt1hzGvbGKmeP7EehxP62L1I/GNRER1zm16EyTKD5GRkZisVjIycmpsT0nJ4eYmJg6HcNqtdK7d2927dpV6/N+fn74+fnVul9DfyF3xjEaqi4ZKirtvLmsalGee4cl4O/n6/IMjU0Z3CuHMiiDszIYnV1ExJVGdY3hnbv7c+9ba1i19whjX13OG+P6GB1LREREvEyYi4uPhk4m4+vrS9++fVm4cGH1NrvdzsKFC2t0N55LZWUlP/30Ey1btmysmB5v7uZsDhw9TkSQLzf1PfPWdBERERFxDwMSIphz/yBahPixLbuIsa+t5PBxo1OJiIiINwn1ryo+Fh6vcMn5DJ/JOjU1lZkzZ/LWW2+xdetWHnzwQUpKSpgwYQIA48aNq7EgzdNPP838+fPJyMhg3bp13HHHHezbt497773XqEtwaw6Hg9cW7wZg3KB4zR0kIiIi4uaSWoXyyQODaRsRyIFjJ/i/LRa2HNQ0FCIiIuIcoQFVN0IXNYXVrgHGjh1Lbm4uU6ZMITs7m169ejFv3rzqRWgyMzMxm3+ukR49epSJEyeSnZ1N8+bN6du3L8uWLSMpKcmoS3Bry3bnszmrEH+rmTsHtTU6joiIiIjUQZuIQD5+YDDj3ljJ1uwi7pi1hjfG92NAQoTR0URERMTD+ZxcB6TS4XDN+VxylvOYNGkSkyZNqvW5RYsW1Xj8z3/+k3/+858uSOUdXl2cAcDN/eIID3LuXI8iIiIi0nhahPjx3j39uOlf37G7qIJxs1bx0m19GJEUbXQ0ERER8WBmU1Xx0UW1R+Nvu5bGs/VQIYt35GI2wb1DE4yOIyIiIiIXKMTfygNdKrmscwvKKuw88O5aPll7wOhYIiIi4smqao/Y7a6pPqr46MVeO9n1OLp7S9pEBBqcRkRERETqw9cCL93Skxv6tKbS7uB3H23k9SUZRscSERERD1Xd+eiq87noPOJiB48d58uNBwG4f5i6HkVEREQ8mY/FzPM39uDeoe0A+MtXW3n+m204XHW/lIiIiHgN86nORxf9HKHio5eatXQPFXYHgxIi6NG6mdFxRETEydLS0khKSiI5OdnoKCLiImaziT/+qgu/H9UJgLTvd/OH/26m0kW3TIk0Jo1rIiKuY0JzPkoDFRy38Z9VmQDcd7G6HkVEvFFKSgrp6emsXr3a6Cgi4kImk4mUS9rz7HXdMZngP6syeeg/6yirqDQ6mkiDaFwTEXEddT5Kg723ch8l5ZV0ig5heMcWRscRERERESe7bUAb0m7rg6/FzNyfsrln9hpKyiqMjiUiIiIewKTVrqUhyioqefPHvQDcNyyh+gtKRERERLzLld1bMuuuZAJ9LSzdlcdtr6/kaEm50bFERETEzZ1acEadj1Ivn63PIreojJhQf8b0bGV0HBERERFpREM7RPL+xIE0C7Sycf8xbnp1OYcKjhsdS0RERNyYqfq2a9ecT8VHL2K3O3htcQYAdw+Nx9dHb6+IiIiIt+sV14yPHxhEyzB/dh0u5sZXlrM7t9joWCIiIuKmzNU3yarzUS7Qd9sOszu3hBA/H27t38boOCIiIiLiIu2jQvj4wcEkRAaRdew4N81Yzk8HCoyOJSIiIm7o1BR9drtrzqfioxc51fV428A2hPhbDU4jIiIiIq4U2yyAjx4YRPfYMI6UlHPrzBUs251ndCwRERFxMyatdi31sS7zKKv2HsFqMXH3kHZGxxERERERA0QE+/H+xAEMSoiguKyCu2atZt7mbKNjiYiIiBs5teCMi6Z8VPHRW7z2Q1XX47W9YokO9Tc4jYiIiIgYJcTfypsTkhnVNZrySju/fm8tH67eb3QsERERcRNmdT7KhdqTV8I36VV/0b5vWILBaURERETEaP5WC2m39WFsvzjsDnjsk028+sNuo2OJiIiIGzBxsvNRq11LXc1ckoHDAZd2jqJDdIjRcURERETEDfhYzPzthu7cf3HVH6enfb2NaV9vxeGq3zRERETELWnOR7kg+cVlfLz2AKCuRxERERGpyWQyMXl0FyaP7gzAqz9k8MQnP1FR6aLlLUVERMTtVM/5qM5HqYt3Vu6nvMJOz7hmDGgXbnQcERFxkbS0NJKSkkhOTjY6ioh4gPsvTuTvN/TAbII5a/aT8v46TtgqjY4lUk3jmoiI62jOR6mzskp4b2XV5OH3D0vAdKpvVkREvF5KSgrp6emsXr3a6Cgi4iFuTo7j5dv74msx882WHCa8uZqiEzajY4kAGtdERFzJpM5HqauVh00cO26jbUQgo7rGGB1HRERERNzcFd1imH13MsF+PizPyOe2mSvJLy4zOpaIiIi40KnORwfqfJRzqKi08/2hqrfv3osSsJjV9SgiIiIi5zc4MZL/TBxIeJAvP2UVcNOry8k6dtzoWCIiIuIipzof7S6aAlrFRw81b0sOR8pMNA+0clPf1kbHEREREREP0r11GB89MIhWYf5k5JZw4yvL2HW4yOhYIiIi4gJa7VrOy+Fw8PqPewG4c0Ab/K0WYwOJiIiIiMdJbBHMxw8Opn1UMIcKTnDTjOVs2H/M6FgiIiLSyKpXu3bV+Vx0HnGiRTty2XKwCKvZwe0D4oyOIyIiIiIeqlWzAD68fxA9W4dxtNTGbTNXsHRnntGxREREpBFVz/mozkepjcPh4MVvdwJwUbSD8CBfgxOJiIiIiCcLD/LlvYkDGdo+ktLySu6evZq5Px0yOpaIiIg0kuo5H7XatdRm0Y5cNu4/hr/VzKWxLpoZVERERES8WrCfD2/c1Y8ru8dQXmkn5f11/GdVptGxREREpBGcWrJYcz7KGU7very9fxwhVoMDiYiIiIjX8POx8O9b+3Br/zY4HDD50594edEul92SJSIiIq5RPeejOh/ll07vepw4NN7oOCIiIiLiZSxmE89e142USxIB+Pu87Tz3zQ6X/XIiIiIijc98shqozkep4fSux3GD4okI9jM4kYiIiIh4I5PJxO9HdeZPv+oCwBs/7uP93WZslZryR0RExBuYUOej1OL0rsf7hiUYHUdEREREvNy9FyXwj5t6YjGbWJVrJuU/GzheXml0LBEREWkg06nVrlHno5z0y67HSHU9ioiIiIgL3NC3NWm39sRqcvD99jzufGMlBaU2o2OJiIiIB1Hx0QOo61FEREREjHJZ5ygeTKok1N+HNfuOcvOry8kuOGF0LBEREfEQKj66udO7Hu8c2FZdjyIiIiLicomh8P49yUSH+rE9p4gbXlnG7txio2OJiIiIB1Dx0c3V7HpMNDqOiIiIiDRRnWJC+PiBwbSLDCLr2HFumrGcjfuPGR1LRERE3JyKj27sl12PLULU9SgiIiIixokLD+SjBwbRPTaMIyXl3DpzBUt25hodS0RERNyYio9uTF2PIiIiIuJuIoP9+M99AxnaPpLS8krunr2aLzceNDqWiIiIuCm3KD6mpaURHx+Pv78/AwYMYNWqVXXa74MPPsBkMnHttdc2bkADqOtRRETOJS0tjaSkJJKTk42OIiJNULCfD2/c1Y9f9WiJrdLBbz5Yz1vL9hodSzyYxjUREe9lePFxzpw5pKamMnXqVNatW0fPnj0ZNWoUhw8fPud+e/fu5dFHH+Wiiy5yUVLXUtejiIicS0pKCunp6axevdroKCLSRPn5WPjXLb0ZN6gtDgdM/WIL0xfswOFwGB1NPJDGNRER72V48XH69OlMnDiRCRMmkJSUxIwZMwgMDGTWrFln3aeyspLbb7+dp556ioSEBBemdQ11PYqIiIiIJ7CYTTx1dVceubwjAP9auJM/fraZSrsKkCIiIlLFx8iTl5eXs3btWiZPnly9zWw2c/nll7N8+fKz7vf0008TFRXFPffcw5IlS855jrKyMsrKyqofFxYWAmCz2bDZbPXKfWq/+u5/Pj+c1vV49+A2tZ6nsTPUhTK4TwZ3yaEMyuDsDEZ/X4mIyPmZTCYevrwDEcG+PPn5Zt5fmcnRknL+ObYX/laL0fFERETEYIYWH/Py8qisrCQ6OrrG9ujoaLZt21brPkuXLuWNN95gw4YNdTrHtGnTeOqpp87YPn/+fAIDAy848+kWLFjQoP1r43DAPzdbABODIitYtXihyzNcKGVwnwzgHjmUQRmclaG0tNSJSUREpDHdMbAt4UG+/PaDDXy9OZtjpat5bVxfQvytRkcTERERAxlafLxQRUVF3HnnncycOZPIyMg67TN58mRSU1OrHxcWFhIXF8fIkSMJDQ2tVw6bzcaCBQsYMWIEVqtzf5havDOPfSvW4W818+y4i4kMrv2W68bMUFfK4D4Z3CWHMiiDszOc6lYXERHPcGX3ljQLsHLfO2tZnpHPLa+tYPaE/ppGSEREpAkztPgYGRmJxWIhJyenxvacnBxiYmLOeP3u3bvZu3cvY8aMqd5mt9sB8PHxYfv27SQm1lycxc/PDz+/M3/YsVqtDf6F3BnHOJ3D4eDf32cAVXM9tmwe7PIM9aEM7pPBXXIogzI4K4PR2UVE5MINbh/JB/cNZPysVWw5WMiNM5bxzt0DaBPRsLuORERExDMZuuCMr68vffv2ZeHCn28tttvtLFy4kEGDBp3x+s6dO/PTTz+xYcOG6o+rr76aSy65hA0bNhAXF+fK+E73w45cNmiFaxERERHxcN1iw/j4wcG0bh7AvvxSbpixjPSD6mYXERFpigy/7To1NZXx48fTr18/+vfvz4svvkhJSQkTJkwAYNy4ccTGxjJt2jT8/f3p1q1bjf2bNWsGcMZ2T6MVrkVERETEm7SLDOLTBwczbtYqtmUXMfbV5bw+vh8DEiKMjiYiIiJUrTviCoZ2PgKMHTuWF154gSlTptCrVy82bNjAvHnzqhehyczM5NChQwanbHzqehQRERERbxMV6s+c+wfRPz6corIK7py1im+2ZBsdS0REpEkzmUwuPZ/hnY8AkyZNYtKkSbU+t2jRonPuO3v2bOcHcjF1PYqIiIiItwoLsPL2Pf156D/rWZCew4PvrmXa9d0Zm9zG6GgiIiLiAoZ3Poq6HkVERETEu/lbLbxyex9u7tcauwMe/+Qn0r7fhcNV93uJiIiIYVR8NNjpXY93DFDXo4iIiIh4Jx+Lmedu6MGDw6v+2P78N9t5+n/p2O0qQIqIiHgzFR8NVqPr8eIEo+OIiIiIiDQak8nE41d05smrkgB488e9PPLhBsor7AYnExERkcai4qOBftn1GBXib3AiEREREZHGd8/Qdrw4thc+ZhOfbzjIvW+vobS8wuhYIiIi0ghUfDSQuh5FREREpKm6tncsM8f3I8BqYfGOXG6buZKjJeVGxxIREREnU/HRIOp6FBEREZGm7pJOUbw3cQDNAq1s2H+MG2csI+vYcaNjiYiIiBOp+GgQdT2KiIiIiECfNs356P5BtAzzZ3duCTe+soydOUVGxxIREREnUfHRAOp6FBERERH5WYfoED55cDCJLYI4VHCCm15dztp9R42OJSIiIk6g4qMB1PUoIiIiIlJTq2YBfPzAYHrFNeNYqY07Xl/J99sPGx1LREREGkjFRxdT16OIiIiISO2aB/ny/sQBDOvYguO2Sia+tYbP1mcZHUtEREQaQMVHF1u8M09djyIiIiIiZxHo68Pr4/pxTa9WVNgd/HbOBt5cts/oWCIiIlJPKj66UFXX4w5AXY8iIiIiImfj62Pmnzf3YsKQeACe/Xo7X+4z43A4jA0mIiIiF0zFRxdavDOP9ZnqehQREREROR+z2cSUq5L4/ahOAHx70MwfPkunotJucDIRERHv4Kq/6an46CLqehQRERERuTAmk4mUS9rz7LVJmHDw8bosHnh3HSdslUZHExERkTpS8dFF1PUoIiIiIlI/N/Vtzd2d7Pj6mPl2aw7j3lhFwXGb0bFERESkDlR8dAF1PYqIyPmUlpbStm1bHn30UaOjiIi4pR7hDmaN60OInw+r9h5h7KvLOVx4wuhYUguNaSIicjoVH11AXY8iInI+f/3rXxk4cKDRMURE3NqAduHMuX8QkcF+bMsu4vpXlrEnr8ToWPILGtNEROR0Kj42MnU9iojI+ezcuZNt27YxevRoo6OIiLi9pFahfPrgYNpGBHLg6HFufGUZm7MKjI4lJ2lMExGRX1LxsZGd6nr081HXo4iIN1q8eDFjxoyhVatWmEwmPvvsszNek5aWRnx8PP7+/gwYMIBVq1bVeP7RRx9l2rRpLkosIuL52kQE8vEDg+naKpT8knJueW0Fy3blGR3L42lMExGRxqDiYyOq0fU4UF2PIiLeqKSkhJ49e5KWllbr83PmzCE1NZWpU6eybt06evbsyahRozh8+DAAn3/+OR07dqRjx46ujC0i4vFahPjxwX0DGZQQQXFZBXe9uZq5Px0yOpZH05gmIiKNwcfoAN7s9K7H+9X1KCLilUaPHn3OW8umT5/OxIkTmTBhAgAzZszgq6++YtasWTzxxBOsWLGCDz74gI8++oji4mJsNhuhoaFMmTKl1uOVlZVRVlZW/biwsBAAm82GzVa/lV9P7Vff/Z1BGZRBGdw3hztn8LfAzDt6kfrxT8xPP0zK++v481VduK1/nMsy1OcY7srVYxo0zrhWG3f4OnY2b7wm0HV5Em+8Jmga11VZUQGAw2Fv8O8QdaHiYyNR16OIiJSXl7N27VomT55cvc1sNnP55ZezfPlyAKZNm1Z9e9rs2bPZvHnzOX9JmzZtGk899dQZ2+fPn09gYGCD8i5YsKBB+zuDMiiDMtTOHXK4c4bRoVAcbWZZjpmpX25lxfrNjGrtwGRyXYa6KC0tdWIS12qMMe3UPo01rtXGHb6Onc0brwl0XZ7EG68JvPu6NuSZAAv5+fnMnTu3Xse5kDFNxcdGoq5HERHJy8ujsrKS6OjoGtujo6PZtm1bvY45efJkUlNTqx8XFhYSFxfHyJEjCQ0NrdcxbTYbCxYsYMSIEVit1nodo6GUQRmUwX1zeEqGXzkc/N93u0lblMHXByxExMbxp9GdsZidU4F0xufhVFefJ2qMMQ0aZ1yrjTt8HTubN14T6Lo8iTdeEzSN67JvzeOtnT8RERHBlVcm1+t4FzKmqfjYCNT1KCIi9XHXXXed9zV+fn74+fmdsd1qtTb4hyNnHKOhlEEZlMF9c3hCht9f0YWo0AD+/OUW3l25n6PHK5h+c0/8fCwuy3C+fZuKuoxp0LjjWm3c4evY2bzxmkDX5Um88ZrAu6/L4lNVDjSZzC4Z07TgTCNQ16OIiABERkZisVjIycmpsT0nJ4eYmBiDUomIeLfxg+P5v1t6Y7WY+GrTIe6ZvYbisgqjY3k8jWkiIlJfKj46mboeRUTkFF9fX/r27cvChQurt9ntdhYuXMigQYMMTCYi4t2u7tmKWXclE+hrYemuPG6buYL84rLz7yhnpTFNRETqS8VHJ1PXo4hI01JcXMyGDRvYsGEDAHv27GHDhg1kZmYCkJqaysyZM3nrrbfYunUrDz74ICUlJdUrhdZXWloaSUlJJCfXb44WERFvd1GHFvxn4kDCg3zZdKCAm2YsZ/8Rz13wxRWMGtNA45qIiDfTnI9O5HA4+D91PYqINClr1qzhkksuqX58atL88ePHM3v2bMaOHUtubi5TpkwhOzubXr16MW/evDMm7L9QKSkppKSkUFhYSFhYWIOOJSLirXrGNeOjBwYx7o1VZOSVcOOMZbx1d386xzhvIRNvYtSYBhrXRES8mYqPTrRkZx7r1PUoItKkDB8+HIfDcc7XTJo0iUmTJrkokYiInC6xRTCfPDiYcbNWsiOnmJtnLGfWXcn0iw83Oprb0ZgmIiKNQbddO4nmehQRERERcU8xYf58eP8g+rZtTuGJCm5/fSULt+acf0cRERFpMBUfnURdjyIiIiIi7qtZoC/v3jOASztHUVZh57531vLx2gNGxxIREfF6Kj46gboeRUTE1TQxv4jIhQvwtfDqnX25oU9rKu0OHv1oI6/+sNvoWILGNRERb6bioxOo61FERFwtJSWF9PR0Vq9ebXQUERGPYrWYeeGmHtw3rOrn9mlfb+PZuVux288916E0Lo1rIiLeS8XHBlLXo4iIiIiIZzGZTPzhyi5MHt0ZgNcWZ/D7jzdhq7QbnExERMT7uEXxMS0tjfj4ePz9/RkwYACrVq0662s//fRT+vXrR7NmzQgKCqJXr1688847Lkxbk7oeRUREREQ80/0XJ/LCTT2xmE18su4A97+zluPllUbHEhER8SqGFx/nzJlDamoqU6dOZd26dfTs2ZNRo0Zx+PDhWl8fHh7OH//4R5YvX86mTZuYMGECEyZM4JtvvnFx8ppdj7cPUNejiIiIiIinubFva167sy9+Pma+23aYO95YSUGpzehYIiIiXsPw4uP06dOZOHEiEyZMICkpiRkzZhAYGMisWbNqff3w4cO57rrr6NKlC4mJiTz88MP06NGDpUuXujg5LN2dX931+IC6HkVEREREPNJlXaJ5794BhPr7sHbfUW56dRnZBSeMjiUiIuIVfIw8eXl5OWvXrmXy5MnV28xmM5dffjnLly8/7/4Oh4PvvvuO7du389xzz9X6mrKyMsrKyqofFxYWAmCz2bDZ6vcXTZvNhsMB/1q4C4Bbk1vTPMBS7+PVN8Pp/zWCMrhPBnfJoQzK4OwMRn9fubO0tDTS0tKorNTtgSIiztAvPpyPHhjMuFkr2ZFTzA2vLOPte/qT2CLY6GhNgsY1ERHvZWjxMS8vj8rKSqKjo2tsj46OZtu2bWfdr6CggNjYWMrKyrBYLLz88suMGDGi1tdOmzaNp5566ozt8+fPJzAwsN7ZtxeY2HCgEKvJQUJ5BnPnZtT7WA2xYMECQ86rDO6ZAdwjhzIog7MylJaWOjGJd0lJSSElJYXCwkLCwsKMjiMi4hU6xYTw8QODGT9rFRl5Jdw0Yzlv3pVMz7hmRkfzehrXRES8l6HFx/oKCQlhw4YNFBcXs3DhQlJTU0lISGD48OFnvHby5MmkpqZWPy4sLCQuLo6RI0cSGhpar/OXl5fzz39+D8DtA9ty65Wd63WchrDZbCxYsIARI0ZgtVpdfn5lcK8M7pJDGZTB2RlOdauLiIi4Slx4IB89MIgJs1ez6UABt85cwat39mVgfDOjo4mIiHgkQ4uPkZGRWCwWcnJyamzPyckhJibmrPuZzWbat28PQK9evdi6dSvTpk2rtfjo5+eHn5/fGdutVmu9fxlesiuPvcUm/HzM/PqSDoYWnBpyHcrgfRncJYcyKIOzMhidXUREmqaIYD/enziQB95Zy9Jdedw9ezXP39Adk9HBREREPJChC874+vrSt29fFi5cWL3NbrezcOFCBg0aVOfj2O32GvM6NiaHw8G/v9sNVM31GBWqFa5FRERERLxNsJ8Pb9zVj6t6tMRW6eCRjzax+JDKjyIiIhfK8NuuU1NTGT9+PP369aN///68+OKLlJSUMGHCBADGjRtHbGws06ZNA6rmcOzXrx+JiYmUlZUxd+5c3nnnHV555RWX5F2yM4/1+wuwmhxMvKidS84pIiIiIiKu5+dj4V+39CYiyJe3lu/jk70WfrUrj0u7tDQ6moiIiMcwvPg4duxYcnNzmTJlCtnZ2fTq1Yt58+ZVL0KTmZmJ2fxzg2ZJSQm//vWvOXDgAAEBAXTu3Jl3332XsWPHuiRvp5gQxg1sw8HMvUSFnHk7t4iIiIiIeA+z2cSfr+5KswAflv+0k6GJEUZHEhERaZC45gHc0Kc17aOCXXI+w4uPAJMmTWLSpEm1Prdo0aIaj//yl7/wl7/8xQWpahcd6s+Tv+ps2OrWIiIiAGlpaaSlpVFZWWl0FBERr2cymZh0SSLtSrdjMunW68agcU1ExHV6t2lO7zbNXXY+Q+d8FBERkfpJSUkhPT2d1atXGx1FRKTJUN2x8WhcExHxXio+ioiIiIiIiIiISKNQ8VFEREREREREREQahYqPIiIiIiIiIiIi0ihUfBQREREREREREZFGoeKjiIiIiIiIiIiINAoVH0VERERERERERKRRqPgoIiIiIiIiIiIijULFRxEREQ+UlpZGUlISycnJRkcRERFpMI1rIiLeS8VHERERD5SSkkJ6ejqrV682OoqIiEiDaVwTEfFeKj6KiIiIiIiIiIhIo/AxOoCrORwOAAoLC+t9DJvNRmlpKYWFhVitVmdFUwZl8PgcyqAMzs5w6t/qU/92y5k0rimDMnhvBnfJoQzOy6Bx7fycMa7Vxh2+hpzNG68JdF2exBuvCXRddXUhY1qTKz4WFRUBEBcXZ3ASERGpq6KiIsLCwoyO4ZY0romIeB6Na2encU1ExLPUZUwzOZrYn93sdjsHDx4kJCQEk8lU47nk5ORa5xj55fbCwkLi4uLYv38/oaGhjZ65Ns7OcLZrd2aGupzjXK+p7blzZajPNdVHY3weLsSp411IjgvJcCHvW20Z6vp9VZ9stWms788LyVWXDPW5zgvZp2/fvuzataveGS70uQv9/qwrh8NBUVERrVq1wmzWTCG1udBxrbHeq4bytHGtoWPa2Z5v6uNafca0C81Q1/flbBk0rp2Zob7X6MxxrT7fbxe6XeOaa5xrXGsIdxjrnM0brwl0XZ7EG68JdF11dSFjWpPrfDSbzbRu3brW5ywWS61vwNm2h4aGGv6F6KwMZ7tGZ2aoyznO9ZpzPVdbhoZcU3048/NwIX55vLrkuJAM9XnfTs9wod9Xzvr8OPv7sz65zpWhPse70PetIRku9LkL/f68EOoMObcLHdca871yBk8Z1xo6pp3v+aY6rtVnTLvQDBf6vvwyg8Y1512jM8e1+n6/1ed907jWuM41rjmDO4x1zuaN1wS6Lk/ijdcEuq66qOuYpj+3nSYlJeWCtnsTV1xjXc5xrtdcaEZ3fd+cnas+x7uQfRrrffO07zdPe98mTpzYoONd6HPu+r41dU35vWrs62zov411PUZDXu8qzsxV32M5c1yr7/NNeVxzxft2vnGtqbxvIiIinqjJ3XbtDIWFhYSFhVFQUGDo7WnKoAzulkMZlMHdMkjduMN7pQzKoAzum0MZ3CeD1J83vn/eeE2g6/Ik3nhNoOtqDOp8rAc/Pz+mTp2Kn5+fMiiDW2RwlxzKoAzulkHqxh3eK2VQBmVw3xzK4D4ZpP688f3zxmsCXZcn8cZrAl1XY1Dno4iIiIiIiIiIiDQKdT6KiIiIiIiIiIhIo1DxUURERERERERERBqFio8iIiIiIiIiIiLSKFR8FBERERERERERkUah4uMFWLx4MWPGjKFVq1aYTCY+++wzl2eYNm0aycnJhISEEBUVxbXXXsv27dtdnuOUv/3tb5hMJn7729+69LyVlZU8+eSTtGvXjoCAABITE3nmmWdozPWT6vL+b926lauvvpqwsDCCgoJITk4mMzPTaRleeeUVevToQWhoKKGhoQwaNIivv/4agCNHjvDQQw/RqVMnAgICaNOmDb/5zW8oKChw2vlPycrK4o477iAiIoKAgAC6d+/OmjVran3tAw88gMlk4sUXX6z3+c71ubfZbDz++ON0796doKAgWrVqxbhx4zh48GCNY+zYsYNrrrmGyMhIQkNDGTp0KN9//32dM9Tle2/48OGYTKYaHw888MAZx5o9ezY9evTA39+fqKgoUlJS6pThz3/+8xnH79y5c/Xzr732GsOHDyc0NBSTycSxY8dq7L93717uueeeGt83U6dOpby8/KznPN/XvcPhYMqUKbRs2ZKAgAAuv/xydu7cWe9z7tq1i5CQEJo1a1anz4k0jNHjmruNaaBx7Zc0rp1J45rGNY1r7istLY34+Hj8/f0ZMGAAq1atOutrP/30U/r160ezZs0ICgqiV69evPPOOy5MWzcXck2n++CDDzCZTFx77bWNG7CeLuS6Zs+efca/Ff7+/i5MW3cX+n4dO3aMlJQUWrZsiZ+fHx07dmTu/7d350FRnVkbwJ8GZImCER0QRBlZ4i46EhmWqCMKGqMYJ66EIWppGbFco5AoHy7RwS0uJC5B1MyEiImRiZM4RMQlcVekRVBREOOauERBNCLQ5/vDoistIN1NL2ieX1VX2Xd7z7lLH/vlvbd37jRRtNrRJafq6oZCocCAAQNMGLF2dD1WK1euVP+fpGXLlpg2bRoePXpkomi1p0teZWVlmD9/Pjw9PWFrawsfHx+kpaUZJS52PurgwYMH8PHxwSeffGK2GPbv34+oqCgcOXIE6enpKCsrQ0hICB48eGDyWI4fP47169ejc+fOJm978eLFWLt2LT7++GOcPXsWixcvxpIlS5CQkGC0Nms7/gUFBQgKCkLbtm2xb98+ZGdnIzY21qCF0c3NDfHx8cjMzMSJEyfQu3dvhIWFITc3F9evX8f169exbNky5OTkYPPmzUhLS8PYsWMN1j4A3L17F4GBgWjQoAH+97//4cyZM1i+fDmaNGlSZdnU1FQcOXIErq6udWrzWfv+4cOHOHnyJGJjY3Hy5Els374deXl5GDRokMZyb7zxBsrLy7Fnzx5kZmbCx8cHb7zxBn7++WetYtD22hs3bhxu3Lihfi1ZskRj/kcffYTZs2cjJiYGubm52L17N0JDQ7XeFx06dNDY/oEDBzT2Rb9+/fDBBx9Uu+65c+egUqmwfv165ObmYsWKFVi3bl2NywO1n/dLlizB6tWrsW7dOhw9ehQNGzZEaGiouhDr0mZZWRlGjhyJ1157Tev9QXVj7rpWn2oawLr2NNY11jWAdY117fmxdetWTJ8+HXFxcTh58iR8fHwQGhqKmzdvVru8o6MjZs+ejcOHDyM7OxujR4/G6NGj8f3335s48prpmlOlS5cu4b333qu3554+eTk4OGh8Vvz0008mjFg7uub1+PFj9O3bF5cuXcK2bduQl5eHxMREtGjRwsSR10zXnLZv365xnHJycmBpaYmhQ4eaOPJn0zWvL774AjExMYiLi8PZs2eRlJSErVu3PrPemIOuec2ZMwfr169HQkICzpw5gwkTJuDNN99EVlaW4YMT0gsASU1NNXcYcvPmTQEg+/fvN2m79+/fF29vb0lPT5eePXvKlClTTNr+gAEDZMyYMRrThgwZIuHh4SZpv7rjP3z4cHn77bdN0v7vNWnSRDZs2FDtvC+//FKsra2lrKzMYO1FR0dLUFBQrctdvXpVWrRoITk5OeLu7i4rVqwwSPvaXHvHjh0TAPLTTz+JiMitW7cEgPzwww/qZYqLiwWApKen6xVHdddebdfCr7/+KnZ2drJ792692oyLixMfH59al9u7d68AkLt379a67JIlS6R169Zatf/0vlepVNK8eXNZunSpetq9e/fExsZGtmzZonObs2bNkrfffls2bdokjRs31iomMpz6UNfMVdNEWNdY11jXWNdY15533bt3l6ioKPX7iooKcXV1lX/+859ab6Nr164yZ84cY4SnF31yKi8vl4CAANmwYYNERkZKWFiYCSLVja55PS/XkK55rV27Vjw8POTx48emClFndb2uVqxYIfb29lJSUmKsEPWia15RUVHSu3dvjWnTp0+XwMBAo8apK13zcnFxkY8//lhjmrH+/8mRj8+5yluPHB0dTdpuVFQUBgwYgD59+pi03UoBAQHIyMjA+fPnAQCnTp3CgQMH0L9/f7PEo1Kp8N133+GVV15BaGgonJyc4OfnZ9RbGCsqKpCSkoIHDx7A39+/2mWKiorg4OAAKysrg7W7Y8cO+Pr6YujQoXByckLXrl2RmJiosYxKpUJERARmzpyJDh06GKxtbRUVFUGhUKhvcWratCnatGmDf/3rX3jw4AHKy8uxfv16ODk5oVu3bnq3AVS99pKTk9GsWTN07NgR77//Ph4+fKiel56eDpVKhWvXrqFdu3Zwc3PDsGHDcOXKFa3bvXDhAlxdXeHh4YHw8PA63/5YVFSk9+dHYWEhfv75Z43PgcaNG8PPzw+HDx/Wqc09e/bgq6++MuvIcjI/c9U0gHXtaaxrrGuVWNdY154Hjx8/RmZmpsaxs7CwQJ8+fZ557CqJCDIyMpCXl4cePXoYM1St6ZvT/Pnz4eTkZPBR4oaib14lJSVwd3dHy5Yt1SPk6xN98tqxYwf8/f0RFRUFZ2dndOzYEYsWLUJFRYWpwn6mul5XAJCUlIQRI0agYcOGxgpTZ/rkFRAQgMzMTPUtzBcvXsTOnTvx+uuvmyRmbeiTV2lpaZU7Wuzs7DTuQjAUw/3PjUxOpVJh6tSpCAwMRMeOHU3WbkpKCk6ePInjx4+brM2nxcTEoLi4GG3btoWlpSUqKiqwcOFChIeHmyWemzdvoqSkBPHx8fjwww+xePFipKWlYciQIdi7dy969uxpsLZOnz4Nf39/PHr0CI0aNUJqairat29fZbnbt29jwYIFGD9+vMHaBp580K5duxbTp0/HBx98gOPHj2Py5MmwtrZGZGQkgCe3D1pZWWHy5MkGbVsbjx49QnR0NEaOHAkHBwcAgEKhwO7duzF48GDY29vDwsICTk5OSEtLq/a2utrUdO2NGjUK7u7ucHV1RXZ2NqKjo5GXl4ft27cDeLLvVCoVFi1ahFWrVqFx48aYM2cO+vbti+zsbFhbWz+zXT8/P2zevBlt2rTBjRs3MG/ePLz22mvIycmBvb29znnk5+cjISEBy5Yt03ldAOpb+5ydnTWmOzs713jbX3Vt3rlzB++88w4+//xz9TGjPx5z1TSAda06rGusawDrWiXWtfrv9u3bqKioqPbYnTt3rsb1ioqK0KJFC5SWlsLS0hJr1qxB3759jR2uVvTJ6cCBA0hKSoJSqTRBhPrRJ682bdpg48aN6Ny5M4qKirBs2TIEBAQgNzcXbm5upgi7VvrkdfHiRezZswfh4eHYuXMn8vPzMXHiRJSVlSEuLs4UYT+TvtdVpWPHjiEnJwdJSUnGClEv+uQ1atQo3L59G0FBQRARlJeXY8KECfXqtmt98goNDcVHH32EHj16wNPTExkZGdi+fbtROsDZ+fgci4qKQk5OjlF6pWty5coVTJkyBenp6WZ9yO+XX36J5ORkfPHFF+jQoQOUSiWmTp0KV1dX9RcFU1KpVACAsLAwTJs2DQDQpUsXHDp0COvWrTPol7Q2bdpAqVSiqKgI27ZtQ2RkJPbv36/xRa24uBgDBgxA+/btMXfuXIO1DTzJ1dfXF4sWLQIAdO3aFTk5OVi3bh0iIyORmZmJVatW4eTJk1AoFAZtuzZlZWUYNmwYRARr165VTxcRREVFwcnJCT/++CPs7OywYcMGDBw4EMePH4eLi4tO7dR07f3+C3GnTp3g4uKC4OBgFBQUwNPTEyqVCmVlZVi9ejVCQkIAAFu2bEHz5s2xd+/eWp+R9fsRUJ07d4afnx/c3d3x5Zdf6vzX7WvXrqFfv34YOnQoxo0bp9O6+qqpzXHjxmHUqFH1ZqQBmYc5ahrAulYT1jXWNYB1Td82WdeeH/b29lAqlSgpKUFGRgamT58ODw8P9OrVy9yh6ez+/fuIiIhAYmIimjVrZu5wDMrf319jRHxAQADatWuH9evXY8GCBWaMrG5UKhWcnJzw6aefwtLSEt26dcO1a9ewdOnSetH5WFdJSUno1KkTunfvbu5Q6mzfvn1YtGgR1qxZAz8/P+Tn52PKlClYsGABYmNjzR2e3latWoVx48ahbdu2UCgU8PT0xOjRo7Fx40bDN2bwG7n/IGDmZ2NFRUWJm5ubXLx40aTtpqamCgCxtLRUvwCIQqEQS0tLKS8vN0kcbm5uVZ5NsGDBAmnTpo1J2n/6+JeWloqVlZUsWLBAY7lZs2ZJQECAUWMJDg6W8ePHq98XFxeLv7+/BAcHy2+//Wbw9lq1aiVjx47VmLZmzRpxdXUVkSfP9ag8H35/jlhYWIi7u3ud26/p2nv8+LEMHjxYOnfuLLdv39aYt3v3brGwsJCioiKN6V5eXjo9B0hEt2uvpKREAEhaWpqIiGzcuFEAyJUrVzSWc3Jykk8//VSnOCr5+vpKTEyMxrTano117do18fb2loiICKmoqNC6raf3fUFBgQCQrKwsjeV69OghkydP1rrNxo0ba5wvFhYW6s+ZpKQkreOjujFnXTNXTRNhXavEusa6xrrGuvY8Ky0tFUtLyyrX0j/+8Q8ZNGiQ1tsZO3ashISEGDg6/eiaU1ZWVpV6plAo1J9f+fn5Jor82Qx1rN566y0ZMWKEgaPTnz559ejRQ4KDgzWm7dy5UwBIaWmpsULVWl2OVUlJiTg4OMjKlSuNGKF+9MkrKChI3nvvPY1p//73v8XOzk6numNMdTlev/32m1y9elVUKpXMmjVL2rdvb/D4+MzH54yIYNKkSUhNTcWePXvQunVrk7YfHByM06dPQ6lUql++vr4IDw+HUqmEpaWlSeJ4+PAhLCw0T19LS0v1SA1Ts7a2xquvvoq8vDyN6efPn4e7u7tR21apVCgtLQXwZGRISEgIrK2tsWPHDqOM4gkMDHxmnhEREcjOztY4R1xdXTFz5kyj/Xpg5ciQCxcuYPfu3WjatKnG/MrnUz19zlhYWGh9zuhz7VXe8lI5AiUwMBAANPbfr7/+itu3b+t1npSUlKCgoECnES7Xrl1Dr1690K1bN2zatKnKPtFF69at0bx5c2RkZKinFRcX4+jRoxp/na6tzcOHD2ucL/Pnz1ePRHjzzTf1jo/qP3PXNIB1rSasa6xr1WFd065N1jXTs7a2Rrdu3TSOnUqlQkZGRo3PkK3O7z9/zE3XnNq2bVulng0aNAh/+9vfoFQq0bJlS1OGXyNDHKuKigqcPn1a51HexqRPXoGBgcjPz9f43D5//jxcXFxqfWyFKdTlWH311VcoLS3F22+/bewwdaZPXjX9Pw14Uk/rg7ocL1tbW7Ro0QLl5eX4+uuvERYWZvgADd6d+QK7f/++ZGVlqf+q9NFHH0lWVpb6lwdN4d1335XGjRvLvn375MaNG+rXw4cPTRbD08zxq6CRkZHSokUL+fbbb6WwsFC2b98uzZo1k1mzZhmtzdqO//bt26VBgwby6aefyoULFyQhIUEsLS3lxx9/NFgMMTExsn//fiksLJTs7GyJiYkRhUIhu3btkqKiIvHz85NOnTpJfn6+xvlhyJE7x44dEysrK1m4cKFcuHBBkpOT5aWXXpLPP/+8xnXq+qugz9r3jx8/lkGDBombm5solUqNvCv/Ynjr1i1p2rSpDBkyRJRKpeTl5cl7770nDRo0EKVSqVUMtV17+fn5Mn/+fDlx4oQUFhbKN998Ix4eHtKjRw+N7YSFhUmHDh3k4MGDcvr0aXnjjTekffv2Wv3K3YwZM2Tfvn1SWFgoBw8elD59+kizZs3k5s2bIiJy48YNycrKksTERPWvoGZlZcmdO3dE5MkvtXp5eUlwcLBcvXpVIw999r2ISHx8vLz88svyzTffSHZ2toSFhUnr1q3Vo5P0afN5+UXDF4G561p9rGkirGusa6xrIqxrrGvPn5SUFLGxsZHNmzfLmTNnZPz48fLyyy/Lzz//LCIiERERGqNqFy1aJLt27ZKCggI5c+aMLFu2TKysrCQxMdFcKVSha05Pq6+/dq1rXvPmzZPvv/9eCgoKJDMzU0aMGCG2traSm5trrhSqpWtely9fFnt7e5k0aZLk5eXJt99+K05OTvLhhx+aK4Uq9D0Hg4KCZPjw4aYOV2u65hUXFyf29vayZcsWuXjxouzatUs8PT1l2LBh5kqhWrrmdeTIEfn666+loKBAfvjhB+ndu7e0bt26xjsN6oKdjzqovOXj6VdkZKTJYqiufQCyadMmk8XwNHN8SSsuLpYpU6ZIq1atxNbWVjw8PGT27NlGHZ6uzfFPSkoSLy8vsbW1FR8fH/nPf/5j0BjGjBkj7u7uYm1tLX/6058kODhYdu3a9cz4AEhhYaFB4/jvf/8rHTt2FBsbG2nbtm2tt1bV9Uvas/Z9YWFhjXnv3btXvY3jx49LSEiIODo6ir29vfz1r3+VnTt3ah1Dbdfe5cuXpUePHuLo6Cg2Njbi5eUlM2fOrHJLXFFRkYwZM0ZefvllcXR0lDfffFMuX76sVQzDhw8XFxcXsba2lhYtWsjw4cM1bqGJi4t7ZoybNm2qMY+a1Hbeq1QqiY2NFWdnZ7GxsZHg4GDJy8tTr69Pm/ySZjrmrmv1saaJsK6xrrGuibCusa49nxISEqRVq1ZibW0t3bt3lyNHjqjn9ezZU+Pzbfbs2erPtyZNmoi/v7+kpKSYIepn0yWnp9XXzkcR3fKaOnWqellnZ2d5/fXX5eTJk2aIuna6Hq9Dhw6Jn5+f2NjYiIeHhyxcuNBkj3zRlq45nTt3TgCo63l9pUteZWVlMnfuXPH09BRbW1tp2bKlTJw40SiddHWlS1779u2Tdu3aiY2NjTRt2lQiIiLk2rVrRolLIVJPxogSERERERERERHRC4XPfCQiIiIiIiIiIiKjYOcjERERERERERERGQU7H4mIiIiIiIiIiMgo2PlIRERERERERERERsHORyIiIiIiIiIiIjIKdj4SERERERERERGRUbDzkYiIiIiIiIiIiIyCnY9EfwCXLl2CQqGAUqk0dyhERER1xrpGREQvsrlz56JLly7q9++88w4GDx5stniI6oqdj0RERERERERERGQU7Hwkes6VlZWZOwQiIiKDYV0jIqL67PHjx+YOgei5w85HIgPr1asXJk+ejFmzZsHR0RHNmzfH3LlztVpXoVBg7dq16N+/P+zs7ODh4YFt27ap51feZrZ161b07NkTtra2SE5Ohkqlwvz58+Hm5gYbGxt06dIFaWlpVbZ/7tw5BAQEwNbWFh07dsT+/fs15ufk5KB///5o1KgRnJ2dERERgdu3b6vnb9u2DZ06dYKdnR2aNm2KPn364MGDB/rtKCIiei6wrhER0R9Zr169MGnSJEydOhXNmjVDaGhorfVFpVJhyZIl8PLygo2NDVq1aoWFCxeq50dHR+OVV17BSy+9BA8PD8TGxvKPb/RCY+cjkRF89tlnaNiwIY4ePYolS5Zg/vz5SE9P12rd2NhY/P3vf8epU6cQHh6OESNG4OzZsxrLxMTEYMqUKTh79ixCQ0OxatUqLF++HMuWLUN2djZCQ0MxaNAgXLhwQWO9mTNnYsaMGcjKyoK/vz8GDhyIO3fuAADu3buH3r17o2vXrjhx4gTS0tLwyy+/YNiwYQCAGzduYOTIkRgzZgzOnj2Lffv2YciQIRARA+wxIiKqz1jXiIjoj+yzzz6DtbU1Dh48iPj4+GfWFwB4//33ER8fj9jYWJw5cwZffPEFnJ2d1fPt7e2xefNmnDlzBqtWrUJiYiJWrFhhjtSITEOIyKB69uwpQUFBGtNeffVViY6OrnVdADJhwgSNaX5+fvLuu++KiEhhYaEAkJUrV2os4+rqKgsXLqzS5sSJEzXWi4+PV88vKysTNzc3Wbx4sYiILFiwQEJCQjS2ceXKFQEgeXl5kpmZKQDk0qVLteZBREQvDtY1IiL6I+vZs6d07dpV/b62+lJcXCw2NjaSmJiodRtLly6Vbt26qd/HxcWJj4+P+n1kZKSEhYXpnQORuVmZqc+T6IXWuXNnjfcuLi64efOmVuv6+/tXef/0r3n6+vqq/11cXIzr168jMDBQY5nAwECcOnWqxm1bWVnB19dXPfrk1KlT2Lt3Lxo1alQlpoKCAoSEhCA4OBidOnVCaGgoQkJC8NZbb6FJkyZa5UVERM8v1jUiIvoj69atm/rftdWXe/fuobS0FMHBwTVub+vWrVi9ejUKCgpQUlKC8vJyODg4GCV2ovqAnY9ERtCgQQON9wqFAiqVymDbb9iwocG2VamkpAQDBw7E4sWLq8xzcXGBpaUl0tPTcejQIezatQsJCQmYPXs2jh49itatWxs8HiIiqj9Y14iI6I/s93Wqtvpy8eLFZ27r8OHDCA8Px7x58xAaGorGjRsjJSUFy5cvN3jcRPUFn/lIVM8cOXKkyvt27drVuLyDgwNcXV1x8OBBjekHDx5E+/bta9x2eXk5MjMz1dv+y1/+gtzcXPz5z3+Gl5eXxquy2CoUCgQGBmLevHnIysqCtbU1UlNT65QvERG92FjXiIjoRVJbffH29oadnR0yMjKqXf/QoUNwd3fH7Nmz4evrC29vb/z0008mzoLItDjykaie+eqrr+Dr64ugoCAkJyfj2LFjSEpKeuY6M2fORFxcHDw9PdGlSxds2rQJSqUSycnJGst98skn8Pb2Rrt27bBixQrcvXsXY8aMAQBERUUhMTERI0eOVP+iaX5+PlJSUrBhwwacOHECGRkZCAkJgZOTE44ePYpbt2498wskERER6xoREb1Iaqsvtra2iI6OxqxZs2BtbY3AwEDcunULubm5GDt2LLy9vXH58mWkpKTg1VdfxXfffcc/fNELj52PRPXMvHnzkJKSgokTJ8LFxQVbtmypMtLjaZMnT0ZRURFmzJiBmzdvon379tixYwe8vb01louPj0d8fDyUSiW8vLywY8cONGvWDADUo0yio6MREhKC0tJSuLu7o1+/frCwsICDgwN++OEHrFy5EsXFxXB3d8fy5cvRv39/o+0LIiJ6/rGuERHRi6S2+gIAsbGxsLKywv/93//h+vXrcHFxwYQJEwAAgwYNwrRp0zBp0iSUlpZiwIABiI2Nxdy5c82YFZFxKUREzB0EET2hUCiQmpqKwYMHmzsUIiKiOmNdIyIiIiI+85GIiIiIiIiIiIiMgp2PRCaSnJyMRo0aVfvq0KGDucMjIiLSCesaEREREWmDt10Tmcj9+/fxyy+/VDuvQYMGcHd3N3FERERE+mNdIyIiIiJtsPORiIiIiIiIiIiIjIK3XRMREREREREREZFRsPORiIiIiIiIiIiIjIKdj0RERERERERERGQU7HwkIiIiIiIiIiIio2DnIxERERERERERERkFOx+JiIiIiIiIiIjIKNj5SEREREREREREREbBzkciIiIiIiIiIiIyiv8HLDeAqlBTzTQAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "fig, ax = plt.subplots(1, 3, figsize=plt.figaspect(1/4))\n", - "\n", - "ax[0].plot(bench_probes, bench_recall)\n", - "ax[0].set_xscale('log')\n", - "ax[0].set_xticks(bench_probes, bench_probes)\n", - "ax[0].set_xlabel('n_probes')\n", - "ax[0].set_ylabel('recall')\n", - "ax[0].grid()\n", - "\n", - "ax[1].plot(bench_probes, bench_qps)\n", - "ax[1].set_xscale('log')\n", - "ax[1].set_xticks(bench_probes, bench_probes)\n", - "ax[1].set_xlabel('n_probes')\n", - "ax[1].set_ylabel('QPS')\n", - "ax[1].set_yscale('log')\n", - "ax[1].grid()\n", - "\n", - "ax[2].plot(bench_recall, bench_qps)\n", - "ax[2].set_xlabel('recall')\n", - "ax[2].set_ylabel('QPS')\n", - "ax[2].set_yscale('log')\n", - "ax[2].grid();" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Internal search types\n", - "Besides `n_probes`, `ivf_pq.SearchParams` contains a couple more parameters, which affect the internal workings of the algorithm.\n", - "\n", - "`internal_distance_dtype` controls the representation of the distance/similarity during the search.\n", - "By default, it's `np.float32`, but you can change it to `np.float16` when appropriate to save the memory bandwidth.\n", - "This can be a good idea when the dataset type is low precision anyway (e.g. `np.uint8`),\n", - "yet it may help with 32-bit float datasets too.\n", - "\n", - "`lut_dtype` is the Look-Up Table Data Type.\n", - "The specifics of the PQ algorithm is that it stores the data in the Product Quantizer (PQ) encoded format,\n", - "which needs to be decoded during the second-phase (in-cluster) search.\n", - "Thus, the algorithm constructs a lookup table for each cluster.\n", - "This is a costly operation, and the table itself can be rather large.\n", - "By default, the individual elements in the table are stored as 32-bit floats,\n", - "but you can change this to `np.float16` or `np.uint8` to reduce the table size.\n", - "\n", - "The exact size of the table is as follows:\n", - "\n", - "$ \\mathtt{lut\\_size} = \\mathtt{pq\\_dim} \\cdot \\mathtt{sizeof(lut\\_dtype) \\cdot 2^{\\mathtt{pq\\_bits}}} $\n", - "\n", - "Ideally, the lookup table should fit in the shared memory of a GPU's multiprocessor,\n", - "but it's not the case for wider datasets.\n", - "The logic of deciding whether this table should stay in the shared or the global memory of the GPU is somewhat complicated.\n", - "Yet, you can see the outcome when you gradually change `pq_dim` and observe a sudden drop in QPS after a certain threshold.\n", - "The shared-memory kernel version is typically 2-5x faster than the global-memory version.\n", - "\n", - "However `pq_dim` strongly affects the recall and requires the index to be re-build on change.\n", - "This is where `lut_dtype` comes in handy: you can halve or quarter the lookup table size by changing it.\n", - "Though it does affect the recall too.\n", - "\n", - "Also note, it does not make sense to set the `lut_dtype` to a more precise type than `internal_distance_dtype`,\n", - "as the former is converted to the latter internally.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "209 ms ± 151 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "178 ms ± 485 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "182 ms ± 297 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "176 ms ± 220 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "181 ms ± 439 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "bench_qps_s1 = np.zeros((5,), dtype=np.float32)\n", - "bench_recall_s1 = np.zeros((5,), dtype=np.float32)\n", - "k = 10\n", - "n_probes = 256\n", - "search_params_32_32 = ivf_pq.SearchParams(n_probes=n_probes, internal_distance_dtype=np.float32, lut_dtype=np.float32)\n", - "search_params_32_16 = ivf_pq.SearchParams(n_probes=n_probes, internal_distance_dtype=np.float32, lut_dtype=np.float16)\n", - "search_params_32_08 = ivf_pq.SearchParams(n_probes=n_probes, internal_distance_dtype=np.float32, lut_dtype=np.uint8)\n", - "search_params_16_16 = ivf_pq.SearchParams(n_probes=n_probes, internal_distance_dtype=np.float16, lut_dtype=np.float16)\n", - "search_params_16_08 = ivf_pq.SearchParams(n_probes=n_probes, internal_distance_dtype=np.float16, lut_dtype=np.uint8)\n", - "search_ps = [search_params_32_32, search_params_32_16, search_params_32_08, search_params_16_16, search_params_16_08]\n", - "bench_names = ['32/32', '32/16', '32/8', '16/16', '16/8']\n", - "\n", - "for i, sp in enumerate(search_ps):\n", - " r = %timeit -o ivf_pq.search(sp, index, queries, k, handle=resources); resources.sync()\n", - " bench_qps_s1[i] = (queries.shape[0] * r.loops / np.array(r.all_runs)).mean()\n", - " bench_recall_s1[i] = calc_recall(ivf_pq.search(sp, index, queries, k, handle=resources)[1], gt_neighbors)" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0UAAAHgCAYAAABqycbBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAACcU0lEQVR4nOzdd1wUx/8/8NfRexGpioBdsGNUrNgANbZoQGzYjYo11mgENPbejRVj+dh7QbE31IgaY29YoiIqCiJSb35/+Lv9ct7RFEW51/PxIHFn52Znd/Z2730zOycTQggQERERERFpKK38rgAREREREVF+YlBEREREREQajUERERERERFpNAZFRERERESk0RgUERERERGRRmNQREREREREGo1BERERERERaTQGRUREREREpNEYFBERERERkUZjUEQqEhIS0LNnT9jZ2UEmk2Hw4MEAgOfPn6Ndu3awsrKCTCbDnDlz8rWeee3OnTvw8vKCubk5ZDIZduzYkd9VyjPBwcGQyWR4+fJlfleFqEDz9PSEp6entPzgwQPIZDKEhobmW52y4uzsjK5du+bb9qdPn47ixYtDW1sblStXBgCkpaVhxIgRcHR0hJaWFlq3bp1lGXK5HOXLl8fEiRNzte2uXbvC2dlZKS2z+x99OsX9J6OcnnehoaGQyWR48ODBl6lcHvD09ET58uU/+fXHjh2DTCbDsWPHlNLXrFmDsmXLQldXFxYWFrkqMywsDCYmJnjx4sUn10sTMSjSEIoLS2Z/Z8+elfJOmjQJoaGh6Nu3L9asWYPOnTsDAIYMGYIDBw5g9OjRWLNmDXx8fPK8npMmTcq3YCQgIAD//vsvJk6ciDVr1qBatWr5Ug/69uzbtw/BwcH5XY189e7dO0yYMAEVK1aEkZERzM3NUbduXaxZswZCCJX8Ga8vWlpacHBwgJeXl8qNPyUlBXPnzkWVKlVgZmYGCwsLuLm5oXfv3rh586ZKuXK5HNbW1pg2bdqX2tV8cf36dQQHB3/TH/7y2sGDBzFixAjUrl0bq1atwqRJkwAAK1euxPTp09GuXTusXr0aQ4YMybKc//3vf3j8+DECAwM/u07q7n9nzpxBcHAw3rx5k+NyNm7ciE6dOqFUqVKQyWRKgXJGf//9NwIDA+Hm5gZjY2MUK1YMvr6+uH37ttr8mzZtQs2aNWFhYQErKyvUr18fe/fu/YQ9pW/ZzZs30bVrV5QoUQLLli3D0qVLAXwI5NV9hitbtqzS6318fFCyZElMnjw5P6r/3dLJ7wrQ1zV+/Hi4uLiopJcsWVL695EjR1CzZk0EBQUp5Tly5AhatWqFYcOGfbH6TZo0Ce3atcv2m8G89v79e0RERGDMmDF5cmOlgmXfvn1YuHChxgZGz58/R6NGjXDjxg20b98egYGBSEpKwtatW9GlSxeEhYVhzZo10NJS/p6tSZMm6NKlC4QQiIqKwqJFi9CwYUPs3bsXTZs2BQC0bdsW+/fvh7+/P3r16oXU1FTcvHkTe/bsQa1atVRu9ufPn8fLly/RvHnzr7b/X8P169cREhICT09Pld6LgurIkSPQ0tLCihUroKenp5RepEgRzJ49O0flTJ8+He3bt4e5uXmutr9s2TLI5XKVOn18/5sxYwZCQkLQtWvXHH9jv3jxYkRGRuKHH37Aq1evMs03depUnD59Gj///DMqVqyI6OhoLFiwAFWrVsXZs2eVeiDmz5+PgQMHonnz5pgyZQqSkpIQGhqKH3/8EVu3bsVPP/2Uq/3PT7du3VK5XtD/OXbsGORyOebOnav0+QwA9PX1sXz5cqU0ded+nz59MGzYMISEhMDU1PSL1regYFCkYZo2bZptD0hMTAxcXV3Vpue2C/d7oehi/p72Ly0tDXK5XOnDxPesoO1PdoQQSEpKgqGhYX5XJVsBAQG4ceMGtm/fjpYtW0rpAwcOxPDhwzFjxgxUrlwZw4cPV3pd6dKl0alTJ2m5TZs2qFixIubMmYOmTZvi77//xp49ezBx4kT89ttvSq9dsGCB2m/m9+3bBycnJ7i5uamt6/d0XD9VQdnHmJgYGBoaqrznc3OvuXTpEv755x/MnDkz19vX1dVVWyd197/cWrNmDYoUKQItLa0sh1YNHToU69evVzoGfn5+qFChAqZMmYK1a9dK6fPnz8cPP/yA3bt3S8PRunfvjiJFimD16tXfVVCkr6+f31X4psXExABQ/5lER0dH6bqambZt22LAgAHYvHkzunfvntdVLJAYppNEMa41KioKe/fulbplFUPvhBBYuHChlK7w5s0bDB48GI6OjtDX10fJkiUxdepUlW/gFN96VKhQAQYGBrC2toaPjw8uXLgA4MNwm3fv3mH16tXSNhRjjt++fYvBgwfD2dkZ+vr6sLGxQZMmTXDx4sVs9+vSpUto2rQpzMzMYGJigkaNGikNFwwODoaTkxMAYPjw4ZDJZNl+Uzt//ny4ubnByMgIlpaWqFatGtavX6+U58mTJ+jevTtsbW2hr68PNzc3rFy5UilPSkoKxo0bB3d3d5ibm8PY2Bh169bF0aNHlfIpnkuYMWMG5syZgxIlSkBfXx/Xr18H8KGr3dfXF9bW1jA0NESZMmUwZswYlXq/efNG+rbT3Nwc3bp1Q2JiYrbHUDFmOjIyErVq1YKhoSFcXFywZMmSPN+fTylj4cKFKF68OIyMjODl5YXHjx9DCIEJEyagaNGiMDQ0RKtWrRAbG6uyb/v370fdunVhbGwMU1NTNG/eHNeuXZPWd+3aFQsXLgSgPCRMQS6XY86cOXBzc4OBgQFsbW3Rp08fvH79Wmk7zs7O+PHHH3HgwAFUq1YNhoaG+PPPPwEA4eHhqFOnDiwsLGBiYoIyZcqoBAnqpKWlYcKECdLxc3Z2xm+//Ybk5GS12z516hSqV68OAwMDFC9eHH/99Ve22zh79iwOHDiArl27KgVECpMnT0apUqUwZcoUvH//PsuyKlSogMKFCyMqKgoAcO/ePQBA7dq1VfJqa2vDyspKJX3v3r1KvURZHde8ujYBwKpVq9CwYUPY2NhAX18frq6uWLx4cZb7m1OhoaH4+eefAQANGjSQzjHFUMOs9jGn9RJC4I8//kDRokVhZGSEBg0aKJ3nGeX0uGUmJ+elTCbDqlWr8O7dO5V7zdGjR3Ht2jWV46DOjh07oKenh3r16iml5+SekfGZoszuf127dpWCfRcXFyk9u2GOiuehslOrVi2VoLBUqVJwc3PDjRs3lNLj4+NhY2OjdP1R3NdyGiDn5L6U2XM8mT37cu7cOTRr1gyWlpYwNjZGxYoVMXfu3Czroe6ZomvXrqFhw4YwNDRE0aJF8ccff2R6zmV33QaAK1euoGvXrihevDgMDAxgZ2eH7t27q/TcKZ55unv37ifdHxWuX7+OBg0awMjICEWKFFE7xPe///5D69atYWxsDBsbGwwZMkTt9VrRU2ltbQ2ZTKYySiE9PR3x8fFZ1sfGxgYVK1bEzp07c7wPmo49RRomLi5O5WF7mUwGKysrlCtXDmvWrMGQIUNQtGhR/PrrrwCAKlWqSGOrFcNhFBITE1G/fn08efIEffr0QbFixXDmzBmMHj0az549U5qMoUePHggNDUXTpk3Rs2dPpKWl4eTJkzh79iyqVauGNWvWoGfPnqhevTp69+4NAChRogQA4JdffsGWLVsQGBgIV1dXvHr1CqdOncKNGzdQtWrVTPf32rVrqFu3LszMzDBixAjo6urizz//hKenJ44fP44aNWrgp59+goWFBYYMGQJ/f380a9YMJiYmmZa5bNkyDBw4EO3atcOgQYOQlJSEK1eu4Ny5c+jQoQOAD8ONatasCZlMhsDAQFhbW2P//v3o0aMH4uPjpYd34+PjsXz5cmno0Nu3b7FixQp4e3vj/Pnz0oPHCqtWrUJSUhJ69+4NfX19FCpUCFeuXEHdunWhq6uL3r17w9nZGffu3cPu3btVHjz29fWFi4sLJk+ejIsXL2L58uWwsbHB1KlTM91fhdevX6NZs2bw9fWFv78/Nm3ahL59+0JPT0/6Fiov9ie3Zaxbtw4pKSkYMGAAYmNjMW3aNPj6+qJhw4Y4duwYRo4cibt372L+/PkYNmyY0geANWvWICAgAN7e3pg6dSoSExOxePFi1KlTB5cuXYKzszP69OmDp0+fIjw8HGvWrFE5Ln369EFoaCi6deuGgQMHIioqCgsWLMClS5dw+vRppW+jb926BX9/f/Tp0we9evVCmTJlcO3aNfz444+oWLEixo8fD319fdy9exenT5/Otk169uyJ1atXo127dvj1119x7tw5TJ48WerVyeju3bto164devTogYCAAKxcuRJdu3aFu7t7pr0uALB7924AUHrfZ6Sjo4MOHTogJCQEZ86cQaNGjTIt6/Xr13j9+rU0HETxZcS6detQu3Zt6OhkfUuKjo7GpUuXMH78eKV0dcc1L69NwIfhUG5ubmjZsiV0dHSwe/du9OvXD3K5HP3798+y3tmpV68eBg4ciHnz5uG3335DuXLlAED6f2b7mJt6jRs3Dn/88QeaNWuGZs2a4eLFi/Dy8kJKSopSXXJz3DKTk/NyzZo1WLp0Kc6fPy8NBVLcayZOnIiEhATpeYiMx+FjZ86cQfny5VV6fXJ7z8js/lehQgWkpKTgf//7H2bPno3ChQsD+PBh9UsRQuD58+cq70tPT09s2bIF8+fPR4sWLZCUlIT58+cjLi4OgwYNyrbcnN6XciM8PBw//vgj7O3tMWjQINjZ2eHGjRvYs2dPjuqkEB0djQYNGiAtLQ2jRo2CsbExli5dqjbYy8l1W1G3+/fvo1u3brCzs8O1a9ewdOlSXLt2DWfPnlWZ/OFz748+Pj746aef4Ovriy1btmDkyJGoUKGCNFT4/fv3aNSoER49eoSBAwfCwcEBa9aswZEjR5TKmjNnDv766y9s374dixcvhomJCSpWrCitT0xMhJmZGRITE2FpaQl/f39MnTpV7ecWd3f3AjVp1BcnSCOsWrVKAFD7p6+vr5TXyclJNG/eXKUMAKJ///5KaRMmTBDGxsbi9u3bSumjRo0S2tra4tGjR0IIIY4cOSIAiIEDB6qUK5fLpX8bGxuLgIAAlTzm5uYq286J1q1bCz09PXHv3j0p7enTp8LU1FTUq1dPSouKihIAxPTp07Mts1WrVsLNzS3LPD169BD29vbi5cuXSunt27cX5ubmIjExUQghRFpamkhOTlbK8/r1a2Frayu6d++uUj8zMzMRExOjlL9evXrC1NRUPHz4UCk943ENCgoSAJTKFEKINm3aCCsrq2z2WIj69esLAGLmzJlSWnJysqhcubKwsbERKSkpebY/uS3D2tpavHnzRkofPXq0ACAqVaokUlNTpXR/f3+hp6cnkpKShBBCvH37VlhYWIhevXopbSs6OlqYm5srpffv31+ou1yePHlSABDr1q1TSg8LC1NJd3JyEgBEWFiYUt7Zs2cLAOLFixcq5Wfl8uXLAoDo2bOnUvqwYcMEAHHkyBGVbZ84cUJKi4mJEfr6+uLXX3/NcjutW7cWAMTr168zzbNt2zYBQMybN09KAyB69OghXrx4IWJiYsS5c+dEo0aNlM4juVwunVu2trbC399fLFy4UOVcVlixYoUwNDSU3j8Z9+3j45rX16aM21Tw9vYWxYsXV0qrX7++qF+/vrSsOE9XrVqldp8UNm/eLACIo0ePqqzLbB9zWq+YmBihp6cnmjdvrrRPv/32mwCgdM3N6XHLTG7Oy4CAAGFsbKxSRv369bO9xioULVpUtG3bViU9J/eMgIAA4eTkpJSm7v43ffp0AUBERUXlqE4fc3NzUzonsrNmzRoBQKxYsUIp/fnz59J7SPFXuHBhcebMmRyVm9P7kuLzwsf7e/ToUaVzNC0tTbi4uAgnJyeV64O6+09GTk5OSufd4MGDBQBx7tw5KS0mJkaYm5sr1SU31211743//e9/KtfCvLo//vXXX1JacnKysLOzUzo358yZIwCITZs2SWnv3r0TJUuWVHnvK+r08X1h1KhRYuTIkWLjxo3if//7nwgICBAARO3atZXudwqTJk0SAMTz58+z3Q8SgsPnNMzChQsRHh6u9Ld///5PLm/z5s2oW7cuLC0t8fLlS+mvcePGSE9Px4kTJwAAW7duhUwmU5m8AYDKtzXqWFhY4Ny5c3j69GmO65aeno6DBw+idevWKF68uJRub2+PDh064NSpU9l2P2dWl//++w9///232vVCCGzduhUtWrSAEELpuHh7eyMuLk4awqGtrS0NnZDL5YiNjUVaWhqqVaumdmhg27Ztlb6hfPHiBU6cOIHu3bujWLFiSnnVHddffvlFablu3bp49epVjo6Djo4O+vTpIy3r6emhT58+iImJQWRkZJ7sz6eU8fPPPys9ZFqjRg0AQKdOnZR6HmrUqIGUlBQ8efIEwIdvEd+8eQN/f3+lNtLW1kaNGjVUhuups3nzZpibm6NJkyZKZbi7u8PExESlDBcXF3h7eyulKcaM79y5M8fDk4APz9YAH55JyEjxDffHM1K5urqibt260rK1tTXKlCmD+/fvZ7mdt2/fAkCWD+oq1inyKqxYsQLW1tawsbFBjRo1cPr0aQwdOlT6Rlomk+HAgQP4448/YGlpif/973/o378/nJyc4Ofnp/JM0b59+9CgQQOVb4/VHde8vjZl3Kaix71+/fq4f/8+4uLiMj02eUXdPua0XocOHZJ6UzPuk7qegZwet8zk9rz8XK9evYKlpaVK+qfcM74FN2/eRP/+/eHh4YGAgACldUZGRihTpgwCAgKwefNmrFy5Evb29vjpp59w9+7dLMvNzX0ppy5duoSoqCgMHjxY5dmXnNzXM9q3bx9q1qyJ6tWrS2nW1tbo2LGjUr7cXLczvjeSkpLw8uVL1KxZEwDU7uvn3B9NTEyUnvPR09ND9erVla6v+/btg729Pdq1ayelGRkZSSNjcmLy5MmYMmUKfH190b59e4SGhmLixIk4ffo0tmzZopJf8d7gz3HkDIfPaZjq1avn6VTTd+7cwZUrVzIdSqB4WPDevXtwcHBAoUKFPmk706ZNQ0BAABwdHeHu7o5mzZqhS5cuSsHOx168eIHExERpmElG5cqVg1wux+PHj7McOqTOyJEjcejQIVSvXh0lS5aEl5cXOnToID0X8eLFC7x58wZLly6VptH8mOK4AMDq1asxc+ZM3Lx5E6mpqVK6ulkCP05TXHBz+hsJHwdOigvm69evYWZmluVrHRwcYGxsrJRWunRpAB+e71HcbD5nfxRyU8bH+6QIkBwdHdWmK571uXPnDgCgYcOGauuQ3fFQlBEXFwcbGxu16zO2M6C+/n5+fli+fDl69uyJUaNGoVGjRvjpp5/Qrl27LJ9JePjwIbS0tFRmJrKzs4OFhQUePnyolP7xcQI+tP/Hzz59LGPAk9nD74pg6OPj0KpVKwQGBkImk8HU1FSadjgjfX19jBkzBmPGjMGzZ89w/PhxzJ07F5s2bYKurq70oHlqairCw8PVTjGr7rjm9bXp9OnTCAoKQkREhMpzBnFxcbme+Sy3Mnuv5KReinOhVKlSSuutra1VAoqcHrcXL14gPT1dSjcxMYGJiUmuz8u8INRMCf8p94zcio2NVRp+aGho+FnnQXR0NJo3bw5zc3Ns2bIF2traSut//vlnaYikQqtWrVCqVCmMGTMGGzduRHp6uspv0xQqVAhv3rzJ1X0pJxTPBH7Ob/QoPHz4UPpCK6OP79+5uW7HxsYiJCQEGzZsUNk3dV9kfM79sWjRoiqBoKWlJa5cuSItP3z4ECVLllTJp+4zSm4MGTIEv//+Ow4dOoT27dsrrVO8N3IbpGoqBkX0WeRyOZo0aYIRI0aoXa/40Py5fH19UbduXWzfvh0HDx7E9OnTMXXqVGzbtk0ar/u1lCtXDrdu3cKePXsQFhaGrVu3YtGiRRg3bhxCQkKkb/s7deqk8k2fgmJ88Nq1a9G1a1e0bt0aw4cPh42NDbS1tTF58mTphpPR58429fFNVkHdh4pPkRf7k9syMtun7PZV0U5r1qyBnZ2dSr7snm9RlGFjY4N169apXf/xB0t1+2toaIgTJ07g6NGj2Lt3L8LCwrBx40Y0bNgQBw8ezHQ/FHJ6s/vUtnd1dcWOHTtw5coVlYfZFRQ3/o8/cBYtWhSNGzfOUf2AD7247du3R9u2beHm5oZNmzYhNDQUOjo6Us9us2bNVF6n7rjm5bXp3r17aNSoEcqWLYtZs2bB0dERenp62LdvH2bPnp2rHr5PpW4fv0S9cnrcfvjhB6UAJygoSOlh8K/1IczKykptYP817hk//fQTjh8/Li0HBAR88o/0xsXFoWnTpnjz5g1OnjwJBwcHpfX3799HWFiYSkBTqFAh1KlTR3oG8fHjxyoB9NGjR6Wp7XNyX8qs7TIGwfklN9dtX19fnDlzBsOHD0flypVhYmICuVwOHx8fte+Nz7k/ful7a1YMDQ1hZWWldiIhxXtD8SwcZY1BEX2WEiVKICEhIdsPPiVKlMCBAwcQGxub5TeyWd1I7e3t0a9fP/Tr1w8xMTGoWrUqJk6cmOkNztraGkZGRrh165bKups3b0JLS0ulNyGnjI2N4efnBz8/P6SkpOCnn37CxIkTMXr0aFhbW8PU1BTp6enZHpctW7agePHi2LZtm9K+qxvKo47iQ+jVq1c/aT9y4+nTp3j37p3SN/2KHxhUPNj6ufuTV2XkhGISDxsbm2zbKbPzskSJEjh06BBq1679WQGrlpYWGjVqhEaNGmHWrFmYNGkSxowZg6NHj2ZaNycnJ8jlcty5c0fpQfTnz5/jzZs30iQGn6tFixaYNGkS/vrrL7VBUXp6OtavXw9bW9tMg6bc0tXVRcWKFXHnzh28fPkSdnZ22Lt3L1xdXXP8Gz55eW3avXs3kpOTsWvXLqVvk3MyxDKnPiWIyGm9FOfCnTt3lALXFy9eqAQUOT1u69atU5ptUFHu1zovFcqWLSvNZvix3N4zMpNZ28ycOVPp+H0cyORUUlISWrRogdu3b+PQoUNqpwR//vw5APWBSWpqKtLS0gB86JELDw9XWl+pUiWYmZnl+L6k6CH5ePjqx718imvo1atXc/XlhzpOTk5SL1BGH9+/c3rdfv36NQ4fPoyQkBCMGzdOSle3ja/FyckJV69ehRBC6ZxS9xklN96+fYuXL1+q7d2NiopC4cKFv+jEIAUJnymiz+Lr64uIiAgcOHBAZd2bN2+kC3Xbtm0hhEBISIhKvozfpBgbG6tciNPT01W6um1sbODg4KAylWVG2tra8PLyws6dO5WmFn3+/DnWr1+POnXq5GiI1Mc+ns5TT08Prq6uEEIgNTUV2traaNu2LbZu3ao2WMk4tEHx7VLGY3Du3DlERETkqC7W1taoV68eVq5ciUePHimty+tvqNLS0qRpgIEP02//+eefsLa2hru7O4DP35+8KiMnvL29YWZmhkmTJikN0VPI2E6KQPDjc9PX1xfp6emYMGGCyuvT0tLU/s7Ox9R9u6eYYS+r81vRY/LxbGCzZs0CgDz7cdOaNWvCy8sLq1atwp49e1TWjxkzBrdv38aIESNy1LuW0Z07d1TOW+DDcY6IiIClpaV0M9+3b1+u9ikvr03qzsm4uDisWrUqx/XJTmbnWFZyWq/GjRtDV1cX8+fPV8qrbia5nB632rVro3HjxtKfIij6WuelgoeHB65evar0XvnUe0ZmMmsbd3d3pWPwKb9vlJ6eDj8/P0RERGDz5s3w8PBQm69kyZLQ0tLCxo0bldrwv//+w8mTJ1GlShUAgIGBgVKdGjduDEtLy1zdlxSBR8bnx9LT01V6qapWrQoXFxfMmTNH5djk9v7TrFkznD17FufPn1eq08e98Dm9bqt7bwDqz/ncePToEW7evPlJr23WrBmePn2q9OxPYmJipsMZP5aUlKTy3CYATJgwAUII+Pj4qKyLjIzM9JwiVewp0jD79+9X+4auVavWJ421Hj58OHbt2oUff/xRmt733bt3+Pfff7FlyxY8ePAAhQsXRoMGDdC5c2fMmzcPd+7ckbqvT548iQYNGiAwMBDAh5vMoUOHMGvWLDg4OMDFxQVlypRB0aJF0a5dO1SqVAkmJiY4dOgQ/v7772x/sO+PP/6QfgOmX79+0NHRwZ9//onk5GS1vyGQE15eXrCzs0Pt2rVha2uLGzduYMGCBWjevLn0/MWUKVNw9OhR1KhRA7169YKrqytiY2Nx8eJFHDp0SPog/OOPP2Lbtm1o06YNmjdvjqioKCxZsgSurq5ISEjIUX3mzZuHOnXqoGrVqujduzdcXFzw4MED7N27F5cvX/6kfVTHwcEBU6dOxYMHD1C6dGls3LgRly9fxtKlS6XpcPNif/KijJwwMzPD4sWL0blzZ1StWhXt27eHtbU1Hj16hL1796J27dpYsGABAEhB38CBA+Ht7Q1tbW20b98e9evXR58+fTB58mRcvnwZXl5e0NXVxZ07d7B582bMnTtX6aFadcaPH48TJ06gefPmcHJyQkxMDBYtWoSiRYuiTp06mb6uUqVKCAgIwNKlS/HmzRvUr18f58+fx+rVq9G6dWs0aNAgz47VX3/9hYYNG6JVq1bo0KED6tati+TkZGzbtg3Hjh1Dp06dMGTIkFyX+88//6BDhw5o2rQp6tati0KFCuHJkydYvXo1nj59ijlz5kBbWxtRUVG4ceNGrn4XKC+vTV5eXtDT00OLFi3Qp08fJCQkYNmyZbCxscGzZ89yvd/qVK5cGdra2pg6dSri4uKgr68v/f5QZnJaL2trawwbNgyTJ0/Gjz/+iGbNmuHSpUvYv3+/yrCanB63zHzN8xL48EzNhAkTcPz4cXh5eQH48M35p94z1FG8/8eMGYP27dtDV1cXLVq0UHk+LqMTJ05IQcWLFy/w7t07/PHHHwA+TMGu6FX99ddfsWvXLrRo0QKxsbFKP9YKQHp439raGt27d8fy5cul5w7fvn2LRYsW4f379xg9enS2+5HT+5Kbmxtq1qyJ0aNHSz2oGzZskAJiBS0tLSxevBgtWrRA5cqV0a1bN9jb2+PmzZu4du2a2sA6MyNGjMCaNWvg4+ODQYMGSVNyOzk5KT2Xk9PrtpmZGerVq4dp06YhNTUVRYoUwcGDBzPtVcypLl264Pjx45/0pWOvXr2wYMECdOnSBZGRkbC3t8eaNWtgZGSUo9dHR0ejSpUq8Pf3l4ZDHjhwAPv27YOPjw9atWqllD8mJgZXrlz57J8M0ChfZY47yndZTcmNj6aLzc2U3EJ8mCJz9OjRomTJkkJPT08ULlxY1KpVS8yYMUOaqlmID9N3Tp8+XZQtW1bo6ekJa2tr0bRpUxEZGSnluXnzpqhXr54wNDSUpopNTk4Ww4cPF5UqVRKmpqbC2NhYVKpUSSxatChH+37x4kXh7e0tTExMhJGRkWjQoIHKFKa5mZL7zz//FPXq1RNWVlZCX19flChRQgwfPlzExcUp5Xv+/Lno37+/cHR0FLq6usLOzk40atRILF26VMojl8vFpEmThJOTk9DX1xdVqlQRe/bsUZkqNrv6Xb16VbRp00ZYWFgIAwMDUaZMGfH7779L6zOb3jOzqVc/ppgi98KFC8LDw0MYGBgIJycnsWDBAqV8ebE/n1uGYtrYzZs3q93Xv//+WyW/t7e3MDc3FwYGBqJEiRKia9eu4sKFC1KetLQ0MWDAAGFtbS1kMpnK9LJLly4V7u7uwtDQUJiamooKFSqIESNGiKdPn0p5MntfHT58WLRq1Uo4ODgIPT094eDgIPz9/VWmRFYnNTVVhISECBcXF6GrqyscHR3F6NGjpWnHs9v2x9NHZ+Xt27ciJCREuLm5CQMDA+nakfE8yyiz60VGz58/F1OmTBH169cX9vb2QkdHR1haWoqGDRuKLVu2SPkWLFggzM3N1U45m9m+KeqcV9emXbt2iYoVKwoDAwPh7Owspk6dKlauXKny/vnUKbmFEGLZsmWiePHiQltbW2mK3qz2Maf1Sk9PFyEhIcLe3l4YGhoKT09PcfXqVZWpkXNz3DKT0/MyL6bkFkKIihUrih49ekjLOb1n5HRKbiE+TFVepEgRoaWllaNrpuKaq+4vKChIaV+zujdnlJqaKubPny8qV64sTExMhImJiWjQoIHSNOfZycl9SQgh7t27Jxo3biz09fWFra2t+O2330R4eLjaaeNPnTolmjRpIh3rihUrivnz56sci4zUnXdXrlwR9evXFwYGBqJIkSJiwoQJYsWKFZlOD57ddfu///6T7ovm5ubi559/Fk+fPlVpg9zcHxXtlVFm56u68+vhw4eiZcuWwsjISBQuXFgMGjRI+gmH7Kbkfv36tejUqZMoWbKkMDIyEvr6+sLNzU1MmjRJ7fty8eLFwsjISMTHx6usI/VkQnyFp8CI6Lvm6emJly9ffpVnl+j78OTJE9SqVQtpaWmIiIhQO7tdXlH8oPKmTZu+2Dbo+7VmzRr0798fjx49ynSGRCJNU6VKFXh6emL27Nn5XZXvBp8pIiKiXCtSpAjCwsKQlJSEpk2bZju19+fw9PT8pOF5pBk6duyIYsWKYeHChfldFaJvQlhYGO7cuZOjYZX0f9hTRETZYk8RERERFWTsKSIiIiIiIo3GniIiIiIiItJo7CkiIiIiIiKNxqCIiIiIiIg0GoMiIsoXwcHBkMlkePnyZX5XhT5B165dYWJikt/VoG/AgwcPIJPJEBoaKqUp3t9ERN8LBkVEpPEmTpyIli1bwtbWFjKZDMHBwZnmffLkCXx9fWFhYQEzMzO0atUK9+/f/3qVpS9q27Zt8PPzQ/HixWFkZIQyZcrg119/xZs3b1TyOjs7QyaTqfz98ssvass+dOgQGjZsCHNzc5iamsLd3R0bN278wntEuXXv3j106NABNjY2MDQ0RKlSpTBmzJhM86empsLV1RUymQwzZsz4ijUlorykk98VICLKb2PHjoWdnR2qVKmCAwcOZJovISEBDRo0QFxcHH777Tfo6upi9uzZqF+/Pi5fvgwrK6uvWGv6Enr37g0HBwd06tQJxYoVw7///osFCxZg3759uHjxIgwNDZXyV65cGb/++qtSWunSpVXKXbVqFXr06IEmTZpg0qRJ0NbWxq1bt/D48eMvuj/5ZezYsRg1alR+VyPXLl++DE9PTxQpUgS//vorrKys8OjRoyzbaf78+Xj06NFXrCURfQkMiohI40VFRcHZ2RkvX76EtbV1pvkWLVqEO3fu4Pz58/jhhx8AAE2bNkX58uUxc+ZMTJo06WtVOc8lJSVBT08PWlqaPYBgy5Yt8PT0VEpzd3dHQEAA1q1bh549eyqtK1KkCDp16pRlmQ8ePED//v0xYMAAzJ07N6+r/E3S0dGBjs739RFDLpejc+fOKFu2LI4ePaoSAKsTExOD8ePHY+TIkRg3btxXqCURfSmaffcjom/Kw4cPUbJkSZQvXx7Pnz//att1dnbOUb4tW7bghx9+kAIiAChbtiwaNWqETZs2fdK2Fc/mPHnyBK1bt4aJiQmsra0xbNgwpKen56osT09PlC9fHpGRkahVqxYMDQ3h4uKCJUuWKOU7duwYZDIZNmzYgLFjx6JIkSIwMjJCfHw8AGDz5s1wd3eHoaEhChcujE6dOuHJkydqt3n//n14e3vD2NgYDg4OGD9+PD7+pQe5XI45c+bAzc0NBgYGsLW1RZ8+ffD69WulfBcuXIC3tzcKFy4s1b179+65Ogaf6+OACADatGkDALhx44ba16SkpODdu3eZlrlkyRKkp6dj/PjxAD70OH7ur2Eontm5efMmfH19YWZmBisrKwwaNAhJSUlKeZOTkzFkyBBYW1vD1NQULVu2xH///ZftUFF13rx5g65du8Lc3BwWFhYICAhQO7RQ3TNFMpkMgYGB2Lx5M1xdXWFoaAgPDw/8+++/AIA///wTJUuWhIGBATw9PfHgwYNc1e1zHTx4EFevXkVQUBAMDQ2RmJiY7Xtw1KhRKFOmTLaBMRF9+76vr3GIqMC6d+8eGjZsiEKFCiE8PByFCxfONG9qairi4uJyVG6hQoXypPdDLpfjypUraj+kV69eHQcPHsTbt29hamqa67LT09Ph7e2NGjVqYMaMGTh06BBmzpyJEiVKoG/fvrkq6/Xr12jWrBl8fX3h7++PTZs2oW/fvtDT01Op+4QJE6Cnp4dhw4YhOTkZenp6CA0NRbdu3fDDDz9g8uTJeP78OebOnYvTp0/j0qVLsLCwUKq3j48PatasiWnTpiEsLAxBQUFIS0uTAgAA6NOnj1TuwIEDERUVhQULFuDSpUs4ffo0dHV1ERMTAy8vL1hbW2PUqFGwsLDAgwcPsG3btmz3OSEhQSUQUEdXVxfm5uY5P5j/X3R0NACoPSePHDkCIyMjpKenw8nJCUOGDMGgQYOU8hw6dAhly5bFvn37MHz4cDx58gSWlpbo378/QkJCPuv89PX1hbOzMyZPnoyzZ89i3rx5eP36Nf766y8pT8+ePbF27Vp06NABtWrVwpEjR9C8efNcb0sIgVatWuHUqVP45ZdfUK5cOWzfvh0BAQE5LuPkyZPYtWsX+vfvDwCYPHkyfvzxR4wYMQKLFi1Cv3798Pr1a0ybNg3du3fHkSNHsiwvL68Fhw4dAgDo6+ujWrVqiIyMhJ6eHtq0aYNFixahUKFCSvnPnz+P1atX49SpU5xUgqggEERE+SAoKEgAEC9evBA3btwQDg4O4ocffhCxsbHZvvbo0aMCQI7+oqKiclynFy9eCAAiKCgo03Xjx49XWbdw4UIBQNy8eTPH21IICAhQW26VKlWEu7t7rsqqX7++ACBmzpwppSUnJ4vKlSsLGxsbkZKSIoT4v+NXvHhxkZiYKOVNSUkRNjY2onz58uL9+/dS+p49ewQAMW7cOJV6DxgwQEqTy+WiefPmQk9PT7x48UIIIcTJkycFALFu3TqluoaFhSmlb9++XQAQf//9d672OWNdsvurX79+rssWQogePXoIbW1tcfv2baX0Fi1aiKlTp4odO3aIFStWiLp16woAYsSIEUr5zMzMhKWlpdDX1xe///672LJli+jQoYMAIEaNGvVJdVK8f1q2bKmU3q9fPwFA/PPPP0IIIS5fviwAiH79+inlU2xf3bmemR07dggAYtq0aVJaWlqatN+rVq1SqV9GAIS+vr7Se/LPP/8UAISdnZ2Ij4+X0kePHp2j929eXgtatmwpAAgrKyvRsWNHsWXLFvH7778LHR0dUatWLSGXy6W8crlcVK9eXfj7+wshhIiKihIAxPTp07PcBhF9u9hTRET56urVq/Dz80PJkiWxf/9+mJmZZfuaSpUqITw8PEfl29nZfW4VAQDv378H8OFb5I8ZGBgo5fkUH89YVrduXaxZsybX5ejo6KBPnz7Ssp6eHvr06YO+ffsiMjISNWvWlNYFBAQoPTdx4cIFxMTEIDg4WNonAGjevDnKli2LvXv3IiQkRGl7gYGB0r8Vw6P27t2LQ4cOoX379ti8eTPMzc3RpEkTpenX3d3dYWJigqNHj6JDhw5SD9SePXtQqVIl6Orq5nifR4wYkaPhS5aWljkuU2H9+vVYsWIFRowYgVKlSimt27Vrl9Jyt27d0LRpU8yaNQsDBgxA0aJFAXzoyZLL5ZgyZQpGjhwJAGjbti1iY2Mxd+5c/Pbbb5/UwwhA6nFRGDBgABYtWoR9+/ahYsWK2LdvHwBg4MCBSvkGDx6M9evX52pb+/btg46OjlLvpba2NgYMGICTJ0/mqIxGjRopDVetUaMGgA/HI+MxUKTfv38/y+GteXktSEhIAAD88MMPWLt2rVQvIyMjjB49GocPH0bjxo0BAKGhofj333+xZcuWHG2biL59DIqIKF+1aNECtra2OHDgQI5/98bS0lL6cPK1KIKH5ORklXWKoVs5eTBbHQMDA5UJHiwtLVWeuckJBwcHGBsbK6UpZkN78OCBUlDk4uKilO/hw4cAgDJlyqiUW7ZsWZw6dUopTUtLC8WLF890WwBw584dxMXFwcbGRm19Y2JiAAD169dH27ZtERISgtmzZ8PT0xOtW7dGhw4d1AaiGbm6usLV1TXLPJ/i5MmT6NGjB7y9vTFx4sRs88tkMgwZMgQHDhzAsWPHpEDN0NAQ7969g7+/v1J+f39/hIWF4dKlS6hXr94n1fHjQK1EiRLQ0tKSjv/Dhw+hpaWFEiVKKOVT18bZefjwIezt7VXep7kpq1ixYkrLiuGMjo6OatOzew/k5bVA8f79uJ06dOiA0aNH48yZM2jcuDHi4+MxevRoDB8+XKXeRPT9YlBERPmqbdu2WL16NdatW6fUw5GVlJQUxMbG5iivtbU1tLW1P6eKAD48j6Cvr49nz56prFOkOTg4fFLZeVG/T/GpQVxuyOVy2NjYYN26dWrXK4JBmUyGLVu24OzZs9i9ezcOHDiA7t27Y+bMmTh79myWAXNcXFyOeun09PRUngvJzD///IOWLVuifPny2LJlS45nUlN8SM54fjo4OODOnTuwtbVVyqsIFD8l+M3Mt/5sS2bnembpIpsJKfLyWqB4/2bXTjNmzEBKSgr8/Pyk4PO///6T8jx48AAODg7Q09PLUb2I6NvAoIiI8tX06dOho6ODfv36wdTUFB06dMj2NWfOnEGDBg1yVL5iuu3PpaWlhQoVKuDChQsq686dO4fixYt/8hCovPT06VO8e/dOqbfo9u3bALKfZc/JyQkAcOvWLTRs2FBp3a1bt6T1CnK5HPfv31f6XZ6Pt1WiRAkcOnQItWvXzlEQVrNmTdSsWRMTJ07E+vXr0bFjR2zYsEFlKuyMBg0ahNWrV2dbdv369XHs2LFs8927dw8+Pj6wsbHBvn37ctyDCUD6Id+MPX/u7u64c+cOnjx5otSz9vTpU5W8uXXnzh2lHr+7d+9CLpdLx9/JyQlyuRz37t1T6tG5detWrrfl5OSEw4cPIyEhQemYfEpZeSUvrwXu7u5YtmyZykyLH7fTo0eP8Pr1a7i5uamUMWnSJEyaNAmXLl1C5cqVc7YTRPRNYFBERPlKJpNh6dKlePv2LQICAmBiYoKWLVtm+Zr8eKYIANq1a4dRo0bhwoULqFatGoAPHwiPHDmCYcOG5dl2PkdaWhr+/PNPDB06FMCHb9L//PNPWFtbw93dPcvXVqtWDTY2NliyZAm6d+8uDVvbv38/bty4ofZ3WBYsWIB58+YB+PCt/oIFC6Crq4tGjRoB+DA72qJFizBhwgSV33FKS0tDQkICLCws8Pr1a1hYWCj1dCg+VKobsphRXj5TFB0dDS8vL2hpaeHAgQOZBiyxsbEwNzdX6nlITU3FlClToKenp/RB3c/PDxs2bMCKFSukYXhyuRyrVq1CoUKFsm2XrCxcuBBeXl7S8vz58wF8+P0sxf9/++03zJs3DwsXLpTyzZkzJ9fbatasGZYuXYrFixdj+PDhAD7MQKjYZn7Iy2tBq1atMGjQIKxatQpdu3aVZqpbvnw5AKBJkyYAPjyf1bp1a6XXxsTEoE+fPujatStatWqlMjSViL59DIqIKN9paWlh7dq1aN26NXx9fbFv3z6VnoqM8vqZojVr1uDhw4dITEwEAJw4cQJ//PEHAKBz585SD0m/fv2wbNkyNG/eHMOGDYOuri5mzZoFW1tb/Prrr0plenp64vjx45/9ezS55eDggKlTp+LBgwcoXbo0Nm7ciMuXL2Pp0qXZTl6gq6uLqVOnolu3bqhfvz78/f2lKbmdnZ0xZMgQpfwGBgYICwtDQEAAatSogf3792Pv3r347bffpGCifv366NOnDyZPnozLly/Dy8sLurq6uHPnDjZv3oy5c+eiXbt2WL16NRYtWoQ2bdqgRIkSePv2LZYtWwYzMzM0a9Ysy3rn5TNFPj4+uH//PkaMGIFTp04pPUdla2srfTDetWsX/vjjD7Rr1w4uLi6IjY3F+vXrcfXqVUyaNEnpA3irVq3QqFEjTJ48GS9fvkSlSpWwY8cOnDp1Cn/++afSM1Ndu3bF6tWrc9zDGRUVhZYtW8LHxwcRERHS1NuVKlUC8CGw9Pf3x6JFixAXF4datWrh8OHDuHv3bq6PTYsWLVC7dm2MGjUKDx48gKurK7Zt25bjKbG/hLy8FtjZ2WHMmDEYN24cfHx80Lp1a/zzzz9YtmwZ/P39pd8nq1q1KqpWrar0WsUwOjc3N5WAiYi+E/k8+x0RaaiMU3IrJCYmivr16wsTExNx9uzZr1YXxVTW6v6OHj2qlPfx48eiXbt2wszMTJiYmIgff/xR3LlzR6VMd3d3YWdnl+22AwIChLGxsUq6uimNc7Ifbm5u4sKFC8LDw0MYGBgIJycnsWDBAqV8immMN2/erLacjRs3iipVqgh9fX1RqFAh0bFjR/Hff/+prfe9e/eEl5eXMDIyEra2tiIoKEikp6erlLl06VLh7u4uDA0NhampqahQoYIYMWKEePr0qRBCiIsXLwp/f39RrFgxoa+vL2xsbMSPP/4oLly4kKtj8LkyOw/w0ZTeFy5cEC1atBBFihQRenp6wsTERNSpU0ds2rRJbblv374VgwYNEnZ2dkJPT09UqFBBrF27ViVf27ZthaGhoXj9+nWW9VScH9evXxft2rUTpqamwtLSUgQGBipNpy6EEO/fvxcDBw4UVlZWwtjYWLRo0UI8fvw411NyCyHEq1evROfOnYWZmZkwNzcXnTt3FpcuXcrxlNz9+/dXSstsKuvsztEvRS6Xi/nz54vSpUsLXV1d4ejoKMaOHStNZ58ZTslN9P2TCfGVv8YkIirg3r59i0KFCmHOnDkqUyZ/SZ6ennj58iWuXr361bZJecvW1hZdunTB9OnTs8wXHByMkJAQvHjxIssfOs6KTCZDUFAQgoODP+n1REQFyef/zDsRESk5ceIEihQpgl69euV3Veg7cu3aNbx//176LSMiIvp6+EwREVEea968OZo3b55n5cXGxiIlJSXT9dra2p81gxl9G9zc3BAfH//Vt5ueno4XL15kmcfExCRXs/AREX1vGBQREX3jfvrpJxw/fjzT9U5OTtKD3kS59fjx42xnS+MwOyIq6PhMERHRNy4yMjLLH/g0NDRE7dq1v2KNqCBJSkpSmmVPneLFiyv9xhIRUUHDoIiIiIiIiDQaJ1ogIiIiIiKNxqCIiIiIiIg0GoMiIiIiIiLSaAyKiIiIiIhIozEoIiIiIiIijcbfKcojcrkcT58+hampKWQyWX5Xh4iIiIhIowkh8PbtWzg4OEBLK+u+IAZFeeTp06dwdHTM72oQEREREVEGjx8/RtGiRbPMw6Aoj5iamgL4cNDNzMzyuTYfpKam4uDBg/Dy8oKurm5+V4c+Edvx+8c2LBjYjgUD27FgYDsWDF+6HePj4+Ho6Ch9Ts8Kg6I8ohgyZ2Zm9k0FRUZGRjAzM+MF4zvGdvz+sQ0LBrZjwcB2LBjYjgXD12rHnDzawokWiIiIiIhIozEoos+WlJSErl27okKFCtDR0UHr1q3V5ktOTsaYMWPg5OQEfX19ODs7Y+XKlSr5QkJC0KlTJwDA0qVL4enpCTMzM8hkMrx580Zt2Xv37kWNGjVgaGgIS0vLTOtARERERPQxDp+jz5aeng5DQ0MMHDgQW7duzTSfr68vnj9/jhUrVqBkyZJ49uwZ5HK5Sr6dO3di1KhRAIDExET4+PjAx8cHo0ePVlvu1q1b0atXL0yaNAkNGzZEWloarl69mjc7R0REREQFHoMi+mzGxsZYvHgxAOD06dNqe3PCwsJw/Phx3L9/H4UKFQIAODs7q+R7/Pgxrl27Bh8fHwDA4MGDAQDHjh1Tu+20tDQMGjQI06dPR48ePaR0V1fXT98hIiIiItIoHD5HX8WuXbtQrVo1TJs2DUWKFEHp0qUxbNgwvH//XiWfYrhcTly8eBFPnjyBlpYWqlSpAnt7ezRt2pQ9RURERESUY+wpoq/i/v37OHXqFAwMDLB9+3a8fPkS/fr1w6tXr7Bq1Sop386dO9GqVatclQsAwcHBmDVrFpydnTFz5kx4enri9u3bUq8UEREREVFm2FNEX4VcLodMJsO6detQvXp1NGvWDLNmzcLq1aul3qL4+HgcP34cLVu2zFW5ADBmzBi0bdsW7u7uWLVqFWQyGTZv3vxF9oWIiIiIChYGRfRV2Nvbo0iRIjA3N5fSypUrByEE/vvvPwDA/v374erqCkdHx1yVCyg/Q6Svr4/ixYvj0aNHeVR7IiIiIirIGBTRV1G7dm08ffoUCQkJUtrt27ehpaWFokWLAsj90DkAcHd3h76+Pm7duiWlpaam4sGDB3BycsqbyhMRERFRgcagiLKVLheIuPcKOy8/QcS9V0iXC5U8169fx+XLlxEbG4u4uDhcvnwZly9fltZ36NABVlZW6NatG65fv44TJ05g+PDh6N69OwwNDZGWlob9+/erDJ2Ljo7G5cuXcffuXQDAv//+K20HAMzMzPDLL78gKCgIBw8exK1bt9C3b18AwM8///yFjggRERERFSScaIGyFHb1GUJ2X8ezuCQpzd7cAEEtXOFT3l5Ka9asGR4+fCgtV6lSBQAgxIcAysTEBOHh4RgwYACqVasGKysr+Pr64o8//gAAHD9+HCYmJqhatarS9pcsWYKQkBBpuV69egCAVatWoWvXrgCA6dOnQ0dHB507d8b79+9Ro0YNHDlyBJaWlnl4JIiIiIiooGJQRJkKu/oMfddexMf9QtFxSei79iIWd6oqBUYPHjzItryyZcsiPDxc7bqdO3eiRYsWKunBwcEIDg7OslxdXV3MmDEDM2bMyLYOREREREQfY1BEaqXLBUJ2X1cJiABAAJABCNl9HU1c7aCtJfvs7ZUvXx4eHh6fXQ4RERERUW4xKCK1zkfFKg2Z+5gA8CwuCeejYuFRwuqzt9e7d+/PLoOIiIiI6FNwogVSK+Zt5gHRp+QjIiIiIvpWMSgitWxMDfI0HxERERHRt4pBEalV3aUQ7M0NkNnTQjJ8mIWuukuhr1ktIiIiIqI8x6CI1NLWkiGohSsAqARGiuWgFq55MskCEREREVF+YlBEmfIpb4/FnarCzlx5iJyduYHSdNxERERERN8zzj5HWfIpb48mrnY4HxWLmLdJsDH9MGSOPUREREREVFAwKKJsaWvJ8mTabSIiIiKibxGHzxERERERkUZjUERERERERBqNQREREREREWk0BkVERERERKTRGBQREREREZFGY1BERERERPQNu3XrFho0aABbW1sYGBigePHiGDt2LFJTU6U8y5YtQ926dWFpaQlLS0s0btwY58+fV1tegwYNsHz5cgDAwIED4e7uDn19fVSuXFltfiEEZsyYgdKlS0NfXx9FihTBxIkT83w/8xOn5CYiIiIi+obp6uqiS5cuqFq1KiwsLPDPP/+gV69ekMvlmDRpEgDg2LFj8Pf3R61atWBgYICpU6fCy8sL165dQ5EiRaSyYmNjcfr0aWzYsEFK6969O86dO4crV66o3f6gQYNw8OBBzJgxAxUqVEBsbCxiY2O/7E5/ZQyKiIiIiIi+YcWLF0fx4sWlZScnJxw7dgwnT56U0tatW6f0muXLl2Pr1q04fPgwunTpIqXv3bsXVatWha2tLQBg3rx5AIAXL16oDYpu3LiBxYsX4+rVqyhTpgwAwMXFJe927hvB4XNERERERN+Ru3fvIiwsDPXr1880T2JiIlJTU1GoUCGl9F27dqFVq1Y53tbu3btRvHhx7NmzBy4uLnB2dkbPnj0LXE8RgyIiIiIiou+AYmhcqVKlULduXYwfPz7TvCNHjoSDgwMaN24spSUnJyMsLAwtW7bM8Tbv37+Phw8fYvPmzfjrr78QGhqKyMhItGvX7rP25VvDoIiIiIiI6DuwceNGXLx4EevXr8fevXsxY8YMtfmmTJmCDRs2YPv27TAwMJDSjxw5AhsbG7i5ueV4m3K5HMnJyfjrr79Qt25deHp6YsWKFTh69Chu3br12fv0reAzRURERERE3wFHR0cAgKurK9LT09G7d2/8+uuv0NbWlvLMmDEDU6ZMwaFDh1CxYkWl1+/atStXvUQAYG9vDx0dHZQuXVpKK1euHADg0aNH0nNG3zv2FBERERERfWfkcjlSU1Mhl8ultGnTpmHChAkICwtDtWrVlPILIbB79+5cPU8EALVr10ZaWhru3bsnpd2+fRvAhwkfCgr2FBERERERfcPWrVsHXV1dVKhQAfr6+rhw4QJGjx4NPz8/6OrqAgCmTp2KcePGYf369XB2dkZ0dDQAwMTEBCYmJoiMjERiYiLq1KmjVPbdu3eRkJCA6OhovH//HpcvXwbwoTdKT08PjRs3RtWqVdG9e3fMmTMHcrkc/fv3R5MmTZR6jxTS5QLno2IR8zYJNqYGqO5SCNpasi97gPIAgyIiIiIiom+Yjo4Opk6ditu3b0MIAScnJwQGBmLIkCFSnsWLFyMlJUVlAoSgoCAEBwdj586daNasGXR0lD/+9+zZE8ePH5eWq1SpAgCIioqCs7MztLS0sHv3bgwYMAD16tWDsbExmjZtipkzZ6rUM+zqM4Tsvo5ncUlSmr25AYJauMKnvH2eHIsvhUEREREREdE3zM/PD35+flnmefDgQZbrd+7cibFjx6qkHzt2LNvtOzg4YOvWrVnmCbv6DH3XXoT4KD06Lgl9117E4k5Vv+nAiM8UEREREREVYCkpKWjbti2aNm36RcpPlwuE7L6uEhABkNJCdl9Hulxdjm9DvgZFwcHBkMlkSn9ly5ZVyhMREYGGDRvC2NgYZmZmqFevHt6/fy+tj42NRceOHWFmZgYLCwv06NEDCQkJSmVcuXIFdevWhYGBARwdHTFt2jSVumzevBlly5aFgYEBKlSogH379n2ZnSYiIiIi+or09PQQFBQEU1PTL1L++ahYpSFzHxMAnsUl4XzUt/uDr/neU+Tm5oZnz55Jf6dOnZLWRUREwMfHB15eXjh//jz+/vtvBAYGQkvr/6rdsWNHXLt2DeHh4dizZw9OnDiB3r17S+vj4+Ph5eUFJycnREZGYvr06QgODsbSpUulPGfOnIG/vz969OiBS5cuoXXr1mjdujWuXr36dQ4CEREREdF3KuZt5gHRp+TLD/n+TJGOjg7s7OzUrhsyZAgGDhyIUaNGSWkZ50K/ceMGwsLC8Pfff0vTDs6fPx/NmjXDjBkz4ODggHXr1iElJQUrV66Enp4e3NzccPnyZcyaNUsKnubOnQsfHx8MHz4cADBhwgSEh4djwYIFWLJkidq6JScnIzk5WVqOj48HAKSmpiI1NfUzjkjeUdTjW6kPfRq24/ePbVgwsB0LBrZjwcB2/LZYGeUspLAy0lFqsy/djrkpN9+Dojt37sDBwQEGBgbw8PDA5MmTUaxYMcTExODcuXPo2LEjatWqhXv37qFs2bKYOHGiNJVgREQELCwslOZhb9y4MbS0tHDu3Dm0adMGERERqFevHvT09KQ83t7emDp1Kl6/fg1LS0tERERg6NChSvXy9vbGjh07Mq335MmTERISopJ+8OBBGBkZfeZRyVvh4eH5XQXKA2zH7x/bsGBgOxYMbMeCge34bZALwEJPG29SAEDd9NsCFnrAi+tnse+G6tov1Y6JiYk5zpuvQVGNGjUQGhqKMmXK4NmzZwgJCUHdunVx9epV3L9/H8CH545mzJiBypUr46+//kKjRo1w9epVlCpVCtHR0bCxsVEqU0dHB4UKFZLmZo+OjoaLi4tSHltbW2mdpaUloqOjpbSMeRRlqDN69GilQCo+Ph6Ojo7w8vKCmZnZpx+UPJSamorw8HA0adJEmsOevj9sx+8f27BgYDsWDGzHgoHt+O3RdX6OARv+AQClCRdk//+/f/xUCd5uyp+3v3Q7KkZy5US+BkUZZ8CoWLEiatSoAScnJ2zatAnlypUDAPTp0wfdunUD8GHe9MOHD2PlypWYPHlyvtRZQV9fH/r6+irpurq639yb81usE+Ue2/H7xzYsGNiOBQPbsWBgO347fqxcFDo62iq/U2SXg98p+lLtmJsy8334XEYWFhYoXbo07t69i4YNGwL48Gu6GZUrVw6PHj0CANjZ2SEmJkZpfVpaGmJjY6XnlOzs7PD8+XOlPIrl7PJk9qwTEREREREp8ylvjyaudjgfFYuYt0mwMTVAdZdC0NZSN6Tu25Lvs89llJCQgHv37sHe3h7Ozs5wcHDArVu3lPLcvn0bTk5OAAAPDw+8efMGkZGR0vojR45ALpejRo0aUp4TJ04oPWgVHh6OMmXKwNLSUspz+PBhpe2Eh4fDw8Pji+wnEREREVFBpK0lg0cJK7SqXAQeJay+i4AIyOegaNiwYTh+/DgePHiAM2fOoE2bNtDW1oa/vz9kMhmGDx+OefPmYcuWLbh79y5+//133Lx5Ez169ADwodfIx8cHvXr1wvnz53H69GkEBgaiffv2cHBwAAB06NABenp66NGjB65du4aNGzdi7ty5Ss8DDRo0CGFhYZg5cyZu3ryJ4OBgXLhwAYGBgflyXIiIiIiI6OvJ1+Fz//33H/z9/fHq1StYW1ujTp06OHv2LKytrQEAgwcPRlJSEoYMGYLY2FhUqlQJ4eHhKFGihFTGunXrEBgYiEaNGkFLSwtt27bFvHnzpPXm5uY4ePAg+vfvD3d3dxQuXBjjxo1T+i2jWrVqYf369Rg7dix+++03lCpVCjt27ED58uW/3sEgIiIiIqJ8ka9B0YYNG7LNM2rUKKXfKfpYoUKFsH79+izLqFixIk6ePJllnp9//hk///xztvUhIiIiIqKC5Zt6poiIiIiIiOhrY1BEREREREQajUERERERERFpNAZFGiQpKQldu3ZFhQoVoKOjg9atW6vNl5ycjDFjxsDJyQn6+vpwdnbGypUrVfKFhISgU6dOAIDo6Gh07twZdnZ2MDY2RtWqVbF169YvuTtERERERHnim/rxVvqy0tPTYWhoiIEDB2YZsPj6+uL58+dYsWIFSpYsiWfPnkEul6vk27lzpzQJRpcuXfDmzRvs2rULhQsXxvr16+Hr64sLFy6gSpUqX2yfiIiIiIg+F4MiDWJsbIzFixcDAE6fPo03b96o5AkLC8Px48dx//59FCpUCADg7Oysku/x48e4du0afHx8AABnzpzB4sWLUb16dQDA2LFjMXv2bERGRjIoIiIiIqJvGofPkZJdu3ahWrVqmDZtGooUKYLSpUtj2LBheP/+vUo+T09PmJmZAfjwW08bN25EbGws5HI5NmzYgKSkJHh6eubDXhARERER5Rx7ikjJ/fv3cerUKRgYGGD79u14+fIl+vXrh1evXmHVqlVSvp07d6JVq1bS8qZNm+Dn5wcrKyvo6OjAyMgI27dvR8mSJfNjN4iIiIiIcoxBESmRy+WQyWRYt24dzM3NAQCzZs1Cu3btsGjRIhgaGiI+Ph7Hjx/HihUrpNf9/vvvePPmDQ4dOoTChQtjx44d8PX1xcmTJ1GhQoX82h0iIiIiomwxKCIl9vb2KFKkiBQQAUC5cuUghMB///2HUqVKYf/+/XB1dYWjoyMA4N69e1iwYAGuXr0KNzc3AEClSpVw8uRJLFy4EEuWLMmXfSEiIiIiygk+U0RKateujadPnyIhIUFKu337NrS0tFC0aFEAqkPnEhMTAQBaWsqnk7a2ttpZ64iIiIiIviUMigqQdLlAxL1X2Hn5CSLuvUK6XKjkuX79Oi5fvozY2FjExcXh8uXLuHz5srS+Q4cOsLKyQrdu3XD9+nWcOHECw4cPR/fu3WFoaIi0tDTs378fLVu2lF5TtmxZlCxZEn369MH58+dx7949zJw5E+Hh4Zn+FhIRERER0beCw+cKiLCrzxCy+zqexSVJaXZm+mhmJ0OzDPmaNWuGhw8fSsuK6bKF+BBAmZiYIDw8HAMGDEC1atVgZWUFX19f/PHHHwCA48ePw8TEBFWrVpXK0NXVxb59+zBq1Ci0aNECCQkJKFmyJFavXo1mzTJunYiIiIjo28OgqAAIu/oMfddexMf9Qs/jk7EyXgtVrz3Hj5U/DH178OBBtuWVLVsW4eHhatft3LkTLVq0UEkvVapUlj8IS0RERET0reLwue9culwgZPd1lYAIgJQ2cf9NtUPpPkX58uXRt2/fPCmLiIiIiL4tt27dQoMGDWBrawsDAwMUL14cY8eORWpqqpRn2bJlqFu3LiwtLWFpaYnGjRvj/Pnzastr0KABli9fDgD4+++/0ahRI1hYWMDS0hLNmzdHVFTUV9mv7DAo+s6dj4pVGjKnSoZncck4HxWbJ9vr3bs3p9gmIiIiKqB0dXXRpUsXHDx4ELdu3cKcOXOwbNkyBAUFSXmOHTsGf39/HD16FBEREXB0dISXlxeePHmiVFZsbCxOnz4tPV7h4+ODYsWK4dy5czh16hRMTEwQEhKiFHDlFw6f+87FvM0qIMp9PiIiIiLSXMWLF0fx4sWlZScnJxw7dgwnT56U0tatW6f0muXLl2Pr1q04fPgwunTpIqXv3bsXVatWha2tLS5cuIDY2FiMHz9e+lmXsWPHYvv27Xj48CHKlSv3hfcsa+wp+s7ZmBrkaT4iIiIiIoW7d+8iLCwM9evXzzRPYmIiUlNTUahQIaX0Xbt2ST/jUqZMGVhZWWHFihVISUnB+/fvERoaiqJFi8LZ2flL7kKOMCj6zlV3KQR7cwPIMs0hYG+uj+ouhTLNQURERESUUa1atWBgYIBSpUqhbt26GD9+fKZ5R44cCQcHBzRu3FhKS05ORlhYmPQzLqampjh27BjWrl0LQ0NDmJiY4MCBAxg3bhx0dPJ/8BqDou+ctpYMQS1cAUAlMFIsj2laFtpamYdNREREREQZbdy4ERcvXsT69euxd+9ezJgxQ22+KVOmYMOGDdi+fTsMDP5vZNKRI0dgY2MDNzc3AMD79+/Ro0cP1K5dG2fPnsXp06fh5uaGP/74A+/fv/8q+5SV/A/L6LP5lLfH4k5VVX+nyFwfTW0T4e1mm4+1IyIiIqLvjeK5H1dXV6Snp6N379749ddfoa2tLeWZMWMGpkyZgkOHDqFixYpKr9+1a5fUSwQA69evx4MHDxAREQEtrQ/9MmvWrIGVlRV27dqFTp06fYW9yhyDogLCp7w9mrja4XxULGLeJsHG1ABVipriQNj+/K4aEREREX3H5HI5UlNTIZfLpaBo2rRpmDhxIg4cOIBq1aop5RdCYPfu3Vi7dq2UlpiYCC0tLchk/zd6SbEsl8u/zo5kgUFRAaKtJYNHCStp+VuY3pCIiIiIvi3pcqH0RXp1l0LSoxbr1q2Drq4uKlSoAH19fVy4cAGjR4+Gn58fdHV1AQBTp07FuHHjsH79ejg7OyM6OhoAYGJiAhMTE0RGRiIxMRF16tSRttmkSRMMHz4c/fv3x4ABAyCXyzFp0iRoaWnB09Pzqx+DjzEoIiIiIiLSEGFXn6k8cmFvboCgFq7wKW8PHR0dTJ06Fbdv34YQAk5OTggMDMSQIUOk/IsXL0ZKSgratWunVHZQUBCCg4Oxc+dONGvWTGkChbJly2L37t0ICQmBh4cHtLS0ULlyZQQFBcHe3v7L73g2GBQREREREWmAsKvP0HftRYiP0qPjktB37UUs7lQVfn5+8PPzy7KcBw8eZLl+586dGDt2rEp6kyZN0KRJE2k5NTUV+/bty2n1vyjOPkdEREREVMClywVCdl9XCYgASGkhu68jXa4uR86lpKSgbdu2aNq06WeV87UxKCIiIiIiKuDOR8UqDZn7mADwLC4J56NiP2s7enp6CAoKgqmp6WeV87UxKCIiIiIiKuBi3mYeEH1KvoKGQRERERERUQFnY2qQfaZc5CtoGBQRERERERVw1V0Kwd7cALJM1svwYRa66i6Fvma1vhkMioiIiIiICjhtLRmCWrgCgEpgpFgOauEq/V6RpmFQRERERESkAXzK22Nxp6qwM1ceImdnboDFnarCp3z+/15QfuHvFBERERERaQif8vZo4mqH81GxiHmbBBvTD0PmNLWHSIFBERERERGRBtHWksGjhFV+V+ObwuFzRERERESk0RgUERERERGRRmNQREREREREGo1BERERERERaTQGRUREREREpNEYFBERERERkUZjUERERERERBqNQREREREREWk0BkVERERERKTRGBQREREREZFGY1BEREREREQajUERERERERFpNAZFRERERESk0RgUERERERGRRmNQREREREREGo1BERERERERaTQGRUREREREpNEYFBERERERkUZjUERERERERBqNQREREREREWk0BkVERERERKTRGBQREREREZFGY1BEREREREQajUERERERERFpNAZFRERERESk0RgUERERERGRRmNQREREREREGo1BERERERERaTQGRUREREREpNEYFBERERERkUbL16AoODgYMplM6a9s2bIq+YQQaNq0KWQyGXbs2KG07tGjR2jevDmMjIxgY2OD4cOHIy0tTSnPsWPHULVqVejr66NkyZIIDQ1V2cbChQvh7OwMAwMD1KhRA+fPn8/LXSUiIiIiom9UvvcUubm54dmzZ9LfqVOnVPLMmTMHMplMJT09PR3NmzdHSkoKzpw5g9WrVyM0NBTjxo2T8kRFRaF58+Zo0KABLl++jMGDB6Nnz544cOCAlGfjxo0YOnQogoKCcPHiRVSqVAne3t6IiYn5MjtNRERERETfDJ18r4CODuzs7DJdf/nyZcycORMXLlyAvb290rqDBw/i+vXrOHToEGxtbVG5cmVMmDABI0eORHBwMPT09LBkyRK4uLhg5syZAIBy5crh1KlTmD17Nry9vQEAs2bNQq9evdCtWzcAwJIlS7B3716sXLkSo0aNUluv5ORkJCcnS8vx8fEAgNTUVKSmpn76AclDinp8K/WhT8N2/P6xDQsGtmPBwHYsGNiOBcOXbsfclJvvQdGdO3fg4OAAAwMDeHh4YPLkyShWrBgAIDExER06dMDChQvVBk4RERGoUKECbG1tpTRvb2/07dsX165dQ5UqVRAREYHGjRsrvc7b2xuDBw8GAKSkpCAyMhKjR4+W1mtpaaFx48aIiIjItN6TJ09GSEiISvrBgwdhZGSUq2PwpYWHh+d3FSgPsB2/f2zDgoHtWDCwHQsGtmPB8KXaMTExMcd58zUoqlGjBkJDQ1GmTBk8e/YMISEhqFu3Lq5evQpTU1MMGTIEtWrVQqtWrdS+Pjo6WikgAiAtR0dHZ5knPj4e79+/x+vXr5Genq42z82bNzOt++jRozF06FBpOT4+Ho6OjvDy8oKZmVnOD8IXlJqaivDwcDRp0gS6urr5XR36RGzH7x/bsGBgOxYMbMeCge1YMHzpdlSM5MqJfA2KmjZtKv27YsWKqFGjBpycnLBp0yZYW1vjyJEjuHTpUj7WMHP6+vrQ19dXSdfV1f3m3pzfYp0o99iO3z+2YcHAdiwY2I4FA9uxYPhS7ZibMvN9ooWMLCwsULp0ady9exdHjhzBvXv3YGFhAR0dHejofIjf2rZtC09PTwCAnZ0dnj9/rlSGYlkx3C6zPGZmZjA0NEThwoWhra2tNk9WzzoREREREVHB8E0FRQkJCbh37x7s7e0xatQoXLlyBZcvX5b+AGD27NlYtWoVAMDDwwP//vuv0ixx4eHhMDMzg6urq5Tn8OHDStsJDw+Hh4cHAEBPTw/u7u5KeeRyOQ4fPizlISIiIiKigitfh88NGzYMLVq0gJOTE54+fYqgoCBoa2vD398f1tbWantqihUrBhcXFwCAl5cXXF1d0blzZ0ybNg3R0dEYO3Ys+vfvLw1t++WXX7BgwQKMGDEC3bt3x5EjR7Bp0ybs3btXKnPo0KEICAhAtWrVUL16dcyZMwfv3r2TZqMjIiIiIqKCK1+Dov/++w/+/v549eoVrK2tUadOHZw9exbW1tY5er22tjb27NmDvn37wsPDA8bGxggICMD48eOlPC4uLti7dy+GDBmCuXPnomjRoli+fLk0HTcA+Pn54cWLFxg3bhyio6NRuXJlhIWFqUy+QEREREREBU++BkUbNmzIVX4hhEqak5MT9u3bl+XrPD09s52wITAwEIGBgbmqDxERERERff++qWeKiIiIiIiIvjYGRUREREREpNEYFBERERERkUZjUERERERERBqNQREREREREWk0BkVERERERKTRGBQREREREZFGY1BEREREREQajUERERERERFpNAZFRERERESk0RgUERERERGRRmNQREREREREGo1BERERERERaTQGRUREREREpNEYFBERERERkUZjUERERERERBqNQREREREREWk0BkVERERERKTRGBQREREREZFGY1BEREREREQajUERERERERFpNAZFRERERESk0RgUERERERGRRmNQREREREREGo1BERERERERaTQGRUREREREpNEYFBERERERkUZjUERERERERBqNQREREREREWk0BkVERERERKTRGBQREREREZFGY1BEREREREQajUERERERERFpNAZFRERERESk0RgUERERERGRRmNQREREREREGo1BERERERERaTQGRUREREREpNEYFBERERERkUZjUERERERERBqNQREREREREWk0BkVERERERKTRGBQREREREZFGY1BEREREREQajUERERERERFpNAZFRERERESk0RgUERERERGRRsuToOjhw4e4fv065HJ5XhRHRERERET01eQqKFq5ciVmzZqllNa7d28UL14cFSpUQPny5fH48eM8rSAREREREdGXlKugaOnSpbC0tJSWw8LCsGrVKvz111/4+++/YWFhgZCQkDyvJBERERER0Zeik5vMd+7cQbVq1aTlnTt3olWrVujYsSMAYNKkSejWrVve1pCIiIiIiOgLylVP0fv372FmZiYtnzlzBvXq1ZOWixcvjujo6LyrHRERERER0ReWq6DIyckJkZGRAICXL1/i2rVrqF27trQ+Ojoa5ubmeVtDIiIiIiKiLyhXw+cCAgLQv39/XLt2DUeOHEHZsmXh7u4urT9z5gzKly+f55UkIiIiIiL6UnIVFI0YMQKJiYnYtm0b7OzssHnzZqX1p0+fhr+/f55WkIiIiIiI6EvKVVCkpaWF8ePHY/z48WrXfxwkERERERERfetyFRQBwMaNG7Fr1y6kpKSgUaNG+OWXX75EvYiIiIiIiL6KXAVFixcvRv/+/VGqVCkYGhpi27ZtuHfvHqZPn/6l6kdERERERPRF5Wr2uQULFiAoKAi3bt3C5cuXsXr1aixatOhL1Y2IiIiIiOiLy1VQdP/+fQQEBEjLHTp0QFpaGp49e5bnFSMiIiIiIvoachUUJScnw9jY+P9erKUFPT09vH//Ps8rRkRERERE9DXkeqKF33//HUZGRtJySkoKJk6cqPSjrbNmzcqb2hEREREREX1huQqK6tWrh1u3biml1apVC/fv35eWZTJZ3tSMiIiIiIjoK8hVUHTs2DGl5ZcvX0JPTw9mZmZ5WSciIiIiIqKvJlfPFAHAmzdv0L9/fxQuXBi2trawtLSEnZ0dRo8ejcTExFyVFRwcDJlMpvRXtmxZAEBsbCwGDBiAMmXKwNDQEMWKFcPAgQMRFxenVMajR4/QvHlzGBkZwcbGBsOHD0daWppSnmPHjqFq1arQ19dHyZIlERoaqlKXhQsXwtnZGQYGBqhRowbOnz+fuwNDRERERETfpVz1FMXGxsLDwwNPnjxBx44dUa5cOQDA9evXMX/+fISHh+PUqVO4cuUKzp49i4EDB2ZbppubGw4dOvR/FdL5UKWnT5/i6dOnmDFjBlxdXfHw4UP88ssvePr0KbZs2QIASE9PR/PmzWFnZ4czZ87g2bNn6NKlC3R1dTFp0iQAQFRUFJo3b45ffvkF69atw+HDh9GzZ0/Y29vD29sbwIcfpB06dCiWLFmCGjVqYM6cOfD29satW7dgY2OTm0NERERERETfmVwFRePHj4eenh7u3bsHW1tblXVeXl7o3LkzDh48iHnz5uWsAjo6sLOzU0kvX748tm7dKi2XKFECEydORKdOnZCWlgYdHR0cPHgQ169fx6FDh2Bra4vKlStjwoQJGDlyJIKDg6Gnp4clS5bAxcUFM2fOBACUK1cOp06dwuzZs6WgaNasWejVqxe6desGAFiyZAn27t2LlStXYtSoUbk5RERERERE9J3JVVC0Y8cO/PnnnyoBEQDY2dlh2rRpaNasGYKCgpR+zygrd+7cgYODAwwMDODh4YHJkyejWLFiavPGxcXBzMxM6k2KiIhAhQoVlOrj7e2Nvn374tq1a6hSpQoiIiLQuHFjpXK8vb0xePBgAB9mz4uMjMTo0aOl9VpaWmjcuDEiIiIyrXdycjKSk5Ol5fj4eABAamoqUlNTc7TvX5qiHt9KfejTsB2/f2zDgoHtWDCwHQsGtmPB8KXbMTfl5iooevbsGdzc3DJdX758eWhpaSEoKChH5dWoUQOhoaEoU6YMnj17hpCQENStWxdXr16FqampUt6XL19iwoQJ6N27t5QWHR2tEqAplqOjo7PMEx8fj/fv3+P169dIT09Xm+fmzZuZ1n3y5MkICQlRST948KDSlOXfgvDw8PyuAuUBtuP3j21YMLAdCwa2Y8HAdiwYvlQ75ma+g1wFRYULF8aDBw9QtGhRteujoqJy9QxO06ZNpX9XrFgRNWrUgJOTEzZt2oQePXpI6+Lj49G8eXO4uroiODg4N1X+YkaPHo2hQ4dKy/Hx8XB0dISXl9c3MxtfamoqwsPD0aRJE+jq6uZ3degTsR2/f2zDgoHtWDCwHQsGtmPB8KXbUTGSKydyFRR5e3tjzJgxCA8Ph56entK65ORk/P777/Dx8clNkUosLCxQunRp3L17V0p7+/YtfHx8YGpqiu3btysdMDs7O5VZ4p4/fy6tU/xfkZYxj5mZGQwNDaGtrQ1tbW21edQ966Sgr68PfX19lXRdXd1v7s35LdaJco/t+P1jGxYMbMeCge1YMLAdC4Yv1Y65KTNXU3KPHz8et27dQqlSpTBt2jTs2rULO3fuxJQpU1CqVCncuHHjs3pyEhIScO/ePdjb2wP4EN15eXlBT08Pu3btgoGBgVJ+Dw8P/Pvvv4iJiZHSwsPDYWZmBldXVynP4cOHlV4XHh4ODw8PAICenh7c3d2V8sjlchw+fFjKQ0REREREBVeueoqKFi2KiIgI9OvXD6NHj4YQAgAgk8nQpEkTLFiwINNJEtQZNmwYWrRoAScnJzx9+hRBQUHQ1taGv7+/FBAlJiZi7dq1iI+Pl7rArK2toa2tDS8vL7i6uqJz586YNm0aoqOjMXbsWPTv31/qxfnll1+wYMECjBgxAt27d8eRI0ewadMm7N27V6rH0KFDERAQgGrVqqF69eqYM2cO3r17J81GR0REREREBVeugiIAcHFxwf79+/H69WvcuXMHAFCyZEkUKlQo1xv/77//4O/vj1evXsHa2hp16tTB2bNnYW1tjWPHjuHcuXNS+RlFRUXB2dkZ2tra2LNnD/r27QsPDw8YGxsjICAA48ePV6rv3r17MWTIEMydOxdFixbF8uXLpem4AcDPzw8vXrzAuHHjEB0djcqVKyMsLEztLHtERERERFSw5DooUrC0tET16tU/a+MbNmzIdJ2np6fUE5UVJycn7Nu3L8s8np6euHTpUpZ5AgMDERgYmO32iIiIiIioYMnVM0VEREREREQFDYMiIiIiIiLSaAyKiIiIiIhIozEoIiIiIiIijcagiIiIiIiINBqDIiIiIiIi0mgMioiIiIiISKMxKCIiIiIiIo3GoIiIiIiIiDQagyIiIiIiItJoDIqIiIiIiEijMSgiIiIiIiKNxqCIiIiIiIg0GoMiIiIiIiLSaAyKiIiIiIhIozEoIiIiIiIijcagiIiIiIiINBqDIiIiIiIi0mgMioiIiIiISKMxKCIiIiIiIo3GoIiIiIiIiDQagyIiIiIiItJoDIqIiIiIiEijMSgiIiIiIiKNxqCIiIiIiIg0GoMiIiIiIiLSaAyKiIiIiIhIozEoIiIiIiIijcagiIiIiIiINBqDIiIiIiIi0mgMioiIiIiISKMxKCIiIiIiIo3GoIiIiIiIiDQagyIiIiIiItJoDIqIiIiIiEijMSgiIiIiIiKNxqCIiIiIiIg0GoMiIiIiIiLSaAyKiIiIiIhIozEoIiIiIiIijcagiIiIiIiINBqDIiIiIiIi0mgMioiIiIiISKMxKCIiIiIiIo3GoIiIiIiIiDQagyIiIiIiItJoDIqIiIiIiEijMSgiIiIiIiKNxqCIiIiIiIg0GoMiIiIiIiLSaAyKiIiIiIhIozEoIiIiIiIijcagiIiIiIiINBqDIiIiIiIi0mgMioiIiIiISKMxKCIiIiIiIo3GoIiIiIiIiDQagyIiIiIiItJoDIqIiIiIiEijMSgiIiIiIiKNxqCIiIiIiIg0GoMiIiIiIiLSaAyKiIiIiIhIozEoIiIiIiIijZavQVFwcDBkMpnSX9myZaX1SUlJ6N+/P6ysrGBiYoK2bdvi+fPnSmU8evQIzZs3h5GREWxsbDB8+HCkpaUp5Tl27BiqVq0KfX19lCxZEqGhoSp1WbhwIZydnWFgYIAaNWrg/PnzX2SfiYiIiIjo25LvPUVubm549uyZ9Hfq1Clp3ZAhQ7B7925s3rwZx48fx9OnT/HTTz9J69PT09G8eXOkpKTgzJkzWL16NUJDQzFu3DgpT1RUFJo3b44GDRrg8uXLGDx4MHr27IkDBw5IeTZu3IihQ4ciKCgIFy9eRKVKleDt7Y2YmJivcxCIiIiIiCjf5HtQpKOjAzs7O+mvcOHCAIC4uDisWLECs2bNQsOGDeHu7o5Vq1bhzJkzOHv2LADg4MGDuH79OtauXYvKlSujadOmmDBhAhYuXIiUlBQAwJIlS+Di4oKZM2eiXLlyCAwMRLt27TB79mypDrNmzUKvXr3QrVs3uLq6YsmSJTAyMsLKlSu//gEhIiIiIqKvSie/K3Dnzh04ODjAwMAAHh4emDx5MooVK4bIyEikpqaicePGUt6yZcuiWLFiiIiIQM2aNREREYEKFSrA1tZWyuPt7Y2+ffvi2rVrqFKlCiIiIpTKUOQZPHgwACAlJQWRkZEYPXq0tF5LSwuNGzdGREREpvVOTk5GcnKytBwfHw8ASE1NRWpq6mcdk7yiqMe3Uh/6NGzH7x/bsGBgOxYMbMeCge1YMHzpdsxNufkaFNWoUQOhoaEoU6YMnj17hpCQENStWxdXr15FdHQ09PT0YGFhofQaW1tbREdHAwCio6OVAiLFesW6rPLEx8fj/fv3eP36NdLT09XmuXnzZqZ1nzx5MkJCQlTSDx48CCMjo5wdgK8kPDw8v6tAeYDt+P1jGxYMbMeCge1YMLAdC4Yv1Y6JiYk5zpuvQVHTpk2lf1esWBE1atSAk5MTNm3aBENDw3ysWfZGjx6NoUOHSsvx8fFwdHSEl5cXzMzM8rFm/yc1NRXh4eFo0qQJdHV187s69InYjt8/tmHBwHYsGNiOBQPbsWD40u2oGMmVE/k+fC4jCwsLlC5dGnfv3kWTJk2QkpKCN2/eKPUWPX/+HHZ2dgAAOzs7lVniFLPTZczz8Yx1z58/h5mZGQwNDaGtrQ1tbW21eRRlqKOvrw99fX2VdF1d3W/uzfkt1olyj+34/WMbFgxsx4KB7VgwsB0Lhi/VjrkpM98nWsgoISEB9+7dg729Pdzd3aGrq4vDhw9L62/duoVHjx7Bw8MDAODh4YF///1XaZa48PBwmJmZwdXVVcqTsQxFHkUZenp6cHd3V8ojl8tx+PBhKQ8RERERERVc+RoUDRs2DMePH8eDBw9w5swZtGnTBtra2vD394e5uTl69OiBoUOH4ujRo4iMjES3bt3g4eGBmjVrAgC8vLzg6uqKzp07459//sGBAwcwduxY9O/fX+rF+eWXX3D//n2MGDECN2/exKJFi7Bp0yYMGTJEqsfQoUOxbNkyrF69Gjdu3EDfvn3x7t07dOvWLV+OCxERERERfT35Onzuv//+g7+/P169egVra2vUqVMHZ8+ehbW1NQBg9uzZ0NLSQtu2bZGcnAxvb28sWrRIer22tjb27NmDvn37wsPDA8bGxggICMD48eOlPC4uLti7dy+GDBmCuXPnomjRoli+fDm8vb2lPH5+fnjx4gXGjRuH6OhoVK5cGWFhYSqTLxARERERUcGTr0HRhg0bslxvYGCAhQsXYuHChZnmcXJywr59+7Isx9PTE5cuXcoyT2BgIAIDA7PMQ0REREREBc839UwRERERERHR18agiIiIiIiINBqDIiIiIiIi0mgMioiIiIiISKMxKCIiIiIiIo3GoIiIiIiIiDQagyIiIiIiItJoDIqIiIiIiEijMSgiIiIiIiKNxqCIiIiIiIg0GoMiIiIiIiLSaAyKiIiIiIhIozEoIiIiIiIijcagiIiIiIiINBqDIiIiIiIi0mgMioiIiIiISKMxKCIiIiIiIo3GoIiIiIiIiDQagyIiIiIiItJoDIqIiIiIiEijMSgiIiIiIiKNxqCIiIiIiIg0GoMiIiIiIiLSaAyKiIiIiIhIozEoIiIiIiIijcagiIiIiIiINBqDIiIiIiIi0mgMioiIiIiISKMxKCIiIiIiIo3GoIiIiIiIiDQagyIiIiIiItJoDIqIiIiIiEijMSgiIiIiIiKNxqCIiIiIiIg0GoMiIiIiIiLSaAyKiIiIiIhIozEoIiIiIiIijcagiIiIiIiINBqDIiIiIiIi0mgMioiIiIiISKMxKCIiIiIiIo3GoIiIiIiIiDQagyIiIiIiItJoDIqIiIiIiEijMSgiIiIiIiKNxqCIiIiIiIg0GoMiIiIiIiLSaAyKiIiIiIhIozEoIiIiIiIijcagiIiIiIiINBqDIiIiIiIi0mgMioiIiIiISKMxKCIiIiIiIo3GoIiIiIiIiDQagyIiIiIiItJoDIqIiIiIiEijMSgiIiIiIiKNppPfFdA0cXFxSExM/CrbSktLQ2JiIqKjo6Gjw6b+XrEdv39sw4KB7fh5jIyMYG5unt/VICJSi1f1ryguLg4LFy5EamrqV93u7du3v+r26MtgO37/2IYFA9vx0+jq6qJ///4MjIjom8Sg6CtKTExEamoq2rRpA2tr6/yuDhER0Vfx4sULbN++HYmJiQyKiOibxKAoH1hbW8Pe3j6/q0FEREREROBEC0REREREpOEYFBERERERkUZjUERERERE9I27desWGjRoAFtbWxgYGKB48eIYO3as0gRey5YtQ926dWFpaQlLS0s0btwY58+fV1tegwYNsHz5crx69Qo+Pj5wcHCAvr4+HB0dERgYiPj4eCnvtm3b0KRJE1hbW8PMzAweHh44cODAF9/nr+mbCYqmTJkCmUyGwYMHS2nR0dHo3Lkz7OzsYGxsjKpVq2Lr1q1Kr4uNjUXHjh1hZmYGCwsL9OjRAwkJCUp5rly5grp168LAwACOjo6YNm2ayvY3b96MsmXLwsDAABUqVMC+ffu+yH5+SaGhobCwsPjscmQyGXbs2PHZ5Sh07doVrVu3zrPyvgUPHjyATCbD5cuX87Tc4OBgVK5cOcs8n3s8v1Tdc2vp0qVwdHSElpYW5syZk6PX5PW5mR9y0sZZ+bj9PT09la6bmuJb3u/8fI8VhPcIEamnq6uLLl264ODBg7h16xbmzJmDZcuWISgoSMpz7Ngx+Pv74+jRo4iIiICjoyO8vLzw5MkTpbJiY2Nx+vRptGjRAlpaWmjVqhV27dqF27dvIzQ0FIcOHcIvv/wi5T9x4gSaNGmCffv2ITIyEg0aNECLFi1w6dKlr7b/X9o3MdHC33//jT///BMVK1ZUSu/SpQvevHmDXbt2oXDhwli/fj18fX1x4cIFVKlSBQDQsWNHPHv2DOHh4UhNTUW3bt3Qu3dvrF+/HgAQHx8PLy8vNG7cGEuWLMG///6L7t27w8LCAr179wYAnDlzBv7+/pg8eTJ+/PFHrF+/Hq1bt8bFixdRvnz5r3Ycunbtijdv3vCGlkdkMhm2b9/+2QHZt9Yuc+fOhRBCWvb09ETlypVzHFg4Ojri2bNnKFy48BeqYfbi4+MRGBiIWbNmoW3btjmejerZs2ewtLTM8XZCQ0MxePBgvHnz5hNr+u3btm0bdHV1c5Q3t+fK98TZ2RmDBw/O10CpW7duKFKkCHr27Jnr1wYHB2PHjh15Gkg9ePAALi4uuHTp0mcF4kT0bShevDiKFy8uLTs5OeHYsWM4efKklLZu3Tql1yxfvhxbt27F4cOH0aVLFyl97969qFq1KmxtbQEAffv2VSq3X79+mD59upT28X1j0qRJ2LlzJ3bv3i19Jv/e5XtPUUJCAjp27Ihly5apfNg5c+YMBgwYgOrVq0tdhBYWFoiMjAQA3LhxA2FhYVi+fDlq1KiBOnXqYP78+diwYQOePn0K4MPJkZKSgpUrV8LNzQ3t27fHwIEDMWvWLGk7c+fOhY+PD4YPH45y5cphwoQJqFq1KhYsWPD1DgRRDpmbm39Wj6C2tjbs7Ozy9ccnHz16hNTUVDRv3hz29vYwMjLK0evs7Oygr6//hWunKj09HXK5/KtvNycKFSoEU1PT/K6GxktPT8eePXvQsmXL/K4KEWmIu3fvIiwsDPXr1880j+LnYAoVKqSUvmvXLrRq1Urta54+fYpt27ZlWa5cLsfbt29Vyv2e5XtQ1L9/fzRv3hyNGzdWWVerVi1s3LgRsbGxkMvl2LBhA5KSkuDp6QkAiIiIgIWFBapVqya9pnHjxtDS0sK5c+ekPPXq1YOenp6Ux9vbG7du3cLr16+lPB9v39vbGxEREZnWOzk5GfHx8Up/AJCamprpX1pa2qcdpP9v1qxZqFChAoyNjeHo6Ih+/fqpDBUEgB07dqBUqVIwMDCAt7c3Hj9+rLR+586dqFq1qjQeNSQkJNO6paSkIDAwEPb29jAwMICTkxMmT56caR3T09MxdOhQWFhYwMrKCiNGjFDq1QA+vJEmT54MFxcXGBoaolKlStiyZYu0/tixY5DJZDh8+DCqVasGIyMj1KpVC7du3VIqZ/HixShRogT09PRQpkwZrFmzRlrn7OwMAGjTpg1kMpm0nNv9Dw4OxurVq7Fz507IZDLIZDIcO3ZMWn///n00aNAARkZGqFSpkso5c+rUKdStWxeGhoZwdHTEwIED8e7du0yPn8Kff/4JR0dHGBkZwdfXF3FxcdK6jMOnunbtiuPHj2Pu3LlS/R48eIDXr1+jY8eOsLa2hqGhIUqVKoVVq1YBUB3a07VrV+m1Gf8U+5mcnIxhw4ahSJEiMDY2Ro0aNZSOgTqPHj1Cq1atYGJiAjMzM/j6+uL58+cAPvTeVKhQAcCHb70Udc6JjEODFPuxbds2tW1w7NgxdOvWDXFxcdI+BQcH52ifFENRd+3aBVdXV+jr6+PRo0dwdnbGpEmT0L17d5iamqJYsWJYunSpUh1HjhyJ0qVLw8jICMWLF8fvv//+yT/YnJP308fDyBYtWiS9/21tbdGuXTsAmZ8r6enp6NGjh/R+LFOmDObOnau0DcU5N2PGDNjb28PKygr9+/dX2q/k5GSMHDkSjo6O0NfXR8mSJbFixQpp/dWrV9G0aVOYmJjA1tYWnTt3xsuXL3N0HN69e4cuXbrAxMQE9vb2mDlzpsoxePjwIYYMGSLt27t372BmZqZ0bQE+XB+NjY3x9u1b6RzasGEDatWqBQMDA5QvXx7Hjx9Xek1O6n7mzBno6urihx9+UKm/uqHNO3bsgEwmk9aHhITgn3/+keofGhqa7XG5c+cO6tWrBwMDA7i6uiI8PFxpvYuLCwCgSpUqkMlk8PT0xIkTJ6Crq4vo6GilvIMHD0bdunWV6puX9xKFtLS0LO+TX+sPyPp+zb/v409T29HDwwMGBgYoVaoUateuLd1n1P0NHz4cDg4OqF+/vpSWkJCAsLAwNG3aVCmvn58fjIyMUKRIEZiYmGDx4sWZljt16lQkJCSgTZs233w75lS+Dp/bsGEDLl68iL///lvt+k2bNsHPzw9WVlbQ0dGBkZERtm/fjpIlSwL48MyRjY2N0mt0dHRQqFAh6YIfHR0t3RgUFF2F0dHRsLS0RHR0tJSWMc/HN42MJk+ejJCQEJX0gwcPZvqtd2JiYqbl5YSWlhbmzZsHFxcX3L9/H/369cOIESOwaNEipW1MnDgRf/31F/T09NCvXz+0b98ep0+fBgCcPHkSXbp0wbx581C3bl3cu3dPGkaYcUyqwrx587Br1y5s2rQJxYoVw+PHj1VujBnNnDkToaGhWLlyJcqVK4eZM2di+/btaNiwoZRn8uTJWLt2LZYsWYJSpUrhxIkT6NSpE6ytrZW+lRgzZgxmzpwJa2tr/PLLL+jevbu0H9u3b8egQYMwZ84cNG7cGHv27EG3bt1QtGhRNGjQAH///TdsbGywatUq+Pj4QFtb+5P2f9iwYbhx4wbi4+OloKJQoUJST+SYMWMwY8YMlCpVCmPGjIG/vz/u3r0LHR0d3Lt3Dz4+Pvjjjz+wcuVKvHjxAoGBgQgMDJTKUufu3bvYtGkTdu/ejfj4ePTo0QP9+vVT6RIHPvRy3r59G+XLl8f48eMBfPgdrEGDBuH69evYv38/ChcujLt37+L9+/dqtzd37lxMmTJFWp4yZQr+97//oWzZsgCAwMBAXL9+HRs2bICDgwO2b98OHx8f/PvvvyhVqpRKeXK5XAqIjh8/jrS0NPTv3x9+fn44duwY/Pz84OjoKD386ejoCGtra3Tt2hUPHjzINuD6WGZtUKtWLcyZMwfjxo2TAmoTE5Mc71NiYiKmTp2K5cuXw8rKSrrWzJw5ExMmTMBvv/2GLVu2oG/fvqhfvz7KlCkDADA1NUVoaCgcHBzw77//olevXjA1NcWIESNytV+KbWX3fsrowoULGDhwINasWYNatWohNjZWGlaR2bkil8tRtGhRbN68GVZWVjhz5gx69+4Ne3t7+Pr6SmUfPXoU9vb2OHr0KO7evQs/Pz9UrlwZvXr1AvBhuHNERATmzZuHSpUqISoqSgoc3rx5g4YNG6Jnz56YPXs23r9/j5EjR8LX1xdHjhzJ9jgMHz4cx48fx86dO2FjY4PffvsNFy9elIaEbdu2DZUqVULv3r2l+hgbG6N9+/ZYtWqVFBgCkJZNTU3x6tUrqfw5c+bA1dUVs2bNQosWLRAVFQUrK6sc133Xrl1o0aKFFOjkhp+fH65evYqwsDAcOnQIALIdUiqXy/HTTz/B1tYW586dQ1xcnMrQwfPnz6N69eo4dOgQ3NzcoKenh0KFCqF48eJYs2YNhg8fDuDDh5F169YpPW+b1/cShVOnTuW4Z/hL+ziIpO+TJrZjjx49kJSUhKioKKxevRopKSn46aefVPJt3boV27dvxx9//KF0vYqMjISxsTEePnyIhw8fSulNmzZFvXr18PTpU6xZswa+vr5KzxUpHD9+HIsWLcJvv/2GCxcu5Mk+fal2zNVnb5FPHj16JGxsbMQ///wjpdWvX18MGjRIWg4MDBTVq1cXhw4dEpcvXxbBwcHC3NxcXLlyRQghxMSJE0Xp0qVVyra2thaLFi0SQgjRpEkT0bt3b6X1165dEwDE9evXhRBC6OrqivXr1yvlWbhwobCxscm0/klJSSIuLk76e/z4sQAgXr58KVJSUtT+PXr0SAQHB4unT5+qLTMgIEC0atUq84P2kc2bNwsrKytpedWqVQKAOHv2rJR248YNAUCcO3dOCCFEo0aNxKRJk5TKWbNmjbC3t5eWAYjt27cLIYQYMGCAaNiwoZDL5Tmqk729vZg2bZq0nJqaKooWLSrtV1JSkjAyMhJnzpxRel2PHj2Ev7+/EEKIo0ePCgDi0KFD0vq9e/cKAOL9+/dCCCFq1aolevXqpVTGzz//LJo1a6Z2PxRysv8fU9cuUVFRAoBYvny5lKY4r27cuCHt08fn3smTJ4WWlpa0Hx8LCgoS2tra4r///pPS9u/fL7S0tMSzZ8/U1ufj940QQrRo0UJ069ZN7TYUdb906ZLKuq1btwoDAwNx6tQpIYQQDx8+FNra2uLJkydK+Ro1aiRGjx6ttvyDBw8KbW1t8ejRIylNcWzOnz8vhBDi0qVLAoCIioqS8owaNUp07txZbZkKGds0J22watUqYW5urlRGTvZJ8V66fPmyUh4nJyfRqVMnaVkulwsbGxuxePHiTOs8ffp04e7uLi0HBQWJSpUqZbmfCtm9n4RQbv+tW7cKMzMzER8fr7Y8deeKOv379xdt27aVlgMCAoSTk5NIS0uT0n7++Wfh5+cnhBDi1q1bAoAIDw9XW96ECROEl5eXUprimnnr1q0s6/L27Vuhp6cnNm3aJKW9evVKGBoaKu2Lk5OTmD17ttJrz507J7S1taVr7vPnz4WOjo44duyYEOL/zqEpU6ZIr1Ec46lTp+aq7qVKlRJ79uxRKlfxHlN3Hm7fvl1kvAXn5rwQQogDBw4IHR0dpfN4//79at8jH7/Xp06dKsqVKyctb926VZiYmIiEhASpvnlxL8no6dOnIjg4WDx69CjTe+TX+nv37p3YsWOHePfuXb7XhX9sx8/9W7VqlTA0NBTv379XSp8yZYowNzcXERERKq/p3bu3GDhwYJblKj6LPXz4UCl9zZo1wtDQUOzYseO7aMeXL18KACIuLi7b62q+9RRFRkYiJiYGVatWldLS09Nx4sQJLFiwALdu3cKCBQtw9epVuLm5AQAqVaqEkydPYuHChViyZAns7OwQExOjVG5aWhpiY2NhZ2cH4MMzCIphOwqK5ezyKNaro6+vr/bZBl1d3Uwfev7cZzgOHTqEyZMn4+bNm4iPj0daWhqSkpKQmJgoffOmo6OjNHyjbNmysLCwwI0bN1C9enX8888/OH36NCZOnCjlSU9PVylHoWvXrmjSpAnKlCkDHx8f/Pjjj/Dy8lJbv7i4ODx79gw1atRQ2udq1apJQ37u3r2LxMRENGnSROm1KSkpKg/qZZx4w97eHgAQExODYsWK4caNG9K3kgq1a9dWGfbzsdzuf3Yyq2PZsmXxzz//4MqVK0o9PEIIyOVyREVFoVy5cmrLLFasGIoUKSIte3h4QC6X49atW1mekxn17dsXbdu2xcWLF+Hl5YXWrVujVq1aWb7m0qVL6Ny5MxYsWIDatWsDAP7991+kp6ejdOnSSnmTk5NhZWWltpwbN27A0dERjo6OUpqrq6t0HqobXgQgy2GZWcmqDdTJ6T7p6empTP7y8fZkMpnKdWjjxo2YN28e7t27h4SEBKSlpcHMzCzX+5WT99PHmjRpAicnJxQvXhw+Pj7w8fFBmzZtsj2vFy5ciJUrV+LRo0d4//49UlJSVB7Md3Nzk3pcgQ/H+t9//wUAXL58Gdra2pmOP//nn39w9OhRqacuo3v37qm0xcfrU1JSlI5DoUKFpJ65rFSvXh1ubm5YvXo1Ro0ahbVr18LJyQn16tVTyufh4SH9W3GMb9y4keO637hxA0+fPkWjRo2yrVNeUbzPHBwcpLSM+5GVrl27YuzYsTh79ixq1qyJ0NBQ+Pr6wtjYWMqT1/eSjOXmdGKQLy2r+zV9PzS9HbW0tJCamgptbW3pOEybNg2TJk3CgQMHULNmTaX8Qgjs3bsXa9euzfK4aWl9eMJGLpdL+f73v/+hV69e2LBhQ6bPI32qL9WOuSkz34KiRo0aSTdUhW7duqFs2bIYOXKk1N2laBQFbW1t6YFnDw8PvHnzBpGRkXB3dwcAHDlyBHK5XLqBenh4YMyYMUhNTZUOTHh4OMqUKSNN7ODh4YHDhw8rDT0IDw/P8Q3ma3jw4AF+/PFH9O3bFxMnTkShQoVw6tQp9OjRAykpKTn+MJ+QkICQkBC13awGBgYqaVWrVkVUVBT279+PQ4cOwdfXF40bN1YZp59Timeg9u7dq/TBH4BKkJnxRFYMSfnch91zu//ZyaqOCQkJ6NOnDwYOHKjyumLFiuV6W7nRtGlTPHz4EPv27UN4eDgaNWqE/v37Y8aMGWrzR0dHo2XLlujZsyd69OghpSckJEBbWxuRkZFKH4gBqP2QmB9ye57kdJ8MDQ3VDoX6+AIrk8mk7UVERKBjx44ICQmBt7c3zM3NsWHDBpVnYL4UU1NTXLx4EceOHcPBgwcxbtw4BAcH4++//850co4NGzZg2LBhmDlzJjw8PGBqaorp06dLz2UqZLXfhoaGWdYrISEBLVq0wNSpU1XWKQLZL6Vnz55YuHAhRo0ahVWrVqFbt265GuKWk7rv2rULTZo0yfQaoqWlpRLI5mace16zsbFBixYtsGrVKri4uGD//v25Hraa19dSIsreunXroKuriwoVKkBfXx8XLlzA6NGj4efnJ12jp06dinHjxmH9+vVwdnaWHgUxMTGBiYkJIiMjkZiYiDp16kjl7tu3D8+fP8cPP/wAExMTXLt2DcOHD0ft2rWlZ7LXr1+PgIAAzJ07FzVq1JDKNTQ0VBnumy4XOB8Vi5i3SbAxNUB1l0LQ1sr90OKvLd+CIlNTU5Xpro2NjWFlZYXy5csjNTUVJUuWRJ8+fTBjxgxYWVlhx44dCA8Px549ewAA5cqVg4+PD3r16oUlS5YgNTUVgYGBaN++vfTtWYcOHRASEoIePXpg5MiRuHr1KubOnYvZs2dL2x00aBDq16+PmTNnonnz5tiwYQMuXLig8gB1foqMjIRcLsfMmTOlQHHTpk0q+dLS0nDhwgVUr14dwIcf+nrz5o3UK1G1alXcunVLei4rJ8zMzODn5wc/Pz+0a9cOPj4+iI2NVZlxxNzcHPb29jh37pz0TWxaWhoiIyOlHsGMD61nNatJdsqVK4fTp08jICBASjt9+jRcXV2lZV1dXaSnpyu97lP2X09PT6WcnKhatSquX7+eq20BHyYpePr0qXQOnz17FlpaWpl+M55Z/aytrREQEICAgADUrVsXw4cPVxsUJSUloVWrVihbtqzSrIzAhwe009PTERMTIz2EnZ1y5cpJz54peouuX7+ON2/eKLXP16Du2HzKPuXUmTNn4OTkhDFjxkhpGcdr50ZO3k/q6OjooHHjxmjcuDGCgoJgYWGBI0eO4KefflJ7PE6fPo1atWqhX79+Utq9e/dyVdcKFSpALpfj+PHjaifNUfzGnLOzc657zEuUKAFdXV2cO3dO+jLh9evXuH37ttI1JLP3QadOnTBixAjMmzcP169fV7pmKJw9e1blGAcGBua47jt37lTpuc7I2toab9++xbt376TemI+n3s7tdUbxPnv27JkUnJ09e1alTABqy+3Zsyf8/f1RtGhRlChRQuodVvgS9xIi+jw6OjqYOnUqbt++DSEEnJycEBgYiCFDhkh5Fi9ejJSUFKVnKYEPz/oFBwdj586daNasmdL1zNDQEMuWLcOQIUOQnJwMR0dH/PTTTxg1apSUZ+nSpdIzwv3795fSAwIClCaGCbv6DCG7r+NZXJKUZv//2rv3qCjr/A/gb+4zwDAIpsJyGRXwfkETXeSEmySbFywrzcWE1TQNVsDNeyi7XiJX07SshcxbGK0nUVMPWESteUNl9cS6aYKpaxqaxuWwIjCf3x/+mBy5wzwgzPt1zhzxme985/udzwzPfHie7+fRqrBsXG/8vq+yfwRrrkfiOkU1sbGxwcGDB7Fw4UKMGzcOJSUl8PHxwbZt2zB69GhDu5SUFERHR2PkyJGwtLTEc889hw0bNhju12q1OHToEKKiojB48GB07NgRS5cuNdqBBQYGYufOnXj99dexePFi+Pr6Ys+ePS16jaIqhYWF1XaWrq6u8PHxQXl5OTZu3Ihx48bhyJEjeP/996s93sbGBn/605+wYcMGWFtbIzo6GsOGDTPs2JYuXYqxY8fCy8sLzz//PCwtLXH27Fnk5uZixYoV1fp766234ObmBn9/f1haWmLXrl3o0qVLrX91jomJQWJiInx9fQ1fsh+8RoxGo8Frr72GuLg46PV6BAUFobCwEEeOHIGTk1ONX1hqMm/ePEycOBH+/v4ICQnBZ599ht27dxsWKQP3K9BlZmZi+PDhsLOzQ4cOHRo9/6p+MjIycP78ebi6ujb4mjoLFizAsGHDEB0djZdffhkODg44d+4cPv/88zrLvatUKkRERGDNmjUoKirCnDlzMHHixFpPndPpdDhx4gR++OEHODo6wsXFBQkJCRg8eDD69OmDsrIy7N+/v9bT9V555RVcvXoVmZmZuHnzpmG7i4sL/Pz8EB4ejqlTp2Lt2rXw9/fHzZs3kZmZif79+2PMmDHV+gsJCUG/fv0QHh6O9evXo6KiAq+++iqCg4ONKkU+bNGiRbh27Rq2b99ea5vG0ul0KCkpQWZmJgYMGAB7e/smzamhfH19ceXKFaSmpmLIkCE4cOAA0tLSmtxffZ+nh+3fvx/5+fl44okn0KFDBxw8eBB6vd6QUNf0XvH19cX27duRkZGBrl27YseOHTh58mS1AjV10el0iIiIwLRp0wyFFi5fvoyCggJMnDgRUVFRSE5OxuTJkzF//ny4uLjg4sWLSE1NxQcffFDtiN2DHB0dMX36dMybN89Q8GLJkiXVziLQ6XT45z//iRdffBF2dnaG63B16NABEyZMwLx58zBq1Ch4eHhUe453330Xvr6+6NWrF9atW4c7d+5g2rRpAFDv2H/++WecOnUK+/btq3UOQ4cOhb29PRYvXow5c+bgxIkT1arL6XQ6XLp0CWfOnIGHhwc0Gk2d5edDQkLg5+eHiIgI/O1vf0NRUZFRMg7cPyKkVquRnp4ODw8PqFQqw++v0NBQODk5YcWKFYbCGw8y9b6EiJqv6g/UdamvmuvevXvx+uuvG2373e9+h6NHj9b5uIYcTU7PvY7ZH+Xg4RO8bxTexeyPcvDelEGPdmJU76ojapDCwsJ6F3JVLTStq9ACgGq36dOni4jIW2+9JW5ubqJWqyU0NFS2b98uAOTOnTsi8uti3k8//VS6desmdnZ2EhISIpcvXzZ6nvT0dAkMDBS1Wi1OTk4SEBAgSUlJhvvxwELdpKQkGThwoDg4OIiTk5OMHDlScnJyap1jeXm5xMTEiJOTkzg7O8vcuXNl6tSpRgvD9Xq9rF+/Xnr06CE2Njby2GOPSWhoqHz99dci8muhhap5idS8MH/Tpk3SrVs3sbGxET8/P9m+fbvRWPbt2yc+Pj5ibW0t3t7eDZ7/wwoKCuSpp54SR0dHASBZWVk1LmC+c+eO4f4q2dnZhsc6ODhI//79ZeXKlbU+V9Vi602bNom7u7uoVCp5/vnn5fbt24Y2DxdaOH/+vAwbNkzUarXhNVq+fLn06tVL1Gq1uLi4yPjx4yU/P19Eqi++9vb2rvF9VzWPe/fuydKlS0Wn04mNjY24ubnJs88+ayh4UpPLly9LWFiYODg4iEajkRdeeEFu3LhhuL+meEZEREhwcHCtfYrUXGihvhjMmjVLXF1dBYAsW7asQXOqaWF81Wv18GL+AQMGGPoVEZk3b564urqKo6OjTJo0SdatW2fUV2MW1Dfk8/Rg8YTDhw9LcHCwdOjQQdRqtfTv318++eQTQ9ua3it3796VyMhI0Wq14uzsLLNnz5aFCxcajbGmYiMxMTFG8frf//4ncXFx4ubmJra2tuLj4yMffvih4f4LFy7Is88+K87OzqJWq6Vnz54SGxvboCIuxcXFMmXKFLG3t5fOnTvL6tWrqxWNOHbsmPTv31/s7Ozk4V1bZmamADAq1iDy63to586dEhAQILa2ttK7d2/58ssvjdrVNfYPPvhAhg8fXmO/D74309LSxMfHR9RqtYwdO1aSkpKMxnn37l157rnnxNnZWQDIli1b6n1dzp8/L0FBQWJrayt+fn6Snp5ercBMcnKyeHp6iqWlZbXPV3x8vFEhiiqm2pc8qL79X0u6d++eYZE4tV2MY+OVlZVJQkJCrcV4mqOiUi/DVn0h3gv213jTLdgvw1Z9IRWVxr/zlY5jQ76fV7EQqWXFLjVKUVERtFotCgsLa11Uff36dSQlJRnK3RIRkfJ27NiBuLg4/Pjjj0bXrPvhhx/QtWtX/Otf/6pWWKKhwsLCEBQU1KSS661t+vTpuHnzZrWjXFu3bkVsbGydRyUb61Ha/5WXl+PgwYMYPXq0WS/Qb+sYx0fLsbyfMTn5eL3tPp4xDL/t/mtRI6Xj2JDv51Ue2dPniIiImqO0tBTXr19HYmIiXnnlFaOEyFSCgoIwefJkk/erpMLCQnz77bfYuXNnnaf9ERE1VEHx3fobNaJda7CsvwkRESmhqhpQTbeqi662d1euXKnzdbhy5UqT+169ejV69uyJLl26YNGiRSYc9a/mz59vVH7eVFJSUmp9TaouU9FU48ePx6hRozBr1qxql0cgImqKTpqGVZ1saLvWwCNFRESt5OGiKg96uGR9e+Xu7l7n6/DgdXgaKyEhAQkJCbXer9Ppar3mU2sLCwszujbTg5p7ikl9C6YjIyMRGRnZrOcgIvMS0NUFbloVbhTerVZoAQAsAHTR3i/P/ahiUkRE1EpYzvh+iVm+DtVpNBpoNJrWHgYRUYNYWVpg2bjemP1RDiwAo8So6gpFy8b1fqSvV8TT54iIiIiIqFl+39cN700ZhC5a41PkumhVj345bvBIERERERERmcDv+7rhqd5dkH3pNgqK76KT5v4pc4/yEaIqTIpawYMXyCQiImrvuN8jMh9WlhZGZbfbCiZFLcje3h42NjbNusI9ERFRW2RjYwN7e/vWHgYRUY2YFLUgrVaLqKgolJaWtsjzVVRU4JtvvkFQUBCsrRnqtopxbPsYw/aBcWwee3t7aLXa1h4GEVGN+Fu9hWm12hbbKZSXl8Pe3h5dunTh1Z7bMMax7WMM2wfGkYio/WL1OSIiIiIiMmtMioiIiIiIyKwxKSIiIiIiIrPGpIiIiIiIiMwakyIiIiIiIjJrTIqIiIiIiMissSS3iYgIAKCoqKiVR/Kr8vJylJaWoqioiOVj2zDGse1jDNsHxrF9YBzbB8axfVA6jlXfy6u+p9eFSZGJFBcXAwA8PT1beSRERERERFSluLi43uuEWkhDUieql16vx48//giNRgMLC4vWHg6A+9mxp6cnrl69Cicnp9YeDjUR49j2MYbtA+PYPjCO7QPj2D4oHUcRQXFxMdzd3WFpWfeqIR4pMhFLS0t4eHi09jBq5OTkxF8Y7QDj2PYxhu0D49g+MI7tA+PYPigZx/qOEFVhoQUiIiIiIjJrTIqIiIiIiMisMSlqx+zs7LBs2TLY2dm19lCoGRjHto8xbB8Yx/aBcWwfGMf24VGKIwstEBERERGRWeORIiIiIiIiMmtMioiIiIiIyKwxKSIiIiIiIrPGpIiIiIiIiMwak6I25N1334VOp4NKpcLQoUORnZ1da9sRI0bAwsKi2m3MmDFG7f7zn/8gLCwMWq0WDg4OGDJkCK5cuaL0VMyaqeNYUlKC6OhoeHh4QK1Wo3fv3nj//fdbYipmrTFxBID169ejR48eUKvV8PT0RFxcHO7evdusPqn5TB3HN954A0OGDIFGo0GnTp3wzDPP4Pz580pPw+wp8XmskpiYCAsLC8TGxiowcqqiRAyvXbuGKVOmwNXVFWq1Gv369cOpU6eUnIbZM3UcKysrER8fj65du0KtVqN79+5Yvnw5FKkTJ9QmpKamiq2trXz44Yfy73//W2bMmCHOzs7y008/1dj+559/luvXrxtuubm5YmVlJVu2bDG0uXjxori4uMi8efMkJydHLl68KHv37q21T2o+JeI4Y8YM6d69u2RlZcmlS5fk73//u1hZWcnevXtbaFbmp7FxTElJETs7O0lJSZFLly5JRkaGuLm5SVxcXJP7pOZTIo6hoaGyZcsWyc3NlTNnzsjo0aPFy8tLSkpKWmpaZkeJOFbJzs4WnU4n/fv3l5iYGIVnYr6UiOHt27fF29tbIiMj5cSJE5Kfny8ZGRly8eLFlpqW2VEijitXrhRXV1fZv3+/XLp0SXbt2iWOjo7y9ttvm3z8TIraiICAAImKijL8v7KyUtzd3eWNN95o0OPXrVsnGo3GaMc8adIkmTJlisnHSrVTIo59+vSRv/71r0btBg0aJEuWLDHNoKmaxsYxKipKnnzySaNtc+fOleHDhze5T2o+JeL4sIKCAgEgX3/9tWkGTdUoFcfi4mLx9fWVzz//XIKDg5kUKUiJGC5YsECCgoKUGTDVSIk4jhkzRqZNm2bUZsKECRIeHm7Ckd/H0+fagHv37uH06dMICQkxbLO0tERISAiOHTvWoD42b96MF198EQ4ODgAAvV6PAwcOwM/PD6GhoejUqROGDh2KPXv2KDEFgjJxBIDAwEDs27cP165dg4ggKysLFy5cwKhRo0w+B2paHAMDA3H69GnDaQT5+fk4ePAgRo8e3eQ+qXmUiGNNCgsLAQAuLi4mHD1VUTKOUVFRGDNmjFHfZHpKxXDfvn14/PHH8cILL6BTp07w9/dHcnKyspMxY0rFMTAwEJmZmbhw4QIA4OzZs/jmm2/w9NNPm3wO1ibvkUzu1q1bqKysROfOnY22d+7cGd999129j8/OzkZubi42b95s2FZQUICSkhIkJiZixYoVePPNN5Geno4JEyYgKysLwcHBJp+HuVMijgCwceNGzJw5Ex4eHrC2toalpSWSk5PxxBNPmHT8dF9T4viHP/wBt27dQlBQEEQEFRUVmDVrFhYvXtzkPql5lIjjw/R6PWJjYzF8+HD07dvX5HMg5eKYmpqKnJwcnDx5UtHxk3IxzM/Px3vvvYe5c+di8eLFOHnyJObMmQNbW1tEREQoOidzpFQcFy5ciKKiIvTs2RNWVlaorKzEypUrER4ebvI58EiRGdi8eTP69euHgIAAwza9Xg8AGD9+POLi4jBw4EAsXLgQY8eO5SL9R1RNcQTuJ0XHjx/Hvn37cPr0aaxduxZRUVH44osvWmmk9LCvvvoKq1atwqZNm5CTk4Pdu3fjwIEDWL58eWsPjRqhsXGMiopCbm4uUlNTW3ikVJf64nj16lXExMQgJSUFKpWqlUdLNWnIZ1Gv12PQoEFYtWoV/P39MXPmTMyYMYPfcR4hDYnjP/7xD6SkpGDnzp3IycnBtm3bsGbNGmzbts30AzL5CXlkcmVlZWJlZSVpaWlG26dOnSphYWF1PrakpEScnJxk/fr11fq0traW5cuXG22fP3++BAYGmmTcZEyJOJaWloqNjY3s37/faPv06dMlNDTUJOMmY02JY1BQkLz22mtG23bs2CFqtVoqKyub9d6gplEijg+KiooSDw8Pyc/PN+m4yZgScUxLSxMAYmVlZbgBEAsLC7GyspKKigqlpmOWlPosenl5yfTp043abNq0Sdzd3U03eDJQKo4eHh7yzjvvGLVZvny59OjRw3SD/388UtQG2NraYvDgwcjMzDRs0+v1yMzMxG9/+9s6H7tr1y6UlZVhypQp1focMmRItVKxFy5cgLe3t+kGTwZKxLG8vBzl5eWwtDT+KFtZWRmOBpJpNSWOpaWlNcYIAESkWe8Nahol4lj1b3R0NNLS0vDll1+ia9euCs2AAGXiOHLkSHz77bc4c+aM4fb4448jPDwcZ86cMbQl01Dqszh8+HB+x2lBSsWxtjaKfMcxeZpFikhNTRU7OzvZunWrnDt3TmbOnCnOzs5y48YNERF56aWXZOHChdUeFxQUJJMmTaqxz927d4uNjY0kJSXJ999/Lxs3bhQrKys5fPiwonMxZ0rEMTg4WPr06SNZWVmSn58vW7ZsEZVKJZs2bVJ0LuassXFctmyZaDQa+fjjjyU/P18OHTok3bt3l4kTJza4TzI9JeI4e/Zs0Wq18tVXXxmV0y8tLW3x+ZkLJeL4MFafU5YSMczOzhZra2tZuXKlfP/995KSkiL29vby0Ucftfj8zIUScYyIiJDf/OY3hpLcu3fvlo4dO8r8+fNNPn4mRW3Ixo0bxcvLS2xtbSUgIECOHz9uuC84OFgiIiKM2n/33XcCQA4dOlRrn5s3bxYfHx9RqVQyYMAA2bNnj1LDp/9n6jhev35dIiMjxd3dXVQqlfTo0UPWrl0rer1eyWmYvcbEsby8XBISEqR79+6iUqnE09NTXn31Vblz506D+yRlmDqOAGq8PXhtMTI9JT6PD2JSpDwlYvjZZ59J3759xc7OTnr27ClJSUktNBvzZeo4FhUVSUxMjHh5eYlKpZJu3brJkiVLpKyszORjtxBR4pKwREREREREbQPXFBERERERkVljUkRERERERGaNSREREREREZk1JkVERERERGTWmBQREREREZFZY1JERERERERmjUkRERERERGZNSZFRERERERk1pgUERERNUNCQgIGDhxo+H9kZCSeeeaZVhsPERE1HpMiIiIiIiIya0yKiIio3bp3715rD4GIiNoAJkVERNRujBgxAtHR0YiNjUXHjh0RGhqK3NxcPP3003B0dETnzp3x0ksv4datW4bH6PV6rF69Gj4+PrCzs4OXlxdWrlxpuH/BggXw8/ODvb09unXrhvj4eJSXl7fG9IiISCFMioiIqF3Ztm0bbG1tceTIESQmJuLJJ5+Ev78/Tp06hfT0dPz000+YOHGiof2iRYuQmJiI+Ph4nDt3Djt37kTnzp0N92s0GmzduhXnzp3D22+/jeTkZKxbt641pkZERAqxEBFp7UEQERGZwogRI1BUVIScnBwAwIoVK3D48GFkZGQY2vz3v/+Fp6cnzp8/Dzc3Nzz22GN455138PLLLzfoOdasWYPU1FScOnUKwP1CC3v27MGZM2cA3C+08Msvv2DPnj0mnRsRESnHurUHQEREZEqDBw82/Hz27FlkZWXB0dGxWru8vDz88ssvKCsrw8iRI2vt75NPPsGGDRuQl5eHkpISVFRUwMnJSZGxExFR62BSRERE7YqDg4Ph55KSEowbNw5vvvlmtXZubm7Iz8+vs69jx44hPDwcf/nLXxAaGgqtVovU1FSsXbvW5OMmIqLWw6SIiIjarUGDBuHTTz+FTqeDtXX1XZ6vry/UajUyMzNrPH3u6NGj8Pb2xpIlSwzbLl++rOiYiYio5bHQAhERtVtRUVG4ffs2Jk+ejJMnTyIvLw8ZGRn44x//iMrKSqhUKixYsADz58/H9u3bkZeXh+PHj2Pz5s0A7idNV65cQWpqKvLy8rBhwwakpaW18qyIiMjUmBQREVG75e7ujiNHjqCyshKjRo1Cv379EBsbC2dnZ1ha3t8FxsfH489//jOWLl2KXr16YdKkSSgoKAAAhIWFIS4uDtHR0Rg4cCCOHj2K+Pj41pwSEREpgNXniIiIiIjIrPFIERERERERmTUmRUREREREZNaYFBERERERkVljUkRERERERGaNSREREREREZk1JkVERERERGTWmBQREREREZFZY1JERERERERmjUkRERERERGZNSZFRERERERk1pgUERERERGRWfs/FNNVXBYe5NQAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "fig, ax = plt.subplots(1, 1, figsize=plt.figaspect(1/2))\n", - "fig.suptitle(\n", - " f'Effects of search parameters on QPS/recall trade-off ({DATASET_FILENAME})\\n' + \\\n", - " f'k = {k}, n_probes = {n_probes}, pq_dim = {pq_dim}')\n", - "ax.plot(bench_recall_s1, bench_qps_s1, 'o')\n", - "ax.set_xlabel('recall')\n", - "ax.set_ylabel('QPS')\n", - "ax.grid()\n", - "annotations = []\n", - "for i, label in enumerate(bench_names):\n", - " annotations.append(ax.text(\n", - " bench_recall_s1[i], bench_qps_s1[i],\n", - " f\" {label} \",\n", - " ha='center', va='center'))\n", - "clutter = [\n", - " ax.text(\n", - " 0.02, 0.08,\n", - " 'Labels denote the bitsize of: internal_distance_dtype/lut_dtype',\n", - " verticalalignment='top',\n", - " bbox={'facecolor': 'white', 'edgecolor': 'grey'},\n", - " transform = ax.transAxes)\n", - "]\n", - "adjust_text(annotations, objects=clutter);" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This figure represents the trade-offs one does by choosing different combintations of the internal search types (the bit sizes of the data types are shown as point labels).\n", - "Depending on the GPU and the selected dataset, you may see different pictures.\n", - "With SIFT-128 (`pq_dim = 64`), reducing the `internal_distance_dtype` comes at a huge cost to recall,\n", - "whereas `lut_dtype` doesn't cost too much while significantly improving QPS.\n", - "\n", - "Also, often you may see `16/16` version being faster than `16/8`.\n", - "This indicates that ALU is the bottleneck in this configuration, and a few extra ALU operations for converting between fp8 and fp16 do more harm than the saved L1 bandwidth does good for the performance.\n", - "\n", - "\n", - "Let's try the same experiment, but with refinement.\n", - "We'll try ratio 2 and 4 and see how it affects recall and QPS." - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "210 ms ± 129 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "181 ms ± 331 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "184 ms ± 536 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "179 ms ± 331 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "182 ms ± 329 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "410 ms ± 203 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "344 ms ± 304 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "338 ms ± 632 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "320 ms ± 269 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "323 ms ± 194 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "425 ms ± 743 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "389 ms ± 688 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "381 ms ± 519 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "325 ms ± 552 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "340 ms ± 876 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "def search_refine(ps, ratio):\n", - " k_search = k * ratio\n", - " candidates = ivf_pq.search(ps, index, queries, k_search, handle=resources)[1]\n", - " return candidates if ratio == 1 else refine(dataset, queries, candidates, k, handle=resources)[1]\n", - "\n", - "ratios = [1, 2, 4]\n", - "bench_qps_sr = np.zeros((len(ratios), len(search_ps)), dtype=np.float32)\n", - "bench_recall_sr = np.zeros((len(ratios), len(search_ps)), dtype=np.float32)\n", - "\n", - "for j, ratio in enumerate(ratios): \n", - " for i, ps in enumerate(search_ps):\n", - " r = %timeit -o search_refine(ps, ratio); resources.sync()\n", - " bench_qps_sr[j, i] = (queries.shape[0] * r.loops / np.array(r.all_runs)).mean()\n", - " bench_recall_sr[j, i] = calc_recall(search_refine(ps, ratio), gt_neighbors)" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0UAAAHgCAYAAABqycbBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAADexklEQVR4nOzdd1gUx/8H8PdxdI6qVEFAsYEigopoVFQElaBGjYrGgC02EtFYE2NN7I3YO0bNN5aYxFhQULFi7L0jlihFY0FU2t3+/uB3G887yimK8d6v5+HRnZ2bnV2G3fvszM5KBEEQQEREREREpKP0yroCREREREREZYlBERERERER6TQGRUREREREpNMYFBERERERkU5jUERERERERDqNQREREREREek0BkVERERERKTTGBQREREREZFOY1BEREREREQ6jUERqcnKykKfPn3g4OAAiUSC6OhoAEB6ejo6deqEcuXKQSKRYO7cuWVaz9J27do1BAcHw9LSEhKJBL///ntZV6nUjB8/HhKJBA8ePCjrqhB90AIDAxEYGCgu37x5ExKJBLGxsWVWp6K4ubkhMjKyzLY/Y8YMVKpUCVKpFD4+PgCA/Px8jBgxAi4uLtDT00P79u2LLEOhUKBmzZr44YcftNp2ZGQk3NzcVNIKu/7R61Nef15W0nYXGxsLiUSCmzdvvp3KlYLAwEDUrFnztT+fmJgIiUSCxMRElfQ1a9agevXqMDAwgJWVlVZlxsXFQSaT4f79+69dL13EoEhHKE8shf0cOXJEzDt58mTExsZiwIABWLNmDXr06AEAGDJkCHbu3InRo0djzZo1aNWqVanXc/LkyWUWjERERODcuXP44YcfsGbNGtStW7dM6kHvn+3bt2P8+PFlXY0y9ezZM0yaNAne3t4wNTWFpaUlGjdujDVr1kAQBLX8L59f9PT04OTkhODgYLULf25uLmJiYlCnTh1YWFjAysoKXl5e+OKLL3D58mW1chUKBWxtbTF9+vS3tatl4uLFixg/fvx7/eWvtO3atQsjRoxAo0aNsGrVKkyePBkAsHLlSsyYMQOdOnXC6tWrMWTIkCLL+d///oc7d+4gKirqjeuk6fp3+PBhjB8/Ho8fPy5xOevXr8dnn32GKlWqQCKRqATKLzt27BiioqLg5eUFMzMzVKxYEZ07d8bVq1c15t+wYQMaNGgAKysrlCtXDk2bNsW2bdteY0/pfXb58mVERkaicuXKWLZsGZYuXQqgIJDX9B2uevXqKp9v1aoVPDw8MGXKlLKo/n+WfllXgN6tiRMnwt3dXS3dw8ND/P+ePXvQoEEDjBs3TiXPnj170K5dOwwbNuyt1W/y5Mno1KlTsXcGS9uLFy+QlJSEb7/9tlQurPRh2b59OxYsWKCzgVF6ejpatGiBS5cuoWvXroiKikJ2djZ+/fVXfP7554iLi8OaNWugp6d6n61ly5b4/PPPIQgCUlJSsHDhQjRv3hzbtm1D69atAQAdO3bEjh07EB4ejr59+yIvLw+XL1/G1q1b0bBhQ7WL/dGjR/HgwQOEhoa+s/1/Fy5evIgJEyYgMDBQrffiQ7Vnzx7o6elhxYoVMDQ0VEmvUKEC5syZU6JyZsyYga5du8LS0lKr7S9btgwKhUKtTq9e/2bOnIkJEyYgMjKyxHfsFy1ahBMnTqBevXr4559/Cs03bdo0HDp0CJ9++im8vb2RlpaG+fPnw9fXF0eOHFHpgZg3bx6++uorhIaGYurUqcjOzkZsbCw+/vhj/Prrr+jQoYNW+1+Wrly5ona+oH8lJiZCoVAgJiZG5fsZABgZGWH58uUqaZrafr9+/TBs2DBMmDAB5ubmb7W+HwoGRTqmdevWxfaAZGRkwNPTU2O6tl24/xXKLub/0v7l5+dDoVCofJn4L/vQ9qc4giAgOzsbJiYmZV2VYkVERODSpUv47bff0LZtWzH9q6++wvDhwzFz5kz4+Phg+PDhKp+rWrUqPvvsM3H5k08+gbe3N+bOnYvWrVvj2LFj2Lp1K3744Qd88803Kp+dP3++xjvz27dvh6urK7y8vDTW9b90XF/Xh7KPGRkZMDExUfub1+Zac+rUKZw5cwazZs3SevsGBgYa66Tp+qetNWvWoEKFCtDT0ytyaNXQoUPx888/qxyDLl26oFatWpg6dSrWrl0rps+bNw/16tXDn3/+KQ5H69WrFypUqIDVq1f/p4IiIyOjsq7Cey0jIwOA5u8k+vr6KufVwnTs2BFffvklNm7ciF69epV2FT9IDNNJpBzXmpKSgm3btondssqhd4IgYMGCBWK60uPHjxEdHQ0XFxcYGRnBw8MD06ZNU7sDp7zrUatWLRgbG8PW1hatWrXC8ePHARQMt3n27BlWr14tbkM55vjp06eIjo6Gm5sbjIyMYGdnh5YtW+LkyZPF7tepU6fQunVrWFhYQCaToUWLFirDBcePHw9XV1cAwPDhwyGRSIq9Uztv3jx4eXnB1NQU1tbWqFu3Ln7++WeVPHfv3kWvXr1gb28PIyMjeHl5YeXKlSp5cnNzMXbsWPj5+cHS0hJmZmZo3Lgx9u7dq5JP+VzCzJkzMXfuXFSuXBlGRka4ePEigIKu9s6dO8PW1hYmJiaoVq0avv32W7V6P378WLzbaWlpiZ49e+L58+fFHkPlmOkTJ06gYcOGMDExgbu7OxYvXlzq+/M6ZSxYsACVKlWCqakpgoODcefOHQiCgEmTJsHZ2RkmJiZo164dHj58qLZvO3bsQOPGjWFmZgZzc3OEhobiwoUL4vrIyEgsWLAAgOqQMCWFQoG5c+fCy8sLxsbGsLe3R79+/fDo0SOV7bi5ueHjjz/Gzp07UbduXZiYmGDJkiUAgPj4eHz00UewsrKCTCZDtWrV1IIETfLz8zFp0iTx+Lm5ueGbb75BTk6Oxm0fPHgQ9evXh7GxMSpVqoSffvqp2G0cOXIEO3fuRGRkpEpApDRlyhRUqVIFU6dOxYsXL4osq1atWihfvjxSUlIAAMnJyQCARo0aqeWVSqUoV66cWvq2bdtUeomKOq6ldW4CgFWrVqF58+aws7ODkZERPD09sWjRoiL3t6RiY2Px6aefAgCaNWsmtjHlUMOi9rGk9RIEAd9//z2cnZ1hamqKZs2aqbTzl5X0uBWmJO1SIpFg1apVePbsmdq1Zu/evbhw4YLacdDk999/h6GhIZo0aaKSXpJrxsvPFBV2/YuMjBSDfXd3dzG9uGGOyuehitOwYUO1oLBKlSrw8vLCpUuXVNIzMzNhZ2encv5RXtdKGiCX5LpU2HM8hT378tdff6FNmzawtraGmZkZvL29ERMTU2Q9ND1TdOHCBTRv3hwmJiZwdnbG999/X2ibK+68DQBnz55FZGQkKlWqBGNjYzg4OKBXr15qPXfKZ56uX7/+WtdHpYsXL6JZs2YwNTVFhQoVNA7x/fvvv9G+fXuYmZnBzs4OQ4YM0Xi+VvZU2traQiKRqI1SkMvlyMzMLLI+dnZ28Pb2xh9//FHifdB17CnSMU+ePFF72F4ikaBcuXKoUaMG1qxZgyFDhsDZ2Rlff/01AKBOnTri2GrlcBil58+fo2nTprh79y769euHihUr4vDhwxg9ejRSU1NVJmPo3bs3YmNj0bp1a/Tp0wf5+fk4cOAAjhw5grp162LNmjXo06cP6tevjy+++AIAULlyZQBA//79sWnTJkRFRcHT0xP//PMPDh48iEuXLsHX17fQ/b1w4QIaN24MCwsLjBgxAgYGBliyZAkCAwOxb98++Pv7o0OHDrCyssKQIUMQHh6ONm3aQCaTFVrmsmXL8NVXX6FTp04YPHgwsrOzcfbsWfz111/o1q0bgILhRg0aNIBEIkFUVBRsbW2xY8cO9O7dG5mZmeLDu5mZmVi+fLk4dOjp06dYsWIFQkJCcPToUfHBY6VVq1YhOzsbX3zxBYyMjGBjY4OzZ8+icePGMDAwwBdffAE3NzckJyfjzz//VHvwuHPnznB3d8eUKVNw8uRJLF++HHZ2dpg2bVqh+6v06NEjtGnTBp07d0Z4eDg2bNiAAQMGwNDQULwLVRr7o20Z69atQ25uLr788ks8fPgQ06dPR+fOndG8eXMkJiZi5MiRuH79OubNm4dhw4apfAFYs2YNIiIiEBISgmnTpuH58+dYtGgRPvroI5w6dQpubm7o168f7t27h/j4eKxZs0btuPTr1w+xsbHo2bMnvvrqK6SkpGD+/Pk4deoUDh06pHI3+sqVKwgPD0e/fv3Qt29fVKtWDRcuXMDHH38Mb29vTJw4EUZGRrh+/ToOHTpU7O+kT58+WL16NTp16oSvv/4af/31F6ZMmSL26rzs+vXr6NSpE3r37o2IiAisXLkSkZGR8PPzK7TXBQD+/PNPAFD5u3+Zvr4+unXrhgkTJuDw4cNo0aJFoWU9evQIjx49EoeDKG9GrFu3Do0aNYK+ftGXpLS0NJw6dQoTJ05USdd0XEvz3AQUDIfy8vJC27Ztoa+vjz///BMDBw6EQqHAoEGDiqx3cZo0aYKvvvoKP/74I7755hvUqFEDAMR/C9tHbeo1duxYfP/992jTpg3atGmDkydPIjg4GLm5uSp10ea4FaYk7XLNmjVYunQpjh49Kg4FUl5rfvjhB2RlZYnPQ7x8HF51+PBh1KxZU63XR9trRmHXv1q1aiE3Nxf/+9//MGfOHJQvXx5AwZfVt0UQBKSnp6v9XQYGBmLTpk2YN28ewsLCkJ2djXnz5uHJkycYPHhwseWW9Lqkjfj4eHz88cdwdHTE4MGD4eDggEuXLmHr1q0lqpNSWloamjVrhvz8fIwaNQpmZmZYunSpxmCvJOdtZd1u3LiBnj17wsHBARcuXMDSpUtx4cIFHDlyRG3yhze9PrZq1QodOnRA586dsWnTJowcORK1atUShwq/ePECLVq0wO3bt/HVV1/ByckJa9aswZ49e1TKmjt3Ln766Sf89ttvWLRoEWQyGby9vcX1z58/h4WFBZ4/fw5ra2uEh4dj2rRpGr+3+Pn5fVCTRr11AumEVatWCQA0/hgZGankdXV1FUJDQ9XKACAMGjRIJW3SpEmCmZmZcPXqVZX0UaNGCVKpVLh9+7YgCIKwZ88eAYDw1VdfqZWrUCjE/5uZmQkRERFqeSwtLdW2XRLt27cXDA0NheTkZDHt3r17grm5udCkSRMxLSUlRQAgzJgxo9gy27VrJ3h5eRWZp3fv3oKjo6Pw4MEDlfSuXbsKlpaWwvPnzwVBEIT8/HwhJydHJc+jR48Ee3t7oVevXmr1s7CwEDIyMlTyN2nSRDA3Nxdu3bqlkv7ycR03bpwAQKVMQRCETz75RChXrlwxeywITZs2FQAIs2bNEtNycnIEHx8fwc7OTsjNzS21/dG2DFtbW+Hx48di+ujRowUAQu3atYW8vDwxPTw8XDA0NBSys7MFQRCEp0+fClZWVkLfvn1VtpWWliZYWlqqpA8aNEjQdLo8cOCAAEBYt26dSnpcXJxauqurqwBAiIuLU8k7Z84cAYBw//59tfKLcvr0aQGA0KdPH5X0YcOGCQCEPXv2qG17//79YlpGRoZgZGQkfP3110Vup3379gIA4dGjR4Xm2bx5swBA+PHHH8U0AELv3r2F+/fvCxkZGcJff/0ltGjRQqUdKRQKsW3Z29sL4eHhwoIFC9TastKKFSsEExMT8e/n5X179biW9rnp5W0qhYSECJUqVVJJa9q0qdC0aVNxWdlOV61apXGflDZu3CgAEPbu3au2rrB9LGm9MjIyBENDQyE0NFRln7755hsBgMo5t6THrTDatMuIiAjBzMxMrYymTZsWe45VcnZ2Fjp27KiWXpJrRkREhODq6qqSpun6N2PGDAGAkJKSUqI6vcrLy0ulTRRnzZo1AgBhxYoVKunp6eni35Dyp3z58sLhw4dLVG5Jr0vK7wuv7u/evXtV2mh+fr7g7u4uuLq6qp0fNF1/Xubq6qrS7qKjowUAwl9//SWmZWRkCJaWlip10ea8relv43//+5/aubC0ro8//fSTmJaTkyM4ODiotM25c+cKAIQNGzaIac+ePRM8PDzU/vaVdXr1ujBq1Chh5MiRwvr164X//e9/QkREhABAaNSokcr1Tmny5MkCACE9Pb3Y/SBB4PA5HbNgwQLEx8er/OzYseO1y9u4cSMaN24Ma2trPHjwQPwJCgqCXC7H/v37AQC//vorJBKJ2uQNANTu1mhiZWWFv/76C/fu3Stx3eRyOXbt2oX27dujUqVKYrqjoyO6deuGgwcPFtv9XFhd/v77bxw7dkzjekEQ8OuvvyIsLAyCIKgcl5CQEDx58kQcwiGVSsWhEwqFAg8fPkR+fj7q1q2rcWhgx44dVe5Q3r9/H/v370evXr1QsWJFlbyajmv//v1Vlhs3box//vmnRMdBX18f/fr1E5cNDQ3Rr18/ZGRk4MSJE6WyP69TxqeffqrykKm/vz8A4LPPPlPpefD390dubi7u3r0LoOAu4uPHjxEeHq7yO5JKpfD391cbrqfJxo0bYWlpiZYtW6qU4efnB5lMplaGu7s7QkJCVNKUY8b/+OOPEg9PAgqerQEKnkl4mfIO96szUnl6eqJx48bisq2tLapVq4YbN24UuZ2nT58CQJEP6irXKfMqrVixAra2trCzs4O/vz8OHTqEoUOHinekJRIJdu7cie+//x7W1tb43//+h0GDBsHV1RVdunRRe6Zo+/btaNasmdrdY03HtbTPTS9vU9nj3rRpU9y4cQNPnjwp9NiUFk37WNJ6JSQkiL2pL++Tpp6Bkh63wmjbLt/UP//8A2tra7X017lmvA8uX76MQYMGISAgABERESrrTE1NUa1aNURERGDjxo1YuXIlHB0d0aFDB1y/fr3IcrW5LpXUqVOnkJKSgujoaLVnX0pyXX/Z9u3b0aBBA9SvX19Ms7W1Rffu3VXyaXPefvlvIzs7Gw8ePECDBg0AQOO+vsn1USaTqTznY2hoiPr166ucX7dv3w5HR0d06tRJTDM1NRVHxpTElClTMHXqVHTu3Bldu3ZFbGwsfvjhBxw6dAibNm1Sy6/82+DrOEqGw+d0TP369Ut1qulr167h7NmzhQ4lUD4smJycDCcnJ9jY2LzWdqZPn46IiAi4uLjAz88Pbdq0weeff64S7Lzq/v37eP78uTjM5GU1atSAQqHAnTt3ihw6pMnIkSORkJCA+vXrw8PDA8HBwejWrZv4XMT9+/fx+PFjLF26VJxG81XK4wIAq1evxqxZs3D58mXk5eWJ6ZpmCXw1TXnCLek7El4NnJQnzEePHsHCwqLIzzo5OcHMzEwlrWrVqgAKnu9RXmzeZH+UtCnj1X1SBkguLi4a05XP+ly7dg0A0Lx5c411KO54KMt48uQJ7OzsNK5/+fcMaK5/ly5dsHz5cvTp0wejRo1CixYt0KFDB3Tq1KnIZxJu3boFPT09tZmJHBwcYGVlhVu3bqmkv3qcgILf/6vPPr3q5YCnsIfflcHQq8ehXbt2iIqKgkQigbm5uTjt8MuMjIzw7bff4ttvv0Vqair27duHmJgYbNiwAQYGBuKD5nl5eYiPj9c4xaym41ra56ZDhw5h3LhxSEpKUnvO4MmTJ1rPfKatwv5WSlIvZVuoUqWKynpbW1u1gKKkx+3+/fuQy+Viukwmg0wm07pdlgZBw5Twr3PN0NbDhw9Vhh+amJi8UTtIS0tDaGgoLC0tsWnTJkilUpX1n376qThEUqldu3aoUqUKvv32W6xfvx5yuVzt3TQ2NjZ4/PixVtelklA+E/gm7+hRunXrlnhD62WvXr+1OW8/fPgQEyZMwC+//KK2b5puZLzJ9dHZ2VktELS2tsbZs2fF5Vu3bsHDw0Mtn6bvKNoYMmQIvvvuOyQkJKBr164q65R/G9oGqbqKQRG9EYVCgZYtW2LEiBEa1yu/NL+pzp07o3Hjxvjtt9+wa9cuzJgxA9OmTcPmzZvF8brvSo0aNXDlyhVs3boVcXFx+PXXX7Fw4UKMHTsWEyZMEO/2f/bZZ2p3+pSU44PXrl2LyMhItG/fHsOHD4ednR2kUimmTJkiXnBe9qazTb16kVXS9KXidZTG/mhbRmH7VNy+Kn9Pa9asgYODg1q+4p5vUZZhZ2eHdevWaVz/6hdLTftrYmKC/fv3Y+/evdi2bRvi4uKwfv16NG/eHLt27Sp0P5RKerF73d+9p6cnfv/9d5w9e1btYXYl5YX/1S+czs7OCAoKKlH9gIJe3K5du6Jjx47w8vLChg0bEBsbC319fbFnt02bNmqf03RcS/PclJycjBYtWqB69eqYPXs2XFxcYGhoiO3bt2POnDla9fC9Lk37+DbqVdLjVq9ePZUAZ9y4cSoPg7+rL2HlypXTGNi/i2tGhw4dsG/fPnE5IiLitV/S++TJE7Ru3RqPHz/GgQMH4OTkpLL+xo0biIuLUwtobGxs8NFHH4nPIN65c0ctgN67d684tX1JrkuF/e5eDoLLijbn7c6dO+Pw4cMYPnw4fHx8IJPJoFAo0KpVK41/G29yfXzb19aimJiYoFy5chonElL+bSifhaOiMSiiN1K5cmVkZWUV+8WncuXK2LlzJx4+fFjkHdmiLqSOjo4YOHAgBg4ciIyMDPj6+uKHH34o9AJna2sLU1NTXLlyRW3d5cuXoaenp9abUFJmZmbo0qULunTpgtzcXHTo0AE//PADRo8eDVtbW5ibm0Mulxd7XDZt2oRKlSph8+bNKvuuaSiPJsovoefPn3+t/dDGvXv38OzZM5U7/coXDCofbH3T/SmtMkpCOYmHnZ1dsb+nwtpl5cqVkZCQgEaNGr1RwKqnp4cWLVqgRYsWmD17NiZPnoxvv/0We/fuLbRurq6uUCgUuHbtmsqD6Onp6Xj8+LE4icGbCgsLw+TJk/HTTz9pDIrkcjl+/vln2NvbFxo0acvAwADe3t64du0aHjx4AAcHB2zbtg2enp4lfodPaZ6b/vzzT+Tk5GDLli0qd5NLMsSypF4niChpvZRt4dq1ayqB6/3799UCipIet3Xr1qnMNqgs9121S6Xq1auLsxm+SttrRmEK+93MmjVL5fi9GsiUVHZ2NsLCwnD16lUkJCRonBI8PT0dgObAJC8vD/n5+QAKeuTi4+NV1teuXRsWFhYlvi4pe0heHb76ai+f8hx6/vx5rW5+aOLq6ir2Ar3s1et3Sc/bjx49wu7duzFhwgSMHTtWTNe0jXfF1dUV58+fhyAIKm1K03cUbTx9+hQPHjzQ2LubkpKC8uXLv9WJQT4kfKaI3kjnzp2RlJSEnTt3qq17/PixeKLu2LEjBEHAhAkT1PK9fCfFzMxM7UQsl8vVurrt7Ozg5OSkNpXly6RSKYKDg/HHH3+oTC2anp6On3/+GR999FGJhki96tXpPA0NDeHp6QlBEJCXlwepVIqOHTvi119/1RisvDy0QXl36eVj8NdffyEpKalEdbG1tUWTJk2wcuVK3L59W2Vdad+hys/PF6cBBgqm316yZAlsbW3h5+cH4M33p7TKKImQkBBYWFhg8uTJKkP0lF7+PSkDwVfbZufOnSGXyzFp0iS1z+fn52t8z86rNN3dU86wV1T7VvaYvDob2OzZswGg1F5u2qBBAwQHB2PVqlXYunWr2vpvv/0WV69exYgRI0rUu/aya9euqbVboOA4JyUlwdraWryYb9++Xat9Ks1zk6Y2+eTJE6xatarE9SlOYW2sKCWtV1BQEAwMDDBv3jyVvJpmkivpcWvUqBGCgoLEH2VQ9K7apVJAQADOnz+v8rfyuteMwhT2u/Hz81M5Bq/zfiO5XI4uXbogKSkJGzduREBAgMZ8Hh4e0NPTw/r161V+h3///TcOHDiAOnXqAACMjY1V6hQUFARra2utrkvKwOPl58fkcrlaL5Wvry/c3d0xd+5ctWOj7fWnTZs2OHLkCI4ePapSp1d74Ut63tb0twFobvPauH37Ni5fvvxan23Tpg3u3bun8uzP8+fPCx3O+Krs7Gy15zYBYNKkSRAEAa1atVJbd+LEiULbFKljT5GO2bFjh8Y/6IYNG77WWOvhw4djy5Yt+Pjjj8XpfZ89e4Zz585h06ZNuHnzJsqXL49mzZqhR48e+PHHH3Ht2jWx+/rAgQNo1qwZoqKiABRcZBISEjB79mw4OTnB3d0d1apVg7OzMzp16oTatWtDJpMhISEBx44dK/aFfd9//734DpiBAwdCX18fS5YsQU5OjsZ3CJREcHAwHBwc0KhRI9jb2+PSpUuYP38+QkNDxecvpk6dir1798Lf3x99+/aFp6cnHj58iJMnTyIhIUH8Ivzxxx9j8+bN+OSTTxAaGoqUlBQsXrwYnp6eyMrKKlF9fvzxR3z00Ufw9fXFF198AXd3d9y8eRPbtm3D6dOnX2sfNXFycsK0adNw8+ZNVK1aFevXr8fp06exdOlScTrc0tif0iijJCwsLLBo0SL06NEDvr6+6Nq1K2xtbXH79m1s27YNjRo1wvz58wFADPq++uorhISEQCqVomvXrmjatCn69euHKVOm4PTp0wgODoaBgQGuXbuGjRs3IiYmRuWhWk0mTpyI/fv3IzQ0FK6ursjIyMDChQvh7OyMjz76qNDP1a5dGxEREVi6dCkeP36Mpk2b4ujRo1i9ejXat2+PZs2aldqx+umnn9C8eXO0a9cO3bp1Q+PGjZGTk4PNmzcjMTERn332GYYMGaJ1uWfOnEG3bt3QunVrNG7cGDY2Nrh79y5Wr16Ne/fuYe7cuZBKpUhJScGlS5e0ei9QaZ6bgoODYWhoiLCwMPTr1w9ZWVlYtmwZ7OzskJqaqvV+a+Lj4wOpVIpp06bhyZMnMDIyEt8/VJiS1svW1hbDhg3DlClT8PHHH6NNmzY4deoUduzYoTaspqTHrTDvsl0CBc/UTJo0Cfv27UNwcDCAgjvnr3vN0ET59//tt9+ia9euMDAwQFhYmNrzcS/bv3+/GFTcv38fz549w/fffw+gYAp2Za/q119/jS1btiAsLAwPHz5UeVkrAPHhfVtbW/Tq1QvLly8Xnzt8+vQpFi5ciBcvXmD06NHF7kdJr0teXl5o0KABRo8eLfag/vLLL2JArKSnp4dFixYhLCwMPj4+6NmzJxwdHXH58mVcuHBBY2BdmBEjRmDNmjVo1aoVBg8eLE7J7erqqvJcTknP2xYWFmjSpAmmT5+OvLw8VKhQAbt27Sq0V7GkPv/8c+zbt++1bjr27dsX8+fPx+eff44TJ07A0dERa9asgampaYk+n5aWhjp16iA8PFwcDrlz505s374drVq1Qrt27VTyZ2Rk4OzZs2/8ygCd8k7muKMyV9SU3HhlulhtpuQWhIIpMkePHi14eHgIhoaGQvny5YWGDRsKM2fOFKdqFoSC6TtnzJghVK9eXTA0NBRsbW2F1q1bCydOnBDzXL58WWjSpIlgYmIiThWbk5MjDB8+XKhdu7Zgbm4umJmZCbVr1xYWLlxYon0/efKkEBISIshkMsHU1FRo1qyZ2hSm2kzJvWTJEqFJkyZCuXLlBCMjI6Fy5crC8OHDhSdPnqjkS09PFwYNGiS4uLgIBgYGgoODg9CiRQth6dKlYh6FQiFMnjxZcHV1FYyMjIQ6deoIW7duVZsqtrj6nT9/Xvjkk08EKysrwdjYWKhWrZrw3XffiesLm96zsKlXX6WcIvf48eNCQECAYGxsLLi6ugrz589XyVca+/OmZSinjd24caPGfT127Jha/pCQEMHS0lIwNjYWKleuLERGRgrHjx8X8+Tn5wtffvmlYGtrK0gkErXpZZcuXSr4+fkJJiYmgrm5uVCrVi1hxIgRwr1798Q8hf1d7d69W2jXrp3g5OQkGBoaCk5OTkJ4eLjalMia5OXlCRMmTBDc3d0FAwMDwcXFRRg9erQ47Xhx2351+uiiPH36VJgwYYLg5eUlGBsbi+eOl9vZywo7X7wsPT1dmDp1qtC0aVPB0dFR0NfXF6ytrYXmzZsLmzZtEvPNnz9fsLS01DjlbGH7pqxzaZ2btmzZInh7ewvGxsaCm5ubMG3aNGHlypVqfz+vOyW3IAjCsmXLhEqVKglSqVRlit6i9rGk9ZLL5cKECRMER0dHwcTERAgMDBTOnz+vNjWyNsetMCVtl6UxJbcgCIK3t7fQu3dvcbmk14ySTsktCAVTlVeoUEHQ09Mr0TlTec7V9DNu3DiVfS3q2vyyvLw8Yd68eYKPj48gk8kEmUwmNGvWTGWa8+KU5LokCIKQnJwsBAUFCUZGRoK9vb3wzTffCPHx8RqnjT948KDQsmVL8Vh7e3sL8+bNUzsWL9PU7s6ePSs0bdpUMDY2FipUqCBMmjRJWLFiRaHTgxd33v7777/F66KlpaXw6aefCvfu3VP7HWhzfVT+vl5WWHvV1L5u3boltG3bVjA1NRXKly8vDB48WHyFQ3FTcj969Ej47LPPBA8PD8HU1FQwMjISvLy8hMmTJ2v8u1y0aJFgamoqZGZmqq0jzSSC8A6eAiOi/7TAwEA8ePDgnTy7RP8Nd+/eRcOGDZGfn4+kpCSNs9uVFuULlTds2PDWtkH/XWvWrMGgQYNw+/btQmdIJNI1derUQWBgIObMmVPWVfnP4DNFRESktQoVKiAuLg7Z2dlo3bp1sVN7v4nAwMDXGp5HuqF79+6oWLEiFixYUNZVIXovxMXF4dq1ayUaVkn/Yk8RERWLPUVERET0IWNPERERERER6TT2FBERERERkU5jTxEREREREek0BkVERERERKTTGBQRUZkYP348JBIJHjx4UNZVodcQGRkJmUxW1tWg98DNmzchkUgQGxsrpin/vomI/isYFBGRzvvhhx/Qtm1b2NvbQyKRYPz48YXmvXv3Ljp37gwrKytYWFigXbt2uHHjxrurLL1VmzdvRpcuXVCpUiWYmpqiWrVq+Prrr/H48WO1vG5ubpBIJGo//fv311h2QkICmjdvDktLS5ibm8PPzw/r169/y3tE2kpOTka3bt1gZ2cHExMTVKlSBd9++22h+fPy8uDp6QmJRIKZM2e+w5oSUWnSL+sKEBGVtTFjxsDBwQF16tTBzp07C82XlZWFZs2a4cmTJ/jmm29gYGCAOXPmoGnTpjh9+jTKlSv3DmtNb8MXX3wBJycnfPbZZ6hYsSLOnTuH+fPnY/v27Th58iRMTExU8vv4+ODrr79WSatatapauatWrULv3r3RsmVLTJ48GVKpFFeuXMGdO3fe6v6UlTFjxmDUqFFlXQ2tnT59GoGBgahQoQK+/vprlCtXDrdv3y7y9zRv3jzcvn37HdaSiN4GBkVEpPNSUlLg5uaGBw8ewNbWttB8CxcuxLVr13D06FHUq1cPANC6dWvUrFkTs2bNwuTJk99VlUtddnY2DA0Noaen2wMINm3ahMDAQJU0Pz8/REREYN26dejTp4/KugoVKuCzzz4rssybN29i0KBB+PLLLxETE1PaVX4v6evrQ1//v/UVQ6FQoEePHqhevTr27t2rFgBrkpGRgYkTJ2LkyJEYO3bsO6glEb0tun31I6L3yq1bt+Dh4YGaNWsiPT39nW3Xzc2tRPk2bdqEevXqiQERAFSvXh0tWrTAhg0bXmvbymdz7t69i/bt20Mmk8HW1hbDhg2DXC7XqqzAwEDUrFkTJ06cQMOGDWFiYgJ3d3csXrxYJV9iYiIkEgl++eUXjBkzBhUqVICpqSkyMzMBABs3boSfnx9MTExQvnx5fPbZZ7h7967Gbd64cQMhISEwMzODk5MTJk6ciFff9KBQKDB37lx4eXnB2NgY9vb26NevHx49eqSS7/jx4wgJCUH58uXFuvfq1UurY/CmXg2IAOCTTz4BAFy6dEnjZ3Jzc/Hs2bNCy1y8eDHkcjkmTpwIoKDH8U3fhqF8Zufy5cvo3LkzLCwsUK5cOQwePBjZ2dkqeXNycjBkyBDY2trC3Nwcbdu2xd9//13sUFFNHj9+jMjISFhaWsLKygoREREahxZqeqZIIpEgKioKGzduhKenJ0xMTBAQEIBz584BAJYsWQIPDw8YGxsjMDAQN2/e1Kpub2rXrl04f/48xo0bBxMTEzx//rzYv8FRo0ahWrVqxQbGRPT++2/dxiGiD1ZycjKaN28OGxsbxMfHo3z58oXmzcvLw5MnT0pUro2NTan0figUCpw9e1bjl/T69etj165dePr0KczNzbUuWy6XIyQkBP7+/pg5cyYSEhIwa9YsVK5cGQMGDNCqrEePHqFNmzbo3LkzwsPDsWHDBgwYMACGhoZqdZ80aRIMDQ0xbNgw5OTkwNDQELGxsejZsyfq1auHKVOmID09HTExMTh06BBOnToFKysrlXq3atUKDRo0wPTp0xEXF4dx48YhPz9fDAAAoF+/fmK5X331FVJSUjB//nycOnUKhw4dgoGBATIyMhAcHAxbW1uMGjUKVlZWuHnzJjZv3lzsPmdlZakFApoYGBjA0tKy5Afz/6WlpQGAxja5Z88emJqaQi6Xw9XVFUOGDMHgwYNV8iQkJKB69erYvn07hg8fjrt378La2hqDBg3ChAkT3qh9du7cGW5ubpgyZQqOHDmCH3/8EY8ePcJPP/0k5unTpw/Wrl2Lbt26oWHDhtizZw9CQ0O13pYgCGjXrh0OHjyI/v37o0aNGvjtt98QERFR4jIOHDiALVu2YNCgQQCAKVOm4OOPP8aIESOwcOFCDBw4EI8ePcL06dPRq1cv7Nmzp8jySvNckJCQAAAwMjJC3bp1ceLECRgaGuKTTz7BwoULYWNjo5L/6NGjWL16NQ4ePMhJJYg+BAIRURkYN26cAEC4f/++cOnSJcHJyUmoV6+e8PDhw2I/u3fvXgFAiX5SUlJKXKf79+8LAIRx48YVum7ixIlq6xYsWCAAEC5fvlzibSlFRERoLLdOnTqCn5+fVmU1bdpUACDMmjVLTMvJyRF8fHwEOzs7ITc3VxCEf49fpUqVhOfPn4t5c3NzBTs7O6FmzZrCixcvxPStW7cKAISxY8eq1fvLL78U0xQKhRAaGioYGhoK9+/fFwRBEA4cOCAAENatW6dS17i4OJX03377TQAgHDt2TKt9frkuxf00bdpU67IFQRB69+4tSKVS4erVqyrpYWFhwrRp04Tff/9dWLFihdC4cWMBgDBixAiVfBYWFoK1tbVgZGQkfPfdd8KmTZuEbt26CQCEUaNGvVadlH8/bdu2VUkfOHCgAEA4c+aMIAiCcPr0aQGAMHDgQJV8yu1rauuF+f333wUAwvTp08W0/Px8cb9XrVqlVr+XARCMjIxU/iaXLFkiABAcHByEzMxMMX306NEl+vstzXNB27ZtBQBCuXLlhO7duwubNm0SvvvuO0FfX19o2LChoFAoxLwKhUKoX7++EB4eLgiCIKSkpAgAhBkzZhS5DSJ6f7GniIjK1Pnz59GlSxd4eHhgx44dsLCwKPYztWvXRnx8fInKd3BweNMqAgBevHgBoOAu8quMjY1V8ryOV2csa9y4MdasWaN1Ofr6+ujXr5+4bGhoiH79+mHAgAE4ceIEGjRoIK6LiIhQeW7i+PHjyMjIwPjx48V9AoDQ0FBUr14d27Ztw4QJE1S2FxUVJf5fOTxq27ZtSEhIQNeuXbFx40ZYWlqiZcuWKtOv+/n5QSaTYe/evejWrZvYA7V161bUrl0bBgYGJd7nESNGlGj4krW1dYnLVPr555+xYsUKjBgxAlWqVFFZt2XLFpXlnj17onXr1pg9eza+/PJLODs7AyjoyVIoFJg6dSpGjhwJAOjYsSMePnyImJgYfPPNN6/VwwhA7HFR+vLLL7Fw4UJs374d3t7e2L59OwDgq6++UskXHR2Nn3/+Wattbd++Hfr6+iq9l1KpFF9++SUOHDhQojJatGihMlzV398fQMHxePkYKNNv3LhR5PDW0jwXZGVlAQDq1auHtWvXivUyNTXF6NGjsXv3bgQFBQEAYmNjce7cOWzatKlE2yai9x+DIiIqU2FhYbC3t8fOnTtL/N4ba2tr8cvJu6IMHnJyctTWKYduleTBbE2MjY3VJniwtrZWe+amJJycnGBmZqaSppwN7ebNmypBkbu7u0q+W7duAQCqVaumVm716tVx8OBBlTQ9PT1UqlSp0G0BwLVr1/DkyRPY2dlprG9GRgYAoGnTpujYsSMmTJiAOXPmIDAwEO3bt0e3bt00BqIv8/T0hKenZ5F5XseBAwfQu3dvhISE4Icffig2v0QiwZAhQ7Bz504kJiaKgZqJiQmePXuG8PBwlfzh4eGIi4vDqVOn0KRJk9eq46uBWuXKlaGnpyce/1u3bkFPTw+VK1dWyafpd1ycW7duwdHRUe3vVJuyKlasqLKsHM7o4uKiMb24v4HSPBco/35f/T1169YNo0ePxuHDhxEUFITMzEyMHj0aw4cPV6s3Ef13MSgiojLVsWNHrF69GuvWrVPp4ShKbm4uHj58WKK8tra2kEqlb1JFAAXPIxgZGSE1NVVtnTLNycnptcoujfq9jtcN4rShUChgZ2eHdevWaVyvDAYlEgk2bdqEI0eO4M8//8TOnTvRq1cvzJo1C0eOHCkyYH7y5EmJeukMDQ3VngspzJkzZ9C2bVvUrFkTmzZtKvFMasovyS+3TycnJ1y7dg329vYqeZWB4usEv4V5359tKaytF5YuFDMhRWmeC5R/v8X9nmbOnInc3Fx06dJFDD7//vtvMc/Nmzfh5OQEQ0PDEtWLiN4PDIqIqEzNmDED+vr6GDhwIMzNzdGtW7diP3P48GE0a9asROUrp9t+U3p6eqhVqxaOHz+utu6vv/5CpUqVXnsIVGm6d+8enj17ptJbdPXqVQDFz7Ln6uoKALhy5QqaN2+usu7KlSvieiWFQoEbN26ovJfn1W1VrlwZCQkJaNSoUYmCsAYNGqBBgwb44Ycf8PPPP6N79+745Zdf1KbCftngwYOxevXqYstu2rQpEhMTi82XnJyMVq1awc7ODtu3by9xDyYA8UW+L/f8+fn54dq1a7h7965Kz9q9e/fU8mrr2rVrKj1+169fh0KhEI+/q6srFAoFkpOTVXp0rly5ovW2XF1dsXv3bmRlZakck9cpq7SU5rnAz88Py5YtU5tp8dXf0+3bt/Ho0SN4eXmplTF58mRMnjwZp06dgo+PT8l2gojeCwyKiKhMSSQSLF26FE+fPkVERARkMhnatm1b5GfK4pkiAOjUqRNGjRqF48ePo27dugAKvhDu2bMHw4YNK7XtvIn8/HwsWbIEQ4cOBVBwJ33JkiWwtbWFn59fkZ+tW7cu7OzssHjxYvTq1UsctrZjxw5cunRJ43tY5s+fjx9//BFAwV39+fPnw8DAAC1atABQMDvawoULMWnSJLX3OOXn5yMrKwtWVlZ49OgRrKysVHo6lF8qNQ1ZfFlpPlOUlpaG4OBg6OnpYefOnYUGLA8fPoSlpaVKz0NeXh6mTp0KQ0NDlS/qXbp0wS+//IIVK1aIw/AUCgVWrVoFGxubYn8vRVmwYAGCg4PF5Xnz5gEoeH+W8t9vvvkGP/74IxYsWCDmmzt3rtbbatOmDZYuXYpFixZh+PDhAApmIFRusyyU5rmgXbt2GDx4MFatWoXIyEhxprrly5cDAFq2bAmg4Pms9u3bq3w2IyMD/fr1Q2RkJNq1a6c2NJWI3n8MioiozOnp6WHt2rVo3749OnfujO3bt6v1VLystJ8pWrNmDW7duoXnz58DAPbv34/vv/8eANCjRw+xh2TgwIFYtmwZQkNDMWzYMBgYGGD27Nmwt7fH119/rVJmYGAg9u3b98bvo9GWk5MTpk2bhps3b6Jq1apYv349Tp8+jaVLlxY7eYGBgQGmTZuGnj17omnTpggPDxen5HZzc8OQIUNU8hsbGyMuLg4RERHw9/fHjh07sG3bNnzzzTdiMNG0aVP069cPU6ZMwenTpxEcHAwDAwNcu3YNGzduRExMDDp16oTVq1dj4cKF+OSTT1C5cmU8ffoUy5Ytg4WFBdq0aVNkvUvzmaJWrVrhxo0bGDFiBA4ePKjyHJW9vb34xXjLli34/vvv0alTJ7i7u+Phw4f4+eefcf78eUyePFnlC3i7du3QokULTJkyBQ8ePEDt2rXx+++/4+DBg1iyZInKM1ORkZFYvXp1iXs4U1JS0LZtW7Rq1QpJSUni1Nu1a9cGUBBYhoeHY+HChXjy5AkaNmyI3bt34/r161ofm7CwMDRq1AijRo3CzZs34enpic2bN5d4Suy3oTTPBQ4ODvj2228xduxYtGrVCu3bt8eZM2ewbNkyhIeHi+8n8/X1ha+vr8pnlcPovLy81AImIvqPKOPZ74hIR708JbfS8+fPhaZNmwoymUw4cuTIO6uLciprTT979+5VyXvnzh2hU6dOgoWFhSCTyYSPP/5YuHbtmlqZfn5+goODQ7HbjoiIEMzMzNTSNU1pXJL98PLyEo4fPy4EBAQIxsbGgqurqzB//nyVfMppjDdu3KixnPXr1wt16tQRjIyMBBsbG6F79+7C33//rbHeycnJQnBwsGBqairY29sL48aNE+RyuVqZS5cuFfz8/AQTExPB3NxcqFWrljBixAjh3r17giAIwsmTJ4Xw8HChYsWKgpGRkWBnZyd8/PHHwvHjx7U6Bm+qsHaAV6b0Pn78uBAWFiZUqFBBMDQ0FGQymfDRRx8JGzZs0Fju06dPhcGDBwsODg6CoaGhUKtWLWHt2rVq+Tp27CiYmJgIjx49KrKeyvZx8eJFoVOnToK5ublgbW0tREVFqUynLgiC8OLFC+Grr74SypUrJ5iZmQlhYWHCnTt3tJ6SWxAE4Z9//hF69OghWFhYCJaWlkKPHj2EU6dOlXhK7kGDBqmkFTaVdXFt9G1RKBTCvHnzhKpVqwoGBgaCi4uLMGbMGHE6+8JwSm6i/z6JILzj25hERB+4p0+fwsbGBnPnzlWbMvltCgwMxIMHD3D+/Pl3tk0qXfb29vj8888xY8aMIvONHz8eEyZMwP3794t80XFRJBIJxo0bh/Hjx7/W54mIPiRv/pp3IiJSsX//flSoUAF9+/Yt66rQf8iFCxfw4sUL8V1GRET07vCZIiKiUhYaGorQ0NBSK+/hw4fIzc0tdL1UKn2jGczo/eDl5YXMzMx3vl25XI779+8XmUcmk2k1Cx8R0X8NgyIiovdchw4dsG/fvkLXu7q6ig96E2nrzp07xc6WxmF2RPSh4zNFRETvuRMnThT5gk8TExM0atToHdaIPiTZ2dkqs+xpUqlSJZV3LBERfWgYFBERERERkU7jRAtERERERKTTGBQREREREZFOY1BEREREREQ6jUERERERERHpNAZFRERERESk0/ieolKiUChw7949mJubQyKRlHV1iIiIiIh0miAIePr0KZycnKCnV3RfEIOiUnLv3j24uLiUdTWIiIiIiOgld+7cgbOzc5F5GBSVEnNzcwAFB93CwkJlXV5eHnbt2oXg4GAYGBiURfXoP4ZthrTFNkPaYpshbbHNkLbKus1kZmbCxcVF/J5eFAZFpUQ5ZM7CwkJjUGRqagoLCwueRKhE2GZIW2wzpC22GdIW2wxp631pMyV5tIUTLRARERERkU5jTxEVKjtPjm9/O4/zd5/g+v0sNK9uh2Wf11XLl5Mvx4+7r+H3U/dw/2kObM2NMLhFFXSup/qM1dyEq7j54Bnmdq2Dn/+6jT9O38WFe5nIysnHmXHBsDRRv4Ow53I6YnZfx+XUTBjp68G/UjmNdSAiIiIiel0MiqhQCkGAsYEeIhu5Ycf5tELzDVp3Cg+ycjCtozdcy5ki42kOBEFQyxd/MR0DAisDAF7kydG0mi2aVrPF9LgrGsvdcS4Vozafw/CQamhYuRzkCgFX0p+Wzs4REREREf0/BkVUKFNDffzwSS0AwPGbj5CZnaeWJ/FKBv5K+QcHRjSDlakhAMDFxlQt373HL3AtPQtNq9oCAHp/5A4ASEr+R+O28+UKTPjzIr5pUx1d6lUU06vYF/+gHBEREb0f5HI58vLUvz+QbsjLy4O+vj6ys7Mhl8tLvXwDAwNIpdJSKYtBEb2RhEvp8Ha2xOJ9N/Dbqb9haqiPoBp2+Dq4GowNpCr5/CvZwNy4ZA/Znb+XibTMbEgkErSJOYD7WTnwdLTAN21qoJoDAyMiIqL3mSAISEtLw+PHj8u6KlSGBEGAg4MD7ty589be42llZQUHB4c3Lp9BEb2R2w9f4NjNRzDSl2JJj7p49CwXY34/j0fP8zDz09pivviL6Wjpaa9Fuc8BADEJ1zAmtAacrU2x7MANdF2ahL3DAsVeKSIiInr/KAMiOzs7mJqa8sX2OkqhUCArKwsymazYl6dqSxAEPH/+HBkZGQAAR0fHNyqPQRG9EUEQIAEwt6sPLP6/F+i7j2tgwLqT+L59TRgbSPE0Ow9/3XiIaR29tSoXAAY180DrWgWNfMan3giYsgfbzqWiu79rqe8LERERvTm5XC4GROXKlSvr6lAZUigUyM3NhbGxcakHRQBgYmICAMjIyICdnd0bDaXjlNz0RmzNjeBgaSwGRADgYSeDIACpT7IBAIlX7sPDTgYnKxOtygWAKvYyMc1IXwoXG1Pce/yilGpPREREpU35DJGpqfozxkSlTdnO3vTZNQZF9EbqutogPTMbz3LyxbQb959BTwI4WhoD0H7oHADUqmAJQ3093LifJablyRW4++g5KljxJEtERPS+45A5ehdKq51x+JyOkisEHE15iIyn2bAzN0Z9dxtI9dQb1bX0p8iVK/DkRS6ycvJx4d4TAICXkyUAoJ2PE+btuYbhm85gSFBVPHyWiyk7LqNzXRcYG0iRL1cg8UoGvmjSQKXcjKfZuP80B7f+eQYAuJL2FGZGUlSwMoGVqSHMjQ3Q3b8i5sRfg6OlCSpYm2DpvhsAgNBabzZmlIiIiIjoZQyKdFDc+VRM+POiOLwNKOjVGRfmiVY1VQOOyFXHcPel4WqhPx4EANycGgoAMDPSx5re/hi/5QLC5h+EtakhQms5YlhINQDAXykPYWakj5oVLFXKXXfkNmJ2XxOXOy9JAgDM6OSNT+sWvPT1mzY1oK8nwdANp5Gdp4CPixV+7tsAlqYlm8GOiKg0JN/Pwre/ncP1jCxkZufD3sII7WpXwOCgKjCQFgy4+N/R29h88m9cSSt4l1otZ0sMD6kOHxcrtfK6Lk1Ce58K6Fq/IsZvuYDjtx7ialoWKtvJsGNwY7X8giBg2YEb+N/RO7j76AWszQzQo4EroppXeav7TUSkSxgU6Zi486kYsPYkXn21atqTbAxYexKLPvNVCYwOjWpebJkedjKs7eOvcV38xXS0qGGnlj6kZVUMaVm1yHINpHr4NtQT34Z6FlsHIqK3xUBPDx18nVHTyRIWJvq4lPoUozefhUIQMKJVdQDAkRv/oG1tJ/i2tYaRvhSL9yWjx4q/ED+kKRz+fygxADx+nosTtx5hXrivmNa5rgtO336MS2maX0494c+L2H/tPr5pUwPVHczx+HkeHr/Ifbs7TfSeKOnIlrImCAL69euHTZs24dGjRzh16hSio6Ph4+ODuXPnlnX1Siw2NhbR0dE6OZU6gyIdIlcImPDnRbWACAAEABIUXHxbejqU2gmnqr05fF2tSqUsIqKyULGcKSqW+/dZRmdrUxy5UQHHbj4U02K61lH5zLSO3og7n4ZD1x+go5+zmL7ncga8nCzFyWTGt/UCAPyTdVVjUHQ94ynWHrmFnUOaoLJtwcQzLjalt29E7zNtRraUtbi4OMTGxiIxMRGVKlVC+fLlsXnzZhgYvL+jW9zc3BAdHY3o6GgxrUuXLmjTpk3ZVQrAhQsXMHbsWJw4cQK3bt3CnDlzVOr4tnCiBR1yNOWhyonlVQIKZow7mvKw0Dza6uZfEdUdLEqtPCKisnbzwTPsu3of/u6FTzX8Ik+OPLkCVq8M9024pN3EMwmXMlDRxhR7LmXgo2l70GjqHozcdBaPn7OniD5sypEtr35vUY5siTuf+k7qkZtbsr+15ORkODo6omHDhnBwcIC+vj5sbGxgbv5uXzgvCALy8/OLz1gIExMT2Nmpj/B5l54/f45KlSph6tSpcHBweGfbZVCkQzKeFh4QvU4+IiJd0mHhIVQdswOBMxNRz80GQ4sYAjx1xyXYWxijkUd5MS0nX459V+5r/SLrvx+/wLZzqZjd2QczP62Nc3efYMDak2+0L0Tvs+JGtgAFI1vkCk053kxgYCCioqIQHR2N8uXLIyQkBABw/vx5tG7dGjKZDPb29ujRowcePHgAAIiMjMSXX36J27dvQyKRwM3NTSzr5R4ONzc3TJ48Gb169YK5uTkqVqyIpUuXqmz/zp076Ny5M6ysrGBjY4N27drh5s2bhdY3MTEREokEO3bsgJ+fH4yMjHDw4EEkJyejXbt2sLe3h0wmQ7169ZCQkKCyn7du3cKQIUMgkUjEGdxiY2NhZWWlso1FixahcuXKMDQ0RLVq1bBmzZrXPLolU69ePcyYMQNdu3aFkZHRW93WyxgU6RA7c+PiM2mRj4hIl8zv5ottX36EmK4+2Hs5A0sP3NCYb2Hidfx5JhVLevjB2ODfFwkeTv4H5WRGqGpf8jvHgiAgN1+B2Z1ro767DQIql8P0Tt5IuvEPkl96ZQHRh6QsRra8bPXq1TA0NMShQ4ewePFiPH78GM2bN0edOnVw/PhxxMXFIT09HZ07dwYAxMTEYOLEiXB2dkZqaiqOHTtWaNmzZs1C3bp1cerUKQwcOBADBgzAlStXABS8ZyckJATm5uY4cOAADh06BJlMhlatWhXbYzVq1ChMnToVly5dgre3N7KystCmTRvs3r0bp06dQqtWrRAWFobbt28DADZv3gxnZ2dMnDgRqampSE3V3PP222+/YfDgwfj6669x/vx59OvXDz179sTevXsLrcu6desgk8kgk8lgYWEBZ2dnWFhYiGkymQwHDhwocn/KAp8p0iH13W3gaGmMtCfZGu++SAA4WBY8xEhERKqUL6CuYm8OhSBg9OZz6Nu4ksozmEv3J2NRYjLW9fFHDUfVocMJF9MRVEO7d7bZmhtDX0+CSrb/vsjaw67g//cevxCfMyL6kJT1yJYqVapg+vTp4vL333+POnXqYPLkyWLaypUr4eLigqtXr6Jq1aowNzeHVCotdrhXmzZtMHDgQADAyJEjMWfOHOzduxfVqlXD+vXroVAosHz5crHnZtWqVbCyskJiYiKCg4MLLXfixIlo2bKluGxjY4PatWuLy5MmTcJvv/2GLVu2ICoqCjY2NpBKpTA3Ny+yzjNnzkRkZKRY56FDh+LIkSOYOXMmmjVrpvEzbdu2hb9/wQRcCoUCWVlZkMlk0NP7ty+mQoUKRR6nssCgSIdI9SQYF+aJAWtPQgKoBEbKS/q4MM/3clYXIqL3iUIB5MsFKAQB0v8/gy7el4wFe65jde/68Ha2UskvCAJ2X8rAnC4+Wm2nrqs18hUCbv3zDK7lzAAUvCAbACr8f5BG9KEp65Etfn5+KstnzpzB3r17IZOp34RITk5G1apFz6b7Mm9vb/H/EokEDg4OyMjIELdz/fp1teeQsrOzkZycXGS5devWVVnOysrC+PHjsW3bNqSmpiI/Px8vXrwQe4pK6tKlS/jiiy9U0ho1aoSYmJhCP2Nubi7ug0KhQGZmJiwsLFSCovcRgyId06qmIxZ95qs2m4vDezqbCxHR21bclL+/n7oLfakE1R3MYSiV4uzdx5i+8zI+9nYU31O0KDEZc+KvIqarD5ytTcQ72GaG+jAz0se5u0/wIk+Oem7WKtu++eAZnuXm435WDnLy5OILsqvYmcNQXw8feZRHzQoWGL7pLMZ+7AlBAL774zwaVymv0ntE9CEp65EtZmZmKstZWVkICwvDtGnT1PI6Omr3venV2egkEgkUCoW4HT8/P6xbt07tc7a2tlrVediwYYiPj8fMmTPh4eEBExMTdOrUqcQTR7yJdevWoV+/fkXm2bFjBxo3Vn8vW1liUKSDWtV0REtPh//EvP9ERG9TSab8lepJsHhfMlLuP4OAgh6azwPc0Psjd/Eza4/cQq5cgQHrVCdAGNyiCoa0rIr4i+loVs0W+lLVO6Ujfz2Lv156LkL5guwDI5rBxcYUenoSrIioh3F/XECXJUkwMdRHYDVbjAmtUdqHgui98b6NbPH19cWvv/4KNzc36Ou/va/Ovr6+WL9+Pezs7GBh8WYz9x46dAiRkZH45JNPABQEXK9O2GBoaAi5XF5kOTVq1MChQ4cQERGhUranZ+HvkOTwOSpz2rzgTKonQUDlwqeTJSL60JX0ZdZhtZ0QVtupyLKKe9F1/MV0RDX3UEtf3y+g2HraWxhjcQ+/YvMRfUjep5EtgwYNwrJlyxAeHo4RI0bAxsYG169fxy+//ILly5dDKpUWX0gJdO/eHTNmzEC7du3EiRtu3bqFzZs3Y8SIEXB2di6+kP9XpUoVbN68GWFhYZBIJPjuu+/EHiklNzc37N+/X5zlrXz58mrlDB8+HJ07d0adOnUQFBSEP//8E5s3b1aZye5Vbzp8Ljc3FxcvXhT/f/fuXZw+fRoymQweHurn0dJSpoP7xo8fL04DqPypXr26uD4wMFBtff/+/VXKuH37NkJDQ2Fqago7OzsMHz5cbX72xMRE+Pr6wsjICB4eHoiNjVWry4IFC+Dm5gZjY2P4+/vj6NGjb2Wf35a486n4aNoehC87gsG/nEb4siP4aNqedzaPPxHRf8m7nPI3N1+BVjUdEFitbN/9QfRf06qmIw6ObI7/9W2AmK4++F/fBjg4svk7H+rv5OSEQ4cOQS6XIzg4GLVq1UJ0dDSsrKxK9TkZU1NT7N+/HxUrVkSHDh1Qo0YN9O7dG9nZ2Vr3HM2ePRvW1tZo2LAhwsLCEBISAl9fX5U8EydOxM2bN1G5cuVCh+e1b98eMTExmDlzJry8vLBkyRKsWrUKgYGBr7ubxbp37x7q1KmDOnXqIDU1FTNnzkSdOnXQp0+ft7ZNAJAIglD6k7yX0Pjx47Fp0yaVaFNfX1+MVAMDA1G1alVMnDhRXG9qaio2DLlcDh8fHzg4OGDGjBlITU3F559/jr59+4ozhKSkpKBmzZro378/+vTpg927dyM6Ohrbtm0T555fv349Pv/8cyxevBj+/v6YO3cuNm7ciCtXrpT4BVaZmZmwtLTEkydP1BpuXl4etm/fjjZt2ryVNxsXdrdT2UekvNtJ/x1vu83Qh4dtRjtJyf8gfNmRYvP9r2+DD7ZXnW2GtFXSNpOdnY2UlBS4u7vD2Jiv+dBl72KihaLaW1Hfz19V5sPn9PX1i5wK0NTUtND1u3btwsWLF5GQkAB7e3v4+Phg0qRJGDlyJMaPHw9DQ0MsXrwY7u7umDVrFoCCsZEHDx7EnDlzxKBo9uzZ6Nu3L3r27AkAWLx4MbZt24aVK1di1KhRGredk5ODnJwccTkzMxNAwQkjLy9PJa9y+dX00iBXCBi/5UKhdzslACb8eQGBVcrxmaH/kLfZZujDxDajndTHz0qcLy/vzcb2v6/YZkhbJW0zeXl5EAQBCoVCbcgW6RZl34uyPbwNCoUCgiAgLy9PbSijNue3Mg+Krl27BicnJxgbGyMgIABTpkxBxYoVxfXr1q3D2rVr4eDggLCwMHz33XcwNTUFACQlJaFWrVqwt//3vQ8hISEYMGAALly4gDp16iApKQlBQUEq2wwJCRHfMJybm4sTJ05g9OjR4no9PT0EBQUhKSmp0HpPmTIFEyZMUEvftWuXWL9XxcfHF39AtHTtiQRpmYWPZS14wVkO5q+PQxXLMusUpNf0NtoMfdjYZkrmxhMJgOKfA7hx4TS2/33q7VeoDLHNkLaKazPKG95ZWVnvZLYzev89ffr0rZWdm5uLFy9eYP/+/WqP0Dx//rzE5ZRpUOTv74/Y2FhUq1YNqampmDBhAho3bozz58/D3Nwc3bp1g6urK5ycnHD27FmMHDkSV65cwebNmwEAaWlpKgERAHE5LS2tyDyZmZl48eIFHj16BLlcrjHP5cuXC6376NGjMXToUHE5MzMTLi4uCA4O1jh8Lj4+Hi1btiz1IQp/nk0FLp4rNl8lLx+08eYQuv+Kt9lm6MPENqMduULApln7kZ6ZU8SUv0aI6tLkg+1lZ5shbZW0zWRnZ+POnTuQyWQcPqfjBEHA06dPYW5uLr6QtrRlZ2fDxMQETZo00Th8rqTKNChq3bq1+H9vb2/4+/vD1dUVGzZsQO/evVVeFlWrVi04OjqiRYsWSE5ORuXKlcuiyiIjIyMYGRmppRsYGBR6oihq3etytDIrPtP/5+NF77/nbbQZ+rCxzRQobjZOAwDj23oVM+WvF4yNDN9hrcsG2wxpq7g2I5fLIZFIoKen996/sJPeLuWQOWV7eBv09PQgkUg0tkttzm1lPnzuZVZWVqhatSquX7+ucb1yzvPr16+jcuXKcHBwUJslLj09HQDE55AcHBzEtJfzWFhYwMTEBFKpFFKpVGOeop51el+U9QvOiIjeNyV59xDwfk35S0REZeu9Ct+zsrKQnJxc6NuBT58+DeDftwcHBATg3LlzyMjIEPPEx8fDwsJCfKlUQEAAdu/erVJOfHw8AgIK3g1haGgIPz8/lTwKhQK7d+8W87zPlC84A/69u6lUFi84IyIqS8rZOF8OcoB/3z306msK3pcpf4mIqGyVaU/RsGHDEBYWBldXV9y7dw/jxo2DVCpFeHg4kpOT8fPPP6NNmzYoV64czp49iyFDhqBJkybw9vYGAAQHB8PT0xM9evTA9OnTkZaWhjFjxmDQoEHi0Lb+/ftj/vz5GDFiBHr16oU9e/Zgw4YN2LZtm1iPoUOHIiIiAnXr1kX9+vUxd+5cPHv2TJyN7n1X0rud2XlyfPvbeZy/+wTX72eheXU7LPu8rlp5Ofly/Lj7Gn4/dQ/3n+bA1twIg1tUQed6Lir55iZcxc0HzzC3ax1kPM3GlO2XceDaAzzLyUclWzNENfNA61r8YkFE70Zx7x4qmI3zIlp6OqjcKOLLrImIqEyDor///hvh4eH4559/YGtri48++ghHjhyBra0tsrOzkZCQIAYoLi4u6NixI8aMGSN+XiqVYuvWrRgwYAACAgJgZmaGiIgIlfcaubu7Y9u2bRgyZAhiYmLg7OyM5cuXi9NxA0CXLl1w//59jB07FmlpafDx8UFcXJza5Avvs1Y1HdHS06HIMfQKQYCxgR4iG7lhx/m0QssatO4UHmTlYFpHb7iWM0XG0xxoep1V/MV0DAgseLbr6w1nkPkiD8sj6sLG1BB/nL6LQT+fxJaoj1CzgmXp7zAR0SuOpjxU6yF6WcFsnNk4mvKQQRAREako06Dol19+KXSdi4sL9u3bV2wZrq6u2L59e5F5AgMDcepU0VOqRkVFISoqqtjtvc+Ku9tpaqiPHz6pBQA4fvMRMrPV525PvJKBv1L+wYERzWBlWvCAsYuN+hTj9x6/wLX0LDStWvAG5BO3HuH79jXh42IFAPiyRRWsOJSC83efMCgionci42nhAdHr5CMiKilBENCvXz9s2rQJjx49wqlTpxAdHQ0fHx/MnTu3rKtXYrGxsYiOjsbjx4/Luirv3Hv1TBGVvYRL6fB2tsTifTfgPzkBzWYm4odtF5GdJ1fL51/JBubGBbN6+LlaY+vZVDx+nguFQsCWM/eQk6dAg0q8G0tE74adecmm/i1pPiJ6DyjkQMoB4Nymgn8V8uI/Uwbi4uIQGxuLrVu3IjU1FTVr1sTmzZsxadKksq5aodzc3NQCti5duuDq1atlU6H/t2zZMjRu3BjW1tawtrZGUFCQ2sRqb8N7Nfsclb3bD1/g2M1HMNKXYkmPunj0LBdjfj+PR8/zMPPT2mK++IvpaOn57/DC+d18EfXzSfhMjIe+ngQmBlIs6eEHt/IlmzKciOhNcTZOog/MxS1A3Egg896/aRZOQKtpgGfbd1KF3NxcGBoWPzW/cqKwhg0bimk2Nu/+XCMIAuRyOfT1X+8rvomJCUxMTEq5VtpJTExEeHg4GjZsCGNjY0ybNg3BwcG4cOECKlSo8Na2y54iUiEIAiQA5nb1gY+LFZpVt8N3H9fAryf/FnuLnmbn4a8bDxFU49+gaPauK8jMzse6Pv7YEvURejd2x6CfT+JyWslfmkVE9CY4GyfRB+TiFmDD56oBEQBkphakX9zyVjYbGBiIqKgoREdHo3z58uIz6OfPn0fr1q0hk8lgb2+PHj164MGDBwCAyMhIfPnll7h9+zYkEgnc3NzEsqKjo8Wy3dzcMHnyZPTq1Qvm5uaoWLEili5dqrL9O3fuoHPnzrCysoKNjQ3atWuHmzdvFlrfxMRESCQS7NixA35+fjAyMsLBgweRnJyMdu3awd7eHjKZDPXq1UNCQoLKft66dQtDhgyBRCIRX6waGxsLKysrlW0sWrQIlStXhqGhIapVq4Y1a9a85tEtmXXr1mHgwIHw8fFB9erVsXz5cnFm6LeJQRGpsDU3goOlMSyM/33ZlYedDIIA8QHmxCv34WEng5NVwZ2EW/88w+qkW5jRyRuNPMrD08kC0UFV4e1siZ+SbpXJfhCRblLOxulgqTpEzsHSGIs+8xVn40y+n4WuS5NQ9/t4VB2zA42n78HMnVeQJ1eIn/nf0dv4dPFheI/fCe/xO9F9+RGcvvNY43a7Lk3CL0dvAwDO3HmMbsuOoNb/f67Hir9w8R5vEBGVmEJe0ENU6FySAOJGvbWhdKtXr4ahoSEOHTqExYsX4/Hjx2jevDnq1KmD48ePIy4uDunp6ejcuTMAICYmBhMnToSzszNSU1Nx7NixQsueNWsW6tati1OnTmHgwIEYMGAArly5AgDIy8tDSEgIzM3NceDAARw6dAgymQytWrVCbm5ukXUeNWoUpk6dikuXLsHb2xtZWVlo06YNdu/ejVOnTqFVq1YICwvD7dsF56nNmzfD2dkZEydORGpqKlJTUzWW+9tvv2Hw4MH4+uuvcf78efTr1w89e/bE3r17C63LunXrIJPJIJPJYGFhAWdnZ1hYWIhpMpkMBw4cKHJ/Xvb8+XPk5eW99Z43Dp8jFXVdbbD9XCqe5eTDzKigedy4/wx6koKXHwLqQ+de/H8P0qs3X/UkEo2z1hERvU0lmY3TQE8PHXydUdPJEhYm+riU+hSjN5+FQhAwolV1AMCRG/+gbW0n+La1hpG+FIv3JaPHir8QP6SpStD1+HkuTtx6hHnhvniWk4+IVUcRVMMek9rXhFwhYE78VXy+8iiSRjeHgZT3IomKdeuweg+RCgHIvFuQz71xqW++SpUqmD59urj8/fffo06dOpg8ebKYtnLlSri4uODq1auoWrUqzM3NIZVK4eDgUGTZbdq0wcCBAwEAI0eOxJw5c7B3715Uq1YN69evh0KhwPLly8Wem1WrVsHKygqJiYkIDg4utNyJEyeiZcuW4rKNjQ1q1/73sYdJkybht99+w5YtWxAVFQUbGxtIpVKYm5sXWeeZM2ciMjJSrPPQoUNx5MgRzJw5E82aNdP4mbZt28Lf3x9Awbs/s7KyIJPJoKf37/lPm2FwI0eOhJOTE4KCgkr8mdfBoEjHXEt/ily5Ak9e5CIrJx8X7j0BAHg5FcwQ187HCfP2XMPwTWcwJKgqHj7LxZQdl9G5rguMDaTIlyuQeCUDXzRpIJZZ2VYGt3Km+GbzeXwTWgPWpgbYdSEdB68/wMqIemWyn0Sk24qbjbNiOVNULPfvzJrO1qY4cqMCjt18KKbFdK2j8plpHb0Rdz4Nh64/QEc/ZzF9z+UMeDlZwtbcCGf/fozHz/MwtGVVsTd9cFAVtJp7AHcfveBzlkQlkZVeuvm05Ofnp7J85swZ7N27FzKZTC1vcnIyqlatWuKyle/aBACJRAIHBwdkZGSI27l+/TrMzc1VPpOdnY3k5OQiy61bV/W9k1lZWRg/fjy2bduG1NRU5Ofn48WLF2JPUUldunQJX3zxhUpao0aNEBMTU+hnzM3NxX1QKBTIzMyEhYWFSlBUUlOnTsUvv/yCxMREGBu/3UlyGBTpmMhVx3D38QtxOfTHgwCAm1NDAQBmRvpY09sf47dcQNj8g7A2NURoLUcMC6kGAPgr5SHMjPRVptk2kOphVc/6mLbjMvqsPoZnOXK4ljPFrE9ro1l1u3e4d0REr+fmg2fYd/U+WnkVfsf0RZ4ceXIFrEwNVNITLv3be17JVgZrUwOsP3YHg5p5QCEIWH/sDjzsZHC2LtuHl4n+M2QlfE9kSfNpycxM9eZFVlYWwsLCMG3aNLW8jo7avaTewED1/CGRSKBQKMTt+Pn5Yd26dWqfs7W11arOw4YNQ3x8PGbOnAkPDw+YmJigU6dOxQ7DKw3r1q1Dv379isyzY8cONG5cdC/fzJkzMXXqVCQkJKgEk28LgyIdc2hU82LzeNjJsLaPv8Z18RfT0aKGeqDjXt4Mi3v4afgEEdH7q8PCQzh/LxO5+QqE16+IoS0Lv+M7dccl2FsYo5FHeTEtJ1+OfVfuIzqo4HMyI3388kUAvlhzHPP2XAMAuJU3w0+96kOfQ+eISsa1YcEsc5mp0PxckaRgvWtDDetKn6+vL3799Ve4ubm99qxuJd3O+vXrYWdnBwsLizcq69ChQ4iMjMQnn3wCoCDgenXCBkNDQ8jlRT+XVaNGDRw6dAgREREqZXt6ehb6mdIYPjd9+nT88MMP2Llzp1ov2NvCMzRppaq9OT5r4FrW1SAiKhXzu/li25cfIaarD/ZezsDSAzc05luYeB1/nknFkh5+MDaQiumHk/9BOZkRqtoXDBXJzpNjxK9n4edqjd8GNsKmAQ1Rzd4cvWKPqb3vjYgKoSctmHYbQKFzSbaaWpDvHRg0aBAePnyI8PBwHDt2DMnJydi5cyd69uxZbFChje7du6N8+fJo164dDhw4gJSUFCQmJuKrr77C33//rVVZVapUwebNm3H69GmcOXMG3bp1E3uklNzc3LB//37cvXtXnEnvVcOHD0dsbCwWLVqEa9euYfbs2di8eTOGDRtW6LbNzc3h4eEh/lSqVEllWdlzVZhp06bhu+++w8qVK+Hm5oa0tDSkpaUhKytLq2OgLQZFpJVu/hVR3eHN7l4QEb0vnKxMUMXeHO18KmBk62qYm3AVcoXqneml+5OxKDEZa3rXRw1H1fNfwsV0ldcT/HH6Lu4+eo6ZnWqjtosVfCtaI6ZrHdx5+AK7Lr6d5x+IPkiebYHOPwEWrwxPs3AqSH9H7ykCACcnJxw6dAhyuRzBwcGoVasWoqOjYWVl9VrPyRTG1NQU+/fvR8WKFdGhQwfUqFEDvXv3RnZ2ttY9R7Nnz4a1tTUaNmyIsLAwhISEwNfXVyXPxIkTcfPmTVSuXLnQ4Xnt27dHTEwMZs6cCS8vLyxZsgSrVq1CYGDg6+5msRYtWoTc3Fx06tQJjo6O4s/MmTPf2jYBDp8jIiICACgUQL5cgEIQIP3/u9GL9yVjwZ7rWN27PrydrVTyC4KA3ZcyMKeLj5j2Ilf+/+/8+DefngSQSMDZOIm05dkWqB5aMMtcVnrBM0SuDd9qD1FiYqLGdGXPS2Gio6NV3kmkqSxN7xs6ffq0yrKDgwNWr15dgpoWCAwM1HhucXNzw549e1TSBg0apLLcoEEDnDlzRiUtMjISkZGRKmkDBgzAgAEDSlynN1XUe5neJgZFRESkc34/dRf6UgmqO5jDUCrF2buPMX3nZXzs7ShOm70oMRlz4q8ipqsPnK1NkPG04F1tZob6MDPSx7m7T/AiT456btZiuR9VscXkHZfx3R/nEdnQDQqhoBypngQBlQqfDY+ICqEnfSvTbhO9ikERERHpHKmeBIv3JSPl/jMIACpYmeDzADf0/shdzLP2yC3kyhUYsO6kymcHt6iCIS2rIv5iOppVs1WZQMHDToYVEXURk3ANnyw8DD2JBF5OFljdqz7sLN7udLJERPT6GBQREZHOCavthLDaTkXmKW62zviL6Yhq7qGW3riKLRpXKXr6XCIier9wogUiIiIt5eYr0KqmAwKr8V1sREQfAvYUERERaclQX098NxEREf33saeIiIiIiIh0GoMiIiIiIiLSaQyKiIiIiIhIpzEoIiIiIiIincagiIiIiIjoDQiCgC+++AI2NjaQSCQ4ffo0AgMDER0dXdZV00psbCysrKzKuhplgkEREREREb2X5Ao5jqUdw/Yb23Es7RjkCnlZV0mjuLg4xMbGYuvWrUhNTUXNmjWxefNmTJo0qayrVig3NzfMnTtXJa1Lly64evVq2VRIg19++QUSiQTt27d/69vilNxERERE9N5JuJWAqUenIv15uphmb2qPUfVHIcg16J3UITc3F4aGhsXmS05OhqOjIxo2bCim2djYvM2qaSQIAuRyOfT1X+8rvomJCUxMTEq5Vq/n5s2bGDZsGBo3bvxOtseeIiIiIiJ6ryTcSsDQxKEqAREAZDzPwNDEoUi4lfBWthsYGIioqChER0ejfPnyCAkJAQCcP38erVu3hkwmg729PXr06IEHDx4AACIjI/Hll1/i9u3bkEgkcHNzE8t6eficm5sbJk+ejF69esHc3BwVK1bE0qVLVbZ/584ddO7cGVZWVrCxsUG7du1w8+bNQuubmJgIiUSCHTt2wM/PD0ZGRjh48CCSk5PRrl072NvbQyaToV69ekhI+PeYBQYG4tatWxgyZAgkEgkkEgkAzcPnFi1ahMqVK8PQ0BDVqlXDmjVrXvPolpxcLkf37t0xYcIEVKpU6a1vD2BQRERERETvEblCjqlHp0KAoLZOmTbt6LS3NpRu9erVMDQ0xKFDh7B48WI8fvwYzZs3R506dXD8+HHExcUhPT0dnTt3BgDExMRg4sSJcHZ2RmpqKo4dO1Zo2bNmzULdunVx6tQpDBw4EAMGDMCVK1cAAHl5eQgJCYG5uTkOHDiAQ4cOQSaToVWrVsjNzS2yzqNGjcLUqVNx6dIleHt7IysrC23atMHu3btx6tQptGrVCmFhYbh9+zYAYPPmzXB2dsbEiRORmpqK1NRUjeX+9ttvGDx4ML7++mucP38e/fr1Q8+ePbF3795C67Ju3TrIZDLIZDJYWFjA2dkZFhYWYppMJsOBAweK3J+JEyfCzs4OvXv3LjJfaeLwOSIiIiJ6b5zMOKnWQ/QyAQLSnqfhZMZJ1HOoV+rbr1KlCqZPny4uf//996hTpw4mT54spq1cuRIuLi64evUqqlatCnNzc0ilUjg4OBRZdps2bTBw4EAAwMiRIzFnzhzs3bsX1apVw/r166FQKLB8+XKx52bVqlWwsrJCYmIigoODCy134sSJaNmypbhsY2OD2rVri8uTJk3Cb7/9hi1btiAqKgo2NjaQSqUwNzcvss4zZ85EZGSkWOehQ4fiyJEjmDlzJpo1a6bxM23btoW/vz8AQKFQICsrCzKZDHp6//bFVKhQodBtHjx4ECtWrMDp06cLzfM2MCgiIiIiovfG/ef3SzWftvz8/FSWz5w5g71790Imk6nlTU5ORtWqVUtctre3t/h/iUQCBwcHZGRkiNu5fv06zM3NVT6TnZ2N5OTkIsutW7euynJWVhbGjx+Pbdu2ITU1Ffn5+Xjx4oXYU1RSly5dwhdffKGS1qhRI8TExBT6GXNzc3EfFAoFMjMzYWFhoRIUFebp06fo0aMHli1bhvLly2tV1zfFoIiIiIiI3hu2pralmk9bZmZmKstZWVkICwvDtGnT1PI6OjpqVbaBgYHKskQigUKhELfj5+eHdevWqX3O1rbofX21zsOGDUN8fDxmzpwJDw8PmJiYoFOnTsUOwysN69atQ79+/YrMs2PHDo0TKCQnJ+PmzZsICwsT05THR19fH1euXEHlypVLt8L/j0EREREREb03fO18YW9qj4znGRqfK5JAAntTe/ja+b6b+vj64tdff4Wbm9trz+pW0u2sX78ednZ2sLCweKOyDh06hMjISHzyyScACgKuVydsMDQ0hFxe9HNZNWrUwKFDhxAREaFStqenZ6GfeZPhc9WrV8e5c+dU0saMGYOnT58iJiYGLi4uRdb3TXCiBSIiIiJ6b0j1pBhVfxSAggDoZcrlkfVHQqonfSf1GTRoEB4+fIjw8HAcO3YMycnJ2LlzJ3r27FlsUKGN7t27o3z58mjXrh0OHDiAlJQUJCYm4quvvsLff/+tVVlVqlTB5s2bcfr0aZw5cwbdunUTe1yU3NzcsH//fty9e1ecSe9Vw4cPR2xsLBYtWoRr165h9uzZ2Lx5M4YNG1bots3NzeHh4SH+VKpUSWVZ2XOlibGxMWrWrKnyY2VlBXNzc9SsWbNE06O/LgZFRERERPReCXINwuzA2bAztVNJtze1x+zA2e/sPUUA4OTkhEOHDkEulyM4OBi1atVCdHQ0rKysSvScTEmZmppi//79qFixIjp06IAaNWqgd+/eyM7O1rrnaPbs2bC2tkbDhg0RFhaGkJAQ+Pqq9qxNnDgRN2/eROXKlQsdnte+fXvExMRg5syZ8PLywpIlS7Bq1SoEBga+7m6+tySCIKj3S5LWMjMzYWlpiSdPnqg13Ly8PGzfvh1t2rRRG0tKpAnbDGmLbYa0xTZD2ippm8nOzkZKSgrc3d1hbGz8RtuUK+Q4mXES95/fh62pLXztfN9ZDxG9OW0nWngdRbW3or6fv4rPFBERERHRe0mqJ30r024TvYrD54iIiIiISKcxKCIiIiIiIp3GoIiIiIiIiHQagyIiIiIiKnWcy4vehdJqZwyKiIiIiKjUKGeme/78eRnXhHSBsp296SyanH2OiIiIiEqNVCqFlZUVMjIyABS8f0cikRTzKfoQKRQK5ObmIjs7u9Sn5BYEAc+fP0dGRgasrKwglb7ZVO0MioiIiIioVDk4OACAGBiRbhIEAS9evICJiclbC4ytrKzE9vYmyjQoGj9+PCZMmKCSVq1aNVy+fBlAwcuYvv76a/zyyy/IyclBSEgIFi5cCHt7ezH/7du3MWDAAOzduxcymQwRERGYMmUK9PX/3bXExEQMHToUFy5cgIuLC8aMGYPIyEiV7S5YsAAzZsxAWloaateujXnz5qF+/fpvb+eJiIiIPlASiQSOjo6ws7NDXl5eWVeHykheXh7279+PJk2avJWXRBsYGLxxD5FSmfcUeXl5ISEhQVx+OZgZMmQItm3bho0bN8LS0hJRUVHo0KEDDh06BACQy+UIDQ2Fg4MDDh8+jNTUVHz++ecwMDDA5MmTAQApKSkIDQ1F//79sW7dOuzevRt9+vSBo6MjQkJCAADr16/H0KFDsXjxYvj7+2Pu3LkICQnBlStXYGdn9w6PBhEREdGHQyqVltqXVvrvkUqlyM/Ph7Gx8VsJikpTmU+0oK+vDwcHB/GnfPnyAIAnT55gxYoVmD17Npo3bw4/Pz+sWrUKhw8fxpEjRwAAu3btwsWLF7F27Vr4+PigdevWmDRpEhYsWIDc3FwAwOLFi+Hu7o5Zs2ahRo0aiIqKQqdOnTBnzhyxDrNnz0bfvn3Rs2dPeHp6YvHixTA1NcXKlSvf/QEhIiIiIqJ3qsx7iq5duwYnJycYGxsjICAAU6ZMQcWKFXHixAnk5eUhKChIzFu9enVUrFgRSUlJaNCgAZKSklCrVi2V4XQhISEYMGAALly4gDp16iApKUmlDGWe6OhoAEBubi5OnDiB0aNHi+v19PQQFBSEpKSkQuudk5ODnJwccTkzMxNAQTfhq93EymV2H1NJsc2QtthmSFtsM6QtthnSVlm3GW22W6ZBkb+/P2JjY1GtWjWkpqZiwoQJaNy4Mc6fP4+0tDQYGhrCyspK5TP29vZIS0sDAKSlpakERMr1ynVF5cnMzMSLFy/w6NEjyOVyjXmUzzZpMmXKFLXnoYCC3itTU1ONn4mPjy+0PCJN2GZIW2wzpC22GdIW2wxpq6zajDbTwpdpUNS6dWvx/97e3vD394erqys2bNgAExOTMqxZ8UaPHo2hQ4eKy5mZmXBxcUFwcDAsLCxU8ubl5SE+Ph4tW7Z878dT0vuBbYa0xTZD2mKbIW2xzZC2yrrNKEdylUSZD597mZWVFapWrYrr16+jZcuWyM3NxePHj1V6i9LT08Vp9xwcHHD06FGVMtLT08V1yn+VaS/nsbCwgImJifgAoKY8RU3vZ2RkBCMjI7V0AwODQn/pRa0j0oRthrTFNkPaYpshbbHNkLbKqs1os80yn2jhZVlZWUhOToajoyP8/PxgYGCA3bt3i+uvXLmC27dvIyAgAAAQEBCAc+fOqcyBHx8fDwsLC3h6eop5Xi5DmUdZhqGhIfz8/FTyKBQK7N69W8xDREREREQfrjINioYNG4Z9+/bh5s2bOHz4MD755BNIpVKEh4fD0tISvXv3xtChQ7F3716cOHECPXv2REBAABo0aAAACA4OhqenJ3r06IEzZ85g586dGDNmDAYNGiT24vTv3x83btzAiBEjcPnyZSxcuBAbNmzAkCFDxHoMHToUy5Ytw+rVq3Hp0iUMGDAAz549Q8+ePcvkuBARERER0btTpsPn/v77b4SHh+Off/6Bra0tPvroIxw5cgS2trYAgDlz5kBPTw8dO3ZUeXmrklQqxdatWzFgwAAEBATAzMwMERERmDhxopjH3d0d27Ztw5AhQxATEwNnZ2csX75cfEcRAHTp0gX379/H2LFjkZaWBh8fH8TFxalNvkBERERERB+eMg2KfvnllyLXGxsbY8GCBViwYEGheVxdXbF9+/YiywkMDMSpU6eKzBMVFYWoqKgi8xARERER0YfnvXqmiIiIiIiI6F1jUERERERERDqNQREREREREek0BkVERERERKTT3quXtxIREb2Pku9n4dvfzuF6RhYys/Nhb2GEdrUrYHBQFRhIC+4v/u/obWw++TeupD0FANRytsTwkOrwcbFSK6/r0iS096mAEC8HDF5/GpdTM/H4eR7KyQzR0tMew0Oqwdy44KWDcedTsfbIbVxMzURuvgJV7GWIDqqKplVt39n+ExF96BgUERERFcNATw8dfJ1R08kSFib6uJT6FKM3n4VCEDCiVXUAwJEb/6BtbSf4trWGkb4Ui/clo8eKvxA/pCkcLI3Fsh4/z8WJW48wL9wXehIJWnraY1hwVdiYGeLWP8/x3R/n8fh5Hn4MrwMA+CvlIT6qUh7DQ6rBwsQAG4/fQZ/Vx/DbwEaoWcGyTI4HEdGHhkERERFRMSqWM0XFcqbisrO1KY7cqIBjNx+KaTFd66h8ZlpHb8SdT8Oh6w/Q0c9ZTN9zOQNeTpawNS94yXiPBq4q5fZo4Iql+2+IaePCvFTKHdGqOuIvpmP3pQwGRUREpYTPFBEREWnp5oNn2Hf1PvzdyxWa50WeHHlyBaxMDVTSEy6lo6Wn5peDp2dmI+58GvzdbQotV6EQ8CwnX61cIiJ6fewpIiIiKqEOCw/h/L2CZ3vC61fE0JZVC807dccl2FsYo5FHeTEtJ1+OfVfuIzpI9XNf/u8U4i+mITtPgaAadpja0bvQcpceuIFnuXKEeju++Q4REREA9hQRERGV2Pxuvtj25UeI6eqDvZczsPTADY35FiZex59nUrGkhx+MDaRi+uHkf1BOZoSq9uYq+b/7uAa2ftkYyz6vi1v/PMf32y5qLPeP03cRk3ANC7r5orzMqPR2jIhIx7GniIiIqIScrEwAAFXszaEQBIzefA59G1eCVE8i5lm6PxmLEpOxro8/ajhaqHw+4WI6gmqoD52zMzeGnTngYSeDlakBPl2chK+aV4Gdxb8TNGw5cw8jfz2Lhd198VGV8mplEBHR62NPERER0WtQKIB8uQCFIIhpi/clY97u61jdqz68na1U8guCgN2XMgp9nujfcgvKy8lXiGl/nL6L4RvP4MeuddC8etGfJyIi7bGniIiIdJpcIeBoykNkPM2Gnbkx6rvbqPT8AMDvp+5CXypBdQdzGEqlOHv3MabvvIyPvR3F9xQtSkzGnPiriOnqA2drE2Q8zQYAmBnqw8xIH+fuPsGLPDnquVmL5e69nIH7WTmo7WwFU0MprmU8xeTtl1HX1RouNgWz3f1x+i6+3nAG48I84VPRSizX2EAKC2NOtkBEVBoYFBERkc6KO5+KCX9eROqTbDHN0dIY48I80armvxMZSPUkWLwvGSn3n0EAUMHKBJ8HuKH3R+5inrVHbiFXrsCAdSdVtjG4RRUMaVkV8RfT0ayaLfSl/w7SMDLQwy9Hb2PS1ovIzVfAycoEIV4OGBBYWczz81+3ka8Q8N0fF/DdHxfE9I6+zpjVuXZpHg4iIp3FoIiIiHRS3PlUDFh7EsIr6WlPsjFg7Uks+sxXDIzCajshrLZTkeUdGtW8yPXxF9MR1dxDJa1h5fLYPLDo54PW9wsocj0REb05PlNEREQ6R64QMOHPi2oBEQAxbcKfFyFXaMqhvdx8BVrVdEBgNbtSKY+IiEoXgyIiItI5R1MeqgyZe5UAIPVJNo6mPCyV7Rnq6yE6qCpkRhygQUT0PmJQREREOkc5WUFp5SMiov82BkVERKRz7MyNi8+kRT4iIvpvY1BEREQ6p767DRwtjSEpZL0EBbPQ1Xe3eZfVIiKiMsKgiIiIdI5UT4JxYZ4AoBYYKZfHhXmqva+IiIg+TAyKiIhIJ7Wq6YhFn/nCwVJ1iJyDpbHKdNxERPTh4zQ4RESks1rVdERLTwccTXmIjKfZsDMvGDLHHiIiIt3CoIiIiHSaVE+CgMrlyroaRERUhjh8joiIiIiIdBqDIiIiIiIi0mkMioiIiIiISKcxKCIiIiIiIp3GoIiIiIiIiHQagyIiIiIiItJpDIqIiIiIiEinMSgiIiIiIiKdxqCIiIiIiIh0GoMiIiIiIiLSaQyKiIiIiIhIpzEoIiIiIiIincagiIiIiIiIdBqDIiIiIiIi0mkMioiIiIiISKcxKCIiIiIiIp3GoIiIiIiIiHTaexMUTZ06FRKJBNHR0WJaYGAgJBKJyk///v1VPnf79m2EhobC1NQUdnZ2GD58OPLz81XyJCYmwtfXF0ZGRvDw8EBsbKza9hcsWAA3NzcYGxvD398fR48efRu7SURERERE75n3Iig6duwYlixZAm9vb7V1ffv2RWpqqvgzffp0cZ1cLkdoaChyc3Nx+PBhrF69GrGxsRg7dqyYJyUlBaGhoWjWrBlOnz6N6Oho9OnTBzt37hTzrF+/HkOHDsW4ceNw8uRJ1K5dGyEhIcjIyHi7O05ERERERGWuzIOirKwsdO/eHcuWLYO1tbXaelNTUzg4OIg/FhYW4rpdu3bh4sWLWLt2LXx8fNC6dWtMmjQJCxYsQG5uLgBg8eLFcHd3x6xZs1CjRg1ERUWhU6dOmDNnjljO7Nmz0bdvX/Ts2ROenp5YvHgxTE1NsXLlyrd/AIiIiIiIqEzpl3UFBg0ahNDQUAQFBeH7779XW79u3TqsXbsWDg4OCAsLw3fffQdTU1MAQFJSEmrVqgV7e3sxf0hICAYMGIALFy6gTp06SEpKQlBQkEqZISEh4jC93NxcnDhxAqNHjxbX6+npISgoCElJSYXWOycnBzk5OeJyZmYmACAvLw95eXkqeZXLr6YTFYZthrTFNkPaYpshbbHNkLbKus1os90yDYp++eUXnDx5EseOHdO4vlu3bnB1dYWTkxPOnj2LkSNH4sqVK9i8eTMAIC0tTSUgAiAup6WlFZknMzMTL168wKNHjyCXyzXmuXz5cqF1nzJlCiZMmKCWvmvXLjFoe1V8fHyh5RFpwjZD2mKbIW2xzZC22GZIW2XVZp4/f17ivGUWFN25cweDBw9GfHw8jI2NNeb54osvxP/XqlULjo6OaNGiBZKTk1G5cuV3VVWNRo8ejaFDh4rLmZmZcHFxQXBwsMoQP6AgSo2Pj0fLli1hYGDwrqtK/0FsM6QtthnSFtsMaYtthrRV1m1GOZKrJMosKDpx4gQyMjLg6+srpsnlcuzfvx/z589HTk4OpFKpymf8/f0BANevX0flypXh4OCgNktceno6AMDBwUH8V5n2ch4LCwuYmJhAKpVCKpVqzKMsQxMjIyMYGRmppRsYGBT6Sy9qHZEmbDOkLbYZ0hbbDGmLbYa0VVZtRpttltlECy1atMC5c+dw+vRp8adu3bro3r07Tp8+rRYQAcDp06cBAI6OjgCAgIAAnDt3TmWWuPj4eFhYWMDT01PMs3v3bpVy4uPjERAQAAAwNDSEn5+fSh6FQoHdu3eLeYiIiIiI6MNVZj1F5ubmqFmzpkqamZkZypUrh5o1ayI5ORk///wz2rRpg3LlyuHs2bMYMmQImjRpIk7dHRwcDE9PT/To0QPTp09HWloaxowZg0GDBom9OP3798f8+fMxYsQI9OrVC3v27MGGDRuwbds2cbtDhw5FREQE6tati/r162Pu3Ll49uwZevbs+e4OCBERERERlYkyn32uMIaGhkhISBADFBcXF3Ts2BFjxowR80ilUmzduhUDBgxAQEAAzMzMEBERgYkTJ4p53N3dsW3bNgwZMgQxMTFwdnbG8uXLERISIubp0qUL7t+/j7FjxyItLQ0+Pj6Ii4tTm3yBiIiIiIg+PO9VUJSYmCj+38XFBfv27Sv2M66urti+fXuReQIDA3Hq1Kki80RFRSEqKqpE9SQiIiIiog9Hmb+8lYiIiIiIqCwxKCIiIiIiIp3GoIiIiIiIiHQagyIiIiIiItJpDIqIiIiIiEinMSgiIiIiIiKdxqCIiIiIiIh0GoMiIiIiIiLSaQyKiIiIiIhIpzEoIiIiIiIincagiIiIiIiIdBqDIiIiIiIi0mkMioiIiIiISKcxKCIiIiIiIp3GoIiIiIiIiHQagyIiIiIiItJpDIqIiIiIiEinMSgiIiIiIiKdxqCIiIiIiIh0GoMiIiIiIiLSaQyKiIiIiIhIpzEoIiIiIiIincagiIiIiIiIdBqDIiIiIiIi0mkMioiIiIiISKcxKCIiIiIiIp3GoIiIiIiIiHQagyIiIiIiItJpDIqIiIiIiEinMSgiIiIiIiKdxqCIiIiIiIh0GoMiIiIiIiLSaQyKiIiIiIhIpzEoIiIiIiIincagiIiIiIiIdBqDIiIiIiIi0mmlEhTdunULFy9ehEKhKI3iiIiIiIiI3hmtgqKVK1di9uzZKmlffPEFKlWqhFq1aqFmzZq4c+dOqVaQiIiIiIjobdIqKFq6dCmsra3F5bi4OKxatQo//fQTjh07BisrK0yYMKHUK0lERERERPS26GuT+dq1a6hbt664/Mcff6Bdu3bo3r07AGDy5Mno2bNn6daQiIiIiIjoLdKqp+jFixewsLAQlw8fPowmTZqIy5UqVUJaWlrp1Y6IiIiIiOgt0yoocnV1xYkTJwAADx48wIULF9CoUSNxfVpaGiwtLV+rIlOnToVEIkF0dLSYlp2djUGDBqFcuXKQyWTo2LEj0tPTVT53+/ZthIaGwtTUFHZ2dhg+fDjy8/NV8iQmJsLX1xdGRkbw8PBAbGys2vYXLFgANzc3GBsbw9/fH0ePHn2t/SAiIiIiov8WrYKiiIgIDBo0CJMmTcKnn36K6tWrw8/PT1x/+PBh1KxZU+tKHDt2DEuWLIG3t7dK+pAhQ/Dnn39i48aN2LdvH+7du4cOHTqI6+VyOUJDQ5Gbm4vDhw9j9erViI2NxdixY8U8KSkpCA0NRbNmzXD69GlER0ejT58+2Llzp5hn/fr1GDp0KMaNG4eTJ0+idu3aCAkJQUZGhtb7QkRERERE/y1aBUUjRoxA3759sXnzZhgbG2Pjxo0q6w8dOoTw8HCtKpCVlYXu3btj2bJlKpM4PHnyBCtWrMDs2bPRvHlz+Pn5YdWqVTh8+DCOHDkCANi1axcuXryItWvXwsfHB61bt8akSZOwYMEC5ObmAgAWL14Md3d3zJo1CzVq1EBUVBQ6deqEOXPmiNuaPXs2+vbti549e8LT0xOLFy+GqakpVq5cqdW+EBERERHRf49WEy3o6elh4sSJmDhxosb1rwZJJTFo0CCEhoYiKCgI33//vZh+4sQJ5OXlISgoSEyrXr06KlasiKSkJDRo0ABJSUmoVasW7O3txTwhISEYMGAALly4gDp16iApKUmlDGUe5TC93NxcnDhxAqNHj1bZz6CgICQlJRVa75ycHOTk5IjLmZmZAIC8vDzk5eWp5FUuv5pOVBi2GdIW2wxpi22GtMU2Q9oq6zajzXa1CoqAgqFmW7ZsQW5uLlq0aIH+/ftrW4Tol19+wcmTJ3Hs2DG1dWlpaTA0NISVlZVKur29vTiZQ1pamkpApFyvXFdUnszMTLx48QKPHj2CXC7XmOfy5cuF1n3KlCkapx/ftWsXTE1NNX4mPj6+0PKINGGbIW2xzZC22GZIW2wzpK2yajPPnz8vcV6tgqJFixZh0KBBqFKlCkxMTLB582YkJydjxowZWlfyzp07GDx4MOLj42FsbKz158va6NGjMXToUHE5MzMTLi4uCA4OVpmhDyiIUuPj49GyZUsYGBi866rSfxDbDGmLbYa0xTZD2mKbIW2VdZtRjuQqCa2Covnz52PcuHEYN24cAGDt2rXo16/fawVFJ06cQEZGBnx9fcU0uVyO/fv3Y/78+di5cydyc3Px+PFjld6i9PR0ODg4AAAcHBzUZolTzk73cp5XZ6xLT0+HhYUFTExMIJVKIZVKNeZRlqGJkZERjIyM1NINDAwK/aUXtY5IE7YZ0hbbDGmLbYa0xTZD2iqrNqPNNrWaaOHGjRuIiIgQl7t164b8/HykpqZqUwwAoEWLFjh37hxOnz4t/tStWxfdu3cX/29gYIDdu3eLn7ly5Qpu376NgIAAAEBAQADOnTunMktcfHw8LCws4OnpKeZ5uQxlHmUZhoaG8PPzU8mjUCiwe/duMQ8REREREX24tOopysnJgZmZmbisp6cHQ0NDvHjxQusNm5ubq03fbWZmhnLlyonpvXv3xtChQ2FjYwMLCwt8+eWXCAgIQIMGDQAAwcHB8PT0RI8ePTB9+nSkpaVhzJgxGDRokNiL079/f8yfPx8jRoxAr169sGfPHmzYsAHbtm0Ttzt06FBERESgbt26qF+/PubOnYtnz56hZ8+eWu8XERERERH9t2g90cJ3332nMpFAbm4ufvjhB5WXts6ePbtUKjdnzhzo6emhY8eOyMnJQUhICBYuXCiul0ql2Lp1KwYMGICAgACYmZkhIiJCZXY8d3d3bNu2DUOGDEFMTAycnZ2xfPlyhISEiHm6dOmC+/fvY+zYsUhLS4OPjw/i4uLUJl8gIiIiIqIPj1ZBUZMmTXDlyhWVtIYNG+LGjRviskQiee3KJCYmqiwbGxtjwYIFWLBgQaGfcXV1xfbt24ssNzAwEKdOnSoyT1RUFKKiokpcVyIiIiIi+jBoFRS9GrQ8ePAAhoaGarOtERERERER/VdoNdECADx+/BiDBg1C+fLlYW9vD2trazg4OGD06NFazQVORERERET0PtCqp+jhw4cICAjA3bt30b17d9SoUQMAcPHiRcybNw/x8fE4ePAgzp49iyNHjuCrr756K5UmIiIiIiIqLVoFRRMnToShoSGSk5PVJiGYOHEigoOD0aNHD+zatQs//vhjqVaUiIiIiIjobdAqKPr999+xZMkSjbOyOTg4YPr06WjTpg3GjRun8j4jIiIiIiKi95VWzxSlpqbCy8ur0PU1a9aEnp4exo0b98YVIyIiIiIiehe0CorKly+PmzdvFro+JSUFdnZ2b1onIiIiIiKid0aroCgkJATffvstcnNz1dbl5OTgu+++Q6tWrUqtckRERERERG+b1hMt1K1bF1WqVMGgQYNQvXp1CIKAS5cuYeHChcjJycFPP/30tupKRERERERU6rQKipydnZGUlISBAwdi9OjREAQBACCRSNCyZUvMnz8fFStWfCsVJSIiIiIiehu0CooAwN3dHTt27MCjR49w7do1AICHhwdsbGxKvXJERERERERvm9ZBkZK1tTXq169fmnUhIiIiIiJ657SaaIGIiIiIiOhDw6CIiIiIiIh0GoMiIiIiIiLSaQyKiIiIiIhIpzEoIiIiIiIincagiIiIiIiIdBqDIiIiIiIi0mkMioiIiIiISKcxKCIiIiIiIp3GoIiIiIiIiHQagyIiIiIiItJpDIqIiIiIiEinMSgiIiIiIiKdxqCIiIiIiIh0GoMiIiIiIiLSaQyKiIiIiIhIpzEoIiIiIiIincagiIiIiIiIdBqDIiIiIiIi0mkMioiIiIiISKcxKCIiIiIiIp3GoIiIiIiIiHQagyIiIiIiItJpDIqIiIiIiEinMSgiIiIiIiKdxqCIiIiIiIh0GoMiIiIiIiLSaQyKiIiIiIhIpzEoIiIiIiIinVamQdGiRYvg7e0NCwsLWFhYICAgADt27BDXBwYGQiKRqPz0799fpYzbt28jNDQUpqamsLOzw/Dhw5Gfn6+SJzExEb6+vjAyMoKHhwdiY2PV6rJgwQK4ubnB2NgY/v7+OHr06FvZZyIiIiIier+UaVDk7OyMqVOn4sSJEzh+/DiaN2+Odu3a4cKFC2Kevn37IjU1VfyZPn26uE4ulyM0NBS5ubk4fPgwVq9ejdjYWIwdO1bMk5KSgtDQUDRr1gynT59GdHQ0+vTpg507d4p51q9fj6FDh2LcuHE4efIkateujZCQEGRkZLybA0FERERERGVGvyw3HhYWprL8ww8/YNGiRThy5Ai8vLwAAKampnBwcND4+V27duHixYtISEiAvb09fHx8MGnSJIwcORLjx4+HoaEhFi9eDHd3d8yaNQsAUKNGDRw8eBBz5sxBSEgIAGD27Nno27cvevbsCQBYvHgxtm3bhpUrV2LUqFEat52Tk4OcnBxxOTMzEwCQl5eHvLw8lbzK5VfTiQrDNkPaYpshbbHNkLbYZkhbZd1mtNlumQZFL5PL5di4cSOePXuGgIAAMX3dunVYu3YtHBwcEBYWhu+++w6mpqYAgKSkJNSqVQv29vZi/pCQEAwYMAAXLlxAnTp1kJSUhKCgIJVthYSEIDo6GgCQm5uLEydOYPTo0eJ6PT09BAUFISkpqdD6TpkyBRMmTFBL37Vrl1i/V8XHxxd/IIhewjZD2mKbIW2xzZC22GZIW2XVZp4/f17ivGUeFJ07dw4BAQHIzs6GTCbDb7/9Bk9PTwBAt27d4OrqCicnJ5w9exYjR47ElStXsHnzZgBAWlqaSkAEQFxOS0srMk9mZiZevHiBR48eQS6Xa8xz+fLlQus9evRoDB06VFzOzMyEi4sLgoODYWFhoZI3Ly8P8fHxaNmyJQwMDLQ5PKSj2GZIW2wzpC22GdIW2wxpq6zbjHIkV0mUeVBUrVo1nD59Gk+ePMGmTZsQERGBffv2wdPTE1988YWYr1atWnB0dESLFi2QnJyMypUrl2GtASMjIxgZGamlGxgYFPpLL2odkSZsM6QtthnSFtsMaYtthrRVVm1Gm22W+ZTchoaG8PDwgJ+fH6ZMmYLatWsjJiZGY15/f38AwPXr1wEADg4OSE9PV8mjXFY+h1RYHgsLC5iYmKB8+fKQSqUa8xT2LBMREREREX04yjwoepVCoVCZwOBlp0+fBgA4OjoCAAICAnDu3DmVWeLi4+NhYWEhDsELCAjA7t27VcqJj48Xn1syNDSEn5+fSh6FQoHdu3erPNtEREREREQfpjIdPjd69Gi0bt0aFStWxNOnT/Hzzz8jMTERO3fuRHJyMn7++We0adMG5cqVw9mzZzFkyBA0adIE3t7eAIDg4GB4enqiR48emD59OtLS0jBmzBgMGjRIHNrWv39/zJ8/HyNGjECvXr2wZ88ebNiwAdu2bRPrMXToUERERKBu3bqoX78+5s6di2fPnomz0RERERER0YerTIOijIwMfP7550hNTYWlpSW8vb2xc+dOtGzZEnfu3EFCQoIYoLi4uKBjx44YM2aM+HmpVIqtW7diwIABCAgIgJmZGSIiIjBx4kQxj7u7O7Zt24YhQ4YgJiYGzs7OWL58uTgdNwB06dIF9+/fx9ixY5GWlgYfHx/ExcWpTb5AREREREQfnjINilasWFHoOhcXF+zbt6/YMlxdXbF9+/Yi8wQGBuLUqVNF5omKikJUVFSx2yMiIiIiog/Le/dMERERERER0bvEoIiIiIiIiHQagyIiIiIiItJpDIqIiIiIiEinMSgi+r/27jwuqup94Phn2HdwYVXEfcFU1HJfMFFM0zJL2wxLrVxKpXL5lrlVLuVWuWS5ZOXPzK1yFxJN3NdUlBQRXEBcQURhmDm/PyYGxwEEBFF53q8XL51zzz1zznDmzjzcc58rhBBCCCFKNQmKhBBCCCGEEKWaBEVCCCGEEEKIUk2CIiGEEEIIIUSpJkGREEIIIYQQolSToEgIIYQQQghRqklQJIQQQgghhCjVJCgSQgghhBBClGoSFAkhhBBCCCFKNQmKhBBCCCGEEKWaBEVCCCGEEEKIUk2CIiGEEEIIIUSpJkGREEIIIYQQolSToEgIIYQQQghRqlmVdAeEEEKUAO1tWDMMEg7BpWio2QleWWJeLzMdtk6Gf5ZB6kVw8oK2w6FRb9N6EZPgSgz0+B72LYQjyyHhMGTcgBFxYO9m3va/Gw1tXzwGVrbg1yrnPgghhBDFTIIiIYQojZQOrO2g6TsQ9Ufu9X7rA6lJ0O0bKFvVEBgpvXm9E2uh1TDD/7W3oHp7w0/4uJzbjfod/ngf2n8KVdqCPhOSou57WEIIIURhSFAkhBClkY0jPDvd8P/43XA72bzOyTA4EwlDDoFDWUNZGT/zesnn4NIJqB5keNx8oOHf2L9zfm5dJqwfCR0nQKM3sss9ahdqKEIIIcT9kqBICCEKqziXoN24CJtHQ8wWyEiFctWhzYfg/9wDGRoA0evAJwAiZ8I/v4K1A9R6Bp7+BKzt76i3Hiq3AjuX/LWbcBhuXACNBcxtZTgT5VUPOkwAT/9iGYoQQgiRFwmKhBCisIpzCdqqdwxnb15ZajhLc2S5oZ23I8C7QTEMJgfXzkD8LrCyg16/QNoVWPsB3LoGz8827XftLgVoN9bwb8QkCP4c3CrBjm9hURd4b3/2WSkhhBDiAZHsc0IIUVhZS9Aa9wEnz5zrZC1Be+03qNbOsPzMtwlUamZa7+4laGf3GIKtio2hbBVo+xHYucKFQ8U5IlNKDxqN4cxVxcZQs6MhiDm0xHDdEMDtFIiLNJxByne7yvBv6w8MZ758GhqCLI0GolYX+TCEEEKIe5GgSAghitOdS9Cm1oavG8HGj7ODCmO9u5ag+TaBoysh7Sro9YYzRZnphjoPirMXOHsbgrEs7rUABSkXDI9PbTaUuVYsQLv/BZDud1xDZGULZSobgkMhhBDiAZPlc0IIUZwKuwTtpUWw/E2YUgUsrAzX8/T6GcpVe3B9920Kx1ZDeirYOhnKrpwyXAvk4vNfv9dBrQIsnQPwDgBLW7hyEvyaG8p0WrgeD66+RdV7IYQQIt/kTJEQQhSnwi5B2/K54ZqiN343XEfUfBD89qbhnj75odcZsr8dWW74V68zr5N0AhL+MQRo6SmG/yf8k7293kuG63t+H2ioeyYSNo2Ghq8bEi3oMg1niu5eOnfjoqGdq6f/e54ow+O0q4bHdi7w5FuwZSKcCofLJw0JKwDqPp+/8QkhhBBFSM4UCSFEcbrXErRy1cyXoF09DXvmwcBd4FHHUOZVD+J2wJ7voeuMvJ8z6g/YMCJ7iRsYzux0mgz+3bLLfnkJkuOzH3/X2vDv2P/Sc9s6Qe/VsP4jmBdoCJDqdjdknwOI2w42ToblgXfatwC2Tsp+vPC/oOm52dDwNcP/O04AC0tDQgntbUPAGPIn2JfJe2xCCCFEMZCgSAghilNhlqBlnUHS3HUy38Iy56x1d9CcWAMr3gSU6YaUBFj2BvRcnB0YDTty7/671zScrcrJiXWGNOR3azfK8JMXS2vDGbPgz+/dByGEEKKYyfI5IYS4H8WxBK18TUPq7j+Hwrn9hjNHO74x3LOo9rPmfdDr0MRtp8LVHViu+wCzgAiyyzaMzHkpXWF41IGn+hZNW0IIIUQJkjNFQghxP4pjCZqlNby2HMLGwP/1goybhiCp+1zDNUl3+m+pnFXKBZ68Z2cVpJw3LMOr0rqwI8725Jv334YQQgjxEJCgSAgh7kdxLUErV82QbS4vUX8YlsTleGYoD6kXC1ZfCCGEeMxJUCSEECXJo47hnkQFpdcZkikUMCDSAQcyk7l0eh3uDu408miEpYVlwZ9fCCGEeIxIUCSEECWpsEvQ4naYZpfLhzAHByaVL8fFf2YYyzwdPBnZZCRBfkGF64cQQgjxGJBEC0II8Sgq4BK4MAcHQj3KcdFSY1KelJZEaEQoYXFhRdk7IYQQ4pEiZ4qEEOJR5OSZ76o6YFL5ciiNxmybQqFBw+Q9k2nn206W0gkhhCiYrBtwXzphuBm5s5ch82rgSGMVzcHFcPQ3w828AbwDoP0Ywz3q7rboWcP+jUPg/H4IGwsXDoMGqNAYOow33LuviMmZIiGEeBT5tfjvPkfmgY6BBhzKwwvfc+C5aWZniO6kUCSmJXIg6UCxdFUIIcRjzMIKGrwMvVfBe/ug0yQ48CNs+SK7SlwkPNEDQtZA3zDDzcp/6m6+DDztKsTvMtymIj0Vfu4Brr7QPxze2mjI1vrTC6DTFv0wirxFIYQQxc/CEjpN/u/B3QHPf4+fnQ71e3LJ1TtfTV5Ku1Rk3RNCCFFKlK1iuPeeVz1wqwS1O0O9nhC/01hF9/x30KQ/eNc3ZGTt9o3hZuSnt5q2dXITeDcAJw+4/K/hHoDt/gflaxgSEwWOhJtJcD2eoiZBkRBCPKr8u0HPxeByV9Dj4mMo9+8GgLuDe76ay289IYQQIldXYuBUGPi1zL2ONg30WrAvY1oevc4QVIEhELIvCwd+gswM0N4y/L98LXDzK/JuyzVFQgjxKPPvBrW7kHl6G4f+3khA62CsqrYxnEn6TyOPRng6eJKUloTKIYW3Bg2eDp408mj0IHsuhBDicfJDB0g4DLp0aNwH2n0MOl3OdTePMVx7VDUwuywzHU6FQ+Aow2NbZ+izFpa+CtumGMrKVoPeK8Gy6EMYCYqEEOJRZ2GJ8mvF+WMpNPBrZRIQAVhaWDKyyUhCI0LRoDEJjDT/LbUb0WSEMclCui6d8TvHE3UlitjkWNpUbMPXT39t9rQZugzmHp7LmtNruHzrMu727rzb4F261+huUm/OoTnE3YhjUutJ/Pbvb6w7vY7jV49zU3uTyFcicbFxMWt727ltzD08l3+v/YuNpQ1Pej6ZYx+EEEI8JF5aaLgO6OJR2DQaynwNTQeZ1/t7GhxdYQh4rO2yy2O3gWN5wzI5MJwZ+mMwVGoGL84HvR52fA2/9IS3t4C1fZF2v0SXz82ZM4f69evj4uKCi4sLzZs3Z/369cbtt2/fZtCgQZQrVw4nJyd69OjBxYumaWjj4+Pp0qULDg4OeHh48NFHH5GZmWlSJyIigkaNGmFra0v16tVZtGiRWV9mzZpF5cqVsbOzo2nTpuzZs6dYxiyEECUhyC+IaYHT8HDwMCn3dPBkWuA0k/sU6fQ67CzteK3OazTzbpZrmx9s/YDdCbsZ12Icf3b/k8ltJlPZtbJZvS1ntxDoGwjA7czbtKzQkn71+uXa7ua4zYz6exTPV3+e5V2X89MzP9G5aueCDVgIIcSD5VoRPGpDvRchaCxETDLcaPxOkV/D9hmGpAxeT5hui14Hte441h/5zXDt0HOzDVnnfJ+CHvPhehycWFvk3S/RM0UVK1Zk0qRJ1KhRA6UUP/74I8899xwHDx6kbt26DBs2jLVr1/Lbb7/h6urK4MGDeeGFF4iMjARAp9PRpUsXvLy82LFjBwkJCbzxxhtYW1vzxReGjBexsbF06dKFd999l19++YXw8HD69euHt7c3wcHBAPz666+EhoYyd+5cmjZtyowZMwgODiY6OhoPD49c+y+EEI+SIL8g2vm240DSAS6lXcLdwZ1GHo3M0nA7WDswuvloAA4mHeRGxg2ztraf387+xP2s77EeV1tXACo4VTCrl3gzkVPXT9HKpxUAvf17A7A3cW+OfczUZzJpzyQ+ePIDXqjxgrG8mlu1QoxYCCFEiVB6wzVDSp9dtn0G/D0VXl8JFe5arq0URG+AF+Zll2lvgcYC7rydhMYC0BjqF7ESDYq6du1q8vjzzz9nzpw57Nq1i4oVKzJ//nyWLFnC008/DcDChQupU6cOu3btolmzZmzatImoqCjCwsLw9PQkICCACRMmMGLECMaOHYuNjQ1z586lSpUqTJ06FYA6deqwfft2pk+fbgyKpk2bRv/+/XnzTcOd5efOncvatWtZsGABI0eOJCfp6emkp6cbH6ekpACg1WrRak3TBGY9vrtciNzInBEFVZA5E1AuAMoZ/q/X6dHr9LnWVXqFUsqs3b/i/qJO2Tr88M8PrI1di72VPW0rtGVA/QHYWWUvhwg7E0Zjj8bYamxN2sg6o5+pzUSryS4/evkoSWlJ6HV6XvzjRa7cukLNMjUZ2nAo1d2q33NsIv/kOCMKSuZMKaXXoTm703DTcCdPlG9zk2XamqO/gYU1ysMfLG3QJBzCMmwsyv95tP99vKjt01Hbv0T3/HcoJ2+4ds6wwcYRbJzQXDiIpTaNTJ8nIWt+VWqN1a3R6P8chv6p/qD0WO6YicbCksyKzbLr5aEgc/WhuaZIp9Px22+/cfPmTZo3b87+/fvRarUEBWUv6ahduzaVKlVi586dNGvWjJ07d1KvXj08PbNvYhgcHMyAAQM4duwYDRs2ZOfOnSZtZNUZOnQoABkZGezfv59Ro0YZt1tYWBAUFMTOnTvJzcSJExk3bpxZ+aZNm3BwcMhxn82bN+frtRAii8wZUVBFPWfO3TzHbXWbdevWmZQfSj1EbGYsyVeS6WHXg5v6m/wZ/SdHTx+lh0MPY73lqcupY13HbP/T2tOA4Zhpb5G9LvyfjH8AmLFnBp3tO+Nm5Ubk5Uj6rO/DUOehOFjkfHwVhSfHGVFQMmdKD+/re6l37hfstVeNZbesy3Kk4mskuD0FgM+1I9S4uA6n9ERAkWZTnnNlWhNjGYz+v7mSufM7bHQZWK1406T9E17PE+39ArUvLMfB3p8DGzaZbHevPIRa0atwOfwrCg1X7P047jeUa3/n7756aWlp+R5riQdFR44coXnz5ty+fRsnJydWrVqFv78/hw4dwsbGBjc3N5P6np6eJCYmApCYmGgSEGVtz9qWV52UlBRu3brFtWvX0Ol0OdY5ceJErv0eNWoUoaGhxscpKSn4+vrSsWNHXFxMLxrWarVs3ryZDh06YG1tnY9XRZR2MmdEQeU1Z3R6HQcvHeTyrcuUty9PQ/eGZkvmcrN7525uaG/QuY3pNT1r/lpD/KV45j0/D2cbZwDqn63P8L+HM+v5WdhZ2ZGqTWXcinF80/EbvBy9TPbfd3EfC8IX0LFjR+P+AJozGpbtWMbgpwbTo7ohuArRhdBpdSeoBZ1ryLVFRUWOM6KgZM6ULpoTa7Bc8S3clbXUTnuNp2K/RddjIar2s0BnYLyxlj1Q47+frDnD0CNoc5gz1f77sfp+ErrAUDr7332M7wwMN/QHcAOaF2AMWSu58qPEg6JatWpx6NAhkpOTWb58OSEhIWzduvXeO5YwW1tbbG1tzcqtra1zPVDktU2InMicEQV195wJiwtj0p5JXEzLTlLj6eDJyCYjTZIr5EZjoUGj0ZjNQw9HDzxuelDWsayxrGbZmigUV7VX8bP3Y/e53VRzq4avm69Zu1ZWho8fK2srk7a9nLyMbWWVW1tb4+vsy6Xbl+T9UAzkOCMKSuZMKaDXweb/cXdABKBBARqsNn8MdbuZZTzNSZ5zJjMD/J/DqnYnKOJ5VZB5WuI3b7WxsaF69eo0btyYiRMn0qBBA2bOnImXlxcZGRlcv37dpP7Fixfx8jJ8aHp5eZllo8t6fK86Li4u2NvbU758eSwtLXOsk9WGEEI8isLiwgiNCDUJiACS0pIIjQglLC6s0G0HeARwKe0SadrspQlnUs5gobHA08Fw5v2vs3/Rzrddgdr1L+ePjYUNZ1LOGMu0ei3nU8/j7eSd+45CCCGKTtwOSLmQRwUFKecN9e6XlQ0EjjTcl6gElXhQdDe9Xk96ejqNGzfG2tqa8PBw47bo6Gji4+Np3txw4qx58+YcOXKEpKQkY53Nmzfj4uKCv7+/sc6dbWTVyWrDxsaGxo0bm9TR6/WEh4cb6wghxKNGp9cxac+kHG/WmlU2ec9kdHenS/1PzPUYTlw9QUp6CqnaVE5cPcGJq9lLirtU6YKrrSufRH5CzPUY9iXuY9r+aXSv3h07Kzsy9ZlsP7/dmIo7y+Vblzlx9QTxKfEAnLx2khNXT5CcngyAk40TPWv1ZNahWew4v4PY5Fg+2/UZAB39Ot736yKEECIfUi/eu05B6j0CSnT53KhRo3jmmWeoVKkSN27cYMmSJURERLBx40ZcXV3p27cvoaGhlC1bFhcXF9577z2aN29Os2aG+2Z07NgRf39/evfuzZQpU0hMTOSTTz5h0KBBxqVt7777Lt9++y3Dhw/nrbfe4q+//mLZsmWsXZud3zw0NJSQkBCefPJJmjRpwowZM7h586YxG50QQjxqDiQdMDtDdCeFIjEtkQNJB3jK6ymz7QPDBnLhZvZfCV/68yUAjoQcAQxpu+d1nMfE3RN5ec3LuNq6Elw5mPcavgcYrhlysHLAv5y/SbvLopcx5/Ac4+M+G/oAMKHlBJ6v/jwAoU+GYqmxZNT2UaTr0qlXvh7zO843pv4WQghRzJw8710H4EpM8fbjASrRoCgpKYk33niDhIQEXF1dqV+/Phs3bqRDhw4ATJ8+HQsLC3r06EF6ejrBwcHMnj3buL+lpSVr1qxhwIABNG/eHEdHR0JCQhg/fryxTpUqVVi7di3Dhg1j5syZVKxYkR9++MGYjhugV69eXLp0iU8//ZTExEQCAgLYsGGDWfIFIYR4VFxKu3Rf9Ta+uPGe+1Z1rcr3Hb/PcduW+C1mZ4kABgYMZGDAwDzbtbaw5sOnPuTDpz68Zx+EEEIUA78W4OIDKQnkdF2RUcQX4FEH/Ls9sK4VlxINiubPn5/ndjs7O2bNmsWsWbNyrePn52eW6vVugYGBHDx4MM86gwcPZvDgwXnWEUKIR4W7g3uR1iuo6mWq08C9QbG0LYQQophZWEKnybDsjXtU1MCGkVC7S74SLjzMHrprioQQQty/Rh6N8HTwRIMmx+0aNHg5eNHIo1GO2+/XSzVfomaZmsXSthBCiAfAvxsEjrpHpSJMuFDCJCgSQojHkKWFJSObjAQwC4yyHo9oMiLf9ysSQghRCpWrlr96j0HChRK/T5EQQjyu0nXpjN85nqgrUcQmx9KmYhu+fvprs3oZugzmHp7LmtNruHzrMu727rzb4F261+huUm/OoTnE3YhjUutJXL51man7prLzwk7SMtPwc/ajYUZDOpN947sgvyCmBU7L8T5FI5qMyNd9ioQQQpRi+U24kN96DzEJioQQopjo9DrsLO14rc5red4T6IOtH3D11lXGtRhHJZdKXEq7lGMq7S1nt/BWvbcA+N/f/+NGxg2+efob3OzcWHNqDXP+mUPXq12p51nPuE+QXxDtfNtxIOkAl9Iu4e7gTiOPRnKGSAghStLlk7BmGFw6AbdTwNkL6r1kuF+P5X83HN2/CA4vhaQow2PvAGg/Bio2Nm9v0bOG/RuHwLrhcHYXJB2H8rVgwHbz+krBjm8Mz5F8FhzKwVN9oc1HpvXumXBBY9ju16LQL8XDQoIiIYQoJg7WDoxuPhqAg0kHuZFxw6zO9vPb2Z+4n/U91htTTldwqmBWL/FmIqeun6KVTysADl06xOhmo6nnbgiA+j3Rj4VHFnL86nGToAgMS+lySrsthBCihFhYQYOXwbsB2LlC4lH4831QeggaY6hzZjs80QN8p4CVHUTOgJ+6w6BdhkAkS9pViN8FLy7ILmvYG87tg4vHcn7+9SMg5i/o+Bl4+sOta4Yfs37emXBBg2lg9N/S7E6THvkkCyBBkRBClKiIsxH4l/dnwdEFrIlZg721PYEVAxnccDB2VnbGelvObuEpr6dwsnECIMA9gA1nNtCmYhucbZzZeGYjmSqTxp45/AVRCCHEw6VsFcNPFrdKhiAofmd2WY8fTPfp9g1E/QGnt0LAK9nlJzcZgisnD8PjzlMM/968nHNQdCka9s2HgbugfA1DWZnKuffVvxv0XAwbRkBK9v3rcPExBESPQTpukKBICCFK1Lkb5zh48SC2lrbMaDeDa+nX+HzX51xPv85nrT4z1tsSv4V2ldoZH38V+BUfbf2IVktbYaWxws7KjlcdX6WSc6WSGIYQQoj7cSUGToVBna6519GmgV4L9mVMy6PXQe3OOe+Tk+j1hiDo3w3w8wuGkz9V20KH8eBQNud9/LsZ0m7H7TAkVXDyNCyZewzOEGWRoEgIIUqQXunRaDRMaj0JZxtnADKeyiA0IpRPmn2CnZUdqRmp7Lu4j/Ets29M/e3Bb7mRcYPvO35PGdsybD6zmR+P/Ejn653xd/cvqeEIIYQoiB86QMJh0KVD4z7Q7uPc624eY7j2qGpgdllmOpwKz0fq7DtcOwPXz8Kx1dD9O9DrYOMowxK5Pmty38/CEqq0zv/zPGIkJbcQQpQgdwd3PBw8jAERQFXXqiiUMWPc9vPbqeZWDS9HLwDOppzl/078H+NbjKeZdzNqla3FO/XewcfKh2X/LiuRcQghhCiElxbCO9ugx3z4dxPsMM9QCsDf0+DoCuj1C1hnL60mdhs4lgePOvl/TqU3BGHdvzOc7anSGrp9C2f+NiSAKKUkKBJCiBIU4BHApbRLpGnTjGVnUs5gobHA08GQ4vSvs3/Rzjd76dwt3S0ALDSmh3ALLNAr/QPotRBCiCLhWhE8akO9FyFoLERMMpy5uVPk17B9BvReBV5PmG6LXge1CrB0DgxnmyysoHz17DL3WoZ/k88WdASPDQmKhBCiGMVcj+HE1ROkpKeQqk3lxNUTnLh6wri9S5UuuNq68knkJ8Rcj2Ff4j6m7Z9G9+rdsbOyI1Ofyfbz2wn0DTTuU8W1CpWcKzFu5ziOXDrC2ZSz/HT8J2IyY2hXsV0OvRBCCPHQU3rDNUN3/nFr+wzY9iW8vgIqNLqrvoLoDQUPinybgj4Trp7OLrtyyvCva+m9LlWuKRJCiGI0MGwgF25mZ+t56c+XADgScgQwpO2e13EeE3dP5OU1L+Nq60pw5WDea/geAPsu7sPBygH/ctnXCVlbWDM7aDYz9s9g8F+DuZV5i4pOFXnB4QVaVWj1AEcnhBCiUP5ZZjhb41kXLG3gwkEIHwd1X8i+T9H26bDlC0MWOrdKcOO/m3DbOIKtk2EfbRpUam7a9pUYyLhpSIiQeQsS/jGUu9cGKxuo2s6Qre73wdBpoiEIW/uhofzOs0eljARFQghRjDa+uPGedaq6VuX7jt/nuG1L/BaTs0RZ/Fz8mN5uuvGxVqtl3bp1he6nEEKIB8jC0nDfoSsxhjM+br7QpD80G5RdZ+8C0GX8d4+gO7QdCe1GGZbO1egIlnd9nf/jfYi744at3/2XHGHIP1DGDyws4JVfYf1HsLAzWDtAjQ6GexaVYhIUCSHEQ6x6meo0cG9Q0t0QQghRlJ7oYfjJy7AjeW8/sQ7afGhe/ubaez+/izf0+vne9UoRCYqEEOIh9lLNl0q6C0IIIR42mRmGewfV6FDSPXlsSFAkhBBCCCHEo8TKBgJHlnQvHiuSfU4IIYQQQghRqklQJIQQQgghhCjVZPmcEEIIIYQQosjEJscyYdcEYq7HkHI7hTm/z6Fz1c4MCBiAtYUh5fjyf5fzZ8yfnLx+EgD/cv4MaTiEeu71zNp7a+NbdKnShR41e3D08lFm7J9B1JUo0EC98vUIbRxKrbK17qvPcqZICCGEEEIIUWSsLKzoWrUrs9vNZojLED5s/CErTq5g9qHZxjp7E/fyTJVnWBC8gJ87/4yXgxfvbH6HizcvmrSVnJ7MwaSDtPVtS5o2jXfD3sXL0YtfuvzC4k6LcbR25J3N76DVa++vz/e1txBCCCGEEELcwdfZF19nX7RaLacsTtG2YlsOXD7AgYsHjHUmt5lsss+4FuMIiw9jd+JuulXrZizfdm4b/mX9KW9fnmOXj5GcnszghoPxcvQC4N0G79Ljjx4kpCZQyaVSofssZ4qEEEIIIYQQxSb+RjyR5yNp7Nk41zq3dbfJ1GfiauNqUr7l7BbaVWoHQGXXyrjZurHy5Eq0Oi23M2+z6uQqqrpWxcfJ5776KGeKhBBCCCGEEEWuz6Y+RF2PIvPPTF6s+SKDGw7Ote70/dNxt3enmU8zY1mGLoPI85EMbDAQAEdrRxYEL2DIliF89893AFRyrsR3Hb7DyuL+who5UySEEEIIIYQocpNaTmKg80C+aPEF285tY9GxRTnW++HID6yPXc+MdjOwtbQ1lu9O2E1Zu7JUL1MdgNuZtxmzYwwNPRryS+dfWPzMYmqUqcGg8EHczrx9X32VoEgIIYQQQghR5LwcvfCw9KBT5U4MbTSUOYfmoNPrTOosOrqIBUcWMK/DPLMMchFnIwj0DTQ+Xhe7jvOp55nQcgJPlH+CBu4NmNx6MudTz7Pl7Jb76qsERUIIIYQQQohipVBk6jPRozeWLTi6gO/++Y45HeZQt3xd0/pKEXEugqcrPW0su5V5CwuNBRo0xjKNxvB/vdJzP+SaIiGEEEIIIUSR0Ol1zDo0i5vam9Ryq8U13TU2xW1i5oGZBFcJNt6naP6R+cw6NIvJbSZTwakCl29dBsDBygEHaweirkRxO/M2DT0aGttu7tOcafum8fnuz3m19qvolZ75R+djpbGiiVeT++q3BEVCCCGEEEKI+xYWF8akPZO4mGZ6ryHPg568UucVevv3NpYti16GVq8lNCLUpO6ABgMYGDCQv87+ReuKrU0SKFR1rco37b9h7uG5vL7udTQaDXXK1mFOhzm4O7jfV98lKBJCCCGEEELcl7C4MEIjQlEos21JaUlUdqlskkRh44sb82xvy9ktvF3/bbPyFj4taOHT4v47fBe5pkgIIYQQQghRaDq9jkl7JuUYEGWZvGeyWZKF3Gh1WjpU6kDrCq2Lqov3JEGREEIIIYQQotAOJB0wWzJ3J4UiMS2RA0kH8tWetaU1AwIG4GjtWFRdvCcJioQQQgghhBCFdintUpHWKwkSFAkhhBBCCCEKLb9JDu43GUJxkqBICCGEEEIIUWiNPBrh6eCZZx0vBy8aeTR6QD0qOAmKhBBCCCGEEIVmaWFJ5yqd86zzTJVnsLSwfEA9KjgJioQQQgghhBCFptPrWBe7Ls8662PX5zv7XEmQoEgIIYQQQghRaPfKPgcUKPtcSZCgSAghhBBCCFFokn1OCCGEEEIIUao9DtnnrEq6A0IIIYQQQoiHT2xyLBN2TSDmegypGam4O7jTuUpnBgQMwNrCGoDl/y7nj5g/0KBBoXJsR4MGTwdPZh+azbNVn6VHzR5M3D2Rg0kHOXX9FFVdq7K823Kz/ZRS/HjsR5afXM6F1AuUsS1Dr9q9eLv+20U+1hINiiZOnMjKlSs5ceIE9vb2tGjRgsmTJ1OrVi1jncDAQLZu3Wqy3zvvvMPcuXONj+Pj4xkwYABbtmzBycmJkJAQJk6ciJVV9vAiIiIIDQ3l2LFj+Pr68sknn9CnTx+TdmfNmsWXX35JYmIiDRo04JtvvqFJkyZFOubk5GTS0tKKtE3x+MnMzCQtLY3ExESTeSxEbmTOiIJ60HPGwcEBV1fXYn8eIUTRsbKwomvVrviX88fZxpnoq9GM3TkWhWJIoyEA7E3cS+cqnWnv256v9n+Va1uDGw5m7M6xfNn2S2NZ9xrdOXLpCP9e+zfHfSbtmcSOCzv4oPEH1ChTg+SMZJLTk4t2kP8p0U/OrVu3MmjQIJ566ikyMzP53//+R8eOHYmKisLR0dFYr3///owfP9742MHBwfh/nU5Hly5d8PLyYseOHSQkJPDGG29gbW3NF198AUBsbCxdunTh3Xff5ZdffiE8PJx+/frh7e1NcHAwAL/++iuhoaHMnTuXpk2bMmPGDIKDg4mOjsbDw6NIxpucnMy8efPQarVF0p54/P37b84HCSFyI3NGFNSDmjPW1tYMGjRIAiMhHiG+zr74OvsaH/s4+bD34l4OXMxOmDC5zWTj/ys4V2Di7okk3UoylrlqXPmk1SdolRb/sv6Uty8PwKimowC4dvtajkHR6eunWRa9jJXPraSKaxUAKlKxaAd4hxINijZs2GDyeNGiRXh4eLB//37atGljLHdwcMDLyyvHNjZt2kRUVBRhYWF4enoSEBDAhAkTGDFiBGPHjsXGxoa5c+dSpUoVpk6dCkCdOnXYvn0706dPNwZF06ZNo3///rz55psAzJ07l7Vr17JgwQJGjhxZJOO9desWWq2W7t274+7+8K6pFEIIIYrSpUuXWLVqFWlpaRIUCfEIi0+JJ/J8JO0rtc9xe5BfEE28mhC4LJDXar9GC+8WJO5LpL1ve0ZEjqBdpXb5fq6IcxFUdK7ItnPbGBA2AKUUzXyaEdo4FFfboj+OPFRrLJKTDafDypYta1L+yy+/8PPPP+Pl5UXXrl0ZPXq08WzRzp07qVevHp6e2XfRDQ4OZsCAARw7doyGDRuyc+dOgoKCTNoMDg5m6NChAGRkZLB//35GjRpl3G5hYUFQUBA7d+7Msa/p6emkp6cbH6ekpACg1WrNzgRlPc7MzATA3d0db2/v/L0oQgghxGMiMzNTVks8wrJ+d/I7LH36bOrDiasnyNBn8EL1F3jniXdynQcz9s/A08GTd+q9g4XegiRNEjdv3yTyfCRv133bbD+dTodSyqw8PjmeC6kX2BC7gXHNxqFXeqYemMrQLUOZ135evvpdkLn60ARFer2eoUOH0rJlS5544glj+auvvoqfnx8+Pj78888/jBgxgujoaFauXAlAYmKiSUAEGB8nJibmWSclJYVbt25x7do1dDpdjnVOnDiRY38nTpzIuHHjzMo3bdpksrzvTrt27crrJRBCCCEea9u3b8/1M1I8OjZv3lzSXRAPWEd9RwIdA0nQJbAxZiM3z92ktV1rs3pbb29le/p2+jr1JXxjuLH8+43fY6u3JXpHNNFEm+xz8tZJUrQprFtnevPXuLQ4MvQZBGUEkbjP8J2+fWZ7Zl+bzY9//oi75b1XXRXkOv6HJigaNGgQR48eZfv27Sblb7+dnV2iXr16eHt70759e2JiYqhWrdqD7qbRqFGjCA0NNT5OSUnB19eXjh074uLiYlJXq9WyefNmmjVrJuv9hRBClFqtWrXKdTm8ePhlfZ/p0KED1tbWJd0d8aDpdWjO7qTB2XQ+O7eez7qNw9LKxrh58fHF7Dy6k+87fo9/OX8ge86keqbyjPUzdG7c2azZ+H/iOX/uPJ07m26L+yeOQ8cO8UbXN4xltzNvM3vZbGo9WYtm3s3u2eWslVz58VAERYMHD2bNmjVs27aNihXzvoCqadOmAJw6dYpq1arh5eXFnj17TOpcvGi4o27WgdfLy8tYdmcdFxcX7O3tsbS0xNLSMsc6uR28bW1tsbW1NSu3trbO9UAhGaGEEEKUZlZWVvJl+jGQ13cd8ZiK+gM2jICUC2icHMksXxbLOU9i3Wky+HdjwdEF/HD0B+Z2mEsD9wYmuyql2J6wnUltJuU4bywtLdFoNGbbnvR6ku+Pfk/irUR8XQzJHk7fOA2Ar6tvvuZgQeZpid68VSnF4MGDWbVqFX/99RdVqlS55z6HDh0CMF6T07x5c44cOUJSUnaWi82bN+Pi4oK/v7+xTnh4uEk7mzdvpnnz5gDY2NjQuHFjkzp6vZ7w8HBjnUfBokWLcHNzu+92NBoNq1evvu92svTp04fnn3++yNp7GJw5cwaNRmOcj0Vl7NixBAQE5Fnnfl/P4up7Qc2bNw9fX18sLCyYMWNGvvYp6rlZEvLzO87L3b//wMBA4/WRpcnDPO6SfI89Du8RIcTDY83pNWzY/jmnV73F2bSLbHB0YGYZV4JvpmGdkgDL3mB++Id8e/BbxrccTwWnCly+dZnLty6TpjUsXbugu8Bt3W0aejQ0aTs+JZ4TV09w+dZl0nXpnLh6ghNXT6DVGa4DaubTjDpl6zB6x2iOXznOsSvHGL9zPM29m1PZtXKRj7VET10MGjSIJUuW8Pvvv+Ps7Gy8BsjV1RV7e3tiYmJYsmQJnTt3ply5cvzzzz8MGzaMNm3aUL9+fQA6duyIv78/vXv3ZsqUKSQmJvLJJ58waNAg45mcd999l2+//Zbhw4fz1ltv8ddff7Fs2TLWrl1r7EtoaCghISE8+eSTNGnShBkzZnDz5k1jNroHoU+fPly/fl0+0IqIRqNh1apV9x2QPWy/l5kzZ6JU9s3RAgMDCQgIyHdg4evrS0JCAuXLly+mHt5bSkoKgwcPZtq0afTo0SPf2agSEhIoU6ZMvp9n0aJFDB06lOvXrxeypw+/lStX5vsvYQWdK4+SypUrM3To0BINlN58800qVKhAv379Crzv2LFjWb16dZEGUmfOnKFKlSocPHjwvgJxIUTpZYUFC6L/jzgfTxTgk5nJKymp9DYuS9OwLG4DWksNoRGhJvsOaDCA/nX7c1x7nJY+LbGyMA07xuwYw76L+4yPX/rzJQA29NhABacKWGgs+Lb9t0zcPZE+G/pgb2VPqwqt+Oipj4pprCVozpw5gOGD+k4LFy6kT58+2NjYEBYWZgxQfH196dGjB5988omxrqWlJWvWrGHAgAE0b94cR0dHQkJCTO5rVKVKFdauXcuwYcOYOXMmFStW5IcffjCm4wbo1asXly5d4tNPPyUxMZGAgAA2bNhglnxBiJJ2v+lsLS0tS3xNf3x8PFqtli5duhQoE2NJ9Vun06HRaLCwKNGT6zm6O1unKBk6nY41a9aY/LFNCCEedZ00znQ6ezaPGoqN8WchZA1UMU+8oNVqOa49zrAKw8y2Ley08J7P7+HgwfR20wvS5UIr8eVzOf306dMHMPxFe+vWrVy5coXbt29z8uRJpkyZYpbIwM/Pj3Xr1pGWlsalS5f46quvzK7fCQwM5ODBg6SnpxMTE2N8jjsNHjyYuLg40tPT2b17t/H6pYfFtGnTqFevHo6Ojvj6+jJw4EBSU1PN6q1evZoaNWpgZ2dHcHAwZ++azL///juNGjXCzs6OqlWrMm7cOGO68LtlZGQwePBgvL29sbOzw8/Pj4kTJ+baR51OR2hoKG5ubpQrV47hw4ebnNUAw9LEiRMnUqVKFezt7WnQoAHLly83bo+IiECj0RAeHs6TTz6Jg4MDLVq0IDraNFvJnDlzqFatGjY2NtSqVYuffvrJuK1y5coAdO/eHY1GY3xc0PGPHTuWH3/8kd9//x2NRoNGoyEiIsK4/fTp07Rr1w4HBwcaNGhglsJ9+/bttG7dGnt7e3x9fXn//fe5efNmrq9flu+++w5fX18cHBzo2bOnMV09mC6f6tOnD1u3bmXmzJnG/p05c4Zr167x2muv4e7ujr29PTVq1GDhQsPB5+6lPX369DHue+dP1jjT09P58MMPqVChAo6OjjRt2tTkNchJfHw8zz33HE5OTri4uNCzZ0/jNXuLFi2iXr16AFStWtXY5/y4c2lQ1jhWrlyZ4+8gIiKCN998k+TkZOOYxo4dm68xZS1F/eOPP/D398fW1pb4+HgqV67MF198wVtvvYWzszOVKlVi3jzTtKAjRoygZs2aODg4ULVqVUaPHl3o9LX5eT/dvYxs9uzZxve/p6cnL774IpD7XNHpdPTt29f4fqxVqxYzZ840eY6sOffVV1/h7e1NuXLlGDRokMm40tPTGTFiBL6+vtja2lK9enXmz59v3H706FGeeeYZnJyc8PT0pHfv3ly+fDlfr8PNmzd54403cHJywtvb23jPuTtfg7i4OIYNG2Yc282bN3FxcTE5toDh+Ojo6MiNGzeMc2jp0qW0aNECOzs7nnjiCbZu3WqyT376vmPHDqytrXnqqafM+p/T0ubVq1ej0WiM28eNG8fhw4eN/V+0aNE9X5eTJ0/Spk0b7Ozs8Pf3N8sIlrUkvWHDhmg0GgIDA9m2bRvW1tbGlRlZhg4dSuvWrU36W5SfJUKIR1TqxXvXyaOeVqelrk1dWvq0LMJOFRMlikRycrICVHJystm2jIwMtXr1ahUfH6/Gjh2rLly4kGMbISEh6rnnnsv1OaZPn67++usvFRsbq8LDw1WtWrXUgAEDjNsXLlyorK2t1ZNPPql27Nih9u3bp5o0aaJatGhhrLNt2zbl4uKiFi1apGJiYtSmTZtU5cqV1dixY411ALVq1SqllFJffvml8vX1Vdu2bVNnzpxRf//9t1qyZEmufZw8ebIqU6aMWrFihYqKilJ9+/ZVzs7OJuP67LPPVO3atdWGDRtUTEyMWrhwobK1tVURERFKKaW2bNmiANW0aVMVERGhjh07plq3bm0yjpUrVypra2s1a9YsFR0draZOnaosLS3VX3/9pZRSKikpSQFq4cKFKiEhQSUlJeV7/He6ceOG6tmzp+rUqZNKSEhQCQkJKj09XcXGxipA1a5dW61Zs0ZFR0erF198Ufn5+SmtVquUUurUqVPK0dFRTZ8+Xf37778qMjJSNWzYUPXp0yfX12/MmDHK0dFRPf300+rgwYNq69atqnr16urVV1811rlznly/fl01b95c9e/f39i/zMxMNWjQIBUQEKD27t2rYmNj1ebNm9Uff/yhlFLGvh88eNDYRta+CQkJasiQIcrDw0MlJCQopZTq16+fatGihdq2bZs6deqU+vLLL5Wtra36999/cxyDTqdTAQEBqlWrVmrfvn1q165dqnHjxqpt27ZKKaXS0tJUWFiYAtSePXuMfQ4JCTHWyc2dc/Nev4P09HQ1Y8YM5eLiYhzbjRs38jWmrPdSixYtVGRkpDpx4oS6efOm8vPzU2XLllWzZs1SJ0+eVBMnTlQWFhbqxIkTxj5OmDBBRUZGqtjYWPXHH38oT09PNXnyZJPfcYMGDfIcZ5b8vJ/atm2rhgwZopRSau/evcrS0lItWbJEnTlzRh04cEDNnDlTKZX7XMnIyFCffvqp2rt3rzp9+rT6+eeflYODg/r111+NzxESEqJcXFzUu+++q44fP67+/PNP5eDgoObNm2es07NnT+Xr66tWrlypYmJiVFhYmFq6dKlSSqlr164pd3d3NWrUKHX8+HF14MAB1aFDB9WuXbt8vQ4DBgxQlSpVUmFhYeqff/5Rzz77rHJ2djaO+8qVK6pixYpq/PjxxrEppVT//v1V586dTdrq1q2beuONN5RS2XOoYsWKavny5SoqKkr169dPOTs7q8uXLxeo7x9++KF6++23TdrNeo8tXLhQubq6mtRftWqVyvoITktLUx988IGqW7eusf9paWl5viY6nU498cQTqn379urQoUNq69atqmHDhibvkT179ihAhYWFqYSEBHXlyhWllFI1a9ZUU6ZMMbaVkZGhypcvrxYsWGDsb1F8ltzpwoULeX7+iUdD1veZjIyMku6KeFBOb1NqjMu9f05vy3H3kp4zeX0/v5sERUXkQQRFd/vtt99UuXLljI8XLlyoALVr1y5j2fHjxxWgdu/erZRSqn379uqLL74waeenn35S3t7exsd3fqi+99576umnn1Z6vT5fffL29jb5sNVqtapixYrGcd2+fVs5ODioHTt2mOzXt29f9corryilsoOisLAw4/a1a9cqQN26dUsppVSLFi1U//79Tdp46aWXTL4A3TmOLPkZ/91y+r1kfen54YcfjGXHjh1TgDp+/LhxTFlfkrL8/fffysLCwjiOu40ZM0ZZWlqqc+fOGcvWr1+vLCwsjF/07u7PnV+Ks3Tt2lW9+eabOT7H3V/Y7rRixQplZ2entm/frpRSKi4uTllaWqrz58+b1Gvfvr0aNWpUju1v2rRJWVpaqvj4eGNZ1muzZ88epZRSBw8eVICKjY011hk5cqTq3bt3jm1mySkoyut3kNOX0fyMKeu9dOjQIZM6fn5+6vXXXzc+1uv1ysPDQ82ZMyfXPn/55ZeqcePGxscFCYru9X5SyvT3v2LFCuXi4qJSUlJybC+nuZKTQYMGqR49ehgfh4SEKD8/P5WZmWkse+mll1SvXr2UUkpFR0crQG3evDnH9iZMmKA6duxoUnb27FkFqOjo6Dz7cuPGDWVjY6OWLVtmLLty5Yqyt7c3GYufn5+aPn26yb67d+9WlpaWxmPuxYsXlZWVlfEPMFlzaNKkScZ9sl7jrEA2v32vUaOGWrNmjUm7+Q2KlCrYvFBKqY0bNyorKyuTebx+/foc3yN3v9cnT56s6tSpY3y8YsUK5eTkpFJTU439LYrPkjtJUPR4KOkvuKIE6DKVmlpbqTGuuQRErkpNrWOol4OSnjMFCYoevgXyIldhYWG0b9+eChUq4OzsTO/evbly5YrJjamsrKxMlm/Url0bNzc3jh8/DsDhw4cZP348Tk5Oxp/+/fuTkJCQ4w2u+vTpw6FDh6hVqxbvv/8+mzZtyrV/ycnJJCQkmCw7tLKy4sknnzQ+PnXqFGlpaXTo0MGkD4sXLyYmJsakvaxkGpCdbTAry+Dx48dp2dL0VGzLli2N48xNQcd/L3n18fDhwyxatMjkuYKDg9Hr9cTGxubaZqVKlahQoYLxcfPmzdHr9WbLB/MyYMAAli5dSkBAAMOHD2fHjh333OfgwYP07t2bb7/91vjaHjlyBJ1OR82aNU3GsXXrVrPfV5bjx4/j6+uLr6+vsczf399kHuZk4sSJLF68ON9jzJLX7yAn+R2TjY2NSds5PZ9Go8HLy8vk+X799VdatmyJl5cXTk5OfPLJJ8THxxd4XPl5P92tQ4cO+Pn5UbVqVXr37s0vv/ySr3k9a9YsGjdujLu7O05OTsybN8+sz3Xr1sXS0tL42Nvb2zjuQ4cOYWlpSdu2bXNs//Dhw2zZssXk9a5duzZArvMoS0xMDBkZGSavQ9myZalVq9Y9x9WkSRPq1q3Ljz/+CMDPP/+Mn58fbdq0Mal3Z5bRrNf4zmPmvfp+/PhxLly4QPv27e/Zp6KS9T7z8fHJcRx56dOnD6dOnTLeUHzRokX07NkTR0dHY52i/iwRQjyiLCyh0+T/Hmju2vjf406TDPUecXLjnEfEmTNnePbZZxkwYACff/45ZcuWZfv27fTt25eMjIx83yE8NTWVcePG8cILL5hts7OzMytr1KgRsbGxrF+/nrCwMHr27ElQUJDZOv38yroGau3atSZf/AGz+z7dmVEra+29Xq8v1PPe+fwFGf+95NXH1NRU3nnnHd5//32z/SpVqlTg5yqIZ555hri4ONatW8fmzZtp3749gwYN4quvvsqxfmJiIt26daNfv3707dvXWJ6amoqlpSX79+83+UIM4OTkVKxjyK+CzpP8jsne3t7YXm7Pl/WcWc+3c+dOXnvtNcaNG0dwcDCurq4sXbrU7BqY4uLs7MyBAweIiIhg06ZNfPrpp4wdO5a9e/fmmq5/6dKlfPjhh0ydOpXmzZvj7OzMl19+ye7du03q5TVue3v7PPuVmppK165dmTx5stm2giTaKIx+/foxa9YsRo4cycKFC3nzzTdz/L3mJj99/+OPP+jQoUOuxxALCwuza8EKe51ZUfDw8KBr164sXLiQKlWqsH79+nteJ3i3oj6WCiEeYv7doOdi432KjFx8DAGRfzeT6jq9jgNJB7iUdokyNmXQq/v77vagSFD0iNi/fz96vZ6pU6caM2AtW7bMrF5mZib79u2jSZMmAERHR3P9+nXq1KkDGIKc6Ohoqlevnu/ndnFxoVevXvTq1YsXX3yRTp06cfXqVbOsV66urnh7e7N7927jX2IzMzPZv38/jRo1AjC5aD23vyrnR506dYiMjCQkJMRYFhkZabw3FRi+xOl0OpP9CjN+Gxsbs3byo1GjRkRFRRXoucCQpODChQvGvwDv2rULCwuLXP8ynlv/3N3dCQkJISQkhNatW/PRRx/lGBTdvn2b5557jtq1azNt2jSTbQ0bNkSn05GUlGS8CPte6tSpw9mzZzl79qzxbFFUVBTXr183+f08CDm9NoUZU37t2LEDPz8/Pv74Y2NZXFxcodrKz/spJ1ZWVgQFBREUFMSYMWNwc3Pjr7/+4oUXXsjx9YiMjKRFixYMHDjQWHavszd3q1evHnq9nq1btxIUFGS2vVGjRqxYsYLKlSsX+CbW1apVw9ramt27dxv/mHDt2jX+/fdfk2NIbu+D119/neHDh/P1118TFRVlcszIsmvXLrPXePDgwfnu+++//87bb7+d6xjc3d25ceMGN2/eNJ6NuTv1dkGPM1nvs4SEBGNwlnXm5842gRzb7devH6+88goVK1akWrVqZmfei+OzRAjxCPPvBrW7QNwOQ1IFJ0/wa2F2higsLoxJeyZxMS078YKLxgX7s/Z0qtrpQfe6QCQoesgkJyebfViWK1eO6tWro9Vq+eabb+jatSuRkZHMnTvXbH9ra2vee+89vv76a6ysrBg8eDDNmjUzfrB9+umnPPvss1SqVIkXX3wRCwsLDh8+zNGjR/nss8/M2ps2bRre3t40bNgQCwsLfvvtN7y8vHL9q/OQIUOYNGkSNWrUMH7JvvMeMc7Oznz44YcMGzYMvV5Pq1atSE5OJjIyEhcXlxy/sOTko48+omfPnjRs2JCgoCD+/PNPVq5cSVhYmLFO5cqVCQ8Pp2XLltja2lKmTJkCjz+rnY0bNxIdHU25cuXynRJ7xIgRNGvWjMGDB9OvXz8cHR2Jiopi8+bNfPvtt7nuZ2dnR0hICF999RUpKSm8//779OzZM9d01JUrV2b37t2cOXMGJycnypYty9ixY2ncuDF169YlPT2dNWvWGL/M3O2dd97h7NmzhIeHc+nSJWN52bJlqVmzJq+99hpvvPEGU6dOpWHDhly6dInw8HDq169Ply5dzNoLCgqiXr16vPbaa8yYMYPMzEwGDhxI27Zt81z6NWrUKM6fP1+oJXS5qVy5MqmpqYSHh9OgQQMcHBwKNab8qlGjBvHx8SxdupSnnnqKtWvXsmrVqkK3d6/3093WrFnD6dOnadOmDWXKlGHdunXo9XpjQJ3TXKlRowaLFy9m48aNVKlShZ9++om9e/fm62baWSpXrkxISAhvvfUWX3/9NQ0aNCAuLo6kpCR69uzJoEGD+P7773nllVcYPnw4ZcuW5dSpUyxdupQffvjB7IzdnZycnOjbty8fffQR5cqVw8PDg48//tgsPXrlypXZtm0bL7/8Mra2tsb7cJUpU4YXXniBjz76iI4dO1KxYkWz55g1axY1atSgTp06TJ8+nWvXrvHWW28B3LPvV65cYd++ffzxxx+5jqFp06Y4ODjwv//9j/fff5/du3ebZZerXLkysbGxHDp0iIoVK+Ls7Gx29vxOQUFB1KxZk5CQEL788ktSUlJMgnEwnBGyt7dnw4YNVKxYETs7O+PxKzg4GBcXFz777DOTW1hkKerPEiFKncsnYc0wuHQCbqeAsxfUewkCR4Llf2fe9y+Cw0shKcrw2DsA2o+Bio3N21v0rGH/Ol1hRT+4eAxuXQVHd6jVGdp/Cnb/ZWeO+gP2zYfEI5CZAR61Dc9b3fyPVgViYZlj2u0sYXFhhEaEojA9M56iUhj+93CsLK0I8rvPPhSn4r/EqXQoqkQLgNlP3759lVJKTZs2TXl7eyt7e3sVHBysFi9erAB17do1pVT2xbwrVqxQVatWVba2tiooKEjFxcWZPM+GDRtUixYtlL29vXJxcVFNmjQxySLFHRfqzps3TwUEBChHR0fl4uKi2rdvrw4cOJDr66DVatWQIUOUi4uLcnNzU6GhoeqNN94wuTBcr9erGTNmqFq1ailra2vl7u6ugoOD1datW5VS2YkWssalVM4X5s+ePVtVrVpVWVtbq5o1a6rFixeb9OWPP/5Q1atXV1ZWVsrPzy/f479bUlKS6tChg3JyclKA2rJlS44XMF+7ds24PcuePXuM+zo6Oqr69eurzz//PNfnyrrYevbs2crHx0fZ2dmpF198UV29etVY5+5EC9HR0apZs2bK3t7e+BpNmDBB1alTR9nb26uyZcuq5557Tp0+fVopZX7xtZ+fX47zLmscWdnJKleurKytrZW3t7fq3r27+ueff3IdR1xcnOrWrZtydHRUzs7O6qWXXlKJiYnG7Tn9Pgubfe5ev4N3331XlStXTgFqzJgx+RpTThfGZ71Wd1/M36BBA2O7Sin10UcfqXLlyiknJyfVq1cvNX36dJO2CnJBfX7eT3cmT/j7779V27ZtVZkyZZS9vb2qX7++SRa5nObK7du3VZ8+fZSrq6tyc3NTAwYMUCNHjjTpY07JRoYMGWLy+7p165YaNmyY8vb2VjY2Nqp69erGbGZKKfXvv/+q7t27Kzc3N2Vvb69q166thg4dmq8kLjdu3FCvv/66cnBwUJ6enmrKlClmSSN27typ6tevr2xtbdXdH23h4eEKMEnWoFT2HFqyZIlq0qSJsrGxUf7+/sYslvnp+w8//KBatmyZY7t3zs1Vq1ap6tWrK3t7e/Xss8+qefPmmfTz9u3bqkePHsrNzc2YOfNeoqOjVatWrZSNjY2qWbOm2rBhg1mCme+//175+voqCwsLs/fX6NGjTRJRZCmqz5I7SaKFx0NJXzT/SLlyWqkDPymV8I9S1+KUOr5WqSnVlNp8R4bG5X2V2j1PqQuHlUqKVmrVAKW+8FUq2TQRkLp5Ralx5ZS6cVGptKtK7fleqXP7De3GbFHq68ZK/fZWdv11I5T6e7pS5/YpdfmU4TnHlVPqgmnyoKKUqctU7Ze1V08seiLHn3qL6qmgZUEqM5eEDMWlIIkWNErdtdBZFEpKSgqurq4kJyeb3UdJq9Wybt06GjVqxIIFC3j77beLfR29EEIIg59++olhw4Zx4cIF45IyMFyrWaVKFQ4ePEhAQECh2u7WrRutWrVi+PDhRdTbB6dv375cunTJ7CzXokWLGDp0aJ5nJQsqISGBefPmyeffIy7r+0znzp3NrjMU+bDhf3DhALy1Iefteh1M8oPOX0LAK9nlh5fCnu+hf3jO++2aCzu+htCo3J97VlOo+wIEjih8//OwN3Evb2186571FgQv4Ckv8/u5FZe8vp/fTZbPCSGEeCylpaWRkJDApEmTeOedd0wCoqLSqlUrXnnllXtXfIgkJydz5MgRlixZkueyPyFEEboSA6fCDMvfcqNNA70W7MuYlkevg9qdc94nJQGO/wl+edwcVa+H9FTzdovQpbRL965UgHolQVJyCyFECbkznfHdP3///XdJd++BiI+Pz/N1KEw68yxTpkyhdu3aeHl5MWrUqCLsdbbhw4ebpJ8vKr/88kuur0ndunXvq+3nnnuOjh078u6779KhQ4ci6rEQIkc/dIAJHvBNI/BrDu0+zr3u5jGGa4+qBmaXZabDqXDDdUN3Wv4WfOYF02qDrTN0+yb3dnd8DRmpULf7fQ0lL+4O7kVaryTImSIhhCghdydVudPdKesfVz4+Pnm+Dnfeh6egxo4dy9ixY3PdXrlyZbNU2Q+Lbt26mdyb6U73u2zpXum3+/TpQ58+fe7rOYQQ/3lpoeEszcWjsGk0lPkaWg01r/f3NDi6AvqsBes70trHbgPH8uBxV7Kk4InQdiRcOQXh42Dj/+BZ0wyyAPzzG2ydDC8vAafiC0gaeTTC08GTpLQks0QLABo0eDp40sgj9+ypJU2CIiGEKCGSztiQQlxeB3POzs44OzuXdDeEEPfL9b+Mlx61DdcM/TkEWrxnmso68mvYPgPeWA1eT5juH73O/CwRgLOn4ce9pmFZ3MJO0Ha44UxTliPL4Y/3oOePUK1dUY/MhKWFJSObjCQ0IhQNmhwDoxFNRmD5EN/kVZbPCSGEEEIIUdyU3nDN0J03M90+A7Z9Ca+vgAp3nUVRCqI35BwU3d0uGJbaZTmyHH4fBC/Oh5rBRdL9ewnyC2Ja4DQ8HDxMyl01rkxpPeXhTseNnCkSQgghhBCiaP2zDCyswLMuWNrAhYOGZW51X8i+T9H26bDlC+jxA7hVghv/3fDUxhFsnQz7aNOgUvPsdv/dBDeTwKeRod6lE4Zleb7NoIzff8/9G6x+FzpNggpPZrdrbQd2+bvXYm5ik2OZsGsCMddjSM1Ixd3Bnc5VOjMgYADWFtYE+QVx9fZVlp5YyrnUc2jQUE6Vw9PeM8f23tr4Fl2qdKF9pfaM/Hsk/177l+vp1ylrV5Z2vu0Y0mgITjZOgOE+SL9G/0r01Wgy9BlUc6vGwAYDaVkhjyQTBSBBUQm48waZQgghxONOPvdEqWNhCZEzDFnnlAI3X2jSH5oNyq6zdwHoMmDZG6b7th0J7UYZls7V6AiWd3xdt7aD/T8a0nvr0sGlgiGjXath2XX2LwJ9Jqz70PCTpcGr0H3OfQ3LysKKrlW74l/OH2cbZ6KvRjN251gUiiGNhhie/uJ+etbqSYBHABZ6Cz7b/BkDtwxk9XOr8XTMDo6S05M5mHSQKW2moNFoaOfbjvcavkcZuzLE34jn812fk7wrmSltphjbbe7TnCGNhuBs48zqU6sZ/NdglnReQp1yOd+gvkBju+8WRL7Z29tjbW19X3e4F0IIIR5F1tbWODg4lHQ3hHgwnuhh+MnLsCN5bz+xDtp8aFpWpQ3025z3fm+uvXf/CsnX2Rdf5+yMmz5OPuy9uJcDFw8Yyya3mWz8v1arpbt9dybfnMzuxN10q9bNuG3buW34l/WnvH15AHrV7mXS7su1X2bh0YXGshFNTO+xNKTRELbEbyHiXIQERY8aV1dXBg0aRFpaWkl3RTzkMjMz2b59O61atcLKSt6m4t5kzoiCetBzxsHBAVfX+1u6I0SpkZkB/t2gxsOdNj8+JZ7I85G0r9Q+1zpatGSqTFxtTN//W85uoV2lnBNAJKUlERYXxpNeT+barl7puZl506zdwpJPzgfM1dVVPhTEPWm1WhwcHPDy8pK7hot8kTkjCkrmjBAPMSsbCBxZ0r3I1evrXuf4leNk6DN4seaLDG44ONe6G29txN3enWY+zYxlGboMIs9HMrDBQJO6w7cOZ8vZLdzW3SawYiDjWozLtd1FxxaRpk0juHLRJJKQ7HNCCCGEEEKIfPuq7Vcs67qMya0ns+3cNhYdW5RjvYXHFnJEe4SvWn+FraWtsXx3wm7K2pWlehnTWzIMbzKcX7v+ytftvubsjbN8uffLHNtde3otcw/P5au2X1HOvlyRjEnOFAkhhBBCCCHyzcvRcD+kam7V0Ckd43eOJ8Q/xOQ+RIuOLmJh1EL6OPahZpmaJvtHnI0g0DfQrN3y9uUpb1+eqq5VcbV1JWRDCO/Ufwd3h+wbz66PXc/YHWOZGjiV5j7NzdooLDlTJIQQQgghhCgUhSJTn4me7PsvLTi6gO/++Y5v231LBasKpvWVIuJcBE9XejrPdvX/3X8pQ59hLFt3eh2jI0czuc1k2lRsU4SjkDNFQgghhBBCiHxYc3oNVhZW1HSribWlNceuHGPm/pkEVwnG2sJwbeL8I/OZdWgWk9tMxsfRh+P641y+dRlXXHGwdiDqShS3M2/T0KOhsd1t57Zx5dYVnij/BA7WDsRcj2Hqvqk09GhIBSdDULX29Fo+2f4JI5qMoL57fS7fugyAraUtzjbO9z02CYqEEEIIIYQQ92SlsWLBkQXEpcShUPg4+vBKnVfo7d/bWGdZ9DK0ei2hEaHGssmrJjOgwQAGBgzkr7N/0bpia6wsssMQO0s7VpxcwZd7vyRDn4GXoxftK7Wnb72+xjrL/11Opsrk892f8/nuz43l3ap14/NW2Y8LPbb7bkEIIYQQQgjx2OtUpROdqnTKs87GFzca/6/Valm3bh2dO3c2ZrnccnYLb9d/22SfJt5N+Nn75zzbXdhpYZ7b75cERUVEKQVASkqK2TatVktaWhopKSmS9lTki8wZUVAyZ0RByZwRBSVzRhTU3XNGq9fSxrMNDZwb5PiduahlPUfW9/S8aFR+aol7OnfuHL6+vveuKIQQQgghhHhgzp49S8WKFfOsI0FREdHr9Vy4cAFnZ2c0Go3JtpSUFHx9fTl79iwuLi4l1EPxKJE5IwpK5owoKJkzoqBkzoiCKuk5o5Tixo0b+Pj4YGGRd9JtWT5XRCwsLO4Zgbq4uMhBRBSIzBlRUDJnREHJnBEFJXNGFFRJzhlXV9d81ZP7FAkhhBBCCCFKNQmKhBBCCCGEEKWaBEUPgK2tLWPGjMHW1rakuyIeETJnREHJnBEFJXNGFJTMGVFQj9KckUQLQgghhBBCiFJNzhQJIYQQQgghSjUJioQQQgghhBClmgRFQgghhBBCiFJNgiIhhBBCCCFEqSZBUSHNmjWLypUrY2dnR9OmTdmzZ0+udQMDA9FoNGY/Xbp0Mdbp06eP2fZOnTo9iKGIB6QgcwZgxowZ1KpVC3t7e3x9fRk2bBi3b9++rzbFo6Wo58zYsWPNjjO1a9cu7mGIB6ggc0ar1TJ+/HiqVauGnZ0dDRo0YMOGDffVpnj0FPWckePM42vbtm107doVHx8fNBoNq1evvuc+ERERNGrUCFtbW6pXr86iRYvM6jw0xxglCmzp0qXKxsZGLViwQB07dkz1799fubm5qYsXL+ZY/8qVKyohIcH4c/ToUWVpaakWLlxorBMSEqI6depkUu/q1asPaESiuBV0zvzyyy/K1tZW/fLLLyo2NlZt3LhReXt7q2HDhhW6TfFoKY45M2bMGFW3bl2T48ylS5ce1JBEMSvonBk+fLjy8fFRa9euVTExMWr27NnKzs5OHThwoNBtikdLccwZOc48vtatW6c+/vhjtXLlSgWoVatW5Vn/9OnTysHBQYWGhqqoqCj1zTffKEtLS7VhwwZjnYfpGCNBUSE0adJEDRo0yPhYp9MpHx8fNXHixHztP336dOXs7KxSU1ONZSEhIeq5554r6q6Kh0RB58ygQYPU008/bVIWGhqqWrZsWeg2xaOlOObMmDFjVIMGDYqlv6LkFXTOeHt7q2+//dak7IUXXlCvvfZaodsUj5bimDNynCkd8hMUDR8+XNWtW9ekrFevXio4ONj4+GE6xsjyuQLKyMhg//79BAUFGcssLCwICgpi586d+Wpj/vz5vPzyyzg6OpqUR0RE4OHhQa1atRgwYABXrlwp0r6LklGYOdOiRQv2799vPIV8+vRp1q1bR+fOnQvdpnh0FMecyXLy5El8fHyoWrUqr732GvHx8cU3EPHAFGbOpKenY2dnZ1Jmb2/P9u3bC92meHQUx5zJIscZAbBz506T+QUQHBxsnF8P2zFGgqICunz5MjqdDk9PT5NyT09PEhMT77n/nj17OHr0KP369TMp79SpE4sXLyY8PJzJkyezdetWnnnmGXQ6XZH2Xzx4hZkzr776KuPHj6dVq1ZYW1tTrVo1AgMD+d///lfoNsWjozjmDEDTpk1ZtGgRGzZsYM6cOcTGxtK6dWtu3LhRrOMRxa8wcyY4OJhp06Zx8uRJ9Ho9mzdvZuXKlSQkJBS6TfHoKI45A3KcEdkSExNznF8pKSncunXroTvGSFD0gM2fP5969erRpEkTk/KXX36Zbt26Ua9ePZ5//nnWrFnD3r17iYiIKJmOihIVERHBF198wezZszlw4AArV65k7dq1TJgwoaS7Jh5S+ZkzzzzzDC+99BL169cnODiYdevWcf36dZYtW1aCPRclZebMmdSoUYPatWtjY2PD4MGDefPNN7GwkK8GImf5mTNynBGPKjnyFVD58uWxtLTk4sWLJuUXL17Ey8srz31v3rzJ0qVL6du37z2fp2rVqpQvX55Tp07dV39FySvMnBk9ejS9e/emX79+1KtXj+7du/PFF18wceJE9Hr9fc1D8fArjjmTEzc3N2rWrCnHmcdAYeaMu7s7q1ev5ubNm8TFxXHixAmcnJyoWrVqodsUj47imDM5keNM6eXl5ZXj/HJxccHe3v6hO8ZIUFRANjY2NG7cmPDwcGOZXq8nPDyc5s2b57nvb7/9Rnp6Oq+//vo9n+fcuXNcuXIFb2/v++6zKFmFmTNpaWlmf621tLQEQCl1X/NQPPyKY87kJDU1lZiYGDnOPAbu55hgZ2dHhQoVyMzMZMWKFTz33HP33aZ4+BXHnMmJHGdKr+bNm5vML4DNmzcb59dDd4x54KkdHgNLly5Vtra2atGiRSoqKkq9/fbbys3NTSUmJiqllOrdu7caOXKk2X6tWrVSvXr1Miu/ceOG+vDDD9XOnTtVbGysCgsLU40aNVI1atRQt2/fLvbxiOJX0DkzZswY5ezsrP7v//5PnT59Wm3atElVq1ZN9ezZM99tikdbccyZDz74QEVERKjY2FgVGRmpgoKCVPny5VVSUtIDH58oegWdM7t27VIrVqxQMTExatu2berpp59WVapUUdeuXct3m+LRVhxzRo4zj68bN26ogwcPqoMHDypATZs2TR08eFDFxcUppZQaOXKk6t27t7F+Vkrujz76SB0/flzNmjUrx5TcD8sxRoKiQvrmm29UpUqVlI2NjWrSpInatWuXcVvbtm1VSEiISf0TJ04oQG3atMmsrbS0NNWxY0fl7u6urK2tlZ+fn+rfv7986DxmCjJntFqtGjt2rKpWrZqys7NTvr6+auDAgSYfPPdqUzz6inrO9OrVS3l7eysbGxtVoUIF1atXL3Xq1KkHOCJR3AoyZyIiIlSdOnWUra2tKleunOrdu7c6f/58gdoUj76injNynHl8bdmyRQFmP1lzJCQkRLVt29Zsn4CAAGVjY6OqVq1qco/OLA/LMUajVC7rKoQQQgghhBCiFJBrioQQQgghhBClmgRFQgghhBBCiFJNgiIhhBBCCCFEqSZBkRBCCCGEEKJUk6BICCGEEEIIUapJUCSEEEIIIYQo1SQoEkIIIYQQQpRqEhQJIYQQQgghSjUJioQQQoj7MHbsWAICAoyP+/Tpw/PPP19i/RFCCFFwEhQJIYQQQgghSjUJioQQQjy2MjIySroLQgghHgESFAkhhHhsBAYGMnjwYIYOHUr58uUJDg7m6NGjPPPMMzg5OeHp6Unv3r25fPmycR+9Xs+UKVOoXr06tra2VKpUic8//9y4fcSIEdSsWRMHBweqVq3K6NGj0Wq1JTE8IYQQxUSCIiGEEI+VH3/8ERsbGyIjI5k0aRJPP/00DRs2ZN++fWzYsIGLFy/Ss2dPY/1Ro0YxadIkRo8eTVRUFEuWLMHT09O43dnZmUWLFhEVFcXMmTP5/vvvmT59ekkMTQghRDHRKKVUSXdCCCGEKAqBgYGkpKRw4MABAD777DP+/vtvNm7caKxz7tw5fH19iY6OxtvbG3d3d7799lv69euXr+f46quvWLp0Kfv27QMMiRZWr17NoUOHAEOihevXr7N69eoiHZsQQojiY1XSHRBCCCGKUuPGjY3/P3z4MFu2bMHJycmsXkxMDNevXyc9PZ327dvn2t6vv/7K119/TUxMDKmpqWRmZuLi4lIsfRdCCFEyJCgSQgjxWHF0dDT+PzU1la5duzJ58mSzet7e3pw+fTrPtnbu3Mlrr73GuHHjCA4OxtXVlaVLlzJ16tQi77cQQoiSI0GREEKIx1ajRo1YsWIFlStXxsrK/COvRo0a2NvbEx4enuPyuR07duDn58fHH39sLIuLiyvWPgshhHjwJNGCEEKIx9agQYO4evUqr7zyCnv37iUmJoaNGzfy5ptvotPpsLOzY8SIEQwfPpzFixcTExPDrl27mD9/PmAImuLj41m6dCkxMTF8/fXXrFq1qoRHJYQQoqhJUCSEEOKx5ePjQ2RkJDqdjo4dO1KvXj2GDh2Km5sbFhaGj8DRo0fzwQcf8Omnn1KnTh169epFUlISAN26dWPYsGEMHjyYgIAAduzYwejRo0tySEIIIYqBZJ8TQgghhBBClGpypkgIIYQQQghRqklQJIQQQgghhCjVJCgSQgghhBBClGoSFAkhhBBCCCFKNQmKhBBCCCGEEKWaBEVCCCGEEEKIUk2CIiGEEEIIIUSpJkGREEIIIYQQolSToEgIIYQQQghRqklQJIQQQgghhCjVJCgSQgghhBBClGr/DxwjjyFRq/BaAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "fig, ax = plt.subplots(1, 1, figsize=plt.figaspect(1/2))\n", - "fig.suptitle(\n", - " f'Effects of search parameters on QPS/recall trade-off ({DATASET_FILENAME})\\n' + \\\n", - " f'k = {k}, n_probes = {n_probes}, pq_dim = {pq_dim}')\n", - "labels = []\n", - "for j, ratio in enumerate(ratios):\n", - " ax.plot(bench_recall_sr[j, :], bench_qps_sr[j, :], 'o')\n", - " labels.append(f\"refine ratio = {ratio}\")\n", - "ax.legend(labels)\n", - "ax.set_xlabel('recall')\n", - "ax.set_ylabel('QPS')\n", - "ax.grid()\n", - "colors = plt.rcParams[\"axes.prop_cycle\"].by_key()[\"color\"]\n", - "annotations = []\n", - "for j, ratio in enumerate(ratios):\n", - " for i, label in enumerate(bench_names):\n", - " annotations.append(ax.text(\n", - " bench_recall_sr[j, i], bench_qps_sr[j, i],\n", - " f\" {label} \",\n", - " color=colors[j],\n", - " ha='center', va='center'))\n", - "clutter = [\n", - " ax.text(\n", - " 0.02, 0.08,\n", - " 'Labels denote the bitsize of: internal_distance_dtype/lut_dtype',\n", - " verticalalignment='top',\n", - " bbox={'facecolor': 'white', 'edgecolor': 'grey'},\n", - " transform = ax.transAxes)\n", - "]\n", - "adjust_text(annotations, objects=clutter);" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Depending on the dataset, you may see very different pictures here. For SIFT-128, we pick three interesting candidates candidates featuring compromizes between the QPS and the recall:\n", - " - `internal_distance_dtype = 16, lut_dtype = 16`\n", - " - `internal_distance_dtype = 32, lut_dtype = 8`\n", - " - `internal_distance_dtype = 32, lut_dtype = 8, refine_ratio = 2`\n", - "\n", - "This is all for the search parameters, but we will come back to the look-up table question in the next section." - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [], - "source": [ - "def search_refine(internal_distance_dtype, lut_dtype, ratio, n_probes):\n", - " k_search = k * ratio\n", - " ps = ivf_pq.SearchParams(\n", - " n_probes=n_probes,\n", - " internal_distance_dtype=internal_distance_dtype,\n", - " lut_dtype=lut_dtype)\n", - " candidates = ivf_pq.search(ps, index, queries, k_search, handle=resources)[1]\n", - " return candidates if ratio == 1 else refine(dataset, queries, candidates, k, handle=resources)[1]\n", - "\n", - "search_configs = [\n", - " lambda n_probes: search_refine(np.float16, np.float16, 1, n_probes),\n", - " lambda n_probes: search_refine(np.float32, np.uint8, 1, n_probes),\n", - " lambda n_probes: search_refine(np.float32, np.uint8, 2, n_probes)\n", - "]\n", - "search_config_names = [\n", - " '16/16', '32/8', '32/8/r2'\n", - "]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Tweaking indexing parameters\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Deciding on the indexing parameters is a bit more involved than on the search parameters. This is obviously because `ivf_pq.IndexParams` has more members than `ivf_pq.SearchParams`, but also because the try-test loop takes longer time when it includes training.\n", - "Since RAFT's IVF-PQ algorithm uses balanced-hierarchical k-means clustering and efficient logic for encoding, we find significantly improved index build times.\n", - "\n", - "First of all, let's pick the parameters we __don't need__ to tweak:\n", - "\n", - " - `metric` - the distance metric often depens on the problem and thus fixed (currently RAFT supports variations of eucliean and inner product distances).\n", - " - `conservative_memory_allocation` only affects how data is allocated - does not affect the search performance.\n", - " - `add_data_on_build` is a convenience flag. When activated, it automatically adds the training data to the index during `ivf_pq.build`. Otherwise, no data is added during `ivf_pq.build` and vectors need to be explicitly added to the index using `ivf_pq.extend`.\n", - " - `force_random_rotation` may slightly affect performance when the data dimensionality is a power of two (see the module docs), but normally you don't need to change the defaults. \n", - "\n", - "The rest of the parameters can be divided in two categories: influencing the coarse search (`kmeans_n_iters`, `kmeans_trainset_fraction` , `n_lists`) and the fine search / product quantization (`codebook_kind`, `pq_dim`, `pq_bits`)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Indexing parameters affecting the coarse search\n", - "\n", - "#### n_lists\n", - "\n", - "`n_lists` is the first parameter to look at. It has a profound impact on overall performance during both training and search.\n", - "`n_lists` defines the number of clusters into which the index data is partitioned; you should keep this in mind when selecting the `n_probes` search parameter.\n", - "\n", - "The ratio `n_probes/n_lists` tells how large fraction of the dataset is compared to each query. If `n_lists == n_probes`, that is like a brute force search: we compare all dataset vectors to all query vectors. One would expect the recall is equal to `1` in such a case, but that does not take into account the PQ compression, which is lossy; in reality the recall is always lower unless you refine the search results.\n", - "\n", - "As `n_probes` approaches `n_lists`, IVF-PQ becomes slower than brute force because of all the extra work the algorithm does: dimension padding / transform, two-step search, extra PQ compute, etc. In practice searching around 0.1-1% of lists is enough for many datasets. But this depends on how well the input can be clustered. (e.g. for uniform random numbers as inputs, IVF methods don't work well).\n", - "\n", - "`n_lists = sqrt(n_samples)` is a good starting point for the balance of coarse/fine search time. To make sure the GPU resources are utilized efficiently, keep in mind:\n", - " - The average cluster size (i.e. `n_smaples / n_lists`) should be in the range of at least ~2k records to keep individual SMs busy\n", - " - Total amount of search work (`n_queries * n_probes`) should be a good multiple of number of SMs\n" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "4.36 ms ± 2.38 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "4.36 ms ± 1.9 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "4.37 ms ± 2.47 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "7.74 ms ± 19.7 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "33.8 ms ± 733 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "44.1 ms ± 714 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "1.83 ms ± 1.66 µs per loop (mean ± std. dev. of 7 runs, 1,000 loops each)\n", - "3.1 ms ± 14.1 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "6.43 ms ± 16.8 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "11.9 ms ± 33 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "45.2 ms ± 622 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "87.3 ms ± 153 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "2.55 ms ± 452 ns per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "5.1 ms ± 11.1 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "9.32 ms ± 15.5 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "16.1 ms ± 34.2 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "74 ms ± 254 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "145 ms ± 295 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "3.92 ms ± 5.94 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "8.12 ms ± 6.62 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "14.7 ms ± 23.5 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "27.8 ms ± 131 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "132 ms ± 289 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "259 ms ± 3.04 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "7.49 ms ± 4.68 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "17.2 ms ± 48.5 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "32.4 ms ± 111 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "63 ms ± 149 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "303 ms ± 2.32 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "603 ms ± 1.78 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "n_list_variants = [100, 500, 1000, 2000, 5000]\n", - "pl_ratio_variants = [500, 200, 100, 50, 10, 5]\n", - "selected_search_variant = 1\n", - "search_fun = search_configs[selected_search_variant]\n", - "search_label = search_config_names[selected_search_variant]\n", - "\n", - "bench_qps_nl = np.zeros((len(n_list_variants), len(pl_ratio_variants)), dtype=np.float32)\n", - "bench_recall_nl = np.zeros_like(bench_qps_nl, dtype=np.float32)\n", - "\n", - "for i, n_lists in enumerate(n_list_variants):\n", - " index_params = ivf_pq.IndexParams(n_lists=n_lists, metric=metric, pq_dim=pq_dim)\n", - " index = ivf_pq.build(index_params, dataset, handle=resources)\n", - " for j, pl_ratio in enumerate(pl_ratio_variants):\n", - " n_probes = max(1, n_lists // pl_ratio)\n", - " r = %timeit -o search_fun(n_probes); resources.sync()\n", - " bench_qps_nl[i, j] = (queries.shape[0] * r.loops / np.array(r.all_runs)).mean()\n", - " bench_recall_nl[i, j] = calc_recall(search_fun(n_probes), gt_neighbors)" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzkAAAHgCAYAAACGvKPXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAADZlklEQVR4nOzdd3hUVfrA8e+0zGTSe2dSKEkAKUmkKE3pTSxgWwHRVde6+rOsu66KbVdR1wKuomvDDhZUQJqgFFFChyQQQhrpvZeZzP39McmQIQEChBbez/PcB3LunXvPvXOnvHPOeY9KURQFIYQQQgghhOgi1Oe6AkIIIYQQQgjRmSTIEUIIIYQQQnQpEuQIIYQQQgghuhQJcoQQQgghhBBdigQ5QgghhBBCiC5FghwhhBBCCCFElyJBjhBCCCGEEKJLkSBHCCGEEEII0aVIkCOEEEIIIYToUiTIEReU6upqbr/9dgIDA1GpVPz1r38FoKCggOuuuw4fHx9UKhWvvfbaOa1nZ0tNTWXs2LF4eHigUqn47rvvzvgx169fj0qlYv369fay2bNnEx4efsaPLc4vH374ISqVioyMDHvZyJEjGTly5Dmr0/E8/fTTqFSqc3b8rVu3MnToUFxcXFCpVOzcuROAn376if79+2MwGFCpVJSXlx93Py+99BLR0dFYrdYOH7u91y3AokWLiI6ORqfT4enpeXInJNrIyMhApVLx4Ycf2stO5r5TqVQ8/fTTZ6ZynaDlNZ+YmHjK+wgPD2f27NkOZafzWWY2mwkLC+Ott9465TqJi4sEOeKca3kzPdayZcsW+7YvvPACH374IX/5y19YtGgRt9xyCwAPPvggK1eu5PHHH2fRokWMHz++0+v5wgsvnJXgoj2zZs1iz549PP/88yxatIj4+PhzUo9TkZSUxNNPP+3wBfl88uOPPzJ+/Hh8fHwwGAz07NmTRx55hNLS0jbbzp492+HedHd3p1+/frzyyis0NDQ4bLtx40YmTJhASEgIBoOBbt26MWXKFD777LN26/F///d/xMbGnpFzPJfO5evmXDCbzUyfPp3S0lL+85//sGjRIkwmEyUlJcyYMQNnZ2cWLFjAokWLcHFxOeZ+KisrefHFF3nsscdQq0/vozolJYXZs2cTFRXFu+++y8KFC6mtreXpp59uEwwdz/79+3nwwQcZOnSoPVBr73VdUlLCvHnzGD58OH5+fnh6ejJ48GC+/PLLdvebmprKDTfcQGhoKEajkejoaJ555hlqa2tP8YzF+aq9z7KWwPxEn/86nY6HHnqI559/nvr6+nN4FuJCoT3XFRCixTPPPENERESb8u7du9v///PPPzN48GCeeuoph21+/vlnrrrqKh5++OEzVr8XXniB6667jmnTpp2xY7Snrq6O3377jX/84x/ce++9Z/XYR3v33XdP6ldlsAU5c+fOZeTIkeddK9DDDz/MK6+8Qr9+/Xjsscfw9vZm+/btvPnmm3z55ZesXbuWHj16ODxGr9fz3nvvAVBeXs7XX3/Nww8/zNatW/niiy8AWLx4Mddffz39+/fngQcewMvLi/T0dH799VfeffddbrrppjZ1WbZsGVOmTDnzJ32WnavXzbmSlpZGZmYm7777Lrfffru9/KeffqKqqopnn32W0aNHn3A/77//PhaLhRtvvPGkjj98+HDq6upwcnKyl61fvx6r1crrr79ufz8tLi5m7ty5AB1ukfvtt9944403iI2NJSYmxt5C1d52//jHP5g4cSJPPPEEWq2Wr7/+mhtuuMH+ftAiOzubSy+9FA8PD+699168vb357bffeOqpp9i2bRtLly49qfM/l5544gn+9re/netqnLeO9Vl28OBBAO6//34SEhIcHtP68x/g1ltv5W9/+xufffYZc+bMOfOVFhc0CXLEeWPChAknbKEoLCxs99fuwsLCLtsFo6ioCOC8OD+dTneuq9BpPv/8c1555RWuv/56Pv30UzQajX3d7NmzGTVqFNOnTycxMRGt9shbpVar5U9/+pP977vvvptBgwbx5Zdf8uqrrxIcHMzTTz9NbGwsW7ZscfiyCbZ79WiHDh1i//79vP3228esb01NzXF/+e8KusI5tjy/R79ej1V+LB988AFTp07FYDCc1PHVanWbx5zssY9l6tSplJeX4+bmxssvv3zMIKd3796kpqZiMpnsZXfffTejR4/mxRdf5NFHH7U/z4sWLaK8vJyNGzfSu3dvAO644w6sVisff/wxZWVleHl5nVa9zxatVuvwXiEcneizbNiwYVx33XXH3Yenpydjx47lww8/lCBHnJB0VxMXhJbm7PT0dJYtW2Zvym7p6qYoCgsWLLCXtygvL+evf/0rYWFh6PV6unfvzosvvtimNaLlV86+fftiMBjw8/Nj/Pjx9v7IKpWKmpoaPvroI/sxWvoaV1VV8de//pXw8HD0ej3+/v6MGTOG7du3n/C8duzYwYQJE3B3d8fV1ZUrr7zSoXn+6aeftn9ReOSRR1CpVMdtDWm5Tl999RXPP/88oaGhGAwGrrzySvuvZaejvTE5X3zxBXFxcbi5ueHu7k7fvn15/fXXAVtXxOnTpwMwatQo+7U7UReZn3/+mWHDhuHi4oKnpydXXXUVycnJDtu09H8/ePAgs2fPxtPTEw8PD2699dYOdXOZO3cuXl5eLFy40CHAAbj00kt57LHH2LVrF998881x96NWq+2/hLd03UlLSyMhIaFNgAPg7+/fpmzZsmV4eHhw+eWXO5xbUlISN910E15eXvZ1AJ988glxcXE4Ozvj7e3NDTfcQHZ2dpv9/v7770ycOBEvLy9cXFy45JJL7M8NwO7du5k9ezaRkZEYDAYCAwOZM2cOJSUlxz3njjre6+Z453gy9dq4cSMJCQkYDAaioqJ45513jlmfjl63YznRfTl79mxGjBgBwPTp01GpVPaxS7NmzQIgISHB4Tq0Jz09nd27d7fb4nO81xu0HZMTHh5ub/n28/OzH9vPzw+wvQ5anpsTjRHx9vbGzc3thNcpIiLCIcAB270wbdo0GhoaOHTokL28srISgICAAIftg4KCUKvV7b6Gjma1Wnnttdfo3bs3BoOBgIAA7rzzTsrKytrUob1zbG/sSHl5OQ8++KD9fT00NJSZM2dSXFx8zHq0NyanoaGBBx98ED8/P9zc3Jg6dSqHDx9u9/E5OTnMmTOHgIAA9Ho9vXv35v3333fYprGxkSeffJK4uDg8PDxwcXFh2LBhrFu3zmG7ljFDL7/8MgsXLiQqKgq9Xk9CQgJbt2495jkcraGhgYceegg/Pz9cXFy4+uqr7cFKC0VReO655+zdDUeNGsW+ffvaXJuOfJZVVVVhsViOW6cxY8awcePGdrsUC9Ga/OQgzhsVFRVtPkBUKhU+Pj7ExMSwaNEiHnzwQUJDQ/m///s/AAYMGGAfmzNmzBhmzpxpf2xtbS0jRowgJyeHO++8k27durF582Yef/xx8vLyHJIT3HbbbXz44YdMmDCB22+/HYvFwoYNG9iyZQvx8fEsWrSI22+/nUsvvZQ77rgDgKioKADuuusulixZwr333ktsbCwlJSVs3LiR5ORkBg4ceMzz3bdvH8OGDcPd3Z1HH30UnU7HO++8w8iRI/nll18YNGgQ11xzDZ6enjz44IPceOONTJw4EVdX1xNey3//+9+o1WoefvhhKioqeOmll7j55pv5/fffO/x8dMTq1au58cYbufLKK3nxxRcBSE5OZtOmTTzwwAMMHz6c+++/nzfeeIO///3vxMTEANj/bc+aNWuYMGECkZGRPP3009TV1fHmm29y2WWXsX379jYfjDNmzCAiIoJ//etfbN++nffeew9/f397fdqTmprK/v37mT17Nu7u7u1uM3PmTJ566il++OEHZsyYcdzrkJaWBoCPjw8AJpOJtWvXcvjwYUJDQ4/7WIDly5czZsyYNr8CT58+nR49evDCCy+gKAoAzz//PP/85z+ZMWMGt99+O0VFRbz55psMHz6cHTt22H8lXb16NZMnTyYoKIgHHniAwMBAkpOT+fHHH3nggQfs2xw6dIhbb72VwMBA9u3bx8KFC9m3bx9btmw57cH7x3vdHO8cO1qvPXv2MHbsWPz8/Hj66aexWCw89dRTbb4wn8x1O5aO3Jd33nknISEhvPDCC/auNy116dWrFwsXLrR3yz36OrS2efNmgDbvHyd6vbXntdde4+OPP+bbb7/lv//9L66urvTt25fBgwfzl7/8hauvvpprrrkGgEsuueS41+B05efnA+Dr62svGzlyJC+++CK33XYbc+fOxcfHh82bN/Pf//6X+++/v0Mte3feeScffvght956K/fffz/p6enMnz+fHTt2sGnTppNuga6urmbYsGEkJyczZ84cBg4cSHFxMd9//z2HDx92qP+J3H777XzyySfcdNNNDB06lJ9//plJkya12a6goIDBgwejUqm499578fPzY8WKFdx2221UVlbak+xUVlby3nvvceONN/LnP/+Zqqoq/ve//zFu3Dj++OMP+vfv77Dfzz77jKqqKu68805UKhUvvfQS11xzDYcOHerQdbnvvvvw8vLiqaeeIiMjg9dee417773XYXzVk08+yXPPPcfEiROZOHEi27dvZ+zYsTQ2Ntq36chn2a233kp1dTUajYZhw4Yxb968dnt3xMXFoSgKmzdvZvLkySc8B3ERU4Q4xz744AMFaHfR6/UO25pMJmXSpElt9gEo99xzj0PZs88+q7i4uCgHDhxwKP/b3/6maDQaJSsrS1EURfn5558VQLn//vvb7Ndqtdr/7+LiosyaNavNNh4eHm2O3RHTpk1TnJyclLS0NHtZbm6u4ubmpgwfPtxelp6ergDKvHnzTrjPdevWKYASExOjNDQ02Mtff/11BVD27NnT4fq17GvdunX2slmzZikmk8n+9wMPPKC4u7srFovlmPtZvHhxm/0cT//+/RV/f3+lpKTEXrZr1y5FrVYrM2fOtJc99dRTCqDMmTPH4fFXX3214uPjc9xjfPfddwqg/Oc//znudu7u7srAgQPtf8+aNUtxcXFRioqKlKKiIuXgwYPKCy+8oKhUKuWSSy6xb/e///1PARQnJydl1KhRyj//+U9lw4YNSlNTU5tj1NTUKAaDQfnggw/anNuNN97osG1GRoai0WiU559/3qF8z549ilartZdbLBYlIiJCMZlMSllZmcO2re/p2traNvX5/PPPFUD59ddf7WUtr9H09HR72YgRI5QRI0a0vWhHOdbr5ljneDL1mjZtmmIwGJTMzEx7WVJSkqLRaJTWH28dvW7H09H7suV1s3jxYofHt1zDrVu3nvBYTzzxhAIoVVVVDuUdeb2197ptudZFRUX2sqKiIgVQnnrqqRPWpz3z5s1rc08cT0lJieLv768MGzaszbpnn31WcXZ2dnjv/8c//tGh/W7YsEEBlE8//dSh/KeffmpTfqzzNZlMDvfok08+qQDKN99802bbltdPy/tye6/bFjt37lQA5e6773bYx0033dSmLrfddpsSFBSkFBcXO2x7ww03KB4eHvbXhMVicXhvVxRFKSsrUwICAhzeC1vq5+Pjo5SWltrLly5dqgDKDz/80ObcWmu5X0ePHu3wnvHggw8qGo1GKS8vVxRFUQoLCxUnJydl0qRJDtv9/e9/VwCH63qsz7JNmzYp1157rfK///1PWbp0qfKvf/1L8fHxUQwGg7J9+/Y2dcvNzVUA5cUXXzzuOQgh3dXEeWPBggWsXr3aYVmxYsUp72/x4sUMGzYMLy8viouL7cvo0aNpamri119/BeDrr79GpVK1SWYAdOiXbE9PT37//Xdyc3M7XLempiZWrVrFtGnTiIyMtJcHBQVx0003sXHjRns3jlNx6623OnTzGDZsGIBDN5HO4OnpSU1NDatXr+6U/eXl5bFz505mz56Nt7e3vfySSy5hzJgxLF++vM1j7rrrLoe/hw0bRklJyXGvX1VVFcAJu964ubnZt21RU1ODn58ffn5+dO/enb///e8MGTKEb7/91r7NnDlz+Omnnxg5ciQbN27k2WefZdiwYfTo0cP+K32Ln3/+mYaGBiZMmHDCc/vmm2+wWq3MmDHD4Z4ODAykR48e9i4rO3bsID09nb/+9a9tWiha39POzs72/9fX11NcXMzgwYMBOtTdsjMcfY4drVdTUxMrV65k2rRpdOvWzb59TEwM48aNc9hfR6/bsZzKfXk6SkpK0Gq1bX7p7uzX29litVq5+eabKS8v580332yzPjw8nOHDh7Nw4UK+/vpr5syZwwsvvMD8+fNPuO/Fixfj4eHBmDFjHJ7buLg4XF1dT/jctufrr7+mX79+XH311W3WnUzrZst9cf/99zuUt7TKtFAUha+//popU6agKIrDeYwbN46Kigr7fa/RaOzv7VarldLSUiwWC/Hx8e2+Zq+//nqHMU0n+1lwxx13OJzzsGHDaGpqIjMzE7C1cDY2NnLfffc5bHf0OR7P0KFDWbJkCXPmzGHq1Kn87W9/s7fYPv744222bzmf43UdFAKku5o4j1x66aWdmho5NTWV3bt32/ueH61lMG5aWhrBwcEOX15OxksvvcSsWbMICwsjLi6OiRMnMnPmTIfg5WhFRUXU1tbSq1evNutiYmKwWq1kZ2fbB+KerNZf+uDIh8LRfdRP1913381XX31lT5U8duxYZsyYccopvFs+OI91XVauXNlmcPrxzvVYXdFagpujA5ijVVVVtekeZzAY+OGHHwBbprWIiIh2u6SNGzeOcePGUVtby7Zt2/jyyy95++23mTx5MikpKfaxOcuWLSM+Pr7dLlZHZxtMTU1FUZQ2Gd9atHQ/aek+16dPn+OeX2lpKXPnzuWLL75okxChoqLiuI/tLO1lVOxIvYqKiqirq2v3WvTq1csh8Ojodauurqa6utpertFo8PPzO6X78kzo7Ndbe+rq6to894GBgae1z/vuu4+ffvqJjz/+mH79+jms++KLL7jjjjs4cOCA/XV0zTXXYLVaeeyxx7jxxhvx8fGhtLTUofuTs7MzHh4epKamUlFR0e5YN2g/0ceJpKWlce211570446WmZmJWq1u0zXx6PuoqKiI8vJyFi5cyMKFC9vdV+vz+Oijj3jllVdISUnBbDbby9t7LZ3uZ8GJHt/y2jj6teXn53daCSO6d+/OVVddxTfffENTU5PDuEmluVvruZwLS1wYJMgRXZbVamXMmDE8+uij7a7v2bNnpxxnxowZDBs2jG+//ZZVq1Yxb948XnzxRb755pt2f50/G44eSN+i5cOhs/j7+7Nz505WrlzJihUrWLFiBR988AEzZ87ko48+6tRjHcupnGtLhr7du3cfc5vMzEwqKyvbBKsajaZDKYBbGI1Ghg0bxrBhw/D19WXu3LmsWLHCPhB9+fLl3Hrrre0+tnWLBtjuaZVKxYoVK9o9746M12ptxowZbN68mUceeYT+/fvj6uqK1Wpl/PjxJ50q/FQdfY5nol4dvW4vv/yyQ3pjk8l0TuZ38vHxwWKxUFVV5dDaeDZeb19++WWb+/F03jfmzp3LW2+9xb///W/7vGatvfXWWwwYMKDNDwVTp07lww8/ZMeOHYwePZprrrmGX375xb5+1qxZfPjhh1itVvz9/fn000/bPf6xfuRqramp6STPqnO13NN/+tOf7O8LR2sZL/XJJ58we/Zspk2bxiOPPIK/vz8ajYZ//etf9h83Wjvdz4Kz9VnSnrCwMBobG6mpqXH4waolwDqZsVHi4iRBjuiyoqKiqK6uPuEX0qioKFauXElpaelxW3OO96tRUFAQd999N3fffTeFhYUMHDiQ559//phBjp+fH0ajkf3797dZl5KSglqtJiws7Lj1Pl84OTkxZcoUpkyZgtVq5e677+add97hn//8J927dz+pX9tasu8c67r4+vp2yq/lPXr0oFevXnz33Xe8/vrr7XZb+/jjjwHs2eE6Q0tLZV5eHgB79+4lKyur3YHI7YmKikJRFCIiIo4bpLf8crx3795j3v9lZWWsXbuWuXPn8uSTT9rLU1NTO1SXjjrZX1s7Wi8/Pz+cnZ3bre/R909Hr9vMmTMdsti1BGBn675sER0dDdiyrB2dDOBEr7eOOtbzMm7cuE7rDrdgwQKefvpp/vrXv/LYY4+1u01BQUG7v/i3tFC0ZNp65ZVXHFofgoODAdtzu2bNGi677LJ2A+bWvLy8KC8vdyhrbGy0vx5bREVFsXfv3uOfXAeYTCasVitpaWkOrTdH30ctmdeamppO+Hm1ZMkSIiMj+eabbxyew/a6W58NLa+N1NRUhx+EioqKTrvnwKFDhzAYDG1+vElPTweOn8BGCJAU0qILmzFjBr/99hsrV65ss668vNz+4XnttdeiKIrDL7gtWv9a5eLi0uYDsqmpqU3XDn9/f4KDg2loaDhm3TQaDWPHjmXp0qUOvxQXFBTw2Wefcfnllx+zq9X55OiUvmq12v6lrOX8W778HX3t2hMUFET//v356KOPHLbfu3cvq1atYuLEiZ1TcWxfCsrKyrjrrrva/JK7bds2XnzxRQYMGHBKrXFr165tt7ylC1XLF57ly5cTEBDQ4W6a11xzDRqNhrlz57b5JVVRFPvzMXDgQCIiInjttdfaXPeWx7X8Qnv0flpnHewM7b1ujqej9dJoNIwbN47vvvuOrKwse3lycnKb13xHr1tkZCSjR4+2L5dddhlwdu9LgCFDhgDYU9i36MjrraOMRiPQ9nUZFBTkcA1OptWytS+//JL777+fm2++mVdfffWY2/Xs2ZMdO3Zw4MABh/LPP//c4fzi4uIc6tTSGjtjxgyampp49tln2+zbYrE4nF9UVJR9LGaLhQsXtnn9X3vttezatcthnF2Lk2nBaHnveOONNxzK27uXr732Wr7++ut2g6vWKZvbe338/vvv/Pbbbx2u19EqKipISUk5pS6qo0ePRqfT8eabbzrU6WTeR45OSQ2wa9cuvv/+e8aOHYta7fhVddu2bahUKvvrRIhjkZYccd5YsWIFKSkpbcqHDh163PEtx/LII4/w/fffM3nyZGbPnk1cXBw1NTXs2bOHJUuWkJGRga+vL6NGjeKWW27hjTfeIDU11d4lZsOGDYwaNco+M3NcXBxr1qyxT/gYERFBr169CA0N5brrrqNfv364urqyZs0atm7dyiuvvHLc+j333HOsXr2ayy+/nLvvvhutVss777xDQ0MDL7300kmf77lw++23U1payhVXXEFoaCiZmZm8+eab9O/f3/4rW//+/dFoNLz44otUVFSg1+u54oorjtmHft68eUyYMIEhQ4Zw22232VP1enh4nHAej5Nx4403kpiYyKuvvkpSUhI333wzXl5ebN++nffffx8/Pz+WLFlySpP7XXXVVURERDBlyhSioqKoqalhzZo1/PDDDyQkJDBlyhTANh5nwoQJHW7tiIqK4rnnnuPxxx8nIyODadOm4ebmRnp6Ot9++y133HEHDz/8MGq1mv/+979MmTKF/v37c+uttxIUFERKSgr79u1j5cqVuLu7M3z4cF566SXMZjMhISGsWrXK/itpZ2nvdTNo0KBjbn8y9Zo7dy4//fQTw4YN4+6778ZisfDmm2/Su3dvh66IHb1ux3O27kuwBVt9+vRhzZo1DhMeduT11lHOzs7Exsby5Zdf0rNnT7y9venTp89xx3FVVFTYEwds2rQJgPnz5+Pp6Ymnp6f9vfKPP/5g5syZ+Pj4cOWVV7bpStb6Pf2RRx5hxYoVDBs2jHvvvRcfHx9+/PFHVqxYwe23325vsTmWESNGcOedd/Kvf/2LnTt3MnbsWHQ6HampqSxevJjXX3/dPsHk7bffzl133cW1117LmDFj2LVrFytXrmzT7emRRx5hyZIlTJ8+nTlz5hAXF0dpaSnff/89b7/9dptxRcfSv39/brzxRt566y0qKioYOnQoa9eubXfOsn//+9+sW7eOQYMG8ec//5nY2FhKS0vZvn07a9assc8JM3nyZL755huuvvpqJk2aRHp6Om+//TaxsbEO48lOxrfffsutt97KBx98cNz5m9rj5+fHww8/zL/+9S8mT57MxIkT2bFjBytWrOhwd7Lrr78eZ2dnhg4dir+/P0lJSSxcuBCj0ci///3vNtuvXr2ayy67zJ6yX4hjOnuJ3IRo3/FSSHNUis6TSSGtKIpSVVWlPP7440r37t0VJycnxdfXVxk6dKjy8ssvK42NjfbtLBaLMm/ePCU6OlpxcnJS/Pz8lAkTJijbtm2zb5OSkqIMHz7cnup01qxZSkNDg/LII48o/fr1U9zc3BQXFxelX79+yltvvdWhc9++fbsybtw4xdXVVTEajcqoUaOUzZs3O2xzKimkj05f2166047u63gppJcsWaKMHTtW8ff3V5ycnJRu3bopd955p5KXl+ewr3fffVeJjIy0p/Y9UTrpNWvWKJdddpni7OysuLu7K1OmTFGSkpIctmkvLa6itJ/u+Hi+//57ZfTo0Yqnp6f9nuvdu7dSUVHRZtuWFNIn8vnnnys33HCDEhUVpTg7OysGg0GJjY1V/vGPfyiVlZWKoihKeXm5otVqla+++qrN4491bi2+/vpr5fLLL1dcXFwUFxcXJTo6WrnnnnuU/fv3O2y3ceNGZcyYMfZ785JLLlHefPNN+/rDhw8rV199teLp6al4eHgo06dPt6dnbZ3e9nRSSLf3ujnROXa0XoqiKL/88osSFxenODk5KZGRkcrbb7/dJpXvyV63Y+nIfdkZKaQVRVFeffVVxdXV1SGddkdebx1NIa0oirJ582b7tWvv2h6t5X2kvaX1+8LJvKcriqL8/vvvyoQJE5TAwEBFp9MpPXv2VJ5//nnFbDZ36FopiqIsXLhQiYuLU5ydnRU3Nzelb9++yqOPPqrk5ubat2lqalIee+wxxdfXVzEajcq4ceOUgwcPtkkhrSi2lNf33nuvEhISojg5OSmhoaHKrFmz7CmeO5JCWlEUpa6uTrn//vsVHx8fxcXFRZkyZYqSnZ3d7vUuKChQ7rnnHiUsLEzR6XRKYGCgcuWVVyoLFy60b2O1WpUXXnhBMZlMil6vVwYMGKD8+OOPbd6bj/e5cazXd+tzOdb92t791dTUpMydO1cJCgpSnJ2dlZEjRyp79+5tc12PVafXX39dufTSSxVvb29Fq9UqQUFByp/+9CclNTW1Td3Ly8sVJycn5b333muzToijqRTlLIweE0KIC8Ttt9/O//73P959911uv/32M3acr776iptvvpni4mI8PDzO2HHEhamiooLIyEheeuklbrvttnNdHSHOC6+99hovvfQSaWlpJxyDJYSMyRFCiFbeeecdJk+ezF/+8pdOn/+kNU9PT9544w0JcES7PDw8ePTRR5k3b95Zy3QnxPnMbDbz6quv8sQTT0iAIzpEWnKEuMi0Nw/G0by9vR0mExVCCCGEuJBI4gEhLjLtzYNxtHXr1jFy5MizUyEhhBBCiE4mLTlCXGTy8vLYt2/fcbeJi4s7rdmqhRBCCCHOJQlyhBBCCCGEEF2KJB4QQgghhBBCdCkS5AghOs3TTz+NSqWiuLj4XFdFNFu/fj0qlYr169fby2bPnk14ePg5q5M48zIyMlCpVLz88svnuipCCHFOSJAjhOiSnn/+eaZOnUpAQAAqleq4s9Ln5OQwY8YMPD09cXd356qrruLQoUNnr7Kiw7Zv387UqVPx9vbGaDTSp08f3njjjWNuX15ejr+/PyqViiVLlpzFmooWdXV13HbbbfTp0wcPDw9cXV3p168fr7/+Omaz2WHbtWvXMmfOHHr27InRaCQyMpLbb7+dvLy8Y+7/zTffxMPDw76vvLw87rjjDiIiInB2diYqKoqHHnqIkpKSM3qeQojzi2RXE0J0SU888QSBgYEMGDCAlStXHnO76upqRo0aRUVFBX//+9/R6XT85z//YcSIEezcuRMfH5+zWOuz4913370g515ZtWoVU6ZMYcCAAfzzn//E1dWVtLQ0Dh8+fMzHPPnkk9TW1p7FWoqj1dXVsW/fPiZOnEh4eDhqtZrNmzfz4IMP8vvvv/PZZ5/Zt33ssccoLS1l+vTp9OjRg0OHDjF//nx+/PFHdu7cSWBgYJv9L1u2jLFjx6LT6aiurmbIkCHU1NRw9913ExYWxq5du5g/fz7r1q1j27ZtqNXy+64QFwMJcoQQXVJ6ejrh4eEUFxfj5+d3zO3eeustUlNT+eOPP0hISABgwoQJ9OnTh1deeYUXXnjhbFX5rNHpdOe6CietsrKSmTNnMmnSJJYsWdKhL6p79+7lv//9L08++SRPPvnkWajlmVdbW4vRaDzX1Tgp3t7ebNmyxaHsrrvuwsPDg/nz5/Pqq6/ag5dXX32Vyy+/3OH5HT9+PCNGjGD+/Pk899xzDvupra3ll19+4b///S8A33//PZmZmfz4449MmjTJoQ7PPPMMu3btYsCAAWfqVIUQ5xH5OUMIcUZlZmbSvXt3+vTpQ0FBwVk7bkfHnCxZsoSEhAR7gAMQHR3NlVdeyVdffXVKx549ezaurq4cOnSIcePG4eLiQnBwMM888wxHJ7QsLy9n9uzZeHh44OnpyaxZs9i5cycqlYoPP/zwpI57+PBhpk2bhouLC/7+/jz44IM0NDS0W7/W16f1+I0FCxYQGRmJ0Whk7NixZGdnoygKzz77LKGhoTg7O3PVVVdRWlp6KpfmlH322WcUFBTw/PPPo1arqampOWFr1AMPPMDVV1/NsGHDTvv4iYmJjBs3Dl9fX5ydnYmIiGDOnDkO21itVl577TV69+6NwWAgICCAO++8k7KyMoftli5dyqRJkwgODkav1xMVFcWzzz5LU1OTw3YjR46kT58+bNu2jeHDh2M0Gvn73/8OQH19PU8//TQ9e/bEYDAQFBTENddcQ1paWpu6L1y4kKioKPR6PQkJCWzduvW0r0dnaLkHy8vL7WXDhw9vE8AOHz4cb29vkpOT2+xj7dq1NDQ0MGHCBMAWDAMEBAQ4bBcUFASAs7NzZ1VfCHGek5YcIcQZk5aWxhVXXIG3tzerV6/G19f3mNuazWYqKio6tF9vb+9O6XJitVrZvXt3my+rAJdeeimrVq2iqqoKNze3k953U1MT48ePZ/Dgwbz00kv89NNPPPXUU1gsFp555hkAFEXhqquuYuPGjdx1113ExMTw7bffMmvWrJM+Xl1dHVdeeSVZWVncf//9BAcHs2jRIn7++ecO7+PTTz+lsbGR++67j9LSUl566SVmzJjBFVdcwfr163nsscc4ePAgb775Jg8//DDvv//+cffX0NBAVVVVh459vHsDYM2aNbi7u5OTk8O0adM4cOAALi4u3HLLLfznP//BYDA4bL948WI2b95McnIyGRkZHarDsRQWFjJ27Fj8/Pz429/+hqenJxkZGXzzzTcO29155518+OGH3Hrrrdx///2kp6czf/58duzYwaZNm+wtaB9++CGurq489NBDuLq68vPPP/Pkk09SWVnJvHnzHPZZUlLChAkTuOGGG/jTn/5EQEAATU1NTJ48mbVr13LDDTfwwAMPUFVVxerVq9m7dy9RUVH2x3/22WdUVVVx5513olKpeOmll7jmmms4dOjQcVv0rFZrhwNZDw+PDrUONjY2UllZSV1dHYmJibz88suYTCa6d+9+3MdVV1dTXV3d7j2yfPly4uLi7EFNS5D0wAMP8MorrxAaGsru3bt5/vnnmTZtGtHR0R06JyFEF6AIIUQneeqppxRAKSoqUpKTk5Xg4GAlISFBKS0tPeFj161bpwAdWtLT0ztcp6KiIgVQnnrqqWOue+aZZ9qsW7BggQIoKSkpHT5Wi1mzZimAct9999nLrFarMmnSJMXJyUkpKipSFEVRvvvuOwVQXnrpJft2FotFGTZsmAIoH3zwQYeP+dprrymA8tVXX9nLampqlO7duyuAsm7dOof6mUwm+9/p6ekKoPj5+Snl5eX28scff1wBlH79+ilms9lefuONNypOTk5KfX39cev0wQcfdPg5PZFLLrlEMRqNitFoVO677z7l66+/Vu677z4FUG644QaHbWtra5Vu3bopjz/+uKIoR+6txYsXn/A47fn2228VQNm6desxt9mwYYMCKJ9++qlD+U8//dSmvLa2ts3j77zzTsVoNDpc0xEjRiiA8vbbbzts+/777yuA8uqrr7bZj9VqVRTlyHPq4+Pj8PpbunSpAig//PDDcc+55fEdWVrfW8fz+eefOzwuPj5e2b179wkf9+yzzyqAsnbt2jbrunXr1ua1/d577ymenp4Ox5o1a5bDPSyE6PqkJUcI0en27t3L9ddfT/fu3VmxYgXu7u4nfEy/fv1YvXp1h/bf3uDjU1FXVweAXq9vs66lZaBlm1Nx77332v+vUqm49957WbZsGWvWrOGGG25g+fLlaLVa/vKXv9i302g03HfffWzYsOGkjrV8+XKCgoK47rrr7GVGo5E77riDRx99tEP7mD59Oh4eHva/Bw0aBMCf/vQntFqtQ/nnn39OTk4OkZGRx9zfuHHjOvycnkh1dTW1tbXcdddd9mxq11xzDY2Njbzzzjs888wz9OjRA4B///vfmM1me9eu0+Xp6QnAjz/+SL9+/dpttVi8eDEeHh6MGTPGIYV6XFwcrq6urFu3jptuuglw7DJVVVVFQ0MDw4YN45133iElJYV+/frZ1+v1em699VaHY3399df4+vpy3333tamHSqVy+Pv666/Hy8vL/ndL170TZQ8MDAzs8HPXur7HM2rUKFavXk15eTlr165l165d1NTUHPcxv/76K3PnzrW3KLa2d+9esrKyHMbeAISEhHDppZcyceJETCYTGzZs4I033sDX11dSagtxEZEgRwjR6aZMmUJAQAArV67E1dW1Q4/x8vJi9OjRZ7hmjlq+bLY3bqW+vt5hm5OlVqvbBAA9e/YEsHefyszMJCgoqM016tWr10kfr2Xs09Ffck9mX926dXP4uyXgCQsLa7f86LEmRwsKCrKPhThdLc/DjTfe6FB+00038c477/Dbb7/Ro0cPMjIymDdvHgsWLOjwvXciI0aM4Nprr2Xu3Ln85z//YeTIkUybNo2bbrrJHiCnpqZSUVGBv79/u/soLCy0/3/fvn088cQT/Pzzz/YxJC2O7rIZEhKCk5OTQ1laWhq9evVyCDyP5ejntCXgOdFzZzAYOv31GBAQYO9Wdt111/HCCy8wZswYUlNT2/3hIiUlhauvvpo+ffrw3nvvtVm/bNkyAgICiI+Pt5dt2rSJyZMns2XLFnv5tGnTcHd3Z+7cucyZM4fY2NhOPS8hxPlJghwhRKe79tpr+eijj/j000+58847O/SYxsbGDo8B8PPzQ6PRnE4VAdvYHr1e3+4cHC1lwcHBp32cC8WxrumxypWjkigcra6ursPjrE7UOhccHMy+ffvaDChvCSpavrQ/+eSThISEMHLkSHswmZ+fD0BRUREZGRl069btpMZ0tcyxs2XLFn744QdWrlzJnDlzeOWVV9iyZQuurq5YrVb8/f359NNP291HS4a/8vJyRowYgbu7O8888wxRUVEYDAa2b9/OY4891iaZwukOlD/V566pqYmioqIOHcPb27tNINYR1113Hf/4xz9YunRpm/eJ7Oxsxo4di4eHB8uXL293XNzy5csZP368Q2D/zjvvtAl8AKZOncrTTz/N5s2bJcgR4iIhQY4QotPNmzcPrVbL3XffjZubm72bzvFs3ryZUaNGdWj/LemhT5daraZv374kJia2Wff7778TGRl5SkkHwDZw+9ChQ/bWG4ADBw4AR7JKmUwm1q5dS3V1tUOrw/79+0/6eCaTib1796IoisOXvlPZV2f58ssv23S1OpYTfemOi4tj9erV5OTkOLRO5ebmAkeCiKysLA4ePNhuN7q7774bsAVELV3QTsbgwYMZPHgwzz//PJ999hk333wzX3zxBbfffjtRUVGsWbOGyy677LiByfr16ykpKeGbb75h+PDh9vL09PQO1yMqKorff/8ds9l8xtKBZ2dnExER0aFt161bx8iRI0/6GC1dQY8OhEtKShg7diwNDQ2sXbu23dbA8vJyNm/e7NAlFKCgoKBNljrAPlGoxWI56XoKIS5MEuQIITqdSqVi4cKFVFVVMWvWLFxdXZk6depxH3MuxuSA7dfkv/3tbyQmJtp//d2/fz8///wzDz/88Gnte/78+fbxI4qiMH/+fHQ6HVdeeSUAEydOZOHChfz3v//lkUceAWy/oL/55psnfayJEyeyatUqlixZwvTp0wHbHCILFy48rXM4HZ05JmfGjBn8+9//5n//+5/D2Iz33nsPrVZr/5L93HPPOYyJAdvYjX/+8588+uijDBkyBBcXl5M6dktQ1Dp47N+/P3Ckq+OMGTN46623ePbZZ9vMrWSxWKiursbT09PestI6qGtsbOStt97qcH2uvfZali1bxvz583nwwQcd1h0d5J6qzhyTU1xcjI+PT5t6tXRBa93qUlNTw8SJE8nJyWHdunX2cVZHW7VqFQBjx451KO/ZsyerVq1i/fr1DoHX559/DiBz5AhxEZEgRwhxRqjVaj755BOmTZvGjBkzWL58eZuBw6119picRYsWkZmZaZ/t/tdff7VPJHjLLbdgMpkA26/77777LpMmTeLhhx9Gp9Px6quvEhAQwP/93/857HPkyJH88ssvJ2x1ANuYhp9++olZs2YxaNAgVqxYwbJly/j73/9ub3WYMmUKl112GX/729/IyMggNjaWb775psNdvFr785//zPz585k5cybbtm0jKCiIRYsWndOJIztzTM6AAQOYM2cO77//PhaLhREjRrB+/XoWL17M448/bu9WePnll7d5bEurTUJCAtOmTXNYp1Kp7Ps6lo8++oi33nqLq6++mqioKKqqqnj33Xdxd3dn4sSJgG3czp133sm//vUvdu7cydixY9HpdKSmprJ48WJef/11rrvuOoYOHYqXlxezZs3i/vvvR6VSsWjRog7dUy1mzpzJxx9/zEMPPcQff/zBsGHDqKmpYc2aNdx9991cddVVHd7XsXTmmJxPPvmEt99+m2nTphEZGUlVVRUrV65k9erVTJkyxeF94eabb+aPP/5gzpw5JCcnO8yN4+rqan/+li1bxuWXX+6QKANsyT4++OADpkyZwn333YfJZOKXX37h888/Z8yYMfZkGkKIi8A5y+smhOhyWqeQblFbW6uMGDFCcXV1VbZs2XLW6tKSfre95eiUt9nZ2cp1112nuLu7K66ursrkyZOV1NTUNvuMi4tTAgMDT3jsWbNmKS4uLkpaWpoyduxYxWg0KgEBAcpTTz2lNDU1OWxbUlKi3HLLLYq7u7vi4eGh3HLLLcqOHTtOOoW0oihKZmamMnXqVMVoNCq+vr7KAw88YE9h3JEU0vPmzXPY37FSL7ekhj5eSuUzobGxUXn66acVk8mk6HQ6pXv37sp//vOfEz7uWOdRVVXVbgrqo23fvl258cYblW7duil6vV7x9/dXJk+erCQmJrbZduHChUpcXJzi7OysuLm5KX379lUeffRRJTc3177Npk2blMGDByvOzs5KcHCw8uijjyorV65s8zyNGDFC6d27d7t1qq2tVf7xj38oERERik6nUwIDA5XrrrtOSUtLUxTl2M+poijHTKl+pmzdulWZPn26/fq5uLgoAwcOVF599dU2aZ1NJtMxX7ct96zValX8/f0dUq+3lpKSolx33XVKWFiYotPpFJPJpDz88MNKTU3NmT5VIcR5RKUoJ/HzkRBCXKSqqqrw9vbmtdde45577jnutrNnz2bJkiVUV1ef0rEyMjKIiIjggw8+YPbs2ae0D3Fiy5cvZ/LkyezatYu+ffue6+qIDvrjjz8YNGgQ+/btkyQCQohjOv0pw4UQ4iLw66+/EhISwp///OdzXRXRSdatW8cNN9wgAc4F6IUXXpAARwhxXDImRwghOmDSpEltJh08GzqSWtvDw+O0Uw1fjObNm3euqyBOwaWXXsqll156rqshhDjPSZAjhBDnsY6k1pZubUIIIYQjGZMjhBDnsbKyMrZt23bcbXr37t1pWcyEEEKIrkCCHCGEEEIIIUSXIokHhBBCCCGEEF2KBDlCCCGEEEKILkWCHCGEEEIIIUSXIkGOEEIIIYQQokuRIEcIIYQQQgjRpVz08+RYrVZyc3Nxc3NDpVKd6+oIIYQQQghxUVMUhaqqKoKDg1GrT61N5qIPcnJzcwkLCzvX1RBCCCGEEEK0kp2dTWho6Ck99qIPctzc3ADbRXR3dz/utmazmVWrVjF27Fh0Ot3ZqJ7oguQ+EqdL7iHRGeQ+Ep1B7iPRGY6+jyorKwkLC7N/Tz8VF32Q09JFzd3dvUNBjtFoxN3dXV7I4pTJfSROl9xDojPIfSQ6g9xHojMc6z46naEkF23igQULFhAbG0tCQsK5rooQQgghhBCiE120Qc4999xDUlISW7duPddVEUIIIYQQQnSiizbIEUIIIYQQQnRNEuQIIYQQQgghuhQJcoQQQgghhBBdigQ5QgghhBBCiC5FghwhhBBCCCFElyJBjhBCCCGEEKJLkSBHCCGEEEII0aVctEGOTAYqhBBCCCFE13TRBjkyGagQQgghhBBd00Ub5AghhBBCCCG6Ju25roAAKg6DooB7MKg157o2QgghhBBCXNAkyDkf/PIibP8Y1DrwCAUvE3iajvzb8n8XP1CpznVthRBCCCGEOK9JkHM+sDSCWgtWM5Sl25b26Izg2c0xALIHQt3A2fOsVlsIIYQQQojzkQQ554Nr3oFpb0FlLpRnQlkmlGe1+n+mbZ25FopSbEt7DB5HBUDhR/72CAMn41k9LSGEEEIIIc4FCXLOF2oNeIbZlvDL2663NNjG7pRl2IKe8qwjAVBZJtQWQ30F5O+2Le1x8W/bAtTyr0coaHRn9BSFEEIIIYQ4GyTIuVBo9eATZVva01DdtvWndYtQQyXUFNqWw+2kzVapwT2k/QDIywSugaCWZHxCCCGEEOL8J0FOV6F3hYBY23I0RYG6snYCoJYgKAss9VCRbVsyN7bdh8bJ1uWt3ZagcDB6S1IEIYQQQghxXrhog5wFCxawYMECmpqaznVVzjyVyhaEGL0heEDb9VarrYXHIQDKOPJ3RQ40NUJpmm1pj5PrkQQI7QVCerczeopCCCGEEEK0uGiDnHvuuYd77rmHyspKPDw8znV1zi21GtwCbUu3QW3XN1mgMqf9lqCyTKjOh8ZqKNxnW9rj7O2YCa6lBaglKYLOcEZPUQghhBBCXDwu2iBHnASN1haMeJkgop315uaubke3ALX8W1cGdaW2JXdH+8dwC2rV+nNUmmz3EFsdhBBCCCGE6AD55ihOn84Avj1sS3vqK4+RFKH5X3MNVOXZluwtbR+v1toCnWNNkuoaIOOBhBBCCCGEnQQ54swzuENgH9tyNEWB2pJWiRCOCoAqsm3jgVrWtUdraNv603pskLOXBEFCCCGEEBcRCXLEuaVSgYuvbQmNa7vearW18LQ3N1B5pm2skKUeig/Ylvbo3dtPjd0SCDm5nNlzFEIIIYQQZ5UEOeL8plaDR4htMQ1tu97SCJWHHVNitw6EagptcwQV7LEt7TH6HmeS1DDQOp3ZcxRCCCGEEJ1KghxxYdM6gXekbWlPY23b8UCt/19fAbXFtiVnWzs7UIF78LEnSXULArXmjJ6iEEIIIYQ4ORLkiK7NyQj+0balPXXlx54ktSwTLHW2LnGVOZC1ue3j1TrwCG0nAAq3/eviK+OBhBBCCCHOMglyxMXN2dO2BPVru05RoKaoVQCU0SoQyrIlRbCaoSzdtrRHZ2yTCEHlFop7bZYt65zO5wyenBBCCCHExUmCHCGORaUCV3/bEpbQdr21CSpzj50auyoPzLVQlGxbmmmBUQD7/wkGz6PmBgp3nC9I53x2zlUIIYQQoguRIEeIU6XWgGeYbQm/vO16SwNUHD6qBSgTa2kG5qKD6C1VUF8OeeWQt6v9Y7gGHHuSVI9Q0OjO4AkKIYQQQlyYJMgR4kzR6sEnyra00mQ289Py5UwcPRxddd6xW4Iaq6C6wLYc/qPt/lVqcA9tf24gLxO4Btqy0wkhhBBCXGQu2iBnwYIFLFiwgKampnNdFXGxcnKFgFjbcjRFgbqyYwdA5VnQ1AAVWbaFDW33odE3tzSZ+MXFyMv1GZiMAZg8Igj37UO4bywmj3D8nP1QSXIEIYQQQnQhF22Qc88993DPPfdQWVmJh4fHua6OEI5UKjB625bgAW3XW622Fh6HuYEyjgRCFTm2IKjkIJQcJM3DjQxvLzIqKqDiAGSttO/KiBqT1pVwZ3/C3U2YvGMIDxqAyScWVyfXs3fOQgghhBCd5KINcoS4oKnV4B5kW7oNbru+yeIwSeq0kgPElh4gsyaHjIYyMmgkU6clR6ulVmUl2VJJclUlVB2EnLXQPG+qr6LCpHEh3OBLuGsYJu+ehAf0JzQwDp3B7eyesxBCCCFEB0mQI0RXpNHaMrV5hQPgDQxuXgAw10F5No2laRwu3E1G2QEyqg6TWV9ChrWGDI2KUo2GYpVCsbWabbXVUJsBhRsgBTSKQogVwlUGTHpvwl1DCPfqjsnvEvz9+6LyCLXVQQghhBDiHJBvIUJcjHTO4NcTJ7+eRPaaQOTR6+srqSxKIit/B+mlKWRWZpJZV0SGpZpMVRN1ahVZGsiiAcx5UJYHZYlw6AucrVbCzRZMKidMTh6EG4MI94zE5NsHN99etsQIrv4ySaoQQgghzhgJcoQQbRnccQ8bTJ+wwfQ5apVitVJYup+M3EQyS/aRUZFBRm0+meZKchQzdWo1yXonbDMDVUBtBdSmQO5yfCxNmCxmwi0K4Vo3TEZ/wt3DCfOJRucdeSQznLPX2T9nIYQQQnQZEuQIIU6KSq0mwDeGAN8YBh21ztxk5nBlNhkF28ks2ktGeRoZNblkNpZRrJgp0Woo0WrYDoAFrLlQnou6bBMhFgsms4Vws5lwRYfJ4Eu4exj+nt1Re4c7zhfk5HLWz1sIIYQQFw4JcoQQnUan0RHhFUmEVyREX+ewrrqxmszKTDLKDpJZvJeM0gNkVB8ms6GUWixk63Rk63RsxLn5ETXQmIJzfhLdsm3Bj8lsIdxiJlzjisk1BHfPCMe5gTxN4BEGWqezf/JCCCGEOG9IkCOEOCtcnVzp7dub3r69ocdV9nJFUSiqKyKzMpP0inQyy9LILN1PZlUW2fXF1KnV7Nc7sV9/dOBSgndVIeGlGzGZLZjMZsLNFsItTYQZ/HFqafVpHQB5mcAtCNSas3vyQgghhDirJMgRQpxTKpUKf6M//kZ/EgITHNaZrWZyqnJsLUCVGWRUZpBZlkZGZTpFDeWUajSUajRsNzjuU60oBFsOYco/QLi9FcgWBAUoatTNk6Ti2a1VABRu+9fFV5IiCCGEEBc4CXKEEOctnVpHuEc44R7hjGCEw7oac40t+KnIcAyCKjKosdRyWKfjsE7HpqP2abBa6WauxVS9h/Cy7YQfaGkFMuNhVUDn4hj8OARCJjDI5MFCCCHE+U6CHCHEBclF50KsTyyxPrEO5YqiUFJfYuv6VplpD4QyKjM4XHWYerWFA3onDrTp/gZeTU3NyQ8KMBXkEH74F8LNFsIsZvRK80YGzyNBj3cE+PYC/2jbv3rXM3/iQgghhDghCXKEEF2KSqXC19kXX2ffNt3fLFYLOdU5DoFPSytQYW0hZRoNZRoNOw16x30qEGy1YmposM0BVJ1GePl+wlPNBFqaULds6BEGftHg16v53+b/G9zPzskLIYQQApAgRwhxEdGqtZjcTZjcTQwPHe6wrtZca2/5Sa9sbgWqsAVA1eZqcjRqcozObD5qn3pFRVhTExEN9ZjMlZjyNxOebWsB8rRabRu5hxwV+DQHP86eZ+W8hRBCiIuNBDlCCAEYdUZifGKI8YlxKG/p/nb0+J/MykyyqrJosFo4qFVzUGtss08Pq0J4YyMmcx3hxYmE5/2GyWyhm8WCQVFsmd7swU8v8Iux/Wv0PlunLYQQQnRJEuQIIcRxtO7+FhcQ57DOYrWQV513pOWnVTe4gtoCKtQqdhn07GrT/U0hyNKEyWLGVLHLFgDttiVACLI0oXHxt43zcej6FgMuPmfz1IUQQogLlgQ5QghxirRqLWHuYYS5h7VZV2uuJasqqznj25HWn4yKDKrMVeTqtOTqtPzm7Pg4J6tCN4sZU00S4eW7MCXZUmCHmy14GrxR+UWj9ulBRFETqgw3COoDLn6S9loIIYRo5aINchYsWMCCBQtoamo611URQnRBRp2RaO9oor2jHcoVRaG0vvRIy09lhr0bXFZVFo2YOejkxEGnttnf3JuaCG9MJTwrCZPZQt63iwk3W+imdcO5datPSyuQa4AEP0IIIS5KF22Qc88993DPPfdQWVmJh4fMeyGEODtUKhU+zj74OPswMGCgw7omaxO5NblHEiC0SoOdV5NHpUbDbo2G3Ud1fwMItGQQnpOKKeM7ws3NE6CqDQR790TjF+PY9c09WIIfIYQQXdpFG+QIIcT5RqPWEOYWRphbGJeHXO6wrs5SR1ZlFpmVmaSVpbE5eTNN7k1kVmVS2VhJvlZLvlbLlqO6v+mUPLoVZGE6vAyT2UKE2YxJ5YTJIxJv32hU/rFHAiCPUAl+hBBCdAkS5AghxAXAWetML+9e9PLuhTnETHBWMBPHTUSr1VLeUN6m5Sej4hBZVdk0Ws2kOTmR1qb7WxFupQWEF6zBZGlu+VG0hLuF0c0nBqN/7JFsbx5hoFa3Wy8hhBDifCRBjhBCXMBUKhVeBi+8DF709+/vsK7J2kR+bb7jxKflh8gsP0RefTFVGjV7NHr20Lr7WxlUbiag9FfC99gyvoVbVZiMgYR79yTY/xK0/rG24MczXIIfIYQQ5yUJcoQQoovSqDWEuIYQ4hrCZSGXOayrt9STVZV1ZPxP+SEyS/eTWX2YckstBVotBVotvzsbmh9RDdXb0VZtIyzFFvxENIHJ4EO4RwQmvz74BA5A5R8DXuGg1pz18xVCCCFaSJAjhBAXIYPWQE+vnvT06tlmXXl9uWPLT0kSGRUZZNUX0UAT6U460p10rAegDuqSICsJ14wvbN3eLFZMOk8i3EIxeUdjCorDGNQfvCJAIx87Qgghzjz5tBFCiC5OaWqiqaICrbd3h7b3NHjS39C/Tfc3q2Ilvybflva6/BCZRXvJLEsloyaXXEs11Wo1e/V69uoBGqAhDfLSIG8Z/hYL4ZYmTBoXTMZAIjy7YwroT3DIIHR+vUCj6/TzFkIIcfGSIEcIIbq4hoMHSb9qGtqAAAwxMRhiY9DHxGCIiUUXEoyqgxnV1Co1wa7BBLsGMzR4qOMxmhrIrswmo+IQGQW7bK0/VVlkNpRSplgo1Gop1Gr5AwuYD0PRYShaj3aPQqjFQrjKYOv65m7C5NuH8OBB+AbHodIZjlEbIYQQ4tgkyBFCiC6uMT0DAEtBAdUFBVSvX29fp/bwwBAdbQ9+DDExOEVEoNKe3MeDXqOnu1d3unt1h/CxDusqGirIKE8ns2A7GYV7yKxII6O2kKymGupVKjJ0OjJoAkshlBZC6VY48AEuVismRYNJ50GEawgm716YAuMID7sMF6PP6V4WIYQQXZgEOUII0cW5jx+HS2IiDftTqE9Kpj7ZtjQcPIi1ooLa33+n9vff7dur9Hr0PXs6BD76Xr1QG06tVcVD70G/gP70C+jvUG5VrBTWFJCeu5XM/O1klu0nvTqHTHMluVioUatJQiHJWg6V5VC5DzK+gS3gZwWTxoVwYwDhHpGY/C8hPHQIIV5R6NTS9U0IIS52EuQIIcRFQOPqgjEuDmNcnL1MaWyk4eBBW9DTHPw0pKRgra2lfs8e6vfsObIDtRqnyAgMMbEOwY/Gw+OU66RWqQl0DSKw51SG9JzqsK7R0kB2/nYycraQWZxERmUmmQ0lZFjrKdWoKVJDkVJDYs0hqDkEuWtgJ2gUCFU5Ea73xuRuwuQbQ0RQAiafaPyc/TrcNU8IIcSFTYIcIYS4SKmcnDDExmKIjYVrbWWK1UpjZiYNza09LcFPU2kpjQfTaDyYRuUPP9j3oQsORt8c8BhiYjHExqANCDjtYMJJqycqdAhRoUMcVygKFaUHyTq8iYyC3WSUpZJZW0BmUzWZGhV1ajWZNJLZkA9F+VD0OyR/CIARNSatG+EuwZi8exIeMIBwn2hM7iZcnVxPq75CCCHOLxLkCCGEsFOp1egjItBHROA+cSIAiqJgKSykPinJ1trTHPyYc3Iw5+Zizs2les1a+z40Xl5tEhw4hZtQdcbEoSoVHj496OvTg76tyxUFa3UBhTm/k5m3jYySFDKqD5PZWE6G2kqOVkutykqypYLkigqoSIb0pfaH+6r1mJz9CHePJNy/jy31tYeJMNcwdJL5TQghLjgS5AghzisNliZ6PfETAA+N6YnJx0iErwsmHxc8nOXL5rmgUqnQBQSgCwjAbdQoe3lTZSX1ySnUJyfZA5+GQ4doKiujZvNmajZvPrIPoxFDr14OwY++Rw/UTk6dVUnUboEERl9FYPRVDGq9rqYYc8FesnP/IKNoL5mVGWTWFZGuspCp1VGi1VBsbaC45jDbag5D3q/2h2pQEeLkgck1jHDfGMK9e2FyNxHuHo6/0V+6vwkhxHlKghwhxHllXUqh/f+vrj7gsM7bxckW9PjYgp5wXyPhPi6E+0oAdC5o3N1xGXQpLoMutZdZ6+tpSE1t7ubW3PKz/wBKbS11O3ZQt2PHkR1otei7d2/u6tYc/ERHo3Ht5K5jLr7oIkcSGTmSyNbltaVQtJ+q/F1kFuxo7vqWT4a1nkydjgydljq1mqzGcrJKy9lQusdht85qHSZjIOFePTB59bAHP+Ee4bg5uXXuOQghhDgpEuQIIc4rl0b4cPOgbiTlVdLdz5WMkhoySmopqmqgtKaR0ppGdmSVt3mcl1FHuK+LLehpHQD5uOBhlADobFEbDDj37Ytz3yOdyRSLhcaMDIcxPvXJyVgrKmhISaEhJYWKb7+1b68zdWuT4EDr69v5lTV6g2kIbqYh9AH6tJTXlUPRfpTCZArtc/4cJqOpmkydjkydlsNaLXVWMynV2aRUZ0P2zw679ta52QIeb1vwY3I3EeEeQahbKE6aTmq9EkIIcUwS5AghziveLk48f3XfNuXVDRYyS2rIKK61BT7FNWSW1JJeUkNRVQNltWbKssrbDYA8jbrmgMd4JBDytf3taZQvnGeaqrnFRt+9Ox5TpgDN43xyc9sEPpb8fMyZWZgzs6j66Sf7PrR+fm0SHOhCQ89MdzFnT+g2CFW3QQQAAcClAPWVUHwACpMxFyZxuGgfmRXpZDSWkdEc/GTodBRrNZSaqygt2cP2EsfWHzUqgl0CCfeMItw93Nb64xFu7/6mVnXCuCUhhBAS5AghLgyuei29gz3oHdw2ZXFNg4XMkubgpzkAyiipJaO4hsKqBsprzeysLWdndnmbx3oadZh8XIjwMdr+9XWxjwOSAOjMUalU6EJC0IWE4DZ6tL3cUlrqkNygPjmZxowMLEVFWH4pouaXI+Nl1K6uGKKjm4MfW+Cjj4xEpTtDLXcGdwiNh9B4dEBE80JDlS34KdoPhclUFyaRWXaAzPpi20SnOq09AKpVqzlck8fhmjw25mx02L2zxkC35lYfk7uJCI8I+/899KeeqlsIIS5GEuQIIS54LnotscHuxAa7t1nXEgBlltSQXlJDZrGt9SezpIaCSlsAVF5bzq52AiAPZ529xccWADX/6+OCp1Eng87PAK23N66XXYbrZZfZy6w1NdTvP3BkjE9SMg2pqVirq6lNTKQ2MdG+rcrJCX2PHq0yu8Vg6NULtdF45iqtd4OQONsCuAK9gd6NNUeCn6IUlIJkiktSyKjNJ0OnsXV909qCn8M6LXVN9ewv28/+sv1tDuFt8MLU0vLjHm5vBerm3k26vwkhRDskyBFCdGnHC4BqG5tbgJpbfjJLakhv7gaXX1lPRZ2ZXdntB0DuBq0961tLINTSFc5LAqBOpXZxwThwAMaBA+xlitlMQ1paq65uSTQkp9gCon37qN+3r9UO1DiFhzuM8dHHxKD18jqzFXdygeABtgVQAX6An7mOhOJUKEppXvZjLkomtzKbDK3aoetbpk5LoVZLaX0ZpfVl7Cjc4XAItUpNkEuQPeFB6/E/AS4B0v1NCHHRkiBHCHHRMjppiQlyJyao/QAoq7TWoetbRvOYoPzKeirrLew6XMGuwxVtHutu0LZKgmALflq6wkkA1DlUOh2G6GgM0dHA1YBtIlNzdnabcT5NxcU0HjpE46FDVC5bZt+HNijIIbObISYGbVDQmX9+dM4QdIltaSkCTOZ6TCUHGdEc+FCUDEX7qSk9RKZWZc/41joIqlFDTnUOOdU5bMrd5HAYvUZPN/duDi0/LeN/pPubEKKrkyBHCCHaYXTSEh3oTnRg2wCorrGJzNIjSRBaJ0TIq7AFQLsPV7C7nQDIrXULkI/RIROct4uTBECnQaVW42Qy4WQy4T5+vL3cUlTUJvAxZ2VhycujOi+P6p+PZEbTeHqij4l2yO7mFB6OSqM58yegM0BgH9vSioulkdiSg8Tagx9bC5CSf5ASldUW9DR3e7ON/9GRrdPS0NRAalkqqWWpbQ7lqffE5GZCXasmf18+UV5R9u5veo3+zJ+rEEKcYRdtkLNgwQIWLFhAU1PTua6KEOIC4+ykOWYAVG9uOpIEoVUrUGZJDbkV9VSdIAAK93FxmAC1ZRyQjwRAp0zr54ernx+uw4fby5qqqmhISXEIfhrS0mgqL6f2ty3U/rbFvq3K2RlDz54O2d30PXug1p+lYEDrBAGxtqUVVZMZ39JD+BalEF94pOsb+alYmhrJ1WodWn4ydTrSdToKtRrKG8opbygHYMeuI13gVKgIdg3G5G6il1cv4gPjGeA/QOb9EUJccFSKoijnuhLnUmVlJR4eHlRUVODu3vYLS2tms5nly5czceJEdGcqe4/o8uQ+unjVm5vIKq1tHvdTQ3pxbXMrkC0AOh43vRZTc4tPmJeBysMHmTJqMFEBHvi6SgDUGawNDTSkHmwe39Mc/Ozfj1JX13ZjrRZ9ZOSRSUybu71p3M6DYKDJAmXpR8b8FDYHP8UHoKmBWpWKLJ1jy0+GzokMJx3V7dxGapWaaO9o4gPiiQ+IZ2DAQOnuJuzkM010hqPvo5P5fn4sF21LjhBCnG0GnYaeAW70DGj7RbglALKP/SmptXeDy62oo6rBwt6cSvbmVDY/QsOnaVsBW3ptkz3xgbHVPEAuEgCdBLVej3Of3jj36W0vU5qaaMzMbG7tORL8NJWX03DgAA0HDlCxdKl9e11YWJsEBzp//7N7Ihot+PawLTFTjpRbm6AsA2NRCtFFKfQsSKYybSse1QWoLCUoQKnalvgg3UnLbr2eRIOBbJ2WpJIkkkqS+DjpY1So6OXVk/jABOID4okLiMPT4Hl2z1EIIU5AghwhhDgPnCgAyi6ttXd9O1RUxbYDWdSojORW1FPdYGFfbiX7civbPNYeADWP/Wk9F5Cfq14CoBNQaTToIyPRR0biMXkS0DyRaX7+UeN8krDk5mHOzsacnU3VqlX2fWh8fdskONCFhaFSn+XMZ2oN+ETZluhJNJnN/LJ8ORPHj0NXk4eqaD8+Rcn4FO0nriiF64r2Q3EpBRoNiQY9Ww0Gthn0ZDjpSCnbT0rZfj5J/gSAHs7+xPsNIN40irigwfg4+5zdcxNCiKNIkCOEEOc5g05DjwA3ejQHQLZm/QwmThxOE2oOl9Xau761pMBOL64ht6LuuAGQi5OmOQV2cxDUKh22n5sEQMeiUqnQBQWhCwrC7Yor7OWWsjLbOJ9WCQ4a09NpKi6mZsMGajZssG+rdnGxJTiIPhL86KOiUDmdgzlv1BrwjrAtvY4kbMBqhYpsAor2M6kohUnN3d+K8g6wTdNkD3wOOelIrSskNWsln2etBCBKpSfeGEq8X3/iw6/ENzjBllhBCCHOEglyhBDiAmbQaeju70Z3/7YtQA2WJrJL61p1gWsVAJXXUdPYRFJeJUl5bQMgY3MA1HoC1JaECBIAtU/r5YV2yBBchgyxl1nr6mjYv98xwcGBA1hraqhL3EZd4jb7tiqdDqce3e3JDQyxzROZurici9MBtRq8TLal51h7sZ+iML4yh/HNyQ5KCnazrTSZxPoCtjqpOOjkRJrSQFpNGl/WpEHG14SbzcRb9cQbQ4j3u4SAoIHg1wt8eoDTGZyoVQhx0ZIgRwghuii9VkN3f1e6+7u2WdcSALVu/WkJhHLK6qhtbCI5r5Lk4wRA4T5Gx0DI1wV/CYAcqJ2dce7fH+f+/e1litlMw6F0xwQHKSlYq6poSEqmISmZCr6xbaxS4WQytUpuYAt+tN7e5+aEmuuER6ht6TEaH2AsMFZRoCqPstxEtmdvILFkL1vr8zmA2ZbkACtLmrIhP5uw7O9IqG8gvr6BeCc/gnxjbEGPXzT4R4NvT9tkqkIIcYokyBFCiIvQiQKgw2V19hTYrQOhw2W1xw2AnHWaVmOAjkyGGu7jQoC7BEDQPJFpr54YevWEadMA2zgfc04O9UlJttae5lYfS2EhjRkZNGZkwPIV9n1oAwKOyuwWiy4k+NxeX5UK3IPxcp/KldFTubK5uKK+gu2Za0nMXk9iyV5S6ovI1unI1un4xs0VsBJSu4u4/b+TsKuB+Pp6QixNqDy72YIeh6Un6M+DDHZCiPOeBDlCCCEc6LUaovxcifJrGwA1WqwcLmuZB6jWngkuo7iGw2W11JmbSMmvIiW/qs1jDTq1feyPydfY3AXuSAuQWn3xBkAqlQqn0FCcQkNxH3uka5iluJj65BR7coOGpGQaMzOxFBRQXVBA9fr19m3VHh4YoqMdEhw4RUSg0p7bj3oPgwejel3DqF7XAFDVWMWOwh0k5m8lMXcLSWUHyNFpydG58r2b7Z4LtFiIr68hIW8T8ek/E2axYL87PMKOtPq0Dn4MktZaCHGEBDlCCCE6zEmrJtLPlcjjBEAt434yS2pIb24JOlxWR73ZesIAyNSq5adlDFCAm+GiDYC0vr64Drsc12GX28uaqmto2O+Y4KDh4EGsFRXU/v47tb//bt9Wpdej79XLIbubvmdP1IZzlwTAzcmN4aHDGR5qm5y1xlzTHPQkkliQyL7ifeRr4UdXLT+62rqs+aMhrr6RhOpy4mvzCD+YjergmqN2HGzr6uYX3RwExdiCH2evs32KQojzgAQ5QgghOkXrAGjUUevMTdZWXeBqHAKh7A4EQCbvI0FP64xwge4XXwCkcXXBGBeHMS7OXmZtbKTx4EHHBAcpKVhra6nfvZv63btb7UCDPjICp17ReCoKtb6+uPbpg8bj3LSEuOhcuDzkci4PsQVyteZadhbtJDE/kW0F29hdvJtCq4UVBg0rDLbU1L5aF+I07sQ3NJJQlk9keR6qqlyoyoW0nx0P4BpoC3r8YxxbgIzncFyTEOKMkyBHCCHEGafTqInwtXVNO5q5yUpOWR3pJTVkNo8DsnWHOxIA7S+oYn9B2wBIr1UfNQaoOSGCrwtBF1EApHZywhAbiyE2Fq61lSlWK42ZmbbkBq2Cn6bSUhpSD9KQehB/IHfZMgB0ISGtxvjEYIiNRevvf9bH+Rh1RoYGD2Vo8FAA6ix17C7aTWJBIon5iewu2k2xpYaVlhpWAnjp8A68hDj3KOK0HiQ0NNG9LAd18QGoPAzV+bYl/RfHA7n4twp6WoKgaHDxPavnK4Q4MyTIEUIIcU7pNGpbgOLrAr0c17UEQC1BT0sAlFlSS3ZpLQ0WKwcKqjlQUN1mv05aNSZvY5sECOEXSQCkUqvRR0Sgj4jAfeJEoHki08JC6pOSqN27j8z16/EqL8OSk4s5JwdzTg5Vq490A9N4ezuM8dHHxOBkMp3ViUydtc4MChrEoKBBADQ0NdiDnm3529hVtIvShnJWF21jdfNjPPQexPUfTbxPH+KdvOlZV4OmaD+0LBVZUFNoWzI2OB7Q6NPc1e2oAMjFz5ZcQQhxQZAgRwghxHnreAGQpclKTnmdPfFBSyCUWVJLVmktjRYrqYXVpBYeOwA6OgW2ycdIsIdzlw2AVCoVuoAAdAEBGC6/nC3dwhgwcSLq2lrHBAfJyTQcSqeptJSaTZuo2bTpyD6MRgwt43yaW370PXqgPksTmeo1ehICE0gITIB+YG4ys7dkL1vzt5KYn8jOop1UNFTwc/bP/Jxt67rm5uRGnH8c8QOmEB8wl14uIWhL05qDnhRonvOH8kyoLYHMjbalNWevo5Id9IKwQTLPjxDnKQlyhBBCXJC0GjWm5gxtI3r6OayzNFnJLa+3dYFrPRdQcQ3ZZScOgLp5G22tPz4umHyPTIYa7OmMpgsGQBoPD1wGD8Jl8CB7mbW+noYDBxwTHOzfj1JbS92OHdTt2HFkBzod+qgoxwQH0TFoXM/8XDc6jY4B/gMY4D+AOy65A7PVTFJJki3oKUhkR8EOqhqrWH94PesPrwfAVefKAP8BxAfGE9//OmJ8YtCpddBYA8WptoCnKOVIEFSaDnVlkPWbbbEf3Ag9xkLvabZ/ZW4fIc4bEuQIIYTocrQaNd18jHTzMQLtB0Atk59mFDfPBVRSQ3ZzC9DBwmoOthcAadSEeTu3SoDgYg+GuloApDYYcL7kEpwvucReplgsNGZkOIzxqU9OxlpRQUNKCg0pKVR8+619e52pm20C01Zd3rS+Z3bMi06to59fP/r59eP2vrdjsVpIKU2xBz3bC7ZTba5mQ84GNuTYuqo5a50Z6D/QFvQExNO7z7XoNLojOzXXNQc/+48EQLk7bWN+kr6zLToj9BgDsdOg5zgJeIQ4xyTIEUIIcVFpHQANPyoAarIq5JYfNQaouStcdmkdjU1W0opqSCuqabPflgCo9USoLd3gukoApNJq0Xfvjr57dzymTAGax/nk5rYJfCz5+ZgzszBnZlH100/2fWj9/NA3BzyGmFgMsTHoQkPPWIIDrVpLH98+9PHtw619bqXJ2sT+sv0OQU9lYyWbcjexKdfWLc9Z60w/v37EB8QTHxhPX9++OAVdAkFHAj4UBXJ32AKcfd/ZurolLbUtWmfoOVYCHiHOIQlyhBBCiGYatYowbyNh3kaG9ThOANQc/LR0hTtRAKTT2PbbMhlqSwpsWwuQAa3m7A3k72wqlQpdSAi6kBDcRo+2l1tKS21d3FoFP40ZGViKirD8UkTNL7/at1W7udkmMrVnd4tFHxV5RiYy1ag1xPrEEusTy6zes7AqVlLLUkksSGRr/la2FWyjvKGcLXlb2JK3BbCNA2od9Fzidwl6jR5CBtqW0XMhb6ct2En6DsoyHAOeHmOau7SNA33bOaaEEJ1PghwhhBCiAxwDIMd1LQFQZqv01y2Z4LJKamlssnKoqIZDxwqAvGzZ31rPBRRxgQdAWm9vXC+7DNfLLrOXWWtqqN9/gPrkJFsAlJRMQ2oq1qoqardupXbrVvu2Kicn9D17OmZ369ULtbNzp9ZTrVLTy7sXvbx7cXPMzVgVK2nlaQ5BT2l9KX/k/8Ef+X/ALluXuL6+fUkITCA+MJ5+fv1wDh4AwQNg9NOQtwv2fXsk4En+3rZonaHH6OYWnvES8AhxBkmQI4QQQpym1gHQ5T0cx5w0WRXyKuocJkBNbx4HlNk8BuhQcQ2HitsGQFp1SwuQYwa4CF8XQjydL7gASO3ignHgAIwDB9jLlMZGGg4datXVLYmG5BRbQLR3L/V797bagRqniAiHBAeGmBg0np6dV0eVmh5ePejh1YMbo29EURTSK9LtQU9iQSLFdcVsL9zO9sLtvLP7HVuXOJ8+tqAnIJ7+/v0xjpl7JOBp6dJWlg7JP9gWrQG6j4beV9u6tOndOu0chBAS5AghhBBnlEatItTLSKiXkcu6OwZAVqtCXmW9fdxP60Aoo8QWAKUX27rEQZHDY1sCIPtkqK3mAgr1unACIJWTk62rWnQ0cDVgm8jUnJ3dZpxPU3ExjWlpNKalUfnjj/Z9aIOD2iY4CAzslHE+KpWKSM9IIj0jmdFrBoqikFmZaZuctDnwKawtZGfRTnYW7eTdPe+iVWmJ9YklLjCOhIAEBgx/GNcrn4L83Ue6tJUegpQfbYsEPEJ0OglyhBBCiHNErVYR4ulMiKdzuwFQvj0Aat0NzhYMNZwgAAr1cj4yAaqP0Z4KO8TLGd15HgCp1GqcTCacTCbcx4+3l5sLC21jfFoFP+bsbCy5eVTn5lG9dq19W42nJ/qYaIfgxyk8HJVGc3p1U6kI9wgn3COc63peh6IoHK467BD05NXksbt4N7uLd/PB3g9Qq9TEeMcQHxBPQq+RDBj2V9xLM4+08JSmHQl4NPojWdp6jZeAR4hTJEGOEEIIcR5Sq1UEezoT7OnM0O6O6+wBUOsU2C1zAZXU0GCxNgdGtbQEQEPqteRprOQ6KQR5O2PydkapUlP0WyZR/u6E+9pagM7nAEjn74/O3x/XESPsZU1VVW0SHDSkpdFUXk7tb1uo/W2LfVuVszOGnj0dsrvpe/ZArdefcp1UKhVh7mGEuYdxdQ9bS1ROdQ6J+UeCnpzqHPaV7GNfyT4+SvoIFSqivaOJC4gj4aqXiVM545G6uv2Ap/toW9KCnuPB4H7K9RTiYiNBjhBCCHGBcQiAohzXWa0KBVX1DkFPzuEqYrbZ5v1prFHIrLJwKLecNJ2aDfn77Y/VNLcA2RIfOI4DCvM2npcBkMbNDZdLL8Xl0kvtZdaGBhpSDzaP72kOfvbvR6mro27XLup27TqyA60WfWTkkUlMm8f7aNxOvQUlxDWEkO4hXNX9KgDya/LtSQwSCxLJrMwkuTSZ5NJkPkn+BBUqenj1IH7QdOL1/sQVHsJ7/09QchD2L7MtEvAIcVIkyBFCCCG6ELVaRZCHM0EeRwKgiqI6tjtnkrmnGCoa6WHR0MOigTqodVFz2KCws6mBzKYmMktqySyp5dej9qtp7lrXkvhgaJQvV0T746Q9/wIftV6Pc5/eOPfpbS9TmppozMxsbu05Evw0lZfTcOAADQcOULF0qX17XViYY2a3mBh0/v6nVJ9Al0CmRE1hSpRtbqHC2kJ7S09iQSLpFekcKDvAgbIDfNb8mO7doojrPYL42mriM7bhW3x0wHNlc5e2CRLwCNEOCXKEEEKILs7Dz5lRf4pGURSKs6vJ2FNMxp5iCjMqMdZY6VkDPXFC76rDNdyVBj89ec6QUVFnHwNUZ24iq7SWrNJaNqQW8/Fvmfi4OHH1gBBmJITRM+D8Hjui0mjQR0aij4zEY/IkoHki0/z8oxIcJGHJzcOcnY05O5uqVavs+9D4+rbJ7KYLC0OlPrlAz9/oz8TIiUyMnAhAcV2xLeDJT2RbwTYOlh/kYHkaB8vT+BLADSICBxOPM/FFmcQXZ+K/fznsXw4aJ4i60tbC02sCGDw66YoJcWGTIEcIIYS4SKhUKvy6ueHXzY3+Y0P54dsV9AoeSHZSGVlJpTRUm2nYWwaAh0bFhB6emPqEY+rrQ6Ozxp74ICW/ih9351FU1cB7G9N5b2M6/cM8mREfxuR+QbgbdOf4TDtGpVKhCwpCFxSE2xVX2MstZWU0pKQ4ZHZrTE+nqbiYmg0bqNmwwb6t2sWlTYIDfVQUKl3Hr4Gvsy/jw8czPtyWZKG0vtTWta25tedA2QHSa3JJBxYbgW4hmLRuxNfWEldeQELaKgIPrGgOeK440sLj7Nk5F0qIC5AEOUIIIcRFSqNX6DkogN6Xh9JksZJ3sJyMPSVk7i2hvKCWwyllHE4pY9OSg3j4OxPe15ehfX24bkAo/5gYw/r9RXyVmM3PKYXszC5nZ3Y5z/y4j4l9gpgeH8bgSO9OSeN8tmm9vNAOGYLLkCH2MmtdHQ379zu0+jQcOIC1poa6xG3UJW6zb6vS6dD36IFx8GDcJ03EEBt7UtfB2+DNGNMYxpjGAFBeX862wm32lp6U0hQyLVVkOsHX/rasfCFWFQk1VcTnbiD+0GpCvlc7dmmTgEdcZCTIEUIIIQQarZrQaG9Co725fHoPygtqydxbQsaeYnIPlFNRWMeutdnsWpuNzqChW6w3pj6+vDbtEmpUCt/tyOHLxGwOFlbzzY4cvtmRg8nHyPS4UK6NCyXIw/lcn+JpUTs749y/P879+9vLFLOZhkPpjgkOUlKwVlVRn5REfVISpe+/j1N4OO6TJuE+aRL6yIiTPranwZMru13Jld2uBKCysZIdBTvsk5MmlyaTo7aS4+bKd26uAARZLCSU/kH8mg3EL/8rod2Go+pzNfSaKAGPuChIkCOEEEKINjwDjHgGGOl3ZRiNdRayk0vJ2FNM5t4S6qrMpG0vIm17EaggINyd/n18uOq6AWQrFpZsO8wPu/LILKnl5VUHeHX1AYb18GNGfBijY/3Ra09vrprzhUqnw9CrJ4ZePWHaNMA2zsd8+DD1e/ZQuWo11evW0ZiRQfGCBRQvWIA+NgaPSZNwnzgRXVDQKR3X3cmdEWEjGBFmS6Vd3VjNjsIdbC3Yyrb8bewr2UeeFr53c+X75qDHv2EvCZu2Ef/zY8T7D8QUOx1VzCRw9uqUayHE+UaCHCGEEEIcl5OzlqiB/kQN9EexKhRmVtkDnqKsKgrSKylIr+SPH9Jx8XBibF9fZk3tzx5zA4t35fBHeim/HCjilwNFeBl1TBsQwoz4MGKCul5WMJVKhVNYGE5hYbhPnEhTdQ3VP6+lYtkyajZtpiEpmcKkZArnvYxzXBzukybiPn48Wm/vUz6mq5Mrw0KHMSx0GAC15lp2Fu5ka8FWEvMT2Vu8h0ItLHPVsswVaDqE347nidvyNAlu4cR3n0JEv1tQGU+9DkKcbyTIEUIIIUSHqdQqAiLcCYhwZ9DUSGrKG+zd2rKTS6mpaCRpYy5JG3PRaNXc0suTv1wezbbGOhan5FNQ2cAHmzL4YFMGfUM8mBEfytT+IXg4XxjJCk6WxtUFj6lT8Zg6FUtZGVUrV1G5bBm1iYnUbdtG3bZtFDz/Ai5DhuA+aRJuY0ajcXU9rWMadUaGhgxlaMhQAOosdewq2kVifiJbs39lT9l+irRaftJq+claAAfewzv5HeI0HiQEDyG+z01EBQ5ErTr/0oML0VES5AghhBDilLl46om9PJjYy4OxmJvIPWBLXpCxp5iqknqy9pXCvlKcgQeC3FB392drQx0/5JSwJ6eCPTkVPLcsmfF9ApkRH8aQSB/U6gsvWUFHaL288LrherxuuB5zfj6Vy1dQuWwZ9fv2UbNxIzUbN5L/1FO4jhiB++TJuI4YjtpgOO3jOmudGRw0mMFBg2HAvdRb6tlTvIfEQz+xNesXdtcXUKrRsJpqVueuhtzVeKImzj2K+MjxJISNoIdXDwl6xAVFghwhhBBCdAqtTkO33j506+3DsOt7UJZXa+/WlpdWQVleDeTVEAk84uyKJUDP1oY6NtfUsHRnLkt35hLi6cz0+FCuiwsl1Mt4rk/pjNEFBuIz51Z85txKQ3o6lcuXU7lsOY2HDlG1ejVVq1ejdnHBbfRo3CdPwmXw4JNKS308Bq2BhMAEEgIT+MvQf9LY1Mieg8tJTFnC1pK97FKZKVfD2spU1u5MhZ1v4q4xMDAgjvjgIcQHxhPtFY1G3TXGVomuSYIcIYQQQnQ6lUqFd7AL3sEuDBxnor7GTHZSc/KCfSU01Fggw0IcEKdyptFTy3ZzA8kl9by2OpXX16ZyeXdfpseHMTY2AIOu636h1kdE4HfPPfjefTcNKSlULltGxfLlWHLzqFi6lIqlS9F4eeE2fhwekybhPHDgSU9AejxOGifiek0jrtc07gTMhUns2/EBiVnrSLSUs92gp5J61uduYn3uJgDcdK4MCBjIAN8BNFgasFgt6OiaXQ7FhalLBDnp6enMmTOHgoICNBoNW7ZswcXF5VxXSwghhBDNDC46eiQE0CMhAKtVoeBQRfOcPMWU5NTgVGZhMBoGo6HeSUUyZtKSSnnwQDEuzlp7soI+IR7n+lTOGJVKZZtQNCYGv4ceom7nTip/XEblTz/RVFpK+edfUP75F2iDgnCfMOGU5uDpCJ1/LP3HzaM/cHvxQcz7viY55VsSaw+TaDCw3aCnylzNr4d/5dfDvwKwaMkiLg26lBujb2Rw0OALcn4k0bV0iSBn9uzZPPfccwwbNozS0lL0ev25rpIQQgghjkGtVhHU3ZOg7p4MuTqKypI6svaWkLGnhMP7yzA0WhmAlgGNWiwqhYwaKzvXH+abTZl0C3FnRnwoV/UPwcvF6VyfyhmjUqsxDhyIceBAAv7+ODVbfqdy2TKqVq/GkpdH6fvvd8ocPCfk2x3diMe4ZMRjXFJ8kDlJ32LZ9x37yw6QaNCTaNCzzWCgihrWZa9jXfY6enr1ZGbsTCZETMBJ03WfI3F+u+CDnH379qHT6Rg2zJY20fs0UjAKIYQQ4uxz93Gmz4hQ+owIxdzYRE5KGRl7isnYU0JNeQPdLRq6WzRQBwU1Daw5dID3l+6nbz8/rk/oxmXdfdF00WQFACqtFtfLL8P18suwPv0U1b/+SuWy5WdkDp7j8u0Owx9BO/wRepek0Xvft8zc9x3WrD3sd9Lxnasr37m5cKDsAE9seoLXtr3KjTE3M6PnDDwNnp1fHyGO45ynyfj111+ZMmUKwcHBqFQqvvvuuzbbLFiwgPDwcAwGA4MGDeKPP/6wr0tNTcXV1ZUpU6YwcOBAXnjhhbNYeyGEEEJ0Jp2ThvBLfBl5czSz/jWU659IYNDUSAIj3W0TjzapGdKg4/pKJ0wby/n8zR3c/OTPvLosmayS2nNd/TNOrdfjPmYMoa/9hx6bNhH84r9xGT4MNBrbHDzzXubgqCvIuPlPlH72GZbS0jNTEZ8oGP4wltvXsS7mJXoNfYy/O4WxOjuHv5aW4W+xUFxfyps73mTMV6N4bsM/yKjIODN1EaId57wlp6amhn79+jFnzhyuueaaNuu//PJLHnroId5++20GDRrEa6+9xrhx49i/fz/+/v5YLBY2bNjAzp078ff3Z/z48SQkJDBmzJh2j9fQ0EBDQ4P978rKSgDMZjNms/m4dW1Zf6LthDgeuY/E6ZJ7SHSGC+U+8ggw0G9MCP3GhFBX1Uh2UhlZ+0rJTCrFpcFKH7MWiqHph1zmrzhMU4CehCHBTBwSgrPTOf+ac2bpnTBOnIhx4kSaysqoXr2aquUrqG+ef6dlDh7j4MG4TpyA6xVXoD7NOXiOZjabqTEE0nDpLegu+yvG0kPMTv6eW5K/Y2XpIT72cCdFD18e+p6v0r5nhFsEN/e7h4HdRsm4HWF39PtRZ7wvqRRFUU57L51EpVLx7bffMm3aNHvZoEGDSEhIYP78+QBYrVbCwsK47777+Nvf/sZvv/3G008/zcqVKwGYN28eAI888ki7x3j66aeZO3dum/LPPvsMo7HrpqoUQgghuhLFCg1lGmoLtVTma9HWO3ZOKVdbqfVoIiDETFhwExdTtmNteQVuu3fhtnMXhpwce7lVq6UmOpqq/v2oiY5G6aSU1MdibCgguOx3Cmu38rWhll+MzvZ1Pc1qRqujCXObQJOT1xmth7jw1NbWctNNN1FRUYG7u/sp7eO8DnIaGxsxGo0sWbLEIfCZNWsW5eXlLF26FIvFQkJCAj///DMeHh5cddVV3HnnnUyePLndY7TXkhMWFkZxcfEJL6LZbGb16tWMGTMG3Rl+YxBdl9xH4nTJPSQ6Q1e7jyqK6tiTWMDexEKUwno0HGklMKvBKdiZgYODiBngh9H94hkM35iRQfWKn6hasQJzerq9XOXiguuVV+A6YQLGQYNOeQ6eDt9HZelk7f6ETzKX8YO6nobmFNgBFgs3aXy5uscMXHtfCy5+p1QPcWE7+j6qrKzE19f3tIKc87odt7i4mKamJgICAhzKAwICSElJAUCr1fLCCy8wfPhwFEVh7NixxwxwAPR6fbvZ13Q6XYff5E9mWyGORe4jcbrkHhKdoavcR77BOkZNdWfU1B7U15r5+Zcsdv6eh7agARerCuVwHduWHGLbkkMYApzpEx9AxCW++IW5oerCSQt0PXrg0qMH/vfd22YOnqrvf6Dq+x86ZQ6eE95H/j2JGv0MT/EM9+ft4qvEV/m8ZAcFWi3/oZy3D7zNNdtf4Wa3XoT1ng4xU8HV/zTOXFyIWu6jznhPOq+DnI6aMGECEyZMONfVEEIIIcR5wGDUMXFCFBMnRFFe3ch369PZ9XseLqUWgprU1BfUkbgsg8RlGejddERd4oupry+h0V44GbrEV6M22p+D50cqf1p5VufgAfAK6sedUz5idlMDy/d8zMfJizjYWMan7m58ruRw5dZ/MXPtP+gfmACxV9kCHreAE+9YiFbO61eyr68vGo2GgoICh/KCggICAwPPUa2EEEIIcaHwdHVi9uReMLkXSbmVLNmUwb7EAgJrIdyshiozSZvySNqUh1qjIqSXF+F9fTD18cXDz/nEB7gAOc7B83dqfttim4NnzRrHOXgiInCfOPGMzcGj1+i5uv+fmdbvdn7L/Y2Pd73DpqLtrHYxstrFyCX1B5i5/p9cufwRtOGXS8AjTsp5HeQ4OTkRFxfH2rVr7WNyrFYra9eu5d577z23lRNCCCHEBSU22J0np19Cw9VNrE0uZPEfWWQklxLRqCHKosazSU12UinZSaVs+DIVr0Aj4X19MfX1ITDKA43mnM+80elUWi2uwy7HddjlWBuepvqXX2xz8KxfT2N6un0OHkNsrG3S0YkTOn0OHpVKxdCQoQwNGUpqWSqLkhbx46Ef2G2Ahw1+hJgt3Fy6k6tXbMR1+SNgugx6T5OARxzXOQ9yqqurOXjwoP3v9PR0du7cibe3N926deOhhx5i1qxZxMfHc+mll/Laa69RU1PDrbfeelrHXbBgAQsWLKCpqel0T0EIIYQQFxC9VsPEvkFM7BtEXkUdX287zFdbs6kurifKrCHSoibUoqEsv5ay/Cx2rM5Cb9QSFutNeF9fuvX2xtm16yUvUOv1uI8di/vYsTRVV1O9di0Vy5ZRs2kz9UlJ1CclUThvHs7xcXhMmoTbuHHg5tapdejh1YNnLnuG+wfez5f7v+TLlC/JoYyXfLx4y9ubaysruPnwbwRlboTlj4BpKMROg9ip4Ca9fMQR5zy72vr16xk1alSb8lmzZvHhhx8CMH/+fObNm0d+fj79+/fnjTfeYNCgQZ1y/MrKSjw8PDqUvcFsNrN8+XImTpzYJQZpinND7iNxuuQeEp1B7iNHVqvCHxmlfJWYzfI9eSgNVsItGrpbNPS0atFajnxdUqkgIMKD8Ets3dp8Qly69JwvlrIyqlaupPLHZdQmJh5ZodFgHDyYtJBgLnvwQfRenZ8Kut5Szw+HfuDjfR+TUZlhOywqxjY5MTM/kz6Njc1bqqDbkCMtPO6d29okzqyj349O5vv5sZzzIOdckyBHnG1yH4nTJfeQ6AxyHx1bZb2ZH3fl8VViNjuzy1EpENSkpo/KiT4qHZpKi8P2rl56e7e20F5eaJ267qQ85vx8KpevoPLHH6lPSrKXq/R6XEeMwH3SJFxHjkDdTibb02FVrGzM2cjH+z7m9/zf7eUDDYHMrK5jZPYejlx1CXguNGciyDnn3dWEEEIIIc4n7gYdNw3qxk2DunGgoIrFidl8sz2HVTX1rKIeN3cVo9xc6avW05RfR3VZA3t/zWHvrzlodWpCo70w9fXF1McHN2/DuT6dTqULDMRnzq34zLmVhvR0yn/4gfzFS3AqKqJq1SqqVq1C7eKC2+jRuE+ehMuQIai0p/91U61SMzx0OMNDh5NSmsLH+z5mRfoKttfns10L3foM5k8ukVyVcwDj4UTI2mxbVjwG3QZD76th4CzQda3nQxybBDlCCCGEEMfQM8CNf0yK5ZFx0fycUsjixGzW7S/k+5oqvqcKN08NVwf70FftRG1GNdVlDWTsKSFjTwkAPiGuhPf1IfwSX/zD3VF3oTl59BEReP/lL2zp1o0rIyOpWbmSyuUrsOTlUbF0KRVLl6Lx8sJ9wnjcJ03CecCAU5qD52jR3tG8MOwFHhj4AF/s/4Kv9n9FVk0uL9TkMt/FneljH+amJj3++1fD4T8g6zfbcugXuP4T6IQ6iPOfBDlCCCGEECfgpFUzvk8g4/sEUlBZzzfbc1icmM2h4ho+ziwEINLHyHWDguiNjuIDFRSkV1CSU01JTjXbfsrE4KrD1NsHU18fusV6ozd2kW6CKhX6mBhcL7kE///7P+p27KBy2TL7HDxln31O2Wef2+bgmTgBj0mT0MfEnPY4pgCXAB4Y+AB/7vtnlqYtZVHSIrKrsvlf6ld8pNYyodcEZo55guicPbD2Gdi/DDa+CsMf7qQTF+czCXKEEEIIIU5CgLuBv4yM4q4RkSRmlvHV1myW7cnjUEktL5UcQqNWMbKnH9eM6km4WU1OUilZSaXUV5vZ/3s++3/PR6VWEdzdA1MfX8Iv8cEzwNglkheo1GqMcXEY4+Ic5+BZvdo2B8//3qf0f81z8EyahPukiegjTm8OHqPOyI3RNzKj5wzWH17Px/s+Znvhdn449AM/HPqBQYGDmDnsL1y+/jXUPz8Hwf2h++jOOWFx3rpogxxJIS2EEEKI06FSqUgI9yYh3JunpvZm+e48vkzMZltmGWtTClmbUoiPixPXDAzhuon9cK22krmnhIw9xZTl15JzoJycA+Vs/uYg7r4Ge/KCkB5eaHQXfpcqhzl45raag2fdOtscPPPnUzx/fqfNwaNRa7iy25Vc2e1K9hbv5eN9H7MqcxW/5//O70BEVDS3FGYz5evbMdzxC3iZOu9kxXlHsqtJdjVxlsl9JE6X3EOiM8h9dOYcLKxm8bZsvt6WQ3F1g728f5gnM+LDmNIvCGuVhcy9xWTsKSHnQBnWVimqtXoN3WK8MfX1wdTHBxePzs1U1plO5T5qqq6mas0aKpctp2bzZmj1g7N9Dp7x49F2QkrqvOo8Pk3+lK9Tv6baXA2AV1MT11uNXD/jO3zdQ0/7GOL0SXY1IYQQQojzXHd/Vx6fEMPDY3vxy/4ivkrM5ueUQnZml7Mzu5xnftzHxL5BtoBnZCjmhiYOp5SRuaeYjL0l1FY0cmhnEYd2FgHgb3LD1MeWvMAvzA3VBZ68QOPqiue0aXhOm4altJSqlSupWLaMusRt9iX/uedxGToUj8mTcL1yNBpXl1M6VpBrEA8nPMxd/e7i24Pf8sneD8mtK+RtTQP/+3Yik7tP45bYW+jh1aOTz1KcaxLkCCGEEEKcATqNmtGxAYyODaCoqoFvdxzmy63ZpBXV8M32HL7ZnoPJx8j0uFCujQtlVH8/FKtC8eFqMvYUk7G7mMLMKvuydVkGRncnW8DT15fQGC+cDBf2VzmttzdeN96I1403Ys7Ls83Bs2wZ9UlJ1GzYQM2GDbY5eEaOxH3SRFxHnNocPK5OrtwSews3Rt/I2q1v8vGu/7Jbr+fbg9/y7cFvuSz4MmbGzmRI8JAuMTZKSJAjhBBCCHHG+bnpuWN4FH8eFsmO7HIWJ2bzw648MktqeXnVAV5dfYDhPf2YER/GlTH+JHSLIGFSBDUVDWTtKyFzTwlZSaXUVjaSvDmP5M15qDUqQnp62pMXePgZz/VpnhZdUBA+t83B57Y5NBxKp3L5ciqXLaMxPZ2qlSupWrkStaurbQ6eSZNwGTL4pOfg0aq1jBv0IOMsGnb++iwfe3iw1sXIptxNbMrdRHfP7syMncmkyEk4aZzO0JmKs0GCHCGEEEKIs0SlUjGwmxcDu3nxz8mxLN+Tz1eJ2fyRXsr6/UWs31+El1HHtAEhzIgPIybInZihwcQMDabJYiU3tdyevKCiqI7s5DKyk8vYuDgVr0CjvZUnsLsHGs2Fm7xAHxmB37334HvP3TQkJ1OxbNmROXi++46K775D4+2N+/hxpzYHz9D76J+zjf5J35HtEcRnCdfzdeYKDpYf5MnNT/L69tdtGdt6zcDLcPpjg8TZJ0GOEEIIIcQ5YHTScl1cKNfFhZJeXMOSbdks2XaYgsoGPtiUwQebMrgk1IPp8WFM7ReMh7OOsBhvwmK8uXxGD8oLam3d2vYUk5daQVl+LWX5texck42Ts5Zusd6E9/WhWx8fnF0vzFYJlUqFITYWQ2ysbQ6e7dupWLaMqqPn4AkOwn3CSczBo1LBVfOhKIWwohQeS93KX274ia/TlvJp8qcU1BYwf+d83t3zLlOjpnJL7C1EeJxeqmtxdkl2NcmuJs4yuY/E6ZJ7SHQGuY/OT5YmKxtSi/kqMZs1yQWYm2xf0/TNk5FeHx/G4Egf1EclH2ios5CdVErmnmIy95VQV2U+slIFgRHu9m5tPiGunTbu5FzdR4rZTM2WLVT+uIyqNWuw1tTY1zlFRuI+cWLH5uApToWFo6CxCgbfDeP/hdlqZlXGKj7a9xHJpcn2TUeEjmBm7EwSAhNk3E4nk+xqnUjmyRFCCCHE+UarUTMq2p9R0f6UVDfw3c5cvtqazf6CKpbuzGXpzlxCvZyZHhfGdfGhhHg6A6B31tI9zp/ucf5YrQqFGZVk7rV1ayvOrib/UCX5hyr5/ftDuHrp7d3aQqK90DlpzvFZnzyVTofrsGG4DhuGtb6e6l9+pXLZMqrXr6fx0CHHOXimTsHrpptQO7XTmuXbA65+G768Gba8BSFx6Ppex6TISUyMmMi2gm18lPQRv2T/wi+HbUuMdwy3xN7C+PDx6DTyA8H5SlpypCVHnGVyH4nTJfeQ6AxyH104FEVh9+EKvkrM5vuduVQ1WABbj6vLu/syIz6MMbEBGHTtByvVZfXNAU8Jh5NLsZit9nUanZrQXl6E9/XB1NcXN2/DSdXtfLuPjjUHj6FPH0Je+w9OoceYF2ftM7DhFdAZ4fY1ENDbYXVGRQafJH/C0oNLqW+qB8Df2Z8bY25kes/peOg9zuh5dXXSkiOEEEIIcZFRqVT0C/OkX5gnT0yKZeW+fL7cms1vh0rYkFrMhtRiPJx1TOsfzPT4MPqEOH7hdvUy0HtYCL2HhWBpbCLnQLltTp49JVSV2gKgzL0l8PkBfEJcMPX1JbyPDwGRHm26xZ3vjp6Dp3LFCorfeJP6vXtJv+Zagv/9b9yuGNX2gaP+Abk7IO1n+OJmuGM9OHvaV4d7hPPE4Ce4t/+9LD6wmM9SPqOwrpDXt7/Owt0LmdZ9GrfE3EKYe9hZO1dxfBLkCCGEEEJcIJydNEwbEMK0ASFkldTakxXkVtTz0W+ZfPRbJrFB7lyfEMZV/YPxNDp20dI6aTD18cHUx4dhNyiU5tXYs7Xlp1VQklNDSU4N23/KxOCio1tvb8L7+hIW643B5dy31JwMrbc33jffjNuoURx+8EHqd+3m8N1343P7bfj99a+O6afVGrj2f/DOCChLh2/vhBs+h6MytnkaPPnzJX9mVu9ZrEhfwcdJH3Og7ACfp3zOFylfcEW3K5gZO5MB/gNk3M45JkGOEEIIIcQFqJuPkYfG9uKB0T3ZdLCYLxOzWb2vgKS8Sp76fh/PL0tmbO8AZsSHcVl3XzRHtcqoVCp8gl3xCXZl4DgT9TVmsvbZurVl7SuhvsbMgT8KOPBHASq1iqAoD0x9fQjv44tXkPGC+RKvCw4mfNEiCl5+mbKPF1Hy3v+o3bmTkFdeRRfgf2RDozdcvwjeHwcHfoJf58HIx9rdp5PGiau6X8XUqKlsydvCx0kfszFnI2uz1rI2ay19ffsyM3Ymo02j0arl6/a5IFddCCGEEOICplGrGN7Tj+E9/SiraWTpzhy+TDxMcl4lP+7O48fdeQR7GLguLpTp8WGEebc/aajBRUfPSwPpeWkg1iYr+Ycqydxr69ZWmltDbmo5uanl/PZNGu6+Bkx9fAmN9US5AHI4qZycCPz73zEOjCPvH/+gLnEb6ddcQ8jL83AZMuTIhsH9YdKrsPRuWP8vCBkIPcYce78qFUOChzAkeAhp5WksSlrED2k/sKd4D4/8+ghBLkHcHHMz1/S4BjcntzN/osJOEg9I4gFxlsl9JE6X3EOiM8h91PXtzalgcWI23+3MpaLuSErpoVE+zIgPY3yfwGMmKzhaZXGdPVtbzv5ymixHkheoNArdYn2J7OeHqY8PLp76Tj+XztSYkcHhvz5IQ0oKqFT43ncvvnfd5TiZ6I8PQuL7YPCwjc/xjuzw/kvqSvhq/1d8sf8LSutLAXDRufDmFW+SEJjQyWfTNUjiASGEEEII0SF9QjzoE+LB4xNjWJVUwOLEbDYeLGZzWgmb00pwW6rlqv7BzIgPo2+Ix3G7n7n7OtN3ZCh9R4ZibmjicEopGc1jeWorGsncU0LmnhIA/Lq52bu1+ZvcUJ1nyQucwsMJ/+JzCp5/nvLFSyh+403qtm0neN5LaL29bRuN/zfk74HDW+HLmXDbKnBqvwXsaD7OPvyl/1+Y03cOP6b9yEdJH5Fekc7r21/nk4mfnMEzE61JkCOEEEII0YUZdBqm9gtmar9gDpfVsmTbYRYnHianvI5PtmTxyZYsogPdmB4fxtUDQvB2aWc+mVZ0eg0R/fyI6OdHY2MjS79YSTevWLKTyijIqKQoq4qirCoSl2Xg7O5km5Onjw9hMd44OZ8fXz3VBgNBzz6L88A48ufOpWbTJtKvvoaQ//wH48ABoNXDjI/hneFQsAd+/Ctc/Y4tb3cH6TV6ru15LSPCRjBm8Rh2Fe3iQNkBenr1PHMnJuzUJ96ka1qwYAGxsbEkJEizoRBCCCEuDqFeRv46uicbHh3Fp7cP4qr+wThp1aTkV/Hsj0kMemENd3+6jXX7C2mynnhEg0qlwsnDysDx3bjusXhuffFyrpwVQ9RAP5wMGuoqG0nZnMdPC/fyv4c3sPS1Hexam019jfmE+z4bPK+eRviXX+IUEYGloIDMmTMp+eBDFEUB92CY/iGoNLD7S/jj3VM6hq+zL6O62dJWL96/uBNrL47nog1y7rnnHpKSkti6dev/t3fncVWW+f/HX+cAh3PYBAQFkhQVU1AUcQkMw1zILKUptbLSprRxcBx10sZSXMq20RZHGnMqtW+aZovNL7cMc1rEPSslzV1TcQNFZTks5/cHeSZySWQ5cHg/H4/zyPu+r/u+P7de6flwXdfndnQoIiIiItXKaDTQpXkAr90XzaanevBMv0ja3FCPwmIby3/I5JG5m+jywhr+sWonB05duObreviYaBkbzO3D2vDH6fH0G9WOtj1C8W3oQUmxjZ93ZvP1kt28m5LOD2t/pqS45PcvWsXMN7WgyZIl+NxxBxQVceLFFzkyciTFOTnQ5Bbo9Uxpw1Xj4WD6dd3j3hb3AvDpvk/JK8qrrNDlKupskiMiIiIiUM/DjYdim/D//nILy0fG80iXJvh5uJGZk0/qF3tJmL6WAW+k8+GWn8m1Fl3zdV1cjTRq6c8t94YzaMrNDJpyM7f0D8cv2JOCC0V8uegnFj27iUMZp6vw6a4xVi9PQmZMp2HKRAxubpxb/Tn777mX/IwMuPnPEPkHKCmCJYPhXGa5r39z8M008mrE+cLzrNy/sgqeQH5LSY6IiIiIABAR4sOkuyJZ/1R3Xh/UnltbBGIwwMb9WfxtyXd0mpbG+I++Z+uhbMpboNe3oQdtu4dy34SOdL2vBWZPN7KPXeD/zfyOZanfkZ157SNGVcFgMOD/wAM0XrgQtxtuoPDwYQ7cdz/Z7y/B1vef0CACzh+H9wdDkbVc1zYajNzT4h4APvjpg6oIX35DSY6IiIiIlOHu6sIdbYKZ/8dOfPPkbTzRqwU3+ntwvqCI9zYe5g+vr6PXK1/y1jcHyCnf932MLkbaJDRi0NSbaXtbKEajgQM/nGbR1I18vWS3w9frWNq0JuyjD/Hq1g2b1UrmpEkcnTiVkr5vgrsPHF4Pn00o93WTmifhanDl+1PfsytrVxVELr+mJEdERERErijE18KI28JZ+0QC7w29mT9E34DZzcjuE+d5YeVPTNrqwp8XbuPzjOMUlWONjdnTjVsGhHNfSicat6lPSYmN79IOsyBlPdv/69j1Oi716tEodRYNxj4BLi7k/Of/sf9PT1LQYWppg41vwHeLy3XNMgUIflIBgqqmJEdEREREfpfRaCC2WX1eHtiOjU/34Lm729C2UT1KbAZW/3iCx97ZTOwLa3hhxU72njx/zdf1C/LkzuS23PWXtvgFe5J/oZD/vvcTi6dt4vCPWVX4RFdnMBqp/+ijNJ4/D9fAQKx79rJ/3D85a+5f2uD//bX0XTrl0L9F6bnL9i0jtzC3skOWX1GSIyIiIiLl4mN244HON/LB4535e9si/hjXmPqeJk6eK2D2f/fSfcZ/ufdf63h/02EuFFxbsYIbI+vb1+u4e7qSdfQC/3ltG8te/54zxx2XEHh06EDY0o/xiL0ZW14eR+d9w7GfIikpyIPFD0Je9jVfq3NwZ0K9Q0sLEBxQAYKqpCRHRERERK5bsAeM730T6eO7M/vBGLq3bIDRAJsPZjPuw+/pOO1zxn3wHZsPZP1usYKL63UenBpL1G2NStfrfH+K96Zu4OsPdlOQ65j1Oq7163Pjm28S8OfhYDBwZms2B78IwXr4MHw4FEqubWqd0WC0l5NWAYKqpSRHRERERCrM5Grk9tZBvDWkI+njuzPu9psIC/Ak11rM+5t/5t7Z6XSf8V/+tXYvJ3Lyr3ots6cb8QNalK7XaV2fkmIb331+mHdT1rP9yyOUXMOLSiubwcWFwJEjCZ0zBxc/P/JPwf5VgZxb+xX894Vrvk6/Zv1wNbryw6kf2Jm1swojrtuU5IiIiIhIpWroY+bPCc1Z87dbWfKnWPrHNMLD5MK+Uxd4ceVOYl9Yw2PzN7FqRyaFVykw4BfkyZ0j2nLnX9riF+RB/vlC/rtwF+9P28jPOx2zXscr/hbCPv4IS3Q0JYVGfv7an+Ov/gvbjmXXdH59S32639gdgCW7VICgqtTZJCc1NZWIiAg6duzo6FBEREREnJLBYKBjE3/+0b8tG5/uwYv3tCGmsR/FJTY+//EEj//fFmKfT2Pasgx2Hz93xes0jqzPwImdiB8YjruHK6ePXOCTV7ex/F/fc+ZE9a/XcQsKovE78/EfMgSArF1eHBw+msIfN1zT+fYCBPtVgKCq1NkkJzk5mYyMDDZt2uToUEREREScnpe7KwM73siHw+P4fMytPH5rUwK83Dl13sq/v9pPz1e+5O7Xv+G9jYc4l3/p2hsXFyNR3UJ58JlY2nRrhMFoYP93p3hvyga++XAPBXnXVuCgshjc3Gj49ye54dWXMZoM5J1wYf/9j3D+v2m/e27HoI7c6H0jFwovsGL/imqItu6ps0mOiIiIiDhG8wZejO/divTxt/HvhzvQM6IhLkYD3x46w/iPfqDjtM8Z8/42Nuw7fUmxArOnG10HtuC+iZ24MdKfkmIb21YfYkFKOju+qv71Oj639yZs0Tu417dRnG/j8J9GcPKf/8RWXHzFc35dgEDvzKkaSnJERERExCHcXIz0jGjIvx/uQPr423jqjpY0C/Qkv7CEj7YeYeCc9XSbvpbUL/aQebZssQL/YE/u+ks77hxRul4n71whaxfs4v1pm/h517WXda4MpogONJn3Br7N88AGp1Jf5/DQoRSdPn3Fc/o174eb0Y0dp3eQcTqjGqOtG5TkiIiIiIjDNfA2M6xrMz4fcysfDo/jvo6heJpcOHA6l3+s2kXcC2kMmbuR5T8cw1r0v2IFjVuXrte5ZcDF9Trn+eSVb1kx+wfOnqy+9S7G8FsJnvB3Qm7OxuBi48K6dPbf/Qdyt2y5bHt/sz89buwBqJx0VVCSIyIiIiI1hsFgIKaxHy/cE8WmCT2Y3r8tncL8KbHB2l0n+fOCrdz8fBpT/18GOzNzgNL1Om1vC+XBqbG0SShdr7Nv20kWTtnAuo/2YK2u9TqdH6feXXcS1uskJl8bRSdOcPDhwZx+663LviPo4pS1ZfuWcaHwQvXEWEcoyRERERGRGsnD5Mq9MY14//FYvngigT8nNKOBtztZF6y8/c1+bn/1K/rO+pp31x/kbF4hZi83ut7XgoETOhIa4U9JkY1vPzvEuynpZHx9tOrX6xgMcNdruLdoSVj3THxaWaC4mBP/mM7PySMoPnu2TPOOQR1p4tOE3KJclu9fXrWx1TFKckRERESkxgsL8GTc7S1Z9/fbmDukI71bB+HmYuD7n88yYel2Ok37nFGLvmXdnlP4BXly11/a0ic5Ct+Gpet1vnh3J0ue38SRn6p4vY7JEwb+H0ZvH0Ki9hJ0b2sMbm6cX7OG/ffcS972HfamBoPBPpqjKWuVS0mOiIiIiNQari5GurVswL8ejGH9+O5M6NOKFg29KCgqYem2ozzw5gZunf4F/1yzB7dQT+6b2Ilb+peu1zl1+DxLX/6WFW/8wNmTeVUXpH9T+MObGAwG/Fw/o/HUwbg1akThzz9z8P77yX7vPfv0tb7N+uJmdCPjdAY7Tu/4nQvLtVKSIyIiIiK1Un0vdx6Lb8qqUV35JLkLgzrfiLe7K4ez8nh59U/c8uIahszfxKEAF+6d2InWt96AwQD7vj3JwinrSf+4CtfrtOgFCX8HwLLjJcL+NQWvHt2xFRaSOWUqR58YS8mFC/iZ/ejRuLQAwZJdKiddWZTkiIiIiEitZjAYaBvqy7S727Dx6R68MrAtsU3rY7PBV7tP8Zf3vuXWmV/yhWcRMY9HENrKj5IiG1tXHeLdSevJ+KaK1ut0HQfhiVCUj8uyx2n04hQaPPkkuLqSs2wZ+/sPoGD3bvq36A/A8v3LOW89X/lx1EFKckRERETEaVhMLtwd3Yj3ht3Ml2O78ZfbmhNcz8zZvELmrTvAwEVbeINzmG9riHegmbwcK1/8X+l6naO7K3m9jtEIf3gD/MLgzCEMHz1G/cEP0fid+bg2bIh13z72DxhI+PqjNPFpQl5RngoQVJI6m+SkpqYSERFBx44dHR2KiIiIiFSBG+t78LdeN/H1k7cx/4+d6BMVjMnFyI5j53hm6wGeKz7L8aZmjO5GTh0+z8czvmXlnB/IOVWJ63UsfjDwXXC1wN418MVzeLRvT9jHH+HZpQu2vDyO/f3vPJHmiVuhjQ9++uCy5aalfOpskpOcnExGRgabNm1ydCgiIiIiUoVcjAZubRFI6gPt2fBUdybfFUGrYB/yS0p4Jyubme4X+MkbbMDerSdZOHkD6Uv3Ys2vpPU6Qa2h7z9Lf/3VdNi5DFd/f0LnvEHAX0aAwUCDz79j2v+VkLVHBQgqQ51NckRERESk7vHzNDGkSxjLR97Cp3+5hYdjG+Pm4conLnnM887noGsxxUUlbF15kHdT1vPjuqPYKmO9TlR/6Dy89Ncf/wlO7cHg4kJgcjKhb/4bF39/mhy38eK8YtLfe7Xi96vjlOSIiIiISJ1jMBhofUM9pvZrzcanezDz/mha3lSfJV5WPvIsINtYQl6OlTXv7OSdqesr5/06vZ6BG+OgIAcWD4KC0iIDXl26EPbxRxS3aYFHAdyS+g2Hn52KzWqt+D3rKCU5IiIiIlKnmd1c6Ns2hHcf68yX47pxV+9mrAo18oW5kAJsnM/MY+nL3zL7uQ0cOpxz/TdycYP+88ArCE7uhP+MgF/W37g1bEjEgiX8t6sfAOfffY+DDw+m8NixSnjCukdJjoiIiIjIL0L9PRjVowX/fbIbo0fE8HOcLz+YiynBRvGhC3w8bROTp31D2g/HKL6eaWzeDWHAO2B0gx0fQ/os+yGjyYTnqOG8eK+RPIuRvG3b2H/3H7iQnl6JT1g3VEqSc/DgQTIyMigpKamMy4mIiIiIOJTRaKBL8wBefjiGF59LwPuuRpz2NOCKgcDDBWx+PYOHJqxh+sqdHDx9oXwXv7Ez3P586a9XT4L9X9kP9W3Wlx9uMvPEEAO2m5pSfOYMR0aPwVZcXIlP5/zKleS8/fbbvPzyy2X2DRs2jKZNm9KmTRtat27N4cOHKzVAERERERFHqufhxpA+NzFxegKtBzaj2MMFL5uBLlkGzv6/n3ng+S8Z+EY6H275mVzrNVZk6/gYtL0fbMWwZAicPVJ6L/d69GrSi5O+Bt4f3Q6jlxfFZ85Q8NNPVfeATqhcSc6cOXPw8/Ozb69cuZK5c+fyzjvvsGnTJnx9fZkyZUqlBykiIiIi4mgGg4FbuzUm+cV4OiU1xWAyElRs5IHz7gRuP8eUxd/TaVoa4z/6gW8PZV/9fTcGA9z5CgS1gdxT8P5DUFQAQP8W/QFYfmQ1bm3bAJC7ZWuVP58zKVeSs3v3bjp06GDf/uSTT+jXrx+DBg2iffv2PPfcc6SlpVV6kCIiIiIiNYWLm5GOtzdhyLNxRMSHgAFaFbry6Dkzbc/ABxsOcffr6+j1ypf8+8t9nDpfcPkLuVlKXxRq9oUjW2DFkwBEN4imWb1m5BXlcaCxGYC8rVuq5+GcRLmSnLy8PHx8fOzb69ato2vXrvbtpk2bkpmZWXnRiYiIiIjUUB4+JroNasnApztyQwtfXG0QV+DGiDwP2hW7svv4eaYt/5Gbn0vj8f/bTNqPxykq/s0adr8mcM9bgAG2zIWt/4fBYKD/Tb+M5njvA0pHcq46MiRllCvJady4MVu2lGaRp06dYseOHXTp0sV+PDMzk3r16lVuhCIiIiIiNVhAI2/6jY6m9+Nt8Akw42q10fOcG39386Wrvw9FJTZW7TjOo/M3E/vCGl5YsZN9J8//7wLhPaDb06W/XvY3OLKVO5veibuLO194/4zNxYWi48cpOnrUMQ9YC5UryRk8eDDJyck888wz9O/fn5YtWxITE2M/vm7dOlq3bl3pQYqIiIiI1GQGg4Gm0YE8MOlmYu9uhpvZheJTBXTeV8j0RjcwNOZG/D1NnDxXwOz/7uW2Gf+l/+x1vL/5MBcKiiD+b9CiNxQXwPsPU6+oiMQmiVjdDJxuXDqIkLtV63KuVbmSnHHjxjF06FA++ugjzGYzS5YsKXP8m2++4f7776/UAEVEREREagsXNyPtExvz4NRYIroEgwGOb8+i/penmdWmKa8PbMdtLRtgNMCmA9mM++B7Ok77nHEf/cC3HV7A5t8Mzh6GD/9I/+Z/AGBT4DkAcrdoXc61ci1PY6PRyNSpU5k6deplj/826RERERERqYs8fEx0e6gVrRMa8fX7uzm6+wzfrjyIZz0TTw1owfN/aMOHW39myeaf2X/qAu9v/pn3N//Mbf5/5Q3jONz2raXtD+1o7tuc7Tf8RG8gb+u3jn6sWqPcLwNdvHgxgwYNon///syePbsqYhIRERERcQqBod4kjYnm9sdb4xNg5sJZK6ve3I71SC5/TmjOmr/dypI/xXJvTCMsbi6syQpgdP5QAAzfvEpscWN2NTIAULB7N8VnzzrycWqNciU5//rXv7j//vvZvHkzu3fvJjk5mbFjx1ZVbCIiIiIitZ7BYKBZdAPun9SZm24OAht8PjeD3BwrBoOBjk38md6/LZsm9ODFe9pwLPQO/l10BwAP/bCIcxY3jvoBNht527Y59Flqi3IlObNmzWLSpEns2rWLbdu2MX/+fF5//fWqiq1KpaamEhERQceOHR0dioiIiIjUAa5uLiQ8cBP+IZ7k5lhJm5+BreR/ZaG93F0Z2PFGPhweR7cRr3PQO5pgWx7dzlvZFVo6mrN/bbqjwq9VypXk7Nu3j8GDB9u3H3jgAYqKijh27FilB1bVkpOTycjIYNOmTY4ORURERETqCFeTC70ei8TVzcihHVls+/zwZds1D/Kj8ePvY/MO4ZHzJ9j5y5S1I199U53h1lrlSnIKCgrw9PT838lGIyaTiby8vEoPTERERETEGdUP8eKWAeEArF+6l+MHci7f0KsBhgHv0LbQRl7DIgCCju3j+30nqivUWqtc1dUAJk6ciIeHh33barUybdq0Mi8BffnllysnOhERERERJxRxSwiHf8xm79YTfPbmdgY+3QmT5TJfzUM7Yuj9Irf9N4WzHj7Uyy1h4XufE/X0A9UfdC1SriSna9eu7Nq1q8y+uLg49u3bZ982GAyVE5mIiIiIiJMyGAx0e/AmThzIIedUPmsX7qLnHyMu/126wx/p8/NGPrnhSzrsBuMPy/jx2J20Cvap/sBriXIlOWvXri2zferUKUwmEz4++g0WERERESkPdw83ej0WyUfTt7J703FCW/nRKi7k0oYGAz53vopxVWfYXUjHs98yOy2D1x68ufqDriXK/Z6cM2fOkJycTEBAAA0bNsTPz4+goCDGjx9Pbm5uVcQoIiIiIuKUgprWo3PfMAC+XPQTWccuXL6hm4WoviMAaJBpo91Pz7LnxLnqCrPWKVeSk5WVRefOnZk/fz733HMPM2bMYMaMGfTt25d//vOfdO3alfz8fDZu3MjMmTOrKmYREREREafRvldjGrX0o8hawmdv7qCosPiy7SK7D6bQ1YB3HnjxPVs/erV6A61FypXkTJ06FZPJxN69e3njjTcYNWoUo0aNYs6cOezZswer1cpDDz1Ez549yxQiEBERERGRyzMYDfR4JAKLtxunj5xn3Yd7L9vO6O5OwU03ArDzjAd9j73KsR1fV2eotUa5kpylS5cyffp0GjZseMmxoKAgXnrpJT788EPGjBlT5n06IiIiIiJyZZ713Ok+JAKAH9b+zL5tJy/bLjjuNgDqHzOyw+yCZekjcP7ybeuyciU5x44dIzIy8orHW7dujdFoZNKkSRUOTERERESkLmkcWZ92PUtHata88yPnsvIvaePbsbTYQMufbcz1DsS38AQFiwZDcVG1xlrTlSvJCQgI4MCBA1c8vn//fho0aFDRmERERERE6qSb+zWlQWNvCnKLWP32DkqKS8oct7Rrh81gIDgbvje4ctRgxv3nbyBtioMirpnKleQkJiby9NNPY7VaLzlWUFDAxIkTuf322ystOBERERGRusTF1UivxyJxM7twbM9ZNi0/UPa4jw/mFi0ACD9SxJ88upceWDcTdnxczdHWXOUuPLBr1y7Cw8N56aWX+M9//sMnn3zCCy+8QHh4OD/++COTJ0+uolBFRERERJxfvUAPEgbdBMCW5Qc4siu7zHGPmPYAtDxs40jACf5V1Kf0wNJkOLGzWmOtqcqV5DRq1Ij09HQiIiIYP348SUlJ3H333Tz99NNERETwzTffcOONN1ZVrCIiIiIidUKLjkG0jAvGZoPVb+8g7/z/ZlJZokuTnIgjBqzG47zs1pn1tkgovACLB0H+WUeFXWOU+2WgYWFhrFixglOnTrF+/XrWr1/PyZMnWblyJc2bN6+KGEVERERE6pyuA1vg29CDC2etrHlnJzabDfjfSE6T4zbcrTbq3/Adfy74CzmmhnB6Dyz9M5SUXO3STq/cSc5Ffn5+dOrUiU6dOuHv71+ZMYmIiIiI1Hlu7i4kDo3ExdXIge9P8f0XP5fuDwnBNTgYY7GN5sds5Ju2ke3iwvDCUdhcTLDzU/jmFQdH71jXneSIiIiIiEjVCmjkTdw9pbOl1n20h5OHzgHgER0NQPypAIpthfTsdJjpox7BcMf00hPTnoEzhx0Sc02gJEdEREREpAZrk3ADYW0DKCmyserN7Vjzi7D8MmWt4wlvAI6VrCXIxwwxg8G/KWCDM4ccGLVjKckREREREanBDAYDtz3cCi8/d86eyOOrRT/hERMDgM/uTDyNFg7kHGDz8c2lJ5g8S/9blOegiB1PSY6IiIiISA1n9nSj5x8jMRhg5/pMDmT7YPTywnbhAveb4gBYsmtJaWNXc+l/C/MdFK3jKckREREREakFQsJ96XhnGABfLtpNcbtbAOh5phEAnx/6nKz8rP8lOUVKckREREREpIaL6d2EkHBfCguK2ebdkxKDK/V2HSOifgSFJYX8Z89/wM1S2lhJjoiIiIiI1HRGo4Gef4zA3dOV7Dwze5v2JXfLFvqH3wvAB7s/wObiXtq4UGty6pzU1FQiIiLo2LGjo0MREREREblmXn5mug+OAOBwaHcyCwPoZW6Ph6sHB3MOsslgLW2okZy6Jzk5mYyMDDZt2uToUEREREREyiUsKoCobqVrcX5s+RAXNv5In6Z9AFhSfKq0kZIcERERERGpTeL+0Bxf0wUKTd589ZWVe5uXTln7vPAUp41GVVcTEREREZHaxcXNyK0JZlyKCzhp9SV3kwet67emCBufeHtqJEdERERERGqfoK7RtPhpMQAbP91HP6/7APjQ24sSFR4QEREREZHaxjUggBvNmTTM3IitBPJX1cev2IdDbm5szM90dHgOoyRHRERERKQW84iJ4abdi/ByK+BCtpW7D/4RbLAk/2dHh+YwSnJERERERGoxj5j2uBYX0O7sKowuBkzHw4g43oU1xdmcyjvl6PAcQkmOiIiIiEgtZoluD4D7d2uJ7dsEgFsO3I3PhWA+2fOJAyNzHCU5IiIiIiK1mCmsCS5+ftgKCmjR4AyNGxditLnRY/cQPv7xE0psJY4OsdopyRERERERqcUMBgOWmNLRnLyt39L9dhsWYxb+eUE02d6J9cfWOzjC6qckR0RERESklvP4Zcpa7tatWHzc6VXvVWyU0OpEHCs+/8bB0VU/JTkiIiIiIrWch30kZys2VzON3H8gLDANAP8NERw5WbfKSSvJERERERGp5cwRERjc3SnOzsZ6LBuA3vX+Q75HDqZiC8s2pDk4wuqlJEdEREREpJYzmExYoqIAyMvYB4Cx+AL1giwAbN77fZ0qQKAkR0RERETECVja/7IuZ8dPpTuKCgi7oVHpL3Ng/dG6U4BASY6IiIiIiBO4uC4n9/uM0h2Fefj6ewLgZfVjyU9LHBVatVOSIyIiIiLiBCzt2oHBQOHhIxTlGwEbXr4uAHgV+PLF4S84mXvSoTFWFyU5IiIiIiJOwMXHB/fwcAByT5oA8PYuPVa/KIhiWzFL9yx1UHTVS0mOiIiIiIiTsL8U9FRpkuP1S5JjLvAGG3y4+8M6UYBASY6IiIiIiJPwaB8DQO4pMwCeHkVgAIoNBBiCOHL+COuOrnNghNVDSY6IiIiIiJO4WHwgP8uFkiIDLrYCPHxKR3V6B/YFYMku5y9AoCRHRERERMRJuIWE4BoUBDYDeafdoCgPb//SUZ3OXl0A2JC5wZEhVgslOSIiIiIiTsTj4vtyTpqgqAAvP3cAjBdK/1tYXOiw2KqLkhwRERERESdSpvhAYR5ev4zk5J0pAlDhARERERERqV0ujuTknTJhK8jF269sklNsK3ZYbNVFSY6IiIiIiBNxb9ECo8lASZGRgn0H7NPV8s6UTlOzYcNmszkyxCqnJEdERERExIkYXFywNPIAIHfHXvt0tdwz/1uL4+yjOUpyREREREScjEdjHwDyftz/v5GcnEKMJaVf/519XY6SHBERERERJ2MJ8wMgd9dhLF5uGF0NYAOPwnqARnJERERERKSWsTQJBIONouwLFGcew8u3dDTHq6A0+dGaHBERERERqVWMnp6Y/UrX4ORu/db+QtCLSY5GckREREREpHZxNeMRaAUgd+sWvH4pI+1l9QW0JkdERERERGqbHlOw/PlNAPK2bLUXH6grIzmujg6gMjRp0gQfHx+MRiN+fn588cUXjg5JRERERMRxXFzxiIkBoGD3bjxLK0rjZS1Ncpx9JMcpkhyAdevW4eXl5egwRERERERqBNeAAEyNG2M9eBC304cB8L44klPi3CM5mq4mIiIiIuKkLO3bA+By8EcAPLUmp3p8+eWX3HXXXYSEhGAwGFi6dOklbVJTU2nSpAlms5nOnTuzcePGMscNBgO33norHTt2ZMGCBdUUuYiIiIhIzeYRU5rkGLZvAsBc5Ilrscnp1+Q4PMm5cOECbdu2JTU19bLHFy9ezJgxY5g0aRJbt26lbdu2JCYmcuLECXubr7/+mi1btvCf//yH5557ju+//766whcRERERqbEs7UvX5RT9sBWT2QUorbDm7O/JcfianN69e9O7d+8rHn/55ZcZOnQojzzyCACzZ89m2bJlvP322/z9738H4IYbbgAgODiYO+64g61btxIVFXXZ6xUUFFBQUGDfzsnJAaCwsJDCwsKrxnrx+O+1E7ka9SOpKPUhqQzqR1IZ1I9qPkOjGzD6+VGSnY2HxYY1v7TCWkFhQY35c/ttP6qMuBye5FyN1Wply5YtjB8/3r7PaDTSo0cP0tPTgdKRoJKSEry9vTl//jxr1qxhwIABV7zm888/z5QpUy7Z/9lnn+Hh4XFNca1evbqcTyJyKfUjqSj1IakM6kdSGdSParaQ4GC8srMpOpsJNMCrwI8v/vsFDVwaODq0Mi72o9zc3Apfq0YnOadOnaK4uJiGDRuW2d+wYUN27twJwPHjx7n77rsBKC4uZujQoXTs2PGK1xw/fjxjxoyxb+fk5BAaGkqvXr3w8fG5ajyFhYWsXr2anj174ubmdr2PJXWc+pFUlPqQVAb1I6kM6ke1Q/aJE5zOyMC/5DznaYCX1Zdb4m+huW9zR4cGXNqPLs60qoganeRci6ZNm/Ldd99dc3t3d3fc3d0v2e/m5nbN/3OWp63IlagfSUWpD0llUD+SyqB+VLN5d+zIacDt2D4IbopXgR9GF2ON+zO72I8qI64aneQEBATg4uLC8ePHy+w/fvw4QUFB1RZHSUkJVquVwsJCXF1dyc/Pp7jYuStSSOVyc3PDxcXF0WGIiIhIHWSOiMDg7o7pzBEILn0hqLNXV6vRSY7JZCImJoa0tDSSkpKA0oQjLS2NESNGVEsMVquV/fv3U1JSgs1mIygoiMOHD2MwGKrl/uI8fH19qzU5FxEREQEwmExY2rTBvCcbAK8CX6d/T47Dk5zz58+zZ88e+/b+/fvZtm0b/v7+3HjjjYwZM4bBgwfToUMHOnXqxKuvvsqFCxfs1daqks1m49ixY7i4uBAaGmqP18vLC6PR4dW3pZaw2Wzk5ubay54HBAQ4OCIRERGpaywxMbhv/xAAT6sfRSVFDo6oajk8ydm8eTPdunWzb18sCjB48GDmzZvHwIEDOXnyJCkpKWRmZtKuXTtWrlx5STGC8kpNTSU1NfWq086KiorIzc0lJCQEDw8P+7Q1s9msJEfKxWKxAHDixAn8/PwcHI2IiIjUNR4x7THPeQsAtxIT1lxNV6tSCQkJv/syohEjRlT69LTk5GSSk5PJycmhXr16l21zMQEymUyVem+pmy6WKC8qcu6fnIiIiEjNY2nXDiPFuFlzKDT5kH/GuZMcDUdcA62/kcpwsR85+xuGRUREpOZx8fHBPTwcc37puhwlOSIiIiIiUutZYtpjLvglyTnr3DNLlOSIiIiIiNQBHu3b416QBUDBGeeurqYkRy5hMBhYunQpAAcOHMBgMLBt2zaHxiQiIiIiFePRvj3m/DMAFGRpJEfqsNDQUI4dO0br1q1/t21NSYimTZtGXFwcHh4e+Pr6XrbNoUOH6NOnDx4eHjRo0ICxY8deUhBg7dq1tG/fHnd3d5o3b868efOqPngRERGRKuIaEkKJy1kAik7kOziaqlVnk5zU1FQiIiLo2LGjo0Op0VxcXAgKCsLV1eGF+K6Z1Wqlf//+DB8+/LLHi4uL6dOnD1arlXXr1jF//nzmzZtHSkqKvc3+/fvp06cP3bp1Y9u2bYwaNYrHHnuMVatWVddjiIiIiFQqg8FAdmABAIU5Dg6mitXZJCc5OZmMjAw2bdp0zefYbDbyrMXkWouq/VOeilwJCQmMHDmScePG4e/vT1BQEJMnT76O36VLR2eys7MZNGgQgYGBWCwWwsPDmTt3LgBhYWEAREdHYzAYSEhIAEpHRDp16oSnpye+vr506dKFgwcPXlc812LKlCmMHj2aNm3aXPb4Z599RkZGBu+++y7t2rWjd+/ePPPMM6SmpmK1WgGYPXs2YWFhzJgxg1atWjFixAjuvfdeXnnllSqLW0RERKSqnWxUWlXNWuROSYnzVnytPT+erwHyCouJfXm9Q+6dMTURD9O1/3HNnz+fMWPGsGHDBtLT0xkyZAhdunShZ8+eFYpj4sSJZGRksGLFCgICAtizZw95eXkAbNy4kU6dOvH5558TGRmJyWSiqKiIpKQkhg4dynvvvYfVamXjxo1XLcsdGRl51SQoPj6eFStWXPczpKen06ZNmzIvlE1MTGT48OHs2LGD6Oho0tPT6dGjR5nzEhMTGTVq1HXfV0RERMTRjjRzocmmYmxGF3LPFuDlZ3Z0SFVCSY6TioqKYtKkSQCEh4cza9Ys0tLSKpzkHDp0iOjoaDp06ABAkyZN7McCAwMBqF+/PkFBQQBkZWVx9uxZ7rzzTpo1awZAq1atrnqP5cuXU1hYeMXjFoulIo9AZmZmmQQHsG9nZmZetU1OTg55eXkVjkFERETEEfJ93HC3niHfXJ9zWUpyBLC4uZA+5ma8fbwxGqt3pp/FzaVc7aOiospsBwcHc+LEiQrHMXz4cO655x62bt1Kr169SEpKIi4u7ort/f39GTJkCImJifTs2ZMePXowYMAAgoODr3hO48aNKxyniIiIiFzKaDDinp9Nvrk+57PzgXqODqlK1Nk1OdfDYDBgMbngYXKt9s/Vpnddjpub2yWxl5RUvB567969OXjwIKNHj+bo0aN0796dJ5544qrnzJ07l/T0dOLi4li8eDEtWrRg/forT/uLjIzEy8vrip/evXtX6BmCgoI4fvx4mX0Xty+OQF2pjY+Pj0ZxREREpNYyGl3sLwQ9n1Xg4GiqjkZypNwCAwMZPHgwgwcPJj4+nrFjxzJ9+nRMJhNQWr3st6Kjo4mOjmb8+PHExsaycOFCbr755stev6qnq8XGxjJt2jROnDhBgwYNAFi9ejU+Pj5ERETY2yxfvrzMeatXryY2NrZC9xYRERFxJIPBiDm/9IWg57Kdt4x0nU1yUlNTSU1NvewXcrmylJQUYmJiiIyMpKCggE8//dS+xqZBgwZYLBZWrlxJo0aNMJvNZGVlMWfOHPr27UtISAi7du1i9+7dPPzww1e8R0Wnqx06dIisrCwOHTpEcXGxvTJc8+bN8fLyolevXkRERPDQQw/x0ksvkZmZyYQJE0hOTsbd3R2AP/3pT8yaNYtx48bxxz/+kTVr1vD++++zbNmyCsUmIiIi4kguBhfc7SM5zpvk1NnpatdTQlrAZDIxfvx4oqKi6Nq1Ky4uLixatAgAV1dXZs6cyRtvvEFISAj9+vXDw8ODnTt3cs8999CiRQuGDRtGcnIyjz/+eJXFmJKSQnR0NJMmTeL8+fP2UaTNmzcDpe/++fTTT3FxcSE2NpYHH3yQhx9+mKlTp9qvERYWxrJly1i9ejVt27ZlxowZvPnmmyQmJlZZ3CIiIiJVzWAw/m+6Wramq0ktsnbt2kv2LV269JrP//U7eZo0aVJme8KECUyYMOGK5z722GM89thjZfZ9/PHH13zvyjBv3jzmzZt31TaNGze+ZDrabyUkJPDtt99WYmQiIiIijjUsahin858E+KXwgHOqsyM5IiIiIiJ1TWPfJvaRnLxzhRRZnXPphpKcOmbBggVXrFoWGRnp6PBEREREpCoZDLgW5eJSXDpVzVmnrGm6Wh3Tt29fOnfufNljvy07LSIiIiJOxmDAALjnZ5PrGcT57Hx8G3o4OqpKpySnjvH29sbb29vRYYiIiIiIA5kLssj1DOKck74rR9PVRERERETqGHsZaSctPqAkR0RERESkjjAYDACY8537XTl1NslJTU0lIiKCjh07OjoUEREREZHq8UuS4+7k78qps0mOXgYqIiIiInXVxTLS5zSSIyIiIiIitdpvp6tlF5R58buzUJIjlzAYDCxduhSAAwcOYDAY2LZtm0NjEhEREZFK8JvpaoUFxVjzihwZUZVQkiNXFRoayrFjx2jduvXvtq0pCVGTJk0wGAxlPi+88EKZNt9//z3x8fGYzWZCQ0N56aWXLrnOkiVLaNmyJWazmTZt2rB8+fLqegQRERGRqvFLkuNSUojZs/RtMs5YRlpJjlyVi4sLQUFBuLrWrlcqTZ06lWPHjtk/f/nLX+zHcnJy6NWrF40bN2bLli384x//YPLkycyZM8feZt26ddx///08+uijfPvttyQlJZGUlMT27dsd8TgiIiIilcRg/5WXnzvgnGWkleSUh80GhblgvVD9n3LMlUxISGDkyJGMGzcOf39/goKCmDx58nU98m9HZ7Kzsxk0aBCBgYFYLBbCw8OZO3cuAGFhYQBER0djMBhISEgAYO3atXTq1AlPT098fX3p0qULBw8evK54rpW3tzdBQUH2j6enp/3YggULsFqtvP3220RGRnLfffcxcuRIXn75ZXub1157jdtvv52xY8fSqlUrnnnmGdq3b8+sWbOqNG4RERGR6uLl+0uS44TFB2rXj+cdrTAX39RWjrn3U0fB5Pn77X4xf/58xowZw4YNG0hPT2fIkCF06dKFnj17ViiMiRMnkpGRwYoVKwgICGDPnj3k5eUBsHHjRjp16sTnn39OZGQkJpOJoqIikpKSGDp0KO+99x5Wq5WNGzfaa7RfTmRk5FWToPj4eFasWHHVOF944QWeeeYZbrzxRh544AFGjx5tH41KT0+na9eumEwme/vExERefPFFsrOz8fPzIz09nTFjxpS5ZmJion2tkoiIiEht9OuvYF6+pd+FzjlhGWklOU4qKiqKSZMmARAeHs6sWbNIS0urcJJz6NAhoqOj6dChA1C6/uWiwMBAAOrXr09QUBAAWVlZnD17ljvvvJNmzZoB0KrV1RPF5cuXU1hYeMXjFovlquePHDmS9u3b4+/vz7p16xg/fjzHjh2zj9RkZmbaR50uatiwof2Yn58fmZmZ9n2/bpOZmXnVe4uIiIjUaL/Kci4mORrJqevcPDiT/CM+3t4YjdU808/No1zNo6KiymwHBwdz4sSJCocxfPhw7rnnHrZu3UqvXr1ISkoiLi7uiu39/f0ZMmQIiYmJ9OzZkx49ejBgwACCg4OveE7jxo0rFOOvR2CioqIwmUw8/vjjPP/887i7u1fo2iIiIiK12q+THL9fkhwnHMnRmpzyMBhKkw2TZ/V/rjK963Lc3Nx+E7qBkpKSCv8W9O7dm4MHDzJ69GiOHj1K9+7deeKJJ656zty5c0lPTycuLo7FixfTokUL1q9ff8X2kZGReHl5XfHTu3fvcsXcuXNnioqKOHDgAABBQUEcP368TJuL2xdHoK7U5uJxERERkdrOq97FJEcjOU4jNTWV1NRUiouLHR1KrRMYGMjgwYMZPHgw8fHxjB07lunTp9vXuFzu9zQ6Opro6GjGjx9PbGwsCxcu5Oabb77s9Ss6Xe23tm3bhtFopEGDBgDExsby9NNPU1hYaE8GV69ezU033YSfn5+9TVpaGqNGjbJfZ/Xq1cTGxpbr3iIiIiI1yuWmq2UXYCuxYTCW74fqNVmdTXKSk5NJTk4mJyeHevXqOTqcWiMlJYWYmBgiIyMpKCjg008/ta+xadCgARaLhZUrV9KoUSPMZjNZWVnMmTOHvn37EhISwq5du9i9ezcPP/zwFe9Rkelq6enpbNiwgW7duuHt7U16ejqjR4/mwQcftCcwDzzwAFOmTOHRRx/lySefZPv27bz22mu88sor9uv89a9/5dZbb2XGjBn06dOHRYsWsXnz5jJlpkVERERqnV8lOR7ebhgMUFJsI/ecFc96zjOtX9PVpFxMJhPjx48nKiqKrl274uLiwqJFiwBwdXVl5syZvPHGG4SEhNCvXz88PDzYuXMn99xzDy1atGDYsGEkJyfz+OOPV0l87u7uLFq0iFtvvZXIyEimTZvG6NGjyyQn9erV47PPPmP//v3ExMTwt7/9jZSUFIYNG2ZvExcXx8KFC5kzZw5t27blgw8+YOnSpdf0UlQRERGRGutXSY7RCJ6/lJE+52TFB+rsSI4zW7t27SX7ylP62Pard/I0adKkzPaECROYMGHCFc997LHHeOyxx8rs+/jjj6/53hXVvn37q673uSgqKoqvvvrqqm369+9P//79Kys0ERERkRrHy8/M+ewCzmcVQNjvt68tNJIjIiIiIlJHlFl1Y7Ph5f/LC0GdrPiAkpw6ZsGCBVesWhYZGeno8ERERESkKv26Yq/NhrefGaB0JMeJaLpaHdO3b186d+582WO/LTstIiIiIk7mN0mOs47kKMmpY7y9vfH29nZ0GCIiIiLiCL9596LXLyM5zlZ4QNPVRERERETqIJvNhrf/L9PVsp1rupqSHBERERGRuuK3Izm/TFfLzbFSXFjiiIiqhJIcEREREZG64jdrcsyebri4laYE5884z2iOkhwRERERkTrC8JuRHIPBgJef8xUfUJIjIiIiIlIX/fLCd/u6HCcqPlBnk5zU1FQiIiLo2LGjo0OpcQwGA0uXLgXgwIEDGAwGtm3b5tCYRERERKSS/ZLkXBzJOedExQfqbJKTnJxMRkYGmzZtcnQoNVpoaCjHjh2jdevWv9u2piRE06ZNIy4uDg8PD3x9fS/b5tChQ/Tp0wcPDw8aNGjA2LFjKSoqKtNm7dq1tG/fHnd3d5o3b868efMuuU5qaipNmjTBbDbTuXNnNm7cWAVPJCIiIlKJLk5Zu5jkaCRH6hoXFxeCgoJwda09r1SyWq3079+f4cOHX/Z4cXExffr0wWq1sm7dOubPn8+8efNISUmxt9m/fz99+vShW7dubNu2jVGjRvHYY4+xatUqe5vFixczZswYJk2axNatW2nbti2JiYmcOHGiyp9RRERE5HoZXF3hV9/tvP2cr4y0kpxysNls5BXlkVuYW+0f2y+Z9rVISEhg5MiRjBs3Dn9/f4KCgpg8efJ1PfNvR2eys7MZNGgQgYGBWCwWwsPDmTt3LgBhYWEAREdHYzAYSEhIAEpHRDp16oSnpye+vr506dKFgwcPXlc812LKlCmMHj2aNm3aXPb4Z599RkZGBu+++y7t2rWjd+/ePPPMM6SmpmK1WgGYPXs2YWFhzJgxg1atWjFixAjuvfdeXnnlFft1Xn75ZYYOHcojjzxCREQEs2fPxsPDg7fffrvKnk1ERESkolr+8D2ttv+Aa2Ag8L8y0s70QtDa8+P5GiCvKI9ey3o55N4bHtiAh5vHNbefP38+Y8aMYcOGDaSnpzNkyBC6dOlCz549KxTHxIkTycjIYMWKFQQEBLBnzx7y8vIA2LhxI506deLzzz8nMjISk8lEUVERSUlJDB06lPfeew+r1crGjRsvqezxa5GRkVdNguLj41mxYsV1P0N6ejpt2rShYcOG9n2JiYkMHz6cHTt2EB0dTXp6Oj169ChzXmJiIqNGjQJKR4u2bNnC+PHj7ceNRiM9evQgPT39umMTERERqW5eTjiSoyTHSUVFRTFp0iQAwsPDmTVrFmlpaRVOcg4dOkR0dDQdOnQAoEmTJvZjgb/8NKB+/foEBQUBkJWVxdmzZ7nzzjtp1qwZAK1atbrqPZYvX05hYeEVj1ssloo8ApmZmWUSHMC+nZmZedU2OTk55OXlkZ2dTXFx8WXb7Ny5s0LxiYiIiFSni4UHrHlFWPOKMFlqf4pQ+5+gGllcLXzW5zO8vb0xGqt3pp/FtXxf7KOiospsBwcHV8pakeHDh3PPPfewdetWevXqRVJSEnFxcVds7+/vz5AhQ0hMTKRnz5706NGDAQMGEBwcfMVzGjduXOE4RUREROTamMyuuHu4UpBbxLnsfOpbvBwdUoVpTU45GAwGLK4WPNw8qv1zteldl+Pm5nZJ7CUlJRX+PejduzcHDx5k9OjRHD16lO7du/PEE09c9Zy5c+eSnp5OXFwcixcvpkWLFqxfv/6K7SMjI/Hy8rrip3fv3hV6hqCgII4fP15m38XtiyNQV2rj4+ODxWIhICAAFxeXy7a5eA0RERGR2sLZpqxpJEfKLTAwkMGDBzN48GDi4+MZO3Ys06dPx2QyAaXVy34rOjqa6Ohoxo8fT2xsLAsXLuTmm2++7PWrerpabGws06ZN48SJEzRo0ACA1atX4+PjQ0REhL3N8uXLy5y3evVqYmNjATCZTMTExJCWlkZSUhIAJSUlpKWlMWLEiArFJyIiIlLdvP3dOX3kvNOUkVaSI+WSkpJCTEwMkZGRFBQU8Omnn9rX2DRo0ACLxcLKlStp1KgRZrOZrKws5syZQ9++fQkJCWHXrl3s3r2bhx9++Ir3qOh0tUOHDpGVlcWhQ4coLi62V4Zr3rw5Xl5e9OrVi4iICB566CFeeuklMjMzmTBhAsnJybi7l85J/dOf/sSsWbMYN24cf/zjH1mzZg3vv/8+y5Yts99nzJgxDB48mA4dOtCpUydeffVVLly4wCOPPFKh+EVERESqm0ZypE4zmUyMHz+eAwcOYLFYiI+PZ9GiRQC4uroyc+ZMpk6dSkpKCvHx8SxevJidO3cyf/58Tp8+TXBwMMnJyTz++ONVFmNKSgrz58+3b0dHRwPwxRdfkJCQgIuLC59++inDhw8nNjYWT09PBg8ezNSpU+3nhIWFsWzZMkaPHs1rr71Go0aNePPNN0lMTLS3GThwICdPniQlJYXMzEzatWvHypUrLylGICIiIlLTOVsZaYOtPC9gcUI5OTnUq1ePs2fP4uPjU+ZYfn4++/fvJywsDLPZTElJCTk5Ofj4+FR74QGp/S72p0aNGrFmzRruuOOOS9ZOiVyLwsJCli9frj4kFaJ+JJVB/ch5ZHx9lC/e3UmTqAD6/Dnq90+oRL/tR1f7fn6t9E1dRERERKSuu1jjyknGP5Tk1DELFiy4YtWyyMhIR4cnIiIiIlJhWpNTx/Tt25fOnTtf9piGmUVERETqNucYx1GSU+d4e3vj7e3t6DBEREREpAaxv5LRSbIcTVcTEREREanzyvfi+ZquziY5qampRERE0LFjR0eHIiIiIiJSIzhJ3YG6m+QkJyeTkZHBpk2bHB2KiIiIiIhD2aerOcl8tTqb5IiIiIiIiHNSkiMiIiIiUtep8IA4O4PBwNKlSwE4cOAABoOBbdu2OTQmEREREak6TpbjKMmRqwsNDeXYsWO0bt36d9vWhITowIEDPProo4SFhWGxWGjWrBmTJk3CarWWaff9998THx+P2WwmNDSUl1566ZJrLVmyhJYtW2I2m2nTpg3Lly8vc9xms5GSkkJwcDAWi4UePXqwe/fuKn0+EREREfl9SnLkqlxcXAgKCsLVtXa8Umnnzp2UlJTwxhtvsGPHDl555RVmz57NU089ZW+Tk5NDr169aNy4MVu2bOEf//gHkydPZs6cOfY269at4/777+fRRx/l22+/JSkpiaSkJLZv325v89JLLzFz5kxmz57Nhg0b8PT0JDExkfz8/Gp9ZhEREZEKu1h5wEnKqynJKQebzUZJXh4lubnV/rGVo8MlJCQwcuRIxo0bh7+/P0FBQUyePPm6nvm3ozPZ2dkMGjSIwMBALBYL4eHhzJ07F4CwsDAAoqOjMRgMJCQkALB27Vo6deqEp6cnvr6+dOnShYMHD15XPL/n9ttvZ+7cufTq1YumTZvSt29fnnjiCT766CN7mwULFmC1Wnn77beJjIzkvvvuY+TIkbz88sv2Nq+99hq33347Y8eOpVWrVjzzzDO0b9+eWbNmAaV94dVXX2XChAn069ePqKgo3nnnHY4ePWqf6iciIiJS2zhJjkPt+PF8DWHLy+N4t9s47oB737R1CwYPj2tuP3/+fMaMGcOGDRtIT09nyJAhdOnShZ49e1YojokTJ5KRkcGKFSsICAhgz5495OXlAbBx40Y6derE559/TmRkJCaTiaKiIpKSkhg6dCjvvfceVquVjRs3YjBc+YVTkZGRV02C4uPjWbFixTXHfPbsWfz9/e3b6enpdO3aFZPJZN+XmJjIiy++SHZ2Nn5+fqSnpzNmzJgy10lMTLQnMPv37yczM5MePXrYj9erV4/OnTuTnp7Offfdd83xiYiIiEjlUpLjpKKiopg0aRIA4eHhzJo1i7S0tAonOYcOHSI6OpoOHToA0KRJE/uxwMBAAOrXr09QUBAAWVlZnD17ljvvvJNmzZoB0KpVq6veY/ny5RQWFl7xuMViueZ49+zZwz//+U+mT59u35eZmWkfdbqoYcOG9mN+fn5kZmba9/26TWZmpr3dr8+7XBsRERGR2uIqP3+ulZTklIPBYqHhF2vw8fbGaKzemX6Gcnyxh9Ik59eCg4M5ceJEheMYPnw499xzD1u3bqVXr14kJSURFxd3xfb+/v4MGTKExMREevbsSY8ePRgwYADBwcFXPKdx48YVjhPgyJEj3H777fTv35+hQ4dWyjVFREREnJJzLcnRmpzyMBgMGC0WjB4e1f652vSuy3Fzc7sk9pKSkgr/HvTu3ZuDBw8yevRojh49Svfu3XniiSeues7cuXNJT08nLi6OxYsX06JFC9avX3/F9pGRkXh5eV3x07t379+N8+jRo3Tr1o24uLgyBQUAgoKCOH687KTDi9sXR6Cu1ObXx3993uXaiIiIiIhjaCRHyi0wMJDBgwczePBg4uPjGTt2LNOnT7evcSkuLr7knOjoaKKjoxk/fjyxsbEsXLiQm2+++bLXr+h0tSNHjtCtWzdiYmKYO3fuJaNusbGxPP300xQWFtqTwdWrV3PTTTfh5+dnb5OWlsaoUaPs561evZrY2FigtMhCUFAQaWlptGvXDiit2rZhwwaGDx9+1fhEREREahqDk70pR0mOlEtKSgoxMTFERkZSUFDAp59+al9j06BBAywWCytXrqRRo0aYzWaysrKYM2cOffv2JSQkhF27drF7924efvjhK96jItPVjhw5QkJCAo0bN2b69OmcPHnSfuziCMsDDzzAlClTePTRR3nyySfZvn07r732Gq+88oq97V//+lduvfVWZsyYQZ8+fVi0aBGbN2+2jwoZDAZGjRrFs88+S3h4OGFhYUycOJGQkBCSkpKuO34RERERh3KOHEdJjpSPyWRi/PjxHDhwAIvFQnx8PIsWLQLA1dWVmTNnMnXqVFJSUoiPj2fx4sXs3LmT+fPnc/r0aYKDg0lOTubxxx+vkvhWr17Nnj172LNnD40aNSpz7GIZ7nr16vHZZ5+RnJxMTEwMAQEBpKSkMGzYMHvbuLg4Fi5cyIQJE3jqqacIDw9n6dKlZV6KOm7cOC5cuMCwYcM4c+YMt9xyCytXrsRsNlfJs4mIiIhUGScrPGCwlecFLE4oJyeHevXqcfbsWXx8fMocy8/PZ//+/YSFhWE2mykpKSEnJwcfH59qLzwgtd/F/tSoUSPWrFnDHXfcccnaKZFrUVhYyPLly9WHpELUj6QyqB85j92bj/PZmzsICffl7r+1r9Z7/7YfXe37+bXSN3UREREREXEqSnLqmAULFlyxallkZKSjwxMRERERByhvJd+aTmty6pi+ffvSuXPnyx7TMLOIiIhI3eYsK1mU5NQx3t7eeHt7OzoMEREREZEqo+lqIiIiIiJ1nJPNVlOSIyIiIiJS5znXu0CV5IiIiIiISCknWZJTd5Oc1NRUIiIi6Nixo6NDERERERFxKIOTvQ20ziY5ycnJZGRksGnTJkeHIiIiIiLiWPYcxzmGcupskiNXZjAYWLp0KQAHDhzAYDCwbds2h8YkIiIiIlVP09WkTggNDeXYsWO0bt36d9vWlISoSZMmGAyGMp8XXnihTJvvv/+e+Ph4zGYzoaGhvPTSS5dcZ8mSJbRs2RKz2UybNm1Yvnx5meM2m42UlBSCg4OxWCz06NGD3bt3V+mziYiIiMjvU5IjV+Xi4kJQUBCurrXrlUpTp07l2LFj9s9f/vIX+7GcnBx69epF48aN2bJlC//4xz+YPHkyc+bMsbdZt24d999/P48++ijffvstSUlJJCUlsX37dnubl156iZkzZzJ79mw2bNiAp6cniYmJ5OfnV+uzioiIiEhZSnLKwWazUWQtprCg+j/leftsQkICI0eOZNy4cfj7+xMUFMTkyZOv65l/OzqTnZ3NoEGDCAwMxGKxEB4ezty5cwEICwsDIDo6GoPBQEJCAgBr166lU6dOeHp64uvrS5cuXTh48OB1xXOtvL29CQoKsn88PT3txxYsWIDVauXtt98mMjKS++67j5EjR/Lyyy/b27z22mvcfvvtjB07llatWvHMM8/Qvn17Zs2aBZT2hVdffZUJEybQr18/oqKieOeddzh69Kh9qp+IiIhIbXHxPTnOMl2tdv143sGKrCUsTvnOIfce9tqtuLm7XHP7+fPnM2bMGDZs2EB6ejpDhgyhS5cu9OzZs0JxTJw4kYyMDFasWEFAQAB79uwhLy8PgI0bN9KpUyc+//xzIiMjMZlMFBUVkZSUxNChQ3nvvfewWq1s3LgRw1XeOBUZGXnVJCg+Pp4VK1ZcNc4XXniBZ555hhtvvJEHHniA0aNH20ej0tPT6dq1KyaTyd4+MTGRF198kezsbPz8/EhPT2fMmDFlrpmYmGhPYPbv309mZiY9evSwH69Xrx6dO3cmPT2d++6776rxiYiIiNQoTvY2UCU5TioqKopJkyYBEB4ezqxZs0hLS6twknPo0CGio6Pp0KEDULr+5aLAwEAA6tevT1BQEABZWVmcPXuWO++8k2bNmgHQqlWrq95j+fLlFBYWXvG4xWK56vkjR46kffv2+Pv7s27dOsaPH8+xY8fsIzWZmZn2UaeLGjZsaD/m5+dHZmamfd+v22RmZtrb/fq8y7URERERqXWcZChHSU45uJqMDJzaFm9vH4zG6p3p52oq3/2ioqLKbAcHB3PixIkKxzF8+HDuuecetm7dSq9evUhKSiIuLu6K7f39/RkyZAiJiYn07NmTHj16MGDAAIKDg694TuPGjSsU469HYKKiojCZTDz++OM8//zzuLu7V+jaIiIiIs7IucZxtCanXAwGA64mF9zcq/9zteldl+Pm5nZJ7CUlJRX+PejduzcHDx5k9OjRHD16lO7du/PEE09c9Zy5c+eSnp5OXFwcixcvpkWLFqxfv/6K7SMjI/Hy8rrip3fv3uWKuXPnzhQVFXHgwAEAgoKCOH78eJk2F7cvjkBdqc2vj//6vMu1EREREak1nCzL0UiOlFtgYCCDBw9m8ODBxMfHM3bsWKZPn25f41JcXHzJOdHR0URHRzN+/HhiY2NZuHAhN99882WvX9Hpar+1bds2jEYjDRo0ACA2Npann36awsJCezK4evVqbrrpJvz8/Oxt0tLSGDVqlP06q1evJjY2FigtshAUFERaWhrt2rUDSqu2bdiwgeHDh5crPhEREZGawklmqynJkfJJSUkhJiaGyMhICgoK+PTTT+1rbBo0aIDFYmHlypU0atQIs9lMVlYWc+bMoW/fvoSEhLBr1y52797Nww8/fMV7VGS6Wnp6Ohs2bKBbt254e3uTnp7O6NGjefDBB+0JzAMPPMCUKVN49NFHefLJJ9m+fTuvvfYar7zyiv06f/3rX7n11luZMWMGffr0YdGiRWzevNleZtpgMDBq1CieffZZwsPDCQsLY+LEiYSEhJCUlHTd8YuIiIg4gtFowNXNiIurc0z0UpIj5WIymRg/fjwHDhzAYrEQHx/PokWLAHB1dWXmzJlMnTqVlJQU4uPjWbx4MTt37mT+/PmcPn2a4OBgkpOTefzxx6skPnd3dxYtWsTkyZMpKCggLCyM0aNHl1mnU69ePT777DOSk5OJiYkhICCAlJQUhg0bZm8TFxfHwoULmTBhAk899RTh4eEsXbq0zEtRx40bx4ULFxg2bBhnzpzhlltuYeXKlZjN5ip5NhEREZGqcmNkfR7/Z4Kjw6g0Blt5XsDihHJycqhXrx5nz57Fx8enzLH8/Hz2799PWFgYZrOZkpIScnJy8PGp/sIDUvtd7E+NGjVizZo13HHHHZesnRK5FoWFhSxfvlx9SCpE/Ugqg/qRVIbf9qOrfT+/VvqmLiIiIiIiTkVJTh2zYMGCK1Yti4yMdHR4IiIiIiIVpjU5dUzfvn3p3LnzZY9pmFlEREREnIGSnDrG29sbb29vR4chIiIiIlJlNF3tGtTx2gxSSS72o/K+2FVEREREykdJzlW4uLgAYLVaHRyJOIPc3FygtNS2iIiIiFQdfdu6CldXVzw8PDh58qR9vYrVaiU/P18lpOWa2Ww2cnNzOXHiBL6+vvbkWURERESqhpKcqzAYDAQHB7N//34OHjyIzWYjLy8Pi8WiKUdSbr6+vgQFBVFUVOToUEREREScmpKc32EymQgPD8dqtVJYWMiXX35J165dVYlMysXNzU0jOCIiIiLVREnONTAajZjNZlxcXCgqKsJsNivJERERERGpobSwREREREREnIqSHBERERERcSpKckRERERExKnU+TU5F1/QmJOT87ttCwsLyc3NJScnR2ty5LqpH0lFqQ9JZVA/ksqgfiSV4bf96OL38ovf069HnU9yzp07B0BoaKiDIxERERERkYvOnTtHvXr1rutcg60iKZITKCkp4ejRo3h7e//uu29ycnIIDQ3l8OHD+Pj4VFOE4mzUj6Si1IekMqgfSWVQP5LK8Nt+ZLPZOHfuHCEhIRiN17e6ps6P5BiNRho1alSuc3x8fPQ/slSY+pFUlPqQVAb1I6kM6kdSGX7dj653BOciFR4QERERERGnoiRHREREREScipKccnB3d2fSpEm4u7s7OhSpxdSPpKLUh6QyqB9JZVA/kspQFf2ozhceEBERERER56KRHBERERERcSpKckRERERExKkoyREREREREaeiJEdERERERJyKkpzfSE1NpUmTJpjNZjp37szGjRuv6bxFixZhMBhISkqq2gClVihPP5o3bx4Gg6HMx2w2V2O0UhOV9++iM2fOkJycTHBwMO7u7rRo0YLly5dXU7RSU5WnHyUkJFzyd5HBYKBPnz7VGLHUROX9++jVV1/lpptuwmKxEBoayujRo8nPz6+maKWmKk8/KiwsZOrUqTRr1gyz2Uzbtm1ZuXJl+W5oE7tFixbZTCaT7e2337bt2LHDNnToUJuvr6/t+PHjVz1v//79thtuuMEWHx9v69evX/UEKzVWefvR3LlzbT4+PrZjx47ZP5mZmdUctdQk5e1DBQUFtg4dOtjuuOMO29dff23bv3+/be3atbZt27ZVc+RSk5S3H50+fbrM30Pbt2+3ubi42ObOnVu9gUuNUt5+tGDBApu7u7ttwYIFtv3799tWrVplCw4Oto0ePbqaI5eapLz9aNy4cbaQkBDbsmXLbHv37rW9/vrrNrPZbNu6des131NJzq906tTJlpycbN8uLi62hYSE2J5//vkrnlNUVGSLi4uzvfnmm7bBgwcryZFy96O5c+fa6tWrV03RSW1Q3j70r3/9y9a0aVOb1WqtrhClFrief9N+7ZVXXrF5e3vbzp8/X1UhSi1Q3n6UnJxsu+2228rsGzNmjK1Lly5VGqfUbOXtR8HBwbZZs2aV2feHP/zBNmjQoGu+p6ar/cJqtbJlyxZ69Ohh32c0GunRowfp6elXPG/q1Kk0aNCARx99tDrClBruevvR+fPnady4MaGhofTr148dO3ZUR7hSA11PH/rPf/5DbGwsycnJNGzYkNatW/Pcc89RXFxcXWFLDXO9fxf92ltvvcV9992Hp6dnVYUpNdz19KO4uDi2bNlin4q0b98+li9fzh133FEtMUvNcz39qKCg4JKp+xaLha+//vqa76sk5xenTp2iuLiYhg0bltnfsGFDMjMzL3vO119/zVtvvcW///3v6ghRaoHr6Uc33XQTb7/9Np988gnvvvsuJSUlxMXF8fPPP1dHyFLDXE8f2rdvHx988AHFxcUsX76ciRMnMmPGDJ599tnqCFlqoOvpR7+2ceNGtm/fzmOPPVZVIUotcD396IEHHmDq1KnccsstuLm50axZMxISEnjqqaeqI2Spga6nHyUmJvLyyy+ze/duSkpKWL16NR999BHHjh275vsqyblO586d46GHHuLf//43AQEBjg5HarHY2Fgefvhh2rVrx6233spHH31EYGAgb7zxhqNDk1qipKSEBg0aMGfOHGJiYhg4cCBPP/00s2fPdnRoUku99dZbtGnThk6dOjk6FKll1q5dy3PPPcfrr7/O1q1b+eijj1i2bBnPPPOMo0OTWuS1114jPDycli1bYjKZGDFiBI888ghG47WnLq5VGF+tEhAQgIuLC8ePHy+z//jx4wQFBV3Sfu/evRw4cIC77rrLvq+kpAQAV1dXdu3aRbNmzao2aKlxytuPLsfNzY3o6Gj27NlTFSFKDXc9fSg4OBg3NzdcXFzs+1q1akVmZiZWqxWTyVSlMUvNU5G/iy5cuMCiRYuYOnVqVYYotcD19KOJEyfy0EMP2UcB27Rpw4ULFxg2bBhPP/10ub6kinO4nn4UGBjI0qVLyc/P5/Tp04SEhPD3v/+dpk2bXvN91dN+YTKZiImJIS0tzb6vpKSEtLQ0YmNjL2nfsmVLfvjhB7Zt22b/9O3bl27durFt2zZCQ0OrM3ypIcrbjy6nuLiYH374geDg4KoKU2qw6+lDXbp0Yc+ePfYftAD89NNPBAcHK8Gpoyryd9GSJUsoKCjgwQcfrOowpYa7nn6Um5t7SSJz8QcwNput6oKVGqsifx+ZzWZuuOEGioqK+PDDD+nXr9+13/i6SiQ4qUWLFtnc3d1t8+bNs2VkZNiGDRtm8/X1tZfzfeihh2x///vfr3i+qquJzVb+fjRlyhTbqlWrbHv37rVt2bLFdt9999nMZrNtx44djnoEcbDy9qFDhw7ZvL29bSNGjLDt2rXL9umnn9oaNGhge/bZZx31CFIDXO+/abfccott4MCB1R2u1FDl7UeTJk2yeXt729577z3bvn37bJ999pmtWbNmtgEDBjjqEaQGKG8/Wr9+ve3DDz+07d271/bll1/abrvtNltYWJgtOzv7mu+p6Wq/MnDgQE6ePElKSgqZmZm0a9eOlStX2hdKHTp0SMOs8rvK24+ys7MZOnQomZmZ+Pn5ERMTw7p164iIiHDUI4iDlbcPhYaGsmrVKkaPHk1UVBQ33HADf/3rX3nyyScd9QhSA1zPv2m7du3i66+/5rPPPnNEyFIDlbcfTZgwAYPBwIQJEzhy5AiBgYHcddddTJs2zVGPIDVAeftRfn4+EyZMYN++fXh5eXHHHXfwf//3f/j6+l7zPQ02m8YORURERETEeWhYQkREREREnIqSHBERERERcSpKckRERERExKkoyREREREREaeiJEdERERERJyKkhwREREREXEqSnJERERERMSpKMkRERERERGnoiRHRETqpMmTJ9OuXTv79pAhQ0hKSnJYPCIiUnmU5IiIiIiIiFNRkiMiIjWO1Wp1dAgiIlKLKckRERGHS0hIYMSIEYwaNYqAgAASExPZvn07vXv3xsvLi4YNG/LQQw9x6tQp+zklJSW89NJLNG/eHHd3d2688UamTZtmP/7kk0/SokULPDw8aNq0KRMnTqSwsNARjyciItVMSY6IiNQI8+fPx2Qy8c033/DCCy9w2223ER0dzebNm1m5ciXHjx9nwIAB9vbjx4/nhRdeYOLEiWRkZLBw4UIaNmxoP+7t7c28efPIyMjgtdde49///jevvPKKIx5NRESqmcFms9kcHYSIiNRtCQkJ5OTksHXrVgCeffZZvvrqK1atWmVv8/PPPxMaGsquXbsIDg4mMDCQWbNm8dhjj13TPaZPn86iRYvYvHkzUFp4YOnSpWzbtg0oLTxw5swZli5dWqnPJiIi1c/V0QGIiIgAxMTE2H/93Xff8cUXX+Dl5XVJu71793LmzBkKCgro3r37Fa+3ePFiZs6cyd69ezl//jxFRUX4+PhUSewiIlKzKMkREZEawdPT0/7r8+fPc9ddd/Hiiy9e0i44OJh9+/Zd9Vrp6ekMGjSIKVOmkJiYSL169Vi0aBEzZsyo9LhFRKTmUZIjIiI1Tvv27fnwww9p0qQJrq6X/lMVHh6OxWIhLS3tstPV1q1bR+PGjXn66aft+w4ePFilMYuISM2hwgMiIlLjJCcnk5WVxf3338+mTZvYu3cvq1at4pFHHqG4uBiz2cyTTz7JuHHjeOedd9i7dy/r16/nrbfeAkqToEOHDrFo0SL27t3LzJkz+fjjjx38VCIiUl2U5IiISI0TEhLCN998Q3FxMb169aJNmzaMGjUKX19fjMbSf7omTpzI3/72N1JSUmjVqhUDBw7kxIkTAPTt25fRo0czYsQI2rVrx7p165g4caIjH0lERKqRqquJiIiIiIhT0UiOiIiIiIg4FSU5IiIiIiLiVJTkiIiIiIiIU1GSIyIiIiIiTkVJjoiIiIiIOBUlOSIiIiIi4lSU5IiIiIiIiFNRkiMiIiIiIk5FSY6IiIiIiDgVJTkiIiIiIuJUlOSIiIiIiIhT+f/yvCq/QSYDNQAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "fig, ax = plt.subplots(1, 1, figsize=plt.figaspect(1/2))\n", - "fig.suptitle(\n", - " f'Effects of n_list on QPS/recall trade-off ({DATASET_FILENAME})\\n' + \\\n", - " f'k = {k}, pq_dim = {pq_dim}, search = {search_label}')\n", - "labels = []\n", - "for i, n_lists in enumerate(n_list_variants):\n", - " ax.plot(bench_recall_nl[i, :], bench_qps_nl[i, :])\n", - " labels.append(f\"n_lists = {n_lists}\")\n", - "\n", - "ax.legend(labels)\n", - "ax.set_xlabel('recall')\n", - "ax.set_ylabel('QPS')\n", - "ax.set_yscale('log')\n", - "ax.grid()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This chart demonstrates that for the given data set (SIFT-128) and the selected parameters, the QPS/recall curves are rather close to each other.\n", - "Yet, two lines, which correspond to 100- and 5000-cluster indices, lag below the others.\n", - "This suggests that 5000 clusters is probably too many and 100 clusters is probably too few for this dataset. In the range of 500-2000 the algorithm performs very similar though.\n", - "Hence, you shouldn't worry about finding the exact single best value of `n_lists`, but rather make sure it's within a reasonable range.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### kmeans_trainset_fraction\n", - "\n", - "This parameter defines how much of the original data should be fed into training.\n", - "This is useful when in conjunction with `add_data_on_build = True`.\n", - "For example, having a 100M-record dataset, it's reasonable to set `kmeans_trainset_fraction = 0.1` to train the index (i.e. run the k-means clustering) using 10M records only (10% of data), and then add the whole dataset to the index.\n", - "Hence, this parameter directly affects the training speed, but can indirectly affect the search performance (depending on how well the training set represents the full dataset).\n", - "\n", - "Note, if `add_data_on_build = False`, setting the trainset fraction less than one is identical to passing a smaller dataset to the `ivf_pq.build`." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### kmeans_n_iters\n", - "\n", - "This parameter is passed directly to the k-means algorithm during training. It's set to a reasonable default of 20, which works for most datasets. However, once in a while you may see a warning complaining that the trained clusters are imbalanced. You can try to fix that by increasing the number of iterations." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Indexing parameters affecting the fine search / product quantization\n", - "\n", - "In the IVF-PQ index, a database vector y is approximated with two level quantization:\n", - "\n", - "$ y = Q_1(y) + Q_2(y - Q_1(y)) $\n", - "\n", - "The first level quantizer ($Q_1$), maps the vector y to the nearest cluster center. The number of\n", - "clusters is `n_lists`.\n", - "\n", - "The second quantizer encodes the residual, and it is defined as a product quantizer\n", - "(see [_\"Product quantization for nearest neighbor search\" by Herve Jegou, Matthijs Douze, Cordelia Schmid_](https://www.researchgate.net/publication/47815472_Product_Quantization_for_Nearest_Neighbor_Search)).\n", - "\n", - "A product quantizer encodes a `dim` dimensional vector with a `pq_dim` dimensional vector.\n", - "First we split the input vector into `pq_dim` subvectors (denoted by u), where each u vector\n", - "contains `pq_len` distinct components of y\n", - "```\n", - "y_1, y_2, ... y_{pq_len}, y_{pq_len+1}, ... y_{2*pq_len}, ... y_{dim-pq_len+1} ... y_{dim}\n", - " \\___________________/ \\____________________________/ \\______________________/\n", - " u_1 u_2 u_{pq_dim}\n", - "```\n", - "Then each subvector encoded with a separate quantizer $q_i$, end the results are concatenated\n", - "\n", - "$ Q_2(y) = q_1(u_1),q_2(u_2),...,q_\\mathtt{pq\\_dim}(u_\\mathtt{pq\\_dim}) $\n", - "\n", - "Each quantizer $q_i$ outputs a code with `pq_bit` bits. The second level quantizers are also defined\n", - "by k-means clustering in the corresponding sub-space: the reproduction values are the centroids,\n", - "and the set of reproduction values is the codebook.\n", - "\n", - "During the search, for every query and probed list, a look-up table (LUT) is constructed using appropriate codebooks and the query coordinates.\n", - "The size of the LUT has profound effect on the performance; here it is one more time:\n", - "\n", - "$ \\mathtt{lut\\_size} = \\mathtt{pq\\_dim} \\cdot \\mathtt{sizeof(lut\\_dtype) \\cdot 2^{\\mathtt{pq\\_bits}}} $\n", - "\n", - "If possible, the LUT is stored fully in GPU L1 (shared) memory during search;\n", - "otherwise, a slower version of the kernel is used, which stores the LUT in the global memory.\n", - "\n", - "\n", - "#### codebook_kind\n", - "\n", - "The second-level quantizers are trained either for each subspace or for each cluster, controlled by parameter `codebook_kind`:\n", - "\n", - " 1. \"subspace\" (C++ api: `codebook_gen::PER_SUBSPACE`): \\\n", - " creates `pq_dim` second-level quantizers - one for each slice of the data along features;\n", - " 2. \"cluster\" (C++ api: `codebook_gen::PER_CLUSTER`): \\\n", - " creates `n_lists` second-level quantizers - one for each first-level cluster.\n", - "\n", - "In either case, the centroids are found using k-means clustering interpreting the data as having `pq_len` dimensions.\n", - "\n", - "There's no definitive way to tell in advance, which of the two options yields better performance for a particular use case.\n", - "A few observations, however, may help:\n", - "\n", - " - A per-cluster codebook tends to take more time to train, since `n_lists` is usually much higher than `pq_dim` - more codebooks to train.\n", - " - Search with a per-cluster codebook usually utilizes L1 cache of the GPU better than with a per-subspace codebook; this may result in a faster search when the LUT is big and occupies a large part of the GPU L1 memory.\n", - " - However, in practice, the recall is slightly higher with a per-subspace codebook.\n", - "\n", - "\n", - "#### pq_dim, pq_bits\n", - "\n", - "`pq_dim` parameter is the main way to control the compression in the database.\n", - "You should choose it depending on your expectations about the sparsity of the information in the data.\n", - "As an experiment, you could start with `pq_dim` in the range of the data dimensionality `[dim / 2, dim]`.\n", - "\n", - "`pq_bits` is the number of bits in a single PQ code.\n", - "Hence, it controls the codebook size - $2^{\\mathtt{pq\\_bits}}$ - the number of possible values a code can take.\n", - "IVF-PQ supports the codebooks sizes from 16 to 256, or the `pq_bits` in the range of `[4, 8]`.\n", - "\n", - "`pq_bits` affects the compression: a database with `pq_bits = 4` is twice smaller than with the `pq_bits = 8`.\n", - "Though much stronger `pq_bits` affects the LUT size, as the LUT size is proportional to $2^{\\mathtt{pq\\_bits}}$ (see the formula above).\n", - "This also means a drastic effect on the recall.\n", - "\n", - "A few observations:\n", - "\n", - " - It's required that `(pq_dim * pq_bits) % 8 == 0`; in general, keeping `pq_dim` in powers of two improves the search performance due to better data alignment.\n", - " - Keeping `pq_dim * pq_bits >= 128` and `(pq_dim * pq_bits) % 32 == 0` maximizes the GPU memory bandwidth utilization.\n", - " - Generally `pq_bits = 8` is a good starting point.\n", - " - The recall loss due to smaller `pq_bits` can be compensated by enabling refinement.\n", - " - For high-dimensional data and large `pq_dims`, lowering `pq_bits` can yield a drastic search speedup due to enabling the faster kernel that keeps the LUT in L1.\n", - " - Alternatively, setting the search parameter `lut_dtype` to `uint8` may be enough to keep the LUT in L1.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "8.25 ms ± 10.5 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "15.5 ms ± 24.8 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "36.7 ms ± 468 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "71.8 ms ± 222 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "9.4 ms ± 16.2 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "16.2 ms ± 32.2 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "38.2 ms ± 520 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "74.4 ms ± 291 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "160 ms ± 48.4 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "168 ms ± 393 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "191 ms ± 139 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "228 ms ± 590 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "12.2 ms ± 24.5 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "25.2 ms ± 73.1 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "59.8 ms ± 167 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "117 ms ± 84.6 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "14.3 ms ± 19.2 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "25.2 ms ± 2.93 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "59.6 ms ± 29.3 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "116 ms ± 17.6 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "165 ms ± 757 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "176 ms ± 168 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "212 ms ± 245 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "270 ms ± 283 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "6.47 ms ± 20.2 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "11 ms ± 13.1 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "24.5 ms ± 285 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "46.2 ms ± 460 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "8.25 ms ± 19.3 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "13.2 ms ± 3.08 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "28.7 ms ± 3.21 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "53.4 ms ± 6.59 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "158 ms ± 135 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "164 ms ± 137 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "180 ms ± 114 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "206 ms ± 322 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "6.29 ms ± 3.05 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "10.7 ms ± 10.8 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "23.8 ms ± 5.83 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "44.6 ms ± 126 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "8.17 ms ± 6.97 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "13 ms ± 35.5 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "28.4 ms ± 11.9 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "52.6 ms ± 69.9 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "159 ms ± 205 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "164 ms ± 121 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "181 ms ± 256 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "207 ms ± 2.57 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "# Let's try a few build configurations.\n", - "# Warning: this will take some time\n", - "\n", - "k = 10\n", - "n_probes_variants = [10, 20, 50, 100]\n", - "n_lists = 1000\n", - "\n", - "build_configs = {\n", - " '64-8-subspace': ivf_pq.IndexParams(n_lists=n_lists, metric=metric, pq_dim=64, pq_bits=8, codebook_kind=\"subspace\"),\n", - " '128-8-subspace': ivf_pq.IndexParams(n_lists=n_lists, metric=metric, pq_dim=128, pq_bits=8, codebook_kind=\"subspace\"),\n", - " '128-6-subspace': ivf_pq.IndexParams(n_lists=n_lists, metric=metric, pq_dim=128, pq_bits=6, codebook_kind=\"subspace\"),\n", - " '128-6-cluster': ivf_pq.IndexParams(n_lists=n_lists, metric=metric, pq_dim=128, pq_bits=6, codebook_kind=\"cluster\"),\n", - "}\n", - "\n", - "bench_qps_ip = np.zeros((len(build_configs), len(search_configs), len(n_probes_variants)), dtype=np.float32)\n", - "bench_recall_ip = np.zeros_like(bench_qps_ip, dtype=np.float32)\n", - "\n", - "for i, index_params in enumerate(build_configs.values()):\n", - " index = ivf_pq.build(index_params, dataset, handle=resources)\n", - " for l, search_fun in enumerate(search_configs):\n", - " for j, n_probes in enumerate(n_probes_variants):\n", - " r = %timeit -o search_fun(n_probes); resources.sync()\n", - " bench_qps_ip[i, l, j] = (queries.shape[0] * r.loops / np.array(r.all_runs)).mean()\n", - " bench_recall_ip[i, l, j] = calc_recall(search_fun(n_probes), gt_neighbors)" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABToAAAhnCAYAAAATE7MkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd1xV9ePH8fdl7yGCoiJuFETAmXuvHFmppaVoZpaalT+1snJkWWmmZmllfdVMKxs23am5cgtqLtw7EZUNMs7vD+LmFVAwja69no+Hj+Lcz/mczzn3cC/3fT/DZBiGIQAAAAAAAACwYjbF3QAAAAAAAAAA+LsIOgEAAAAAAABYPYJOAAAAAAAAAFaPoBMAAAAAAACA1SPoBAAAAAAAAGD1CDoBAAAAAAAAWD2CTgAAAAAAAABWj6ATAAAAAAAAgNUj6AQAAAAAAABg9Qg6AeA/KCkpSY8//rhKly4tk8mkZ599VpL0xx9/qHv37vLx8ZHJZNK0adOKtZ23W0xMjNq1aydPT0+ZTCZ99913+ZY7fvy4TCaT5s6de9uOfSfqBHD7jBs3TiaTyWJbhQoV1K9fv+Jp0E3069dPFSpUKLbjL1u2TOHh4XJycpLJZNKVK1ckSfPnz1f16tVlb28vLy+vm9YzePBgtW3btkjHnjt3rkwmk44fP26xffLkyapUqZJsbW0VHh5epDqR19q1a2UymbR27VrztsLed9bwnpf7O3/x4sVbrsNkMmncuHEW27Zt26ZGjRrJ1dVVJpNJUVFRha4vLi5Orq6uWrJkyS23CQD+6wg6AeAukfvBr6B/mzdvNpedOHGi5s6dq6eeekrz589Xnz59JEnPPfecli9frhdffFHz589Xhw4dbns7J06cWGDAeKdFRkZqz549ev311zV//nzVrVu3WNqBf86+ffs0bty4PIHIf4lhGJo/f76aNWsmLy8vubi4KDQ0VK+99ppSUlLylG/RooXFa0eJEiVUr149/e9//1N2drZF2R9//FHNmzeXn5+fXFxcVKlSJfXs2VPLli3Lty0PPvig7r333jtynsUlJSVF48aNswiD7nZxcXHq2bOnnJ2d9f7772v+/PlydXXVgQMH1K9fP1WuXFmzZ8/WRx99dMN6jh07po8//lijR4/+221asWKFRo0apcaNG2vOnDmaOHGizp49q3HjxhUpaNq6dasGDx6sOnXqyN7ePk/4nevUqVMaP3686tevL29vb5UsWVItWrTQqlWr8i2/Y8cOde7cWaVLl5abm5tq1aqld999V1lZWbdyuviXysjIUI8ePXTp0iVNnTpV8+fPV2Bg4A3/Rjt//rx5fx8fHz3++ON65ZVXivEsAMC62RV3AwAAt9err76qihUr5tlepUoV8/+vXr1a99xzj8aOHWtRZvXq1brvvvs0YsSIO9a+iRMnqnv37urWrdsdO0Z+UlNT9dtvv+mll17S0KFDb1g2MDBQqampsre3/4dahztl3759Gj9+vFq0aFGsvd+KS1ZWlnr37q1FixapadOmGjdunFxcXLR+/XqNHTtWixYt0qpVq+Tn52exX7ly5fTGG29IkmJjY/Xpp59qwIABOnTokN58801J0ttvv62RI0eqefPmevHFF+Xi4qLDhw9r1apV+uKLL/J8UZKRkaGVK1ea671bpKSkaPz48ZJyQuL/gm3btikxMVETJkxQmzZtzNvXrl2r7OxsTZ8+3eI9pyDTp09XxYoV1bJlyyIdv0+fPnr44Yfl6Oho3rZ69WrZ2Njok08+kYODgyRp+/btGj9+vCpUqFDoHp5LlizRxx9/rFq1aqlSpUo6dOhQvuW+//57vfXWW+rWrZsiIyOVmZmpTz/9VG3bttX//vc/9e/f31x2x44datSokapWrarnn39eLi4uWrp0qZ555hkdOXJE06dPL9L5F6fZs2fn+cIDfzly5IhOnDih2bNn6/HHH8/zeH5/o13f8/nJJ5/Uu+++q9WrV6tVq1Z3srkAcFci6ASAu0zHjh1v2lPxwoULCg4Oznd7YYYaWqPY2FhJeT9Q5MdkMsnJyekOt+juYRiG0tLS5OzsXNxN+cckJyfL1dW1uJtxU5MmTdKiRYs0YsQITZ482bz9iSeeUM+ePdWtWzf1799fP//8s8V+np6eevTRR80/Dxo0SEFBQXrvvfc0YcIEmUwmTZgwQW3bttWKFSvyHPfChQt5tq1fv16JiYnq1KlTge21luv6d9wN55j7/F7/elrQ9vxkZGRowYIFevLJJ4t8fFtbW9na2uY5trOzsznkvFVPPfWUnn/+eTk7O2vo0KEFBp0tW7bUyZMnVbJkSfO2J598UuHh4RozZoxF0Pnhhx9KktatW6cSJUpIyvmdat68uebOnWtVQSdfAN7YzX4HCvM3Wo0aNVSzZk3NnTuXoBMAbgFD1wHgPyR3vq1jx47p559/Ng+byh1SZRiG3n//ffP2XFeuXNGzzz6rgIAAOTo6qkqVKnrrrbfy9OrI7ckTGhoqJycn+fr6qkOHDtq+fbuknAAxOTlZ8+bNMx8jd/67xMREPfvss6pQoYIcHR3l5+entm3baufOnTc9r127dqljx47y8PCQm5ubWrdubTFUf9y4cQoMDJQkjRw5UiaT6Ya9+/KbW6xfv35yc3PTmTNn1K1bN7m5ucnX11cjRozIM/TwypUr6tevnzw9PeXl5aXIyEjz/HXXO3DggLp3764SJUrIyclJdevW1Q8//GB+/MKFC/L19VWLFi1kGIZ5++HDh+Xq6qqHHnrohtcmdw6yAwcOqGfPnvLw8JCPj4+eeeYZpaWlWZSdM2eOWrVqJT8/Pzk6Oio4OFizZs3KU2eFChXUuXNnLV++XHXr1pWzs7P5g3xR61i7dq25jtDQUPPw32+//dZ8H9WpU0e7du0q8rWbO3euevToISknlMi9564dYrx06VI1bdpUrq6ucnd3V6dOnfT7779bHCf3uT9y5Ijuvfdeubu765FHHpGUM+/rgw8+qNKlS8vJyUnlypXTww8/rPj4+Bs+L5L01VdfqU6dOnJ2dlbJkiX16KOP6syZM/keuzD33fVSU1M1efJkVatWLd9elF26dFFkZKSWLFmirVu33rAuFxcX3XPPPUpOTlZsbKwuXryohIQENW7cON/y1/cQlaSff/5ZwcHB5t+9G13X7OxsTZs2TSEhIXJyclKpUqU0aNAgXb58OU+9S5cuVfPmzeXu7i4PDw/Vq1dPCxcuND++fv169ejRQ+XLl5ejo6MCAgL03HPPKTU19YbnXBjHjx+Xr6+vJGn8+PHmeyx3zr4bnWNR2vXdd9+pZs2acnJyUs2aNbV48eJ821OU61aQm92XLVq0UGRkpCSpXr165tfxChUqmEcJ+Pr65jt34bU2bNigixcvWvQIzTVjxgyFhITIxcVF3t7eqlu3rsVzev0cnSaTSXPmzFFycrLF+1q9evUkSf3797fYfiOlSpUq1Jc2ISEhFiGnJDk6Ouree+/V6dOnlZiYaN6ekJAgJyenPOGXv79/ob8gSk9P19ixY1WlShXz/TJq1Cilp6eby9xoXsz8no8zZ85owIABKlOmjBwdHVWxYkU99dRTunr1aoHtyG+Oztv5nidJly5d0ogRIxQaGio3Nzd5eHioY8eOio6OtiiX+zfNokWL9Prrr6tcuXJycnJS69atdfjw4QLP4Xq57ffy8pKnp6f69++fZ1qP9PR0Pffcc/L19ZW7u7u6du2q06dP57k2zZs3lyT16NFDJpMp317eiYmJN339btu2rX788UeL930AQOHQoxMA7jLx8fF5JtY3mUzy8fFRjRo1NH/+fD333HMqV66c/u///k+SFBERYZ6rs23bturbt69535SUFDVv3lxnzpzRoEGDVL58eW3atEkvvviizp07Z7Fg0YABAzR37lx17NhRjz/+uDIzM7V+/Xpt3rxZdevW1fz58/X444+rfv36euKJJyRJlStXlpTTE+brr7/W0KFDFRwcrLi4OG3YsEH79+9X7dq1Czzf33//XU2bNpWHh4dGjRole3t7ffjhh2rRooV+/fVXNWjQQA888IC8vLz03HPPqVevXrr33nvl5uZW5GublZWl9u3bq0GDBnr77be1atUqTZkyRZUrV9ZTTz0lKad343333acNGzboySefVI0aNbR48WJzMHB92xs3bqyyZcvqhRdekKurqxYtWqRu3brpm2++0f333y8/Pz/NmjVLPXr00IwZMzRs2DBlZ2erX79+cnd318yZMwvV9p49e6pChQp64403tHnzZr377ru6fPmyPv30U3OZWbNmKSQkRF27dpWdnZ1+/PFHDR48WNnZ2RoyZIhFfQcPHlSvXr00aNAgDRw4UEFBQUWu4/Dhw+rdu7cGDRqkRx99VG+//ba6dOmiDz74QKNHj9bgwYMlSW+88YZ69uypgwcPysbGptDXrlmzZho2bJjeffddjR49WjVq1JAk83/nz5+vyMhItW/fXm+99ZZSUlI0a9YsNWnSRLt27bL4MJ+Zman27durSZMmevvtt+Xi4qKrV6+qffv2Sk9P19NPP63SpUvrzJkz+umnn3TlyhV5enoW+HzMnTtX/fv3V7169fTGG2/ojz/+0PTp07Vx40bt2rXLIhApzH2Xnw0bNujy5ct65plnZGeX/598ffv21Zw5c/Tjjz+qfv36BdYlSUePHpWtra28vLzk5OQkZ2dn/fjjj3r66afNvdRuZMmSJercubPFtvyuq5TT2y33Gg0bNkzHjh3Te++9p127dmnjxo3mXmVz587VY489ppCQEL344ovy8vLSrl27tGzZMvXu3VtSTnCXkpKip556Sj4+Ptq6datmzJih06dP66uvvrppu2/E19dXs2bN0lNPPaX7779fDzzwgCSpVq1aNz3HwrZrxYoVevDBBxUcHKw33nhDcXFx6t+/v8qVK5enPYW9bgUpzH350ksvKSgoSB999JF5GG7lypXVrVs3ffrpp1q8eLFmzZplnoeyIJs2bZLJZFJERITF9tmzZ2vYsGHq3r27+QuZ3bt3a8uWLebn9Hrz58/XRx99pK1bt+rjjz+WJFWtWlWvvvqqxowZoyeeeEJNmzaVJDVq1OiG1+DvOn/+vFxcXMzPs5QTDn/55ZcaNGiQhg8fbh66/u2331r0tC5Idna2unbtqg0bNuiJJ55QjRo1tGfPHk2dOlWHDh26pXmvz549q/r16+vKlSt64oknVL16dZ05c0Zff/21UlJSCt0z9na/50k5rzXfffedevTooYoVK+qPP/7Qhx9+qObNm2vfvn0qU6aMRb1vvvmmbGxsNGLECMXHx2vSpEl65JFHtGXLlkKdQ8+ePVWxYkW98cYb2rlzpz7++GP5+fnprbfeMpd5/PHH9dlnn6l3795q1KiRVq9enad3+qBBg1S2bFlNnDhRw4YNU7169VSqVCmLMi1btlRSUpIcHBzUvn17TZkyRVWrVs3Tpjp16mjq1Kn6/fffVbNmzUKdBwDgTwYA4K4wZ84cQ1K+/xwdHS3KBgYGGp06dcpThyRjyJAhFtsmTJhguLq6GocOHbLY/sILLxi2trbGyZMnDcMwjNWrVxuSjGHDhuWpNzs72/z/rq6uRmRkZJ4ynp6eeY5dGN26dTMcHByMI0eOmLedPXvWcHd3N5o1a2beduzYMUOSMXny5JvWmVt2zpw55m2RkZGGJOPVV1+1KBsREWHUqVPH/PN3331nSDImTZpk3paZmWk0bdo0T52tW7c2QkNDjbS0NPO27Oxso1GjRkbVqlUtjtOrVy/DxcXFOHTokDF58mRDkvHdd9/d9FzGjh1rSDK6du1qsX3w4MGGJCM6Otq8LSUlJc/+7du3NypVqmSxLTAw0JBkLFu2LE/5otaxadMm87bly5cbkgxnZ2fjxIkT5u0ffvihIclYs2aNeVthr91XX32VZ1/DMIzExETDy8vLGDhwoMX28+fPG56enhbbc5/7F154waLsrl27DEnGV199leecb+Tq1auGn5+fUbNmTSM1NdW8/aeffjIkGWPGjMlz7Jvdd/mZNm2aIclYvHhxgWUuXbpkSDIeeOAB87bmzZsb1atXN2JjY43Y2Fhj//79xrBhwwxJRpcuXczlxowZY0gyXF1djY4dOxqvv/66sWPHjnyPc/To0TzPQ0HXdf369YYkY8GCBRbbly1bZrH9ypUrhru7u9GgQQOL62gYlq85+d2Tb7zxhmEymSzus9zflWsFBgbm+3p1rdjYWEOSMXbs2DyPFXSORWlXeHi44e/vb1y5csW8bcWKFYYkIzAw0LytsNetIEW5L3Pfb7Zt22ZRR+41jI2NveGxDMMwHn30UcPHxyfP9vvuu88ICQm54b65xz927Jh5W2RkpOHq6mpRbtu2bXled4tiyJAhee6JG4mJiTGcnJyMPn36WGzPzMw0hg4datjb25vfl21tbY1Zs2YVqt758+cbNjY2xvr16y22f/DBB4YkY+PGjYZh5P/elev6e7Rv376GjY1NnufQMP76/VmzZk2+v7fX3nd34j0vLS3NyMrKsmjTsWPHDEdHR4vXwtz21ahRw0hPTzdvnz59uiHJ2LNnT55zu1bu/frYY49ZbL///vst7s2oqChDkjF48GCLcr17985zXXPbdP37wpdffmn069fPmDdvnrF48WLj5ZdfNlxcXIySJUua/4661qZNmwxJxpdffnnDcwAA5MXQdQC4y7z//vtauXKlxb+lS5fecn1fffWVmjZtKm9vb128eNH8r02bNsrKytK6deskSd98841MJlOeBY4kFbhq7bW8vLy0ZcsWnT17ttBty8rK0ooVK9StWzdVqlTJvN3f31+9e/fWhg0blJCQUOj6CuP6+eSaNm2qo0ePmn9esmSJ7OzsLHra2dra6umnn7bY79KlS1q9erV69uypxMRE83WNi4tT+/btFRMTYzFc9L333pOnp6e6d++uV155RX369NF9991X6HZf35sytz1Lliwxb7t2CGVuz+DmzZvr6NGjeYZiV6xYUe3bt89znKLUERwcrIYNG5p/btCggSSpVatWKl++fJ7tude5qNcuPytXrtSVK1fUq1cvi/va1tZWDRo00Jo1a/Lsc33vydwem8uXL8939fKCbN++XRcuXNDgwYMt5oLt1KmTqlevnme+TOnm911+cofOuru7F1gm97Frh9lKOcNLfX195evrqxo1amjGjBnq1KmT/ve//5nLjB8/XgsXLlRERISWL1+ul156SXXq1FHt2rW1f/9+i/p+/vlneXp6qkmTJnnacP11/eqrr+Tp6am2bdtaPDd16tSRm5ub+blZuXKlEhMT9cILL+SZU/fa15xr78nk5GRdvHhRjRo1kmEY+U6JcCfk1/O2MO06d+6coqKiFBkZadFDuG3btnnmWS7sdSvIrdyXf0dcXJy8vb3zbPfy8tLp06e1bdu223q8Oy0lJUU9evSQs7OzecGuXLa2tqpcubLat2+vefPm6csvv1SXLl309NNPF6o35ldffaUaNWqoevXqFs9t7vyNN3tur5edna3vvvtOXbp0yXe+yMK8Z+e6E+95jo6O5t77WVlZiouLk5ubm4KCgvKdzqZ///4WPVBze+/e7DUyV36vr3Fxcea/H3LfJ4cNG2ZR7tlnny1U/VJOr9E5c+aob9++6tatmyZMmKDly5crLi5Or7/+ep7yub8b14/QAQDcHEPXAeAuU79+/ZtOdF8UMTEx2r17t3keuuvlTrx/5MgRlSlTplBDWPMzadIkRUZGKiAgQHXq1NG9996rvn37WgSY14uNjVVKSop52PS1atSooezsbJ06dUohISG31Kbr5c47ei1vb2+L+e9OnDghf3//PEPjr2/j4cOHZRiGXnnlFb3yyiv5Hu/ChQsqW7asJKlEiRJ699131aNHD5UqVUrvvvtukdp+/dC4ypUry8bGxjzHnSRt3LhRY8eO1W+//ZYnuIuPj7cIWq5fNfZW6rg2zJT+Cg4DAgLy3Z57nYt67fITExMjSQUu9ODh4WHxs52dXZ6hwhUrVtTw4cP1zjvvaMGCBWratKm6du2qRx999IbD1k+cOCEp7z0hSdWrV9eGDRssthXmvstPQSHmtXIfu35OzQoVKmj27NnmhbmqVq2a77ybvXr1Uq9evZSQkKAtW7Zo7ty5Wrhwobp06aK9e/eaA7Off/5Z7dq1yzOEPr/rGhMTo/j4+HyPJ1m+5ki66bDOkydPasyYMfrhhx/yXLPCzKX6d+V3joVtV+69kt/Q1utDn8Jet/j4eIt5QB0cHFSiRIki35e3g5HP/IPPP/+8Vq1apfr166tKlSpq166devfuXeB8sLciKSlJSUlJ5p9tbW0LfI8rjKysLD388MPat2+fli5dmu/Q6unTpysmJsb83tCzZ0+1bNlSQ4YMUefOnWVnZ6fY2FiLuRvd3Nzk5uammJgY7d+//6bvw4UVGxurhISE2zIk+k685+XO9z1z5kwdO3bM4pr4+Pjk2e/695LckLCwc9PeaH8PDw+dOHFCNjY25ql2CjrHomrSpIkaNGigVatW5Xks93ejKKEzACAHQScA4Iays7PVtm1bjRo1Kt/Hq1WrdluO07NnTzVt2lSLFy/WihUrNHnyZL311lv69ttv1bFjx9tyjL/r+lV+/47chZxGjBiRb89ISapSpYrFz8uXL5eU8+Hr9OnThVrZuCDXf3g6cuSIWrdurerVq+udd95RQECAHBwctGTJEk2dOjXPwlP5LaBR1DoKup4Fbc/94Hcr1+56uXXMnz9fpUuXzvP49YHctT2MrjVlyhT169dP33//vVasWKFhw4aZ50HNL9y6Fbd63+X2+Nu9e7e6deuWb5ndu3dLUp4vFFxdXfNdJKYgHh4eatu2rdq2bSt7e3vNmzdPW7ZsUfPmzZWSkqK1a9fmuyhVftc1Oztbfn5+WrBgQb7HKkoglZWVpbZt2+rSpUt6/vnnVb16dbm6uurMmTPq169fnnvyTsjvHO9Euwp73Z555hnNmzfPvL158+YWC3T9U3x8fPINomrUqKGDBw/qp59+0rJly/TNN99o5syZGjNmjMaPH39bjv32229b1BUYGGjxpU9RDRw4UD/99JMWLFiQ75cnM2fOVKtWrfKEgV27dtXw4cN1/PhxValSRfXq1TMHzpI0duxYjRs3TtnZ2QoNDdU777yT7/FzvxwqKBS72cI3/4SivG5PnDhRr7zyih577DFNmDBBJUqUkI2NjZ599tl8fzdu9p5xM393/78jICBABw8ezLM993fj+gWvAAA3R9AJALihypUrKykp6aahR+XKlbV8+XJdunTphr06b9Q7wd/fX4MHD9bgwYN14cIF1a5dW6+//nqBQaevr69cXFzy/ZBw4MAB2djY5OkdeKcFBgbql19+UVJSksWH2uvbmBss2dvbFypQWrZsmT7++GONGjVKCxYsUGRkpLZs2VLgIjPXi4mJseiFefjwYWVnZ5sX3Pnxxx+Vnp6uH374waJ3S1GGRN6OOgqjKNeuoPstt2eOn59fkQK9/ISGhio0NFQvv/yyNm3apMaNG+uDDz7Qa6+9lm/5wMBASTn3xPWhyMGDB82P/12NGzeWl5eXFi5cqJdeeinfD/O5i1Hlrk5/O9StW1fz5s3TuXPnJEmrV69Wenp6ob+wqFy5slatWqXGjRvfcEXq3Odw7969BQbbe/bs0aFDhzRv3jyLRdZWrlxZ2NO5qVvpcVXYduXeC7k9kK91/WtKYa/bqFGj9Oijj5p/zu299k/dl7mqV6+uBQsW5OnpLeUE7Q899JAeeughXb16VQ888IBef/11vfjii3mmKbiRgp6bvn37WkyjUNiVz/MzcuRIzZkzR9OmTVOvXr3yLfPHH3/kGzZmZGRIylmwSpIWLFhg0ds297WucuXKio6OVuvWrW94v+U+l9eveH5teCrlvHd6eHho7969Nzm7m7sT73lff/21WrZsqU8++cRi+5UrV4ol+AsMDFR2draOHDli0Yszv789iuro0aP5fnlz7NgxSX8tngcAKDzm6AQA3FDPnj3122+/mXsTXuvKlSvmD2gPPvigDMPIt8fNtb0iXF1d83wIy8rKyjOE1M/PT2XKlFF6enqBbbO1tVW7du30/fffW/TG+eOPP7Rw4UI1adIkzxDkO+3ee+9VZmamRe+1rKwszZgxw6Kcn5+fWrRooQ8//NAcCF0rNjbW/P9Xrlwxr1Y/ceJEffzxx9q5c6cmTpxY6Ha9//77Fj/ntic3fMoNwa59ruLj4zVnzpxCH+N21FEYRbl2rq6ukvJ+8G/fvr08PDw0ceJEc9hQUB0FSUhIMN//uUJDQ2VjY3PD+7Zu3bry8/PTBx98YFFu6dKl2r9/f56VfG+Vi4uLRo0apYMHD+qll17K8/jPP/+suXPnqkuXLgoNDS1S3SkpKfrtt9/yfSx3TuDcQGDJkiWqW7duntWHC9KzZ09lZWVpwoQJeR7LzMw0P5ft2rWTu7u73njjDaWlpVmUy70H87snDcPQ9OnTC9WWwshdXfv6e+xGCtsuf39/hYeHa968eRavkStXrtS+ffssyhb2ugUHB6tNmzbmf3Xq1JH0z92XuRo2bCjDMLRjxw6L7XFxcRY/Ozg4KDg4WIZh5Pu7eiMF/f5XqlTJ4hrc6rD4yZMn6+2339bo0aP1zDPPFFiuWrVqWrlypcW5ZWVladGiRXJ3dzeH9o0bN7ZoV2442LNnT505c0azZ8/OU3dqaqqSk5Ml5fSsLlmypHnu7FwzZ860+NnGxkbdunXTjz/+qO3bt+epsyg9Ge/Ee56trW2eNnz11Vc3nXv5Ri5evKgDBw4UaT7lXLnvk9dPGTNt2rRC15Hfe8qSJUu0Y8cOdejQIc9jO3bskKen522begcA/kvo0QkAd5mlS5fqwIEDebY3atTohvNdFmTkyJH64Ycf1LlzZ/Xr10916tRRcnKy9uzZo6+//lrHjx9XyZIl1bJlS/Xp00fvvvuuYmJi1KFDB2VnZ2v9+vVq2bKlhg4dKkmqU6eOVq1apXfeeUdlypRRxYoVFRQUpHLlyql79+4KCwuTm5ubVq1apW3btmnKlCk3bN9rr72mlStXqkmTJho8eLDs7Oz04YcfKj09XZMmTSry+f5dXbp0UePGjfXCCy/o+PHjCg4O1rfffpvvXIDvv/++mjRpotDQUA0cOFCVKlXSH3/8od9++02nT59WdHS0pJyhpnFxcVq1apVsbW3VoUMHPf7443rttdd03333KSws7KbtOnbsmLp27aoOHTrot99+02effabevXub923Xrp0cHBzUpUsXDRo0SElJSZo9e7b8/Pzy/VCan9tRR2EV9tqFh4fL1tZWb731luLj4+Xo6KhWrVrJz89Ps2bNUp8+fVS7dm09/PDD8vX11cmTJ/Xzzz+rcePGeu+9927YhtWrV2vo0KHq0aOHqlWrpszMTM2fP1+2trZ68MEHC9zP3t5eb731lvr376/mzZurV69e+uOPPzR9+nRVqFBBzz333G27TqNGjVJUVJTeeust/fbbb3rwwQfl7OysDRs26LPPPlNISIjmzp1b5HpTUlLUqFEj3XPPPerQoYMCAgJ05coVfffdd1q/fr26deumiIgISTkf5vv371/oups3b65BgwbpjTfeUFRUlNq1ayd7e3vFxMToq6++0vTp09W9e3d5eHho6tSpevzxx1WvXj317t1b3t7eio6OVkpKiubNm6fq1aurcuXKGjFihM6cOSMPDw998803hZ67rzCcnZ0VHBysL7/8UtWqVVOJEiVUs2bNG85/WJR2vfHGG+rUqZOaNGmixx57TJcuXdKMGTMUEhJiMc9kYa9bQf7J+1LKmZ/Qx8dHq1atsuhB2q5dO5UuXVqNGzdWqVKltH//fr333nvq1KnTDRfWyk/lypXl5eWlDz74QO7u7nJ1dVWDBg0KnGNYyun9OH/+fEkyh4C5vbMDAwPVp08fSdLixYs1atQoVa1aVTVq1NBnn31mUU/btm3N4f4LL7ygRx99VA0aNNATTzwhZ2dnff7559qxY4dee+012dvb3/A8+vTpo0WLFunJJ5/UmjVr1LhxY2VlZenAgQNatGiRli9fbp6b+/HHH9ebb76pxx9/XHXr1tW6det06NChPHVOnDhRK1asUPPmzfXEE0+oRo0aOnfunL766itt2LCh0FOj3In3vM6dO+vVV19V//791ahRI+3Zs0cLFiy4pb9hcr333nsaP3681qxZoxYtWhRp3/DwcPXq1UszZ85UfHy8GjVqpF9++UWHDx8udB2NGjVSRESE6tatK09PT+3cuVP/+9//FBAQoNGjR+cpv3LlSnXp0oU5OgHgVvyDK7wDAO6gOXPmGJIK/Ddnzhxz2cDAQKNTp0556pBkDBkyJM/2xMRE48UXXzSqVKliODg4GCVLljQaNWpkvP3228bVq1fN5TIzM43Jkycb1atXNxwcHAxfX1+jY8eOxo4dO8xlDhw4YDRr1sxwdnY2JBmRkZFGenq6MXLkSCMsLMxwd3c3XF1djbCwMGPmzJmFOvedO3ca7du3N9zc3AwXFxejZcuWxqZNmyzKHDt2zJBkTJ48+ab15Za99ppFRkYarq6uecqOHTvWuP7tNC4uzujTp4/h4eFheHp6Gn369DF27dqVp07DMIwjR44Yffv2NUqXLm3Y29sbZcuWNTp37mx8/fXXhmEYxvfff29IMqZMmWKxX0JCghEYGGiEhYVZPAcFtW/fvn1G9+7dDXd3d8Pb29sYOnSokZqaalH2hx9+MGrVqmU4OTkZFSpUMN566y3jf//7nyHJOHbsmLlcQffP7agjv3uwoOfuZtcu1+zZs41KlSoZtra2hiRjzZo15sfWrFljtG/f3vD09DScnJyMypUrG/369TO2b99uLlPQc3/06FHjscceMypXrmw4OTkZJUqUMFq2bGmsWrUq32tzvS+//NKIiIgwHB0djRIlShiPPPKIcfr0aYsyRbnvCpKdnW3MnTvXaNy4seHu7m5+TWjTpo2Rnp6ep3zz5s2NkJCQG9aZkZFhzJ492+jWrZsRGBhoODo6Gi4uLkZERIQxefJkc7179+41JBlbt27NU0dB55bro48+MurUqWM4Ozsb7u7uRmhoqDFq1Cjj7NmzFuV++OEHo1GjRoazs7Ph4eFh1K9f3/j888/Nj+/bt89o06aN4ebmZpQsWdIYOHCgER0dnef3Mb9rGhgYaERGRt7wWhiGYWzatMmoU6eO4eDgYEgyxo4de9NzLGy7DMMwvvnmG6NGjRqGo6OjERwcbHz77bdGZGSkERgYeMvXrSCFuS9z32+2bdtmsT33GsbGxhbqWMOGDTOqVKlise3DDz80mjVrZvj4+BiOjo5G5cqVjZEjRxrx8fF5jn/ta0pB1/r77783goODDTs7u3yv7fXWrFlT4Pto8+bN85xrQf+ufZ0xDMNYtmyZ0bx5c6NkyZKGg4ODERoaanzwwQeFuk6GYRhXr1413nrrLSMkJMRwdHQ0vL29jTp16hjjx4+3uDYpKSnGgAEDDE9PT8Pd3d3o2bOnceHCBYv7MteJEyeMvn37Gr6+voajo6NRqVIlY8iQIebf39xrce255Hff3c73PMMwjLS0NOP//u//DH9/f8PZ2dlo3Lix8dtvvxnNmze3eA5y2/fVV19ZHCO/9/Dc5+vacynofs3v/kpNTTWGDRtm+Pj4GK6urkaXLl2MU6dO5bmuBbXppZdeMsLDww1PT0/D3t7eKF++vPHUU08Z58+fN663f/9+Q1Kh30sAAJZMhvEPzLIMAACKxbhx4zR+/HjFxsayqAEk5cwL2KVLF/3yyy/68ccf8x02ebtMmjRJ77zzjs6dO0fPJORx9OhRVa9eXUuXLlXr1q2LuznAv8Kzzz6rdevWaceOHbxuAsAtYI5OAACA/xB7e3t98803Cg8PV48ePbRz5847dqwKFSpo6tSpfFhHvipVqqQBAwbozTffLO6mAP8KcXFx+vjjj/Xaa6/xugkAt4g5OgEAAP5jXF1dtW3btjt+nJ49e97xY8C6XbuIDfBf5+PjYzH3LgCg6OjRCQAAAAAAAMDqMUcnAAAAAAAAAKtHj04AAAAAAAAAVo+gEwAAAAAAAIDVI+gEAAC4DcaNGyeTyaSLFy8Wd1PuGv369VOFChUstplMJo0bN65Y2gMAAIB/N4JOAACAu8zrr7+url27qlSpUjcNBs+cOaOePXvKy8tLHh4euu+++3T06NF/rrH/gIULF2ratGnF3YwCffnll3r00UdVtWpVmUwmtWjRosCy6enpev7551WmTBk5OzurQYMGWrlyZb5lN23apCZNmsjFxUWlS5fWsGHD8l3RuSh1AgAA/JsRdAIAANxlXn75ZW3btk0RERE3LJeUlKSWLVvq119/1ejRozV+/Hjt2rVLzZs3V1xc3D/U2qJJTU3Vyy+/XKR9/u1B56xZs/T9998rICBA3t7eNyzbr18/vfPOO3rkkUc0ffp02dra6t5779WGDRssykVFRal169ZKSUnRO++8o8cff1wfffSRevTocct1AgAA/NvZFXcDAAAAcHsdO3ZMFSpU0MWLF+Xr61tguZkzZyomJkZbt25VvXr1JEkdO3ZUzZo1NWXKFE2cOPGfanKhOTk5FXcTbrv58+erbNmysrGxUc2aNQsst3XrVn3xxReaPHmyRowYIUnq27evatasqVGjRmnTpk3msqNHj5a3t7fWrl0rDw8PSVKFChU0cOBArVixQu3atStynQAAAP929OgEAAC4Q06cOKEqVaqoZs2a+uOPP/6x414/r2VBvv76a9WrV88cckpS9erV1bp1ay1atOiWjt2vXz+5ubnpzJkz6tatm9zc3OTr66sRI0YoKyvrluq81vVD8RMTE/Xss8+qQoUKcnR0lJ+fn9q2baudO3dKklq0aKGff/5ZJ06ckMlkkslksrg+M2bMUEhIiFxcXOTt7a26detq4cKFf7udRREQECAbm5v/Wf7111/L1tZWTzzxhHmbk5OTBgwYoN9++02nTp2SJCUkJGjlypV69NFHzSGnlBNgurm5WTy3ha0TAADAGtCjEwAA4A44cuSIWrVqpRIlSmjlypUqWbJkgWUzMjIUHx9fqHpLlChRqFDsZrKzs7V792499thjeR6rX7++VqxYocTERLm7uxe57qysLLVv314NGjTQ22+/rVWrVmnKlCmqXLmynnrqqb/d9ms9+eST+vrrrzV06FAFBwcrLi5OGzZs0P79+1W7dm299NJLio+P1+nTpzV16lRJkpubmyRp9uzZGjZsmLp3765nnnlGaWlp2r17t7Zs2aLevXvf8LiFXXTK3d1djo6Of+8k/7Rr1y5Vq1bNIryUcp4vKWe4ekBAgPbs2aPMzEzVrVvXopyDg4PCw8O1a9euItcJAABgDQg6AQAAbrMDBw6odevWKlu2rJYvX37TeRc3btyoli1bFqru3GHpf9elS5eUnp4uf3//PI/lbjt79qyCgoKKXHdaWpoeeughvfLKK5JywsjatWvrk08+ue1B588//6yBAwdqypQp5m2jRo0y/3/btm1VtmxZXb58WY8++miefUNCQvTVV18V+bg3mhLgWnPmzFG/fv2KXH9+zp07d9PnK7fctduvL7t+/foi1wkAAGANCDoBAABuo7179+qhhx5SlSpVtHTp0jw95fITFhZW6FWuS5cu/XebKClnUR9J+fY2zJ0HM7fMrXjyySctfm7atKnmz59/y/UVxMvLS1u2bNHZs2dVpkyZIu97+vRpbdu2zWL4fmEU9vkKCQkpUr03kpqaWqjn62bP7bXPa2HrBAAAsAYEnQAAALdRly5dVKpUKS1fvtw8RPpmvL291aZNmzvcMkvOzs6SpPT09DyPpaWlWZQpKicnpzw9Hr29vXX58uVbqu9GJk2apMjISAUEBKhOnTq699571bdvX1WqVOmm+z7//PNatWqV6tevrypVqqhdu3bq3bu3GjdufNN9/+nnS8p5PgrzfN3sub32eS1snQAAANaAxYgAAABuowcffFBHjhzRggULCr3P1atXdf78+UL9ux0L+kg5c306OjqahzlfK3dbUXtI5rK1tf1bbSuKnj176ujRo5oxY4bKlCmjyZMnKyQkREuXLr3pvjVq1NDBgwf1xRdfqEmTJvrmm2/UpEkTjR079qb7Fvb5up09Iv39/Qv1fOUOOy+o7LXPa2HrBAAAsAYEnQAAALfR5MmTNWDAAA0ePLjQq3dv2rRJ/v7+hfp3u1bBtrGxUWhoqLZv357nsS1btqhSpUq3tBBRcfD399fgwYP13Xff6dixY/Lx8dHrr79uftxkMhW4r6urqx566CHNmTNHJ0+eVKdOnfT666+bezTe6JiF+ffll1/etvMMDw/XoUOHlJCQYLF9y5Yt5sclqWbNmrKzs8vz3F69elVRUVHmckWpEwAAwBowdB0AAOA2MplM+uijj5SYmKjIyEi5ubmpa9euN9ynOObolKTu3bvrhRde0Pbt280rdB88eFCrV6/WiBEjbttx7pSsrCwlJSXJ09PTvM3Pz09lypSxGI7t6uqa76r2cXFx8vHxMf/s4OCg4OBgLV26VBkZGeZ5KvNTHHN0du/eXW+//bY++ugj8/OTnp6uOXPmqEGDBubV0T09PdWmTRt99tlneuWVV8yB9fz585WUlKQePXoUuU4AAABrQNAJAABwm9nY2Oizzz5Tt27d1LNnTy1ZskStWrUqsPztnqNz/vz5OnHihFJSUiRJ69at02uvvSZJ6tOnjwIDAyVJgwcP1uzZs9WpUyeNGDFC9vb2euedd1SqVCn93//9n0WdLVq00K+//irDMG5bO/+uxMRElStXTt27d1dYWJjc3Ny0atUqbdu2zWIV9jp16ujLL7/U8OHDVa9ePbm5ualLly5q166dSpcurcaNG6tUqVLav3+/3nvvPXXq1OmmvVlv5/O1bt06rVu3TpIUGxur5ORk8/PVrFkzNWvWTJLUoEED9ejRQy+++KIuXLigKlWqaN68eTp+/Lg++eQTizpff/11NWrUSM2bN9cTTzyh06dPa8qUKWrXrp06dOhgLleUOgEAAP71DAAAAPxtY8eONSQZsbGx5m0pKSlG8+bNDTc3N2Pz5s3/WFuaN29uSMr335o1ayzKnjp1yujevbvh4eFhuLm5GZ07dzZiYmLy1FmnTh2jdOnSNz12ZGSk4erqmmd77vUpisjISCMwMNBimyRj7NixhmEYRnp6ujFy5EgjLCzMcHd3N1xdXY2wsDBj5syZFvskJSUZvXv3Nry8vAxJ5jo//PBDo1mzZoaPj4/h6OhoVK5c2Rg5cqQRHx9fpHb+XbnXJr9/ueeaKzU11RgxYoRRunRpw9HR0ahXr56xbNmyfOtdv3690ahRI8PJycnw9fU1hgwZYiQkJOQpV5Q6AQAA/s1MhvEv+loeAAAA/zqJiYkqUaKEpk2bpiFDhhR3cwAAAIB8sRgRAAAAbmjdunUqW7asBg4cWNxNAQAAAApEj04AAAD8oy5duqSrV68W+Litra18fX3/wRYBAADgbkDQCQAAgH9U7sJGBQkMDNTx48f/uQYBAADgrkDQCQAAgH/Ujh07dPny5QIfd3Z2VuPGjf/BFgEAAOBuQNAJAAAAAAAAwOqxGBEAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAACAu06LFi1Us2bN4m4GAAAA/kEEnQAAAEAhzZo1Sz169FD58uVlMpnUr1+/G5ZftWqVWrVqJU9PT7m7u6tOnTr68ssv8y37f//3fwoODpYkJSUlaezYserQoYNKlCghk8mkuXPnFnic7OxszZo1S+Hh4XJ2dpaPj49atWql6OjoWz1VAAAAq2NX3A0AAAAArMVbb72lxMRE1a9fX+fOnbth2Tlz5mjAgAFq27atJk6cKFtbWx08eFCnTp3Kt/zPP/+sLl26SJIuXryoV199VeXLl1dYWJjWrl17w2M99thjWrBggfr27auhQ4cqOTlZu3bt0oULF27pPAEAAKwRQScAAACsQnJyslxdXYu1Db/++qu5N6ebm1uB5Y4fP64hQ4bo6aef1vTp029a79GjR3Xw4EF98MEHkiR/f3+dO3dOpUuX1vbt21WvXr0C9120aJHmzZunb7/9Vvfff3/RTwoAAOAuwdB1AAAA3FBiYqKeffZZVahQQY6OjvLz81Pbtm21c+dOi3JbtmxRhw4d5OnpKRcXFzVv3lwbN260KHPixAkNHjxYQUFB5iHWPXr00PHjxy3KzZ07VyaTSb/++qsGDx4sPz8/lStXzvz40qVL1bx5c7m7u8vDw0P16tXTwoUL87R93759atmypVxcXFS2bFlNmjQpT5mTJ0/qwIEDhboWgYGBMplMNy33wQcfKCsrS6+++qqknKHohmEUWP7nn3+Wp6enmjRpIklydHRU6dKlC9Wmd955R/Xr19f999+v7OxsJScnF2o/AACAuw1BJwAAAG7oySef1KxZs/Tggw9q5syZGjFihJydnbV//35zmdWrV6tZs2ZKSEjQ2LFjNXHiRF25ckWtWrXS1q1bzeW2bdumTZs26eGHH9a7776rJ598Ur/88otatGihlJSUPMcePHiw9u3bpzFjxuiFF16QlBOCdurUSZcuXdKLL76oN998U+Hh4Vq2bJnFvpcvX1aHDh0UFhamKVOmqHr16nr++ee1dOlSi3J9+/ZVjRo1bucl06pVq1S9enUtWbJE5cqVk7u7u3x8fPTKK68oOzs7T/klS5aobdu2srMr2oCrhIQEbd26VfXq1dPo0aPl6ekpNzc3VapUSYsWLbpdpwMAAGAVGLoOAACAG/r55581cOBATZkyxbxt1KhR5v83DENPPvmkWrZsqaVLl5p7PA4aNEghISF6+eWXtWLFCklSp06d1L17d4v6u3TpooYNG+qbb75Rnz59LB4rUaKEfvnlF9na2kqS4uPjNWzYMNWvX19r166Vk5OTRTuudfbsWX366afmOgcMGKDAwEB98skn6tix49+9LDcUExMjW1tb9e/fX6NGjVJYWJi+/fZbvfbaa8rMzNQbb7xhLpuSkqK1a9dq1qxZRT7OkSNHZBiGvvjiC9nZ2WnSpEny9PTU9OnT9fDDD8vDw0MdOnS4nacGAADwr0WPTgAAANyQl5eXtmzZorNnz+b7eFRUlGJiYtS7d2/FxcXp4sWLunjxopKTk9W6dWutW7fO3IvR2dnZvF9GRobi4uJUpUoVeXl55RkKL0kDBw40h5yStHLlSiUmJuqFF16wCDkl5RlS7ubmpkcffdT8s4ODg+rXr6+jR49alFu7du0Nh5XfiqSkJF2+fFnjx4/Xq6++qgcffFALFixQhw4dNH36dCUmJprLrl69Wunp6bcUviYlJUmS4uLi9P333+upp55S79699csvv8jHx0evvfbabTsnAACAfzuCTgAAANzQpEmTtHfvXgUEBKh+/foaN26cRVgYExMjSYqMjJSvr6/Fv48//ljp6emKj4+XJKWmpmrMmDEKCAiQo6OjSpYsKV9fX125csVc5loVK1a0+PnIkSOSpJo1a9603eXKlcsTfnp7e+vy5ctFuwC3IDfQ7dWrl8X2Xr16KTU1Vbt27TJv+/nnn1W3bl2VKlXqlo9TsWJFNWjQwLzdzc1NXbp00datW5WZmXkrpwAAAGB1GLoOAACAG+rZs6eaNm2qxYsXa8WKFZo8ebLeeustffvtt+rYsaO5t+bkyZMVHh6ebx25K5Q//fTTmjNnjp599lk1bNhQnp6eMplMevjhh/Odu/LaHqBFdW1P0Gvd7t6b+SlTpoxiYmLyhJd+fn6SZBG2LlmyRP3797/l40jKNyT18/NTRkaGkpOT5enpeUv1AwAAWBOCTgAAANyUv7+/Bg8erMGDB+vChQuqXbu2Xn/9dXXs2FGVK1eWJHl4eKhNmzY3rOfrr79WZGSkxXyfaWlpunLlSqHakXusvXv3qkqVKrd2Mv+AOnXqKCYmRmfOnFGlSpXM23OH//v6+krKOY+TJ0+qU6dOt3ScMmXKqHTp0jpz5kyex86ePSsnJye5u7vfUt0AAADWhqHrAAAAKFBWVlaeIeV+fn4qU6aM0tPTJeWEepUrV9bbb79tnjPyWrGxseb/t7W1zdOjcsaMGcrKyipUe9q1ayd3d3e98cYbSktLs3jsVntqnjx5UgcOHLilfQvy0EMPSZI++eQT87bs7GzNmTNHJUqUUJ06dSTl9OYsVaqU6tat+7eOderUKa1cudK87eLFi/r+++/VqlUr2djwJz8AAPhvoEcnAAAACpSYmKhy5cqpe/fuCgsLk5ubm1atWqVt27aZe2Xa2Njo448/VseOHRUSEqL+/furbNmyOnPmjNasWSMPDw/9+OOPkqTOnTtr/vz58vT0VHBwsH777TetWrVKPj4+hWqPh4eHpk6dqscff1z16tVT79695e3trejoaKWkpGjevHlFPse+ffvq119/LVRQ+uOPPyo6OlpSzmJKu3fvNi/407VrV9WqVUuSdN9996l169Z64403dPHiRYWFhem7777Thg0b9OGHH8rR0VFSzvycHTt2zDOXqCS99957unLlirkX6I8//qjTp09LypkCIHc4+osvvqhFixbpwQcf1PDhw+Xp6akPPvhAGRkZmjhxYpGvBwAAgLUi6AQAAECBXFxcNHjwYK1YsULffvutsrOzVaVKFc2cOVNPPfWUuVyLFi3022+/acKECXrvvfeUlJSk0qVLq0GDBho0aJC53PTp02Vra6sFCxYoLS1NjRs31qpVq9S+fftCt2nAgAHy8/PTm2++qQkTJsje3l7Vq1fXc889d1vPPT/ffPONRZi6a9cu88JC5cqVMwedJpNJ3333nV5++WV9+eWXmjt3roKCgvTZZ5/pkUcekSTFx8dr06ZNGjp0aL7Hevvtt3XixAnzz99++62+/fZbSdKjjz5qDjpLlSqlDRs2aMSIEZo6daoyMjLUsGFDffbZZwoLC7v9FwEAAOBfymT8E7OxAwAAALCwaNEiPfLII7p48SKLBQEAANwGTNgDAAAAFAMvLy+9++67hJwAAAC3CT06AQAAAAAAAFg9enQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6tkVdwPuZtnZ2Tp79qzc3d1lMpmKuzkAAAAAAACAVTEMQ4mJiSpTpoxsbG7cZ5Og8w46e/asAgICirsZAAAAAAAAgFU7deqUypUrd8MyBJ13kLu7u6ScJ8LDw+MfOWZGRoZWrFihdu3ayd7e/h85JnAncC/jbsL9jLsJ9zPuJtzPuFtwL+Nuwv2M6yUkJCggIMCcs90IQecdlDtc3cPD4x8NOl1cXOTh4cELAqwa9zLuJtzPuJtwP+Nuwv2MuwX3Mu4m3M8oSGGmhWQxIgAAAAAAAABWj6ATAAAAAAAAgNUj6AQAAAAAAABg9ZijEwAAAAAAAJKkrKwsZWRkFNvxMzIyZGdnp7S0NGVlZRVbO/DPsre3l62t7d+uh6ATAAAAAAAASkpK0unTp2UYRrG1wTAMlS5dWqdOnSrU4jO4O5hMJpUrV05ubm5/qx6CTgAAAAAAgP+4rKwsnT59Wi4uLvL19S22kDE7O1tJSUlyc3OTjQ0zLv4XGIah2NhYnT59WlWrVv1bPTsJOgEAAAAAAP7jMjIyZBiGfH195ezsXGztyM7O1tWrV+Xk5ETQ+R/i6+ur48ePKyMj428FndwxAAAAAAAAkCSGi6NY3K77jqATAAAAAAAAgNUj6AQAAAAAAAD+Bfr166du3boVdzOsFkEnAAAAAAAArNaZM2f06KOPysfHR87OzgoNDdX27dvzLfvkk0/KZDJp2rRpN61327Ztat26tby8vOTt7a327dsrOjr6NrcetxNBJwAAAAAAAKzS5cuX1bhxY9nb22vp0qXat2+fpkyZIm9v7zxlFy9erM2bN6tMmTI3rTcpKUkdOnRQ+fLltWXLFm3YsEHu7u5q3769MjIy7sSp4DYg6AQAAAAAAIBVeuuttxQQEKA5c+aofv36qlixotq1a6fKlStblDtz5oyefvppLViwQPb29jet98CBA7p06ZJeffVVBQUFKSQkRGPHjtUff/yhEydOFLhfdHS0WrZsKXd3d3l4eKhOnTrm3qXjxo1TeHi4Rflp06apQoUKeeoZP368fH195eHhoSeffFJXr141P/b1118rNDRUzs7O8vHxUZs2bZScnCzpr6HvN9p/2bJlatKkiby8vOTj46POnTvryJEjFsc/ffq0evXqpRIlSsjV1VV169bVli1bzI9///33ql27tpycnFSpUiWNHz9emZmZN72ud5pdcTcAAAAAAAAA/y6GYSg1I+sfP252drYMwyh0+R9++EHt27dXjx499Ouvv6ps2bIaPHiwBg4caFFnnz59NHLkSIWEhBSq3qCgIPn4+OiTTz7R6NGjlZWVpU8++UQ1atTIN5jM9cgjjygiIkKzZs2Sra2toqKiChWsXuuXX36Rk5OT1q5dq+PHj6t///7y8fHR66+/rnPnzqlXr16aNGmS7r//fiUmJmr9+vUW1+xG+0tScnKyhg8frlq1aikpKUljxozR/fffr6ioKNnY2CgpKUnNmzdX2bJl9cMPP6h06dLauXOnsrOzJUnr169X37599e6776pp06Y6cuSInnjiCUnS2LFji3SutxtBJwAAAAAAACykZmQpeMzyYjn2b8PvkWchyx49elSzZs3S8OHDNXr0aG3btk3Dhg2Tg4ODIiMjJeX0+rSzs9OwYcMK3QZ3d3etXbtW3bp104QJEyRJVatW1fLly2VnV3CcdvLkSY0cOVLVq1c371NUDg4O+t///icXFxeFhITo1Vdf1ciRIzVhwgSdO3dOmZmZeuCBBxQYGChJCg0NLfT+NjY2evDBBy3K/+9//5Ovr6/27dunmjVrauHChYqNjdW2bdtUokQJSVKVKlXM5cePH68XXnjBfH0rVaqkCRMmaNSoUcUedDJ0HQAAAAAAAFYpOztbtWvX1sSJExUREaEnnnhCAwcO1AcffCBJ2rFjh6ZPn665c+fKZDLlW0fHjh3l5uYmNzc3c4/P1NRUDRgwQI0bN9bmzZu1ceNG1axZU506dVJqaqokmfdxc3PTk08+KUkaPny4Hn/8cbVp00ZvvvlmniHhhREWFiYXFxfzzw0bNlRSUpJOnTqlsLAwtW7dWqGhoerRo4dmz56ty5cvF3p/SYqJiVGvXr1UqVIleXh4mHuonjx5UpIUFRWliIgIc8h5vejoaL366qsW5z9w4ECdO3dOKSkpRT7f24kenQAAAAAAALDgbG+rfa+2/8ePm52drYzU5EKX9/f3V3BwsMW2GjVq6JtvvpGUM8z6woULKl++vPnxrKws/d///Z+mTZum48eP6+OPPzaHl7nDzBcuXKjjx4/rt99+k42NjXmbt7e3vv/+ez388MOKiooy1+nh4SEpZx7O3r176+eff9bSpUs1duxYffHFF7r//vtlY2OTZ1h+URc2srW11cqVK7Vp0yatWLFCM2bM0EsvvaQtW7aoYsWKhaqjS5cuCgwM1OzZs1WmTBllZ2erZs2a5nk8nZ2db7h/UlKSxo8frwceeCDPY05OTkU6n9uNoBMAAAAAAAAWTCaTXBz++dgoOztbCWn597zMT+PGjXXw4EGLbYcOHTIP6+7Tp4/atGlj8Xj79u3Vp08f9e/fX5JUtmzZPPWmpKTIxsbGohdo7s+5c1VeO5z7WtWqVVO1atX03HPPqVevXpozZ47uv/9++fr66vz58zIMw1zvtWFprujoaKWmppoDx82bN8vNzU0BAQGScp6bxo0bq3HjxhozZowCAwO1ePFiDR8+/Kb7x8XF6eDBg5o9e7aaNm0qSdqwYYPF8WvVqqWPP/5Yly5dyrdXZ+3atXXw4MECz784MXQdAAAAAAAAVum5557T5s2bNXHiRB0+fFgLFy7URx99pCFDhkiSfHx8VLNmTYt/9vb2Kl26tIKCggqst23btrp8+bKGDBmi/fv36/fff1f//v1lZ2enli1b5rtPamqqhg4dqrVr1+rEiRPauHGjtm3bpho1akiSWrRoodjYWE2aNElHjhzR+++/r6VLl+ap5+rVqxowYID27dunJUuWaOzYsRo6dKhsbGy0ZcsWTZw4Udu3b9fJkyf17bffKjY21nyMm+3v7e0tHx8fffTRRzp8+LBWr15tDkhz9erVS6VLl1a3bt20ceNGHT16VN98841+++03SdKYMWP06aefavz48fr999+1f/9+ffHFF3r55ZeL9uTdAQSdAAAAAAAAsEr16tXT4sWL9fnnn6tmzZqaMGGCpk2bpkceeeRv1Vu9enX9+OOP2r17txo2bKimTZvq7NmzWrZsmfz9/fPdx9bWVnFxcerbt6+qVaumnj17qmPHjho/fryknCH1M2fO1Pvvv6+wsDBt3bpVI0aMyFNP69atVbVqVTVr1kwPPfSQunbtqnHjxknKGSK/bt063XvvvapWrZpefvllTZkyRR07dizU/jY2Nvriiy+0Y8cO1axZU88995wmT55scXwHBwetWLFCfn5+uvfeexUaGqo333xTtra2knJ6xP70009asWKF6tWrp3vuuUdTp04196ItTibj+skBcNskJCTI09NT8fHx5rka7rSMjAwtWbJE9957r3leCaAwriz+Tilbt8o5PFzOEeFyrFJFJpvi+y6Eexl3E+5n3E24n3E34X7G3YJ7GbdDWlqajh07pooVKxbrPIvZ2dlKSEiQh4eHeW5MFF6/fv105coVfffdd8XdlCK50f1XlHyNOToBSJKSVq9W4sqVil+8WJJk4+Ym57AwOUdE5ISfYbVk6+5ezK0EAAAAAADIH0EnAEmS9yOPyKFKZaXuilLq7t3KTkpS8saNSt64MaeAySTHKlX+Cj4jwuVQoYLFxMwAAAAAAADFhaATgCTJ9Z4Gcr2ngSTJyMxUekyMUnbtUmpUlFJ3RSnj1Cmlx8QoPSZGVxYtkiTZennlhJ7h4TkBaGhN2bi4FOdpAAAAAADwnzV37tzibkKxIugEkIfJzk5ONWrIqUYNqXdvSVLmxYtKjYr6M/yMVtqePcq6ckVJa9cqae3anB1tbeUUFHRNr88I2ZctQ6/Pf4Gkq0lyc3Ar7mYAAAAAAHDHEHQCKBS7kiXl3qaN3Nu0kSQZV68q7cABpe7apZQ/e31mnj+vtH37lLZvny4vWCBJsvUtKZfwv4JPp5Bg2Tg6Fuep/OfEp8er6RdNVdGzosL9whXmG6Zw33BV8KwgGxOTewMAAAAA7g4EnQBuicnBQc61asm5Vi2ViIyUJGWcO5cz1D0qSim7opS2b5+yYi8qceVKJa5cmbOfvb2cgoMte32W8ivOU7nrHbh0QIYMHY0/qqPxR/VtzLeSJA8HD4X5huUEn37hCi0ZKhd7ph4AAAAAAFgngk4At429v7/s/f3l0bGjJCk7LU1pv/9u0eszKy5OqdHRSo2ONu9nV8bfstdn9aDiOoW7UgP/Bvr1oV+1O3a3oi5EKTo2Wnsv7lXC1QStP7Ne68+slyTZmGxUzbuaOfgM8w1TObdyTD0AAAAAALAKBJ0A7hgbJye51Kkjlzp15CPJMAxlnD6t1D8XOUrZFaX0gweVefacEs6eU8KSJZIkk5OTHENCVNLdXcnOznKrW1d2JUoU78lYuRJOJdQioIVaBLSQJGVkZ+jQpUOKio1S9IVoRcVG6VzyOR24dEAHLh3Qlwe/lCT5OPko3C9c4b7hCvMLU7BPsBxtmXoAAAAAAPDvQ9AJ4B9jMpnkEBAgh4AAeXbtKknKSkpW2t49f/X6jIpWdny80nbsUAlJ5/5c6Mg+sHxOr8+InF6fjlWqyGRrW3wnY+XsbewVUjJEISVD9EiNRyRJfyT/oejYaHP4ue/SPsWlxemXk7/ol5O/SJLsbOwU7BOcE3z+2fPTz4WpBwAAAAAAxY+gE0CxsnVzles998j1nnskSUZ2tq4eP66k7dsV8+NP8r10SVePHFHGiZOKP3FS8d9/L0mycXWVc1gt83B357Aw2Xp4FOepWL1SrqXUzrWd2lVoJ0lKz0rXvrh9iroQZR7yHpcWp92xu7U7drd5vzKuZXLm+vTLCT6reVeTvY19cZ0GAAAAANzVWrRoofDwcE2bNq24m/KvQ9AJ4F/FZGMjx0qVZBMQoD8cHVXn3ntlk5Ki1N27zUPeU6OilZ2crORNvyl502/mfR2qVJbLNYscOVSoIJMNq4rfKkdbR0X4RSjCL0JSztQDp5NOm0PP6NhoHbp8SGeTz+ps8lktPb5UkuRk66SaJWuah7zX8q0lbyfv4jwVAAAAAHexdevWafLkydqxY4fOnTunxYsXq1u3bpKkjIwMvfzyy1qyZImOHj0qT09PtWnTRm+++abKlCljruPQoUMaOXKkNm7cqKtXr6pWrVqaMGGCWrZsecNjL1++XGPHjtXvv/8uJycnNWvWTFOmTFGFChXu4BmjIASdAP71bD095da0qdyaNpUkGVlZSj98OCf43BWllKhdyjhxUlcPH9HVw0d05auvJUk2np5yDg+TS26vz9BQ2bi6FuepWDWTyaQA9wAFuAeoS+UukqTkjGTtvbg3p9dnbE4Amng1Udv/2K7tf2w371vBo8JfvT59w1XZq7JsTITQAAAAAP6+5ORkhYWF6bHHHtMDDzxg8VhKSop27typV155RWFhYbp8+bKeeeYZde3aVdu3//WZpXPnzqpatapWr14tZ2dnTZs2TZ07d9aRI0dUunTpfI977Ngx3XfffRo+fLgWLFig+Ph4Pffcc3rggQe0c+fOO3rOyB9BJwCrY7K1lVNQkJyCguT98MOSpMzc1dz/DD9T9+xRdny8kn9dp+Rf1+XsaGMjx6AguUSEm3t92pdjVfG/w9XeVQ38G6iBfwNJUraRrePxxxUV+9dw96PxR3U84biOJxzX90dyph5wt3dXqG+oeZGjWiVryc3BrThPBQAAAICV6tixozp27JjvY56enlq5cqXFtvfee0/169fXyZMnVb58eV28eFExMTH65JNPVKtWLUnSm2++qZkzZ2rv3r0FBp07duxQVlaWXnvtNdn8OZpwxIgRuu+++5SRkSF7+/yn9Fq7dq1GjRql33//Xfb29goJCdHChQsVGBiofv366cqVK/ruu+/M5Z999llFRUVp7Z9rWEhSZmamhg4dqvnz58ve3l5PPfWUXn31VfPn25kzZ2rq1Kk6deqUPD091bRpU339dU6noBYtWqhmzZqSVOD+8+fP1/Tp03Xw4EG5urqqVatWmjZtmvz8/lqj4ffff9fzzz+vdevWyTAMhYeHa+7cuapcubIk6eOPP9aUKVN07NgxVahQQcOGDdPgwYPzvSa3C0EngLuCnY+P3Fu1knurVpIkIyNDaQcO/rXCe9QuZZ49p/T9+5W+f78uL/xckmTr4yPniHBzr0+nkBDZODkV56lYNRuTjSp5VVIlr0p6oGrON6nx6fHmoe7RF6K1++JuJWYkatPZTdp0dpMkySSTqnhXUbhvuML9chY6Ku9enhAaAAAAKC6GIWWk/PPHzc7OOfYdFB8fL5PJJC8vL0mSj4+PgoKC9Omnn6p27dpydHTUhx9+KD8/P9WpU6fAeurUqSMbGxvNmTNH/fr1U1JSkubPn682bdoUGHJmZmaqW7duGjhwoD7//HNdvXpVW7duLfJnn3nz5mnAgAHaunWrtm/frieeeELly5fXwIEDtX37dg0bNkzz589Xo0aNdOnSJa1fv77Q+0s5Q/4nTJigoKAgXbhwQcOHD1e/fv20ZMkSSdKZM2fUrFkztWjRQqtXr5aHh4c2btyozMxMSdKCBQs0ZswYvffee4qIiNCuXbs0cOBAubq6KjIyskjnWhQEnQDuSiZ7ezmH1pRzaE2pbx9JUsYff+T09syd63PfPmXFxSlp1S9KWpWzqrjs7eVUo4Zlr88Cvr1D4Xg6eqpZuWZqVq6ZJCkzO1Mxl2PMK7xHXYjSmaQzirkco5jLMfrq0FeSJG9Hb/NQ9zDfMIWUDJGznXNxngoAAADw35GRIk0sc/Nyt5mNJA3ZL8nzjtSflpam559/Xr169ZLHnwvamkwmrVq1St26dZO7u7tsbGzk5+enZcuWydu74PUGKlasqBUrVqhnz54aNGiQsrKy1LBhQ3MYmJ+EhATFx8erc+fO5p6PNWrUKPJ5BAQEaOrUqTKZTAoKCtKePXs0depUDRw4UCdPnpSrq6s6d+4sd3d3BQYGKiIiotD7S9Jjjz1mLlupUiW9++67qlevnpKSkuTm5qb3339fnp6e+uKLL8yhbrVq1cz7jB07VlOmTDFPJVCxYkXt27dPH374IUEnANwO9qVKyb5De3l0aC9Jyk5PV9rv+yx6fWbFXlTa7t1K271bmvepJMmudGnLXp/Vq8vk4FCcp2LV7GzsVMOnhmr41NDD1XOmHriYelHRF/4KPvfF7dPl9Mtae2qt1p5am7OfyU7VS1Q3h5/hfuEq7UoIDQAAAKBwMjIy1LNnTxmGoVmzZpm3G4ahIUOGyM/PT+vXr5ezs7M+/vhjdenSRdu2bZO/v79CQkJ04sQJSVLTpk21dOlSnT9/XgMHDlRkZKR69eqlxMREjRkzRt27d9fKlSt16tQpBQcHm48zevRojR49Wv369VP79u3Vtm1btWnTRj179pS/v3+RzuWee+6x6AXasGFDTZkyRVlZWWrbtq0CAwNVqVIldejQQR06dND9998vFxeXQu1va2urHTt2aNy4cYqOjtbly5eVnZ0tSTp58qSCg4MVFRWlpk2b5ttzNTk5WUeOHNGAAQPMwamU05vV0/POBNi5CDoB/GfZODrKpXaEXGr/tap4xpkzFr0+0w4eVOb580pcukyJS5dJkkyOjnKqWfOvXp/h4bIrWbI4T8XqlXQuqdaBrdU6sLUk6WrWVe2/tN88z2fUhSjFpsZqb9xe7Y3bqwX7F0iS/Fz8LIa71yhRQ/a2+Q8RAQAAAFAE9i7S6LP/+GGzs7Ol1MzbXm9uyHnixAnzUOtcq1ev1k8//aTLly+bt8+cOVMrV67UvHnz9MILL2jJkiXKyMiQJDk754w0y+3VOGnSJHNdn332mQICArRlyxbVrVtXUVFR5sdKlCghSZozZ46GDRumZcuW6csvv9TLL7+slStX6p577pGNjY2M64bu5x63sNzd3bVz506tXbtWK1as0JgxYzRu3Dht27bNPFz/RpKTk9W+fXu1b99eCxYskK+vr06ePKn27dvr6tWrFtcgP0lJSZKk2bNnq0GDBhaP2draFulcioqgEwD+ZDKZ5FCunBzKlZNnl86SpOyUFKXu2fvXcPddu5QVH6/UHTuUumOHeV/7gAA5/xl8ukREyLFqVZnseIm9VQ62DjmrtPuGScoJoc8lnzOHnlGxUTp46aAupFzQihMrtOLECkmSo62jQnxCFOYXZt6/pDMhNAAAAFBkJpPk4PrPHzc7W0pLuK1V5oacMTExWrNmjXx8fCweT0nJmYs0d0GhXDY2NuaejIGBgXnqTUlJybNPbpCXnZ0tOzs7ValSJd82RUREKCIiQi+++KIaNmyohQsX6p577pGvr6/27t1rUTYqKipPz8ktW7ZY/Lx582ZVrVrVfHw7Ozu1adNGbdq00dixY+Xl5aXVq1ebh5LfaP8DBw4oLi5Ob775pgICAiTJYoV6SapVq5bmzZuX76JLpUqVUpkyZXT06FE98sgj+Z7/ncKncAC4ARsXF7k2qC/XBvUl5QRuV48fz+n1+WfwmX74sDJOnVLGqVNK+OFHSZLJxUXOtWrJOTxMLhERcg4Lk20hvjlD/kwmk8q4lVEZtzLqWDFnNcWUjBT9Hve7OfyMjo3WlfQr2nlhp3Ze2GneN8A9wDzPZ7hfuKp4VZGtzZ39FhEAAADAPycpKUmHDx82/3zs2DFFRUWpRIkS8vf3V/fu3bVz50799NNPysrK0vnz5yXl9LB0cHBQw4YN5e3trcjISI0ZM0bOzs6aPXu2jh07pk6dOhV43E6dOmnq1Kl69dVXzUPXR48ene+cmNe27aOPPlLXrl1VpkwZHTx4UDExMerbt68kqVWrVpo8ebI+/fRTNWzYUJ999pn27t2bp76TJ09q+PDhGjRokHbu3KkZM2ZoypQpkqSffvpJR48eVbNmzeTt7a0lS5YoOztbQUFBhdq/fPnycnBw0IwZM/Tkk09q7969mjBhgsXxhw4dqhkzZujhhx/Wiy++KE9PT23evFn169dXUFCQxo8fr2HDhsnT01MdOnRQenq6tm/frsuXL2v48OGFfWqLjKATAIrAZDLJsWJFOVasKK8H7pckZSUmKjV691+9PqOjlZ2UpJTNm5WyebPi/tzXoVIli16fDpUqyXTdt38oPBd7F9UrXU/1SteTlBNCn0g4YZ7nMzo2WkeuHNGpxFM6lXhKPx7NCaFd7FwU6htqHvJey7eWPBw8bnQoAAAAAP9i27dvV8uWLc0/5wZpkZGRGjdunH744QdJUnh4uMV+a9asUYsWLVSyZEktW7ZML730klq1aqWMjAyFhITo+++/V1hYWIHHbdWqlRYuXKhJkyZp0qRJcnFxUcOGDbVs2bICh3a7uLjowIEDmjdvnuLi4uTv768hQ4Zo0KBBkqT27dvrlVde0ahRo5SWlqbHHntMffv21Z49eyzq6du3r1JTU1W/fn3Z2trqmWee0RNPPCFJ8vLy0rfffqtx48YpLS1NVatW1eeff66QkJBC7e/r66u5c+dq9OjRevfdd1W7dm29/fbb6tq1q3l/Hx8frV69WiNHjlTz5s1la2ur8PBwNW7cWJL0+OOPy8XFRZMnT9bIkSPl6uqq0NBQPfvsswVez9vBZFw/8B+3TUJCgjw9PRUfH28x98OdlJGRoSVLlujee+/Nd0JYwFpY871sZGUp/cgRi7k+rx4/nqecjYeHnMPCzAsdOdWqJVs3t3++wXexhKsJ2hO7x9zrc/fF3UrOSM5TrrJnZfM8n2F+YaroUdFiYu6/y5rvZ+B63M+4m3A/427BvYzbIS0tTceOHVPFihXl5ORUbO3Izs5WQkKCPDw88gwLx+3RokULhYeHa9q0acXdFLMb3X9Fydfo0QkAt5nJ1lZO1arJqVo1eT/UU5KUefnyn0Pd/xzyvmePshMSlLx+vZLXr8/Z0cZGjlWrWvT6tC9f/rYGbv81Hg4ealy2sRqXzflWMSs7S0fij1gscnQy8aSOxB/Rkfgj+ibmG0mSp6OneY7PcN9w1SxZUy72Ljc6FAAAAACgmBF0AsA/wM7bW+4tW8r9z+EURmam0g4etJjrM+PMGaUfPKj0gwd15YsvJUm2JUqYV3Z3iQiXU82asrnB6na4MVsbW1XzrqZq3tXUMygnhI5LjdPu2N2Kis0JP/de3Kv49HitO71O606vy9nPlLNf7jyf4X7hKuNahhAaAAAAAP5FCDoBoBiY7OzkHBIi55AQ6dGcVegyLlyw6PWZtnevsi5dUtLq1UpavTpnRzs7OVWvLueICPNCR3b+/gRuf4OPs49alm+pluVzQuiMrAwdvHzwr16fsVE6n3xe+y/t1/5L+/XFwS8kSSWdS5rn+QzzDVMNnxpytHUszlMBAAAAgJtau3ZtcTfhjiHoBIB/CXs/P9m3ayePdu0kSdlXryp93z6lXNPrM/PCBaXt3au0vXt1ef58SZKdn9+fwWdOr0/H4GDZODgU56lYNXtbe9UsWVM1S9bUo3pUknQ++bzF6u77L+3XxdSLWnVylVadXJWzn429gn2Cc1Z498sZ8u7r4lucpwIAAAAA/ykEnQDwL2Xj4GAeti7lrCqeefasUq7t9bl/vzIvXFDi8uVKXL5ckmRycJBTSIi516dzeLjs/fyK8UysX2nX0irtWlrtK7SXJKVlpmlf3D6LFd4vpV1SdGy0omOjpX05+5V1K6tavrUUWiJUSZlJyszOlL1YIAAAAAAA7gSCTgCwEiaTSfZly8qzbFl5duokScpOTVXa3r05vT7/XOE96/LlnP/ftcu8r33ZsuZen84R4XIKCpLJjreAW+Vk56TapWqrdqnaknJC6NOJp83zfEZdiFLMlRidSTqjM0lntPTYUknS/776n0J9Q81D3muVrCUvJ69iPBMAAAAAuHvwKRcArJiNs7Nc6tWTS716knICt4wTJyx6faYfOqSMM2eUceaMEn76SZJkcnaWc2ioOfh0Dg+Xnbd3cZ6KVTOZTArwCFCAR4C6VO4iSUrOSNaei3sUdSFKu/7YpZ3ndiotK03bzm/TtvPbzPtW8Khgnucz3DdclbwqycZkU1ynAgAAAABWi6ATAO4iJpNJDhUqyKFCBXl16yZJykpKUtru3UrZtUupUdFKjYpSdmKiUrZuVcrWreZ9HSpUsOj16Vilikw2BG63ytXeVff436N7/O9RRkaGfvr5JwU3CdbeS3vNPT+PxR/T8YTjOp5wXN8d/k6S5G7vrlp+tczBZ2jJULk5uBXvyQAAAACAFSDoBIC7nK2bm1wbNZJro0aSJCM7W1ePHs0JPv/s9Xn16FFdPX5cV48fV/zixZIkGzc3OYeF/RV+htWSrbt7cZ6KVbMx2aiSZyUFlQzSg9UelCRdSbui3Rd3m+f53HNxjxIzErXxzEZtPLPRvF8VryoWK7wHuAfIZDIV5+kAAAAAwL8OQScA/MeYbGzkWKWKHKtUkXePHpKkrCtXlBod/Vevz927lZ2UpOSNG5W8ceOfO5rkWKWKRa9PhwoVCNz+Bi8nLzUr10zNyjWTJGVmZ+rQ5UMWK7yfSTqjQ5cP6dDlQ1p0aJEkqYRTiZwen38GnyE+IXKycyrOUwEAAADwD2nRooXCw8M1bdq04m7Kvw5BJwBAtl5ecmveXG7Nm0uSjMxMpR86ZDHXZ8apU0qPiVF6TIyuLFpk3i93ZXjniAg5h9aUjYtLcZ6KVbOzsVOwT7CCfYLVq3ovSVJsSqw5+IyKjdK+uH26lHZJa06t0ZpTa3L2M9mphk8NhfmGKcwvZ8h7adfSxXkqAAAAwD9m3bp1mjx5snbs2KFz585p8eLF6vbnVF4ZGRl6+eWXtWTJEh09elSenp5q06aN3nzzTZUpU8Zcx6FDhzRy5Eht3LhRV69eVa1atTRhwgS1bNnyhsc2DENTpkzRRx99pBMnTqhkyZIaPHiwXnrppTt5yigAQScAIA+TnZ2cgoPlFBws9e4tScq8eFGpUVHmXp9pe/Yo68oVJa1dq6S1a3N2tLWVU1DQNb0+I2Rftgy9Pv8GXxdftQlsozaBbSRJV7Oual/cPovw82LqRe25uEd7Lu7RZ/s/kySVdi1tnucz3C9cQSWCZG9jX5ynAgAAANwRycnJCgsL02OPPaYHHnjA4rGUlBTt3LlTr7zyisLCwnT58mU988wz6tq1q7Zv324u17lzZ1WtWlWrV6+Ws7Ozpk2bps6dO+vIkSMqXbrgTgTPPPOMVqxYobfffluhoaG6dOmSLl26dMfOFTdG0AkAKBS7kiXl3qaN3NvkBG7G1atK27//z/AzSqm7dinzjz+Utm+f0vbt0+UFCyRJtr4l5RL+V/DpFBIsG0fH4jwVq+Zg66Bwv5zwMjIkUoZh6GzyWfNQ96gLUTp0+ZDOJ5/X+eTzWn58uSTJ0dZRIT4h5uHuYb5h8nH2KeazAQAAAP6+jh07qmPHjvk+5unpqZUrV1pse++991S/fn2dPHlS5cuX18WLFxUTE6NPPvlEtWrVkiS9+eabmjlzpvbu3Vtg0Ll//37NmjVLe/fuVVBQkCSpYsWKN23v2rVrNWrUKP3++++yt7dXSEiIFi5cqMDAQPXr109XrlzRd999Zy7/7LPPKioqSmtzO5hIyszM1NChQzV//nzZ29vrqaee0quvvmruZDJz5kxNnTpVp06dkqenp5o2baqvv/5aUs7Q95o1a0pSgfvPnz9f06dP18GDB+Xq6qpWrVpp2rRp8vPzM7fh999/1/PPP69169bJMAyFh4dr7ty5qly5siTp448/1pQpU3Ts2DFVqFBBw4YN0+DBg296ff4Ogk4AwC0xOTjkLFYUFqYSkZGSpIxz5yx7fe7bp6zYi0pcuVKJf/5xYbK3l1NwsGWvz1J+NzoUbsBkMqmsW1mVdSurTpU6SZJSMlL0e9zv5h6f0bHRik+P184LO7Xzwk7zvuXdy1sEn1W8qsjWxra4TgUAAAD/IoZhKDUz9R8/bnZ2tgzDuKPHiI+Pl8lkkpeXlyTJx8dHQUFB+vTTT1W7dm05Ojrqww8/lJ+fn+rUqVNgPT/++KMqVaqkn376SR06dJBhGGrTpo0mTZqkEiVK5LtPZmamunXrpoEDB+rzzz/X1atXtXXr1iKPgps3b54GDBigrVu3avv27XriiSdUvnx5DRw4UNu3b9ewYcM0f/58NWrUSJcuXdL69esLvb+UM+R/woQJCgoK0oULFzR8+HD169dPS5YskSSdOXNGzZo1U4sWLbR69Wp5eHho48aNyszMlCQtWLBAY8aM0XvvvaeIiAjt2rVLAwcOlKurqyL//Px4JxB0AgBuG3t/f9n7+8vjz29Ts9PSlPb770rdtcs832dWXJxSo6OVGh1t3s+ujL9lr8/qQTLZM8z6VrnYu6he6XqqV7qepJw/Uo8nHDf3+oyOjdbhK4d1MvGkTiae1A9HfpAkudq7KrRkaE6PUd9whfqGysPBozhPBQAAAMUkNTNVDRY2KJZjr+i0Qp7yvCN1p6Wl6fnnn1evXr3k4ZHzt67JZNKqVavUrVs3ubu7y8bGRn5+flq2bJm8vb0LrOvo0aM6ceKEvvrqK3366afKysrSc889p+7du2v16tX57pOQkKD4+Hh17tzZ3POxRo0aRT6PgIAATZ06VSaTSUFBQdqzZ4+mTp2qgQMH6uTJk3J1dVXnzp3l7u6uwMBARUREFHp/SXrsscfMZStVqqR3331X9erVU1JSktzc3PT+++/L09NTX3zxhez//OxWrVo18z5jx47VlClTzFMJVKxYUfv27dOHH35I0AkAsE42Tk5yqVNHLnXqyEc5gVvGqVMWvT7TDx5U5tlzSjh7Tgl/fjtocnKSc82aOb0+I3IWO7Ir4BtR3JzJZFJFz4qq6FlR91e9X5IUnx6vPRf3mIe7747dreSMZG0+t1mbz23O2U8mVfaqbLHCewWPCsy5CgAAAKuUkZGhnj17yjAMzZo1y7zdMAwNGTJEfn5+Wr9+vZydnfXxxx+rS5cu2rZtm/z9/RUSEqITJ05Ikpo2baqlS5cqOztb6enp+vTTT80h3yeffKI6dero4MGDcnZ2VnBwsPk4o0eP1ujRo9WvXz+1b99ebdu2VZs2bdSzZ0/5+/sX6Vzuuecei7/LGzZsqClTpigrK0tt27ZVYGCgKlWqpA4dOqhDhw66//775XLNwrE32t/W1lY7duzQuHHjFB0drcuXLys7O1uSdPLkSQUHBysqKkpNmzY1h5zXSk5O1pEjRzRgwABzcCrl9Gb19LwzAXYugk4AwD/GZDLJoXx5OZQvL8+uXSVJWUnJStu7J6fX565dSo3erez4eKVs366UayYHtw8sn9PrMyKn16djlSoy2TLM+lZ5OnqqSdkmalK2iSQpKztLh68ctljk6FTiKR2+cliHrxzWNzHfSJK8HL3MQ93D/cIV4hMiF3uXGx0KAAAAVsjZzllbem/5x4+bnZ2tjJSM215vbsh54sQJ81DrXKtXr9ZPP/2ky5cvm7fPnDlTK1eu1Lx58/TCCy9oyZIlysjIaZezs7Mkyd/fX3Z2dhY9GXN7Z548eVItW7ZUVFSU+bHc4exz5szRsGHDtGzZMn355Zd6+eWXtXLlSt1zzz2ysbHJM3Q/97iF5e7urp07d2rt2rVasWKFxowZo3Hjxmnbtm3m4fo3kpycrPbt26t9+/ZasGCBfH19dfLkSbVv315Xr161uAb5SUpKkiTNnj1bDRpY9gq2vcOf4Qg6AQDFytbNVa733CPXe+6RJBnZ2bp67Ng1vT6jdPXwEWWcOKn4EycV//33kiQbV1c5h9UyD3d3DguTrQfDrG+VrY2tgkoEKahEkHoG9ZQkxaXG5QSfsVGKvhCt3+N+15X0K/r19K/69fSvOfuZcva7doV3f1d/en0CAABYOZPJVCxfaGdnZyvBlHBb68wNOWNiYrRmzRr5+FguypmSkiJJsrGxsdhuY2Nj7skYGBiYp97GjRsrMzNTR44cMQ9DP3TokLm8nZ2dqlSpkm+bIiIiFBERoRdffFENGzbUwoULdc8998jX11d79+61KBsVFZWn5+SWLZYh9ObNm1W1alVzkGhnZ6c2bdqoTZs2Gjt2rLy8vLR69WrzUPIb7X/gwAHFxcXpzTffVEBAgCRZrFAvSbVq1dK8efOUkZGRp22lSpVSmTJldPToUT3yyCP5nv+dQtAJAPhXMdnYyLFyZTlWriyvBx+UJGXFxyt1925zr8+06N3KTk5W8qbflLzpN/O+DlUqy+WaRY4cKlSQ6bo/VlB4Ps4+alW+lVqVbyVJysjK0IFLB8wLHO26sEsXUi5oX9w+7Yvbp88PfC5J8nX2tVjkKNgnWA62DsV5KgAAALiLJSUl6fDhw+afjx07pqioKJUoUUL+/v7q3r27du7cqZ9++klZWVk6f/68pJwelg4ODmrYsKG8vb0VGRmpMWPGyNnZWbNnz9axY8fUqVOnAo/bpk0b1a5dW4899pimTZum7OxsDRkyRG3btrXo5XmtY8eO6aOPPlLXrl1VpkwZHTx4UDExMerbt68kqVWrVpo8ebI+/fRTNWzYUJ999pn27t2bZ47NkydPavjw4Ro0aJB27typGTNmaMqUKZKkn376SUePHlWzZs3k7e2tJUuWKDs727wy/M32L1++vBwcHDRjxgw9+eST2rt3ryZMmGBx/KFDh2rGjBl6+OGH9eKLL8rT01ObN29W/fr1FRQUpPHjx2vYsGHy9PRUhw4dlJ6eru3bt+vy5csaPnx4YZ/aIiPoBAD869l6esqtaVO5NW0qSTKyspQeE6PUqCjzQkcZJ07q6uEjunr4iK589bUkycbTU87hYXLJ7fUZGiobV9fiPBWrZm9rr1DfUIX6hqqP+kiSziefN/f4jLoQpQOXDig2NVYrT6zUyhMrc/azsVeIT4g5/Az3C1dJ55LFeSoAAAC4i2zfvl0tW7Y0/5wbpEVGRmrcuHH64YecxTfDw8Mt9luzZo1atGihkiVLatmyZXrppZfUqlUrZWRkKCQkRN9//73CwsIKPK6NjY1+/PFHPf3002rWrJlcXV3VsWNHc2CYHxcXFx04cEDz5s1TXFyc/P39NWTIEA0aNEiS1L59e73yyisaNWqU0tLS9Nhjj6lv377as2ePRT19+/ZVamqq6tevL1tbWz3zzDN64oknJEleXl769ttvNW7cOKWlpalq1ar6/PPPFRISUqj9fX19NXfuXI0ePVrvvvuuateurbfffltd/5x+TMpZqX716tUaOXKkmjdvLltbW4WHh6tx48aSpMcff1wuLi6aPHmyRo4cKVdXV4WGhurZZ58t8NrcDibj+oH/uG0SEhLk6emp+Ph4i7kf7qSMjAwtWbJE9957b74TwgLWgnsZRZWZu5r7rl1K3RWl1D17ZKSnWxaysZFjUJBc/lzgyDkiQvblyt3xYdb/pfs5NTNV++L2mef53B27W5fSLuUpV9at7F/Bp2+4qnpXlZ0N379ag//S/Yy7H/cz7hbcy7gd0tLSdOzYMVWsWFFOTk7F1o7s7GwlJCTIw8Mjz1By3B4tWrRQeHi4pk2bVtxNMbvR/VeUfI1PFACAu4Kdj4/cW7WSe6ucYdbG1atKO3gwJ/SMyun1mXn2nNL371f6/v26vDBnmLWtj4+cI8LNvT6dQkJkU4x/2Fk7Zztn1SlVR3VK1ZGUs4LlqcRTf/X6jI1SzOUYnUk6ozNJZ/Tz0Z/N+4WWDLVY4d3T8c6uyAgAAADg7kLQCQC4K5kcHOQcGirn0FCpb84w64w//sgJPnftUkrULqXt26+suDglrfpFSat+ydnR3l5ONWpY9vosXboYz8S6mUwmlfcor/Ie5dW1cs5Ql6SrSdp9cbeiY6MVfSFa0bHRSspI0tbzW7X1/FbzvhU9Kyrc96/h7hU9K8rGxLf6AAAAAPJH0AkA+M+wL1VK9h3ay6NDe0lSdnq60n7//a9en7uilHXxotJ271ba7t3SvE8lSXalS1v2+qxeXSYHFte5VW4ObmpUppEalWkkSco2snX0ylFFxUYp6kLOQkfHE47rWPwxHYs/psWHF0uS3B3czQschfuFK7RkqFztmXMVAAAAKIq1a9cWdxPuGIJO3LLoU1f08nd71TLIVy2q+ymsnJdsbe7sPHcAcDvZODrKpXZtudSuLSlnmHXGmTPmXp+pUVFKO3hQmefPK3HpMiUuXSZJMjk6yqlmzb96fYaHy64ki+vcKhuTjap4V1EV7yrqXq27JOly2mXtjt1tXuF978W9SryaqA1nNmjDmQ3m/ap6VbVY5Kic252fcxUAAADAvxNBJ27ZmoMXtOdMvPacide7qw/L28Vezav5qkWQn5pV81UJV3o7AbAuJpNJDuXKyaFcOXl26SxJyk5JUeqevX8ucpQTfmbFxyt1xw6l7thh3tc+IEDOfwafLhERcqxaVSY73mZvlbeTt5oHNFfzgOaSpIzsDB26fMjc4zP6QrTOJp/VwcsHdfDyQX158EtJUgmnEgr3DTeHn8E+wXKyY85VAAAA4L+AT2C4ZY80CFRZL2etPRirdTGxupySoe+izuq7qLMymaTwAC+1DPJTyyA/hZTxkA29PQFYIRsXF7k2qC/XBvUl5fT6vHrsuFKj/ur1mX74sDJOnVLGqVNK+OFHSZLJxUXOtWrJOTxMDqG1ZJOSUpynYfXsbewV4hOiEJ8QPVLjEUnShZQLio6NNq/wvj9uvy6lXdLqU6u1+tRqSZKdjZ2CSwSrlm8thfuFK9w3XKVcSxXnqQAAAAC4Qwg6cct83R3Vo26AetQNUEZWtnaeuKy1h2K15sAFHTifqF0nr2jXySt6Z+UhlXRzVIsgX7UM8lOTqiXl6Wxf3M0HgFtiMpnkWKmiHCtVlNcD90uSshISlBq9+6/wc/duZSclKWXzZqVs3ixJqiLpxKfz5VI7wtzr06FSJZlsWFznVvm5+KltYFu1DWwrSUrPStf+uP3mXp+7LuxSXFqcdl/crd0Xd+uz/Z9Jkvxd/c1D3cN9w1WtRDXZ2/C+BAAAAFg7gk7cFva2NmpQyUcNKvno+Q7VdS4+VWsP5oSeGw5f1MWkdH2947S+3nFatjYm1Qn0VssgP7UI8lX10u7MpwbAqtl6eMitaRO5NW0iSTKyspR++Ig5+EyJ2qWM4yeUceyY4o8dU/w330qSbDw85BwWJufwMLlERMipVi3ZurkV56lYNUdbx5zw0i9cUk7v2zNJZ3Lm+fxzdfeDlw/qXPI5nUs+p2XHc+ZcdbJ1UkjJEIsh795O3sV4JgAAAABuBUEn7gh/T2f1ql9eveqXV3pmlrYfv6w1By5ozcELOhKbrK3HLmnrsUt6a9kBlfZwUsvqOXN7Nq5SUm6O3JYArJvJ1lZOQdXkFFRN3g/1VEZGhpYvWqQmfn66mjvf5969yk5IUPL69Upevz5nRxsbOVatajHXp3358nwZdItMJpPKuZdTOfdy6lwpZ87VlIwU7b2412KF94SrCdrxxw7t+OOvOVcDPQItVniv7FlZtja2xXUqAAAAAAqBRAl3nKOdrRpXKanGVUrq5c7BOnUpRWsPXtCag7HadOSiziek6fOtp/T51lOytzWpfsUSf/b29FNlX1c+4AO4K2S5/T979x1W9Xn+cfz9PYs9ZCkCgoKCE9wxjkj2Mjsam2GSZptp9mxWs2Nsotm7WZo0TX4x00Rwxa2IiyXTyd4bzu+PL2Js00z1MD6v63quFj0H7qecIny4n+f2xmvyZPxPMI9ZO5uaqE/POOiuz6Zdu2hIT6chPZ3yj8zhOtaAgPbJ7p7DE3AfMgSLh4crt9Kpedo9GRM6hjGh5p2rrc5Wcitz2VS4qb3zc0fFDvIq88irzOP/dvwfAN52b4YGDW0/7j40eCg+Dh9XbkVEREREuijDMPj3v//NWWed5epSOh0FnXLERQR4cvG4KC4eF0V9UwursktITi9icVoh+aW1rMgqYUVWCY9+uZ3wHh7mQKO4YMb1C8LDoW4aEekaDLsdjyGD8RgyGC4yh+s0FRa2BZ9m+Fm/dSstpaVUL15M9WJzuA42G+5xcXgMH95+5N0WGqpfCv1BFsNCP79+9PPrx9n9zTtXKxoqSC1KNQcdFaWwuWgz1U3VrNyzkpV7VgJgYBDtH90efMYHxxPpG6nPg4iIiIgLLF26lKeffpr169ezZ8+eg0LCpqYm7rvvPr766iuys7Px8/Pj+OOP54knnqB3797t7yMjI4Pbb7+dFStW0NjYyLBhw3jkkUdITEz8xY/tdDp59tlnefXVV8nLyyMoKIjrrruOe++993Bu+TdLTk4mMTGRsrIy/P39XV3OYaegU1zK3W5lclv35t+mDCKnuIak9CKS0wtZnV3KzrI6/rkqj3+uysNhszCuXyCJscEkxoUQGejl6vJFRA4pe0gI9hNPxPfEEwFobWykfutW6lI2mV2fGzfSXFRE/ZYt1G/ZQtk//wmALSSkLfg0uz7dBg3C4nC4ciudmp+bHxPDJzIxfCIAza3NZJVntXd9phSmsLN6J1nlWWSVZ/FJxicA9HDrYR53DzGPvA8JGoKHTd23IiIiIodbTU0N8fHxXH755ZxzzjkH/V1tbS0bNmzg/vvvJz4+nrKyMm666SbOOOMM1q1b1/64008/nf79+7N48WI8PDyYM2cOp59+Ojt27KBXr17/82PfdNNNfPfddzzzzDMMHTqU0tJSSktLD9teXcXpdNLS0oLN1rGjxI5dnXQrhmHQL9ibfsHe/HVCX2oamlm5o4Sk9EKS0grZXVHPkowilmQU8eAX2+gX5MXktm7PMX0DcLOp21NEuhaLw4Hn8OF4Dh8Ol12K0+mkefduajemtB95r09Lo7mwkKpvv6Xq228BMBwO3AcPbu/69EhIwB4S4uLddF42i424gDjiAuKYFjcNgOK6YjYVbWofcrSleAtlDWUk70wmeWey+TzDRmxA7EET3nt59VLXp4iIiMghdsopp3DKKaf87N/5+fmxaNGig/5s7ty5jBkzhvz8fPr06UNxcTGZmZm88cYbDBs2DIAnnniCF198kS1btvzPoHP79u289NJLbNmyhdjYWAD69u37m2p+8803efbZZ8nKyiIgIIBzzz2XuXPn/tfjfq4jMyUlheHDh5OTk0NUVBR5eXlcf/31LF++nMbGRqKionj66acZNGhQe0dqjx7msM0ZM2bw9ttv09raypNPPsmrr77K3r17GTBgAPfffz/nnXfeQR/3q6++4r777mPz5s189913TJ48+Tftz1UUdEqH5eVm4/hBPTl+UE+cTieZhdXtA43W5ZaRXVxDdnEOb67IwcNu3gM6ua3bM8xfHTQi0vUYhoE9LAy/sDD8Tj8NgNbaWuq2bDnQ9ZmSQktZWXsH6H72sLD2rk+P4Qm4x8ZidPDfxnZkQR5BHNfnOI7rcxwATS1NbC/d3j7gKKUwhcK6QraWbGVryVY+SPsAgBDPEDP4bJvwPjBgIHar3ZVbEREREflZTqcTZ13dEf+4ra2tOJ3Ow/oxKioqMAyjPTgMDAwkNjaWd999lxEjRuDm5sYrr7xCSEgII0eO/J/v54svvqBfv34sXLiQk08+GafTyfHHH89TTz1FQEDA/3zeSy+9xKxZs3jiiSc45ZRTqKioYMWKFX94PzNnzqSxsZGlS5fi5eXFtm3b8Pb2JiIign/961+ce+65pKen4+vri0fbff+PP/447733Hi+//DL9+/dn6dKlXHTRRQQHB3PMMce0v++77rqLZ555hn79+rWHpR2ZfsKRTsEwDAb09GFATx+uPiaayvomVmQWk5ReSHJ6EYVVDXy/fR/fb98HwICe3u0DjUZF9cButbh4ByIih4fF0xOvMWPwGmMO13E6nTTl5VH7k7s+GzIzadq1i6Zdu6hcuBAAw8MDj6FD24NPj4QEbJ3gG5eOym61Myx4GMOCzQ4Ap9PJ3pq95oCjtuAzrTSNwtpCFuUtYlGe2VXgsDgYHDS4/Z7P+JB4gjyCXLkVEREREQCcdXWkj/jfId/h1DNpMfj5HZb3XV9fz5133sn06dPx9fUFzMzh+++/56yzzsLHxweLxUJISAjffPPNL4Z72dnZ5OXl8fHHH/Puu+/S0tLCLbfcwnnnncfi/Xfs/4xHH32UW2+9lZtuuqn9z0aPHv2H95Sfn8+5557L0KFDAejXr1/73+0PXENCQtqD3YaGBh577DG+//57xo0b1/6c5cuX88orrxwUdD788MOc0DZQtTNQ0Cmdkq+7nVOGhnLK0FCcTifb9lSSnF5EUlohG/LLyNhXTca+al5Zmo2Pm40J/YNIjA3hmNhgevq6u7p8EZHDxjAMHFFROKKi8G+7gL2lupq6TZsODDratInWqipq16yhds2a9uc6oqIO6vp0i4nBsOgXRX+EYRiEeocS6h3KKX3NY1R1zXVsLd7aPt19U9EmyhrK2Fi4kY2FB7pvw73DSQhJaD/yHuMfg82ib9lERERE/qympiamTp2K0+nkpZdeav9zp9PJzJkzCQkJYdmyZXh4ePD6668zZcoU1q5dS2hoKIMHDyYvLw+AiRMn8vXXX9Pa2kpDQwPvvvsuAwYMAOCNN95g5MiRpKen4+HhwaBBg9o/zj333MMVV1zB7t27Oe644w7Zvm688UauvfZavvvuO44//njOPffc9iP4PycrK4va2tr/CjAbGxsZPnz4QX82atSoQ1bnkaDvmqXTMwyDwb39GNzbj5mJMZTXNrI0s5jktEKSM4oorWnk6y17+XrLXgAGhfqSGBdMYmwICRH+2NTtKSJdnNXbG+/x4/EePx4AZ2srjTt2HOj6TEmhMTubxtxcGnNzqfj3vwGweHvjER9/IPyMH4bVx8eVW+nUPGwejOo1ilG9zG8WnU4n+VX5B467F6WQVZbFzuqd7KzeycJss/vW0+bJ0KChxIeYR96HBQ/Dz+3wdDiIiIiI7Gd4eBC7Yf0R/7itra1UNTUd8ve7P+TMy8tj8eLF7d2cAIsXL2bhwoWUlZW1//mLL77IokWLeOedd7jrrrv46quvaGqra//x79DQUGw2W3vICTBw4EDA7LJMTEwkJSWl/e8CAgKw23/ftUWWtsaDnx7nb/qP/32uuOIKTjrpJL788ku+++47Hn/8cZ599lluuOGGn32f1dXVAHz55ZeEhYUd9Hdubm4Hve3l1bkGQSvolC7H39PBGfG9OSO+N62tTlJ3VZCcXkhSehGpO8vZtqeSbXsqmZe0Az8PO5MGBJMYG8ykAcEEebv9+gcQEenkDIsFt/79cevfnx7nnw9Ac1nZwV2fmzfTWl1NzYoV1Oy/L8gwcIuJOajr0xEVpeE6f5BhGET6RhLpG8mZMWcCUNVYxeaize3BZ2pRKtVN1azeu5rVe1e3P7efX7/2AUfxwfFE+UVhMfSLOxERETl0DMPA8PQ88h+4tRWjsvKQvsv9IWdmZiZJSUkEBgYe9Pe1tbXAgVBxP4vFQmtrKwCRkZH/9X7Hjx9Pc3MzO3bsIDo6GoCMjIz2x9tsNmJiYv7reVFRUfzwww/tg4J+SXBwMAB79uxpP0b/0/B0v4iICK655hquueYa7r77bl577TVuuOEGHA4HAC0tLe2PHTRoEG5ubuTn5x90TL0rUNApXZrFYpAQ4U9ChD83Hz+A4uoGlmYUkZRexNKMIirqmvhi026+2LQbw4Bh4f4kxprdnkPD/LBY9MO7iHQPth498Jk8GZ+2KYrO5mYaMjIOuuuzaedOGjIzacjMpHzBAgCs/v5m6JmQYAagQ4dgccU3xF2Ej8OHo8OO5uiwowFoaW1hR8WO9ns+NxVtIq8yj+yKbLIrsvk081MAfB2+5h2fbcfdhwYNxdOuz4OIiIh0D9XV1WRlZbW/nZOTQ0pKCgEBAYSGhnLeeeexYcMGFi5cSEtLC3v3mic+AwICcDgcjBs3jh49ejBjxgweeOABPDw8eO2118jJyeG00077nx/3+OOPZ8SIEVx++eXMmTOH1tZWZs6cyQknnHBQl+d/evDBB7nmmmsICQnhlFNOoaqqihUrVvxsB2ZMTAwRERE8+OCD/P3vfycjI4Nnn332oMfcfPPNnHLKKQwYMICysjKSkpLaO0sjIyMxDIOFCxdy6qmn4uHhgY+PD7fddhu33HILra2tTJgwoX0gkq+vLzNmzPhd//t3JAo6pVsJ8nbjnBHhnDMinOaWVlIKytsHGm3dXcmmgnI2FZQz5/tMAr0cHDMgmMlxIUzqH4S/p8PV5YuIHDGGzYb7oEG4DxoEf/kLAM1FRWbw2RZ+1m/ZQkt5OdXJyVQnJ5tPtFpxj439SdfncOxhvdX1+QdZLVYG9BjAgB4DOH+A2X1bWl9KalFqe/C5pXgLlY2VLNu1jGW7lgFgMSwM6DGgPfhMCE4gzDtMnwcRERHpktatW3dQd+SsWbMAmDFjBg8++CD/93//B0BCQsJBz0tKSmLy5MkEBQXxzTffcO+993LsscfS1NTE4MGD+fzzz4mPj/+fH9disfDFF19www03MGnSJLy8vDjllFP+K4j8TzNmzKC+vp7nnnuO2267jaCgIM4777yffazdbufDDz/k2muvZdiwYYwePZpHH32U89tOZoHZrTlz5kx27tyJr68vJ598Ms899xwAYWFhPPTQQ9x1111cdtllXHLJJbz99ts88sgjBAcH8/jjj5OdnY2/vz8jRozgnnvu+cXaOzrD+dND/nJIVVZW4ufnR0VFxUF3PxxOTU1NfPXVV5x66qm/+96H7m5fZT1L0otISi9kWWYx1Q3N7X9nMWBEnx4kxoUwOTaYQaG++mHxMNNrWbqSrvp6djY2Ur99O3UpKdS2dX0279v3X4+zBgfhmXAg+HQfPAiLm64KOVSaWpvIKM1oH3KUUpTCnpo9//W4QPfAA8fdQ+IZFDgIN+vv/zx01dezdE96PUtXodeyHAr19fXk5OTQt29f3N1dN8S3tbWVyspKfH19/+souXRdv/T6+z35mjo6Rdr09HVn6ugIpo6OoLG5lfV5ZW13exaSsa+adXllrMsr4+lv0wnxcSMx1gw9x/cPwtdd30yISPdjOBzmsKL4eALajrc07dlD3caN7Ufe67dvp6WomKpFi6hatMh8nt2O+6BBB3d99gxx5VY6NbvFzuCgwQwOGsyFAy8EYF/NvvZ7PjcVbmJb6TZK6kv4If8Hfsj/AQCbxcagwEHt93wmhCQQ4qnPg4iIiIh0Xgo6RX6Gw2ZhXHQg46IDufvUgewsqyU5vYjk9EJWZJVQWNXA/HUFzF9XgM1iMCqqB4mxISTGhdA/xFvdniLSbdlDQ7GHhuJ76qkAtNbXU79ly4Guz5QUWkpKzMFHmza1P8/WO/Tgrs+4WAx1pPxhPb16cqLXiZwYdSIADS0NbCvZRkphSvuR95L6ElKLUkktSm1/Xm+v3uZdnyFm8DmgxwDsFn0eRERERKRzUNAp8huE9/DkoqMiueioSOqbWlibW0pSmhl8ZhfXsCq7lFXZpTz+dRph/h5MbhtodHRMIJ4O/d9MRLovi7s7nqNG4TlqFIGA0+mkqaDgoK7PhowMmnfvoXL3Hiq/+goAw90djyFDzK7P4eawI1tAgGs304m5Wd0YHjKc4SHDAfPzsLN6Z3voualoExllGeyu2c3umt18nfs1AB42DwYHDm4/8j4seBjeVm9XbkVERERE5H9SAiPyO7nbrUzsH8zE/sE8MGUQucU1bUfci1iZXcKu8jreX53P+6vzcVgtjO0X0N7t2TfIy9Xli4i4lGEYOPr0wdGnD35nnglAS3UN9ZtT27o+N1KXsonWykpq162jdt269ufaI/uYXZ/Dza5Pt5gYDKvVVVvp1AzDIMInggifCKZETwGgpqmGLcVbzK7PIjMArWqsYt2+dazbd+DzEOkTSUBDAHVZdYzsNZJo/2gshu7PEhERERHXU9Ap8idFBXlxaVBfLh3fl7rGFlZmF5OcXsTitEJ2ltWxLLOYZZnFPLxwG5GBnu13ex7VLxB3u35AFxGxenvhNW4cXuPGAeBsbaUxJ+egrs/GHTtoysunIi+fis8/B8Di5YVH/LD24+4e8fFYj9Dwv67Iy+7F2NCxjA0dC0Crs5XcilxSig4cd8+uyCavKo888ti4ZiMAPnYfhgUPaz/yPixoGN4OdX2KiIiIyJGnoFPkEPJwWDk2rifHxvXkoTOc7CiqaR9otCanlLySWt7+MZe3f8zF3W7h6OggEmODmRwbQkSAp6vLFxHpEAyLBbfoaNyio/E/7zwAWioqzHs927o+6zel0lpTQ82PK6n5cWX7cx0x0Xj+ZMiRIyoKQ9M6/xCLYaGffz/6+ffjnP7nAFDRUMGGPRv4dNWn1PjXsKVkC1VNVazYvYIVu1cAYGAQ0yOGhOCE9iPvET4Rur9aRESkk3A6na4uQbqhQ/W6U9ApcpgYhkFMiDcxId5cMbEf1Q3NrMgqNoPPtCL2VtazOK2QxWmFwFZiQryZPCCYxLgQRkcF4LDpB3MRkf2sfn54T5qE96RJADhbWmjIzKRu48b2QUdN+fk0Zu2gMWsH5R9/AoDFzw+PhHg893d9Dh2KxUvXiPxRfm5+TAibQKVHJacedyqG1SCzLLN9wntKYQq7qneRWZZJZlkmH2d8DEAPtx7mgKO2Ce+DgwbjYfNw8W5ERETkp6xtVwI1Njbi4aF/p+XIamxsBA68Dv8oBZ0iR4i3m42TBvfipMG9cDqdpO+rIimtiKT0QtbnlZFVWE1WYTWvL8/By2FlfEwQiXHmMfdQP/0jIyLyU4bVintcHO5xcfSYPh2A5pIS6lJSDnR9bt5Ca0UFNUuWUrNkqflEiwW32Fg82wYceQwfjj08XN2Gf5DNYmNg4EAGBg7kgrgLACiuK2ZT4YHgc1vJNsoaykguSCa5INl8nmEjLiCuPfxMCEmgl1cvl+1DREREwGaz4enpSVFREXa7HYuLTsW0trbS2NhIfX29y2qQI6u1tZWioiI8PT2x2f5cVKmgU8QFDMMgrpcvcb18uXZyNBV1TSzPLCYpvZDk9CKKqxv4bts+vtu2D4C4Xj4kxoWQGBvCiD7+2Kz6Yi8i8p9sgYH4HHccPscdB4CzsZH69PSDuj6b9+yhYft2GrZvp+yDDwGwBgbiMTyhvevTffBgLO7urtxKpxbkEcRxkcdxXKT5eWhsaWR76fb2ez5TClMoqitiS8kWtpRs4f3t7wMQ4hly0HH3uIA47Fa7K7ciIiLSrRiGQWhoKDk5OeTl5bmsDqfTSV1dHR4eHvpldDdisVjo06fPn/6cK+gU6QD8POycNiyU04aF0trqZOvuSpLa7vZMKSgnbW8VaXureCl5Bz7uNib1D2ZybDDHxAYT4qMfxkVEfo7hcOAxdCgeQ4fCJZcA0LR3r9n1uTGF2pSN1G/bTktJCdXf/0D19z+YT7TbcR848OCuz17qNvyjHFaHOagoOB4wf3jZU7OnPfRMKUohvTSdwtpCvsv7ju/yvgPAzerG4MDBxIfEtz8/yCPIlVsRERHp8hwOB/37928/RuwKTU1NLF26lEmTJmG365ee3YXD4TgkHbwKOn9FTk4Ol19+Ofv27cNqtbJq1Sq8dLeXHEYWi8HQcD+Ghvtx43H9Ka1pZFlmEUlphSzJKKKstokvN+/hy817ABga5mcONIoLIT7cH6tFv/ESEflf7L16YT/5ZHxPPhmA1oYG6rdupW5jCnUpG6ndmEJLcTH1qanUp6bCO+8CYOvV6+Cuz7g4DIfDlVvptAzDoLd3b3p79+aUvqcAUNtUy9aSre3h56aiTZQ3lLOhcAMbCje0PzfCJ6L9ns+EkARi/GOwWv7cPU4iIiJyMIvFgrsLT7dYrVaam5txd3dX0Cm/m4LOX3HppZfy6KOPMnHiREpLS3Fzc3N1SdLNBHg5ODMhjDMTwmhpdbJpZznJaYUkpRexeVdF+3p+cRY9PO0c0zbQaFL/YHp46YdwEZFfYnFzw3PECDxHjADMbsOmXbvM4+5tXZ8N6Rk0791L1dffUPX1NwAYbm64DxlyoOszIQFbkLoN/yhPuyeje41mdK/RgPl5yKvMa7/nc1PRJnaU76CgqoCCqgK+yP7CfJ7Nk6HBQ9uPvA8LHoavw9eVWxERERERF1LQ+Qu2bt2K3W5n4sSJAAQEBLi4IunurBaDEX16MKJPD2adGEthVT1L0otITi9iaabZ7flZym4+S9mNYUBChD+JsebdnoN7+2JRt6eIyC8yDANHeDiO8HD8pkwBoLWmhrrNW9qOvJv3fbZUVFC3fj1169e3P9ceEYFHW/DpOXw4bv37Y/zJy9S7K8MwiPKLIsovirNizgKgsrGSzUWb27s+U4tTqWmqYfWe1azes7r9udF+0SSEmF2f8SHx9PXtq/u9RERERLqJLv3d99KlS3n66adZv349e/bs4d///jdnnXXWQY+ZN28eTz/9NHv37iU+Pp4XXniBMWPGAJCZmYm3tzdTpkxh165dnHfeedxzzz0u2InIzwvxcef8URGcPyqCppZWNuaXm3d7phWStreKjfnlbMwvZ/aiDIK83ZgcG0xibAgT+gfh56EjACIiv4XFywuvo8biddRYwOw2bMzJbQ8961I20pC1g6aCApoKCqj8P7Pb0PD0xGPYMDwS4vEcPhyP+His/v4u3Enn5uvwZXzYeMaHjQegpbWFHRU7DhpylF+Vz46KHeyo2MG/Mv8FgJ+bX/sdnwnBCQwJGoKn3dOVWxERERGRw6RLB501NTXEx8dz+eWXc8455/zX38+fP59Zs2bx8ssvM3bsWObMmcNJJ51Eeno6ISEhNDc3s2zZMlJSUggJCeHkk09m9OjRnHDCCS7Yjcgvs1stjOkbwJi+Adx5chx7KupITjfv9lyRVUxxdQOfrN/JJ+t3YrUYjIzsQWJsCJNjg4nr5aNuFxGR38gwDNz69cWtX1/8zzW/v2iprKRuU+qB8HPTJlpraqhdtYraVasoaXuuo1+/g7o+Hf36YRyCS9e7I6vFyoAeAxjQYwBTY6cCUFJXQmpRKilFZvi5pXgLFQ0VLN25lKU7l5rPM8zn7b/nMyEkgd5evfXvoIiIiEgX0KWDzlNOOYVTTjnlf/797NmzufLKK7nssssAePnll/nyyy958803ueuuuwgLC2PUqFFEREQAcOqpp5KSkvI/g86GhgYaGhra366srATMiWFNTU2Halu/aP/HOVIfTzquIE8b5w0P5bzhoTQ0t7I+r4wlGcUsySxmR1ENa3JKWZNTypPfpNHT143JA4I4pn8w46ID8HZz/ZcGvZalK9HruRvw8MDtqLG4HTUWf8DZ0kLjjh3Up2yifpO5mvLyaMzOpjE7m4p/fQqAxccH92HDcI+Pxz0hAfehQ7B4e7t0K7+mI7+efW2+TAidwITQCQA0tTSRUZ7BpqJNpBanklqcyt7avWwv3c720u18lP4RAEHuQQwLHkZ8UDzDgoYxMGAgDqvuue4OOvLrWeT30GtZuhK9nuU//Z7XguF0Op2HsZYOwzCMg46uNzY24unpySeffHLQcfYZM2ZQXl7O559/TnNzM6NHj2bx4sX4+flx5plncvXVV3P66af/7Md48MEHeeihh/7rzz/44AM8PXVESjqOknrYVm6wrcwgs9KgqfVAF4vVcBLt62SQv5NBPZyEuIOaXERE/jxrdTXu+fm45+fjkZeHe8FOLP/xTZvTMGjs1ZO6yEjq+vShPjKSpsBAfSE+hCpaKyhoLiC/JZ/85nz2tOyhhZaDHmPFSm9rb/rY+tDH2oc+tj74WHxcVLGIiIhI91ZbW8tf/vIXKioq8PX95cGTrm/bcpHi4mJaWlro2bPnQX/es2dP0tLSALDZbDz22GNMmjQJp9PJiSee+D9DToC7776bWbNmtb9dWVlJREQEJ5544q9+Ig6VpqYmFi1axAknnIDdrjsY5dfVN7WwJreM5IxiktOLKCirI6PCIKMCPsuD8B4eZrfngCDGRgXg4bAekbr0WpauRK9n+TnOpiYaMjKpT0lp7/ps3r0btz17cduzF/9V5oAda0AP3IfFt3V9xuM2eDAWDw+X1d3VXs/1zfVsL93OpuIDXZ+l9aUUtBRQ0FLAClYA0NurN8OChjEsaBjxwfH09++PzdJtv5XuMrra61m6L72WpSvR61n+0/4T07+Fvjv7Fb92/P2n3NzccHNz+68/t9vtR/z/nK74mNI52e12jhsUynGDQnE6neQU15CUXkRyeiGrs0vZWVbHe6sLeG91AW42C0f1CyQxNpjEuBAiA72OSH16LUtXodezHMRux5EQj09CfPsfNe0rbBtwZE54r9+6lZbSMmqSk6lJTjYfZLPhHheHx/Dh7YOObKGhR/yOya7yerbb7YwJG8OYMHMYpdPpZGfVzvZ7PlMKU8gsz2R3zW521+zmm7xvAPCweTAkaAgJweY9n8OChuHv7u/Cncif0VVezyJ6LUtXotez7Pd7XgfdNugMCgrCarWyb9++g/5837599OrVy0VVibiWYRj0C/amX7A3f53Ql5qGZlbuKCEpvZDk9CJ2ldexJKOIJRlFPPjFNvoFeTE5NoTEuGDG9A3AzXZkuj1FRLoqe88Q7CediO9JJwLQ2thI/dat1G08EH42FxVRv2UL9Vu2UPbPfwJgCwlpCz4T8ByegNugQVgcumPyjzAMgwjfCCJ8I5gSPQWAmqYaNhdvJqUwhZSiFFILU6lqqmLt3rWs3bu2/blRvlHmgKPgBOKD4+nn3w+LoWFTIiIiIkdKtw06HQ4HI0eO5Icffmi/o7O1tZUffviB66+/3rXFiXQQXm42jh/Uk+MH9cTpdJJZWE1SWiFJ6YWsyy0ju7iG7OIc3lyRg4fdyviYICa3dXuG+bvuWKWISFdhcTjwHD4cz+HDAbPbsHn3bmp/EnzWp6XRXFhI1bffUvXttwAYDgfugwe3d316JCRgDwlx5VY6NS+7F0eFHsVRoUcB0OpsJacipz343FS0iZyKHHIrc8mtzOWzrM8A8LH7MCzEPOqeEJzA0KCheDs69rApERERkc6sSwed1dXVZGVltb+dk5NDSkoKAQEB9OnTh1mzZjFjxgxGjRrFmDFjmDNnDjU1Ne1T2EXkAMMwGNDThwE9fbj6mGiq6ptYkVVMUloRSemFFFY18P32fXy/3eySHtDTm8TYECbHhjAqqgd2qzpaRET+LMMwsIeF4RcWht/ppwHQWltL3ZYtB3V9tpSXU7dxI3UbN7Y/1x4W1t716TE8AffYWAxbl/5W8LCxGBai/aOJ9o/m3AHnAlBeX05qcSophWbwubl4M1VNVazYtYIVu1a0Py/GP6b9uHtCcALhPuFH/NoBERERka6qS393u27dOhITE9vf3j8oaMaMGbz99ttMmzaNoqIiHnjgAfbu3UtCQgLffPPNfw0oEpH/5uNu5+QhoZw8xLzbc9ueSpLTi0hKK2RDfhkZ+6rJ2FfNK0uz8XGzMaF/EImxIRwTG0xPX3dXly8i0mVYPD3xGjMGrzEH7phsyss7qOuzITOTpl27aNq1i8qFCwEwPDzwGDq0Pfj0SEjA1qOHK7fSqfm7+zMpfBKTwicB0NzaTEZZRvs9n5uKNrGrehcZZRlklGWwIGMBAAHuAWbHZ4h53H1w4GDcbfp3UkREROSP6NJB5+TJk3E6nb/4mOuvv15H1UX+JMMwGNzbj8G9/ZiZGEN5bSNLM4tJTiskOaOI0ppGvt6yl6+37AVgcG/ftm7PYBIi/LGp21NE5JAxDANHVBSOqCj8zz4LgJaqKupSUw90faak0FpdTe2aNdSuWdP+XEdU1EFdn24xMRgWfY3+I2wWG4MCBzEocBDT46YDUFRb1B58phSlsK1kG6X1pSQVJJFUkGQ+z7AxMHAg8cHxxIeYR957een+eBEREZHfoksHnSLiGv6eDs6I780Z8b1pbXWSuquC5PRCktKLSN1ZztbdlWzdXcncpCz8POxMGhBMYmwwxwwIJtDbzdXli4h0OVYfH7zHj8d7/HgAnK2tNO7YQe3GjdSlbKJu40Yac3JozM2lMTeXin//GwCLtzce8fEHws/4YeCubsM/KtgzmOMjj+f4yOMBaGxpZFvJtoPCz+K6YjYXb2Zz8Wbe2/4eAL28erXf85kQkkBsQCx2i6bQioiIiPwnBZ0iclhZLAYJEf4kRPhz8/EDKK5uYGlGEUnpRSzNKKKirokvNu3mi027MQwYFu5PYmwwE6MDaP3lhmwREfmDDIsFt/79cevfnx5TpwLQXFZG3aZNB7o+U1Npra6mZsUKalasaHuigSM6mpCAACobm/AeNRJHVJTumPyDHFaHeVdnSAIzBs/A6XSyu2Z3+1H3lMIUMsoy2Fuzl701e/k21xw25W51Z1DgoAMT3kPiCXAPcPFuRERERFxPQaeIHFFB3m6cMyKcc0aE09zSSkpBOUnphSSnF7F1dyWbCsrZVFDOnO/B22ZlSd1mjh3Ui0n9g/D3dLi6fBGRLsvWowc+kyfjM3kyAM7mZhoyMg7q+mzauZPGrCz8gcI1aygErP7+eCQk0GP6BXgfc4wLd9D5GYZBmHcYYd5hnNbPHDZV21TL1pKtB014r2ioYEPhBjYUbmh/bh+fPu33fMYHxxPjH4PVYnXVVkRERERcQkGniLiMzWphVFQAo6ICuP2kOPZV1rMk3ZzivjSziOqGFj7btIfPNu3BYsCIPj1IjDPv9hwU6qsOIhGRw8iw2XAfNAj3QYPgwgsBaC4qomr9erZ9+m96V1XRsHUrLeXlVCcnU52cTPDNNxF49dX6+nwIedo9Gd1rNKN7jQbMYVO5lbntXZ+bijaRVZ5FflU++VX5/N+O/wPAy+7F0KCh7V2fQ4OH4uvwdeVWRERERA47BZ0i0mH09HVn6ugIpo6OoKaugZc+/pb6gGiWZhaTsa+adXllrMsr4+lv0wnxcWsfaDS+fxC+7rqrTETkcLMFB+N93HEUNzQw5tRTsTqdNGzfTvmn/6Z8/nyK5vyDhqwdhD76CBbd5XlYGIZBX7++9PXry9n9zwagoqGCzcWb24+7pxalUtNUw6o9q1i1Z5X5PAyi/aMPmvAe5atrB0RERKRrUdApIh2Sw2ahv5+TU08awH2nD2ZXeZ050CitiBVZxRRWNTB/XQHz1xVgsxiMiupBYmwIiXEh9A/x1g9uIiJHgMXhMIcVxcfjPnAgex99lMqFC2nMzyd87gvYQ0JcXWK34Ofmx4SwCUwImwBAS2sLWeVZBw05KqgqIKs8i6zyLP6V+S8A/N38Dwo+BwcOxtPu6cqtiIiIiPwpCjpFpFMI8/fgwrGRXDg2kvqmFtbmlpKUVkRyeiHZxTWsyi5lVXYpj3+dRpi/B5Njg0mMDeHomEA8HfpSJyJyuPW4YBqOqCh23nQT9amp5E6dRvi8uXgMHuzq0rodq8VKbEAssQGxTI01h02V1JWYwWdRCpsKN7G1ZCvlDeUs2bmEJTuXmM8zzOfFB8dzRvQZDAka4sptiIiIiPxu+ulfRDodd7uVif2Dmdg/mAemDCK3uMbs9kwvYmV2CbvK63h/dT7vr87HYbUwtl9Ae7dn3yAvV5cvItJleR01lr4L5lNw7XU0ZmeTd+FF9H7ySXxPOtHVpXV7gR6BHNvnWI7tcywATS1NpJWmtQ842li4kcLaQraVbGNbyTY+Tv+YW0fdyoUDL9QpCREREek0FHSKSKcXFeTFpUF9uXR8X+oaW1iZXUxyehGL0wrZWVbHssxilmUW8/DCbUQGerbf7XlUv0Dc7ZpIKyJyKDkiI4n66EN2zbqVmuXL2XXTTTTceANB116rwKwDsVvtDA0eytDgoVzMxQDsrdlLSlEK3+R8ww/5P/Dk2idJLU7lwXEP6ki7iIiIdAoKOkWkS/FwWDk2rifHxvXkoTOc7Cja3+1ZyJqcUvJKann7x1ze/jEXd7uFo6ODSIwNZnJsCBEB+iFORORQsPr6EvHyS+x76inK3v0nxc+/QGPWDkIf+7uGFHVgvbx6cbLXyZwUeRIfpH3AM2uf4eucr8ksy2RO4hwifSNdXaKIiIjIL1LQeRjMmzePefPm0dLS4upSRLo1wzCICfEmJsSbKyb2o7qhmRVZxe1DjfZW1rM4rZDFaYXAVmJCvJk8IJjEuBBGRwXgsFlcvQURkU7LsNnodc89uEXHsPeRR6j86isaCwoInzsXe08NKerIDMPgwoEXMjBgILcuuZWs8iwuWHgBj014jMQ+ia4uT0REROR/UtB5GMycOZOZM2dSWVmJn5+fq8sRkTbebjZOGtyLkwb3wul0kr6viqS0IpLSC1mfV0ZWYTVZhdW8vjwHL4eV8TFBJMaZx9xD/TxcXb6ISKfUY9pUHFFR7LrxRuo3byZ36lTCX5ynIUWdwIieI1hw+gJuW3IbGwo3cGPSjVw59EpmJszEatHVLyIiItLxKOgUkW7JMAzievkS18uXaydHU1HXxPLMYpLSC0lOL6K4uoHvtu3ju237AIjr5UNiXAiJsSGM6OOPzapuTxGR38pr7BiiFsyn4LqZNO7YYQ4peuIJfE8+ydWlya8I9gzm9ZNe59l1z/L+9vd5bfNrbC3ZypMTn8Tf3d/V5YmIiIgcREGniAjg52HntGGhnDYslNZWJ1t3V7aFnoVsLCgnbW8VaXureCl5Bz7uNiYNCGbygGCOiQ0mxEf3zYmI/Jr2IUW33krN0mXsuvlmGm64nqDrrtOQog7ObrFz15i7GBo0lIdWPsSPu39k2sJpzE6czeBAdeaKiIhIx6GgU0TkP1gsBkPD/Rga7seNx/WntKaRZZlFJKUVsiSjiLLaJr5M3cOXqXsAGBrmZw40igshPtwfq0U/sIuI/Byrjw8RL71E4VNPU/rOOxS/MJfGHTsIfewxDSnqBE7rdxr9e/Tn5qSbKagq4JKvLuG+o+7j7P5nu7o0EREREUBBp4jIrwrwcnBmQhhnJoTR0upk085yktMKSUovYvOuivb1/OIsenjaOaZtoNGk/sH08HK4unwRkQ7FsFrpefddOGKi2fvQw1R+9TWN+QWEz5unIUWdwIAeA/jo9I+4d9m9JO9M5oEfHyC1OJW7x9yNw6p/80RERMS1FHSKiPwOVovBiD49GNGnB7NOjKWwqp4l6UUkpxexNNPs9vwsZTefpezGMCAhwp/EWPNuz8G9fbGo21NEBIAe55+PIzKSXTfeRP2WLeSefz7h8+bhMXSIq0uTX+Hr8OUfx/6D11JfY17KPD7J+IT00nRmT55NL69eri5PREREujFN0xAR+RNCfNw5f1QE8y4cwYb7T2DB1eO4dnI0cb18cDphY345sxdlMGXucsY89gO3fbyJL1P3UFHX5OrSRURczmvMGKI+XoAjJprmwkLyLr6Yyq+/dnVZ8htYDAtXx1/Ni8e/iK/Dl83Fm5n6xVRW71nt6tJERESkG1PQKSJyiNitFsb0DeDOk+P45uZJrLz7WB4/ZygnDuqJl8NKcXUDn6zfycwPNjDikUVMfWUlLyXvIG1vJU6n09Xli4i4hCMigqiPPsJr0kSc9fXsumUWRXPn6etiJzEhbALzT5/PwICBlDWUcdWiq3hzy5v6/ImIiIhLKOgUETlMQv08mD6mD69eMooND5zA+1eM5cqJfYkJ8aal1cmanFKe/CaNk+cs4+gnFnP3p6l8u3Uv1Q3Nri5dROSIsnp7E/HSSwRceikAxXPnsmvWLFrr6lxbmPwm4T7hvHvKu5wZfSatzlaeW/8cs5JnUd1Y7erSREREpJvRHZ0iIkeAm83K+JggxscEce9pUFBaS3K6OdDoxx3F7Kmo58M1BXy4pgC71WBM3wASY0OYHBtCdLAXhqG7PUWkazOsVnredSduMdHseehhqr7+hryCnYTPm4u9Z09Xlye/wt3mziPjH2FY8DAeX/M43+d/z46KHcyZPId+/v1cXZ6IiIh0Ewo6RURcICLAk4vHRXHxuCjqm1pYlV1CcnoRSemF5JXUsiKrhBVZJTz65XYiAjzaBxod1S8QD4fV1eWLiBw2/uedhyMykp033GgOKTrvfMJfnIfH0KGuLk1+hWEYTI2dSlxAHLck30JORQ7Tv5zOI+Mf4cSoE11dnoiIiHQDOrouIuJi7nYrk2NDePCMwSy5PZGk2ybzwOmDmNg/CIfVQkFpHe+uzOOyt9eS8PB3zHhzDW+vyCGvpMbVpYuIHBaeo0cfGFJUVETeRRdT+dVXri5LfqNhwcNYcPoCxvQaQ21zLbcuuZXZ62bT3KqrWUREROTwUkeniEgH0zfIi74T+nL5hL7UNjbzY1YJSemFJKcXsau8jiUZRSzJKOLBL7bRL8iLybEhJMYFM6ZvAG42dXuKSNewf0jRrltvpWbJUnbNupWGrB0EXT8Tw6Lf1Xd0gR6BvHLCKzy/4Xne2voWb219iy0lW3h60tMEegS6ujwRERHpohR0ioh0YJ4OG8cP6snxg3ridDrJLKwmKa2QpPRC1uWWkV1cQ3ZxDm+uyMHDbt4DmhgXzOTYEML8PVxdvshvs28rBMeBRUG9HMzq7U3Eiy9S+MyzlL71FsUvvkjDjh30fuJxLB76GtfR2Sw2Zo2axZCgIdy/4n7W7l3L1IVTmT15NvHB8a4uT0RERLogBZ0iIp2EYRgM6OnDgJ4+XH1MNFX1TazIKiYpzbzbs7Cqge+37+P77fsAGNDTu32g0aioHtit6oCSDqhqH7x0NLj5QeTREDUB+k6EnkNBXXtC25CiO+8whxQ9+BBV335LXkEB4S/Ow96rl6vLk9/gxKgTifGP4ebkm8mpyOHSby7l7jF3c/6A8zVsT0RERA4pBZ0iIp2Uj7udk4eEcvKQUJxOJ9v2VJoDjdIK2ZBfRsa+ajL2VfPK0mx83GxM6B9EYmwIx8QG09PX3dXli5hKssyQs6ECMr42F4C7vxl6Rk2AqIkQMkjBZzfnf+65B4YUbdtG7vlTCZ83F49hw1xdmvwG/fz78eFpH3L/ivtZlLeIR1Y9wqaiTdx/1P242/RvkoiIiBwaCjpFRLoAwzAY3NuPwb39mJkYQ3ltI0szi0lOL2RJehElNY18vWUvX2/ZC8Dg3r5t3Z7BJET4Y1O3p7hK1Hi4Mwf2bILc5ZC7DPJWQn05pC00F4BnIESOh76TzPAzOA7UCdbteI4aRdTHC9h57XU0ZGaSd/ElhD72d/xOO83Vpclv4GX34tljnuXtrW8zZ8Mc/m/H/5FZlsnsybMJ9wl3dXkiIiLSBSjoFBHpgvw9HZwR35sz4nvT2upk864KktILSUovInVnOVt3V7J1dyVzk7Lw87AzaUAwibHBHDMgmEBvN1eXL92NxQphI8w1/kZoaYY9KZCz1Aw/81dBbQls/z9zAXgFH+j2jJoIQf0VfHYTjvBwIj/8gN233U51cjK7b72Nxh07CLr+eg0p6gQMw+CyIZcxKHAQty+5ne2l25m2cBpPTnqSCWETXF2eiIiIdHIKOkVEujiLxSA+wp/4CH9uPn4AxdUNLM0oIim9iKUZRVTUNfHFpt18sWk3hgHDwv1JjA0mMTaEoWF+WCwKj+QIs9ogfJS5Js6ClibYtQFyl0LOMihYDTVFsPXf5gLw7nXgfs+oiRDQT8FnF2b19iZ83lwKZ8+m9I03KX7xJRqy2oYUeXq6ujz5DcaGjmXBlAXMSp7F5uLNXPf9dVyXcB1XDbsKi6HAWkRERP4YBZ2Hwbx585g3bx4tLS2uLkVE5L8EebtxzohwzhkRTnNLK5t2lrcPNNq6u5JNBeVsKihnzveZBHo5OGZAMJPjQpjUPwh/T4ery5fuyGqHPmPNNel2aG6AXevN0DN3GRSsgeq9sOUTcwH49D4QevadCD2iXLoFOfQMq5Wet9+OW3QMe/72N6q++47cnQVEvPiihhR1Er28evH2yW/zxJon+DjjY+alzGNL8RYem/gYvg5fV5cnIiIinZCCzsNg5syZzJw5k8rKSvz8/FxdjojI/2SzWhgZGcDIyABuOymWfZX1LEk3Q89lmcWU1DTy6cZdfLpxFxYDRvTpQWKcebfnoFBfTcsV17C5mRPaI48G7oSmeti51gw9c5aZ/71qN6TONxeAX5+DOz79I1y6BTl0/M85G0dkH3ZefwMN27aTc/75RMydi0d8vKtLk9/AYXXwwLgHGBo0lEdXPcqSnUu4YOEFPDf5OWIDYl1dnoiIiHQyCjpFRKRdT193po6OYOroCBqbW1mfV0ZyeiFJ6YVk7KtmXV4Z6/LKePrbdHr6ujF5QAiJccGMjwnCx93u6vKlu7K7mwFm34mQCDTWws41Bzo+d62HinzY9IG5wOzwjJoAUZPM5/n2duUO5E/yHDmSqI8/Zue11/5kSNFj+J2uIUWdxdn9zyY2IJZbkm6hoKqAi766iL8d/TdO73e6q0sTERGRTkRBp4iI/CyHzcK46EDGRQdy96kD2VVeZ4aeaUWsyCpmX2UD89cVMH9dATaLweioACbHBpMYF0L/EG91e4rrODyh32RzATTWmAON9nd87t4IZbnm2vie+ZiA6LaOz0lmx6dPT9fULn+YIzyMyA8/ZPftt1OdlMTu226jYUcWwTfcoCFFncSgwEHMP30+dy67kx93/8jdy+5mc9Fmbht1G3arfpkmIiIiv05Bp4iI/CZh/h5cODaSC8dGUt/UwtrcUpLSikhOLyS7uIaV2SWszC7h8a/TCPP3MEPP2BCOjgnE06F/bsSFHF4Qc5y5AOorzYFGOUvN8HPPJijdYa4N75iPCRrQNtG9bbK7d7Dr6pffzOrtRfjcFyh67jlKXn+DkpdepjFrB72ffEJDijoJf3d/XjzuRV7c9CKvpr7KB2kfsL10O88e8yzBnvr/oYiIiPwy/eQpIiK/m7vdysT+wUzsH8wDUwaRW1zTdsS9iFXZJewqr+P91fm8vzofh9XC2H4BJMaGkBgXQt8gL1eXL92duy/0P8FcAHXlkL8Scpeb4efezVCcYa51b5iPCR7Ydr/nBIicAF6BLitffplhtRJy2204omPY+8ADVC1aRO6unUTMm4c9NNTV5clvYLVYuWH4DQwJHMI9y+9hY+FGpi6cyjPHPMPIniNdXZ6IiIh0YAo6RUTkT4sK8uLSoL5cOr4vdY0trMouISm9kMVphewsq2NZZjHLMot5eOE2IgM9SYw1Bxod1S8Qd7vV1eVLd+fhD7GnmAugthTyfjSDz9xlsG8LFG0315pXzcf0HPKTjs/x4NHDZeXLz/M/+6z/GFI0lYh5GlLUmST2SeSj0z/i5qSbySrP4opvr+DWUbdy4cALdT2KiIiI/CwFnSIickh5OKwkxpndmw+d4WRHUU37QKM1OaXkldTy9o+5vP1jLu52C0dHB5EYG8zk2BAiAnS0VDoAzwAYeLq5AGpKIG9523Cj5WbguW+LuVa/BBjQa+iB+z0jx4G7n0u3ICbPESPo+/ECCq69joaMDHNI0d//jt8UDbjpLCJ9I3n/1Pd5cOWDfJ3zNU+ufZLU4lQeHPcgnnb9myEiIiIHU9ApIiKHjWEYxIR4ExPizRUT+1Hd0MyKrGKS0827PfdU1LM4zez8hK3EhHiT2Ha356ioABw2DRCRDsArEAadaS6A6iKz0zO3LfgszoC9qeZaORcMC4TGm6Fn30nQ5yhw83HtHroxe1gYkR98wO477qB68WJ23347DVlZBN90o4YUdRKedk+enPgkw4KG8ey6Z/k652syyzKZkziHSN9IV5cnIiIiHYiCThEROWK83WycNLgXJw3uhdPpJH1fFUlpRSSlF7I+r4yswmqyCqt5bVkOXg4r42OCSIwzj7mH+nm4unwRk3cwDDnHXABVew/c75m73BxqtHujuX58Hgwr9B7edsfnRDP4dOiu2iPpwJCiOZS89holr7xCY/YOej/xBBYvfS46A8MwuGjQRQwMHMhtS24jqzyLCxZewOMTH2dyxGRXlyciIiIdhIJOERFxCcMwiOvlS1wvX66dHE1FXRPLM4tJSi8kOb2I4uoGvtu2j++27QMgrpePeSQ+NoQRffyxWdWJJR2ETy8Yep65ACp2td3v2RZ8luXCrnXmWv4cWGwQNrKt43MiRIwFu4L8w82wWAi5dRZuMdHsue9+qhZ9T27BRUS8OA97796uLk9+o5E9R7Lg9AXcuuRWNhZu5IbFN3DVsKu4Lv46rBbd+SwiItLdKegUEZEOwc/DzmnDQjltWCitrU627q5sCz0L2VhQTtreKtL2VvFS8g583G1MGhDM5AHBHBMbTIiPu6vLFznALwzip5kLoDy/reOz7bh7RQEUrDbXsmfA6oCwUQc6PsNHg12v6cPF78wzsUf0YecNN9CQlkbO1GmEv/A8nsOHu7o0+Y2CPYN548Q3eHb9s7y//X1eTX2VLcVbeHLik/i7+7u6PBEREXEhBZ0iItLhWCwGQ8P9GBrux43H9ae0ppFlmUUkpRWyJKOIstomvkzdw5epewAYGuZnDjSKCyE+3B+rRdN4pQPx7wMJfzGX02l2eO6f6J6zDKp2Q/6P5lryJFjdIGLMgY7PsFFgc7h6F12K54jh9F0wn4LrZtKQnk7+jEsJffQR/M44w9WlyW9kt9q5a8xdDAkawkM/PsSPu39k2sJpPJf4HIMCB7m6PBEREXERBZ0iItLhBXg5ODMhjDMTwmhpdbJpZznJaYUkpRexeVdF+3p+cRY9PO0cMyCYxLgQJvUPpoeXAiLpQAwDAvqaa8TFZvBZmn0g9MxdBtX7Dgw7SgZsHtBnLERNgKhJEDYCrHZX76TTs4eFEfXB++y6806qv/+B3XfcSUPWDoJvvklDijqR0/udTn///tySfAsFVQVc/NXF3HfUfZzd/2xXlyYiIiIuoKBTREQ6FavFYESfHozo04NZJ8ZSWFXP0gzzbs+lbd2en6Xs5rOU3VgMSIjwJzE2hMS4EAaF+mJRt6d0JIYBgdHmGnmpGXwWZx481b2mCLKTzQVg9zIHGkVNMKe6hyaAVd/S/REWLy/Cn3+eojn/oOTVVyl59VUasncQ9uSTGlLUicQGxPLR6R9xz7J7WLJzCQ/8+ACpxancPeZuHFb9sktERKQ70XfFIiLSqYX4uHPeyHDOGxlOc0srG/LLSUovJCmtkLS9VWzIL2dDfjnPLsogyNuNybHBJMaGMKF/EH4e6oqTDsYwIHiAuUb/1Qw+i9IOdHvmLoe6Utjxg7kAHD4QOc486h41AULjQUNZfjPDYiFk1i3tQ4qqv/+B3As1pKiz8XX48vyxz/Nq6qu8mPIin2R8QnppOrMnz6aXVy9XlyciIiJHiIJOERHpMmxWC2P6BjCmbwB3nhzHnoo6ktOLSE4vZHlmMcXVDXyyfiefrN+J1WIwMrJHW7dnMLE9fTAMdXtKB2MYEDLQXGOvgtZWKNx2IPTMXQ715ZD5nbkA3Pwg8ugDw416DgEdxf5VfmecgT0igp3Xtw0pOn8q4XNf0JCiTsRiWLgm/hqGBA3hzqV3srl4M1O/mMrTxzzN2NCxri5PREREjgAFnSIi0mWF+nkwfUwfpo/pQ2NzK+tyS81uz/QisgqrWZNTypqcUp78Jo1QP3cmxwYzOTaE8TFBeLvpn0jpgCwW6DXEXEddC60tsG9LW8fncshbAQ0VkPG1uQDc/dvu92wbbhQ8UMHn/+A5fDh9P15gDilKSyP/khnmkKIzz3R1afI7TAibwPzT5zMreRbbS7dz1aKruGnETVw2+DL9QktERKSL009xIiLSLThsFo6OCeLomCDuPQ0KSmtJbgs9f9xRzJ6Kej5cU8CHawqwWw3G9A0gMTaEybEhRAd76Ydj6ZgsVvOoemg8HH29GXzu2XRguFH+SrPjM22huQA8AyFyvHm/Z9RECI41O0cFAHvv3kS9/96BIUV33kVDVhbBt9yiIUWdSLhPOO+e8i6PrnqUz3d8znPrn2Nz0WYeGf8I3g5vV5cnIiIih4mCThER6ZYiAjy5eFwUF4+Lor6phVXZJSSnF5GUXkheSS0rskpYkVXCo19uJyLAwzziHhvCUf0C8XDo/kPpoCxWcyp72AgYfxO0NJnBZ85SM/zMXwW1JbD9/8wF4BX8k47PSRAY0+2Dz/YhRf94npJXXqHktddpyM4h7CkNKepM3G3uPDL+EYYFD+PxNY/zff737KjYwZzJc+jn38/V5YmIiMhhoKBTRES6PXe7lclt3ZsPMpic4hqS0gpJSi9kdXYpBaV1vLsyj3dX5uFms3BUv0ASY4NJjAshMlChh3RgVjuEjzLXxFnQ3Ai7N0LuUrPjs2C1OdV967/NBeDdq22ie9sdnwH9umXwaVgshNxyszmk6N77qP7hB3L/cqE5pCgszNXlyW9kGAZTY6cSGxDLrORZ5FTkMP3L6Twy/hFOjDrR1eWJiIjIIaagU0RE5D/0DfKi74S+XD6hL7WNzfyYVUJSeiHJ6UXsKq9jSUYRSzKKePCLbfQL8mJy20CjMX0DcLOp21M6MJsD+ow116TbobkBdq0/MNW9YA1U74Utn5gLwDfs4Ds+vbtXyOc3ZQqOiAgKrr+BhvR0cqZOI/yFF/AcoSFFnUl8cDwLTl/A7UtvZ+3etdy65FYuK76Ma4de6+rSRERE5BBS0HkYzJs3j3nz5tHS0uLqUkRE5E/ydNg4flBPjh/UE6fTSWZhtXm3Z1oRa3NLyS6uIbs4hzdX5ODpsHJ0dBCJceZQozB/D1eXL/LLbG7mhPbIo4E7oakedq49cMfnzrVQuQtS55sLsPlFMNwahZFaCdGTwT/CpVs4EjwSEg4MKdq+nfwZM+j1yMP4n3WWq0uT3yHQI5BXT3iV5zc8z1tb3+KtrW+xuXgzx7ce7+rSRERE5BBR0HkYzJw5k5kzZ1JZWYmfn5+ryxERkUPEMAwG9PRhQE8frpoUTVV9EyuyiklKM+/2LKxq4Pvt+/h++z4ABvT0JjE2hIkxAbS0urh4kd/C7m52bfadCIlAY615vD13uRl+7lqPUVFAHwrgi2Xmc3pEHbjfM2oC+PZ25Q4OG3toKFHvv8fuO++iatEi9tx1N437hxRZ1cndWdgsNmaNmsWQoCHcv+J+1u1bR4aRwcDigYwIHeHq8kRERORPUtApIiLyB/m42zl5SCgnDwnF6XSybU+lOdAorZAN+WVk7KsmY181ryzNxt1q5duqFI4b2ItjYoPp6evu6vJFfp3DE6ITzQXQUE1zzgqyk94lxroHy54UKMs118Z/mo8JiD5wv2fURPDp6aLiDz2Lpydh/5hD0QsvUPLSy5S8/gYNO7Lp/fTTWL11X29ncmLUicT4x3BT0k3kVuby1+//yt1j7ub8AedjdMM7aUVERLoKBZ0iIiKHgGEYDO7tx+DefsxMjKG8tpFlmcVtd3sWUlrTxLfbCvl2WyEAg3v7khgbwuTYYBIi/LFZLS7egchv4OaNM/pYtqfX0/fUU7G01JmT3HOXml2fezZB6Q5zrX/bfE7QgAP3e0ZOAO9gl27hzzIsFkJuugm3ftHsufdeqpOSyPvLXwh/8UUc4d3r/tLOrp9/P9496V2u+ewatjVt45FVj5BalMp9R92Hu02/jBIREemMFHSKiIgcBv6eDqbE92ZKfG8aGhp55ZOvaQ6KZUlWCak7y9m6u5KtuyuZm5SFn4edSQOCSYwN5pgBwQR6u7m6fJHfxt0XBpxoLoC6cshf2TbcaCns3QLFGeZa94b5mOCBP+n4nACeAS4r/8/wm3I6jj4RFFx/PQ0ZGeROnUr4C8/jOXKkq0uT38Hb7s10z+kU9S1i7qa5fL7jczLKMpg9eTbhPuGuLk9ERER+JwWdIiIih5nFYhDpDaceG82sk+Iorm5gaUYRSelFLM0ooqKuiS827eaLTbsxDBgW7k9ibDCJsSEMDfPDYtExSukkPPwh9hRzAdSWQt6PB4YbFW6Fou3mWvOq+ZieQ37S8Xk0ePRwWfm/l0d8PH0//piC666jYdt28i69jNCHHsL/nLNdXZr8DoZhcOmgSxkaMpQ7ltzB9tLtTFs4jacmPcX4sPGuLk9ERER+BwWdIiIiR1iQtxvnjAjnnBHhNLe0smlneftAo627K9lUUM6mgnLmfJ9JoJeDY9pCz0n9g/HztLu6fJHfzjMABp5uLoCaEshb3tbxuQyK0mDfFnOtfgkwoNfQtsFGEyFyHLh37MGO9l69iHrvPXbfdTdV333HnnvuoWFHFiGzZmlIUSdzVOhRzD99PrOSZ7GlZAvXfn8tMxNmcuWwK7EYul5ERESkM1DQKSIi4kI2q4WRkQGMjAzgtpNi2VdZz5J0M/RclllMSU0jn27YxacbdmExYESfHiTGmXd7Dgr11dAM6Vy8AmHQmeYCqC48MNE9ZxmUZMLeVHOtnAuGBUITzCPufSdBn6PAzcelW/g5Fk9PwuY8R/HcuRS/+BKlb7xJ445sej/zjIYUdTKh3qG8c8o7PLHmCT7O+Ji5KXPZUryFv0/8O74OX1eXJyIiIr9CQaeIiEgH0tPXnamjI5g6OoKmllbW5ZaRnF5IcnoR6fuqWJdXxrq8Mp7+Np2evm5MHhBCYlww42OC8HFXt6d0Mt4hMOQccwFU7TWDz5ylZvhZmg27N5jrx+fBsELYCDP4jJpoBp+OjhEkGhYLwTfeiCM6mj333Et1cjJ506cT/tJLGlLUyTisDh4Y9wBDg4by6KpHSd6ZzPSF03ku8TkG9Bjg6vJERETkFyjoFBER6aDsVgvjogMZFx3I3acOZFd5HcnphSSlFbEiq5h9lQ3MX1fA/HUF2CwGo6MCmBwbTGJcCP1DvNXtKZ2PTy8Yep65ACp2tXV8LjU7PsvzYOdacy1/Dix2CBvZNtxoAkSMBbuHS7fgd9ppOCIi2DnzehoyM8k9/3zC576gIUWd0Nn9z2ZAwABmJc0ivyqfC7+8kAePfpDT+p3m6tJERETkf1DQKSIi0kmE+Xtw4dhILhwbSUNzC2tySklKKyI5vZDs4hpWZpewMruEx79OI8zfwww9Y0M4OiYQT4f+yZdOyC8M4qeZC6A8v+1+z7bj7hUFULDKXEufBqsDwkcfmOgePhrs7ke8bI9hw4j6eAE7r5tJ/bZt5pCiBx/E/9xzjngt8ucMDhzM/NPnc8fSO1i5ZyV3LbuLzcWbuXXUrdgt6qIXERHpaPRTj4iISCfkZrMysX8wE/sH88CUQeSV1JDcdrfnyh0l7Cqv4/3V+by/Oh+H1cLYfgEkxoaQGBdC36COcdRX5Hfz7wPDLzSX0wlluWbgmds24KhqN+StMNcSwOZuhp37hxuFjQSb44iUau/Vi8j324YUffste+69l4asLEJuu1VDijoZf3d/Xjr+JealzOO1za/x/vb32VayjWePeZZgz2BXlyciIiI/oaBTRESkC4gM9GLG0V7MODqKusYWVmWXkJReyOK0QnaW1bEss5hlmcU8vHAbUYGeTI41Bxod1S8Qd7tCF+mEDAMC+pprxCVm8FmafeB+z9zlUL2v7b8vM59j84A+Y83Qs+8k6D0crIevK8/i4UHYc7MpnjuP4hdfpPStt2jMzqb3s89g9fY+bB9XDj2rxcqNI25kaNBQ7ll+DxsLNzJ14VSePeZZRvQc4eryREREpI2CThERkS7Gw2ElMc7s3nzoDCc7imrMuz3TC1mTU0puSS1v/5jL2z/m4m63cHR0ENNGR3DS4F6uLl3kjzMMCIw216jLzOCzOPPA/Z65y6G2GLKTzQVg9zIHGvWdCFGTIDQerIf222NzSNENuMVEs/vue6hessQcUvTiizgiIg7px5LDL7FPIh+d/hE3J91MVnkWf/32r9w2+jb+EvcX3YssIiLSASjoFBER6cIMwyAmxJuYEG+umNiP6oZmVmQVk5xu3u25p6KexWlm5+fDZw7mknFRri5Z5NAwDAgeYK7RV5jBZ1FaW+i5FHJXQF0p7PjBXAAOH4gc19bxORF6DQPLoel49j31VOwREey8biYNmVnkTp1G+AvP4zlq1CF5/3LkRPpG8v6p7/Pgjw/yde7XPLHmCVKLUvnbuL/hafd0dXkiIiLdmoJOERGRbsTbzcZJg3tx0uBeOJ1O0vdV8c+Veby/Op8HPt8KoLBTuibDgJCB5hp7FbS2QuE281h7zjLIWw71FZD5nbkA3Pwg8ui2js+J0HMIWCx/uASPoUOJ+uRjc0jR1q3kXXY5oQ/+Df9zzz1Em5QjxdPuyZOTnmRY8DCeWfcMX+V8RUZZBnMS5xDpG+nq8kRERLotBZ0iIiLdlGEYxPXy5dGzhuDtbuOVJdk88PlWDOBihZ3S1Vks0GuIuY66FlpbYN+Wto7PZZD3IzRUQMbX5gLw6AGR4w90fAYP/N3Bp71nTyLf+ye777mHqq+/Yc+999GQmUXI7bdpSFEnYxgGFw26iIGBA7k1+VayyrOYvnA6j018jMkRk11dnoiISLekoFNERKSbMwyDu06OAye8sjSb+z/fCobBxUepK0m6EYvVvKMzNB6Ovh5ammFv6oGOz/yVUFcGaQvNBeAZCFETzOAzaiIEx5qdo7/2oTw8CJs9m+LoGIrnzqX07bdpyMkm7NlnNaSoExrZcyQLpizgtiW3sbFwIzcsvoGrhl3FdfHXYT1EVx+IiIjIb6OgU0RERMyw85Q4nMCrS7O5/7MtGMBFCjulu7LaIGyEucbfBC1NsDvlwBT3/FVQWwLbPjcXgFdIW/A5wZzqHhjzP4NPwzAIvn6mOaTorrupWbKU3AsuIOKllzSkqBMK8QzhjRPf4Jl1z/BB2ge8mvoqW4u38sTEJ/B393d1eSIiIt2Ggk4REREBzODl7lPicDqdvLYsh/s+24JhwIVjFXaKYLVDxGhzTZwFzY2we8OBjs+C1VBTCFs/NReAd68D93tGTYCAfv8VfPqefDL2sHB2zpxJY9YOcs+fStjz/8BrzBgXbFL+DLvVzt1j72ZI0BAeXvkwK3av4IIvL2D25NkMChzk6vJERES6hT9+m7qIiIh0OYZhcM+pA7lyYl8A7v33Fj5Yne/iqkQ6IJsD+hwFk26HGf8Hd+XDpV/B5LvNYNPqBtV7YfPH8MWN8MIIeG4wfHo1bHwPyvLa35XH0CFEffwx7kOG0FJeTv7lf6Xs449duDn5M6ZET+G9U98jwieCXdW7uPiri/l35r9dXZaIiEi3oI5OEREROcj+sNPphNeX53DPvzcD8JexfVxcmUgHZnODqPHmAmiqg51r24YbLTf/e+UuSP3IXAB+fdo7Pu19JxL5z3fZc++9VH71NXvvf4DGrCxC7rhDQ4o6odiAWD487UPuXX4vS3Yu4YEfH2Bz8WbuGnMXDqvD1eWJiIh0WQo6RURE5L8YhsG9pw3ECbzRFnYaBkwfo7BT5Dexe5j3dPadZL7dWGseb89tCz53rYeKfEh531yApUcUvSdNwOGdSPGCJErfeZeG7BzCZj+L1cfHhZuRP8LPzY/nj32eV1Nf5cWUF/k442PSStOYPXk2vbx6ubo8ERGRLklH10VERORnGYbBfacN5PLx5jH2uz/dzEdrdIxd5A9xeEJ0Ihz3APz1O7gzDy76F4y/GcJGgWGFslyMlPcItrxP2NGlGDaoWbaM3LNOo3H7BlfvQP4Ai2HhmvhrmHfcPHwdvmwu3sy0hdNYvWe1q0sTERHpktTRKSIiIv+TYRjcf/pAnDh5a0Uud31qdnZOG63OTpE/xc0bYo43F0B9pTnJPXcp5CzD10jF7l3EzmUBNO4qInfadMJO9cJrwuQDA468gly6BfntJoZP5KPTP2JW8izSStO4atFV3DziZi4dfCnGfwyoEhERkT9OQaeIiIj8IsMweOB0c2Jwe9iJwdTRES6uTKQLcfeFASeaC6CuHI/8lUSN/46dLy6ifh/kf15Lr10f0iP6DfMxIYPMae77p7p7BriufvlVET4R/POUf/LIqkf4vx3/x+z1s9lcvJlHxj+Cl93L1eWJiIh0CTq6fhjMmzePQYMGMXr0aFeXIiIickjsDzsvPToKpxPu/DSVBWsLXF2WSNfl4Q+xp2Cf9hyR367D96TjwWmwd60/e9P64mwFCrfBmldhwcXwVD94aQJ8czekfQl1Za7egfwMd5s7j45/lPvG3ofNYmNR3iKmfzmd7PJsV5cmIiLSJSjoPAxmzpzJtm3bWLt2ratLEREROWQMw+BvU/4j7FynsFPkcLO4u9N7zvME3XgDAGUpDRQUnEXLaa/C6CshOA5wwr7NsOpF+Ogv8GRfeGUSfHsvpH8D9RWu3YS0MwyDaXHTePvktwnxDCGnIofpX07nu9zvXF2aiIhIp6ej6yIiIvKb7Q87nU4n76zM485/pWIA54/SMXaRw8kwDIKvuw636Bh233UXNSvXkLuvmIiXXsRx2jNQXXhgonvOMijJhD2bzLVyLhgWCE1ou99zEvQZC26a5O5K8cHxzD99PncsvYO1e9dy65Jbuaz4Mm4ccSM2i35MExER+SP0L6iIiIj8LoZh8OAZg3EC767M445/pWIYBueNDHd1aSJdnu9JJ2IPD2PndTNpzM4md+o0wv7xD7yOGgtDzjUXQOUeM/TMXWau0mzYvcFcK/5hTnkPG2He79l3IkSMBYfuiTzSgjyCePWEV/nHhn/w9ta3eWvrW2wt2cpTk54i0CPQ1eWJiIh0Ojq6LiIiIr+bYRg8dMZgLj4qEqcTbv9kE5+s3+nqskS6BY/Bg4n6eAHuw4bRUlFB/hVXUDZ/wcEP8g2FYefDGc/DjRvhlq1w9isw/CLwjwRnC+xcC8tnwz/Phici4Y2TYPGjkL0Emupcs7luyGaxceuoW3nmmGfwsHmwZu8api2cRmpRqqtLExER6XTU0SkiIiJ/iGEYPHzmYAD+uSqP2z/ZhAGcq85OkcPOHhJC5LvvsOfe+6j88kv2/u1vNGRl0fPOOzBsP/Mtvl84xF9gLoDyfPOIe+4y8z8rd0LBKnMtfRqsDggffaDjM3w02NyO7Ca7mZOiTiLGP4abk24mtzKXS7+5lLvG3MX5A87HMAxXlyciItIpKOgUERGRP2x/2OnEyXur8rntk00YBpwzQmGnyOFmcXen9zNP49Y/hqI5/6Dsn/+kMSeHsNnPYvX1/eUn+/eB4Reay+mEstwDoWfuMqjaA3krzLXkCbC5m2Fn30lm+Bk2EmyOI7LP7iTaP5oPT/uQ+1bcxw/5P/DIqkdILUrlvqPuw93m7uryREREOjwFnSIiIvKnGIbBw2cMwemE91fnc+vHZth59nCFnSKHm2EYBF1zDY5+/dh9513ULF9O7rQLiHj5JRyRkb/1nUBAX3ONuMQMPkuzIWfpgfCzpvDAfZ8ANg9zoFHURDP87D0crPbDt9FuxNvhzXOTn+PNLW/y/Mbn+XzH52SUZTB78mzCffR1VURE5Jco6BQREZE/zWIxeOTMITiBD1bnc+uCTYDCTpEjxffEE3GEh1Nw3Uwac3LImTqN8H/Mweuoo37/OzMMCIw216jLzOCzOBNyl7Z1fC6H2mLITjYXgN0LIsdB1ARzqntoPFj1o8YfZRgGfx36VwYHDeaOJXewvXQ7F3x5AU9OfJLxYeNdXZ6IiEiHpWFEIiIickhYLAaPnjmE6WP60OqEWxds4rONu1xdlki34T5oEH0/XoB7/DBaKyrIv+JKyj766M+/Y8OA4AEw+gqY+g7cngXXrYJTnoaBU8CjBzTVQNb38P2D8Pqx8FRfeH8q/PgC7N4IrS1/vo5u6KjQo5h/+nyGBA6hoqGCa7+/llc2vUKrs9XVpYmIiHRI+jWriIiIHDIWi8HfzxoCOPlwTQGzFqRgGHBmQpirSxPpFmzBwUS++y577rufyi++YO+DD9GQmUXPu+/6+SFFf4RhQMhAc429ClpboXCr2emZswzylkN9BWR+ay4Adz+IHG8edY+aAD2HgEU9F79FqHcob5/yNk+seYJPMj5hbspcthRv4e8T/46v41fuYhUREelmFHSKiIjIIWWGnUNxOuGjtQXcMj8FUNgpcqRY3Nzo/dSTuEVHUzRnDmXvv28OKXpuNlY/v8PwAS3Qa6i5jrrW7N7cu9kMPnOXQd6PZvCZ/pW5wOwCjRx/YLhRcJyCz1/gZnXjb+P+xrCgYTy66lGSdyYzfeF0nkt8jgE9Bri6PBERkQ5DQaeIiIgcchaLwWNnDwUUdoq4gjmk6Goc0f3Yfced1Pz444EhRVFRh/eDW6zQO8FcR18PLc2wd9OB+z3zV0JdGaQtNBeAZ2Db/Z5tw42CBpido3KQs/ufzYAeA7gl+Rbyq/K58MsLefDoBzmt32muLk1ERKRDUNApIiIih8X+sNPphPnrzLDTMAzOiO/t6tJEug3fE07A8UHbkKLcXHKmXUD4nOfwGjfuyBVhtUHYSHNNuBlammB3yoHhRgWrobYEtn1uLgCvEDP47DvRHG4UGK3gs83goMHMP30+dy69k5V7VnLXsrvYXLyZW0fdit1id3V5IiIiLqXzISIiInLYWCwGj58zlKmjwml1ws0fbeSLTbtdXZZIt+I+cCB9F8zHIz7+wJCiDz90XUFWO0SMhom3wiWfwZ15cPm3kHif2c1pc4eaQtj6KSy8BeaOhNkD4V9XwPp3oDTbnATfjfVw78FLx7/ElUOvBOD97e9zxbdXUFRb5OLKREREXEtBp4iIiBxWFovBE+cMOxB2zk9hYarCTpEjyRYcTJ9338H3jCnQ0sLehx5m78OP4GxudnVpYHNAn6PgmNthxhdwVz5c+hVMvhsiJ4DVAVV7YPPH8MWN8PxweG4IfHo1bHwPyvJcvQOXsFqs3DjiRv6R+A+87d5sKNzA1IVT2bBvg6tLExERcRkFnSIiInLY7Q87zx8ZTkurk5s+SuHL1D2uLkukW7G4udH7yScJnjULDIOyDz6g4KqraKmocHVpB7O5QdR4mHwXXPalGXzO+AIm3QF9xoHFDpU7IfUj+Hwm/GMYzBkKn82ElA+hYqerd3BEHdvnWD487UNi/GMorivmr9/+lfe3v4+zm3e9iohI96Q7OkVEROSIsFgMnjx3GE7gk/U7ufGjjQCcNizUtYWJdCOGYRB01ZW4Rfdj1+13UPPjSnKnXUD4Sy/i1revq8v7eXYP80h730nm24215r2eucvMOz53b4DyfEh5z1wAPfq23e/Ztny79teZKL8o3j/1ff7249/4JvcbnljzBKlFqfxt3N/wtHu6ujwREZEjRkGniIiIHDHtYacT/rXBDDsNA04d2rVDCJGOxue444j64H0KrruOxtxcM+yc8xxeRx/t6tJ+ncMTohPNBdBQDQWr2qa6L4PdG6Esx1wb3jUfExjTFnq2TXb36em6+g8TT7snT016imHBw3h23bN8lfMVmeWZzJk8hz6+fVxdnoiIyBGhoFNERESOKKvF4KnzhuHEyacbdnHDh2Znp8JOkSPLPS6OvgsWsPP6G6hLSSH/yqvoee89BPzlL64u7fdx84aY480FUF8J+SsPdHzuTYWSLHOtf8t8TFAs9J2IEXE0jqZa19V+iBmGwcWDLmZgwEBuW3IbmWWZXLDwAh6b+BiTIya7ujwREZHDTnd0ioiIyBFntRg8fV485wwPo6XVyQ0fbuTrzbqzU+RIswUF0eedt/E78wxoaWHfw4+w9+GHcTY1ubq0P87dFwacBCc+ClcvgTty4IIP4aiZ0GsoYEBxOqx9Hdunl3PKluuxvToRvroDtn8BtaWu3sGfNqrXKBZMWUBCcAJVTVXcsPgGXtj4Ai2tLa4uTURE5LBSR6eIiIi4hNVi8PT58QB8utHs7JxrwMlD1NkpciRZ3NwIfeIJHDExFM1+jrIPPqQhJ4fwOXOw+vm5urw/z8Mf4k41F5hBZt4KyFmGM3cZRuE2jKLtULQd1rwCGNBzyIE7PiOPNt9HJxPiGcKbJ73J0+ue5sO0D3k19VW2Fm/liYlP4O/u7+ryREREDgt1dIqIiIjL7A87zx4eRnOrk+s/2Mg3W/a6uiyRbscwDIKuvJLweXMxPD2pXbmK3KnTaMjOcXVph55nAAycAqc+RfOVS/l6yFyaz3kTRl8BwXGAE/ZthlUvwkfT4ckoeGUSfHsvZHxrHo3vJOxWO/eMvYfHJjyGu9WdFbtXcMGXF7CtZJurSxMRETksFHSKiIiIS1ktBs+cH89ZCb3bws4NCjtFXMTn2GOJ+vADbL1DaczLI/eCC6hescLVZR1WjXZfnAPPgNOehZmr4bZMOO9NGHkZBPYHnLBnE6ycCx9MhScj4dVEWPQAZH5vDkPq4KZET+G9U98j3DucXdW7uOTrS/gs6zNXlyUiInLIKegUERERl7NaDJ6dmnBQ2PntVoWdIq7gHhtL348/xmP4cForKym46mpK33/f1WUdOd4hMORcmDIHblgHs9LgnNdhxCUQ0A+crbB7A6z4B7x/rhl8vn4CfP8Q7FgMjR1zuFFsQCwfnf4Rk8In0dDSwP0r7ufhlQ/T2NLo6tJEREQOGQWdIiIi0iHsDzvPbAs7Z76/ge8Udoq4hC0wsG1I0ZnmkKJHHmXPQw917iFFf5RvKAw7H854AW7cCLdshbNfgYSLwL8PtDbDzjWwfDb882x4og+8eTIsfhSyl0BTnat30M7PzY8Xjn2B6xKuw8Dg44yPueyby9hbo6+1IiLSNSjoFBERkQ7DajF49vx4zohvCzs/2MCibftcXZZIt2RxOAh94nFCbr8NDIPyDz8i/8qraCkvd3VpruUXDvEXwFnz4ObNcFMqnPkixE8H33BobYL8lbD0aXj3DDP4fOs0SHoccpdDc4NLy7cYFq6Nv5Z5x83Dx+FDanEq0xZOY/We1S6tS0RE5FBQ0CkiIiIdis1qYfbUeKbE96apxcl1769X2CniIoZhEPjXvxI+by4WT09qV60iZ9o0GrKzXV1ax9EjEoZfCGe/DLdsMbs+z3gBhk4Fn1BoaYS85bDkCXj7NDP4fGcKLHka8lZCs2uOjk8Mn8j80+cTFxBHaX0pVy26ire2vIXT6XRJPSIiIoeCgk4RERHpcGxWC8/9R9j5vcJOEZfxOfZYIj/8EHtYGE15+eROu4Dq5V17SNEfYhjmPZ4jLoFzX4NZ2+H69XD6HPPeT68QaK6HnKWQ9Ci8dbJ5x+e7Z8GyZ6FgDbQcuesBInwiePeUdzkj+gxana3MXj+bW5fcSk1TzRGrQURE5FBS0CkiIiId0v6w8/RhoTS1OLlWYaeIS7nHDiBqwXw8RoygtaqKgquuovSf76kD8JcYBgTFwKjLzEnut2XAzDVw6jMw6CzwDIKmWshOgh8ehjdOgCej4L1zYfkc2LUeWpoPa4keNg8eHf8o9429D5vFxqK8RUz/cjrZFeraFRGRzkdBp4iIiHRYNquFOdMSOO0nYecP2xV2iriKLTCQPm+/hd/ZZ0NrK/v+/nf2PthNhxT9EYYBwbEw5kqY+g7cngXXroRTnoKBU8CjBzRWQ9b38P3f4LVj4am+8P5U+PEF2J0CrS2HoSyDaXHTeOuktwjxCCGnIofpC6ezKG/RIf9YIiIih5OCzsNg3rx5DBo0iNGjR7u6FBERkU7PZrXwj2kJnDa0Lex8bwOL0xR2iriKxeEg9LG/E3L77eaQovnzyb/iSprLylxdWudjGNBzEIy9Gqa9B7dnwzXL4aTHIfY0cPeDhkrI/Ba+uw9ePcYMPj+cDitfhL2bobX1kJWTEJLA/CnzGdVzFLXNtcxKnsXsdbNpbj28XaUiIiKHioLOw2DmzJls27aNtWvXuroUERGRLsFmtTDnggROHdqLxpZWrvnnBpLSCl1dlki3ZQ4pupzwF+eZQ4pWryZ32gU07Njh6tI6N4sFeg2FcdfB9A/gjhy4agmc+CgMOBncfKG+AtK/gm/vhpcnwNP94KMLYfUrsG8b/MmrBII8gnjtxNeYMWgGAG9tfYurF11NSV3JodihiIjIYaWgU0RERDoFu9XCPy4Y3h52Xv3P9Qo7RVzMJzGRyI/ahhTltw0pWrbc1WV1HRYr9E6Ao2+Av8w3g88rF8PxD0HM8WD3groySFsIX98BL42Dp2NgwSWw5jUoSv9DwafNYuO20bfx9DFP42HzYM3eNUxbOI3UotRDv0cREZFDSEGniIiIdBr7w85Thvwk7ExX2CniSu4DBhD18QI8Ro2ktbqagquvpvTddzWk6HCw2iBsJEy4GS76F9yVB3/9Ho57APolgt0Taoth2+fw1W0wbww8MwA+vgzWvQnFWb8r+Dw56mQ+PO1Donyj2Fe7j0u/uZQF6Qv0uRURkQ5LQaeIiIh0KnarheenHxx2JivsFHEpW0AAkW++id8555hDih57nL1/e1BDig43qx0iRsPEW+GSz+DOPLj8W0i8D/pOAps71BTC1k9h4S0wdyTMHgj/uhLWvwOl2b8afEb7R/PhaR9yXJ/jaGpt4pFVj/DAjw9Q31x/ZPYoIiLyOyjoFBERkU5nf9h58uBeNDa3cpXCThGXMxwOQv/+KCF33GEOKVqwgPy/XqEhRUeSzQF9joJjbocZX8Bd+XDpVzD5boicAFYHVO2BzQvgixvh+eHw3BD49zWw8X0oy/vZd+vt8Oa5yc9x84ibsRgWPsv6jEu+voRd1buO8AZFRER+mYJOERER6ZTsVgsv/GU4Jw3u2R52LskocnVZIt2aYRgEXn4Z4S+9iMXLi9o1azSkyJVsbhA1HibfBZd9aQafM76ASXdAn3FgsUPlTtj0IXx+HfxjGMwZCp/NhE0fQcXO9ndlGAZ/HfpXXj7+ZXq49WB76XamLZzGil0rXLhBERGRgynoFBERkU7LbrXwwvQRnDjIDDuvfHedwk6RDsBn8mSiPvoQe3j4T4YULXN1WWL3MI+0H3svXP6Necfnxf82j76HjwGLDcrzIeU9+PfV8Nxg+EcC/N8NkPoxVO5hXO9xzD99PoMDB1PRUMG131/LK5teodXZ6urdiYiIKOgUERGRzs1hszD3LyM44Sdh51KFnSIu59a//38MKbqG0nfe0SCbjsThBdHHmsOMrlhk3vF54b9g/M3m0CPDAmU5sOFd+PQKmB0HL4wkNOlJ3ulzFudGnYYTJ3NT5nLT4puobKx09Y5ERKSbU9ApIiIinZ7DZmHef4SdyzIVdoq4mq1HD3NI0XnnmkOKHn+CvQ88gLOx0dWlyc9x84b+x8MJD8GVi83g8y8LYNz1EJoAGFCSBevfwu3Tq3kw6SUeqrfjwELyzmSmfzGVjLIMV+9CRES6MQWdIiIi0iXsDzuPH9iThuZWrnhnHcszi11dlki3ZzgchD7yCCF33QkWC+Uff6IhRZ2Fuy8MOAlO+jtcvQTuzIULPoSjroOeQwE4Z88O3t21m9DmZvKrd3HR5+fy1acXwfYvoLbUtfWLiEi3o6BTREREugyHzcKLF47g+IEhNDS38td31irsFOkADMMg8NJLiXj5JXNI0dq15E6dRkNWlqtLk9/Dwx/iToWTH4drl8MdOTDtPQYnXMb8Bj/G1dVRZ8CdVZt4YtENND3VD16eAN/cDWlfQV25q3cgIiJdnIJOERER6VLMsHPkQWHniiyFnSIdgfekSUTN/wh7RARNBQXmkKIlS1xdlvxRngEwcAqc+hQ9rlvFSxet4MpekwB438+HK0KDKSraCqtehI+mw1N94ZVj4Nt7IeNbqNedniIicmgp6BQREZEux2GzMO/CERwXdyDs/FFhp0iH4BYTQ9SC+XiOHk1rTQ0F115Hydtva0hRF2D17smNJ81jTuIcvO3ebHB3Z2q/ODYMOxsCY8DZCntSYOVc+GAqPBkFrx0Li/4Gmd9DQ7WrtyAiIp2cgk4RERHpktxsVl68aATHxoVQ39TK5Qo7RToMW48e9HnjdfzPPw9aWyl84kn23H+/hhR1Ecf1OY4PT/uQaL9oipur+Wv1Jt4/fhbOW7bDOa/DiEugR19wtsCu9bBiDrx/LjwZCa+fAD88DDsWQ2Otq7ciIiKdjIJOERER6bLcbFZe+s+wc4fCTpGOwHA46PXww/S8+y6wWKj45F/kX/5XDSnqIqL8ovjgtA84Oepkmp3NPLHmCe7a9A9qB54GZ7wAN6XALVvh7Fcg4SLw7wOtzbBzDSx7Fv55NjzRB948GRb/HXKWQlOdq7clIiIdnIJOERER6dL2h52JscFm2Pm2wk6RjsIwDAJmzDCHFHl7U7tuHbnnT6UhM9PVpckh4Gn35KlJT3HH6DuwGla+yvmKi76+iPzKfPMBfuEQfwGcNQ9u3gw3pcKZ8yB+OviGQ2sT5K+EpU/BO1PgiUh46zRIfgJyV0Bzg2s3KCIiHY6CThEREenyzLBzJJN/Enau3FHi6rJEpM1BQ4p27iT3gulUJSe7uiw5BAzD4OJBF/P6ia8T6B5IZlkmFyy8gOSC5P9+cI9IGH4RnP0y3LIFbtwIU56HoVPBJxRaGiBvOSQ/Dm+fanZ8vjMFljwN+augWVcfiIh0dwo6RUREpFtwt1t5+aKRHDPgQNi5Klthp0hH4RYdbQ4pGjOG1poadl57HSVvvqUhRV3EqF6jmH/6fOKD46lqquKGxTfwwsYXaGlt+fknGAYE9IORM+Dc12DWdrh+PZz+HAw+B7xCoLnePNKe9Ci8eZJ5x+e7Z5lH3wvWQkvTEd2jiIi4noJOERER6Tbc7VZeudgMO+uaWrjsLYWdIh2JrUcP+rz+Gv7nnw9OJ4VPPcWee+/TkKIuoqdXT9466S0uiL0AgFdTX2XmDzOpaKj49ScbBgTFwKjL4fy34LYMmLkGTn0GBp0JnoHQVAvZSeYwozeON6e6v3cuLJ9jDj1qaT6s+xMREddT0CkiIiLdyv6wc9JPws7VCjtFOgxzSNFD9LznHnNI0aefknf55TSXlrq6NDkE7FY79x51L49NeAx3qzsrdq9g2sJpbCvZ9vvekWFAcCyMuRKmvgu3ZcG1K+GUpyDudPDoAY3VkPU9fP83eO1YeKovfDANfnwBdqfA/+omFRGRTktBp4iIiHQ77nYrr/407Hx7LWtyFKKIdBSGYRBwycVEvPIyFm9v6tatJ/f8qdRnZLi6NDlEpkRP4b1T3yPcO5xd1bu45OtL+Czrsz/+Di0W6DkIxl4NF7wPt2fDNcvhpMch9lRw84OGSsj4Br67D149xgw+P/wLrHwR9m6G1tZDtj8REXENBZ0iIiLSLe0POyf2D6K2sYVL31qjsFOkg/GeONEcUtSnD027dpF3wXSqkpJcXZYcIrEBsXx0+kdMDJtIQ0sD96+4n0dWPkJjyyG4qsBigV5DYdx1MP1DuDMHrloCJz4K/U8Chw/UV0D6l/Dt3fDyBHg6GuZfBKtfgX3bQPfDioh0Ogo6RUREpNtyt1t57ZJRB4Wda3MVdop0JG7R0UTN/wjPsWNpra1l53UzKXnjTQ0p6iL83PyYe9xcrou/DgODBRkLuOyby9hbs/fQfiCLFXonwNE3wIUL4M5cuGIxHP8QxBwPdi+oK4XtX8DXd8BL4+DpGFgwA9a+DkXpCj5FRDoBBZ0iIiLSrf1X2Pmmwk6RjqZ9SNG0aeaQoqefZs+999GqIUVdgsWwcG3Ctcw9bi4+Dh9Si1OZtnAaa/asOXwf1GqD8JEw4Wa46F9wVx78dREc9wD0SwSbB9QWw7bP4MtbYd4YeDYWPrkc1r0FxVkKPkVEOiAFnSIiItLt7Q87J8QEUdMWdq5T2CnSoRh2O70e/Bs97723fUhR/mUaUtSVTAqfxPzT5xPbI5bS+lKuXHQlb21568h071rtEDEGJt4Kl3wGd+XDZd9A4n3QdxLY3KF6H2z5Fyy8GeaOhNmD4F9XwoZ3oTRHwaeISAegoFNERESEA2Hn+JhAahpbmKGwU6TDMQyDgIsvIuLVV7H4+FC3fj25551PfbqGFHUVET4R/PPUfzKl3xRana3MXj+bW5fcSk1TzZEtxOaAyHFwzO0w4wu4Mw8u/RKOuQsiJ4DVAVW7YfMC+L8b4PkEmDMU/n0NbHwfyvOPbL0iIgIo6BQRERFp5+Gw8volozk6+kDYuT5PYadIR+M9Ybw5pCiyD027d5M3fTpVizWkqKvwsHnw9wl/596x92Kz2FiUt4jpX04nuyLbdUXZ3SFqAiTeDZd9aXZ8XvJ/MOl26DMOLHaoKIBNH8Ln15mh55xh8NlM2PQRVOxyXe0iIt2Igk4RERGRn/BwWHljxk/DzrWszytzdVki8h/c+vWj7/z5eB51lDmkaOZMyhYscHVZcogYhsEFcRfw1klvEeIRQk5FDtMXTmdR3iJXl2aye0C/Y+DY++Dyb8w7Pi/+N0yYBeFjwGKD8jxIeQ/+fTU8NwieH252f6Z+DFWHeNiSiIgACjpFRERE/sv+sHNcv0CqG5rbOjsVdop0NFZ/f/q89mr7kKKiOf/QNPYuJiEkgflT5jOq5yhqm2uZlTyL2etm09za7OrSDubwguhj4fi/wRWLzKPuF/4Lxt8EvUeAYYHSbPM+z0+vMAcbvTAKvrjZvPezutDVOxAR6RIUdIqIiIj8DA+HlTcuHcVR/QLaw84N+Qo7RToaw26n5913gcVCS2kpLSUlri5JDrEgjyBePfFVLhl0CQBvbX2LaxZdQ0ldB/5cu3lD/+PhhIfhqiS4Mxf+sgDGXQ+h8YABJZmw/i1zkvsz/WHeWPjyNtj6GdR04L2JiHRgCjpFRERE/gdPh403Lx19IOx8Yw0bFXaKdDgWd3ccffoA0JChwURdkd1i5/bRt/P0pKfxsHmweu9qpi2cxuaiza4u7bdx94MBJ8FJf4erl8KdOXDBh3DUddBzqPmYojRY+xp8PAOe7gcvHg1f3wnbv4Ba3RctIvJbKOgUERER+QU/DTurGpq5RGGnSIfk1r8/AA2ZmS6uRA6nk/uezAenfkCUbxT7avcx45sZfJzxcee7ssCjB8SdCic/DtcuhztyYNp7MOZqCBlkPqZwK6x+GeZfBE/1g5cnwDf3QNpXUFfu0vJFRDoqBZ0iIiIiv2J/2Dm274GwM6Wg3NVlichPuA0YAEC9Ojq7vJgeMXxw2gccG3EsTa1NPLzyYV7Y+IKry/pzPANg4BQ49Sm4biXcvgPOfwdGXwFBsYAT9m6GVfPgo+nwVF945Rj47j7I+A4aqly9AxGRDkFBp4iIiMhv4Omw8dZloxnTFnZe/PpqhZ0iHciBjs4sF1ciR4KPw4c5iXO4PuF6AN7b/h4trS0uruoQ8gqCwWfBac/C9Wvg1gw49w0YeRkExoCzFfakwI8vwAfnY3s2hknpD2JZ/DBkfQ8N1a7egYiISyjoFBEREfmNPB023rp0NGOi2sLON1azSWGnSIewv6OzISsLZ2uri6uRI8EwDK4YegUeNg/qmuvIq8xzdUmHj09PGHoeTJkDN6yHWdvhnNdgxCXQoy+Gs4UetdlYVz4P750LT0bC6yfADw/DjiRorHX1DkREjggFnSIiIiK/g5dbW2dnVABV9c1cpLBTpENw9InAcDhw1tbStGuXq8uRI8RqsTKghxlyby/d7uJqjiDf3jBsKpzxAtyUQtMNm9jQ5ypah/0F/PtAazPsXAPLnoV/ngVP9IE3T4bFf4ecpdBU7+odiIgcFgo6RURERH6n/WHn6Kge7WFn6s5yV5cl0q0ZNhuOmGhAk9e7m4EBAwHYXtKNgs7/5BtGQeAEWqY8DzdvhptS4cx5MOwC8A2D1ibIXwlLn4J3ppjB51unQfITkLsCmhtcvQMRkUNCQedhMG/ePAYNGsTo0aNdXYqIiIgcJmbYOYZRkW1h5+ur2byzwtVliXRr7pq83i0NCjSnlHerjs5f0yMShl8E57wCt2yFGzfClOdh6Png3QtaGiBvOSQ/Dm+fCrMHwdbPXF21iMifpqDzMJg5cybbtm1j7dq1ri5FREREDiNvNxtvX26GnZX1zVz4+iqFnSIu1H5Ppzo6u5WBgQc6Op1Op4ur6YAMAwL6wcgZcO7rcGsaXL8eTn8OBp8DXsFQWwwfz4B/XQl1Za6uWETkD+sUQWdeXh7btm2jVZeKi4iISAezP+wc2RZ2XvTGarbsUtgp4gr7J6/XK+jsVqL9orFb7FQ1VbGzeqery+n4DAOCYmDU5XD+W3DLNph4KxgW2LwAXjwasn5wdZUiIn9Ihwo633zzTWbPnn3Qn1111VX069ePoUOHMmTIEAoKClxUnYiIiMjP83az8U5b2FlR18SFryvsFHGF/R2djbl5tDY2urgaOVLsVjsx/jEApJWmubiaTsjmgOMegMu/g4BoqNoN750DC2dBY42rqxMR+V06VND56quv0qNHj/a3v/nmG9566y3effdd1q5di7+/Pw899JALKxQRERH5ed5uNt6+bDQj+vgr7BRxEVvPnlh8faG5mcacHFeXI0dQ+z2d3Xkg0Z8VMRquWQZjrjLfXvcGvDQe8le5ti4Rkd+hQwWdmZmZjBo1qv3tzz//nDPPPJMLL7yQESNG8Nhjj/HDD2qhFxERkY7Jx93OO5ePaQ87dYxd5MgyDKP9+HpDhgYSdSf7J69vK93m4ko6OYcXnPo0XPyZOa29LAfeOgUW/U2T2UWkU+hQQWddXR2+vr7tb//4449MmjSp/e1+/fqxd+9eV5QmIiIi8pvsDzuH9/GnvNYMO7fuVtgpcqS4DdgfdOqezu5EA4kOsehEuPZHiJ8OzlZYMQdeTYQ9qa6uTETkF3WooDMyMpL169cDUFxczNatWxk/fnz73+/duxc/Pz9XlSciIiLym+wPOxMizLDzwtcVdoocKe0dnZnq6OxOBvQYgNWwUlpfSmFtoavL6Ro8/OHsl2Hae+AZBIVb4bVjYekz0NLs6upERH5Whwo6Z8yYwcyZM3nkkUc4//zziYuLY+TIke1//+OPPzJkyBAXVigiIiLy2/i623n3rweHndt2V7q6LJEuz71tIJE6OrsXd5s7ff36AvD/7N13fJXl+cfxzznZARJG2CNsMEy1gIgoKoq7aK2K1i1W0Vq1tVatu8Ofo9qqaN3WOlqtq05cKIIoQ/aQjbLDTsgiOb8/DjkaQQVJ8uQkn/frxYvknPOccz14l8KX676vuRs8p7NS7XM8jJoI3Y+DshJ4/1Z4/CjIXRh0ZZK0kxoVdP7ud79j5MiRvPjii6SmpvL8889XeH78+PGMGDEioOokSZL2THnY2ScWdk407JSqWHlHZ8nKlZTm5QVcjapT+TmdBp1VoH7TaGfn8AchJQO+mgQPHgSfPgRlZUFXJ0kxNSroDIfD3HLLLXz++ee8+eab7LPPPhWef/755zn//PMDqk6SJGnPZaQm8c/z+tOnTSYbd4Sdc1cZdkpVJSEzk8TmzQG3r9c13zynU1UgFIK+I6Jnd3Y4BLYXwJtXwVPDYfNXQVcnSUANCzoB/v3vf3PGGWfw85//nAcffDDociRJkvZaZloS/zx/wDfCzk+Zt9qwU6oqTl6vm+zorCYN20ansh99BySmwZIPYfSBMO1ZcBCUpIDVqKDzgQceYMSIEUyePJkFCxZwySWXcNVVVwVdliRJ0l4rDzt7t8lkQ34xpz9s2ClVlZTyczrt6KxTujfuDsDq/NVsKNwQcDW1XDgMAy6Eiz6G1j+Bos3w8kXw719A3rqgq5NUh9WooPO+++7jxhtvZP78+UybNo0nn3yS0aNHB12WJElSpchMS+Kpb4Wd81dvDbosqdb5uqPTgUR1Sf3k+mRnZAMwb/28gKupI7I6w3lvw2HXQzgJ5r0Gow+Aua8FXZmkOqpGBZ2LFy/m7LPPjn1/+umns337dlatWhVgVZIkSZUnMy2Jp84bQK/W5WHnRMNOqZKldP066Iy4lbZOcft6ABIS4eDfwsj3oVkObMuFf58BL10MhZuDrk5SHVOjgs6ioiLq1asX+z4cDpOcnExBQUGAVUmSJFWuzPQk/nV+NOxcb9gpVbqUTp0gHKZ00yZKc3ODLkfVqHz7ukFnAFr2hgvHwqBfAyGY/kz07M7FYwMuTFJdkhh0Ad92/fXXk56eHvu+uLiYP/3pT2RmZsYe++tf/xpEaZIkSZWmPOw849GJzFqxhdMfnsizFx5A1+YNgi5Ninvh1FSS27WjeOlSihYsILFp06BLUjVx8nrAElPgiFug2zHw0kWwcQn886fQ/0IYejMkp//we0jSXqhRHZ0HH3ww8+fP5/PPP4/9OPDAA1m8eHHs+2nTpgVdpiRJUqUoDzt7tMqIdXYuWGNnp1QZygcSFXpOZ51SvnV9+dblbC3299PAtDsgOqjoJ+dHv//sIfjHYPhqcrB1Sar1alRH59ixYyt8n5ubS3JyMhkZGcEUJEmSVMUapifz9AUDOOORT5m9cgsjHp7IsyMPoIudndJeSenSha1jxjh5vY5plNqIlvVasip/FfM2zKNfi35Bl1R3pdSH4/4K3Y+BVy6F9Qvh0SPgoCvhkKshMTnoCiXVQjWqoxNg06ZNXHLJJWRlZdG8eXMaNWpEixYtuOaaa9i2bVvQ5UmSJFW68rAzp2UGuXnFjHj4UxautRNJ2hvlHZ1FXxh01jWxgURuX68ZOg+FUZ9Ar59DpAzG3QmPHAZrZgddmaRaqEYFnRs2bGDAgAE8+eST/OxnP+Ouu+7irrvu4oQTTuDee+/l4IMPprCwkM8++4y///3vQZcrSZJUaSqGnUWc9pBhp7Q3UrrsmLy+cCGRsrKAq1F16t4kOpBo3oZ5AVeimLRG8LNH4OdPQlpjWD0THhoCH98DZaVBVyepFqlRQectt9xCcnIyixYt4h//+AeXX345l19+OQ899BALFy6kuLiYM888kyOOOKLCcCJJkqTaoFG9aNi5T4WwMy/osqS4lJzdjlByMpGCAkq++iroclSNchrnAE5er5F6DIdRE6HrUVBaDO/eCI8fAxsWB12ZpFqiRgWdL7/8MnfeeSfNmzff6bkWLVpw++2389///pcrr7ySs88+O4AKJUmSqta3w84RD0807JR+hFBCAsmdOwFQ5ECiOqV88vrizYsp2F4QcDXaSYPmMOI5OOE+SK4PX06EBw6CSY9CJBJ0dZLiXI0KOletWkWPHj2+8/mePXsSDoe58cYbq7EqSZKk6tV4R9jZvUUD1m2Nhp2L1hl2SnsqtcuOczodSFSnNE1rSpPUJpRFyvhioyF3jRQKwX5nwsUTIPsgKMmH16+Ef/0MtqwMujpJcaxGBZ1ZWVksXbr0O59fsmQJzZo1q76CJEmSAtK4XjLPjDzg67DzIcNOaU+ldI2e01loR2edEgqFYl2dDiSq4Rplw9n/g2F/gcRUWPQejD4AZjxvd6ekH6VGBZ3Dhg3juuuuo7i4eKfnioqKuP766znqqKMCqEySJKn6fTPsXGvYKe2x2OR1OzrrnPLJ6w4kigPhMAwcBb/8CFrtC4Wb4cUL4PmzIX990NVJijM1Kui85ZZbmD9/Pl26dOH222/n1Vdf5ZVXXuG2226jS5cuzJ07l5tuuinoMiVJkqrNN7exl4ediw07pd1SHnQWL1lK2S6aKVR7lXd0zlk/J+BKtNuadoPz34Eh10A4Eea8Eu3unP9W0JVJiiM1Kuhs06YNn3zyCTk5OVxzzTUMHz6cE088keuuu46cnBzGjx9Pu3btgi5TkiSpWjWpn8LTFwygW/MdYefDE1mSmx90WVKNl9isGeGMDCgtpXjJkqDLUTUq7+hcsGkBJaUlAVej3ZaQBEN+Dxe8C027Q/5aePZUeOVSKNwSdHWS4kCNCjoBOnTowJtvvklubi4TJ05k4sSJrFu3jrfeeovOnTsHXZ4kSVIgmtRP4emRA+javD5rthRx2kOfGHZKPyAUCsXO6XTyet3Sun5rGiQ3YHvZdhZuWhh0OdpTrfaFCz+EgZcCIfj8KXhgECwZF3Rlkmq4Ghd0lmvUqBH9+/enf//+NG7cOOhyJEmSApdVP4VnRh4QCztHPDSRpYad0vdK6VIedHpOZ10SCoXIaZwDwNwNDiSKS0mpMOxPcM7r0LAdbF4OTx4Hb10DJQVBVyephqqxQackSZJ2Vh52dmlWn9VbCjnNsFP6XqnlA4ns6KxzPKezlmg/CC6eAPudFf1+4mj4x8GwYmqwdUmqkQw6JUmS4sy3w84RD09k2XrDTmlXYh2dTl6vc5y8XoukNIAT7oXT/wP1m0PuF/DIUPjgL+AZrJK+waBTkiQpDjVt8HXYuWpztLPTsFPaWXnQWbJyJaV5eQFXo+rUvUl3AOZvmE9pWWnA1ahSdB0GoyZCjxMhUgof3hYNPNcaZkuKMuiUJEmKU+VhZ+cdYecIw05pJwmZmSQ2bw54Tmddk90gm7TENApLC1m6ZWnQ5aiypDeGnz8BP3sUUhvCqmnRrewT7gUDbanOM+iUJEmKY9GwcwCdmtZj5Y6wc/n6bUGXJdUoKeXndLp9vU5JCCfQvXG0q9NzOmuhXidHuzs7D4XSIhjzB3jyeNi4NOjKJAXIoFOSJCnONWuQyrMXHhALO0976BPDTukbvp687kCiuqb8nE4nr9dSGS3hjBfguHsgqR4sGw8PDIIpT0IkEnR1kgJg0ClJklQLNGuQyrMjD6BjeWfnwxP5coNhpwSQ0tWBRHVV+eR1BxLVYqEQ/ORcuHg8tBsIxXnwv8vgmVNh6+qgq5NUzQw6JUmSaolmGak8tyPsXLGpgNMeMuyUAFLLt65/8QURu7zqlFhH5/q5lEXKAq5GVapxBzjndTjiVkhIhgVvw+gDYNaLQVcmqRoZdEqSJNUisbAzy7BTKpfcsSOEw5Ru2kRpbm7Q5agadWzYkeRwMnkleazYuiLoclTVwgkw6DK48ENo0RsKNsIL58IL58G2DUFXJ6kaGHRKkiTVMs0yomd2GnZKGUvo5QABAABJREFUUeHUVJKzswEo9JzOOiUpnESXRtGjC+ZscCBRndE8By54Dw7+HYQSYNZ/YfRAWPBO0JVJqmIGnZIkSbVQ8x1hZ4cdYeeIhyfy1UbDTtVdsYFEntNZ55Sf0zl3vQOJ6pTEZDjsOjj/HWjSBfJWw9Mnw/9+DUV5QVcnqYoYdEqSJNVSzTOiA4o6ZNXjq43Rzk7DTtVVKbFzOg066xonr9dxbfaHX34EAy6Ofj/lCXjgQFg2IdCyJFUNg05JkqRarEVmxbDTzk7VVXZ01l05TXKA6OR1h1HVUcnpcPRtcNarkNkWNi2Dx4+BMX+AksKgq5NUiQw6JUmSarnysLN9k3S+3BANO1dsKgi6LKlapXTdEXQuXEikzOnbdUmXRl1ICCWwoXADa7atCbocBanjIXDxeOj7CyACE+6Fh4bAqulBVyapkhh0SpIk1QEtMqNndmbvCDtPe+gTw07VKcnt2hFKSSFSUEDJl18GXY6qUUpCCh0bdgQ8p1NAaiYMvx9OexbqNYV1c+Hhw+DD26F0e9DVSdpLBp2SJEl1RMvMNJ77Rtg54qGJrDTsVB0RSkggpVMnwO3rdZHndGon3Y+BURNhn+OhbDt88Cd47EjI9fcHKZ4ZdEqSJNUhLTPTeHbkAbRrnM7yDds4zbBTdUj5OZ2FX3wRcCWqbuXndNrRqQrqZcEpT8GJD0FKJqyYAg8eBBMfBI+4kOKSQackSVId06phtLOzPOwc8fBEVm027FTtF5u8bkdnnWNHp75TKAR9ToVRn0DHQ2F7Ibx1NfzzBNi0POjqJO0hg05JkqQ6qFXDNJ698ADaNk5j2fpoZ6dhp2q72ECiLww665pujbsRIsSabWtYX7A+6HJUE2W2hjNfgmPuhKR0WDoORh8In/8LIpGgq5O0mww6q8D9999PTk4O/fr1C7oUSZKk79S6YRrPXTgwFnaOeGgiqzcXBl2WVGXKOzqLly6lrLg44GpUneol1SM7IxuAeRvmBVyNaqxQCPqPhIs+hjb9oXgrvHIJPHc65K0NujpJu8GgswpccsklzJkzh0mTJgVdiiRJ0vdq3TB6ZmebRmksXb+N0x76xLBTtVZis2aEMzOhtJTixYuDLkfVzO3r2m1NOsF5b8HhN0I4Cea/AaMPgDmvBF2ZpB9g0ClJklTHtWmUznMXfh12jnjYzk7VTqFQiJQunQHP6ayL9mkSDTrnrJ8TcCWKC+EEGHwlXPgBNO8J29bDf86CFy+Egk1BVyfpOxh0SpIkqULYuSQ3nxEPT2TNFsNO1T6p5QOJnLxe55QHnU5e1x5p0QtGvg8HXQmhMMz4N4weCIveD7oySbtg0ClJkiQgGnY+O/IAWjeMhp2nPWTYqdonpYsDieqq8q3rX+V9xZbiLQFXo7iSmAJDb4Tz3obGHWHrSnjqRHj9N1CcH3R1kr7BoFOSJEkxbRtHOzvLw84Rhp2qZcoHEhUusKOzrslMyaR1/dYAzN8wP+BqFJfa9o8OKuo3Mvr9pEfgwYNg+afB1iUpxqBTkiRJFXwz7Fy8I+xca9ipWiKlc/SMzu0rV1G6dWvA1ai6lXd1ek6nfrTkenDsnXDmS9CgFWxYDI8fBe/eBNuLgq5OqvMMOiVJkrSTb4edpz1s2KnaISEzk8QWLQAoXrQo4GpU3bo37g44eV2VoNNhMOoT6H0aRMrg47vh4cNg9aygK5PqNINOSZIk7VLbxtEzO1tlprJ4XXRA0bqtdqso/pWf01nsOZ11jgOJVKnSGsJJ/4BTnoL0JrBmFjw0BMbdBaXbg65OqpMMOiVJkvSd2jVJ57kLB9IqM5VF6/L5xWOT2VIcdFXS3knpuiPoXLgw4EpU3XKa5ACwdMtStpVsC7ga1Ro5J8CoidDtWCgrgfdugcePhvV2jUvVzaBTkiRJ36tdk3SevfAAWmamsjg3n/vmJJCbZ2en4lds8voCOzrrmqy0LJqmNaUsUsYXGx1IpUpUvxmc9jT8dDSkZMBXn0UHFX32MJSVBV2dVGcYdEqSJOkHZTepx3MXHkCLjBTWFIT4xWOT3cauuJW6Y/J68cKFEIkEXI2qW2z7uud0qrKFQrDvGXDxBOhwMJRsgzd+C/86CTavCLo6qU4w6JQkSdJuyW5Sj3+d34+GyREWrcvndM/sVJxK7tQJwmHKNm0iwcnrdU5sIJHndKqqNGwLZ74CR98Oiamw+AMYPRCmP+c/rkhVzKBTkiRJuy27cTqX5pTSPCOFBWvzDDsVl8IpKSRnZwOQsnpNwNWouuU0jp7TaUenqlQ4DAN+Cb8cB633h6LN8NIv4T9nQn5u0NVJtZZBpyRJkvZI0zR4+rx+tMhIjYWdntmpeJOyY/t68urVAVei6la+dX3hxoUUlzpdTVWsaVc4bwwc+gcIJ8Lc/8HoA2De60FXJtVKBp2SJEnaY9k7BhSVd3aO/OfkoEuS9kj5QKKUNQaddU3Lei3JTMlke2Q7CzctDLoc1QUJiXDIVTDyfWiWA/nr4LnT4eVRULg56OqkWsWgU5IkST9Kh6x6PDvyAAA+X76Jjfl2Ril+pHTdEXS6db3OCYVC7NN4x0Aiz+lUdWrZB0Z+AAdeBoRg2tPwwCBY/GHQlUm1hkGnJEmSfrSOTevTKjMVgMW5eQFXI+2+8o7O5DVriJSWBlyNqpuT1xWYpFQ48lY49w1o1B42fwn/PAHevBqKtwVdnRT3DDolSZK0Vzo2rQ/AorX5AVci7b7kdu0IpaQQLimhZMWKoMtRNbOjU4HLPhAuGg/7nxv9/tMH4R+D4aspwdYlxTmDTkmSJO2Vjk3rAbDIjk7FkVBCAskdOwJQvGBBwNWoupUHnfM3zmd72faAq1GdlVIfjr8HzvgvNGgJ6xfCo0fA+3+E7R4HI/0YBp2SJEnaK512dHQuXmdHp+JL8o7t68ULHEhT17TLaEd6YjpFpUUs3bw06HJU13UZChdPgJ4nQ6QUProDHjkc1swJujIp7hh0SpIkaa+Ud3QuXmdHp+JLcpfOgB2ddVE4FKZ74+6A53SqhkhvDCc/Cic/DmmNYPUMeOgQGP83KPMcYWl3GXRKkiRpr5Sf0bl8wza2l5YFXI20+8o7OosW2tFZF+U0yQFgznq75lSD9DwJRk2ELsOgtBjeuQGeOBY2LAm6MikuGHRKkiRpr7TMSCU1KUxJaYQvNxYEXY6028onr5csW0ZZsefh1TVOXleN1aAFnP5vOOFeSK4Pyz+BBwbB5McgEgm6OqlGM+iUJEnSXgmHQ3TIKp+87vZ1xY+Epk0pTUuD0lKKFy8OuhxVs/Kt6/M2zKMsYje6aphQCPY7Cy4eD9mDoCQfXrsCnj4ZtqwMujqpxjLolCRJ0l6LndPp5HXFkVAoRFGLFgAUffFFwNWounXM7EhKQgr5Jfl8ufXLoMuRdq1Rezj7NTjyT5CQAgvfhdEDYeYLQVcm1UgGnZIkSdprTl5XvCpu0RyAIgcS1TmJ4US6NuoKuH1dNVw4DAdeCr/8CFr2hcJN8N/z4flzYNuGgIuTahaDTkmSJO21TrHJ6wadii/lHZ2FdnTWSfs03nFO53qDTsWBZt3hgndhyDUQSoDZL8HoA+CLt4OuTKoxDDolSZK01zruOKPTreuKN0XNy7eu29FZF8UGEhl0Kl4kJMGQ30cDz6xukLcGnjkFXv0VFG0NujopcAadkiRJ2msddnR05uYVs3lbScDVSLuvfOv69lWrKN1qSFDXfHPyesRp1oonrfeDX34IB1wChGDqP+GBA2Hpx0FXJgXKoFOSJEl7rX5KIs0zUgBYZFen4khZWhqJzT2ns67q0rALiaFENhVtYnX+6qDLkfZMUhoc9Wc45zXIbAeblsMTx8Hb10FJYdDVSYEw6JQkSVKliG1f95xOxZnkLl0At6/XRckJyXRq2AmAORvmBFyN9CO1PwguHg/7nglE4JP74B8Hw4qpQVcmVTuDTkmSJFWKjrGBRHZ0Kr4kd+kMQJEDieqk8u3r8zbMC7gSaS+kZsBP74MR/4Z6zSB3PjwyFMbeBqUeKaO6w6BTkiRJlaJTUzs6FZ9iHZ1uXa+TnLyuWqXbUTBqIuQMh0gpjP0LPHoErJsfdGVStTDolCRJUqWIdXR6RqfiTEps6/oXDqSpg3Ka5AAGnapF6jWBnz8BP3sUUjNh5efw4GD45H4oKwu6OqlKGXRKkiSpUpR3dC7N3UZpmWGR4kdShw6QkEDp5s1sX7cu6HJUzbo26kqIEGsL1pJbkBt0OVLlCIWg18nR7s5Oh0NpEbx9LTx5PGxcFnR1UpUx6JQkSVKlaNUwjeTEMMWlZXy1cVvQ5Ui7LZySQnJ2NuBAorooPSmd9pntAbs6VQtltIJf/BeOuxuS6sGyj+GBA2HqP8EOdtVCBp2SJEmqFAnhEB2alA8k8pxOxZdvbl9X3VN+TqcDiVQrhULwk/Pg4o+h7QFQnAev/gqePQ22rgm6OqlSGXRKkiSp0pSf07nIyeuKMyldHUhUl8XO6dxgR6dqscYd4dw34IhbICEZvngLRh8As18KujKp0hh0SpIkqdLEJq/n2tGp+GJHZ91W3tE5Z/2cgCuRqlg4AQb9Gi4cCy16QcEGeP4c+O8FULAx6OqkvWbQKUmSpEoTm7xuR6fiTGrXrgAULVpEpLQ04GpU3bo36Q7AirwVbC7aHHA1UjVo3gMueB8OvgpCYZj5PIweCAvfDboyaa8YdEqSJKnSdNzR0bnIMzoVZ5LatiWUmkqksJCSL78MuhxVs4zkDNrUbwN4TqfqkMRkOOwPcP470KQzbF0F//oZvHYFFPkPlopPBp2SJEmqNOUdneu2FrG1sCTgaqTdF0pIIKVTJwAKPaezTtqnSXT7upPXVee0+Qn8chz0/2X0+8mPwYODYNknwdYl/QgGnZIkSao0GalJZNVPAZy8rvjjOZ11W/k5nQ4kUp2UnA7H3A5nvQIZbWDjUnj8aHjnBtheFHR10m4z6JQkSVKlip3Tmeu2N8WXlPJzOhcsDLgSBSHW0WnQqbqs4xAYNQH6ngFEYPzf4KEhsGpGwIVJu8egU5IkSZUqNnndjk7FmVjQaUdnndS9cXQg0dLNS9lWsi3gaqQApWbC8NFw2jOQngVr58DDh8JHd0Dp9qCrk76XQackSZIqVafY5HWDTsWX8q3rxcuWUVbkVs26Jisti2bpzYgQYf7G+UGXIwWv+7EwaiJ0Pw7KtsP7f4THhkGu5xir5jLolCRJUqUq37q+aJ1b1xVfEps1JSEzE0pLKV68OOhyFICcxjkAzFk/J+BKpBqiflM49V9w4j8gJRNWTIYHB8On/4CysqCrk3Zi0ClJkqRK1TErunV9SW4+ZWWRgKuRdl8oFHIgUR3XvUl0+/q8DfMCrkSqQUIh6HNa9OzOjkNgewG8+Tt46qew6cugq5MqMOiUJElSpWrTKI2khBBF28tYsakg6HKkPZLWtw+pvXsTSkkJuhQFIDZ5fb0DiaSdZLaBX7wEx9wJiWmw5CN44ECY9gxE/IdN1QwGnZIkSapUiQlhspuUT173nE7Fl2a//S0d/vNvMo46KuhSFICcJtGt64s2LaKo1HNapZ2Ew9B/JFz0MbTpB0Vb4OWL4bkzIG9d0NVJBp2SJEmqfF8PJPKcTknxo3l6cxqlNGJ7ZDsLNy4Muhyp5srqDOe+BYffAOEkmP86jD4A5v4v6MpUxxl0SpIkqdJ1bBo9p9PJ65LiSSgUYp8m0e3rczY4kEj6XgmJMPg3cOEH0KwHbMuFf/8CXvwlFGwKujrVUQadkiRJqnQds5y8Lik+dW8cHUjkOZ3SbmrRKxp2HnQFhMIw47no2Z2LPgi6MtVBBp2SJEmqdHZ0SopX5R2dTl6X9kBiCgy9KbqdvVEH2LICnhoOb1wFxduCrk51iEGnJEmSKl35GZ2rtxSSX7Q94GokafflNM6hYUpDmqQ2IeIkaWnPtBsAF4+HfhdEv//sIXjwIPhyUrB1qc4w6JQkSVKla5ieTON6yQAscfK6pDjStkFbPjr1I+49/F5CoVDQ5UjxJ7keHHsX/OJFaNAKNiyCx46Ed2+G7cVBV6dazqBTkiRJVaK8q9NzOiXFk1AoZMApVYbOh8OoCdD7VIiUwcd/hYcPg9Wzgq5MtZhBpyRJkqpExyzP6ZQkqU5LawQnPQSn/BPSGsOamfDQEPj4bigrDbo61UIGnZIkSaoSHe3olCRJADk/hUs+ha5HQ1kJvHsTPH40rF8UdGWqZQw6JUmSVCWcvC5JkmLqN4MRz8JP74fkBvDlp9FBRZMeAQd/qZIYdEqSJKlKlHd0LsnNp6zMv8BIklTnhUKw7y+iZ3e2Hwwl2+D138C/ToLNK4KuTrWAQackSZKqRLvG6SSGQxSUlLJ6S2HQ5UiSpJqiYTs461U46jZITIVF78MDA2HGf+zu1F4x6JQkSVKVSEoI065xOuD2dUmS9C3hMBxwMfxyHLTaDwo3w4sjSXjxPJJLtgRdneKUQackSZKqTOyczlwHEkmSpF1o2hXOfwcOvQ7CiYTn/Y9D511L6Is3g65MccigU5IkSVWmU/nk9bUGnZIk6TskJMIhv4ML3iOS1Y3U7VtIfP5MePkSKLS7U7vPoFOSJElVpnwg0eJct65LkqQf0Kov289/jwXNjiZCCKb9Cx4YBEs+CroyxQmDTkmSJFWZ2NZ1z+iUJEm7IzGVOa1HUHrmq9AwGzYvhyePhzd/DyUFQVenGs6gswrcf//95OTk0K9fv6BLkSRJClTHrGhH54pNBRQUlwZcjSRJiheRdgPh4vGw/znRBz59AP5xMKyYEmhdqtkMOqvAJZdcwpw5c5g0aVLQpUiSJAWqcb1kMtOSAFji9nVJkrQnUhrA8X+D05+H+i0g9wt45Ah4/09QWhJ0daqBDDolSZJUZUKhUGwgkZPXJUnSj9L1SBj1CfT8GURK4aPb4ZHDYe3coCtTDWPQKUmSpCpVfk7norV2dEqSpB8pvTGc/Fj0R1ojWDUd/nEITLgXyjweR1EGnZIkSapSHe3olCRJlaXnz2DUROhyJJQWwZg/wBPHwYYlQVemGsCgU5IkSVWqY5aT1yVJUiVq0AJO/0/0/M7k+rB8AjwwCCY/DpFI0NUpQAadkiRJqlKxMzrX5RHxLx+SJKkyhELRiewXj4d2B0JJPrx2OTz9c9iyKujqFBCDTkmSJFWpdk3SCYcgv7iUtVuLgi5HkiTVJo3awzmvwZF/hIQUWPgOjD4AZv036MoUAINOSZIkVamUxATaNU4HYNE6z+mUJEmVLJwAB/4KfvkhtOwDhZvghfPg+XNh24agq1M1MuiUJElSlYtNXvecTkmSVFWa7QMXvAeHXA2hBJj9YrS784sxQVemamLQKUmSpCrXMevrczolSZKqTEISHHotXPAOZHWFvDXwzM/h1cugaGvQ1amKGXRKkiSpypV3dDp5XZIkVYvW+8MvP4IDRkW/n/pkdDL7sgnB1qUqZdApSZKkKtexfPJ6rh2dkiSpmiSlwVF/gbNfg8x2sGkZPH4MvH0dlBQGXZ2qgEGnJEmSqlx50PnVxgIKS0oDrkaSJNUpHQbDxeNh318AEfjkPnjoEFj5edCVqZIZdEqSJKnKNa2fQoPURCIRWLZ+W9DlSJKkuiY1A356P4x4Duo1g3Xz4JGhMPb/oLQk6OpUSQw6JUmSVOVCodA3Jq+7fV2SJAWk29EwaiLk/BTKtsPYP8OjR8K6L4KuTJXAoFOSJEnVopOT1yVJUk1Qrwn8/Ek46RFIzYSVU+Efg+GT0VBWFnR12gsGnZIkSaoWsYFETl6XJElBC4Wg98+j3Z2dDoPthfD2NfDPE2DT8qCr049k0ClJkqRqEdu6nmvQKUmSaoiMVvCLF+HYuyApHZaOg9EHwuf/gkgk6Oq0hww6JUmSVC2+7ujMI+JfHCRJUk0RCkG/C+Cij6HtACjeCq9cAs+OgK1rgq5Oe8CgU5IkSdWifZN6hEKwtXA7uXnFQZcjSZJUUZNOcO6bMPRmSEiGL96E0QfA7JeDrky7yaBTkiRJ1SI1KYE2jdIAJ69LkqQaKpwAB10OF46F5r2gYAM8fzb8dyQUbAy6Ov0Ag05JkiRVm45Z0XM6HUgkSZJqtOY9YOT7MPg3EArDzP9Ez+5c+F7Qlel7GHRKkiSp2nzznE5JkqQaLTEZDr8BzhsDjTvB1pXwr5PgtSuh2H+0rYkMOiVJklRtyievL3byuiRJihdt+0UHFfW/MPr95EfhgUGwfGKwdWknBp2SJEmqNp2y7OiUJElxKDkdjrkDznwZMlrDxiXw+NHwzo2wvSjo6rSDQackSZKqTXlH55cbCyjaXhpwNZIkSXuo06Fw8QToMwIiZTD+HnjoUFg9M+jKhEGnJEmSqlHzjBTqJSdQWhZh+fptQZcjSZK059IawokPwqlPQ3oWrJ0dDTs/uhNKtwddXZ1m0ClJkqRqEwqFYl2di5y8LkmS4tk+x8GoidD9OCgrgfdvhcePgvWLgq6szjLolCRJUrWKTV7P9ZxOSZIU5+o3hVP/BcMfhJQM+GpSdFDR1KeCrqxOMuiUJElSteqYtWPyuh2dkiSpNgiFoO+I6NmdHQ6B7QXwv8tgy8qgK6tzDDolSZJUrWIdnU5elyRJtUnDttGp7C16RwcVLf046IrqHINOSZIkVavyoHPRunwikUjA1UiSJFWicBg6HBz92qCz2hl0SpIkqVqVb13fXFDChvzigKuRJEmqZO0Piv68bHywddRBBp2SJEmqVmnJCbRumAbA4lzP6ZQkSbVMu4FACNYvhK2rg66mTjHolCRJUrXznE5JklRrpTWEFr2iX9vVWa0MOiVJklTtOmaVB512dEqSpFqofPv6UoPO6mTQKUmSpGrXsWn0nM5FBp2SJKk2yh4U/dmBRNXKoFOSJEnVLrZ1Pdet65IkqRbKPjD6c+58yFsXbC11iEGnJEmSql2nHR2dy9dvo6S0LOBqJEmSKll6Y2jWI/q153RWG4NOSZIkVbsWGamkJSWwvSzC8g3bgi5HkiSp8rXfsX3doLPaGHRKkiSp2oXDITo4kEiSJNVmsXM6DTqri0GnJEmSAhE7p3Od53RKkqRaqDzoXDsbtm0ItpY6wqBTkiRJgSifvG5HpyRJqpXqN4WsbtGvl00ItpY6wqBTkiRJgejk5HVJklTbtT8o+rPndFYLg05JkiQFonzy+iI7OiVJUm1VPpBo6bhg66gjDDolSZIUiPJhRBvyi9m0rTjgaiRJkqpA9o6OztWzoGBjsLXUAQadkiRJCkS9lERaZKQCdnVKkqRaqkFzaNIZiMDyiUFXU+sZdEqSJCkwTl6XJEm1Xvn09aUfB1tHHWDQKUmSpMDEgs5cOzolSVIt5UCiamPQKUmSpMB0zIoOJLKjU5Ik1VrlHZ2rpkPhlmBrqeUMOiVJkhSY8o5Oz+iUJEm1VmZraNQeImXw5adBV1OrGXRKkiQpMJ2aRjs6l63PZ3tpWcDVSJIkVZHy7etLxwVbRy1n0ClJkqTAtG6YRkpimJLSCF9tLAi6HEmSpKqRXR50ek5nVTLolCRJUmDC4RAdssoHEnlOpyRJqqXa7zinc+XnUOSfeaqKQackSZICFZu87jmdkiSptmrYDjLbQaTUczqrkEGnJEmSAlU+ed2BRJIkqVYr7+pc5vb1qmLQKUmSpEB9PXndbVySJKkWy94RdHpOZ5Ux6JQkSVKgyievu3VdkiTVauUdnSumQPG2YGuppQw6JUmSFKjyjs7cvCK2FJYEXI0kSVIVadQBGrSCshL4alLQ1dRKBp2SJEkKVIPUJJo2SAHs6pQkSbVYKATtD4p+vfTjYGuppQw6JUmSFLiOWeWT1z2nU5Ik1WIOJKpSiUEXIEmSJJ0+oB3DerSgb9uGQZciSZJUdbJ3dHR+NRlKCiEpNdh6ahmDTkmSJAXup31bB12CJElS1WvSCeo3h7w1sGLy11vZVSncui5JkiRJkiRVh1AIsndsX1/q9vXKZtApSZIkSZIkVZfYOZ0OJKpsBp2SJEmSJElSdSk/p/PLSbC9ONhaahmDTkmSJEmSJKm6NO0G6VmwvQBWTg26mlrFoFOSJEmSJEmqLqHQ19vXl44LtpZaxqBTkiRJkiRJqk7l29cdSFSpDDolSZIkSZKk6lTe0fnlZ1BaEmwttYhBpyRJkiRJklSdmu4DaY2gJB9WTgu6mlrDoFOSJEmSJEmqTuEwZO/o6lz2cbC11CIGnZIkSZIkSVJ1Kw86Paez0hh0SpIkSZIkSdWt/JzO5ROhdHuwtdQSBp2SJEmSJElSdWveE1IyoXgrrJ4RdDW1gkGnJEmSJEmSVN3CCZB9YPTrpZ7TWRkMOiVJkiRJkqQglG9fX+Y5nZXBoFOSJEmSJEkKQmzy+idQVhpsLbWAQackSZIkSZIUhBa9IbkBFG2GNbOCribuGXRKkiRJkiRJQUhIhHYHRL9e6vb1vWXQKUmSJEmSJAXFczorjUGnJEmSJEmSFJTsg6I/LxsPZWXB1hLnDDolSZIkSZKkoLTqC0n1oGAjrJsbdDVxzaBTkiRJkiRJCkpCErTtH/166cfB1hLnDDolSZIkSZKkILXfsX3doHOvGHRKkiRJkiRJQSoPOpdNgEgk2FrimEGnJEmSJEmSFKRW+0FiGmzLhXXzg64mbhl0SpIkSZIkSUFKTIa2/aJfL3P7+o9l0ClJkiRJkiQFLbv8nM7xwdYRxww6JUmSJEmSpKC1HxT9edl4z+n8kQw6JUmSJEmSpKC1/gkkpEDeGli/KOhq4pJBpyRJkiRJkhS0pFRo85Po10vHBVtLnDLolCRJkiRJkmqC9jvO6VzmOZ0/hkGnJEmSJEmSVBNk7zinc6nndP4YBp2SJEmSJElSTdCmH4STYOtK2Lgk6GrijkGnJEmSJEmSVBMkp0Pr/aNfL3X7+p4y6JQkSZIkSZJqivY7tq97TuceM+iUJEmSJEmSaopvntOpPWLQKUmSJEmSJNUUbQdAKAE2L4eNy4KuJq4YdEqSJEmSJEk1RUp9aLVv9Gu3r+8Rg05JkiRJkiSpJml/UPRnt6/vEYNOSZIkSZIkqSYpDzqXfRxsHXHGoFOSJEmSJEmqSdoOgFAYNi6FzSuCriZuGHRKkiRJkiRJNUlqBrTsE/3aczp3m0GnJEmSJEmSVNNkD4r+vNTt67vLoFOSJEmSJEmqaWLndNrRubsMOiVJkiRJkqSapt1AIATrF8LW1UFXExcMOiVJkiRJkqSaJq0htOgZ/drt67vFoFOSJEmSJEmqidoPjv7s9vXdYtApSZIkSZIk1USxgUQGnbvDoFOSJEmSJEmqibIPjP6cOx/y1gVbSxww6PwB7du3p3fv3vTt25dDDz006HIkSZIkSZJUV6Q3hmY9ol+7ff0HJQZdQDyYMGEC9evXD7oMSZIkSZIk1TXtB8Ha2dGgs8fwoKup0ezolCRJkiRJkmoqz+ncbbU66Pzoo484/vjjadWqFaFQiJdffnmn19x///20b9+e1NRUBgwYwGeffVbh+VAoxCGHHEK/fv14+umnq6lySZIkSZIkia+DzrWzYduGYGup4Wp10Jmfn0+fPn24//77d/n8v//9b6688kpuvPFGpk6dSp8+fRg2bBhr166Nvebjjz9mypQpvPrqq/z5z39mxowZ1VW+JEmSJEmS6rr6TSGrW/Rrz+n8XrX6jM6jjz6ao48++juf/+tf/8rIkSM599xzAXjwwQd5/fXXeeyxx/j9738PQOvWrQFo2bIlxxxzDFOnTqV37967fL+ioiKKiopi32/ZsgWAkpISSkpKKuWefkj551TX50lVxbWs2sT1rNrE9azaxPWs2sK1rNrE9bxr4XYHkpA7n9LF4yjrfFTQ5VSrPVkLtTro/D7FxcVMmTKFa665JvZYOBxm6NChfPLJJ0C0I7SsrIwGDRqQl5fH+++/zymnnPKd7/mXv/yFm2++eafHx4wZQ3p6euXfxPd45513qvXzpKriWlZt4npWbeJ6Vm3ielZt4VpWbeJ6rqjVxjT6AVtnvcmH2wcFXU612rZt226/ts4Gnbm5uZSWltK8efMKjzdv3px58+YBsGbNGk488UQASktLGTlyJP369fvO97zmmmu48sorY99v2bKFtm3bcuSRR5KRkVEFd7GzkpIS3nnnHY444giSkpKq5TOlquBaVm3ielZt4npWbeJ6Vm3hWlZt4nr+Dnn7w99Gk1nwJccceiCkNQy6ompTvmN6d9TZoHN3dOzYkenTp+/261NSUkhJSdnp8aSkpGr/H2cQnylVBdeyahPXs2oT17NqE9ezagvXsmoT1/O3NGoDTToTWr+QpFWTodt3H9VY2+zJOqjVw4i+T1ZWFgkJCaxZs6bC42vWrKFFixYBVSVJkiRJkiTtQvn09aUfB1tHDVZng87k5GT2339/3nvvvdhjZWVlvPfeewwcODDAyiRJkiRJkqRvaX9Q9Gcnr3+nWr11PS8vj4ULF8a+X7JkCdOmTaNx48a0a9eOK6+8krPPPpuf/OQn9O/fn3vuuYf8/PzYFHZJkiRJkiSpRijv6Fw1HQq3QGr1zIOJJ7U66Jw8eTKHHnpo7PvyQUFnn302TzzxBKeeeirr1q3jhhtuYPXq1fTt25e33nprpwFFkiRJkiRJUqAyW0Oj9rBxKSyfCF2PDLqiGqdWB51DhgwhEol872suvfRSLr300mqqSJIkSZIkSfqR2h8UDTqXfWzQuQt19oxOSZIkSZIkKa5k7zinc6nndO6KQackSZIkSZIUD9rvOKdz5edQlBdsLTWQQackSZIkSZIUDxq2g8x2ECmFLz8Nupoax6BTkiRJkiRJihflXZ3L3L7+bQadkiRJkiRJUrzI3hF0ek7nTgw6JUmSJEmSpHhR3tG5YgoUbwu2lhrGoFOSJEmSJEmKF406QINWUFYCX30WdDU1ikGnJEmSJEmSFC9Coa+7Ot2+XoFBpyRJkiRJkhRP2h8U/dmBRBUYdFaB+++/n5ycHPr16xd0KZIkSZIkSaptsncEnV9NhpLCYGupQQw6q8All1zCnDlzmDRpUtClSJIkSZIkqbZp0gnqN4fSIlgxOehqagyDTkmSJEmSJCmehEKQ7Tmd32bQKUmSJEmSJMWb8oFEyz4Oto4axKBTkiRJkiRJijfl53R+OQm2FwVbSw1h0ClJkiRJkiTFm6bdID0LthfAiqlBV1MjGHRKkiRJkiRJ8SYUguwDo1+7fR0w6JQkSZIkSZLiU/vB0Z8dSAQYdEqSJEmSJEnxqXwg0ZefQWlJsLXUAIlBFyBJkiRJkiTpR2i6D+xzArTqGx1IlJAUdEWBMuiUJEmSJEmS4lE4DKc+FXQVNYZb1yVJkiRJkiTFPYNOSZIkSZIkSXHPoFOSJEmSJElS3DPolCRJkiRJkhT3DDolSZIkSZIkxT2DTkmSJEmSJElxz6BTkiRJkiRJUtwz6JQkSZIkSZIU9ww6JUmSJEmSJMU9g84qcP/995OTk0O/fv2CLkWSJEmSJEmqEww6q8All1zCnDlzmDRpUtClSJIkSZIkSXWCQackSZIkSZKkuGfQKUmSJEmSJCnuGXRKkiRJkiRJinsGnZIkSZIkSZLinkGnJEmSJEmSpLhn0ClJkiRJkiQp7hl0SpIkSZIkSYp7Bp2SJEmSJEmS4p5BpyRJkiRJkqS4Z9ApSZIkSZIkKe4lBl1AbRaJRADYsmVLtX1mSUkJ27ZtY8uWLSQlJVXb50qVzbWs2sT1rNrE9azaxPWs2sK1rNrE9axvK8/VynO272PQWYW2bt0KQNu2bQOuRJIkSZIkSYpfW7duJTMz83tfE4rsThyqH6WsrIyVK1fSoEEDQqFQtXzmli1baNu2LV9++SUZGRnV8plSVXAtqzZxPas2cT2rNnE9q7ZwLas2cT3r2yKRCFu3bqVVq1aEw99/CqcdnVUoHA7Tpk2bQD47IyPD3xBUK7iWVZu4nlWbuJ5Vm7ieVVu4llWbuJ71TT/UyVnOYUSSJEmSJEmS4p5BpyRJkiRJkqS4Z9BZy6SkpHDjjTeSkpISdCnSXnEtqzZxPas2cT2rNnE9q7ZwLas2cT1rbziMSJIkSZIkSVLcs6NTkiRJkiRJUtwz6JQkSZIkSZIU9ww6JUmSJEmSJMU9g05JkiRJkiRJcc+gs4a7//77ad++PampqQwYMIDPPvvsO187ZMgQQqHQTj+OPfbY2GsikQg33HADLVu2JC0tjaFDh7JgwYLquBWpUtdzSUkJV199Nb169aJevXq0atWKs846i5UrV1bX7aiOq+zfn7/poosuIhQKcc8991RR9dLXqmItz507lxNOOIHMzEzq1atHv379WL58eVXfilTp6zkvL49LL72UNm3akJaWRk5ODg8++GB13Iq0R+sZ4J577qFbt26kpaXRtm1brrjiCgoLC/fqPaXKUtnr+S9/+Qv9+vWjQYMGNGvWjOHDhzN//vyqvg3Fg4hqrOeeey6SnJwceeyxxyKzZ8+OjBw5MtKwYcPImjVrdvn69evXR1atWhX7MWvWrEhCQkLk8ccfj73mtttui2RmZkZefvnlyPTp0yMnnHBCpEOHDpGCgoJquivVVZW9njdt2hQZOnRo5N///ndk3rx5kU8++STSv3//yP7771+Nd6W6qip+fy734osvRvr06RNp1apV5O67767aG1GdVxVreeHChZHGjRtHrrrqqsjUqVMjCxcujLzyyivf+Z5SZamK9Txy5MhIp06dIh988EFkyZIlkX/84x+RhISEyCuvvFJNd6W6ak/X89NPPx1JSUmJPP3005ElS5ZE3n777UjLli0jV1xxxY9+T6myVMV6HjZsWOTxxx+PzJo1KzJt2rTIMcccE2nXrl0kLy+vum5LNZRBZw3Wv3//yCWXXBL7vrS0NNKqVavIX/7yl926/u677440aNAg9j/0srKySIsWLSJ33HFH7DWbNm2KpKSkRJ599tnKLV76lspez7vy2WefRYDIsmXL9rpe6ftU1Xr+6quvIq1bt47MmjUrkp2dbdCpKlcVa/nUU0+N/OIXv6j0WqUfUhXruUePHpFbbrmlwuv222+/yHXXXVc5RUvfYU/X8yWXXBI57LDDKjx25ZVXRgYNGvSj31OqLFWxnr9t7dq1ESDy4YcfVk7RiltuXa+hiouLmTJlCkOHDo09Fg6HGTp0KJ988sluvcejjz7KaaedRr169QBYsmQJq1evrvCemZmZDBgwYLffU/oxqmI978rmzZsJhUI0bNhwb0uWvlNVreeysjLOPPNMrrrqKnr06FHpdUvfVhVruaysjNdff52uXbsybNgwmjVrxoABA3j55Zer4hakmKr6vfnAAw/k1VdfZcWKFUQiET744AO++OILjjzyyEq/B6ncj1nPBx54IFOmTIltB168eDFvvPEGxxxzzI9+T6kyVMV63pXNmzcD0Lhx40qsXvHIoLOGys3NpbS0lObNm1d4vHnz5qxevfoHr//ss8+YNWsWF1xwQeyx8ut+7HtKP1ZVrOdvKyws5Oqrr2bEiBFkZGTsdc3Sd6mq9fx///d/JCYmctlll1VqvdJ3qYq1vHbtWvLy8rjttts46qijGDNmDCeeeCInnXQSH374YaXfg1Suqn5vvvfee8nJyaFNmzYkJydz1FFHcf/993PwwQdXav3SN/2Y9Xz66adzyy23cNBBB5GUlESnTp0YMmQI11577Y9+T6kyVMV6/raysjIuv/xyBg0aRM+ePSv9HhRfDDprqUcffZRevXrRv3//oEuR9toPreeSkhJOOeUUIpEIDzzwQDVXJ+2ZXa3nKVOm8Le//Y0nnniCUCgUYHXS7tvVWi4rKwPgpz/9KVdccQV9+/bl97//Pccdd5wDXFSjfdefNe69914mTpzIq6++ypQpU7jrrru45JJLePfddwOqVNq1sWPH8uc//5nRo0czdepUXnzxRV5//XVuvfXWoEuT9tierudLLrmEWbNm8dxzz1VzpaqJEoMuQLuWlZVFQkICa9asqfD4mjVraNGixfdem5+fz3PPPcctt9xS4fHy69asWUPLli0rvGffvn0rp3BpF6piPZcrDzmXLVvG+++/bzenqlxVrOdx48axdu1a2rVrF3ustLSU3/zmN9xzzz0sXbq00uqXylXFWs7KyiIxMZGcnJwKj++zzz58/PHHlVO4tAtVsZ4LCgq49tpreemll2KT2Hv37s20adO48847K2zDlCrTj1nP119/PWeeeWasK7lXr17k5+dz4YUXct111+3V/0akvVEV6zkc/rpn79JLL+W1117jo48+ok2bNlV3I4obdnTWUMnJyey///689957scfKysp47733GDhw4Pde+/zzz1NUVMQvfvGLCo936NCBFi1aVHjPLVu28Omnn/7ge0p7oyrWM3wdci5YsIB3332XJk2aVHrt0rdVxXo+88wzmTFjBtOmTYv9aNWqFVdddRVvv/12ldyHVBVrOTk5mX79+jF//vwKj3/xxRdkZ2dXXvHSt1TFei4pKaGkpKTCX6gBEhISYt3LUlX4Met527Ztu1yrAJFIZK/+NyLtjapYz+U/X3rppbz00ku8//77dOjQoYruQHEn0FFI+l7PPfdcJCUlJfLEE09E5syZE7nwwgsjDRs2jKxevToSiUQiZ555ZuT3v//9TtcddNBBkVNPPXWX73nbbbdFGjZsGHnllVciM2bMiPz0pz+NdOjQIVJQUFCl9yJV9nouLi6OnHDCCZE2bdpEpk2bFlm1alXsR1FRUZXfj+q2qvj9+ducuq7qUBVr+cUXX4wkJSVFHnroociCBQsi9957byQhISEybty4Kr0XqSrW8yGHHBLp0aNH5IMPPogsXrw48vjjj0dSU1Mjo0ePrtJ7kfZ0Pd94442RBg0aRJ599tnI4sWLI2PGjIl06tQpcsopp+z2e0pVpSrW88UXXxzJzMyMjB07tsLfBbdt21bt96eaxaCzhrv33nsj7dq1iyQnJ0f69+8fmThxYuy5Qw45JHL22WdXeP28efMiQGTMmDG7fL+ysrLI9ddfH2nevHkkJSUlcvjhh0fmz59flbcgxVTmel6yZEkE2OWPDz74oIrvRKr835+/zaBT1aUq1vKjjz4a6dy5cyQ1NTXSp0+fyMsvv1xV5UsVVPZ6XrVqVeScc86JtGrVKpKamhrp1q1b5K677oqUlZVV5W1IkUhkz9ZzSUlJ5Kabbop06tQpkpqaGmnbtm1k1KhRkY0bN+72e0pVqbLX83f9XfDxxx+vvptSjRSKRHb0/UqSJEmSJElSnPKMTkmSJEmSJElxz6BTkiRJkiRJUtwz6JQkSZIkSZIU9ww6JUmSJEmSJMU9g05JkiRJkiRJcc+gU5IkSZIkSVLcM+iUJEmSJEmSFPcMOiVJkiRJkiTFPYNOSZIkaQ/cdNNN9O3bN/b9Oeecw/DhwwOrR5IkSVEGnZIkSZIkSZLinkGnJEmSao3i4uKgS5AkSVJADDolSZIUt4YMGcKll17K5ZdfTlZWFsOGDWPWrFkcffTR1K9fn+bNm3PmmWeSm5sbu6asrIzbb7+dzp07k5KSQrt27fjTn/4Ue/7qq6+ma9eupKen07FjR66//npKSkqCuD1JkiTtAYNOSZIkxbUnn3yS5ORkxo8fz2233cZhhx3Gvvvuy+TJk3nrrbdYs2YNp5xySuz111xzDbfddhvXX389c+bM4ZlnnqF58+ax5xs0aMATTzzBnDlz+Nvf/sbDDz/M3XffHcStSZIkaQ+EIpFIJOgiJEmSpB9jyJAhbNmyhalTpwLwxz/+kXHjxvH222/HXvPVV1/Rtm1b5s+fT8uWLWnatCn33XcfF1xwwW59xp133slzzz3H5MmTgegwopdffplp06YB0WFEmzZt4uWXX67Ue5MkSdKeSQy6AEmSJGlv7L///rGvp0+fzgcffED9+vV3et2iRYvYtGkTRUVFHH744d/5fv/+97/5+9//zqJFi8jLy2P79u1kZGRUSe2SJEmqPAadkiRJimv16tWLfZ2Xl8fxxx/P//3f/+30upYtW7J48eLvfa9PPvmEM844g5tvvplhw4aRmZnJc889x1133VXpdUuSJKlyGXRKkiSp1thvv/3473//S/v27UlM3PmPul26dCEtLY333ntvl1vXJ0yYQHZ2Ntddd13ssWXLllVpzZIkSaocDiOSJElSrXHJJZewYcMGRowYwaRJk1i0aBFvv/025557LqWlpaSmpnL11Vfzu9/9jn/+858sWrSIiRMn8uijjwLRIHT58uU899xzLFq0iL///e+89NJLAd+VJEmSdodBpyRJkmqNVq1aMX78eEpLSznyyCPp1asXl19+OQ0bNiQcjv7R9/rrr+c3v/kNN9xwA/vssw+nnnoqa9euBeCEE07giiuu4NJLL6Vv375MmDCB66+/PshbkiRJ0m5y6rokSZIkSZKkuGdHpyRJkiRJkqS4Z9ApSZIkSZIkKe4ZdEqSJEmSJEmKewadkiRJkiRJkuKeQackSZIkSZKkuGfQKUmSJEmSJCnuGXRKkiRJkiRJinsGnZIkSZIkSZLinkGnJEmSJEmSpLhn0ClJkiRJkiQp7hl0SpIkSZIkSYp7Bp2SJEmSJEmS4p5BpyRJkiRJkqS4Z9ApSZIkSZIkKe4ZdEqSJEmSJEmKewadkiRJkiRJkuKeQackSZIkSZKkuGfQKUmSJEmSJCnuGXRKkiRJkiRJinsGnZIkSZIkSZLinkGnJEmSJEmSpLhn0ClJkiRJkiQp7hl0SpIkSZIkSYp7Bp2SJEmSJEmS4p5BpyRJkiRJkqS4Z9ApSZIkSZIkKe4ZdEqSJEmSJEmKewadkiRJkiRJkuKeQackSZIkSZKkuGfQKUmSJEmSJCnuGXRKkiRJkiRJinsGnZIkSZIkSZLinkGnJEmSJEmSpLhn0ClJkiRJkiQp7hl0SpIkSZIkSYp7Bp2SJEmSJEmS4p5BpyRJkiRJkqS4Z9ApSZIkSZIkKe4ZdEqSJEmSJEmKewadkiRJkiRJkuKeQackSZIkSZKkuGfQKUmSJEmSJCnuGXRKkiRJkiRJinsGnZIkSZIkSZLinkGnJEmSJEmSpLhn0ClJkqQ6YciQIfTs2TPoMiRJklRFDDolSZKkH6mgoIDzzz+fnj17kpmZSf369enTpw9/+9vfKCkpqfDa9957j/POO4+uXbuSnp5Ox44dueCCC1i1atV3vv+9995LZmZm7L1WrVrFhRdeSIcOHUhLS6NTp05ceeWVrF+/vkrvU5IkKR4kBl2AJEmSFK8KCgqYPXs2xxxzDO3btyccDjNhwgSuuOIKPv30U5555pnYa6+++mo2bNjAz3/+c7p06cLixYu57777eO2115g2bRotWrTY6f1ff/11jjzySJKSksjLy2PgwIHk5+czatQo2rZty/Tp07nvvvv44IMPmDJlCuGwfQySJKnuMuiUJElS3MrPz6devXqBfX7jxo2ZOHFihccuuugiMjMzue+++/jrX/8aCzD/+te/ctBBB1UII4866igOOeQQ7rvvPv74xz9WeJ9t27bx4Ycf8sADDwDw6quvsmzZMl577TWOPfbYCjXccsstTJ8+nX333beqblWSJKnG8598JUmStMe2bt3K5ZdfTvv27UlJSaFZs2YcccQRTJ06tcLrPv30U4466igyMzNJT0/nkEMOYfz48RVes2zZMkaNGkW3bt1IS0ujSZMm/PznP2fp0qUVXvfEE08QCoX48MMPGTVqFM2aNaNNmzax5998800OOeQQGjRoQEZGBv369avQUVluzpw5HHrooaSnp9O6dWtuv/32nV6zfPly5s2b96N/fdq3bw/Apk2bYo8dfPDBO3VcHnzwwTRu3Ji5c+fu9B7vvfceRUVFHH300QBs2bIFgObNm1d4XcuWLQFIS0v70fVKkiTVBnZ0SpIkaY9ddNFFvPDCC1x66aXk5OSwfv16Pv74Y+bOnct+++0HwPvvv8/RRx/N/vvvz4033kg4HObxxx/nsMMOY9y4cfTv3x+ASZMmMWHCBE477TTatGnD0qVLeeCBBxgyZAhz5swhPT29wmePGjWKpk2bcsMNN5Cfnw9EQ9DzzjuPHj16cM0119CwYUM+//xz3nrrLU4//fTYtRs3buSoo47ipJNO4pRTTuGFF17g6quvplevXrFAEeCss87iww8/JBKJ7NavR3FxMVu2bKGgoIDJkydz5513kp2dTefOnb/3ury8PPLy8sjKytrpuTfeeIP9998/FmyWB6W//vWvueuuu2jTpg0zZszgT3/6E8OHD6d79+67VaskSVJtFYrs7p/eJEmSpB0aNmzIL37xC+67775dPh+JROjWrRsdO3bkzTffJBQKAdEzLXv06EHnzp0ZM2ZM7LFvdyNOnDiRgQMH8s9//pMzzzwTiIaZ5557LgcddBBjx44lISEBgM2bN9O2bVtycnIYO3YsqampFeoo/+whQ4bw4YcfVnjP4uJisrOzGTRoEC+88ELsuvLX7u4flZ977jlGjBgR+/4nP/kJjz32GL169fre6/74xz9y/fXX895773HYYYdVeC47O5tzzz2Xm266KfbYo48+ym9/+9sKnaJnn302jzzyCImJ9jBIkqS6zT8NSZIkaY81bNiQTz/9lJUrV9KqVaudnp82bRoLFizgD3/4w04TwQ8//HCeeuopysrKCIfDFULOkpIStmzZQufOnWnYsCFTp06NhZLlRo4cGQs5Ad555x22bt3K73//+wohJxALOcvVr1+fX/ziF7Hvk5OT6d+/P4sXL67wurFjx+7eL8QOhx56KO+88w6bNm3ivffeY/r06bFu0+/y0UcfcfPNN3PKKafsFHLOmjWL5cuXVziLE6B169b079+fY445huzsbMaNG8ff//53srKyuPPOO/eoZkmSpNrGoFOSJEl77Pbbb+fss8+mbdu27L///hxzzDGcddZZdOzYEYAFCxYA0W7D77J582YaNWpEQUEBf/nLX3j88cdZsWJFhS7KzZs373Rdhw4dKny/aNEiAHr27PmDdbdp02an8LNRo0bMmDHjB6/9Ps2bN49tMT/55JP585//zBFHHMGCBQt2OU193rx5nHjiifTs2ZNHHnlkp+dff/11mjdvzk9+8pPYY+PHj+e4445j4sSJsceHDx9ORkYGN998M+eddx45OTl7dR+SJEnxzGFEkiRJ2mOnnHIKixcv5t5776VVq1bccccd9OjRgzfffBOAsrIyAO644w7eeeedXf6oX78+AL/61a/405/+xCmnnMJ//vMfxowZwzvvvEOTJk1i7/NNezN055udoN9U2ac5nXzyyeTl5fHKK6/s9NyXX37JkUceSWZmJm+88QYNGjTY6TVvvPEGRx11VIVQ9h//+MdO4SfACSecQCQSYcKECZV6D5IkSfHGjk5JkiT9KC1btmTUqFGMGjWKtWvXst9++/GnP/2Jo48+mk6dOgGQkZHB0KFDv/d9XnjhBc4++2zuuuuu2GOFhYUVzqH8PuWfNWvWrB8c/lNdCgoKgJ07UtevX8+RRx5JUVER7733Xmxi+jdt2rSJCRMmcOmll1Z4fM2aNZSWlu70+pKSEgC2b99eWeVLkiTFJTs6JUmStEdKS0t3CvCaNWtGq1atKCoqAmD//fenU6dO3HnnneTl5e30HuvWrYt9nZCQsFNH5b333rvLUG9XjjzySBo0aMBf/vIXCgsLKzz3Yzs1ly9fzrx5837wdbm5ubv8jPLt6N/svszPz+eYY45hxYoVvPHGG3Tp0mWX71k+pOnII4+s8HjXrl1Zs2bNTueHPvvsswDsu+++P1ivJElSbWZHpyRJkvbI1q1badOmDSeffDJ9+vShfv36vPvuu0yaNCnWlRkOh3nkkUc4+uij6dGjB+eeey6tW7dmxYoVfPDBB2RkZPC///0PgOOOO46nnnqKzMxMcnJy+OSTT3j33Xdp0qTJbtWTkZHB3XffzQUXXEC/fv04/fTTadSoEdOnT2fbtm08+eSTe3yPZ5111m5NXf/Xv/7Fgw8+yPDhw+nYsSNbt27l7bff5p133uH444+vMGTojDPO4LPPPuO8885j7ty5zJ07N/Zc/fr1GT58OBA9n/Oggw4iMzOzwmddeumlPP744xx//PH86le/Ijs7mw8//JBnn32WI444ggEDBuzxfUqSJNUmBp2SJEnaI+np6YwaNYoxY8bw4osvUlZWRufOnRk9ejQXX3xx7HVDhgzhk08+4dZbb+W+++4jLy+PFi1aMGDAAH75y1/GXve3v/2NhIQEnn76aQoLCxk0aBDvvvsuw4YN2+2azj//fJo1a8Ztt93GrbfeSlJSEt27d+eKK66o1Hv/toMOOogJEybw7LPPsmbNGhITE+nWrRt//etf+dWvflXhtdOmTQPgscce47HHHqvwXHZ2NsOHDycSifDWW2/x29/+dqfP6tatG1OmTOEPf/gD//rXv1i9ejWtWrXit7/9LTfffHOV3aMkSVK8CEUq++R1SZIkST/KZ599xoABA5g9e7YT1CVJkvaQZ3RKkiRJNcif//xnQ05JkqQfwY5OSZIkSZIkSXHPjk5JkiRJkiRJcc+gU5IkSZIkSVLcM+iUJEmSJEmSFPcMOiVJkiRJkiTFvcSgC6jNysrKWLlyJQ0aNCAUCgVdjiRJkiRJkhRXIpEIW7dupVWrVoTD39+zadBZhVauXEnbtm2DLkOSJEmSJEmKa19++SVt2rT53tcYdFahBg0aANH/EBkZGQFXE6ySkhLGjBnDkUceSVJSUtDlKM65nlSZXE+qTK4nVSbXkyqT60mVyfWkyuaa0vfZsmULbdu2jeVs38egswqVb1fPyMgw6CwpIT09nYyMDH/T0l5zPakyuZ5UmVxPqkyuJ1Um15Mqk+tJlc01pd2xO8dCOoxIkiRJkiRJUtwz6JQkSZIkSZIU9ww6JUmSJEmSJMU9z+iUJEmSJEkSAGVlZRQXF1frZ5aUlJCYmEhhYSGlpaXV+tmqGZKTkwmH974f06BTkiRJkiRJFBcXs2TJEsrKyqr1cyORCC1atODLL7/crYEzqn3C4TAdOnQgOTl5r97HoFOSJEmSJKmOi0QirFq1ioSEBNq2bVsp3XW7q6ysjLy8POrXr1+tn6uaoaysjJUrV7Jq1SratWu3V2G3QackSZIkSVIdt337drZt20arVq1IT0+v1s8u3y6fmppq0FlHNW3alJUrV7J9+3aSkpJ+9Pu4eiRJkiRJkuq48rMx93brsPRjlK+7vT2j1aBTkiRJkiRJAJ6RqUBU1roz6JQkSZIkSZIU9ww6JUmSJEmSpBrgnHPOYfjw4UGXEbcMOiVJkiRJkhS3VqxYwS9+8QuaNGlCWloavXr1YvLkybt87UUXXUQoFOKee+75wfedNGkShx9+OA0bNqRRo0YMGzaM6dOnV3L1qkwGnZIkSZIkSYpLGzduZNCgQSQlJfHmm28yZ84c7rrrLho1arTTa1966SUmTpxIq1atfvB98/LyOOqoo2jXrh2ffvopH3/8MQ0aNGDYsGGUlJRUxa2oEhh0SpIkSZIkKS793//9H23btuXxxx+nf//+dOjQgSOPPJJOnTpVeN2KFSv41a9+xdNPP01SUtIPvu+8efPYsGEDt9xyC926daNHjx7ceOONrFmzhmXLln3nddOnT+fQQw+lQYMGZGRksP/++8e6S2+66Sb69u1b4fX33HMP7du33+l9br75Zpo2bUpGRgYXXXQRxcXFsedeeOEFevXqRVpaGk2aNGHo0KHk5+cDX299/77r33rrLQ466CAaNmxIkyZNOO6441i0aFGFz//qq68YMWIEjRs3pl69evzkJz/h008/jT3/yiuvsN9++5GamkrHjh25+eab2b59+w/+ula1xKALkCRJkiRJUs0SiUQoKCmtls8qKyujoLiUxOLthMNh0pISdnsK96uvvsqwYcP4+c9/zocffkjr1q0ZNWoUI0eOrPD+Z555JldddRU9evTYrfft1q0bTZo04dFHH+Xaa6+ltLSURx99lH322WeXwWS5M844g3333ZcHHniAhIQEpk2btlvB6je99957pKamMnbsWJYuXcq5555LkyZN+NOf/sSqVasYMWIEt99+OyeeeCJbt25l3LhxRCKR3boeID8/nyuvvJLevXuTl5fHDTfcwIknnsi0adMIh8Pk5eVxyCGH0Lp1a1599VVatGjB1KlTKSsrA2DcuHGcddZZ/P3vf2fw4MEsWrSICy+8EIAbb7xxj+61shl0SpIkSZIkqYKCklJybng7kM+ec8sw0pN3L7JavHgxDzzwAFdeeSXXXnstkyZN4rLLLiM5OZmzzz4biHZ9JiYmctlll+12DQ0aNGDs2LEMHz6cW2+9FYAuXbrw9ttvk5j43bUtX76cq666iu7du8eu2VPJyck89thjpKen06NHD2655Rauuuoqbr31VlatWsX27ds56aSTyM7OBqBXr167fX04HOZnP/tZhdc/9thjNG3alDlz5tCzZ0+eeeYZ1q1bx6RJk2jcuDEAnTt3jr3+5ptv5ve//33s17djx47ceuut/O53vws86HTruiRJkiRJkuJSWVkZ++23H3/+85/Zd999ufDCCxk5ciQPPvggAFOmTOFvf/sbTzzxxHd2iR599NHUr1+f+vXrxzo+CwoKOP/88xk0aBATJ05k/Pjx9OzZk2OPPZaCggKA2DX169fnoosuAuDKK6/kggsuYOjQodx22207bQnfHX369CE9PT32/cCBA8nLy+PLL7+kT58+HH744fTq1Yuf//znPPzww2zcuHG3rwdYsGABI0aMoGPHjmRkZMQ6VJcvXw7AtGnT2HfffWMh57dNnz6dW265pcL9jxw5klWrVrFt27Y9vt/KZEenJEmSJEmSKkhLSmDOLcOq5bPKysrYumUrDTIaxLau766WLVuSk5NT4bF99tmH//73v0B0m/XatWtp165d7PnS0lJ+85vfcM8997B06VIeeeSRWHhZvs38mWeeYenSpXzyySeEw+HYY40aNeKVV17htNNOY9q0abH3zMjIAKLncJ5++um8/vrrvPnmm9x4440899xznHjiiYTD4QpbzIE9HmyUkJDAO++8w4QJExgzZgz33nsv1113HZ9++ikdOnTYrfc4/vjjyc7O5uGHH6ZVq1aUlZXRs2fP2DmeaWlp33t9Xl4eN998MyeddNJOz6Wmpu7R/VQ2g05JkiRJkiRVEAqFdnv7+N4qKytje3IC6cmJsVBxdw0aNIj58+dXeOyLL76Ibes+88wzGTp0aIXnhw0bxplnnsm5554LQOvWrXd6323bthEOhyt0gZZ/X35W5Te3c39T165d6dq1K1dccQUjRozg8ccf58QTT6Rp06asXr2aSCQSe99vhqXlpk+fTkFBQSxwnDhxIvXr16dt27ZA9L/NoEGDGDRoEDfccAPZ2dm89NJLXHnllT94/fr165k/fz4PP/wwgwcPBuDjjz+u8Pm9e/fmkUceYcOGDbvs6txvv/2YP3/+d95/kNy6LkmSJEmSpLh0xRVXMHHiRP785z+zcOFCnnnmGR566CEuueQSAJo0aULPnj0r/EhKSqJFixZ069btO9/3iCOOYOPGjVxyySXMnTuX2bNnc+6555KYmMihhx66y2sKCgq49NJLGTt2LMuWLWP8+PFMmjSJffbZB4AhQ4awbt06br/9dhYtWsT999/Pm2++udP7FBcXc/755zNnzhzeeOMNbrzxRi699FLC4TCffvopf/7zn5k8eTLLly/nxRdfZN26dbHP+KHrGzVqRJMmTXjooYdYuHAh77//fiwgLTdixAhatGjB8OHDGT9+PIsXL+a///0vn3zyCQA33HAD//znP7n55puZPXs2c+fO5bnnnuMPf/jDnv3HqwIGnZICt23KFDY+/zyF878gUlo9U/0kSZIkSfGvX79+vPTSSzz77LP07NmTW2+9lXvuuYczzjhjr963e/fu/O9//2PGjBkMHDiQwYMHs3LlSt566y1atmy5y2sSEhJYv349Z511Fl27duWUU07h6KOP5uabbwaiW+pHjx7N/fffT58+ffjss8/47W9/u9P7HH744XTp0oWDDz6YU089lRNOOIGbbroJiG6R/+ijjzjmmGPo2rUrf/jDH7jrrrs4+uijd+v6cDjMc889x5QpU+jZsydXXHEFd9xxR4XPT05OZsyYMTRr1oxjjjmGXr16cdttt5GQED1SYNiwYbz22muMGTOGfv36ccABB3D33XfHumiDFIp8+3AAVZotW7aQmZnJ5s2bY2c11FUlJSW88cYbHHPMMbHzLqRyq264kU3/+Q8AofR00nJySO3Tm7RevUnr3YvEli0rbBdwPakyuZ5UmVxPqkyuJ1Um15Mqk+updiosLGTJkiV06NCh2s9ZLCsrY8uWLWRkZOzx1nVVdM4557Bp0yZefvnloEvZI9+3/vYkX/OMTkmBS+nWlfQDDqBw5kzK8vPZNnky2yZPjj2f0DQrFnqm9upFYvfuAVZbM6zdtpYxS8fQM6sn+zTZh5SElKBLkiRJkiQpUAadkgLX+IwzaHzGGURKSylesoSCGTMpmDGdwhkzKfziC0rX5ZL3/vvkvf9+7Jr2TZuy5uOPSe/bl7TevUnp1o1wcnKAd1G9Jq+ezP9N+j8AEsOJdGvUjZ5ZPendtDc9s3rSPqM94ZD/EipJkiRJqjsMOiXVGKGEBFI6dyalc2cannQiAGWFhRTOnUvhjBnRAHTmTEqWLyd53Tq2/u81tv7vtei1SUmk5OxTofMzuX37Clvea5OGqQ0Z0mYIM3JnsKFwA7PXz2b2+tn8e/6/AWiQ1IAeWT3oldUr+qNpL7LSsgKuWpIkSZJUlZ544omgSwiUQaekGi2cmkr6vvuSvu++sccK165l3ONP0DstlaLZsymcPoPSTZsonD6Dwukz2Fh+bWYmaT17ktq7F2m9e5PWuzeJTZoEcyOV7MBWB3JgqwOJRCKsyl/FjNwZzFo3i5m5M5mzfg5bS7YycdVEJq6aGLumZb2WFYLPfRrvQ3pSeoB3IUmSJElS5THolBR3Eho1Ylv3bjTecfh5JBKh5KuvKJgxI9b5WThnDmWbN5M/fjz548fHrk1q1arCoKPUnBzC6fEb9oVCIVrVb0Wr+q04qv1RAGwv287CTQuZmTuTmetmMjN3Jos2LWJV/ipW5a9izLIxACSEEujcsDO9mvaKBaAdMzuSEE4I8pYkSZIkSfpRDDolxb1QKERy27Ykt21L5rHHAhApKaFowQIKyoPPmTMoWriIkpUrKVm5kq1vvhW9OCGBlC5dSOvVa0fnZx9SOncilBC/YV9iOJHujbvTvXF3ft715wDkl+QzO3d2NPzc8WPttrXM3zif+Rvn88IXLwCQnphOj6we0fM+s6Lnfbao1yLI25EkSZIkabcYdEqqlUJJSaTm5JCak0Oj004DoDQvj8JZsymY+XXn5/Y1ayiaN4+iefPg+eej16ank5aTQ+qO7e5pvXuR2LJlXJ/3WS+pHv1b9qd/y/6xx9bkr2FW7qzotvfcWczKncW27duYtHoSk1ZPir2uWVozemb1jHV+9mjSg/rJ9YO4DUmSJEmSvpNBp6Q6I6F+feodMIB6BwyIPVayZg2FM2dSMH0GBTNnUjhzJmX5+WybPJltkyd/fW1WFmm9epHWpzepvXqR1qsXCRkZQdxGpWlerznN6zXn8OzDASgtK2XJ5iUVuj4XbFzA2oK1vP/l+7z/ZXTqfYgQnRp2ioafO7a8d27UmaRwUpC3I0mSJEmq4ww6JdVpSc2bk9S8OQ2GDgUgUlZG8eLFOya8R4cbFX7xBaW5ueR98AF5H3wQuza5Q4cdE957k9anNynduhFOTg7qVvZaQjiBzo0607lRZ07sEp16X7C9gLnr534dfq6bycr8lSzctJCFmxby8sKXAUhNSGWfJvtUGHbUql6ruO6ClSRJkiTFF4NOSfqGUDhMSufOpHTuTMOTomFfWWEhhXPnVuj8LFm+nOIlSyhesoTNr7wavTYpiZR99qnQ+ZmcnU0oHA7ylvZKWmIa+zXfj/2a7xd7LLcgl1m5s2LB56zcWWwt2crnaz/n87Wfx17XOLUxvbJ6xc777JHVg8yUzCBuQ5IkSZJqjSFDhtC3b1/uueeeoEupcQw6JekHhFNTSd93X9L33Tf22PaNG6PB5zc6P0s3baJwx+T3jU8/Hb02I+PrQUc7Jr0nZmUFdSuVIistiyFthzCk7RAAyiJlLNuyLHre57roeZ/zNs5jQ+EGPvzqQz786sPYte0z2lfY8t6tcTeSE+K3C1aSJElS8D766CPuuOMOpkyZwqpVq3jppZcYPnw4ACUlJfzhD3/gjTfeYPHixWRmZjJ06FBuu+02WrVqFXuPL774gquuuorx48dTXFxM7969ufXWWzn00EO/97PffvttbrzxRmbPnk1qaioHH3wwd911F+3bt6/CO9Z3MeiUpB8hsVEj6h98MPUPPhiASCRCyVdfUTBjBoUzZkZ/njOHsi1byB8/nvzx42PXJrVqFR10VN75mZNDOD09qFvZa+FQmA6ZHeiQ2YHjOx0PQFFpEfM3zGdm7sxY+Ll863KWblnK0i1LeW3xawAkhZPo3rh7bLt7r6xetGvQzi3vkiRJknZbfn4+ffr04bzzzuOkk06q8Ny2bduYOnUq119/PX369GHjxo38+te/5oQTTmDyN+YyHHfccXTp0oX333+ftLQ07rnnHo477jgWLVpEixYtdvm5S5Ys4ac//SlXXnklTz/9NJs3b+aKK67gpJNOYurUqVV6z9o1g05JqgShUIjktm1JbtuWzGOPBSBSUkLRggUU7JjwXjhzBkULF1GyciUlK1ey9a23oheHw6R06UJa797Rzs/evUnp1IlQYvz+Fp2SkELvpr3p3bQ3Z+xzBgCbCjcxa/0sZq77etjRpqJNsa+ZF702IzmjQvDZM6snjVMbB3g3kiRJkmqyo48+mqOPPnqXz2VmZvLOO+9UeOy+++6jf//+LF++nHbt2pGbm8uCBQt49NFH6d27NwC33XYbo0ePZtasWd8ZdE6ZMoXS0lL++Mc/Et5xZNlvf/tbfvrTn1JSUkJS0q4Hto4dO5bf/e53zJ49m6SkJHr06MEzzzxDdnY255xzDps2beLll1+Ovf7yyy9n2rRpjB07NvbY9u3bufTSS3nqqadISkri4osv5pZbbok1jYwePZq7776bL7/8kszMTAYPHswLL7wARLe+9+zZE+A7r3/qqaf429/+xvz586lXrx6HHXYY99xzD82aNYvVMHv2bK6++mo++ugjIpEIffv25YknnqBTp04APPLII9x1110sWbKE9u3bc9lllzFq1Khd/ppUlvj9W7Qk1XChpCRSc3JIzcmh0WmnAVCal0/hrFnR7e47Oj+3r1lD0fz5FM2fD88/H702PZ20nJxo5+eO8DOxZcu47nRsmNqQg1ofxEGtDwKiXbBf5X1VIficu34uW4q3MH7leMav/LoLtnX91vTO6h0977Npb7o37k5qYmpQtyJJkiTVfpEIlGyrns8qK4t+VnEChMOQlA5V+HefzZs3EwqFaNiwIQBNmjShW7du/POf/2S//fYjJSWFf/zjHzRr1oz999//O99n//33JxwO8/jjj3POOeeQl5fHU089xdChQ78z5Ny+fTvDhw9n5MiRPPvssxQXF/PZZ5/t8d/1nnzySc4//3w+++wzJk+ezIUXXki7du0YOXIkkydP5rLLLuOpp57iwAMPZMOGDYwbN263r4folv9bb72Vbt26sXbtWq688krOOecc3njjDQBWrFjBwQcfzJAhQ3j//ffJyMhg/PjxbN++HYCnn36aG264gfvuu499992Xzz//nJEjR1KvXj3OPvvsPbrXPWHQKUnVKKF+PeodMIB6BwyIPVayZk2FQUeFM2dSlp/PtsmT2faNrRQJWVkVBh2l9epFQkZGELdRKUKhEG0btKVtg7Yc0/EYAEpKS/hi0xfMWjeLGbnRLe+LNy9mRd4KVuSt4M2lbwKQGEqkS6Mu9G7aOzbsqH1me8Kh+B38JEmSJNUoJdvgz61++HWVIAw0/OYD166E5HpV8lmFhYVcffXVjBgxgowdf58KhUK8++67DB8+nAYNGhAOh2nWrBlvvfUWjRo1+s736tChA2PGjOGUU07hl7/8JaWlpQwcODAWBu7Kli1b2Lx5M8cdd1ys83GfffbZ4/to27Ytd999N6FQiG7dujFz5kzuvvtuRo4cyfLly6lXrx7HHXccDRo0IDs7m32/MXPih64HOO+882Kv7dixI3//+9/p168feXl51K9fn/vvv5/MzEyee+65WKjbtWvX2DU33ngjd911V+wogQ4dOjBnzhz+8Y9/GHRKUm2W1Lw5Sc2b02DoUAAiZWUUL1myI/iMdn4Wzp9PaW4ueR98QN4HH8SuTW7ffkfwGe38TOnenXBy/A73SUpIokeTHvRo0oNTORWArcVbmZU7KzrsKHcGM9fNZH3heuZumMvcDXP59/x/A1A/qT49snrEBh31yupF0/SmQd6OJEmSpBqkpKSEU045hUgkwgMPPBB7PBKJcMkll9CsWTPGjRtHWloajzzyCMcffzyTJk2iZcuW9OjRg2XLlgEwePBg3nzzTVavXs3IkSM5++yzGTFiBFu3buWGG27g5JNP5p133uHLL78kJycn9jnXXnst1157Leeccw7Dhg3jiCOOYOjQoZxyyim0bNlyj+7lgAMOqNAFOnDgQO666y5KS0s54ogjyM7OpmPHjhx11FEcddRRnHjiiaR/YzbE912fkJDAlClTuOmmm5g+fTobN26krKwMgOXLl5OTk8O0adMYPHjwLjtX8/PzWbRoEeeff34sOIVoN2tmZuYe3eeeMuiUpBomFA6T0qkTKZ060fCkEwEoKyykcO7crye9z5hByfLlFC9dSvHSpWx+5dXotUlJpOyzT4XOz+TsbELh+O10bJDcgIGtBjKw1UAg+oeQ1fmrY9vdZ+bOZM76OeSV5PHpqk/5dNWnsWtb1GsRCz17ZvWkR5MepCfF7+AnSZIkqdokpUc7K6tBWVkZW7ZuJWNHNyVV8Gf28pBz2bJlsa3W5d5//31ee+01Nm7cGHt89OjRvPPOOzz55JP8/ve/54033qCkpASAtLQ0gFhX4+233x57r3/961+0bduWTz/9lJ/85CdMmzYt9lzjxtHZA48//jiXXXYZb731Fv/+97/5wx/+wDvvvMMBBxxAOBwmEonsVPueaNCgAVOnTmXs2LGMGTOGG264gZtuuolJkybFtut/n/z8fIYNG8awYcN4+umnadq0KcuXL2fYsGEUFxdX+DXYlby8PAAefvhhBgwYUOG5hISEPbqXPWXQKUlxIJyaSvq++5L+je0G2zdujJ73OWNGbNp76caNFM6YQeGMGWx8+unotRkZpPXsSWqf3qTt6PxMzMoK6lb2WigUomX9lrSs35Ij2x8JwPay7SzatIiZuTNjnZ+LNi1idf5qVuev5p1l0cPHw6EwnRp2ip332SurF+3qtQvydiRJkqSaKRSqsu3jOykrg6TS6OdVQZNGeci5YMECPvjgA5o0aVLh+W3bomeRhr/12eFwONbJmJ2dvdP7btu2badryoO8srIyEhMT6dy58y5r2nfffdl333255pprGDhwIM888wwHHHAATZs2ZdasWRVeO23atJ06Jz/99NMK30+cOJEuXbrEPj8xMZGhQ4cydOhQbrzxRho2bMj7778f20r+fdfPmzeP9evXc9ttt9G2bVuAChPqAXr37s2TTz65y6FLzZs3p1WrVixevJgzzjhjl/dfVQw6JSlOJTZqRP3Bg6k/eDAQ7XQsWbGCgunTo4OOZs6kcPZsyrZsIX/CBPInTIhdm9SqVXTQUa9epPXuRWqPHoTT47fTMTGcSLfG3ejWuBsndz0ZgG0l25i9fvbX4ee6GazZtoYFGxewYOMC/rvgvwCkJabRPNKcLz7/gj7N+tC7aW+apzeP68FPkiRJUl2Sl5fHwoULY98vWbKEadOm0bhxY1q2bMnJJ5/M1KlTee211ygtLWX16tVAtMMyOTmZgQMH0qhRI84++2xuuOEG0tLSePjhh1myZAnHHnvsd37usccey913380tt9wS27p+7bXX7vJMzG/W9tBDD3HCCSfQqlUr5s+fz4IFCzjrrLMAOOyww7jjjjv45z//ycCBA/nXv/7FrFmzdnq/5cuXc+WVV/LLX/6SqVOncu+993LXXXcB8Nprr7F48WIOPvhgGjVqxBtvvEFZWRndunXbrevbtWtHcnIy9957LxdddBGzZs3i1ltvrfD5l156Kffeey+nnXYa11xzDZmZmUycOJH+/fvTrVs3br75Zi677DIyMzM56qijKCoqYvLkyWzcuJErr7xyd//T7jGDTkmqJUKhEMlt2pDcpg2ZO/7POFJSQtGCBdHt7jOjnZ5FCxdRsnIlJStXsvWtt6IXh8OkdOkSDT179yatd29SOnUilBi//zeRnpROvxb96NeiX+yxtdvWxoLPmetmMmv9LPJL8lnKUpbOXQpzo6/LSsv6+qzPpr3o0aQHDZIbBHMjkiRJkr7X5MmTOfTQQ2PflwdpZ599NjfddBOvvho96qtv374Vrvvggw8YMmQIWVlZvPXWW1x33XUcdthhlJSU0KNHD1555RX69OnznZ972GGH8cwzz3D77bdz++23k56ezsCBA3nrrbe+c2t3eno68+bN48knn2T9+vW0bNmSSy65hF/+8pcADBs2jOuvv57f/e53FBYWct5553HWWWcxc+bMCu9z1llnUVBQQP/+/UlISODXv/41F154IQANGzbkxRdf5KabbqKwsJAuXbrw7LPP0qNHj926vmnTpjzxxBNce+21/P3vf2e//fbjzjvv5IQTTohd36RJE95//32uuuoqDjnkEBISEujbty+DBg0C4IILLiA9PZ077riDq666inr16tGrVy8uv/zy7/z1rAyhyLc3/qvSbNmyhczMTDZv3lzh7Ie6qKSkhDfeeINjjjlmlwfVSnvC9bR3SvPyKZw9m4IZX3d+bt/xL5rfFEpLI61Hjx3BZ3TKe2KrVrWq07EsUsaC9Qt45v1nCLcOM3vDbBZsXMD2yPYKrwsRokNmh9iE955Ne9K1UVeSwq4/VeTvT6pMridVJteTKpPrqXYqLCxkyZIldOjQgdTU1Gr97LKyMrZs2UJGRsZOW8FV+YYMGULfvn255557gi4l5vvW357ka/HbqiNJ+lES6tej3oD+1BvQP/ZYyZq1FM6c8XXn58xZlOXlsW3yZLZ94yyWhKysr7e79+pNWq+eJFTx1LyqFA6F6ZjZkf1S9uOY/tE/qBduL2TuhrnMXPf1sKMVeStYvHkxizcv5tVF0X8NTklIoXvj7hU6P9vUb1OrgmBJkiRJiicGnZIkkpo3I6n5UBoMHQpApKyM4iVLdkx4j3Z+Fs6fT2luLnkffEDeBx/Erk1u357U3r1I692HtN69SOnenXByclC3stdSE1PZt9m+7Nvs6zNw1hesZ/b62cxYNyO67T13JluKtzB93XSmr5see12jlEbRIUdNe8UC0MyU+A2CJUmSJCmeGHRKknYSCodJ6dSJlE6daHjicADKiooomjt3x5T3aOdnybLlFC9dSvHSpWx59X/Ri5OSSO3enbQdW95Te/UmuX02oTjegtIkrQkHtzmYg9scDEQHPy3furxC8Dlvwzw2Fm1k3IpxjFsxLnZtuwbtKgSf3Rt3JzkhfoNgSZIkSfFt7NixQZdQZQw69aNtyC9m0tINDOzUhIxUz2WRartwSgppffuS9o0DvLdv3EjhrFk7ws8ZFM6YSenGjRTOnEnhzJlsfHrHtRkZpPXsWaHzMzErK5gbqQShUIjsjGyyM7I5vtPxABSXFjN/w/zYdveZuTNZtmUZy7cuZ/nW5by++HUgOiG+e6Pu0fM+m/amV1Yv2mW0IxyK3yBYkiRJkmoCg079aGPnr+XK/0wnIRxiv3YNGdylKYO7ZNG7TUMSwp5RJ9UFiY0aUX/wYOoPHgxEOx1LVqygcMYMCqbPoGDmTApnz6ZsyxbyJ0wgf8KEr69t1ZK0Xr2/7vzMySFcr15Qt7LXkhOSo52bTXvFHttctDnW8TkzdyYz181kY9FGZq2fxaz1s3hu/nMANEhuQK+sXl8PO8rqSZO0JkHdiiRJkiTFJYNO7ZWOWfVYnJvPpKUbmbR0I3995wsy05I4qHMWg7tkMbhrU1o3TAu6TEnVJBQKkdymDclt2pBxzDEAREpKKFqw4OtBRzNmULRwEdtXrmLrylVsffvt6MXhMClduuzY7t6LtD59SOnUiVBi/P5fVWZKJoNaD2JQ60FANAhekbeCWbmzmJEb3fY+Z/0cthZvZcLKCUxY+XUQ3Lp+6+h5nzu2vO/TZB/SEv39VJIkSZK+S/z+7VGBO2m/Npy0Xxu+3LCNcQtyGbdgHeMX5rK5oITXZ67i9ZmrAOjYtB6DOjUhbVOIQ4q20zDJbe5SXRJKSiI1J4fUnBwanXYqAKV5+RTOnh2d9L6j83P76tUUzZ9P0fz58PwL0WvT0kjtkRPt/OzTm7RevUhs1SpuJ5uHQiHaNGhDmwZtOKrDUQCUlJWwcOPCCl2fizcvZkXeClbkreDtpdEgOCGUQJdGXb6e8p7Viw6ZHUgIJwR5S5IkSZJUYxh0aq+1bZzO6QPacfqAdmwvLWPGis2M+yKXjxasY9qXm1i8Lp/F6/KBBB7/ywfsn92IwV2acnCXpvRolUHYbe5SnZNQvx71BvSn3oD+scdK1qyNBp/lnZ8zZ1GWl0fB5CkUTJ7y9bVZWaT16hUbdJTWqycJmfE72TwpnMQ+TfZhnyb7cEq3UwDIK85j9vrZseBzZu5M1hWsY96GeczbMI/nv3gegHpJ9ejRpEeFLe/N6zUP8nYkSZIkKTAGnapUiQlh9mvXiP3aNeLXQ7uwuaCETxat58P5axgz40vWF8HExRuYuHgDd7w9n8b1kr/e5t6lKS0yU4O+BUkBSWrejKTmQ2kwdCgAkbIyipcsoWDGzFjnZ+H8+ZTm5pL3wQfkffBB7Nrk9u2jg452dH6mdO9OODl+J5vXT67PgJYDGNByABDd8r5m25oKXZ+z188mvySfz1Z/xmerP4td2yy9WSz07N20NzlNcqiXFL9nn0qSJEnS7jLoVJXKTEviqJ4tOLxbEwYkLKXnAUP4ZMlGPlqQyyeL1rMhv5hXp6/k1ekrAejavD4Hd2nK4K5N6d++MWnJbsmU6qpQOExKp06kdOoEJw4HoKyoiKK5c3dMeY92fpYsW07x0qUUL13Kllf/F704KYnU7t2/HnTUqzfJ7bMJheNzsnkoFKJFvRa0qNeCI7KPAKC0rJRFmxdFz/tcFz3vc8GmBazdtpZ3l7/Lu8vfBSAcCtMxs2N0u3vT6Jb3zg07kxj2jwCSJElSPBoyZAh9+/blnnvuCbqUGse/5fyAJUuWcN5557FmzRoSEhKYOHEi9eJ4KnCQQiHIbpJO5xaZnDmwPSWlZXy+fBPjFqzjowW5zPhqE1+syeOLNXk88vESkhPD9G/fmIO7Rrs9u7doELfn8kmqHOGUFNL69iWtb9/YY9s3bqRw1iwKZsygcMZMCmbMoHTjRgpnzqRw5kw2Pr3j2owM0nr2jHZ+9o5Oe0/MygrmRipBQjiBro260rVRV07qchIA20q2MWf9nArDjlblr2LhpoUs3LSQlxa+BEBqQio5TXKik96bRre9t6zX0t9jJUmSFJc++ugj7rjjDqZMmcKqVat46aWXGD58OAAlJSX84Q9/4I033mDx4sVkZmYydOhQbrvtNlq1ahV7jy+++IKrrrqK8ePHU1xcTO/evbn11ls59NBDv/ezI5EId911Fw899BDLli0jKyuLUaNGcd1111XlLes7GHT+gHPOOYc//vGPDB48mA0bNpCSkhJ0SbVGUkKY/h0a079DY35zZDc25hczYdF6PvpiHeMWrGPl5kI+XpjLxwtzgXk0bZDC4M5ZDO6axUGdm9K0gf8tJEFio0bUHzyY+oMHA9E/aJSsWEFhedfnjBkUzp5N2ZYt5E+YQP6EryebJ7ZqSUqPnjRKSqKgWTMSevcmHMf/mJWelM5PWvyEn7T4Seyx3ILc2DmfM3NnMit3FnkleUxdO5Wpa6fGXtcktUms67NnVk96ZvUkIzkjiNuQJEmS9kh+fj59+vThvPPO46STTqrw3LZt25g6dSrXX389ffr0YePGjfz617/mhBNOYPLkybHXHXfccXTp0oX333+ftLQ07rnnHo477jgWLVpEixYtvvOzf/3rXzNmzBjuvPNOevXqxYYNG9iwYUOV3au+n0Hn95g9ezZJSUkM3vGX58aNGwdcUe3WqF4yx/ZuybG9WxKJRFi0Lp9xC9Yxbsc293Vbi3jx8xW8+PkKAHJaZjC4axYHd2nK/tmNSE1ym7uk6Dbv5DZtSG7ThoxjjgEgUlJC0cKFOya8Rzs/ixYuZPvKVWxfuYqmwIo33oBwmJTOnUnr05vUXtHOz5TOnQklxu//XWalZXFou0M5tF30X6LLImUs3bK0Qvj5xYYvWF+4nrFfjWXsV2Nj17bPaE/vpr1jw466NupKUkJSQHciSZKk6hSJRCjYXlAtn1VWVkbB9gISSxIJh8OkJabt0W6jo48+mqOPPnqXz2VmZvLOO+9UeOy+++6jf//+LF++nHbt2pGbm8uCBQt49NFH6d27NwC33XYbo0ePZtasWd8ZdM6dO5cHHniAWbNm0a1bNwA6dOjwg/WOHTuW3/3ud7HcqUePHjzzzDNkZ2dzzjnnsGnTJl5++eXY6y+//HKmTZvG2LFjY49t376dSy+9lKeeeoqkpCQuvvhibrnlltiv2+jRo7n77rv58ssvyczMZPDgwbzwwgtAdOt7z549Ab7z+qeeeoq//e1vzJ8/n3r16nHYYYdxzz330KxZs1gNs2fP5uqrr+ajjz4iEonQt29fnnjiCTp16gTAI488wl133cWSJUto3749l112GaNGjfrBX5+9Eb9/c9sN39e6XO7+++/njjvuYPXq1fTp04d7772X/v2jU4AXLFhA/fr1Of7441mxYgUnn3wy1157bQB3UveEQiE6N6tP52b1OXdQB4q2lzJl2UbGLchl3IJ1zFqxhTmroj/+8eFiUpPCHNCxyY5p7ln/z959x0dVZg0c/90pmZn0MumF9EIqJQGlCGIBxNVdAXUt2Bs2WHt3RUUFxIK9NwR1d1VE14IQQHpLIZBCekJ678nM+8cNg6zltQCThPPdz/PZ5Xpv5jzZMSQn55yHSB9nacEUQtgoej3GuDiMcXF4XHA+AH2tbXRmZ9O2exdF33yLR00NvVVVdOXm0pWbCx+p3wQoJhPG+OHqQUf9be+6gIBB+zXm0MzOcLdwzok8B4Cuvi5y6nJsLe+ZNZmUtZZR1FxEUXMRnxV8BoCDxoFYr1i18rN/BbsED9rPhRBCCCGE+GUdvR2M+WCMXV57y9+34Kh3PGYfv6mpCUVRcHd3B8DLy4uYmBjeeecdRo4cicFg4OWXX8bHx4dRo0b94sf5/PPPCQ8PZ9WqVUydOhWr1cppp53Gk08++YvFcr29vZx77rlcffXVLF++nO7ubrZu3fq7v6d+++23ufLKK9m6dSvbt2/nmmuuISQkhKuvvprt27dz88038+6773LyySdTX1/P+vXrf/PzoLb8P/LII8TExFBdXc38+fO57LLLWL16NQDl5eVMnDiRSZMmsWbNGlxdXdm4cSO9vb0AvP/++zzwwAM8//zzjBgxgl27dnH11Vfj5OTEnDlzftdef48hnej8tdJlgBUrVjB//nxeeuklxowZw9KlSznzzDPZv38/Pj4+9Pb2sn79enbv3o2Pjw9Tp04lNTWV008/3Q67ObEZdFpOjjBzcoSZO6fGUtvaxcb8WtJz1cRndUsXa/fXsHZ/DQD+bkbbSe7jIs14Og3e05eFEMeG1tkJpzFpOIwcwSY/P0ZMnw71DXRmZfa3vO+hMzMLS2srHdt30LF9x+FnvbwwJSb2z/tMxpSYgNbNzY67+XMMWgMpPimk+KTYrjV0Ntha3Q/N+2zqaiKjJoOMmgzbfW4GN1vFZ6JZbXv3MHrYYRdCCCGEEEL8/zo7O7nzzju58MILcXVVRzUpisK3337Lueeei4uLCxqNBh8fH7766is8PH75e9sDBw5QXFzMRx99xDvvvENfXx/z5s1j5syZrFmz5mefaW5upqmpiRkzZtgqH+Pi4n73PoKDg3n66adRFIWYmBgyMzN5+umnufrqqykpKcHJyYkZM2bg4uLCsGHDGDFixG9+HuCKK66w3RseHs6zzz5Lamoqra2tODs7s2zZMtzc3Pjwww/R69Wur+joaNszDz74IIsXL7bl48LCwti7dy8vv/yyJDr/qF8rXQZYsmQJV199NZdffjkAL730El988QVvvPEGd911F4GBgYwePZrg4GAApk+fzu7du38x0dnV1UVXV5ftz83NzYCaBe/p6Tla2xqUDu3/aH0e3Awapsf7MD3eB6vVSl51K+vz69iQX8e2ogYqmzpZub2MldvLUBRICHBlfIQX46O8SAlyx0E3OE9eFqqj/X4SJ7Yfv5/0nh4YJ07EOHEiHoDVYqGnqJjOzAy6MrPozMqia/9++urqaF27ltYftY7ohw3DmJiIISEBY2IChthYFIfB+0sWZ60zJ/mexEm+JwFq61JpaynZddlk1WaRVZfF/ob9NHU1sbF8IxvLN9qeDXIOIsErgQSvBOK94on1jMWgPTHmKsvXJ3E0yftJHE3yfhJHk7yfhqaenh6sVisWiwWLxYJBY2DTBZuO2+u3tLTg4uICgEFjwGKx/OGPdWgP/6unp4dZs2ZhtVpZtmyZ7R6r1coNN9yAt7c369atw2Qy8frrr3P22WezZcsW/P39SUxMpLi4GIDx48ezevVq+vr66Orq4q233rIl+V599VVSU1PJycnBZDLZWsQB7r77bu6++27mzJnDmWeeyWmnncZpp53GrFmz8Pf3t8Vy6P+HQ6xWq21fh4wZM8Z276E/L168mJ6eHqZMmcKwYcMIDw/nzDPP5Mwzz+Svf/0rjo6Ov+l5rVbLjh07ePjhh8nIyKChocH22kVFRQwfPpxdu3Yxfvx4tFrtTz7XbW1tFBQUcOWVV9oSp6BWs7q5uf3s/zcWi0U9b6H/9f/3/7ffakgnOn9Nd3c3O3bs4O6777Zd02g0nHbaaWzapP6LnJqaSnV1NQ0NDbi5uZGens611177ix/z8ccf5+GHH/7J9a+//vqIN9OJ7H/nYhxN/sAsbzjHEw60KOxrVNjXpFDZrpBZ3kxmeTMvphdi0FiJcrMS42Yl1t2Kt1E9EV4MPsfy/SROPL/6ftLrYeQIGDkCpacHQ0UlxrJSjKWlGEtKcairo6e4mJ7iYlpWrQLAqtXS5e9PZ3AwHcFBdIaE0OPlBZrB/4uW+P7/9Lr0UtVXRWlfKWW9ZZT1lVFrqaWstYyy1jK+Kv4KAA0a/LR+BGuDCdQFEqwNxkvjhUYZ/J+LXyJfn8TRJO8ncTTJ+0kcTfJ+Glp0Oh1+fn60trbS3d193F/fpDPR26G2PbfQ8qc+VkdHh6347JCenh4uv/xyioqK+OwzdTTToXvWrVvHF198QWFhoa3K8/HHH+frr7/mlVdeYd68eSxfvtzWlm00GmlubsbT09P2eTv0sQIDAwHYt28fEyZMID093RaDh4cHzc3NLF26lCuuuIJvv/2WDz74gPvvv59//etfpKam0tfXR09PzxHxt7W10dvba7vW29v7k3s6Ojpse9JqtaxZs4YNGzawZs0aHnjgAR566CHWrFmDm5vb//t8Z2cnU6dO5dRTT+Wll17CbDZTVlbGeeedR0NDA83Nzej1+p98jEOqq6sBWLp0KaNHjz7in2m12p99pru7m46ODtLT022f50Pa29t/cv8vOWETnbW1tfT19eHr63vEdV9fX/bt2weo/5I/9thjTJw4EavVyhlnnMGMGTN+8WPefffdzJ8/3/bn5uZmgoODOeOMM2z/opyoenp6+Oabbzj99NNtJc3HS1VzJz8U1LMhv44NBbXUt/WQ1aCQ1aD+8yB3I+MizYyP9OKkcE/cTHLQxkBnz/eTGHqOxvupr7GRzuxsujIy6czKojMzE0tDA8ayMoxlZbj3/yJc4+KCIT4eY2KiWvWZmIjObD6Ku7G/lu4WteqzLsu26jvrqeiroKKvAvq/Z3bWOxPvFU+8VzyJXonEe8VjNg3+z4V8fRJHk7yfxNEk7ydxNMn7aWjq7OyktLQUZ2dnjEbjcX1tq9Vqq+g8GvPfTSbTEXmYnp4eLrvsMoqKivjuu+/w9vb+2efc3d1xdna2/Vmn0+Hg4ICrq+sRlZmHTJ48mSeffJKamhpbG3phYSGgtqN7enr+4qzO8ePHM378eB566CHGjRvHZ599xpQpUwgICCA3N/eI+HNyctDr9bZrOp2OXbt2HXFPRkYGUVFRR7Ta/+Uvf+Evf/kLjz76KJ6enmzbto2//e1v/+/zO3bsoL6+nkWLFtm6nA/lypycnHB1dWXkyJG88847mEymn3wdcHV1JSAggIMHD5KSkvKz+/9fnZ2dmEwmJk6c+JP3388lRn/JCZvo/K3+v/b3HzMYDBgMP23N0+v18sW/nz0+F0FeemZ7uTA7bRgWi5W9lc22Q422FzVQ1tjJiu1lrNhehkaBlGB39VCjaDPJQe7otEO34miwk3+3xNH0Z95Pem9vjJMmwaRJgPqNWk95OZ0ZGeq8z8xMOrOzsbS00LF5Mx2bN9ue1QX42w46MiYmYoqPR+PkdBR2ZB+eek8mOE1gQsgEQP1cVLZVqnM+a7LIrM1kb91eWnta2XJwC1sObrE96+/kf3jep3cicZ5xx3QI/bEkX5/E0STvJ3E0yftJHE3yfhpa+vr6UBQFjUaD5jh3IR1qZT70+r9Xa2sr+fn5tj8XFxeTkZGBp6cn/v7+zJ49m507d7Jq1SqsVqut4tDT0xMHBwfGjRuHh4cHl19+OQ888AAmk4lXX32VwsJCZsyY8YsxnXHGGYwcOZKrrrqKpUuXYrFYmDt3LqeffjqxsbE/+0xhYSGvvPIKf/nLXwgICGD//v3k5eVx6aWXotFomDJlCosWLeK9997jpJNO4r333iMrK4sRI0YcEUdJSQm33XYb1157LTt37uT5559n8eLFaDQaVq1axYEDB5g4cSIeHh6sXr0ai8VCXFyc7WP82vOhoaE4ODiwbNkyrrvuOrKysnj00UcBbO+Pm266ieeff56///3v3H333bi5ubF582bS0tKIiYnh4Ycf5uabb8bd3Z2pU6fS1dXF9u3baWhoOKJI8BCNRoOiKD/7deX3fJ05YROdZrMZrVZLVVXVEderqqrw8/OzU1TiWNNoFBIC3UgIdOP6SRG0d/ey5UA96Xk1rM+rJb+6lZ0ljewsaeSZ7/JwMeo4OcKLidHeTIzyJthzcP7ALYQ4vhRFwSEoCIegIFynTwfA2tNDV37+4YOOMjLpys+nt6KSlopKWv77X/VhjQZDZGT/QUdJmJKSMERGougG51/ZiqIQ4BxAgHMAU0OnAtBr6SW/MZ/M2kwyazLJrM2koLGAyrZKKtsq+aZYbYPTKloi3SPV5Kd3EgnmBCLcItBqtL/2kkIIIYQQ4gSzfft2Jk+ebPvzoUTanDlzeOihh2yt6v9bXfj9998zadIkzGYzX331Fffeey+nnnoqPT09xMfH8+mnn5KcnPyLr6vRaPj888+56aabmDhxIk5OTkybNo3Fixf/4jOOjo7s27ePt99+m7q6Ovz9/Zk7d65tVOKZZ57J/fffzx133EFnZydXXHEFl156KZmZmUd8nEsvvZSOjg7S0tLQarXccsstXHPNNYBamfqvf/2Lhx56iM7OTqKioli+fDnx8fG/6Xlvb2/eeust7rnnHp599llGjhzJokWL+Mtf/mJ73svLizVr1nD77bdzyimnoNVqSUlJYdy4cQBcddVVODo68tRTT3H77bfj5OREYmIit9566y9+bo4GxXpo6ugQpygK//73vzn33HNt18aMGUNaWhrPPfccoP4GISQkhBtvvJG77rrrT79mc3Mzbm5uNDU1Set6Tw+rV69m+vTpA/o3fhWNHWzIq2VdXg0b82tpbD9y4G2olyMToryZEGXmpAgvXIwDdy9D2WB5P4nBwZ7vp77WNjr3Zh9R+dlbWfmT+xSjEWN8fH/iMxFjYhL6wICj0tYzULT1tJFdm60mP/tXdXv1T+4z6Uxqu7t3Iolmdfk5DZxfUMrXJ3E0yftJHE3yfhJHk7yfhqbOzk4KCwsJCws77q3rFouF5uZmXF1dj3s16Ylo0qRJpKSksHTpUnuHYvNr77/fk18bnOUhv9H/li4XFhaye/duPD09CQkJYf78+cyZM4fRo0eTlpbG0qVLaWtrs53CLk48Ae4mZqcGMzs1mD6LlazyJtbn1ZCeW8vOkgaK6topqivm3c3F6DQKI0M8mBBlZkK0N4mBbmg1QyfpIIQ49rTOTjilpeGUlma71lNdTWdm5uHKz8wsLK2tdOzYQceOHYef9fLClJjYX/mZjCkxAa2bmz22cVQ46Z1I808jzf/w56KqrYqs2iy17b02i6zaLNp729letZ3tVdtt9/mYfEgwJ9iSn/Fe8Tg7OP/cywghhBBCCCGGsCGd6Py10uW33nqL888/n5qaGh544AHbgNSvvvrqJwcUiROTVqOQHOxOcrA7N54aRUtnD5sP1LO+v829sLaNrUX1bC2qZ/E3ubg76hkXaWZilJkJUd4EuJvsvQUhxCCk9/FBP2UKLlOmAGC1WOguKqIjI8NW+dm5fz99dXW0rl1L69q1tmcdhg3D2N/ubkpKxBAXh8bBwU47+fN8nXzxdfJlyjD1c9Fn6aOwqfCIqs+8hjyqO6pZU7qGNaVrAFBQCHcLP6LqM9IjEr1GKk6EEEIIIYQYyoZ0onPSpEn8f535N954IzfeeONxikgMZi5GPacP9+X04WoivLS+XZ3tmVvLxgK1zf2LjEq+yFDbTiO8nWyzPceEe+LoMKT/dRNCHCOKRoMhPBxDeDj0j1+xdHXRlZNja3fvyNhDT3EJ3cXFdBcX0/z55+rDej3G2NgjKj8dQoehDNJ2IK1GS6RHJJEekfw16q8AdPR2kFOXczj5WZNJRVsFBU0FFDQV8J/8/wBg1BqJ84qzHXaUYE4g0DlwSLX/CyGEEEII8Vus/VGxxFAjmRch/qBgT0cuGjOMi8YMo7fPwp6yRtJz1dPcd5c2UlDTRkFNG29uLMJBq2HUMA8mRqvzPYf7u6KRNnchxB+kMRgwpaRg+tEw9b7GRjoys+jIzKBzTwYdmZn01dfTmZlJZ2YmfND/rIsLpsQEjIlJmJKTMCUmovP2ts9GjgKTzsRI35GM9B1pu1bbUUtWbZYt8ZlVm0VLTwu7qnexq3qX7T5Po6fa8t5f9ZlgTsDNMHjb/4UQQgghhDjRSaJTiKNAp9Uwapgno4Z5Mu/0aJo6ethUUMu63FrSc2sob+xg04E6Nh2o44mvwMvJgfH9Le4To8z4uB7fQc9CiKFH6+6O84TxOE8YD4DVaqWnvILOjD22ys/O7GwsLS20/bCJth822Z7V+fv/6KCjREzx8WicnOy1lT/NbDIzKXgSk4InAWCxWihuLlbnfdao8z73NeyjvrOe9LJ00svSbc8Ocx1mS3wmmhOJ8YzBQTt42/+FEEIIIYQ4kUiiU4hjwM2kZ2qCP1MT/LFarRTVtdsONdpUUEtdWzef7q7g090VAMT6uaiHGkV5kxbmiVGvtfMOhBCDnaIoOAQF4hAUiOv06QBYe3vpysvrT3xm0JmRSVd+Pr2VlbRUVtLy3/+qD2s0GCIj1Xb3xP55n1FRKLrB+W2DRtEQ5hZGmFsYZ0ecDUBXXxf76/eTWZtpS36WtJRQ3FxMcXMxqw6sAkCv0RPrGWur+EzyTiLEJURa3oUQQgghhBiABudPLEIMIoqiEGZ2IszsxKUnhdLda2FXSQPr89Q294zyJvYdbGHfwRZeXV+IQachLcyTiVHeTIg2E+PrIj9QCyGOCkWnwxgXhzEuDo/zZwPQ19pG595s9aT3/pb33spKunJz6crNpenjT9RnjUaM8fGYEhMxJSdhTExCHxgwaL8+GbQGkryTSPJO4qK4iwBo7Gwkqy6LzJrDhx01djXa/vchrg6uRyQ+E8wJeBo97bUVIYQQQgghRD9JdApxnDnoNIwJ92JMuBe3nRlDQ1s3G/Jrbae5VzZ19idBa2E1+LgYGB9l5pRob8ZFmjE7G+y9BSHEEKJ1dsIpLQ2ntDTbtZ7qajXxmZFJZ6Z60rultZWOHTvo2LHj8LNeXocPOkpMwpSYgNbd3Q67ODrcje6MDxzP+MDD7f9lrWVHJD5z6nJo7m5mY8VGNlZstD0b6BxIojmR4Z7Dae1tpbO3E71eTnkXQgghhBDieJJEpxB25uHkwNnJAZydHIDVaqWgppV1/YcabT5QR3VLF//aWc6/dpYDEB/gqs72jDYzapgHBp20uQshji69jw/6KVNwmTIFAKvFQndRER0Zart7R0YGnfv301dXR+vatbT+6NRGh2HDMCYl2So/DbGxaAyD8xc0iqIQ7BJMsEsw08PV9v+evh5yG3PJqskio1ZteT/QdIDy1nLKW8v5qugrAN746A2iPKLUWZ/e6rzPMLcwNMrgPPFeCCGEEEKIwUASnUIMIIqiEOnjQqSPC1eOD6Ort48dRQ2k56mHGu2tbCa7Ql0vrSvApNcyNtzTlviM8HYetG2kQoiBS9FoMISHYwgPh3PPBcDS1UXXvn22dvfOjAy6i4ttq/nzz9WH9XqMMTGYkpLUys+kJBxCQ1E0gzPhp9fqifeKJ94rnvM5H4CW7hay67LJrMlkT/UedlTsoNXaSk59Djn1OazMXQmAk96JBK8EEr37297NSXg7Dt4T74UQQgghxLGhKAr//ve/Obf/e2/x20miU4gBzKDTcnKkmZMjzdw1LZaali425teS3t/mXtPSxff7a/h+fw0AAW5GJvTP9hwXYcbDSU4KFkIcGxqDAVNyMqbkZNu1vsZGOjKzbAcddWRk0FdfT2dWFp1ZWfBB/7MuLpgSEzD2H3RkSkpC5z14E34uDi6M9R/LWP+x9PT08MUXXzBy0kj2Ne6ztbzvrdtLW08bWw5uYcvBLbZnfR19bXM+E82JxHvF46h3tONuhBBCCCEGn/T0dJ566il27NhBZWXlEUnCnp4e7rvvPlavXs2BAwdwc3PjtNNOY+HChQQEBNg+Rm5uLrfffjsbN26ku7ubpKQkHnnkESZPnvyrr221Wlm8eDGvvPIKxcXFmM1mbrjhBu69995jueXfbO3atUyePJmGhgbcB/GYqd9KEp1CDCLeLgbOHRHIuSMCsVqt7DvYYpvtuaWwnoqmTlZsL2XF9lIUBZKC3JnYf5r7iBB39NrBWUElhBgctO7uOE8Yj/OEwzMue8orbHM+OzIy6MzOxtLSQtsPm2j7YZPtWZ2/v9runpSotr7Hx6NxcrLXVv4URVHwd/InxD2EM0LPAKDX0ktBYwGZtZlk1apt7wWNBVS1V/FN8Td8U/wNoJ4QH+EeQZL5cPIzwj0CnUa+ZRNCCCGE+CVtbW0kJydzxRVX8Le//e2If9be3s7OnTu5//77SU5OpqGhgVtuuYW//OUvbN++3XbfjBkziIqKYs2aNZhMJpYuXcqMGTMoKCjAz8/vF1/7lltu4euvv2bRokUkJiZSX19PfX39MdurvVitVvr6+tDpBvb3pQM7OiHEL1IUhTh/V+L8XblmYgSdPX1sKaxnfa6a+Nxf1cKe0kb2lDby3Jp8nA06xoZ7cUq0mvgc5uUobe5CiGNKURQcggJxCArEddo0AKy9vXTl5/e3vKuVn135+fRWVtJSWUnL11+rD2s0GCIiMCYnqQcdJSViiIpCGeDfWP0SnUZHjGcMMZ4xzIyeCUB7T7va8n4o+VmTQVV7FXkNeeQ15PFJnnrivUlnYrjXcHXeZ//yc/KTr+FCCCGEOKasVivWjo7j8loWiwVLRwcWnQ40GhST6Xd9rzNt2jSm9X+/+b/c3Nz45ptvjrj2/PPPk5aWRklJCSEhIdTW1pKXl8frr79OUlISAAsXLuSFF14gKyvrFxOdOTk5vPjii2RlZRETEwNAWFjYb4r5jTfeYPHixeTn5+Pp6cl5553H888//5P7fq4ic/fu3YwYMYLCwkJCQ0MpLi7mxhtvZMOGDXR3dxMaGspTTz3F8OHDbRWpHh4eAMyZM4e33noLi8XCE088wSuvvMLBgweJjo7m/vvvZ+bMmUe87urVq7nvvvvIzMzk66+/ZtKkSb9pf/YyOH9aEEL8hFGv5ZRob06JVts/q5rV09vTc2vYkF9LfVs33+ZU8W1OFQDBniZ1tmeUmZMizLiZ5HRgIcSxp+h0GGNjMcbG4nH+bAAsbW10ZGfbTnrvyMigt7KSrrw8uvLyaPpYTfgpRiPG+PgfVX4mow8MGLQJP0e9I6l+qaT6pdquVbdX2xKfmTWZZNVl0dbTxo6qHeyoOnzivdlkts35TDAnkGBOwMXBxR7bEEIIIcQQZe3oYP/IUcf1Nav6/ztm5w4Ux2M3zqepqQlFUWyJQy8vL2JiYnjnnXcYOXIkBoOBl19+GR8fH0aN+uXPweeff054eDirVq1i6tSpWK1WTjvtNJ588kk8PT1/8bkXX3yR+fPns3DhQqZNm0ZTUxMbN278w/uZO3cu3d3dpKen4+TkxN69e3F2diY4OJhPPvmE8847j/379+Pq6orJZALg8ccf57333uOll14iKiqK9PR0Lr74Yry9vTnllFNsH/uuu+5i0aJFhIeH25KlA5kkOoUYonxdjcwcFcTMUUFYLFb2Vjarsz1za9leXE9pfQcfbCnhgy0laDUKKcHuTOhvc08OckMnbe5CiONE4+SEU1oaTmlptms91dV0ZmXRsSdDbX3PzMLS0kLHjh107Dic8NN6emJKTDxc+ZmYgHYQzx7ycfRhSsgUpoSoJ95brBYKmwrVWZ816rzPvIY8ajtqWVu6lrWla23PhrmFHa769E4k2j0avVZ+iSWEEEII8WOdnZ3ceeedXHjhhbi6ugJqJ9K3337Lueeei4uLCxqNBh8fH7766qtfTe4dOHCA4uJiPvroI9555x36+vqYN28eM2fOZM2aNb/43IIFC/jHP/7BLbfcYruWmpr6i/f/f0pKSjjvvPNITEwEIDw83PbPDiVcfXx8bIndrq4uHnvsMb799ltOOukk2zMbNmzg5ZdfPiLR+c9//pPTTz/9D8d2vEmi8xhYtmwZy5Yto6+vz96hCAGARqOQEOhGQqAbN0yKpK2rly2FdaTn1rI+r4aCmjZ2FDewo7iBpd/m4WrUMS5STXpOiDIT7CkHYwghji+9jw/6U0/F5dRTAbBaLHQXFalzPjMy1ZPe9+2jr76e1nXraF23zvasw7Bh6pzP/spPQ1wcGoPBXlv5Uw7N7Ixwj+DcyHMB6OztJKc+x5b4zKzNpLy1nMKmQgqbCvms4DMAHDQOxHnFHZH8DHIOGrQVsEIIIYQ4vhSTiZidO/7/G48Ci8VCc0sLrv1JRqW/6vBo6+npYfbs2VitVl588UXbdavVyty5c/Hx8WH9+vWYTCZee+01zj77bLZt24a/vz/x8fEUFxcDMGHCBL788kssFgtdXV288847REdHA/D6668zatQo9u/fj8lkYvjw4bbXueeee7jqqquoqKhgypQpR21fN998M9dffz1ff/01p512Guedd56tBf/n5Ofn097e/pMEZnd3NyNGjDji2ujRo49anMeDJDqPgblz5zJ37lyam5txc3OzdzhC/ISTQcepsb6cGusLQFlDOxvyalmfV8uG/FqaOnr4MusgX2YdBCDM7GQ71GhshBfOBvnSIYQ4vhSNBkN4OIbwcOg/QdPS1UXXvn2HDzrKyKC7uNi2mj//XH1Yr8cYE3P4oKOkJBxCQ1E0g7Ny3agzMsJnBCN8Dn8TWtdRR3ZdNhk1GWrbe20mzd3N7KnZw56aPbb7PAwetkOOEr0TSfBKwN3oboddCCGEEGKgUxTlmLaPH8FiQdPbi8bREc0x+h7tUJKzuLiYNWvW2Ko5AdasWcOqVatoaGiwXX/hhRf45ptvePvtt7nrrrtYvXo1PT09ALb2b39/f3Q6nS3JCRAXFweoVZaTJ09m9+7dtn/m6emJXv/7Om4OfT6sVusRe/mxq666ijPPPJMvvviCr7/+mscff5zFixdz0003/ezHbG1tBeCLL74gMDDwiH9m+J8CAadBdkCoZCuEEAR5OHJBWggXpIXQZ7GSWd5Eem4N6/Nq2FnSSGFtG4W1bby9qRidRmHkMA8mRpmZGO1NfIAbWo1UBwkhjj+NwYApORlTcrLtWl9jIx1Z2XRk7FErPzMy6KuvpzMri86sLPhgufqsszPGxARMSclqAjQxEb2Pj7228qd5mbyYGDSRiUETAfUb4ZKWkiMSn/vq99HQ1cD68vWsL19vezbEJUSd9+mdRKI5kRjPGAzawVkBK4QQQgjxcw4lOfPy8vj+++/x8vI64p+3t7cD/CTJqtFosFgsAAwbNuwnH3fcuHH09vZSUFBAREQEALm5ubb7dTodkZGRP3kuNDSU7777znZQ0K/x9lbP4aisrLS10f84eXpIcHAw1113Hddddx133303r776KjfddBMODg4AR3QdDx8+HIPBQElJyRFt6kOBJDqFEEc4NK8zJdidm6dE0dLZw6aCOvVgo7waiuva2VpYz9bCehZ9nYuHo55xkWYmRnkzIdqMv9uxaTEQQojfQuvujvP4cTiPHweoCb+e8gp1zuehys/sbCytrbRv2kz7ps22Z3X+/ocPOkpMwpQQj2aQ/Qb7EEVRGOY6jGGuwzg74mwAuvu62V+/39bunlWbRVFzESUtJZS0lLC6cDWgnhAf6xFrS34mmBMY5joMjTI4K2CFEEIIMfS1traSn59v+3NhYSG7d+/G09MTf39/Zs6cyc6dO1m1ahV9fX0cPKh2L3p6euLg4MBJJ52Eh4cHc+bM4YEHHsBkMvHqq69SWFjIWWed9Yuve9pppzFy5EiuuOIKli5disViYe7cuZx++ulHVHn+r4ceeojrrrsOHx8fpk2bRktLCxs3bvzZCszIyEiCg4N56KGHePTRR8nNzWXx4sVH3HPrrbcybdo0oqOjaWho4Pvvv7dVlg4bNgxFUVi1ahXTp0/HZDLh4uLCbbfdxrx587BYLIwfP952IJKrqytz5sz5XZ//gUQSnUKIX+Vi1HNGvB9nxPsBUFzXxvo8dbbnD/l1NLT3sCqjklUZlQBE+Tirsz2jzYwJ88TRQb7MCCHsR1EUHIICcQgKxHXaNACsvb105efTkZFhm/nZlZ9Pb2UlLZWVtHz9tfqwRoMhIgJjUqKt8tMQFYWiG5xf1xy0DiR6qy3rhzR1NdkqPg8deNTQ1UBWXRZZdVl8uP9DAFwcXEjwSlCf75/56WXy+qWXEkIIIYQ4rrZv335EdeT8+fMBmDNnDg899BCffabOME9JSTniue+//55JkyZhNpv56quvuPfeezn11FPp6ekhPj6eTz/9lOQfdQ/9L41Gw+eff85NN93ExIkTcXJyYtq0aT9JRP6vOXPm0NnZydNPP81tt92G2Wxm5syZP3uvXq9n+fLlXH/99SQlJZGamsqCBQuYNWuW7Z6+vj7mzp1LWVkZrq6uTJ06laeffhqAwMBAHn74Ye666y4uv/xyLr30Ut566y0eeeQRvL29efzxxzlw4ADu7u6MHDmSe+6551djH+gU64+b/MVRdWhGZ1NT0xGzH05EPT09rF69munTp//ueRRi4Orps7CntJH0/sTnntJGLD/6iuKg1ZAa5mE71CjOzxXNUWhzl/eTOJrk/SQALG1tdO7d25/8zKQjM4Peisqf3KcYjRiHD8eUlGSb+akPDLQd8DPY309Wq5Xy1nKyarPIqFXb3vfW7aWrr+sn9wY4BRyR+IzzisOkk6r+o2mwv5/EwCLvJ3E0yftpaOrs7KSwsJCwsDCMRuNxfW2LxUJzczOurq7HbEanGNh+7f33e/Jrg7MkQQgxIOi1GkaHejI61JP5p0fT1N7DxgI16ZmeW0t5Ywcb8+vYmF/Hwi/B7OzA+Eh1tuf4KDM+Lsf3L08hhPglGicnHFNTcUxNtV3rqa6mMyvLdtBRR2YWlpYWOnbupGPnTtt9Wk9PTImJGJMS0cfHo+mf8TQYKYpCkEsQQS5BTA2bCkCPpYf8hvwjqj4PNB2goq2CirYK/lv0XwC0ipYojyi15d2szvsMcwtDq9Hac0tCCCGEEOIEIolOIcRR4+aoZ3qiP9MT/bFarRTWqm3u6bk1bDpQR21rN//ZXcF/dlcAEOvnwsRobyZGeTM61AOjXn4YFkIMHHofH/SnnorLqacCYLVY6C4qVud97smgIzOTzn376Kuvp3XdOlrXrQMgEih+863+qk+18tMQF4fGMDgP+NFr9MR5xRHnFcfsmNkAtHa3kl2XbUt8ZtZmUtNRw776feyr38fHuR8D4KR3It4r3pb8TDAn4Ovka8/tCCGEEEKIIUwSnUKIY0JRFMK9nQn3dmbOyaF091rYWdLA+rwa1ufVklnexL6DLew72MIr6Qcw6DSMCfdiYpSZCVHeRPs621pBhRBiIFA0GgzhYRjCw3A75xwALN3ddOXk2NrdOzIy6CkqpqekhJ6SEppXrVIf1usxxsQcPugoKRGHsDCUQdqa5ezgzBj/MYzxHwOoLe9V7VVHVH1m12XT1tPG1oNb2Xpwq+1ZH0cfW7t7ojmReHM8TvrBeeiTEEIIIYQYWCTRKYQ4Lhx0GsaGezE23Ivbz4S61i42FtSxPreG9Lwaqpq7SM+tIT23BsjB19Vgm+05PtKMl/PgrIQSQgxtGgcHTMnJmPqH1Pf09PDVxx8zMSCAnr171crPjAz66uvpzMqiMysLWK4+6+yMMTEBU2ISpuQkjImJ6H187LibP05RFPyc/PBz8uP0YacD0Gfpo6CpQJ33WaPO+8xrzKO6vZrvSr7ju5Lv1GdRiHCPINGcaDvpPdI9Ep1Gvk0VQgghhBC/j3wHKYSwCy9nA39JDuAvyQFYrVbyqltJz1WrPbcU1lHV3MXHO8r4eEcZAAmBrkyM8mZClDdJAc52jl4IIX6ZxdERx5NPRn/KKYBa7dhbUXHEQUed2XuxtLbSvmkz7Zs2257V+fkdPugoMQljfDxa58FZ7ajVaIn2iCbaI5q/Rf0NgPaedvbW7T3isKPKtkryG/PJb8zn3/n/BsCoNTLca7ia/PRW2979nfyl0l8IIYQ4DuTMamEPR+t9J4lOIYTdKYpCtK8L0b4uXDUhnM6ePnYUN6gVnnm15FQ2k1WurhfWFuDooCXUUUONRzGT4/wINzvJD79CiAFLURT0gYHoAwNxnTYNAGtvL135+epBR5mZdOzJoCs/n96DB2k5eJCWr79WH9ZoMEREYExKtFV+GqKiUHSD81s4R70jo/1GM9pvtO1abUetbc5nZm0mWbVZtPa0srN6JzurDx/65Gn0tM35TPRWqz9dHX791E0hhBBC/HZarXpmQnd3NyaTyc7RiBNNd3c3cPh9+EcNzu+ShRBDmlGvZVykmXGRZu4Gqls62ZhfS3queqJ7bWs3e7s17F29nwWr9xPobmJC/2zPcZFeuDs62HsLQgjxqxSdDmNsLMbYWJitHvBjaWujc+/eIyo/eysq6crLoysvj6ZP/qU+azRiHD78cOVnUhL6wMBB+wsfs8nM5JDJTA6ZDIDFaqGoueiI5GdufS71nfWsLVvL2rK1tmdDXUPVWZ/e6rzPGI8Y9Fq9nXYihBBCDG46nQ5HR0dqamrQ6/VojuMscYvFQnd3N52dncf1dcXAYLFYqKmpwdHREd2f/IW+JDqFEAOej4uRv44I4q8jgrBYrGSVNfDaqg3U6r3ZXtxIeWMHH24r5cNtpWgUSApyVw81ivYmJdgdvVb+ohRCDHwaJyccU1NxTE21XeutqaEjM1Ot/MzIpCMzE0tLCx07d9Kx83C1o9bTE1Niolr5mZSEKTERrbu7HXbx52kUDeFu4YS7hXNOpHroU1dfFzl1ObaW98yaTMpayyhqLqKouYjPD3wOgIPGgViv2CMOOwp2CR60SWAhhBDieFIUBX9/fwoLCykuLj6ur221Wuno6MBkMsnf2ycojUZDSEjIn/7/XxKdQohBRaNRiPN3YUqglenTR9Nr1bClsI71eWq1Z25VK7tLG9ld2siza/JxMeg4KcKLCdHeTIwyM8xrcM66E0KcmHTe3riceioup54KgNViobuomM7M/qrPjAw69+2jr76e1nXraF23zvasfliI2u7en/w0xMWhMQzOg90MWgMpPimk+KTYrjV0Ntha3Q/N+2zqaiKjJoOMmgzbfW4GN/WQo0Nt7+ZEPIwedtiFEEIIMfA5ODgQFRVlayM+Xnp6ekhPT2fixIno9dKdcSJycHA4KtW8kugUQgxqJgctk2J8mBSjnlR8sKmT9Dz1UKMNeTU0tPfw9d4qvt5bBUCIpyMTo9U295MivHA1yl+iQojBQ9FoMISHYQgPw+0ctdrR0t1N17596gnvmWrlZ3dRET3FJfQUl9C8apX6sE6HMSam/4R3NQHqEBaGMkjbwzyMHkwMmsjEoImAWglS2lJqa3fPrM1kX90+mrqa2Fi+kY3lG23PBjkH2drdE82JxHrGYtQZ7bUVIYQQYkDRaDQYjcf370WtVktvby9Go1ESneJPkUSnEGJI8XMzMnt0MLNHB2OxWMmuaCY9r4b03Bp2FDdQUt/Oe5tLeG9zCVqNwohgdyZEeTMx2kxSkDtajbRJCCEGF42DQ/+8ziTbtb7GRjqyso+o/Oyrq6MzO5vO7GxgufqsszPGxARb5acxKQm9j4+ddvLnKIpCiGsIIa4hnBV+FgA9fT3kNuTaKj4zajIoai6irLWMstYyviz8EgCdoiPaM/qIlvdQt1A0yuBMAgshhBBCnKgk0SmEGLI0GoXEIDcSg9yYOzmS1q5ethxQ29zTc2s4UNvG9uIGthc38PS3ubgadYzvP9RoQpSZIA9He29BCCH+EK27O87jx+E8fhygVjv2VlSo8z4PVX5m78XS2kr7ps20b9pse1bn5/ejeZ/JGOPj0ToPzrEfeq2eeHM88eZ427Xm7mayarPIrDnc9l7fWc/eur3srdvLiv0rAHDWOxNvjifJnGQ78MhsMttrK0IIIYQQ4jeQRKcQ4oThbNAxJc6XKXG+AJTWt7MhX53tuSGvlubOXlZnHmR15kEAwr2dmNif9Bwb7oWTQb5kCiEGJ0VR0AcGog8MxHXqVACsvb105eercz4zM+nIyKQrL4/egwdpOXiQlm++OfQwhsgIjElJtspPQ1QUyiBtK3N1cOXkgJM5OeBkQE0CV7ZVqlWfNVlk1mayt24vrT2tbKncwpbKLbZn/Z38j5j3OdxrOI56+aWYEEIIIcRAIT+1CyFOWMGejlyYFsKFaSH0WazsKWtkfa6a+NxV2siBmjYO1LTx1g9F6LUKI0M8mBjtzcQob+IDXNFIm7sQYhBTdDqMsbEYY2Nh9mwALG1tdO7dq7a7Z2bSkbGH3opKuvLy6crLp+mTf6nPGo0Yhw8/XPmZnIw+MHBQnpKqKAoBzgEEOAcwNVRNAvdaeslvzFdnfdao8z4LGguobKuksq2Sb4rVJLBG0RDpHnm45d07kQi3CLQarT23JIQQQghxwpJEpxBCAFqNmsgcGeLBLadF0dzZww/5dazPqyE9r4bS+g62FNazpbCep/67H08nB8ZFmpnY3+ru5yaHWAghBj+NkxOOqak4pqbarvXW1NCRmUVHxh46+xOglpYWOnbupGPnTtt9Wg+P/nZ3dV6oMSEBncfgPN1cp9ER6xlLrGcss6JnAdDW00Z2bfYRhx1Vt1eT25BLbkMun+R9AoBJZyLeK/6Iw458HX0HZRJYCCGEEGKwkUSnEEL8DFejnqkJfkxN8AOguK6N9P7ZnpsK6qhv6+bzPRV8vqcCgGhfZ9tszzFhXpgcpJpHCDE06Ly9cTl1Mi6nTgbAarHQXVxMZ0aGrfKzKyeHvoYG2tal07Yu3fasPiSkP/GZiDExEePw4WgMBntt5U9x0juR5p9Gmn+a7VpVW5VtzmdWbRZZtVm097azvWo726u22+7zNnnbKj4TzYnEe8Xj7OBsj20IIYQQQgxpkug8BpYtW8ayZcvo6+uzdyhCiKNkmJcTl3g5ccnYYfT0Wdhd2sj63BrS82rZU9ZIblUruVWtvL6hEAedhrRQTyb0V3vG+btIJY8QYshQNBoMYWEYwsJwO+ccACzd3XTt29d/wrta+dldVERPSQk9JSU0r1qlPqzTYYyJsR10ZEpKxCEsDEUzOE8393XyxdfJlynDpgDQZ+mjsKnwiKrPvIY8ajpqWFO6hjWlawBQUAh3C1fnfXqr8z6jPKLQawbn3FMhhBBCiIFCEp3HwNy5c5k7dy7Nzc24ubnZOxwhxFGm12pIDfUkNdST+WfE0NjezcZDbe65NVQ0dbIhv5YN+bU8/uU+zM4GtcU92sz4SG+8XQZnNZMQQvwSjYODrWUdLgKgr6mJjqysw5WfGRn01dXRmZ1NZ3Y2jcs/VJ91dsaYkPCjys8k9L4+dtzNH6fVaIn0iCTSI5K/Rv0VgI7eDnLqcg4nP2syqWiroKCpgIKmAj4t+BQAg9ZAnGfcES3vPobB+XkQQgghhLAXSXQKIcSf5O7owFlJ/pyV5I/VauVAbRvpuTWsz6tlU0Edta1d/GtXOf/aVQ5AnL8rE6PNTIzyZtQwD4x6aXMXQgw9Wjc3nMeNw3ncOEA93by3oqL/kKP+ys/svVhaW2nfvJn2zZttz+r8/A4fdJSUjDE+Hq2zk7228qeYdCZG+o5kpO9I27XajlqyarNsic+s2ixaelrYXbOb3TW7bfd5GDzw6fOhPLOcZN9kEswJuBnkl+hCCCGEEL9EEp1CCHEUKYpChLczEd7OXD4ujK7ePnYWN5KeV8P6vBqyypvJqVTXy+sOYNRrGBPmxYQoM6dEexPp4yxt7kKIIUlRFPSBgegDA3Gdqp5ubu3tpauggI6MDFvlZ1deHr0HD9Jy8CAt33xz6GEMkREYE5NslZ+GqCgU/eBs9TabzEwKnsSk4EkAWKwWipuL1XmfNeq8z30N+2joaqCBBvZn7odM9dlhrsNINCeqbe/mJGI8Y3DQOthvM0IIIYQQA4gkOoUQ4hgy6LScFOHFSRFe3Dk1lrrWLjbk17I+r5b1eTVUNXexLreGdbk1LPgiBz9XozrbM9qb8ZFmPJ3kh1chxNClHJrZGRMDs9TTzS3t7XRmZ9sOOurMyKCnooKuvHy68vJp+te/1GeNRozDh/+o8jMJfVDQoPxlkUbREOYWRphbGGdHnA1AV18X2dXZrFy/Eouvhey6bEpaSihuLqa4uZhVB9S5p3qNnljPWBLMCbaW92Guwwbl50EIIYQQ4s+SRKcQQhxHXs4GzkkJ5JyUQKxWK7lVrepsz7xathyo42BzJx/tKOOjHWUoCiQGutkONRoZ4oGDbnAe2CGEEL+VxtERx9RUHFNTbdd6a2royMyiIzODzj0ZdGRmYmlpoWPnTjp27rTdp/XwUJOeiUmYkpMwJiSg8/Cwxzb+NIPWQKI5kVJDKdNPno5er6exs5Gsuiwyaw4fdtTY1Wj738tZDoCrg+sRic9E70Q8jZ523pEQQgghxLEniU4hhLATRVGI8XMhxs+FqyaE09nTx7aietbn1ZKeW8O+gy1klDWRUdbEsu8LcHJQq0MnRHkzIcpMmNlJKnaEECcEnbc3LqdOxuXUyQBYLRa6i4sPH3SUmUlXTg59DQ20rUunbV267VmX00/D/7HH0Lq42Cv8o8bd6M74wPGMDxwPqHNPy1rLjkh85tTl0NzdzA8VP/BDxQ+2ZwOdA49IfMZ5xmHUGe21FSGEEEKIY0ISnUIIMUAY9dr+JKY390yPo7q509biviG/ltrWbr7NqebbnGoAAt1NtkONTo4w4+Y4OGfVCSHE76VoNBjCwjCEheF2zjkAWLq76dq3j46MTDozM+jYk0F3UREt33xLV14+QS8swxAebufIjy5FUQh2CSbYJZjp4dMB6OnrIbcxl6yaLDJq1XmfB5oOUN5aTnlrOV8VfQWAVtES7RF9eN6ndxJhbmFoFOkcEEIIIcTgJYlOIYQYoHxcjZw3KojzRgVhsVjJOdhsq/bcXtRAeWMHy7eWsnxrKRoFkoPdmRDlzcQoMynB7ui08sOqEOLEoXFw6D+oKAm4CICOrGzKbryR7qIiimafT8BTT+IyebJ9Az3G9Fo98V7xxHvFcz7nA9DS3UJ2XTaZNZlk1GaQWZNJXWcdOfU55NTnsDJ3JQBOeicSvBLUtnfvRMb6j8VJPzhPuxdCCCHEiUkSnUIIMQhoNArxAW7EB7hx3SkRtHf3sqWwnvW5asVnXnUru0oa2VXSyLPf5eFi0HFypFd/4tObEC9He29BCCGOO1NCPGEff0TZLbfSsWMHZTfMxfuWW/C69poTavSHi4MLY/3HMtZ/LKC2vB9sO2hrd8+szWRv3V7aetrYcnALWw5uAcDX0ZfFkxaT7J1sz/CFEEIIIX4zSXQKIcQg5OigY3KMD5NjfACoaOxgQ14t6f1t7o3tPfw3u4r/ZlcBEOrlaJvteVKEFy5GaXMXQpwYdGYzw958g4OPP07j8g+pWbqUzn37CHjsUTSOJ+YvgRRFwd/ZH39nf84IPQOAXksvBY0FZNZmklWbxcaKjRxsO8hlX13GHal3cEHMBSdUclgIIYQQg5MkOoUQYggIcDcxOzWY2anB9FmsZJU32U5z31ncQFFdO0V1xby7uRitRmFkiDsTo7yZEO1NYqAbWo388CqEGLoUBwf8H3wQY2wcBxcsoOWrrygqLCRo2fM4BAXZO7wBQafREeMZQ4xnDDOjZ9LW08b9G+/nm+JveGzLY+yu3s2DJz2Io/7ETA4LIYQQYnCQRKcQQgwxWo1CcrA7ycHu3HhqFK1dvWwuqCM9r4b1ebUU1raxraiBbUUNLP4mFzeTnvGRZiZGm5kQ5U2Au8neWxBCiGPC4/zZGCIjKLvlVrr276do5iwClz6N09ix9g5twHHSO7H4lMW8u/ddluxYwurC1eQ25LJk0hLC3MLsHZ4QQgghxM+SRKcQQgxxzgYdpw335bThvgCU1rcfcZp7U0cPX2RW8kVmJQAR3k7qbM9oM2PDvXB0kL8qhBBDh+OoUerczhtvojMri5Irr8L3zjvxuORiac3+H4qicGn8pcSb47l93e3kN+Zz4RcX8si4Rzh92On2Dk8IIYQQ4ifkp1chhDjBBHs68vcxIfx9TAi9fRb2lKlt7uvzatlV0kBBTRsFNW289UMReq3C6GGeTIg2MzHKm+H+rmikzV0IMcjp/fwY9t67HHzwQZo+/Yyqxx6jMycHv4ceRGMw2Du8AWeU7yhWnr2S29fdzvaq7cxfO585w+dwy6hb0Gtk5rMQQgghBg5JdAohxAlMp9UwapgHo4Z5cOtp0TR19LCpoJb0vFrSc2soa+hg04E6Nh2o48mv9uPl5MD4KLPtYCNfV6O9tyCEEH+IxmjEf+FCDHFxVD/5FE3//jddBQUEPfccel8fe4c34JhNZl4941We3fUsb2a9ydt73yazNpNFpyzC29Hb3uEJIYQQQgCS6BRCCPEjbiY9UxP8mZrgj9VqpbiunfS8GtJza9lUUEtdWzef7q7g090VAMT4ujAhyszEaG/Swjwx6rV23oEQQvx2iqLgddllGKKiKJ//DzozMiiceR5Bzz6L44gR9g5vwNFpdMwfNZ9kczL3bryXndU7mfX5LBadsojRfqPtHZ4QQgghhCQ6hRBC/DxFUQg1OxFqduLSk0Lp6bOwq6SR9Nwa1ufVkFHexP6qFvZXtfDahkIcdBrGhHnaEp8xvi4y704IMSg4jxtH2EcrKZt7I115eZRcOge/Bx/AfeZMe4c2IE0ZNoUI9wjmrZ1HfmM+V319FbeOvJU58XPk674QQggh7EoSnUIIIX4TvVZDWpgnaWGe3HZmDA1t3WwsqGV9bi3peTVUNnX2H3JUy2Or9+HtYlCTnlHejI8yY3aWuXdCiIHLISSE0A+XU3HX3bR88w2V991PZ84+fO+6E0Uvcyj/V6hbKO9Pf59HNj/CqgOrWLxjMXtq9vDPcf/ExcHF3uEJIYQQ4gQliU4hhBB/iIeTAzOSApiRFIDVaqWgppX0XPU0980H6qlp6eJfO8v5185yAOIDXNXT3KPMjAr1wKCTNnchxMCicXIi8Jml1L70ErXPPkfD++/TlZdH4NKn0Xl62ju8AcdR78hj4x8jxTuFhdsW8m3Jt+Q15vH0pKeJ8oiyd3hCCCGEOAFJolMIIcSfpigKkT4uRPq4cMX4MLp6+9hR1EB6npr4zK5otq2X1hVg0msZG+6pJj6jzUR4O0u7oxBiQFA0GrxvuAFjTAwVt99B+9atFM2cRdCy5zHGxdk7vAFHURTOjz2f4V7Dmb9uPsXNxVy0+iIeOOkBZoTPsHd4QgghhDjBSKJTCCHEUWfQaTk50szJkWbumhZLbWsXG/NrWZdbw/q8Wmpauvh+fw3f768BwN/NaJvtOS7CjIeTg513IIQ40blMmULoyhWUzp1LT3EJRRf+nYDHHsV1+nR7hzYgJXonsnLGSu5afxc/VPzA3evvZnf1bu5IvQMHrXxNF0IIIcTxIYlOIYQQx5zZ2cA5KYGckxKI1Wplf1WLbbbnlsJ6Kps6Wbm9jJXby1AUSAp0Y0KUNxOizIwc5oFeq7H3FoQQJyBDZCRhK1dS/o/baNuwQT2ZPScH71tvRdHK+I3/5WH04IUpL/BSxku8tOclVuxfwd66vSw+ZTH+zv72Dk8IIYQQJwBJdAohhDiuFEUh1s+VWD9Xrp4YTmdPH1sL61mfp1Z77jvYwp6yJvaUNfH89/k4OWg5KcLMxGgzE6K8CfVylDZ3IcRxo3VzI/jll6h5+mnqXnuduldfo3P/fgIXLULr6mrv8AYcrUbL3JS5JJoTuXv93WTWZjJ71WyemPAEJweebO/whBBCCDHESaJTCCGEXRn1WiZGezMx2huAquZDp7fXsCGvlrq2br7NqeLbnCoAgjxM6v1RZk6KMONmktOQ7aZkM+R/B0GpEDQaHOWwFjE0KVotPrfdhiE2jsp776UtfT1Fs2YT9MIyDBER9g5vQJoYNJGVZ69k/tr57K3by3XfXsfclLlcnXQ1GkWq9IUQQghxbEii8xhYtmwZy5Yto6+vz96hCCHEoOPramTmqCBmjgrCYrGyt7KZ9Lwa1ufWsr24nrKGDj7YUsIHW0rQKJAS7M7EaG8mRHmTHOSGTtrcj5/9q2HjM4f/7BV5OOkZlAY+w0Er32qIocNtxlk4hIVSduNNdBcXUzT7fAKeegqXUyfbO7QBKdA5kHemvcPCrQv5OPdjnt/9PHtq9vD4hMdxM7jZOzwhhBBCDEHy08cxMHfuXObOnUtzczNubvJNnBBC/FEajUJCoBsJgW7cMCmStq5ethbW9x9qVENBTRs7SxrZWdLI0m/zcDHqGBdhZkK0mYlR3gR7Otp7C0Nb8FhIroGyrVCXf3jtWa7+c70jBIxUE5/BaRA4Glx87RuzEH+SKT6esI8/ovyWW2nfvp2yuXPxvvkmvK67TsZq/AyD1sCDJz1IsncyCzYvYH35es5fdT6LJy0m3ive3uEJIYQQYoiRRKcQQohBw8mgY3KsD5NjfQAob+xgQ14N6Xm1bMirpamjh6+yD/JV9kEAwsxOTIhSZ3ueFOGFs0H+2juqYqerC6C9Hsp3QNk2KN2q/u+uZijeoK5D3EP6qz7T1P/2SwSdnMgsBhedlxchb75B1eOP0/DBcmqeeZbOnH0EPP4YGicne4c3IJ0beS6xnrHM+34eZa1lXLr6Uu4Zcw/nRZ9n79CEEEIIMYTIT3xCCCEGrUB3E+enhnB+agh9FiuZ5U2sz1UPNdpZ0kBhbRuFtW28s6kYnUZh5DAPJvYnPhMC3dBqpPrqqHH0hKjT1QVgsUBtrpr4LNsKZduhOgcaS9SV9Yl6n9YA/slq0jM4Vf1v10CQyjgxwCl6PX4PPIAhNpaDjyyg5euvKSoqImjZ8zgEB9s7vAEp1jOWFWev4N7197K2bC0PbXqI3TW7uXfMvRh1RnuHJ4QQQoghQBKdQgghhgStRiEl2J2UYHdumhJFS2cPmwrqbAcbFdW1s7Wwnq2F9Sz6OhcPRz3jIs2cHO5Jd5e9ox+CNBrwiVXXyEvUa53NULETSrf1J0C3QUd9fyJ0K2zuf9bF//Ccz6BUCEgBvcleOxHiV3nMno0hMpKym2+hKzeXopmzCFz6NE4nnWTv0AYkVwdXnjn1Gd7IeoPndj3Hf/L/w776fSw5ZQnBrpIgFkIIIcSfI4lOIYQQQ5KLUc8Z8X6cEe8HQEldu3qoUV4NP+TX0dDew6qMSlZlVAI63i3daDv9fUyYJ44O8lfkUWd0hfBJ6gKwWqH+gFrtWbZVTXwezIKWSsj5XF0AGh34JqhzPg8dduQRJlWfYsBwHDmSsI8/ouymm+nMzKTkqqvxveN2PC69VOZ2/gyNouGqxKtIMCdwZ/qd7Kvfx/mrzuexCY8xKXiSvcMTQgghxCAmP8UJIYQ4IYR4OXKx1zAuHjuM3j4Le8oaSc+tZV1uNXtKGymoaaOgpo03NxbhoNUwOtSDCVHeTIgyM9zfFY20uR99igJeEepKPl+91t0OlbvVOZ+Hqj5bq9Rrlbth6yvqfY5e/UnP/hU4EgwudtqIEKD382PYe+9y8IEHafr0U6oeX0hnzj78Hn4IjcFg7/AGpLH+Y1kxYwW3rbuNPTV7uGnNTVydeDVzU+ai1WjtHZ4QQgghBiFJdAohhDjh6LQaRg3zZNQwT26cFMbHn63GOWIUPxxoID23hvLGDn4oqOOHgjqe+ArMzg6MjzTbEp8+rjJL7phxcIRhJ6sL1KrPprLDcz7LtkHlHmivg9yv1AWgaMA77vCcz6BU8IpSW+iFOE40BgP+Cx/HODyOqiefouk//6GroICg559D7+tr7/AGJD8nP948800W71jM+znv82rmq2TUZvDEhCfwMnnZOzwhhBBCDDKS6BRCCHHCc9TB1Hhfzk4Jwmq1UljbZpvt+UNBHbWt3fxndwX/2V0BQKyfCxOj1aRnaqgnRr1UHh0zigLuwepK6D+dubcLDmb+qOpzOzSVQHW2una8pd5ndIPA0T+q/BwFJg+7bUWcGBRFwXPOHAzR0ZTfOo/OzEwKZ84k6JlncRw5wt7hDUh6rZ670u4i2TuZB394kC2VW5i9ajaLT1lMik+KvcMTQgghxCAiiU4hhBDiRxRFIdzbmXBvZ+acHEp3r4WdJQ2sz1NPc88sb2LfwRb2HWzhlfQDGHQaxoR72U5zj/Z1lpl8x5rO0H9Y0ejD11oOHm51L9sO5TuhswkKvlPXIV5R/bM++xOg3nGglW+HxNHndNJJhH78EWU3zKUrL4/iOXPwe+B+PGbNsndoA9a0sGlEe0Qzb+08CpsKufyry7kt9Tb+Hvt3+boqhBBCiN9EvrMXQgghfoWDTsPYcC/Ghntx+5lQ39bNxvxa0nPVxOfB5k7Sc2tIz60BcvBxMTAhypuJ0WbGR5rxcpbZfMeFix/Ena0ugL4eqMo+nPgs2wb1BVCXp67d76v36Z3U+Z4/nvfp7G2/fYghxSE4mNAPl1Nx9z20fP01B+9/gK6cHHzvvhtFr7d3eANShHsEy89azgMbH+Dr4q9ZuHUhe2r28NBJD+God7R3eEIIIYQY4CTRKYQQQvwOnk4OnJ0cwNnJAVitVvKrW0nvb3PffKCO6pYuPtlZxic7ywBICHS1zfYcPcwTB53MjDwutHoISFFX2tXqtbY6KN/+o8rPHdDdAkXr1XWIR+iPEp+jwTcRdA522IQYCjROTgQ+s5S6l16i5plnafhgOV25eQQ+sxSdl8yg/DlOeicWnbKI93LeY8n2JXxZ+CW59bksmbyEcLdwe4cnhBBCiAFMEp1CCCHEH6QoClG+LkT5unDl+DA6e/rYUdxAel4N63Nr2VvZTFa5ul5cW4Cjg5ax4V5M6G9zj/B2knbM48nJC6LPVBeApQ9q9h/Z8l6zDxqK1JX5kXqfzgj+KYfb3YPTwDXATpsQg5GiKJivvx5DTCwVt99O+/btFM6aRdBzz2GKj7d3eAOSoihcMvwS4r3iuW3dbRQ0FXDhqgt5ZNwjnBF6hr3DE0IIIcQAJYlOIYQQ4igx6rWMizQzLtLM3dOguqWTjfm1rM+tJT2vltrWLtbsq2bNvmoAAt1NtqTnuEgv3B2lavC40mjBd7i6Rs1Rr3U2QfmOw+3uZdugowFKN6vrENfAw4nPoFQ1Eao32mUbYvBwOXUyoStXUHbDXLqLiym+6GL8FyzAbcZZ9g5twBrpO5KVZ6/kjvQ72HZwG/9Y9w8urbmUW0fdil4j7f9CCCGEOJIkOoUQQohjxMfFyF9HBPHXEepp7vsOtthme24tqqe8sYMPt5Xy4bZSFAWSgtw5JcrMhGhvUoLd0Wulzf24M7pBxKnqArBaoa7gR1Wf29TZn83lsLcc9n6q3qfRg18imoBRBNZroTEezBHqqfFC/IghIoLQj1ZS/o/baFu/norbbqNrXw7e8+ahaLX2Dm9AMpvMvHL6Kzy36zneyHqDd/a+Q1ZtFk+d8hQ+jj72Dk8IIYQQA4gkOoUQQojjQFEU4vxdifN35dpTIujo7mNrUT3rc2tIz6sht6qVPaWN7Clt5Nk1+TgbdJwUoZ7mPjHam2FeTvbewolJUcAcqa6UC9Vr3W1Qsetwu3vpVmirhoqdaCt2Mhpg2Uvg5H14zmdQGgSMAIOzPXcjBgitqyvBL71IzdKl1L36GnWvvU7n/lwCFz2F1s3N3uENSDqNjnmj5pHkncR9G+5jZ/VOZn8+m6dOeYpUv1R7hyeEEEKIAUISnUIIIYQdmBy0nBLtzSnR6gnfB5s6WZ+nVntuyK+lvq2bb/ZW8c3eKgBCPB1tbe4nR3rhapSWTbtxcILQ8eoCteqzsQTKttFXspWm7G/x6CxBaauB/avVBaBowCf+yFmfnhGgkcrdE5Gi1eLzj39giI2l8t77aFu/nsLZswletgxDZKS9wxuwpoRMIXJGJPPWziOvIY+rv76aW0bewmXxl8nMYyGEEEJIolMIIYQYCPzcjMwaHcys0cFYLFayK5rVQ43yathR3EBJfTvvbynh/S0laDUKI4Ld1dPco80kBbqhkzZ3+1EU8BgGHsOwxJ7D+t5xTD/jVPS1OYfb3Uu3QXMZVGWqa8eb6rNG9yNnfQaOApO7PXcjjjO3s87CEBZG6Y030lNcQtH5FxDw1JO4nHqqvUMbsIa5DuP96e/zyKZH+PzA5yzZsYQ9NXt4ZNwjuDi42Ds8IYQQQtiRJDqFEEKIAUajUUgMciMxyI25kyNp6+pl84E61ufVkp5Xw4GaNrYXN7C9uIGnv83F1ahjXKTa4j4hykyQh6O9tyB0RrViMzjt8LXmiiMPOarYBZ2NkP+tug4xx/RXfPYnP71j1YOTxJBlHD6csI8/pvyWW2nfto2yG+Zivvkm3K680t6hDVgmnYlHxz9Kik8KC7cu5LuS78hryGPJpCXEeMbYOzwhhBBC2IkkOoUQQogBzsmgY0qcL1PifAEoa2hnQ3/Sc0NeLc2dvXyZdZAvsw4CEG52YkL/bM+x4V44GeSv+wHBNQCG/0VdAH09UJV1eM5n2TZoKITa/era/Z56n4MzBI5U53wemvnpZLbfPsQxofP0JOSN16la+AQN779P7bPP0bF3L8rEifYObcBSFIXZMbMZ7jWc+WvnU9JSwsWrL+aBkx7g7Iiz7R2eEEIIIexAfvIRQgghBpkgD0cuSAvhgrQQ+ixWMsoa1WrP3Bp2lTZyoLaNA7VtvL2pGL1WYWSIh63aMyHADY1G5tgNCFq9ekBRwAhIu1q91lbbX/XZn/gs3wndrVCYrq5DPMIOz/kMGg2+CerHE4Oaotfjd/99GONiqXz4n7R9+x0hWVn0jE5FHx5m7/AGrARzAitmrOCu9XfxQ8UP3LPhHnZX7+bOtDtx0DrYOzwhhBBCHEeS6BRCCCEGMa1GYUSIByNCPLh5ShTNnT1sKqhjfV4N6bm1lNS3s6Wwni2F9Tz13/14OOoZH+Xdf7CRGX83k723IH7MyQwxU9UFYOmDmn2H53yWbVOrPRsK1ZW5Ur1PZ1QTpodmfQalgqu//fYh/hT3mTNxiIig7KabMRysovSCCwha+jROJ59s79AGLA+jBy9MeYGXM17mpT0vsTJ3JXvr9rJk0hL8neXfBSGEEOJEIYlOIYQQYghxNeo5M96PM+P9ACiuayM9r5b1uTX8UFBHQ3sPn++p4PM9FQBE+zozIUo9/X18pFmqPQcajRZ849U16jL1WkcjlO84POuzbBt0NkHJJnUd4hp0eM5nUCr4JYHeaI9diD/AccQIgj9cTs7lV2AqLaXkqqvxueN2POfMkdPFf4FWo+WGlBtINCdy94a7yarLYvaq2SycsJBxgePsHZ4QQgghjgNJdAohhBBD2DAvJy7xcuKSscPo6bOwp7SR9Nwa0vNqyShrJLeqldyqVl7fUMhZif48c0GKnOA+0JncIXKKugAsFqgvODzns2w7VGerp7xnl0H2v9X7NHrwT+qf9dl/0rt7iHpqvBiQdL6+lF17DSO3bqPls8+oXvgEXTk5+D38MBqjJK1/yYSgCayYsYL5a+ezt24v1397PdenXM+1SdeiUeTrmxBCCDGUSaJTCCGEOEHotRpGh3oyOtST+WfE0NjezQ8FdaTn1vCvneV8kVmJRqPw9OxkSXYOJhoNmKPUNeIi9VpXq3qqe9nWw4cdtdeqlaDlO2BL/7POvocPOApKVdvfHZzsthXxU1a9Hp8Fj+CYEE/VE0/S9OlndBUcIOj559D7+dk7vAEr0DmQd6a9w8KtC/k492Ne2P0CGTUZLJywEDeDm73DE0IIIcQxIolOIYQQ4gTl7ujA9ER/pif6c1qcL9e/v4PP91SgVWDx7BS00sY+eBmcIWyCugCsVmgsPjzns2wbHMyA1irYt0pdAEp/q/yPZ316RUjVp50pioLnpZdiiI6m/JZb6czKonDmLIKefQbHkSPtHd6AZdAaePCkB0n2TmbB5gVsKN/A7M9ns2TyEuK94u0dnhBCCCGOAUl0CiGEEILThvvy3IUjufGDnfxndwVajYanZibJzM6hQlHAI1RdSbPUaz0dULnnR7M+t0NzuZoAPZgB219X7zN5/CjxORoCR4FRKuLswWnsWEI/+ZiyuTfStX8/xXMuw+/++/CYPdveoQ1o50aeS5xnHPPWzqO0pZRLVl/CPWPu4byo82TeqRBCCDHESKJTCCGEEABMTfDj2QtHcNPyXXyyswydRuHxvyVKsnOo0psgZKy6DmkqPzLxWbELOhog72t1AaCAd+zhdvegVPXPGhl3cDw4BAURuvwDKu65l5avvuLgAw/SmZOD3913ozg42Du8ASvGM4YPZ3zIvRvuZW3pWh7e9DC7q3dz39j7MOpk3qkQQggxVEiiUwghhBA20xP96bNYueXDXazYXopGo/DouQmS7DxRuAWqK/5c9c+93VCVqSY9DyVAG4qgJkddu95V7zO4QuDII1veHT3ttYshT+PoSODTS6iLjaXmmWdoXP4hXXl5BD3zDDovL3uHN2C5OrjyzORneDPrTZ7d9SyfFnzKvvp9PD3paYJdg+0dnhBCCCGOAkl0HgPLli1j2bJl9PX12TsUIYQQ4nc7OzkAi9XKvBW7Wb61BJ1G4Z/nxEuL54lI56C2qgeOgjHXqtdaq49MfJbvhK5mOLBWXYd4Rhxudw9OA5940Mq3nkeLoiiYr7sWQ0w0FbfdTsf2Herczueew5Qg8yd/iUbRcGXilSSaE7k9/Xb2N+zn/FXn8+j4R5kcMtne4QkhhBDiT5LvNo+BuXPnMnfuXJqbm3FzkxlWQgghBp9zUgLp7bNy28d7eHdzMVqNwoNnD5dkpwBnH4idri6Avl61uvNQu3vZNqjNhfoCdWV8qN6nd1RPdbe1vKeBi6/99jFEuEyeTOhHKym7YS7dRUUUX3QR/gsW4Hb2DHuHNqCl+aexcsZK/rHuH+yp2cPN39/MVYlXMTdlLjqN/IgkhBBCDFbyt7gQQgghftZ5o4Los1i545MM3vqhCK1G4b6z4iTZKY6k1YFforpGX6Fea69XKz1/PO+zqwmKN6rrELeQw4nP4DT1Y+gM9tnHIGYIDyd05QrKb7+dtnXpVNx+O505Ofj8Yz6KVmvv8AYsXydf3jzzTZbsWMJ7Oe/xWuZrZNZk8sTEJ/AyyQgAIYQQYjCSRKcQQgghftHs1GB6LVbu+Xcmr28oRKdRuGtarCQ7xa9z9ISo09QFYLFAXd6Ric/qvdBUoq7sf6n3aR3AP/nIWZ9uQeqp8eJXaV1dCX7hBWqeeZa6V16h/o036Nq/n8Ali9FKh9Ev0mv13Jl2J8neyTzwwwNsObiF2atms/iUxaT4pNg7PCGEEEL8TpLoFEIIIcSv+vuYEPqsVu7/TxYvpx9Ap1W47YwYSXaK306jAe8YdY24WL3W1fKjqs/tULYV2usOJ0MPcfY7surTPwUcHO2yjYFO0WrxmT8PY1wsFffcS9vGjRTOmk3wsucxREXZO7wBbWrYVKI8opi3dh6FTYVc/tXl3JZ6G3+P/bt8rRNCCCEGEUl0CiGEEOL/dcnYYfT1WXjo870s+74AnUbDvNOj7R2WGMwMLhB+iroArFZoKDw857N0K1RlQetB2LdKXQCKFvwSDs/5DBoNnuFS9fkjrtOm4RAaStncG+kpKaHo/AsIeOpJXKZMsXdoA1qEewTLz1rOgz88yH+L/svCrQvZU72Hh05+CEe9JNeFEEKIwUASnUIIIYT4TS4bF0avxcqCL3J45rs8tBqFm6dIlZg4ShRFTVh6hkPSbPVadztU7umv8twKpdvUxGflHnVte029z+TZX/HZ3+4eMBKMrvbbywBgjIsj9OOPKL91Hu1bt1I290bMN96I+YbrUTQae4c3YDnpnXhq4lOkeKewePtiviz6kv0N+3l68tOEu4XbOzwhhBBC/D8k0SmEEEKI3+yqCeFYrFYeW72PJd/kotUozJ0cae+wxFDl4AjDTlIXqFWfzeX9FZ/9Le6Vu6GjHvL+qy4AFPCJO3LWpzlabaE/geg8PQl5/TWqnniShvfeo/b55+ncl0PAwifQOjvZO7wBS1EULh5+MfHmeG5bexsHmg5w4aoLeXjcw0wNnWrv8IQQQgjxKyTRKYQQQojf5ZqJEfRarDz51X6e+u9+dBqFa0+JsHdY4kSgKOrhRG5BEP9X9VpvFxzMOlz1WbYNGkvUw46q98LOt9X7DG4QNOpw4jNwlHpo0hCn6PX43Xcvxrg4Dj70EK3ffkfxhRcQtGwZDiEh9g5vQBvhM4IVZ6/gjvQ72HZwG7evu5091XuYP3o+eo3e3uEJIYQQ4mdIolMIIYQQv9sNkyLp67Oy+JtcHv9yH1qNwlUTpK1T2IHO0J/AHAVcp15rqYLy7eqcz7LtULETupqgYI26DvGKPDznMygVfIaDdmh+e+x+3t8wRIRTdtPNdOXlUzhrNoFLFuM8bpy9QxvQzCYzr5z+Cs/teo43st7gvZz3yK7LZtEpi/Bx9LF3eEIIIYT4H0PzOzkhhBBCHHM3TYmix2Ll2e/yWPBFDjqNwmXjwuwdlhDg4guxZ6kLoK9Xre4s23r4sKO6/MNrzwfqfXonCBx5OPEZlArOQyeZZUpJIfTjjym7+SY692RQevU1+Nx2G56XXyYni/8KnUbHvFHzSPJO4r4N97GrehezPp/FolMWkeqXau/whBBCCPEjkugUQgghxB8277Qo+iwWln1fwEOf70Wr1XDJ2GH2DkuII2l14J+krtSr1Gvt9VC+o7/qc5v6v7uaoWi9ug5xD+mv+uxPfPolgs7BPvs4CvS+Pgx75x0OPvxPmv71L6qffJLOfTn4//OfaIxGe4c3oE0JmULUjCjmrZ1HbkMuV319FbeMvIXL4y+XRLEQQggxQEiiUwghhBB/mKIo3HZGDL0WKy+vO8D9/8lCp1G4ME1m/4kBztETok5XF4DFArW5h+d8lm2H6hx13mdjCWR9rN6nNUBASn/is7/y0y3Ibtv4IzQGA/6PLsAYF0fVwoU0f/Y53QUHCHr+OfT+/vYOb0ALcQ3hvenvsWDzAj4r+IyndzzNnuo9LBi/ABcHF3uHJ4QQQpzwJNEphBBCiD9FURTumhpLX5+V1zYUcve/MtEqCrNTg+0dmhC/nUYDPrHqGnmpeq2zWa30PNTuXrZNPeG9dIu6DnEJOLLdPSAF9Ca7bOO3UhQFz0suxhAVRfmtt9KZnU3hzFkEPfsMjqNG2Tu8Ac2kM7Fg3AJSfFJ4fMvjrCldwwWrLmDJpCXEeMbYOzwhhBDihCaJTiGEEEL8aYqicO9ZcfRarLz1QxF3/isDrUbhvFGDq9JNiCMYXSFisroArFaoP3A46Vm2TT3xvaUCcj5TF4BGp7a4H0p8BqWCR6h6avwA4zR2jDq3c+5cuvbvp/iyy/G79148Ljjf3qENaIqiMCt6FsM9hzN/7XxKWkq4ePXFPHDSA5wdcba9wxNCCCFOWJLoFEIIIcRRoSgKD549nD6LlXc3F3Pbx3vQahTOHRFo79CEODoUBbwi1JV8gXqtuw0qdh+Z/Gytgopd6tr6inqfo/nIdvfAkWAYGK3ODkGBhC7/gIp776Xly684+NBDdObk4HfvPSgOg3ce6fEQb45nxYwV3LX+LjZWbOSeDfewu3o3d6bdiYNWPndCCCHE8SaJTiGEEEIcNYqi8PBf4umzWvlgSwnzV+5Gq1E4OznA3qEJcWw4OEHoOHWBWvXZVHp4zmfZNqjcA+21kPulugAUDfgM/1HLexp4Raot9HagcXQkcMkS6mLjqFm6lMYVK+jKzyfomaXozGa7xDRYuBvdWTZlGa9kvMKLe15kZe5KsuuyWTJpCQHO8rVPCCGEOJ4k0SmEEEKIo0qjUVhwTgJ9fVZWbC/l1hVqsnN6ohxyIk4AiqKe1O4eAgnnqdd6u6Ay40dVn9uhqQSqstS14y31PqMbBPYnPoNTwSf5OIeuYL72Ggwx0VTcdjsdO3aoczufew5TYsJxjWWw0Wq0XJ9yPYneidy1/i6y67KZvWo2T0x4gnGB4+wdnhBCCHHCkESnEEIIIY46jUbh8b8l0mux8snOMm5evguNojA1wc/eoQlx/OkMauIyOPXwteZKKN9+OPFZvhM6m6DgO3UBeuBUgz/avi8hZIyaAPWJA432mIbrMmkSoStXUjZ3Lt2FhRRffDH+Cx7B7WyZPfn/GR84npUzVjJ/7Xyy67K5/tvruT75eq5NvhaNYp9qXSGEEOJEIolOIYQQQhwTGo3CkzOT6LNY+M/uCm5avpMXLxrFacN97R2aEPbn6g+uZ0Ncf/Kwrweqso9sea8vwKWrEjKWqwvAwRkCRvRXfaapFaDO3kc9PEN4GKErV1Bx2+20rltHxe130Lk3B59/zEfRyY8QvybAOYB3pr3Dwq0L+Sj3I17Y8wJ7avewcPxC3I3u9g5PCCGEGNLkuxQhhBBCHDNajcKiWcn0WeHzPRXc8P5OXr5kFJNjfewdmhADi1YPASnqSrsagJ6mg+z49GVS/RW0Ff1Vn90tULReXYd4hB6e8xk0GnwTQPfnD8LRurgQ9MIyap59jrqXX6b+zTfp2r+fwCWL0bq7/+mPP5Q5aB144KQHSPZO5pHNj7CxfCPnrzqfJZOWEG+Ot3d4QgghxJAliU4hhBBCHFM6rYanZydjsVj5IrOSa9/bwauXjuaU6KNfhSbEkOLoRZVbCpZJ09Hq9WDpg5r9R57wXrMPGorUlfmR+pzOCP4patIzOE1Ngrr+sUNxFK0Wn3m3YoyLpeLue2j74QcKZ59P0PPPYYyOPlo7HbLOiTyHWM9Y5q2dR2lLKZd8eQl3j7mbmVEzURTF3uEJIYQQQ44kOoUQQghxzOm0GpZekEKvxcJ/s6u45p3tvD4nlfFRcpqzEL+ZRgu+w9U1ao56raMRKnaq7e6lW9XkZ2cjlG5W16b+Z10D+0947098+ieD3vibX9p16lQcQkMpm3sjPSUlFF1wIQFPLMT19NOP9i6HnBjPGD6c8SH3bbiP70u/55+b/snu6t3cN/Y+TDqTvcMTQgghhhSZiC2EEEKI40Kv1fDchSM5Lc6Xrl4LV72zjR8Kau0dlhCDm8kdIk6FU+6Aiz+GO4vgxh1w7ksw+grwSwRFA83lsPdT+PpeeOMMeDwIXpkMX94JmR+rFaFW66++lDE2ltCPP8JxzBis7e2U33QzNc89j9ViOR47HdRcHVxZOnkpt468FY2i4bOCz7h49cWUNJfYOzQhhBBiSJFEpxBCCCGOGwedhmUXjeDUWB86eyxc+dZ2thyos3dYQgwdigLmSEi5EGY8DddtgLtK4bIv4LSHIOYscPIGS49aCbrlJfjkSngmGRZFw/K/w/olULgeulp/8uF1Hh6EvPYqHpdeAkDtsmWU3Xwzfa1tx3mjg49G0XBl4pW8evqreBo9yW3I5YJVF7CmZI29QxNCCCGGDEl0CiGEEOK4Mui0vHDRSCZGe9PR08flb21je1G9vcMSYugyOEPoeBg/Dy78AG7Lg1sy4LzXYcx1EDgKNHpoq4b9X8B3D8PbM2BhMLw0HlbNg90fQG0eWK0oej1+99yD/2OPoej1tH77HUUXnE93cbG9dzoopPmnsXLGSlK8U2jpaeGW729h6Y6l9Fp67R2aEEIIMehJolMIIYQQx51Rr+WVS0YxPtJMe3cfc97Yyo7iBnuHJcSJQVHAYxgkzoRpT8DVa+DuMrjyGzjjURh+LrgGgdUCBzNh+xvwn+vh+dHwRCi8NxPWPoF7kivDXn8RnY8P3fkFFM6aTev6Dfbe3aDg6+TLG1Pf4OK4iwF4Pet1rv3mWmo7ZJyHEEII8WdIolMIIYQQdmHUa3n10tGcFO5FW3cfl72xld2ljfYOS4gTk96ontB+8o0w+22Ynw3zc2D2u3DyTRByknqae2cj5H8Dax+D9/6GafVZhM5oxxTiiqW5mdJrr6Xutdew/j/zPgXoNXruTLuTp055CpPOxNaDWzn/8/PZXb3b3qEJIYQQg5YkOoUQQghhNyYHLa9fNpq0ME9aunq59PUtZJY12TssIQSAawAM/wucsQCu+Eqt+rxmLUx7ChJng0cYYEXfnkvImH24hbeBxUL1osVUnDcGy5cPwv6voE2qFH/N1NCpfHjWh4S7hVPdUc3lX13O+znvS7JYCCGE+AMk0SmEEEIIu3J00PHmZamMHuZBc2cvF7++hewKSXYKMeBo9RAwAsZcA+e9CrfshtsL4MIP0Uz6B/6zk/BN6wDFSvPeFor/+R49r/0dnoqAZ0fAv66Bra9CxS7o67H3bgaUcPdwlp+1nKmhU+m19rJw60LuSL+D9p52e4cmhBBCDCo6ewcghBBCCOFk0PHWFWlc+voWdpY0cvFrW/jg6rHE+bvaOzQhxK9xMkPMNIiZhgJ4XtqH4b8fU/7AE3Q2QOG3fgSdVIMjB6D+AGSsUJ/TmdSkadBoCEpV2+Zd/Oy6FXtz1Dvy5MQnSfFJYdG2RXxV9BW5Dbk8Pelpwt3D7R2eEEIIMShIRacQQgghBgTn/mRncrA7De09XPTaFvYfbLF3WEKI30OjxWna+YT953MMcXH0dVgpTvelwXMeTLobIk8Doxv0dkDJD/DDs7DyElgcA08nwEeXwaZlULoNervsvZvjTlEULoq7iDenvomPyYcDTQe44IsL+KroK3uHJoQQQgwKkugUQgghxIDhatTzzhVpJAa6Ud/WzUWvbSa/WpKdQgw2+sBAQj94H9fp06C3j4MvrKByTQfW2cvhjiKYuw3OeQFGXQ6+iaBooKkUsv8N/70HXj8NHg+CV6fAl3dB5sfQWAInyNzKFJ8UVp69kjS/NDp6O7h93e08sfUJeqTlXwghhPhV0rp+DCxbtoxly5bR19dn71CEEEKIQcfNpOfdK9P4+6tb2FvZzIWvbuHDa8YS4e1s79CEEL+DxmQiYPFiDHFx1Cx5msaVK+nKyyPo2WfQeUeDdzSMuEi9uatFnd1Ztg3KtkPpVmivhfLt6trS/0GdfdVW96DREJQGASng4GSvLR5TXiYvXj79ZZ7f9TyvZ73OeznvkVWbxaJTFuHr5Gvv8IQQQogBSSo6j4G5c+eyd+9etm3bZu9QhBBCiEHJ3dGB968aQ6yfCzUtXVz4ymYKa9vsHZYQ4ndSFAXz1VcT/NKLaFxc6Ni1i8KZs+jIzDryRoMLhE2ECf+AC5fD7flw827422uQdi0EjASNDlqrYN8q+PYheGs6PB4ML02AVfNhz4dQVzCkqj51Gh23jrqVZyY/g4vehd01u5m9ajZbK7faOzQhhBBiQJJEpxBCCCEGJA8nNdkZ7etMdUsXf391MyV1cgKxEIOR8ymnELpyBQ7h4fRWVVF80UU0ffrpLz+gKOAZBkmzYPqTcM33cHcZXPFfOGMBDD8HXALA2gcHM2D76/Dva+G5kfBkOLw/C9Y9BQXfQ2fT8dvoMXJqyKl8OONDoj2iqe+s5+pvrub1zNexDqGkrhBCCHE0SOu6EEIIIQYsL2cD7181lgtf3Ux+dSsXvrqZD68ZS7Cno71DE0L8ToawMEJXrqDi9jto/f57Ku68i869OfjcfhuK7jf8WKI3QchYdR3SVN7f7t6/KnZDRz3kfa0uABTwjlXb3YPT1NZ3cwxoBlfNR4hrCO9Nf48FmxfwWcFnLN25lD01e1gwfgGuDq72Dk8IIYQYECTRKYQQQogBzdvFwAdXj+GCVzZzoKaNC1/dzIprTyLQ3WTv0IQQv5PW2ZmgZc9T89xz1L34EvVvv01XXi6BS5agdXf//R/QLVBd8eeqf+7thqpMdc5n2TZ11mdjMdTkqGvXu+p9BlcIHKnO+Tw089PR82ht85gx6UwsGLeAFJ8UHt/yON+Xfs8Fqy7g6UlPE+MZY+/whBBCCLsbXL/GFEIIIcQJycfFyPKrxxJmdqKsoYMLX9lMZVOHvcMSQvwBikaDzy23ELh0KYqjI20/bKJw1mw69+f++Q+uc4DAUTDmWjjvNbg1A27LgwuWw/j5EDoB9E7Q1QwH1kL6k/DBLHgyDJ4dCf++Dra9BpV7oK/3z8dzDCiKwqzoWbw77V0CnAIobSnlotUX8Wn+r4wCEEIIIU4QUtEphBBCiEHB19XIB1eP4fyXN1NS386Fr6iVnb6uRnuHJoT4A1ynnolDWBhlc+fSU1pK0YUXErDwcVzPOOPovpCzD8ROVxeoCcyanP6Kz/6W97o8qC9Q157l6n16R/UQpKDR/VWfqeAycE47jzfHs2LGCu7acBcbyzdy38b72FOzhzvT7sSgNdg7PCGEEMIupKJTCCGEEIOGv5uJ5deMJcjDRFGdmuysbu60d1hCiD/IGBNN6EcrcRw7Fmt7O+U330LNs89itViO3YtqdeCXCKOvgL++CDdthzsK4aJP4JS7IOJUMLhBTzsUb4CNS2HFRbA4Gp5OhI+vgM0vqu3xvV3HLs7fwN3ozgtTXuCGlBtQUPgo9yPmfDmH8tZyu8YlhBBC2ItUdAohhBBiUAl0N7H86rHqzM7aNv7+2haWXz0WbxepYBJiMNJ5eBDy2qtUP/UU9W+/Q+0LL9K5bz8BTz6B1tn5+ATh6AlRp6kLwGJRqzwPzfks2w7Ve6GpRF1Zn6j3aQ3gn3x4zmdQKrgFqafGHycaRcP1ydeTZE7izvV3kl2XzfmrzmfhhIWMDxx/3OIQQgghBgKp6BRCCCHEoBPs6cgHV4/B381IfnUrF722mbpW+1ZWCSH+OEWnw/fuu/Ff+DiKgwOta9ZQdP4FdBcV2ScgjQa8Y2DExfCXZ+GGH+CuErj0Mzj1PoieCo5e0NcFZVth8zL4+HJYmgBL4mDFxbDxGSj+Abrbj0vI4wLHsXLGShK8EmjqauKGb2/gxd0vYrEew+pYIYQQYoCRik4hhBBCDErDvJz44OqxXPDKJnKrWrmov7LTw8nB3qEJIf4g93PPxRAeTtmNN9FdUEDh7PMJXLwI5wkT7B0aGF0h/BR1AVit0FCoVnuWblWrP6uyoKUScj5XF4BGB74Jh+d8Bo0Gz/BjUvUZ4BzA29Pe5sltT7Ji/wpe2PMCe2r3sHD8Qpy0Tkf99YQQQoiBRio6hRBCCDFohZnVZKe3i4F9B1u46LUtNLZ32zssIcSfYEpKIvTjjzClpGBpbqb02uuoe+01rFarvUM7kqKoCcuk2XDWIrh2HdxVCpd/Baf/E+LOBmc/sPRC5W7Y9ir8+xp4biQ8FQEfnA/pT6mnv3e1HLWwHLQO3Df2Ph4b/xhGrZGN5RuZvWo22XXZR+01hBBCiIFKKjqFEEIIMahFeDuz/OoxXPDKZvZWNnPJ61t576oxuJn09g5NCPEH6X18CHnnbaoeeYTGjz6metFiOnP24b/gETQmk73D+2UOjjDsJHWBWvXZXH54zmfZNjXp2V4HuV+pCwAFfIYfnvMZnAZeUWoL/R90dsTZRHtEM3/tfEpaSrjimyuYZpjGNOu0P71NIYQQYqCSRKcQQgghBr1IH5f+NvbNZJY3cekbW3n3yjRcjZLsFGKw0jg44PfPf2KIi6Pqscdp/uILugoPEPz88+gDAuwd3m+jKOrhRG5BkPA39VpvFxzMUmd7lm1TV2MJVGera+fb6n0GNwga1d/ungaBI9VDk36HGM8YPpzxIfdtuI81pWv4rOMzLJstPHDyA5h0AzhhLIQQQvxBkugUQgghxJAQ7evC+1eN4cJXN7OntJE5b2zlnSvScJFkpxCDlqIoeP797xgiIym/dR5de3MonDmLoGeW4piaau/w/hidoT+BOQq4Xr3WUnU46Vm2HSp2QlcTFKxR1yFeUYfnfAangXccaH/9RzoXBxeWTl7Kaxmv8dzu51hVuIrcxlyWTFrCMNdhx26fQgghhB3IjE4hhBBCDBlx/q68d6Xatr6rpJHL39xGW1evvcMSQvxJTmlphH38EYbhcfTV11N8+RXUf/DBwJvb+Ue5+ELcDDj9Ybj8C3XW57XpcNZiSL4QvCLV++ryYM8H8MV8eGk8LAyBt2bAtw/BvtXQWv2zH15RFC4bfhmXO12Op9GT3IZcLlh1Ad+VfHf89iiEEEIcB5LoFEIIIcSQkhDoxntXjsHFqGN7cQOXv7WN9m5Jdgox2OkDAgh9/31czzoLenup+ucjHHzgASzdQ/AAMq0O/JMh9Sr460tw0w64oxD+/hFMvAPCJ4PBFXraoGg9bHgaPrwQFkXB0iT4+ErY/BKU74Dew5+fcH04y6cuZ4TPCFp7Wrn1+1tZsmMJvRb5GimEEGJokNZ1IYQQQgw5iUFuvHvlGC55bQtbC+u58q3tvHFZKiYHrb1DE0L8CRqTiYBFT2EcHkf1osU0fvQxXfkFBD6zFL2Pj73DO7YcPSH6DHUBWCxQu//IlvfqHGgsVlfWx+p9WgMEpKAJGIl/gxbv3hReP/N1nt7xNO/ufZc3s94kqzaLJyc+idlktt/+hBBCiKNAKjqFEEIIMSSlBLvz9pVpOBt0bDpQx9XvbKezp8/eYQkh/iRFUfC68kqCX3kZjYsLHbt2UTRzFh0ZGfYO7fjSaMAnDkZeCn95Dm7YBHcVwyX/gcn3QdSZYPKEvi4o3YJ2y4ukFT2P/rkk9E8ncceBTBb5nYaj1sC2g9uY/flsdlXvsveuhBBCiD9FEp1CCCGEGLJGhnjw1uWpODpo2ZBfy7Xv7pBkpxBDhPOECYR9tBKHiAh6q6spvvgSGv/zH3uHZV9GN4iYDKfcDhethDsOwE074a8v0zfqChpNw7AqWmipgJzPOHPTGywvLiS8u4eajhqu+HIO7377D6x1B2CozD8VQghxQpFEpxBCCCGGtNGhnrx5WSomvZZ1uTXc8P5Ounol2SnEUOAQGkroig9xPvVUrN3dVN51N1WPP461V2ZOAqAo4BUByRdgmfok62Ifofe2A3DZajjtYYidQbjBi+UVB5nW2kYvVp4s/5rbPzyNtkWR8MEFsH4xFKZDV6u9dyOEEEL8v2RGpxBCCCGGvDHhXrxxWSqXv7WVNfuqmfv+Ll64aCQOOvmdrxCDndbZmaDnn6P2+WXUvvAC9W+/Q2duLoFLlqDz8LB3eAOPgxOEjlMXgNWKY1MpT5RuJTnvYxY1Z/NfZydyHXp4uvAbInK/VO9TNOAzHIJSDy+vSLWFXgghhBgg5G8lIYQQQpwQTorw4rVLUzHoNHybU8VNy3fS02exd1hCiKNA0WjwvvkmAp99BsXRkfZNmymaNZvO/bn2Dm3gUxRwD0FJnMlFf/uQN6e/i4/Jh0IHPRcGh/BlzCngFgxWC1RlwY434dMbYFkqPBkG750HaxdC/rfQ0WDv3QghhDjBSaJTCCGEECeM8VFmXrl0NA5aDf/NruKWD3fRK8lOIYYM1zPOIHT5cvTBwfSUlVF04YU0//dre4c1qKT4pLDy7JWM8RtDh7WXO7oLWTjuYnpuzYTz34Nxt0DIyaAzQWejmuBc+7ia8HwiFJ5Pg//Mhe1vwsEssMioECGEEMePtK4LIYQQ4oRySrQ3L18yimve3c7qzINoNXt4enYyOq38/leIocAYE03YRyspnz+fth82UX7LLXRefx3eN92EIm3Wv4mXyYuXT3+ZZbuX8Wrmq7yf8z7ZtdksOmURvnFnqzf19UBVNpRtO7zqD0DtfnXtfk+9z8EZAkce2fLuZLbf5oQQQgxpkugUQgghxAlncqwPL140iuvf38HneyrQKrB4dgpajWLv0IQQR4HW3Z3gV16hetFi6t96i7oXX6Jr334CnnoSrbOzvcMbFLQaLTePvJlEcyL3briX3TW7mb1qNk9OfJIx/mNAq4eAFHWlXa0+1FYH5duhdKua+CzfCd0t6mFGhemHP7hH2I8Sn6PBL1H9eEIIIcSfJIlOIYQQQpyQThvuy3MXjuTGD3byn90VaDUanpqZhEaSnUIMCYpOh+9dd2KMi6Xy/gdo/f57imafT9Cy5zGEhdk7vEFjcshkVsxYwby189jfsJ9rvrmGm0bcxBUJV6BR/qdC1skLos9UF6ht6zX7+ys+t0LZdqjZBw2F6spcqd6nM0LACDXpeSgB6hpwfDcqhBBiSJBEpxBCCCFOWFMT/Hj2whHctHwXn+wsQ6dRePxviZLsFGIIcTvnHBzCwym78Sa6DxygaPb5BC5ehPPEifYObdAIdg3mvenvsWDzAj4t+JRndj7Dnpo9PDr+UVwdXH/5QY0WfIera9Qc9VpHI1TshNIftbx3NkLJJnUd4hp0ZOLTPxn0xmO5TSGEEEOAJDqFEEIIcUKbnuhPn8XKLR/uYsX2UjQahUfPTZBkpxBDiCkxkbCPP6Lsllvp2LmT0muvw3vePLyuvgpFkX/Xfwujzsgj4x4hxSeFx7Y8xtrStVyw6gKWTFpCrGfsb/9AJneIOFVdAFYr1BX0V3z2Jz6rsqG5DPaWwd7/qPdp9OCfdOSsT/cQ9dR4IYQQop8kOoUQQghxwjs7OYA+i5V5K3ezfGsJOo3CP8+JlwSIEEOIztubYW+9ycEFj9K4ciU1S5bQtS8H/0cfRWMy2Tu8QUFRFGZGzyTOM475a+dT2lLKxasv5r6x93Fu5Ll/9IOCOVJdKX9Xr3W1QuXu/lmf29UkaFsNlO9Q15aX1PucfA7P+QxOU9vfHZyOxlaFEEIMUpLoFEIIIYQAzh0RSK/Fyu0f7+HdzcVoNQoPnj1ckp1CDCGKgwP+/3wY4/A4Di54lObVX9JVWETw88+hDwy0d3iDRrw5npVnr+Su9XexoXwD92+8n93Vu7l7zN0YtIY//wIGZwgdry5Qqz4bS4484b0yA9qqYf8X6gJQ+lvlg1IhKE39b68IqfoUQogTiCQ6hRBCCCH6zRwVhMVi5Y5PMnjrhyK0GoX7zoqTZKcQQ4zHBRdgiIig7JZb6crJoXDmLAKXLsVpTJq9Qxs03AxuLJuyjFcyXuGF3S/wSd4n5NTnsGTSEgKdj3LSWFHAY5i6Emeq13o6oXLPj5Kf29V294OZ6tr+hnqfyQMC+2d9BqdC4Cgwuh3d+IQQQgwYkugUQgghhPiR2anB9Fqs3PPvTF7fUIhOo3DXtFhJdgoxxDimpqpzO2+8ic69eym54gp8774bj4v+Lv++/0YaRcN1ydeRZE7izvV3srduL7M/n83CCQuZEDTh2L643gghY9R1SHPFkYnPil3Q0QD536gLAAW8Y/oPOuqv+vSOUQ9OEkIIMehJolMIIYQQ4n/8fUwIfVYr9/8ni5fTD6DTKtx2RowkP4QYYvQBAQx7/z0q73+A5lWrqFqwgM59Ofg98AAaBwd7hzdonBx4MitmrOAfa/9BVl0Wc7+by7XJ13Jd0nVoj2cC0TUAhp+jLoC+HrW6s2z74QRoQyHU7FPXrvfU+xxcIHCkOuczKFWtAHXyOn5xCyGEOGok0SmEEEII8TMuGTuMvj4LD32+l2XfF6DTaJh3erS9wxJCHGUak4mAp57EGBdH9eLFNH38Cd35BQQ++wx6Hx97hzdoBDgH8Pa0t3ly25Os2L+Cl/a8REZNBgsnLMTD6GGfoLR6NYEZOBLGXKNea62B8h8lPst3QncLFK5T1yGe4Uee8O4br348IYQQA5okOoUQQgghfsFl48LotVhZ8EUOz3yXh1ajcPOUKHuHJYQ4yhRFwevKKzBER1P+j3/QsXs3RTNnEfTcs5iSk+0d3qDhoHXgvrH3keydzD83/ZMfKn7g/FXns2TSEhLMCfYOT+XsDTHT1AVg6YPqnMPt7mVboTYX6g+oK2OFep/OpJ7qHvyj5KeLn/32IYQQ4mdJolMIIYQQ4ldcNSEci9XKY6v3seSbXLQahbmTI+0dlhDiGHCeMJ6wj1ZSOncu3fkFFF98CX4P/x979x0fdX34cfx1lx1Iwt4bke0AwYF7UVxF6164cAC16s/VqnVVa23dA/fede9FVRQQGSIIgspQZMqG7OTu98eRSAQV8JLvXfJ6/h55JDfzvvPT/MI7n3E1DY44POhoSeXQzofStVFXzv/gfL5f+z0nv3Uyl/a/lKO2PSrxtgAJp0CLXrGPnU6NXVe4EhZMqrrkvWg1fD829lEhr23VvT5bbgepcTh1XpK01Sw6JUmSfsOZe3amtDzKv9+Zxb/fmUVqOMRZe3UOOpakapDevj0dnnmWhZdewrr3R7Hob3+j6KuvaH7xRYTSXLq8ubZtuC3PHPIMV4y5glHfj+LaT69lytIpXLHrFWSlZgUd79dlNYRt9o99AEQisPzbDQ46mgBLZ8Dq+bGP6S/F7peSDi23Xz/jc/1J73ltY6fGS5JqhEWnJEnSZhi+zzaUR6Lc/N7X/POtmaSEQ5yxR6egY0mqBin169Hm9ttZdtfdLLvrLlY+/jjFX39N61tvIbVhQPtNJqGc9Bxu2fsWHp3+KLdOvpXX5rzGzJUzuWXvW2if2z7oeJsvHIam28Y+djwhdl3x2tip7j9MgPnry8+CZT8VoRXqt/ip9GzTD1rtAOn1AnkZklQXWHRKkiRtpnP360JZJMrto77hH298RWo4xCkDOgYdS1I1CIXDNP3zCDK6dWXRJZdSMH58bN/Ou+8is2vXoOMljVAoxCm9TqFnk55c9NFFfLPyG459/Vj+sfs/2K/dfkHH23oZOdBxz9gHQDQKK+dtsNz9s9iJ7+sWw8zXYx8AofVL5Tc86KhRJ2d9SlKcWHRKkiRtgfP370J5JMJdH8zmqtdmkBIOcdKuHYKOJama5B5wABkdOjB/+AhKv/+eecceR6t/Xk/uH/4QdLSk0q9FP5479Dku+ugiJi+dzHkfnMepvU7l3B3PJTVcC/5ZGgpBo46xj+2Oil1XWgiLvlg/6/Oz2Oe1i2LXLfoCJjwQu19Wow2Kz52gdV/IzA3utUhSEqsF/x9FkiSp5oRCIS48sCtlkSj3fjSHK16ZTko4zPE7tws6mqRqktGlCx2fe5YFF/wf+WPHsuC88yk6ayZN/3IuoXA46HhJo1l2Mx4Y+AC3TrqVx2Y8xsNfPsy0H6fx773+TZOsJkHHi7+0LGi3S+yjwuoFVff6XDgFClfAN+/EPgAIQbPuVZe8N+kaW0IvSfpVFp3V4K677uKuu+6ivLw86CiSJKkahEIhLv1DN8rLozzwyVz+9tI0UsMhDt+hRdDRJFWTlAYNaHvfvSy96WZWPPwwy++9l+JZs2j17xtJyckJOl7SSAuncVG/i9i+6fZcMeYKJi6ZyNGvHc1/9voPfZr3CTpe9ctrHfvoOTh2uawElkyLLXmvmPW56rvYYUdLZ8Dkx2L3y8iNzfRs0y9WnHbax+JTkjbBorMaDB8+nOHDh7NmzRry8vKCjiNJkqpBKBTisoO7UxaJ8sjYeVzy4lSIRsgIOpikahNKTaX5JReT2aM7iy6/gnUffsi8o4+hzV13kdHJ/Xq3xIEdDqRLwy6c/8H5zF49m9PeOY0L+l7AST1OIlSX9qtMTY8VmK37ws5nxa5bt3T9Xp+fxT4vmAzFa2DOB7EPgPa7w+EjoYGrCSRpQ/4JSJIkaSuFQiGuPLQHJ+3SnmgULnnpSyb+WIf+gS7VUXmHHkr7J58ktUULSubOZd7RR7Puo4+CjpV0OuZ15KmDn2JQx0GUR8v598R/838f/R/5pflBRwtW/WbQ7SDY/yo45XW49Hs4+xM45BbY/nhIqwfffQIjB8CUp2MHIUmSAItOSZKk3yUUCnH1YT05rn87olF44tswb0xbHHQsSdUsq1dPOj7/X7L69iWybh3zzz6HZffeR9TSaYtkp2Xzrz3+xV/7/5XUcCrvffcex75+LLNXzQ46WuJISYUWvWGn02KzOM/5BNruHJvl+fLZ8NzJkL886JSSlBAsOiVJkn6ncDjEdYN7cWSf1kQJ8X/PT+PNaYuCjiWpmqU2aUL7hx+iwbHHQDTKj7fcwoILLiBSUBB0tKQSCoU4vvvxPDzwYZplN2Pemnkc98ZxvDnnzaCjJaZGneDUt2C/v0M4Fb56FUbuCt+8F3QySQqcRackSVIchMMhrvtjD/o3jVAeiXLu05/z9pfO7JRqu1B6Oi2vuooWV10Fqamsfett5h1/AiU/LAg6WtLZodkOPHfIc+zcYmcKywq55ONL+Of4f1JaXhp0tMQTToE9/g/OGBU7kX3dEnjySHj9Aiip40v/JdVpFp2SJElxEg6HOK5zhMO2a0lZJMqIpybz3owlQceSVAMaHnsM7R99hJTGjSmeOZN5Rx5J/vjPgo6VdBpnNebeA+5laO+hADw18ylOfedUFuf7h6NNarUDnPUR7HxO7PLEB+GePWKHGElSHWTRKUmSFEfhEPzriJ4cun0ryiJRhj05iQ9mLg06lqQakN23Lx2f/y+ZPXtSvmoV3592Gisef8J9O7dQSjiFc/ucyx373kFOWg5f/PgFx7x+DJ8u+jToaIkpLQsG3QAnvQy5rWHFbHjwQPjgenA2rKQ6xqJTkiQpzlJTwtxy9PYc3LslpeVRznpiEh99/WPQsSTVgLSWLWn/5BPkHnYolJez5LrrWHTZ5URKSoKOlnT2brs3zx76LN0adWNF0QrOeu8sHpj2AJFoJOhoianzPnDOGOh9FETL4aN/xQrPZd8EnUySaoxFpyRJUjVITQlz67E7MLBnc0rKIpz52EQ++WZZ0LEk1YBwZiat/vUvml18MYTDrH7xRb476SRKlzi7e0u1zWnL44MeZ/A2g4lEI9w2+Tb+8r+/sLp4ddDRElNWQ/jTA/CnByEzDxZOji1l/+x+cGaxpDrAolOSJKmapKWEueO4PuzfvRnFZRHOeGwCY2dbdkp1QSgUovFpp9L2/vsI5+VR9MVU5h15JIVTpgQdLelkpmZy7YBruXq3q0kPp/PhDx9y7OvHMnPFzKCjJa7eR8I546DTPlBWCG9eCE/8CdYsCjqZJFUri05JkqRqlJ4a5q4T+rBP16YUlUY4/ZGJjJ+zPOhYkmpI/QED6Pjf58josg1lP/7IdyedzKoXXgw6VlI6ossRPHbQY7Su35of1v3AiW+eyMvfvhx0rMSV1xpOfBEG3QipmTB7FIzcFaa/FHQySao2Fp2SJEnVLCM1hZEn9mXPbZtSWFrOqY9MYOK8FUHHklRD0tu1o/3Tz5BzwP5ES0tZdNllLP7HdURLPShmS/Vs3JNnD3mWPVrvQXF5MVeMuYKrxl5FcXlx0NESUzgMO58FZ42GlttD4Ur47ynw4plQuCrodJIUdxadkiRJNSAzLYX7TurL7ts0oaCknCEPfcak71YGHUtSDUmpX4/Wt91Gkz+PAGDlE0/w/RlDKVvpz4EtlZeRx5373cmIHUYQIsQL37zASW+exA9rfwg6WuJq2hVOfx/2vAhCYZj6LIwcAHNHB51MkuLKolOSJKmGZKalcP/JO7Frp8bkl5RzykOfMWX+qqBjSaohoXCYpsOH0+bOOwhnZ1Mwfjzz/nQkRV99FXS0pBMOhTlr+7O454B7aJDRgK9WfMUxrx/D6B8s7n5Rajrsezmc9g407AhrfoBHD4V3LoPSoqDTSVJcWHRKkiTVoKz0FB48ZSf6d2zE2uIyTn5wPNN+8PRgqS7J2X9/Ojz7DGnt21G6cCHzjjueNW++GXSspLRbq9147pDn6N2kN2tK1jB81HDu/PxOyiPlQUdLXG37w9mfQN9TYpfH3Qn37Q2LpgaZSpLiwqJTkiSphmWnp/LwKf3YqX1D1hSVceKD45m+0LJTqksyunSh43PPUW/AAKJFRSy44P9YevMtRMst6LZUy/oteeQPj3BM12MAuHfqvZzz/jmsLHJbgF+UUR8OvQ2OexbqNYUfv4L794VPbgFLYklJzKJTkiQpAPUyUnnktP7s2K4BqwtLOfGB8Xy1aE3QsSTVoJS8PNredy+NTj8NgOX33cf8YcMoX7s24GTJJz0lnct3uZzrd7+ezJRMxi0ax9GvH820H6cFHS2xdf0DDPsUuh0CkVJ4/yp45GBYOS/oZJK0VSw6JUmSAlI/I5VHT+vP9m3yWFlQygkPjGfWYgsOqS4JpaTQ/KKLaPXvfxPKyCD/o9HMO+poiufMCTpaUjq086E8efCTtM9tz+L8xZz89sk8O/NZotFo0NESV70mcMwT8Me7IT0Hvh8XO6jo8yfA901SkrHolCRJClBuZhqPnb4zvVvnsSK/hBMe+JRvl1p2SnVN3qGH0P7JJ0lt2ZKSefOYd/QxrP3gg6BjJaVtG27L0wc/zX7t9qMsUsY/xv+Dv33yNwpKC4KOlrhCIdjxBDjnE2i3K5Ssg1eGw7MnQv6yoNNJ0maz6JQkSQpYXlYaj5/enx4tc1m2roTj7h/P7B/XBR1LUg3L6tWTjs//l6yd+hJZt44fhg1n2T33OhtxK+Sk53DL3rfwf33/j5RQCq/PeZ0T3jyBeavnBR0tsTXsAKe8AftfDeE0mPk63L0LzHo76GSStFksOiVJkhJAg+x0njxjZ7q1yOHHtcUcd9+nzF2WH3QsSTUstXFj2j/0EA2OOxaiUX689VYWnHc+kXx/HmypUCjEKb1O4f4D76dxZmO+XfUtx75xLO9/937Q0RJbOAV2Pw/O/ACa9YD8H+HpY+DVc6HYP8JJSmwWnZIkSQmiYb1Y2blt8/osXVvM8fd/yvfLXWop1TWh9HRaXnklLa65GtLSWPvOO8w77nhKlywNOlpS6teiH/899L/0adaH/NJ8zv/wfG6eeDNlkbKgoyW2Fr1h6Aew6wggBJMfhXt2h/mfBZ1Mkn6RRackSVICaVw/gyfP2IVtmtVn0eoijrv/U+avsOyU6qKGRx9N+0cfIaVJE4q//ppld98ddKSk1TS7KQ8MfIAhPYYA8PD0hznj3TNYVuj+k78qLRMGXgdDXoXcNrByLjw0kPCH/yQUtSiWlHgsOiVJkhJM05wMnhq6M52a1mPBqkKOu/9TFqwqDDqWpABk9+lDiyv/DkDh558HnCa5pYXTuLDfhdy0103US6vHpCWTOOq1o5i0ZFLQ0RJfxz3hnDGw3bEQjZAy5ib2nHUNLPs66GSSVIVFpyRJUgJqlpPJ00N3oUPjbH5YWchx933KotWWnVJdlLXd9gAUf/ute3XGwYEdDuTpg59mmwbbsKxwGcNHDWdNyZqgYyW+rAZwxL1w1CNEsxrSoHAeqQ/uC5/eA5FI0OkkCbDolCRJSljNczN5+sxdaNcom+9XFHDcfZ+yZE1R0LEk1bC05s1Ibd4cIhGKZswIOk6t0DGvI08e9CSt6rUivzSfaT9OCzpS8uh5OGVDR7MkpzehsiJ4+xJ44nBYvSDoZJKUHEXnd999x4wZM4j4VyJJklTHtMzL4ukzd6FNwyzmLY+VnUstO6U6J2u73gAUTrWQi5fstGx2aLYDANOW+b5ukZyWfNr5QsoH3gipWTDnQxi5K0x7Puhkkuq4hCo6H3roIW6++eYq15155pl06tSJ3r1706tXL+bPnx9QOkmSpGC0bpDF00N3oXWDLOYsy+f4B8bz49rioGNJqkGZvbcDoHCahVw89W4SK5CnL5secJIkFAoR2ek0OPtjaNUHilbDC6fD86dD4cqg00mqoxKq6Lzvvvto2LBh5eW3336bhx9+mMcee4wJEybQoEEDrr766gATSpIkBaNto2yeGrozLfMy+XbpOk544FOWr7PslOqKihmdRRadcdWrSS8gNqMzGo0GnCZJNekCp78Le10KoRT48nm4ezeY/UHQySTVQQlVdH7zzTfstNNOlZdfeeUV/vjHP3LCCSfQp08frr/+ekaNGhVgQkmSpOC0b1yPp4buQvPcDL5eso4THhjPyvySoGNJqgGZPXsCULpgAWXLlwecpvbo1qgbKaEUlhctZ3H+4qDjJK+UNNjnr7HCs1FnWLsQHh8Mb10KpR6kJ6nmJFTRWVhYSG5ubuXlsWPHsueee1Ze7tSpE4sX+/98JElS3dWxSazsbJqTwczFaznhgfGsKrDslGq7lJwc0jt1Aly+Hk+ZqZls23BbAL5c/mXAaWqBNjvFlrLvdHrs8viRcO9esHBKoLEk1R0JVXS2b9+eSZMmAbBs2TKmT5/OgAEDKm9fvHgxeXl5QcWTJElKCJ2b1ufpoTvTpH46Mxat4aQHP2N1YWnQsSRVs6ze65eveyBRXPVsEpst64FEcZJeDw65GU54Huo3h2Wz4IH9YPR/oLws6HSSarmEKjqHDBnC8OHDufbaaznqqKPo1q0bffv2rbx97Nix9OrVK8CEkiRJiWGbZjk8ecYuNKqXzrQFqzn5oc9YU2TZKdVmmRUnrzujM64qDiT6cpkzOuOqywFwzjjofhhEyuB/18IjB8GKOUEnk1SLJVTRefHFFzN06FBefPFFMjMz+e9//1vl9jFjxnDccccFlE6SJCmxdG2Rw5Nn7EyD7DS+mL+KIQ99xlrLTqnWytoudvJ60dSpHpwTRxUHEs1YPoPySHnAaWqZeo3h6Mfg8HshIxfmj4eRu8OkR8AxLKkaJFTRGQ6Hueaaa/j8889566236N69e5Xb//vf/3L66acHlE6SJCnxdG+ZyxOn70xeVhqff7+KUx+eQH6xSwOl2iija1dCaWmUr15N6fz5QcepNTrldSIrNYv80nzmrZkXdJzaJxSC7Y+Fc8ZA+92hNB9e+ws8fSysWxp0Okm1TEIVnQDPPvssJ5xwAkcddRT33HNP0HEkSZISXq/WeTxx+s7kZKYy8buVnPrIBApKLDul2iacnk7G+skghe7TGTep4VS6N4q9r+7TWY0atIMhr8GB/4CUdPj6bbh7V5j5RtDJJNUiCVV0jhw5kuOOO46JEyfyzTffMHz4cC666KKgY0mSJCW83m3yePz0ncnJSOWzuSs4/ZGJFJa4BFOqbSoPJJo2NeAktYv7dNaQcBh2+zOc+SE07wUFy+CZ4+GVEVC8Nuh0kmqBhCo677zzTq688kpmzZrFlClTePTRR7n77ruDjiVJkpQUdmjbgEdP70/9jFTGzVnO0McmUlRq2SnVJlmVBxJZyMVTxT6dFp01pHlPGPo/GPAXIASfPw4jB8B344JOJinJJVTROWfOHIYMGVJ5+fjjj6esrIxFixYFmEqSJCl59GnXkEdO7Ud2egqffLuMsx6fZNkp1SKZvdcfSDRjBtFSDx+Ll4qic9bKWZSUlwScpo5IzYADroFT3oC8drDqO3h4ELx/FZT530DS1kmoorO4uJh69epVXg6Hw6Snp1NYWBhgKkmSpOSyU4dGPHxKP7LSUvjo6x8Z9uRkisssO6XaIL1De8L16xMtKqL422+DjlNrtK7fmoYZDSmLlDFrxayg49QtHQbEDira4QQgCp/cAg/sC0u/CjqZpCSUGnSAn7viiivIzs6uvFxSUsJ1111HXl5e5XU333xzENEkSZKSxs6dGvPgKTtx2iMT+N/MpQx/8nPuPqEP6akJ9XduSVsoFA6T2bsXBeM+pXDqNDLXH06k3ycUCtGzSU8+WfAJ05ZNo3fT3kFHqlsyc2Hw3dB1UOxE9sXT4N69YP8rYedzYnt7StJmSKifFnvuuSezZs3i888/r/zYbbfdmDNnTuXlKVOmBB1TkiQpKezWuQkPnNyPjNQw73+1hD8/PZnS8kjQsST9Tlnrl68XeiBRXFUcSDR9+fSAk9Rh3Q+Fc8ZBl4FQXgzv/A0eOwxWzQ86maQkkVAzOj/88MMql5ctW0Z6ejq5ubnBBJIkSUpyu3dpwn0n78TQRyfyzvQl/OWZz7n92B1JTUmov3dL2gIVBxIVTZ0WcJLapWKfzmnLfF8DldMcjn8WJj0M71wG8z6OHVR08H+g91EQCgWdUFICS7jfcFetWsXw4cNp0qQJzZs3p2HDhrRo0YK//vWvFBQUBB1PkiQp6ey1bVPuPakvaSkh3py2mPOf+4IyZ3ZKSaviQKLib78lkp8fcJrao6LonLt6LmtL1gacpo4LhWCn0+DsT6D1TlC8Gl4cCs+fCgUrgk4nKYElVNG5YsUKdt55Zx599FH+9Kc/cdNNN3HTTTdx2GGHcccdd7DnnntSVFTEZ599xu233+lPqsgAAQAASURBVB50XEmSpKSxT7dmjDwhVna+9sVCLvzvF5RHokHHkrQV0po3I7V5c4hEKJoxI+g4tUajzEa0rt8agBnLfV8TQuPOcNo7sM9lEE6F6S/ByN3g2/eDTiYpQSVU0XnNNdeQnp7O7NmzuffeeznvvPM477zzuO+++/j2228pKSnhpJNO4oADDqhyOJEkSZJ+2/49mnPHcX1IDYd4ecpCLn5+KhHLTikpVSxfL3T5ely5fD0BpaTCXhfD6e9B4y6wdhE88Sd48yIocdWnpKoSquh8+eWX+c9//kPz5s03uq1FixbceOONvPDCC1xwwQUMGTIkgISSJEnJ7Q+9WnD7cTuSEg7xwuQf+OuL0yw7pSRUsXy98EsLuXjq1ThWdH657MuAk2gjrfvAWaOh/1mxy5/dB/fuCQsmB5tLUkJJqKJz0aJF9OzZ8xdv79WrF+FwmCuvvLIGU0mSJNUuB/VuyS3H7EA4BM9OnM9lL39p2SklGQ8kqh4VMzotOhNUejYcdCOc+CLktITl38CDB8BHN0J5WdDpJCWAhCo6mzRpwrx5837x9rlz59KsWbOaCyRJklRLHbZ9K24+egdCIXj6s++58tXpRKOWnVKyyFw/QaR0wQLKli8POE3t0aNxD8KhMEsKlrC0YGnQcfRLttkPzhkLPQ+HSBl8cB08NBCWzw46maSAJVTROXDgQC677DJKSko2uq24uJgrrriCP/zhDwEkkyRJqn0G79iafx+5PaEQPP7pd1z92gzLTilJpOTkkN6pEwCF05zVGS/Zadl0you9r87qTHDZjeDIh+GIByAjDxZMhHt2h4kPgf+/TKqzEqrovOaaa5g1axZdunThxhtv5NVXX+WVV17hhhtuoEuXLnz11VdcddVVQceUJEmqNY7s24Z/HRHb6++RsfP4xxtfWXZKSSKrt8vXq0PvJrH31aIzCYRCsN1RMGwsdNwTSgvg9fPhqaNh7ZKg00kKQEIVnW3atGHcuHH06NGDv/71rwwePJjDDz+cyy67jB49ejBmzBjatWsXdExJkqRa5eh+bbn+8Ng/7B/8ZC43vDXTslNKApkVJ687ozOu3KczCeW1gZNegYH/hJQM+OZduHsXmPFq0Mkk1bDUoAP8XMeOHXnrrbdYuXIl33zzDQDbbLMNjRo1CjiZJElS7XX8zu0oj0a54uUvuXf0HFJTQlx4YFdCoVDQ0ST9gqztYrOxi6ZOJRqN+r/XOKksOpd/6fuaTMJh2HUYdN4HXhwKi6fBcyfB9sfDoBsgMy/ohJJqQELN6NxQw4YN6d+/P/3790+6kvOuu+6iR48e9OvXL+gokiRJm+2kXdpz1aE9ALjrg9nc+v43ASeS9GsyunYllJZG+erVlM6fH3ScWqNLwy6kh9NZW7KW79d+H3Qcbalm3eGM/8HuF0AoDF88BSN3h3ljgk4mqQYkbNGZzIYPH86MGTOYMGFC0FEkSZK2yCkDOnL5wd0BuG3UN9w+yrJTSlTh9HQyusf+9+ry9fhJC6fRrXE3AKYt831NSqnpsP+VcOpb0LADrP4eHjkY3r0CyoqDTiepGll0SpIkqYoz9ujEXwfF/pF/83tfc9cH3wacSNIv8UCi6uGBRLVEu13g7E+gz8lAFMbeDvfvC4v97yrVVhadkiRJ2shZe3XmooFdAfj3O7O496PZASeStClZHkhULTyQqBbJyIHD7oBjn4bsJrDkS7h/HxhzO0TKg04nKc4sOiVJkrRJw/fZhgsO2BaAf741kwc+nhNwIkk/l1kxo3PGDKKlpQGnqT16NY4VnTNXzKQ04vtaK3Q7CIaNg20HQXkJvHcFPHoYrHIfVqk2seiUJEnSLzp3vy6cu18XAP7xxlc8MmZuwIkkbSi9QwfC9esTLSqi+Fu3mYiXdrntyEnPobi8mG9X+r7WGvWbwXFPw6G3Q1o9+O4TGDkApjwN0WjQ6STFgUWnJEmSftX5+3dh+D6dAbjqtRk8Pm5esIEkVQqFw2T2js0+LHSfzrgJh8KVszo9kKiWCYWg7xA45xNouzMUr4GXz4bnTob85UGnk/Q7WXRKkiTpV4VCIS48sCtn7dUJgCtemc5T413qJyWKrN7bAVA4bWrASWoX9+ms5Rp1ip3Kvt/fIZwKX70KI3eFb94LOpmk38GiU5IkSb8pFApx6R+6ccbuHQH420vTeG7C/IBTSYKfDiTy5PX4qiw6l1t01lrhFNjj/+CMUdC0G6xbAk8eCa9fACX5QaeTtBUsOiVJkrRZQqEQlx3cnVN26wDAJS9O5YVJPwQbShKZ62d0Fn/7LZGCgoDT1B69m8QK5NmrZlNQ6vtaq7XaAc78EHYZFrs88UG4Zw/4YWKQqSRtBYtOSZIkbbZQKMSVh/bgxF3aEY3Chc9/wcufLwg6llSnpTVvRmrz5hCJUDRjRtBxao2m2U1plt2MSDTCjOW+r7VeWhb84Z9w8iuQ2xpWzIYHD4QProfy0qDTSdpMFp2SJEnaIqFQiGsO68Vx/WNl5wXPTeG1LxYGHUuq0yqWr3sgUXxVzOqcvnx6wElUYzrtDeeMgd5HQbQcPvpXrPBc9k3QySRtBotOSZIkbbFwOMR1g3tx9E5tiEThvGen8Oa0RUHHkuqsTA8kqhYV+3R68nodk9UQ/vQA/OlByMyDhZNjS9k/ux+i0aDTSfoVFp2SJEnaKuFwiBuO2I4/9WlDeSTKuU9/zttfLg46llQneSBR9fDk9Tqu95Ew7FPotA+UFcKbF8ITf4I1/mFPSlQWnZIkSdpq4XCIG4/cjsE7tKIsEmXEU5N5b8aSoGNJdU5mz54AlC5YQNny5QGnqT16No69rwvWLWBF0YqA0ygQua3gxBdh0I2QmgmzR8HIXWH6S0Enk7QJFp2SJEn6XVLCIf5z1PYcun2s7Bz25CQ+mLk06FhSnZKSk0N6p04AFE5zVme85KTn0DGvI+CszjotHIadz4KzRkPLHaBwJfz3FHjxTChcFXA4SRuy6JQkSdLvlpoS5pajt+eg3i0oLY9y1hOT+OjrH4OOJdUpWb1dvl4dejV2+brWa9oVzngf9rwYQmGY+iyMHABzRwedTNJ6Fp2SJEmKi9SUMLcduyMDezanpCzCmY9N5JNvlgUdS6ozMitOXndGZ1y5T6eqSEmDfS+D096Bhh1hzQ/w6KHwzmVQWhR0OqnOs+iUJElS3KSlhLnjuD7s370ZxWURznhsAmNnW3ZKNSFru9jJ60VTpxL1ZOi46d0kViB/uexL31f9pG1/OPsT6HtK7PK4O+G+vWHR1CBTSXWeRackSZLiKj01zF0n9GGfrk0pKo1w+iMTGT/Hw1Gk6pbRtSuhtDTKV6+m9Icfgo5Ta3Rt1JXUcCori1eyYN2CoOMokWTUh0Nvg+OehXrN4Mev4P594ZNbIFIedDqpTrLolCRJUtxlpKYw8sS+7LltUwpLyzn1kQlMnOeJxVJ1Cqenk9G9OwCFU51VFi/pKel0bdgVgC+Xu3xdm9D1DzBsHHQ7BCKl8P5V8MjBsHJe0MmkOseiU5IkSdUiMy2F+07qy+7bNKGgpJwhD33GpO9WBh1LqtU8kKh6VO7T+aNFp35BvSZwzBPwx7shPQe+Hxc7qOjzJ8AtD6QaY9EpSZKkapOZlsL9J+/Erp0ak19SzikPfcaU+auCjiXVWlkeSFQtKorOact8X/UrQiHY8QQ45xNotxuUrINXhsOzJ0K++1VLNcGiU5IkSdUqKz2FB0/Zif4dG7G2uIyTHhzPtB9WBx1LqpUyK2Z0zphBtLQ04DS1R8WBRF+t+IqySFnAaZTwGnaAU16H/a+GcBrMfB3u3gVmvR10MqnWs+iUJElStctOT+XhU/qxU/uGrC0qY8jDn7Gu2LJAirf0Dh0I169PtKiI4m+/DTpOrdEhtwPZqdkUlhUyZ/WcoOMoGYRTYPfz4MwPoFkPyP8Rnj4GXj0XitcFnU6qtSw6JUmSVCPqZaTy8Kn9aJ6bwYr8Er5wCbsUd6FwmMzesWXWhe7TGTcp4RR6NukJwPRl0wNOo6TSojcM/QB2HQGEYPKjcM/uMP+zoJNJtZJFpyRJkmpMTmYaO7ZtCMCMhWsCTiPVTlm9twOgcJonr8eT+3Rqq6VlwsDrYMirkNsGVs6FhwbC//4B5W4xIcWTRackSZJqVM9WuQBMX+g+nVJ1qDiQyJPX46tX4/Unry/z5HVtpY57wrCxsN2xEI3A6H/DA/vBj7OCTibVGhadkiRJqlE9W1cUnc7olKpD5voZncXffkukoCDgNLVHxYFE36z8hqKyooDTKGll5sER98JRj0BWQ1j0Bdy7J3x6D0QiQaeTkp5FpyRJkmpUz1Z5AMz+cR2FJeUBp5Fqn7TmzUht3hwiEYpmzAg6Tq3Rol4LGmc2pixaxswVM4OOo2TX83A4Zxx03g/KiuDtS+CJw2H1gqCTSUnNolOSJEk1qllOBk3qpxOJwszFzuqUqkPF8nUPJIqfUChUuU+ny9cVF7kt4cQX4OCbIDUL5nwII3eFac8HnUxKWhadkiRJqlGhUIge62d1unxdqh6ZHkhULSqLzuUWnYqTUAj6nQFnfwyt+kDRanjhdHj+dChcGXQ6KelYdEqSJKnG/XQgkUWnVB2yescKOQ8kiq+KfTqd0am4a9IFTn8X9roUQinw5fNw924w+4Ogk0lJxaJTkiRJNa5Hy1jROcOT16VqkdkrVnSWLlhA2fLlAaepPXo27gnAd2u+Y3WxP78UZylpsM9f4fT3oFFnWLsQHh8Mb10KpYVBp5OSgkWnJEmSalzFjM6Zi9dSVu4ps1K8peTkkN6pEwCF05zVGS8NMhvQNqctANOXTw84jWqtNn1jS9n7nRG7PH4k3LsXLJwSaCwpGVh0SpIkqcZ1aFyPeukpFJdFmLMsP+g4Uq2U1Tu2zNrl6/HlgUSqEen1YocUnfA81G8Oy2bBA/vB6P9AeVnQ6aSEZdEpSZKkGhcOh+jesmKfTpd/StUhs+LkdWd0xlWvxrGic9oy31fVgC4HwDnjoPthECmD/10LjxwEK+YEnUxKSBadkiRJCkTlgUQLPJBIqg5Z28VOXi+aNo1oNBpwmtqjd9OfDiTyfVWNqNcYjn4MDr8XMnJh/ngYuTtMegQcg1IVFp2SJEkKRM9WeYAnr0vVJaNrV0JpaZSvWkXZDwuCjlNrdGvUjZRQCssKl7GkYEnQcVRXhEKw/bFwzhhovzuU5sNrf4Gnj4V1S4NOJyUMi05JkiQFokern5auOytKir9wejoZ3bsDUPSly6zjJSs1i20abAO4T6cC0KAdDHkNDvwHpKTD12/D3bvCzDeCTiYlBItOSZIkBWLb5jmkpYRYU1TGDysLg44j1UqVBxJNs5CLJw8kUqDCYdjtz3Dmh9C8FxQsg2eOh1dGQPHaoNNJgbLolCRJUiDSU8N0aZYDuHxdqi5Z6w8kKv7SQi6eejf5aZ9OKTDNe8LQ/8GAvwAh+PxxGDkAvhsXdDIpMBadkiRJCkzF8vUZnrwuVYvM9TM6i7/6CsrLA05Te1TM6Jy+fDqRaCTgNKrTUjPggGvglDcgrx2s+g4eHgTvXwVlJUGnk2qcRackSZICU3nyujM6pWqR3qED4fr1iRYVkbHEg3PipXODzmSmZLKudB3z1swLOo4EHQbEDira4QQgCp/cAg/sC0u/CjqZVKMsOiVJkhSYipPXZyyy6JSqQygcJrN3bPZh5vz5AaepPVLDqfRo3ANw+boSSGYuDL4bjnkCshvD4mlw714w7i6IOPNYdYNFpyRJkgLTvWVsj85Fq4tYke8SO6k6ZPXeDrDojLeeTXoCFp1KQN0PhXPGQZeBUF4M7/wNHv8jrPJngGo/i05JkiQFJiczjQ6NswGY7j6dUrWoOJAo84cfAk5Su3ggkRJaTnM4/lk45FZIy4a5o2MHFU19DqLRoNNJ1caiU5IkSYGqWL7uPp1S9chcP6MzffESIgUFAaepPXo1jm0JMHPFTErLSwNOI21CKAQ7nQpnfwJt+kHxanhxKDx/KhSsCDqdVC0sOiVJkhSoHh5IJFWrtObNSGnWjFA0Gjt9XXHRJqcNeRl5lEZK+Xrl10HHkX5Z485w6tuwz+UQToXpL8HI3eDb94NOJsWdRackSZIC9dPJ6y5dl6pLZu/YMuuiL11mHS+hUIheTWKzOqctmxZwGuk3pKTCXhfB6e9B4y6wdhE88Sd48yIocaa3ag+LTkmSJAWqYun63GX55BeXBZxGqp0ye8UKueJpFp3xVLF83aJTSaN1HzhrNPQ/K3b5s/vg3j1hweRgc0lxYtEpSZKkQDXNyaBpTgbRKMxc7PJ1qTpk9IqdEF70pYVcPFUcSDR92fSAk0hbID0bDroRTnwRclrC8m/gwQPgoxuh3D84KrlZdEqSJClwPd2nU6pWGT1jRWfZgoWULV8ecJrao2eT2Ps6Z/Uc1pWsCziNtIW22Q/OGQs9D4dIGXxwHTw0EJbPDjqZtNUsOiVJkhS4iqJzhkWnVC1ScnIobtoUgMJpzuqMlyZZTWhZryVRosxYPiPoONKWy24ERz4MRzwAGXmwYCLcsztMfAii0aDTSVvMolOSJEmBq9in0xmdUvUpats29nmqRWc8VRxI9OVy9z9VkgqFYLujYNhY6LgnlBbA6+fDU0fD2iVBp5O2iEWnJEmSAlcxo3PW4rWUlkcCTiPVThVFZ6H7dMZVxT6dXy6z6FSSy2sDJ70CA/8JKRnwzbtw9y4w49Wgk0mbzaJTkiRJgWvbMJucjFRKyiN8u9R97qTqUNS2Tezz1GlEXZIaNxUzOj15XbVCOAy7DoOzPoIWvaFwBTx3Erx0DhStDjqd9JssOiVJkhS4cDhEdw8kkqpVScuWkJZG+apVlP7wQ9Bxao0ejXsQIsTi/MUsK1wWdBwpPpp1hzP+B7tfAKEwfPEUjNwd5o0JOpn0qyw6JUmSlBB+OnndGSNSdYimppLRrSsAhVOnBpym9qiXVo/ODToDLl9XLZOaDvtfCae+BQ07wOrv4ZGD4d0roKw46HTSJll0SpIkKSF4IJFU/TJ7xfaT9ECi+OrZuCfg8nXVUu12gbM/gT4nA1EYezvcvy8stthX4rHolCRJUkLo0TI2o/OrhWuIRNw/UKoOGb1ihVzhNAu5eKo4kGj6sukBJ5GqSUYOHHYHHPs0ZDeBJV/C/fvAmNshUh50OqmSRackSZISQpfm9UlPCbO2uIwfVhYGHUeqlSpndM6YQbS0NOA0tUevpj8dSORBT6rVuh0Ewz6FrgdBeQm8dwU8ehis+j7oZBJg0SlJkqQEkZYSZtsW9QH36ZSqS1qH9oTr1ydaVETxt98GHafW2LbBtqSF01hTsob5a+cHHUeqXvWbwrFPxWZ4ptWD7z6BkQNgytNg0a+AWXRKkiQpYfRs6T6dUnUKhcNk9o7NPix0n864SUtJo3uj7oAHEqmOCIVie3ae8wm03RmK18DLZ8NzJ0P+8qDTqQ6z6JQkSVLC6Nnak9el6pbVezsAir606IynXk1+Wr4u1RmNOsVOZd/v7xBOha9ehZG7wjfvBZ1MdZRFpyRJkhJGz1YVRaczOqXqkrVdbJ9OZ3TGV0XROX25BxKpjgmnwB7/B0P/B027wbol8OSR8PoFUJIfdDrVMRadkiRJShjdWuQSCsHStcX8uLY46DhSrZS5fkZn8TffECkoCDhN7VFRdH61/CtKIx70pDqo5fZw5oewy7DY5YkPwj17wA8TA42lusWiU5IkSQmjXkYqHZvUA1y+LlWXtObNSG3eHCIRimbMCDpOrdE+tz05aTkUlRcxe9XsoONIwUjLgj/8E05+BXJbw4rZ8OCB8MH1UO4fAFT9LDolSZKUUHq28kAiqbq5fD3+wqEwPZr0ADyQSKLT3nDOWOh9NETL4aN/xQrPZd8EnUy1nEWnJEmSEkqPlrF9OmdYdErVpmL5euG0qQEnqV16N4kVyBadEpDVAP50Pxz5EGTmwcLJsaXsn90P0WjQ6VRLWXRKkiQpoVQcSDRjkUWnVF2yesf2kyxyRmdc9WrsyevSRnr9CYZ9Cp32gbJCePNCeOJPsGZR0MlUC1l0SpIkKaFUFJ1zl+Wzrrgs4DRS7ZTZK1bIlS5YQNny5QGnqT0qDiSavWo2BaUe9CRVym0FJ74Ig26E1EyYPQpG7grTXwo6mWoZi05JkiQllMb1M2iRmwnAV87qlKpFSk4O6Z06AVA4zdmH8dK8XnOaZTWjPFrOzBUzg44jJZZwGHY+C876GFruAIUr4b+nwItnQpEHECo+LDolSZKUcCpmdU5f4D98pOqS1Tu2n2TRNPeTjKeeTXoCLl+XflHTbeGM92HPiyEUhqnPknr/njRZOyPoZKoFLDolSZKUcCqLTg8kkqpNZsXJ6x5IFFcVBxJNXzY94CRSAktJg30vg9PegUadCK1ZwIBvbyD8/hVQWhR0OiUxi85qcNddd9GjRw/69esXdBRJkqSk1KNVHmDRKVWnrO1iJ68XTZ1G1BOQ46Zin05ndEqboW1/OOtjynccAkDK+JFw396wyD/AaOtYdFaD4cOHM2PGDCZMmBB0FEmSpKRUMaPzm6VrKSmLBJxGqp0yunYllJZG+apVlP7wQ9Bxao2Kpes/rPuBlUUrA04jJYGM+kQOuolPO11AtF4z+PEruH9f+OQWiJQHnU5JxqJTkiRJCadNwyxyM1MpLY/y9ZK1QceRaqVwejoZ3bsDUDjV2VPxkpueS4fcDgBMX+7ydWlzLcnbgbKho6HbIRAphfevgkcOhpXzgo6mJGLRKUmSpIQTCoXosX5W5wyXr0vVpvJAoqkus44nl69LW6leEzjmCfjj3ZCeA9+Pg5ED4PMnwC02tBksOiVJkpSQeq7fp3PGIotOqbpk9o4VcoXTLOTiqaLo/HKZJ9pLWywUgh1PgHM+gXa7Qck6eGU4PHsi5C8LOp0SnEWnJEmSEtJPJ6+vDjiJVHtVHkg0YwbR0tKA09QeGxadHvQkbaWGHeCU12H/qyGcBjNfh7t3gVlvB51MCcyiU5IkSQmpckbnwjVEIhYFUnVI79CBcP36RIuKKP7226Dj1BrdGnUjNZTKiqIVLMpfFHQcKXmFU2D38+DMD6BZD8j/EZ4+Bl49F4rXBZ1OCciiU5IkSQmpc9N6ZKSGyS8p57sVBUHHkWqlUDjs8vVqkJGSQZeGXQD36ZTiokVvGPoB7PZnIASTH4V7dof5nwWdTAnGolOSJEkJKTUlTLcWOYDL16XqlNV7/fJ1i8646t0kdtDT9GWevC7FRVomHPgPGPIa5LWFlXPhoYHwv39AuVtvKMaiU5IkSQmrx/rl69M9eV2qNlnbxQq5Qk9ejytPXpeqScc94JwxsN2xEI3A6H/DA/vBj7OCTqYEYNEpSZKkhPXTgUQWnVJ1yey9HfX32Yfcgw7y4Jw4qig6ZyyfQXmkPOA0Ui2TmQdH3AtHPQpZDWHRF3DvnvDpPRCJBJ1OAbLolCRJUsKqKDpnLFxtASNVk7TmzWg78m6anHUmoVAo6Di1Rqe8TmSlZlFQVsDc1XODjiPVTj0Hw7BPYZv9oawI3r4EnjgcVi8IOpkCYtEpSZKkhNWtRS7hECxbV8LStcVBx5GkzZYSTqFn456Ay9elapXTAk54Hg6+CVKzYM6HMHJXmPZ80MkUAItOSZIkJays9BQ6Na0PwAyXr0tKMqf2OpWb976ZPdvsGXQUqXYLhaDfGXD2x9CqDxSthhdOh+dPh8KVQadTDbLolCRJUkL7aZ9OT16XlFz2bLMnB7Q/gMZZjYOOItUNTbrA6e/C3n+FUAp8+TzcvRvM/iDoZKohFp2SJElKaB5IJEmSNltKGux9KZz+HjTqDGsXwuOD4a1LobQw6HSqZhadkiRJSmg9W+UBFp2SJGkLtOkbW8re74zY5fEj4d69YOGUQGOpell0SpIkKaFVzOj8fkUBa4pKA04jSZKSRnq92CFFJzwP9ZvDslnwwH4w+j9QXhZ0OlUDi05JkiQltAbZ6bRukAV4IJEkSdoKXQ6AYZ9C98MgUgb/uxYeOQhWzAk6meLMolOSJEkJr4f7dEqSpN8juxEc/Rgcfi9k5ML88TByd5j0CESjQadTnFh0SpIkKeH1aOnJ65Ik6XcKhWD7Y+GcMdBhDyjNh9f+Ak8fC+uWBp1OcWDRKUmSpIRXsU+nS9clSdLv1qAdnPwqHHgdpKTD12/D3bvCzDeCTqbfyaJTkiRJCa9n69jJ698uXUdxWXnAaSRJUtILh2G3EXDmh9C8FxQsg2eOh1dGQPHaoNNpK1l0SpIkKeG1ysukQXYaZZEoXy9eF3QcSZJUWzTvCUP/BwP+AoTg88dh5AD4blzQybQVLDolSZKU8EKhUOXydffplCRJcZWaAQdcA6e8EVvWvuo7eHgQvH8VlJUEnU5bwKJTkiRJSaFnq9jydU9elyRJ1aLDADh7DOxwIhCFT26BB/aFpV8FnUybyaJTkiRJScEZnZIkqdpl5sLgu+CYJyC7MSyeBvfuBePugkgk6HT6DRadkiRJSgoVRedXi9ZSHokGnEaSJNVq3Q+Fc8ZBl4FQXgzv/A0e/yOsmh90Mv0Ki05JkiQlhY5N6pOZFqawtJy5y/KDjiNJkmq7nOZw/LNwyK2Qlg1zR8cOKpr6HET9o2sisuiUJElSUkgJh+jWwuXrkiSpBoVCsNOpcPYn0KYfFK+GF4fC86dCwYqg0+lnLDolSZKUNCqWr89Y5IFEkiSpBjXuDKe+DftcDuFUmP4SjNwNvn0/6GTagEWnJEmSkkbFyeszPHldkiTVtJRU2OsiOP09aLItrF0ET/wJ3rwISgqCTicsOiVJkpREfjp5fQ1R98aSJElBaN0HzvwI+p8Vu/zZffDKsGAzCbDolCRJUhLp2iKHlHCIFfklLF5TFHQcSZJUV6Vnw0E3wrFPxy7PfBNKC4PNJItOSZIkJY/MtBS2aVofgOkLXL4uSZIC1nUQ5LSC8mKYPz7oNHWeRackSZKSyobL1yVJkgIVCkHHPWNfzx0dbBZZdEqSJCm59KgsOlcHnESSJAnotFfs85yPgs0hi05JkiQll4qT153RKUmSEkLFjM6Fk6HIP8QGyaJTkiRJSaVHy9iMzgWrCllVUBJwGkmSVOfltYFGnSEage/GBZ2mTrPolCRJUlLJy06jTcMsAGYsclanJElKAJX7dLp8PUgWnZIkSUo6FQcSzXD5uiRJSgQeSJQQLDolSZKUdNynU5IkJZSKonPJl7Dux2Cz1GEWnZIkSUo6PT15XZIkJZJ6TaB5r9jX8z4ONksdZtEpSZKkpFMxo3P2j/kUlZYHnEaSJAnouFfss8vXA2PRKUmSpKTTPDeDxvXSKY9Embl4bdBxJEmSPJAoAVh0SpIkKemEQiF6uHxdkiQlkva7QSgFVsyBVfODTlMnWXRKkiQpKXkgkSRJSiiZudC6T+xrl68HwqJTkiRJSemnGZ0WnZIkKUFULl+36AyCRackSZKSUsXJ6zMXraGsPBJwGkmSJDY4kOgjiEaDzVIHWXRKkiQpKXVsXI/s9BSKyyLMXZYfdBxJkiRo2x9SMmDtIlj+bdBp6hyLTkmSJCWlcDhE95YuX5ckSQkkLQva7Rz72tPXa5xFpyRJkpJWT09elyRJiaZin845Fp01zaJTkiRJSaunBxJJkqREU7FP57yPIeI+4jXJolOSJElJq2erPCBWdEbd8F+SJCWCVn0gPQcKV8KSaUGnqVMsOiVJkpS0ujSvT2o4xOrCUhasKgw6jiRJEqSkQvvdYl/PHR1sljrGolOSJElJKyM1hS7NcwCXr0uSpATSaf3ydYvOGmXRKUmSpKTWY/3J6zMsOiVJUqKoOJDou7FQXhpsljrEolOSJElJzQOJJElSwmnWE7IbQ8k6WDA56DR1hkWnJEmSklpF0Tlj4eqAk0iSJK0XDkOHPWJfz/0o2Cx1iEWnJEmSklqP9UXnwtVFrMwvCTiNJEnSehXL192ns8ZYdEqSJCmp5WSm0b5xNuDydUmSlEA67R37PH88lBQEGqWusOiUJElS0vtpn06Xr0uSpATRqBPktobykljZqWpn0SlJkqSk17NVHuCMTkmSlEBCIZev1zCLTkmSJCW9Hs7olCRJiajjXrHPHkhUIyw6JUmSlPQqlq7PWZZPQUlZwGkkSZLWq5jRufBzKPIPstXNolOSJElJr1lOJk3qZxCNwszFa4OOI0mSFJPXGhpvA9EIzBsTdJpaz6JTkiRJtcJPBxK5T6ckSUog7tNZYyw6JUmSVCtUFJ0z3KdTkiQlksp9Oi06q5tFpyRJkmoFT16XJEkJqcMesc9Lp8O6H4PNUstZdEqSJKlWqJjROXPxWkrLIwGnkSRJWq9eY2jeO/b1PGd1VieLTkmSJNUK7RplUz8jlZKyCLN/XBd0HEmSpJ90Wr98fc5Hweao5Sw6JUmSVCuEwyF6tFx/INECl69LkqQE4oFENcKiU5IkSbVGD09elyRJiaj9bhBKgZVzYdX3QaeptSw6JUmSVGv0rCw6PXldkiQlkIwcaN039rWzOquNRackSZJqjYoZnXOX5RONRgNOI0mStAGXr1c7i05JkiTVGts2z+Hd8/dk7KX7EgqFgo4jSZL0kw0PJPIPstXColOSJEm1RlpKmG2b55Ca4q+5kiQpwbTpD6mZsG4xLPsm6DS1kr8BSpIkSZIkSdUtLRPa7hz7eu5HwWappSw6JUmSJEmSpJpQuU+nRWd1sOiUJEmSJEmSakKnvWOf534MkUigUWoji05JkiRJkiSpJrTcAdJzoGgVLJ4adJpax6JTkiRJkiRJqgkpqdBhQOzruaODzVILWXRKkiRJkiRJNaXjXrHP7tMZdxadkiRJkiRJUk2pOJDou3FQVhJsllrGolOSJEmSJEmqKc16QHYTKM2HBZOCTlOrWHRKkiRJkiRJNSUcho57xL52n864suiUJEmSJEmSalLF8nWLzriy6JQkSZIkSZJqUsWBRD98BiUFwWapRSw6JUmSJEmSpJrUqBPktoHyEpj/adBpag2LTkmSJEmSJKkmhULQaf2szjkfBZulFrHolCRJkiRJkmqa+3TGnUWnJEmSJEmSVNMqis5FU6BwVZBJag2LTkmSJEmSJKmm5baCxl0gGoHvxgSdplaw6JQkSZIkSZKC4PL1uLLolCRJkiRJkoLggURxZdEpSZIkSZIkBaHDHkAIfvwK1i0NOk3Ss+iUJEmSJEmSgpDdCFr0jn3t8vXfzaJTkiRJkiRJCkrlPp0uX/+9LDolSZIkSZKkoHRcv0+nMzp/N4tOSZIkSZIkKSjtd4VwKqycByu/CzpNUrPolCRJkiRJkoKSkQOt+8a+dlbn72LRKUmSJEmSJAWpcvm6+3T+HhadkiRJkiRJUpAqDyQaDdFosFmSmEWnJEmSJEmSFKQ2/SA1E9YtgWVfB50maVl0SpIkSZIkSUFKy4R2u8S+nuPy9a1l0SlJkiRJkiQFrXL5ukXn1rLolCRJkiRJkoLWce/Y53kfQ6Q8yCRJy6JTkiRJkiRJClrL7SEjF4pWw+KpQadJShadkiRJkiRJUtBSUqH9gNjX7tO5VSw6JUmSJEmSpETQaa/Y57mjg82RpCw6JUmSJEmSpERQcSDR9+OgrCTYLEnIolOSJEmSJElKBM16QHYTKC2ABRODTpN0LDolSZIkSZKkRBAK/TSr0+XrW8yiU5IkSZIkSUoUFft0eiDRFrPolCRJkiRJkhJFxYzOHyZASX6wWZKMRackSZIkSZKUKBp2hLy2ECmF7z8NOk1SseiUJEmSJEmSEkUoBB3XL1+f6/L1LWHRKUmSJEmSJCUSDyTaKhadkiRJkiRJUiKpKDoXToHClYFGSSYWnZIkSZIkSVIiyW0JTbYFojBvTNBpkoZFpyRJkiRJkpRoKpevu0/n5rLolCRJkiRJkhJN5YFE7tO5uSw6JUmSJEmSpETTYXcgBD/OhLVLgk6TFCw6f0OHDh3Ybrvt2GGHHdhnn32CjiNJkiRJkqS6ILsRtNwu9rWzOjdLatABksHYsWOpX79+0DEkSZIkSZJUl3TcExZ9Edunc7ujgk6T8JzRKUmSJEmSJCWijnvHPnsg0Wap1UXn6NGjOfTQQ2nVqhWhUIiXX355o/vcdddddOjQgczMTHbeeWc+++yzKreHQiH22msv+vXrx5NPPllDySVJkiRJklTntdsFwqmw6ntYOS/oNAmvVhed+fn5bL/99tx1112bvP3ZZ5/lggsu4Morr2Ty5Mlsv/32DBw4kKVLl1be55NPPmHSpEm8+uqrXH/99UydOrWm4kuSJEmSJKkuy6gPrXeKfe0+nb+pVu/ROWjQIAYNGvSLt998880MHTqUU089FYB77rmHN954g4ceeohLL70UgNatWwPQsmVLDjroICZPnsx22223yecrLi6muLi48vKaNWsAKC0tpbS0NC6vKVlVvP66/j4oPhxPiifHk+LJ8aR4cjwpnhxPiifHk+LNMfXrwu13J2X+p0Rmf0B57+OCjlPjtmRchKLRaLQasySMUCjESy+9xODBgwEoKSkhOzub559/vvI6gCFDhrBq1SpeeeUV8vPziUQi5OTksG7dOvbaay/uuece+vXrt8nvcdVVV3H11VdvdP1TTz1FdnZ2dbwsSZIkSZIk1WKN185k92+vpyg1j3d63Q6hUNCRalRBQQHHH388q1evJjc391fvW6tndP6aZcuWUV5eTvPmzatc37x5c2bOnAnAkiVLOPzwwwEoLy9n6NChv1hyAvz1r3/lggsuqLy8Zs0a2rZty4EHHvib/yFqu9LSUt577z0OOOAA0tLSgo6jJOd4Ujw5nhRPjifFk+NJ8eR4Ujw5nhRvjqnfULYf0ZtuIbNsNQf17wxNuwWdqEZVrJjeHHW26NwcnTp14osvvtjs+2dkZJCRkbHR9Wlpaf4PdT3fC8WT40nx5HhSPDmeFE+OJ8WT40nx5HhSvDmmfkFaWuxQojkfkDZ/LLTqHXSiGrUlY6JWH0b0a5o0aUJKSgpLliypcv2SJUto0aJFQKkkSZIkSZKkn+m4Z+yzBxL9qjpbdKanp9O3b19GjRpVeV0kEmHUqFHsuuuuASaTJEmSJEmSNtBpr9jneR9DpDzYLAmsVi9dX7duHd9++23l5blz5zJlyhQaNWpEu3btuOCCCxgyZAg77bQT/fv359ZbbyU/P7/yFHZJkiRJkiQpcC13gIw8KFoNi76A1n2CTpSQanXROXHiRPbZZ5/KyxUHBQ0ZMoRHHnmEY445hh9//JG///3vLF68mB122IG33357owOKJEmSJEmSpMCEU6DD7jDrDZj7kUXnL6jVRefee+9NNBr91fuMGDGCESNG1FAiSZIkSZIkaSt03HN90Tkadj8/6DQJqc7u0SlJkiRJkiQljYoDib4bB2XFwWZJUBadkiRJkiRJUqJr1h3qNYWyQvhhYtBpEpJFpyRJkiRJkpToQqGfZnXOHR1slgRl0SlJkiRJkiQlg457xT7P/SjYHAnKolOSJEmSJElKBhUzOn+YACX5wWZJQBadkiRJkiRJUjJo1BEatINIWexQIlVh0SlJkiRJkiQli8p9Ol2+/nMWnZIkSZIkSVKyqNyn0wOJfs6iU5IkSZIkSUoWFTM6F30BBSuCzZJgLDolSZIkSZKkZJHTApp0BaLw3Zig0yQUi05JkiRJkiQpmXRav3x9jvt0bsiiU5IkSZIkSUomlQcSuU/nhiw6q8Fdd91Fjx496NevX9BRJEmSJEmSVNu0HwCEYNksWLMo6DQJw6KzGgwfPpwZM2YwYcKEoKNIkiRJkiSptsluBC23j3097+NgsyQQi05JkiRJkiQp2VQuX3efzgoWnZIkSZIkSVKyqTyQaDREo8FmSRAWnZIkSZIkSVKyabcrhFNh9fewcl7QaRKCRackSZIkSZKUbNLrQZv1B2G7fB2w6JQkSZIkSZKSU8f1y9fnjg42R4Kw6JQkSZIkSZKSUeWBRO7TCRadkiRJkiRJUnJq0w/qNYNWfaBoddBpApcadABJkiRJkiRJWyE1Hf5vFoSdywjO6JQkSZIkSZKSlyVnJd8JSZIkSZIkSUnPolOSJEmSJElS0rPolCRJkiRJkpT0LDolSZIkSZIkJT2LTkmSJEmSJElJz6JTkiRJkiRJUtKz6JQkSZIkSZKU9Cw6JUmSJEmSJCU9i05JkiRJkiRJSc+iU5IkSZIkSVLSs+iUJEmSJEmSlPQsOqvBXXfdRY8ePejXr1/QUSRJkiRJkqQ6waKzGgwfPpwZM2YwYcKEoKNIkiRJkiRJdYJFpyRJkiRJkqSkZ9EpSZIkSZIkKelZdEqSJEmSJElKehadkiRJkiRJkpKeRackSZIkSZKkpGfRKUmSJEmSJCnpWXRKkiRJkiRJSnqpQQeozaLRKABr1qwJOEnwSktLKSgoYM2aNaSlpQUdR0nO8aR4cjwpnhxPiifHk+LJ8aR4cjwp3hxT+jUVvVpFz/ZrLDqr0dq1awFo27ZtwEkkSZIkSZKk5LV27Vry8vJ+9T6h6ObUodoqkUiEhQsXkpOTQygUCjpOoNasWUPbtm2ZP38+ubm5QcdRknM8KZ4cT4onx5PiyfGkeHI8KZ4cT4o3x5R+TTQaZe3atbRq1Ypw+Nd34XRGZzUKh8O0adMm6BgJJTc31x9aihvHk+LJ8aR4cjwpnhxPiifHk+LJ8aR4c0zpl/zWTM4KHkYkSZIkSZIkKelZdEqSJEmSJElKehadqhEZGRlceeWVZGRkBB1FtYDjSfHkeFI8OZ4UT44nxZPjSfHkeFK8OaYULx5GJEmSJEmSJCnpOaNTkiRJkiRJUtKz6JQkSZIkSZKU9Cw6JUmSJEmSJCU9i05JkiRJkiRJSc+iU1vlrrvuokOHDmRmZrLzzjvz2Wef/er9b731Vrp27UpWVhZt27bl/PPPp6io6Hc9p2qPeI+nf/7zn/Tr14+cnByaNWvG4MGDmTVrVnW/DCWI6vj5VOGGG24gFApx3nnnVUNyJarqGFMLFizgxBNPpHHjxmRlZdG7d28mTpxYnS9DCSLe46m8vJwrrriCjh07kpWVRefOnbn22mvxvNG6YUvGU2lpKddccw2dO3cmMzOT7bffnrfffvt3Padql3iPJ38nr9uq4+dTBX8n16+KSlvomWeeiaanp0cfeuih6PTp06NDhw6NNmjQILpkyZJN3v/JJ5+MZmRkRJ988sno3Llzo++88060ZcuW0fPPP3+rn1O1R3WMp4EDB0Yffvjh6JdffhmdMmVK9KCDDoq2a9cuum7dupp6WQpIdYynCp999lm0Q4cO0e222y76l7/8pZpfiRJFdYypFStWRNu3bx895ZRTouPHj4/OmTMn+s4770S//fbbmnpZCkh1jKfrrrsu2rhx4+jrr78enTt3bvS///1vtH79+tHbbrutpl6WArKl4+niiy+OtmrVKvrGG29EZ8+eHb377rujmZmZ0cmTJ2/1c6r2qI7x5O/kdVd1jKcK/k6u32LRqS3Wv3//6PDhwysvl5eXR1u1ahX95z//ucn7Dx8+PLrvvvtWue6CCy6IDhgwYKufU7VHdYynn1u6dGkUiH700UfxCa2EVV3jae3atdEuXbpE33vvvehee+3lL1V1SHWMqUsuuSS6++67V09gJbTqGE8HH3xw9LTTTqtynyOOOCJ6wgknxDG5EtGWjqeWLVtG77zzzirX/Xys+Dt53VUd4+nn/J287qiu8eTv5NocLl3XFikpKWHSpEnsv//+ldeFw2H2339/xo0bt8nH7LbbbkyaNKlyqvqcOXN48803Oeigg7b6OVU7VMd42pTVq1cD0KhRozimV6KpzvE0fPhwDj744CrPrdqvusbUq6++yk477cRRRx1Fs2bN2HHHHbn//vur98UocNU1nnbbbTdGjRrF119/DcAXX3zBJ598wqBBg6rx1ShoWzOeiouLyczMrHJdVlYWn3zyyVY/p2qH6hhPm+Lv5HVDdY4nfyfX5kgNOoCSy7JlyygvL6d58+ZVrm/evDkzZ87c5GOOP/54li1bxu677040GqWsrIyzzz6bv/3tb1v9nKodqmM8/VwkEuG8885jwIAB9OrVK+6vQYmjusbTM888w+TJk5kwYUK15lfiqa4xNWfOHEaOHMkFF1zA3/72NyZMmMC5555Leno6Q4YMqdbXpOBU13i69NJLWbNmDd26dSMlJYXy8nKuu+46TjjhhGp9PQrW1oyngQMHcvPNN7PnnnvSuXNnRo0axYsvvkh5eflWP6dqh+oYTz/n7+R1R3WNJ38n1+ZyRqeq3Ycffsj111/P3XffzeTJk3nxxRd54403uPbaa4OOpiS0peNp+PDhfPnllzzzzDM1nFTJ4LfG0/z58/nLX/7Ck08+udFfmaVN2ZyfUZFIhD59+nD99dez4447cuaZZzJ06FDuueeeAJMrEW3OeHruued48skneeqpp5g8eTKPPvoo//nPf3j00UcDTK5EdNttt9GlSxe6detGeno6I0aM4NRTTyUc9p+E2nJbOp78nVy/5rfGk7+Ta0s4o1NbpEmTJqSkpLBkyZIq1y9ZsoQWLVps8jFXXHEFJ510EmeccQYAvXv3Jj8/nzPPPJPLLrtsq55TtUN1jKcNf7kaMWIEr7/+OqNHj6ZNmzbV90KUEKpjPE2aNImlS5fSp0+fyseUl5czevRo7rzzToqLi0lJSam+F6VAVdfPqJYtW9KjR48qj+vevTsvvPBC9bwQJYTqGk8XXXQRl156Kccee2zlfb777jv++c9/OkO4Ftua8dS0aVNefvllioqKWL58Oa1ateLSSy+lU6dOW/2cqh2qYzxtyN/J65bqGE/+Tq4t4Z/vtEXS09Pp27cvo0aNqrwuEokwatQodt11100+pqCgYKO/7FX8EIpGo1v1nKodqmM8VXweMWIEL730Ev/73//o2LFjNb0CJZLqGE/77bcf06ZNY8qUKZUfO+20EyeccAJTpkzxF6parrp+Rg0YMIBZs2ZVuc/XX39N+/bt4xlfCaa6xtMv3ScSicQzvhLM7/n9OTMzk9atW1NWVsYLL7zAH//4x9/9nEpu1TGewN/J66rqGE/+Tq4tEswZSEpmzzzzTDQjIyP6yCOPRGfMmBE988wzow0aNIguXrw4Go1GoyeddFL00ksvrbz/lVdeGc3JyYk+/fTT0Tlz5kTffffdaOfOnaNHH330Zj+naq/qGE/nnHNONC8vL/rhhx9GFy1aVPlRUFBQ469PNas6xtPPecJj3VIdY+qzzz6LpqamRq+77rroN998E33yySej2dnZ0SeeeKLGX59qVnWMpyFDhkRbt24dff3116Nz586Nvvjii9EmTZpEL7744hp/fapZWzqePv300+gLL7wQnT17dnT06NHRfffdN9qxY8foypUrN/s5VXtVx3jyd/K6qzrG08/5O7l+iUWntsodd9wRbdeuXTQ9PT3av3//6Kefflp521577RUdMmRI5eXS0tLoVVddFe3cuXM0MzMz2rZt2+iwYcM2+qH1a8+p2i3e4wnY5MfDDz9ccy9KgamOn08b8pequqc6xtRrr70W7dWrVzQjIyParVu36H333VdDr0ZBi/d4WrNmTfQvf/lLtF27dtHMzMxop06dopdddlm0uLi4Bl+VgrIl4+nDDz+Mdu/ePZqRkRFt3Lhx9KSTToouWLBgi55TtVu8x5O/k9dt1fHzaUP+Tq5fEopG1697kSRJkiRJkqQk5R6dkiRJkiRJkpKeRackSZIkSZKkpGfRKUmSJEmSJCnpWXRKkiRJkiRJSnoWnZIkSZIkSZKSnkWnJEmSJEmSpKRn0SlJkiRJkiQp6Vl0SpIkSZIkSUp6Fp2SJEnSFrjqqqvYYYcdKi+fcsopDB48OLA8kiRJirHolCRJkiRJkpT0LDolSZJUa5SUlAQdQZIkSQGx6JQkSVLS2nvvvRkxYgTnnXceTZo0YeDAgXz55ZcMGjSI+vXr07x5c0466SSWLVtW+ZhIJMKNN97INttsQ0ZGBu3ateO6666rvP2SSy5h2223JTs7m06dOnHFFVdQWloaxMuTJEnSFrDolCRJUlJ79NFHSU9PZ8yYMdxwww3su+++7LjjjkycOJG3336bJUuWcPTRR1fe/69//Ss33HADV1xxBTNmzOCpp56iefPmlbfn5OTwyCOPMGPGDG677Tbuv/9+brnlliBemiRJkrZAKBqNRoMOIUmSJG2NvffemzVr1jB58mQA/vGPf/Dxxx/zzjvvVN7nhx9+oG3btsyaNYuWLVvStGlT7rzzTs4444zN+h7/+c9/eOaZZ5g4cSIQO4zo5ZdfZsqUKUDsMKJVq1bx8ssvx/W1SZIkacukBh1AkiRJ+j369u1b+fUXX3zBBx98QP369Te63+zZs1m1ahXFxcXst99+v/h8zz77LLfffjuzZ89m3bp1lJWVkZubWy3ZJUmSFD8WnZIkSUpq9erVq/x63bp1HHroofzrX//a6H4tW7Zkzpw5v/pc48aN44QTTuDqq69m4MCB5OXl8cwzz3DTTTfFPbckSZLiy6JTkiRJtUafPn144YUX6NChA6mpG/+q26VLF7Kyshg1atQml66PHTuW9u3bc9lll1Ve991331VrZkmSJMWHhxFJkiSp1hg+fDgrVqzguOOOY8KECcyePZt33nmHU089lfLycjIzM7nkkku4+OKLeeyxx5g9ezaffvopDz74IBArQr///nueeeYZZs+eze23385LL70U8KuSJEnS5rDolCRJUq3RqlUrxowZQ3l5OQceeCC9e/fmvPPOo0GDBoTDsV99r7jiCv7v//6Pv//973Tv3p1jjjmGpUuXAnDYYYdx/vnnM2LECHbYYQfGjh3LFVdcEeRLkiRJ0mby1HVJkiRJkiRJSc8ZnZIkSZIkSZKSnkWnJEmSJEmSpKRn0SlJkiRJkiQp6Vl0SpIkSZIkSUp6Fp2SJEmSJEmSkp5FpyRJkiRJkqSkZ9EpSZIkSZIkKelZdEqSJEmSJElKehadkiRJkiRJkpKeRackSZIkSZKkpGfRKUmSJEmSJCnpWXRKkiRJkiRJSnoWnZIkSZIkSZKSnkWnJEmSJEmSpKRn0SlJkiRJkiQp6Vl0SpIkSZIkSUp6Fp2SJEmSJEmSkp5FpyRJkiRJkqSkZ9EpSZIkSZIkKelZdEqSJEmSJElKehadkiRJkiRJkpKeRackSZIkSZKkpGfRKUmSJEmSJCnpWXRKkiRJkiRJSnoWnZIkSZIkSZKSnkWnJEmSJEmSpKRn0SlJkiRJkiQp6Vl0SpIkSZIkSUp6Fp2SJEmSJEmSkp5FpyRJkiRJkqSkZ9EpSZIkSZIkKelZdEqSJEmSJElKehadkiRJkiRJkpKeRackSZIkSZKkpGfRKUmSJEmSJCnpWXRKkiRJkiRJSnoWnZIkSZIkSZKSnkWnJEmSJEmSpKRn0SlJkiRJkiQp6Vl0SpIkSZIkSUp6Fp2SJEmSJEmSkp5FpyRJkiRJkqSkZ9EpSZIkSZIkKelZdEqSJEmSJElKehadkiRJkiRJkpKeRackSZIkSZKkpGfRKUmSpKS1995706tXr6BjSJIkKQFYdEqSJEkbKCws5PTTT6dXr17k5eVRv359tt9+e2677TZKS0ur3HfUqFGcdtppbLvttmRnZ9OpUyfOOOMMFi1a9IvPf8cdd5CXl1f5XIsWLeLMM8+kY8eOZGVl0blzZy644AKWL1++yce/9tprhMNhFi9e/KuvY+bMmVx88cXssMMO5OTk0LJlSw4++GAmTpy4he+IJElSckgNOoAkSZKUSAoLC5k+fToHHXQQHTp0IBwOM3bsWM4//3zGjx/PU089VXnfSy65hBUrVnDUUUfRpUsX5syZw5133snrr7/OlClTaNGixUbP/8Ybb3DggQeSlpbGunXr2HXXXcnPz2fYsGG0bduWL774gjvvvJMPPviASZMmEQ6HN3p83759N/ncG3rggQd48MEH+dOf/sSwYcNYvXo19957L7vssgtvv/02+++/f3zeMEmSpAQRikaj0aBDSJIkSRXy8/OpV6/eZt137733ZtmyZXz55ZfVnAr+/Oc/c+edd7Jo0aLKknH06NHsvvvuVcrI0aNHs9dee3HZZZfxj3/8o8pzFBQU0LhxY0aOHMkpp5zCU089xQknnMDrr7/OwQcfXHm/K6+8kmuuuYbJkyez4447VnmOdu3acdppp3HVVVdtMmdRURHp6el8/vnndO3alfr161fetnz5crp37862227LJ5988nvfEkmSpITi0nVJkiQBsHbtWs477zw6dOhARkYGzZo144ADDmDy5MlV7jd+/Hj+8Ic/kJeXR3Z2NnvttRdjxoypcp/vvvuOYcOG0bVrV7KysmjcuDFHHXUU8+bNq3K/Rx55hFAoxEcffcSwYcNo1qwZbdq0qbz9rbfeYq+99iInJ4fc3Fz69etXZUZlhRkzZrDPPvuQnZ1N69atufHGGze6z/fff8/MmTO3+v3p0KEDAKtWraq8bs8999xoxuWee+5Jo0aN+OqrrzZ6jlGjRlFcXMygQYMAWLNmDQDNmzevcr+WLVsCkJWVVeX6adOmMX/+/MpS9MMPPyQUCvHMM89w+eWX07p1a7Kzs1mzZg19+/atUnICNG7cmD322GOT2SRJkpKdS9clSZIEwNlnn83zzz/PiBEj6NGjB8uXL+eTTz7hq6++ok+fPgD873//Y9CgQfTt25crr7yScDjMww8/zL777svHH39M//79AZgwYQJjx47l2GOPpU2bNsybN4+RI0ey9957M2PGDLKzs6t872HDhtG0aVP+/ve/k5+fD8RK0NNOO42ePXvy17/+lQYNGvD555/z9ttvc/zxx1c+duXKlfzhD3/giCOO4Oijj+b555/nkksuoXfv3pWFIsDJJ5/MRx99xOYuaCopKWHNmjUUFhYyceJE/vOf/9C+fXu22WabX33cunXrWLduHU2aNNnotjfffJO+fftWFpsVRelf/vIXbrrpJtq0acPUqVO57rrrGDx4MN26ddvo8c2aNWOnnXaqcv21115Leno6F154IcXFxaSnp/9ivsWLF28ymyRJUrKz6JQkSRIQ2/tx6NCh3HTTTZXXXXzxxZVfR6NRzj77bPbZZx/eeustQqEQAGeddRY9e/bk8ssv59133wXg4IMP5sgjj6zy/Iceeii77rorL7zwAieddFKV2xo1asSoUaNISUkBYPXq1Zx77rn079+fDz/8kMzMzCo5NrRw4UIee+yxyuc8/fTTad++PQ8++GCVonNLvfjiixx33HGVl3faaSceeughUlN//VfoW2+9lZKSEo455piNbnvzzTc59dRTKy/36NGD++67jwsvvJBdd9218vohQ4bwwAMPbPT4N954g0GDBlW+9xWKioqYOHHiRjNAf+7jjz9m3LhxXH755b96P0mSpGRk0SlJkiQAGjRowPjx41m4cCGtWrXa6PYpU6bwzTffcPnll290Ivh+++3H448/TiQSIRwOVyncSktLWbNmDdtssw0NGjRg8uTJGxWdQ4cOrSw5Ad577z3Wrl3LpZdeWqXkBDYq+erXr8+JJ55YeTk9PZ3+/fszZ86cKvf78MMPN++NWG+fffbhvffeY9WqVYwaNYovvviicrbpLxk9ejRXX301Rx99NPvuu2+V27788ku+//77KntxArRu3Zr+/ftz0EEH0b59ez7++GNuv/12mjRpwn/+85/K+61atYpx48bx5z//eaPvO2TIkN8sOZcuXcrxxx9Px44dqxTYkiRJtYVFpyRJkgC48cYbGTJkCG3btqVv374cdNBBnHzyyXTq1AmAb775BoiVar9k9erVNGzYkMLCQv75z3/y8MMPs2DBgiqzMFevXr3R4zp27Fjl8uzZswHo1avXb+Zu06bNRuVnw4YNmTp16m8+9tc0b968con5kUceyfXXX88BBxzAN998s8kTz2fOnMnhhx9Or169fnE2ZvPmzassOx8zZgyHHHIIn376aeX1gwcPJjc3l6uvvprTTjuNHj16APDOO+8AcOCBB2703D9//34uPz+fQw45hLVr1/LJJ59stHenJElSbeBhRJIkSQLg6KOPZs6cOdxxxx20atWKf//73/Ts2ZO33noLgEgkAsC///1v3nvvvU1+VBRof/7zn7nuuus4+uijee6553j33Xd57733aNy4ceXzbOi3ZiP+mg1ngm5oc/fi3FxHHnkk69at45VXXtnotvnz53PggQeSl5fHm2++SU5Ozkb3efPNN/nDH/5QpZS99957Nyo/AQ477DCi0Shjx46t8vgBAwaQl5e30XP/2vtXUlLCEUccwdSpU3nllVc2qzyWJElKRs7olCRJUqWWLVsybNgwhg0bxtKlS+nTpw/XXXcdgwYNonPnzgDk5uay//77/+rzPP/88wwZMqTKfp9FRUVVTiz/NRXf68svv/zNw39qSmFhIbDxjNTly5dz4IEHUlxczKhRoypPTN/QqlWrGDt2LCNGjKhy/ZIlSygvL9/o/qWlpQCUlZUBsdL27bff5sILL9yizJFIhJNPPplRo0bx3HPPsddee23R4yVJkpKJMzolSZJEeXn5RgVes2bNaNWqFcXFxQD07duXzp0785///Id169Zt9Bw//vhj5dcpKSkbzai84447NlnqbcqBBx5ITk4O//znPykqKqpy29bO1Pz++++ZOXPmb95v2bJlm/weFcvRN5x9mZ+fz0EHHcSCBQt488036dKlyyafs+KQpp8vO992221ZsmTJRvuHPv300wDsuOOOQOwU+6VLl260v+dv+fOf/8yzzz7L3XffzRFHHLFFj5UkSUo2zuiUJEkSa9eupU2bNhx55JFsv/321K9fn/fff58JEyZUzsoMh8M88MADDBo0iJ49e3LqqafSunVrFixYwAcffEBubi6vvfYaAIcccgiPP/44eXl59OjRg3HjxvH+++/TuHHjzcqTm5vLLbfcwhlnnEG/fv04/vjjadiwIV988QUFBQU8+uijW/waTz75ZD766KPfLEqfeOIJ7rnnHgYPHkynTp1Yu3Yt77zzDu+99x6HHnpolUOGTjjhBD777DNOO+00vvrqK7766qvK2+rXr8/gwYOB2P6cu++++0bLzkeMGMHDDz/MoYceyp///Gfat2/PRx99xNNPP80BBxzAzjvvXPn4Dh06VO7XuTluvfVW7r77bnbddVeys7N54oknqtx++OGHU69evc1+PkmSpERn0SlJkiSys7MZNmwY7777Li+++CKRSIRtttmGu+++m3POOafyfnvvvTfjxo3j2muv5c4772TdunW0aNGCnXfembPOOqvyfrfddhspKSk8+eSTFBUVMWDAAN5//30GDhy42ZlOP/10mjVrxg033MC1115LWloa3bp14/zzz4/ra/+53XffnbFjx/L000+zZMkSUlNT6dq1KzfffPNGJ55PmTIFgIceeoiHHnqoym3t27dn8ODBv7rsvGvXrkyaNInLL7+cJ554gsWLF9OqVSsuvPBCrr766sr7vfnmmxx00EFb9Doqso0bN45x48ZtdPvcuXMtOiVJUq0SisZ7l3ZJkiRJlT777DN23nlnpk+fvkUzMissWbKEli1b8vrrr29x2SlJklSXuEenJEmSVM2uv/76rSo5IXb40d///nf22WefOKeSJEmqXZzRKUmSJEmSJCnpOaNTkiRJkiRJUtKz6JQkSZIkSZKU9Cw6JUmSJEmSJCW91KAD1GaRSISFCxeSk5NDKBQKOo4kSZIkSZKUVKLRKGvXrqVVq1aEw78+Z9OisxotXLiQtm3bBh1DkiRJkiRJSmrz58+nTZs2v3ofi85qlJOTA8T+Q+Tm5gacRjWhtLSUd999lwMPPJC0tLSg40hbzbGs2sBxrNrCsazawrGs2sKxrNoiWcbymjVraNu2bWXP9mssOqtRxXL13Nxci846orS0lOzsbHJzcxP6h4T0WxzLqg0cx6otHMuqLRzLqi0cy6otkm0sb862kB5GJEmSJEmSJCnpWXRKkiRJkiRJSnoWnZIkSZIkSZKSnnt0SpIkSZIkCYBIJEJJSUnQMVQDSktLSU1NpaioiPLy8kCzpKenEw7//vmYFp2SJEmSJEmipKSEuXPnEolEgo6iGhCNRmnRogXz58/frIN+qlM4HKZjx46kp6f/ruex6JQkSZIkSarjotEoixYtIiUlhbZt28Zldp0SWyQSYd26ddSvXz/Q/96RSISFCxeyaNEi2rVr97tKV4tOSZIkSZKkOq6srIyCggJatWpFdnZ20HFUAyq2KcjMzAy82G7atCkLFy6krKyMtLS0rX4e63lJkiRJkqQ6rmKPxt+7dFjaGhXj7vfuFWrRKUmSJEmSJIDA92pU3RSvcWfRKUmSJEmSJCnpWXRKkiRJkiRJCeCUU05h8ODBQcdIWhadkiRJkiRJSloLFizgxBNPpHHjxmRlZdG7d28mTpy4yfueffbZhEIhbr311t983gkTJrDffvvRoEEDGjZsyMCBA/niiy/inF7xZNEpSZIkSZKkpLRy5UoGDBhAWloab731FjNmzOCmm26iYcOGG933pZde4tNPP6VVq1a/+bzr1q3jD3/4A+3atWP8+PF88skn5OTkMHDgQEpLS6vjpSgOLDolSZIkSZKUlP71r3/Rtm1bHn74Yfr370/Hjh058MAD6dy5c5X7LViwgD//+c88+eSTpKWl/ebzzpw5kxUrVnDNNdfQtWtXevbsyZVXXsmSJUv47rvvfvFxX3zxBfvssw85OTnk5ubSt2/fytmlV111FTvssEOV+99666106NBho+e5+uqradq0Kbm5uZx99tmUlJRU3vb888/Tu3dvsrKyaNy4Mfvvvz/5+fnAT0vff+3xb7/9NrvvvjuNGjWiU6dOHHroocyePbvK9//hhx847rjjaNSoEfXq1WOnnXZi/Pjxlbe/8sor9OnTh8zMTDp16sTVV19NWVnZb76v1S016ACSJEmSJElKLNFolMLS8kC+d1Zaymafwv3qq68ycOBAjjrqKD766CNat27NsGHDGDp0aOV9IpEIJ510EhdddBE9e/bcrOft2rUrjRs35sEHH+Rvf/sb5eXlPPjgg3Tv3n2TxWSFE044gR133JGRI0eSkpLClClTNqtY3dCoUaPIzMzkww8/ZN68eZx66qk0btyY6667jkWLFnHcccdx4403cvjhh7N27Vo+/vhjotHoZj0eID8/nwsuuIBevXqxZMmSyueaMmUK4XCYdevWsddee9G6dWteffVVWrRoweTJk4lEIgB8/PHHnHzyydx+++3ssccezJ49mzPPPBOAK6+8cotea7xZdEqSJEmSJKmKwtJyevz9nUC+94xrBpKdvnmV1Zw5cxg5ciQXXHABf/vb35gwYQLnnnsu6enpDBkyBIjN+kxNTeXcc8/d7Aw5OTl8+OGHDB48mGuvvRaALl268M4775Ca+svZvv/+ey666CK6detW+ZgtlZ6ezkMPPUR2djY9e/bkmmuu4aKLLuLaa69l0aJFlJWVccQRR9C+fXsAevfuvdmPD4fD/OlPfwJiBXCzZs148MEHad68OTNmzKBXr1489dRT/Pjjj0yYMIFGjRoBsM0221Q+/9VXX82ll15a+f526tSJa6+9losvvjjwotOl65IkSZIkSUpKkUiEPn36cP3117Pjjjty5plnMnToUO655x4AJk2axG233cYjjzzyi7NEBw0aRP369alfv37ljM/CwkJOP/10BgwYwKeffsqYMWPo1asXBx98MIWFhQCVj6lfvz5nn302ABdccAFnnHEG+++/PzfccMNGS8I3x/bbb092dnbl5V133ZV169Yxf/58tt9+e/bbbz969+7NUUcdxf3338/KlSs3+/EA33zzDccddxzbbLMN7dq1o1OnTkCspAWYMmUKO+64Y2XJ+XNffPEF11xzTZXXP3ToUBYtWkRBQcEWv954ckanJEmSJEmSqshKS2HGNQMD+96bq2XLlvTo0aPKdd27d+eFF14AYsusly5dSrt27SpvLy8v5//+7/+49dZbmTdvHg888EBleVmxzPypp55i3rx5jBs3jnA4XHldw4YNeeWVVzj22GOZMmVK5XPm5uYCsX04jz/+eN544w3eeustrrzySp555hkOP/xwwuFwlSXmwBYfbJSSksJ7773H2LFjeffdd7njjju47LLLGD9+PB07dtys5zj00ENp37499957L7m5uWRnZ7PddttV7uOZlZX1q49ft24dV199NUccccRGt2VmZm7R64k3i05JkiRJkiRVEQqFNnv5eJAGDBjArFmzqlz39ddfVy7rPumkk9h///2r3D5w4EBOOukkTj31VABat2690fMWFBQQDoerzAKtuFyxV+WGy7k3tO2227Ltttty/vnnc9xxx/Hwww9z+OGH07RpUxYvXkw0Gq183g3L0gpffPEFhYWFlYXjp59+Sv369Wnbti0Q+28zYMAABgwYwN///nfat2/PSy+9xAUXXPCbj1++fDmzZs3i/vvvZ8CAAaxZs4apU6dW+f7bbbcdDzzwACtWrNjkrM4+ffowa9asX3z9QXLpuiRJkiRJkpLS+eefz6effsr111/Pt99+y1NPPcV9993H8OHDAWjcuDG9evWq8pGWlkaLFi3o2rXrLz7vAQccwMqVKxk+fDhfffUV06dP59RTTyU1NZV99tlnk48pLCxkxIgRfPjhh3z33XeMGTOGCRMm0L17dwD23ntvfvzxR2688UZmz57NXXfdxVtvvbXR85SUlHD66aczY8YM3nzzTa688kpGjBhBOBxm/PjxXH/99UycOJHvv/+eF198kR9//LHye/zW4xs2bEjjxo257777+Pbbbxk9ejQXXnhhle9/3HHH0aJFCwYPHsyYMWOYM2cOL7zwAuPGjQPg73//O4899hhXX30106dP56uvvuKZZ57h8ssv37L/eNXAolO12rzV8/hh7Q+sLFpJcXnxRlPEJUmSJElS8urXrx8vvfQSTz/9NL169eLaa6/l1ltv5YQTTvhdz9utWzdee+01pk6dyq677soee+zBwoULefvtt2nZsuUmH5OSksLy5cs5+eST2XbbbTn66KMZNGgQV199NRBbUn/33Xdz1113sf322/PZZ59tVDIC7LfffnTp0oU999yTY445hsMOO4yrrroKiC2RHz16NAcddBDbbrstl19+OTfddBODBg3arMeHw2GeeeYZJk2axHbbbcff/vY3/vWvf1X5/unp6bz77rs0a9aMgw46iN69e3PDDTeQkhLbUmDgwIG8/vrrvPvuu/Tr149ddtmFW265pXIWbZBCUZufarNmzRry8vJYvXp15V4Nqln7/Xc/lhYsrbycGkolOy2b7LRs6qXWo15aPbLSsqiXWi92XVrsc3bq+q9TN75uw8tZqVlVprGXlpby5ptvctBBB1Xu6yElI8eyagPHsWoLx7JqC8eyaovaOpaLioqYO3cuHTt2DHyfRW29U045hVWrVvHyyy//5n0jkQhr1qwhNze3ch/SoPza+NuSfi3xN1uQfofMlEyyUrMoLIttKlwWLWNNyRrWlKyJy/OHCFWWptlpseKzcG0h73z4DvUz6lctR1OzNypZKy5vWKymhDd/02VJkiRJkiTFWHSqVnvjiDcAKI+UU1hWSH5pPvll+RSWrv+6NJ+CsgLyS/N/un3D60oLyS9bf11pAQWlBeSXxb6Orv+/isdQ+NP3nbdw3lZnzkzJ/Gl26foCtGLW6YazSSvus2GJWmUW6vrPaSm15y+MkiRJkiRJv8SiU3VCSjiF+un1qZ9ePy7PF41GKSwrpKBsffm5vhxdU7iGMRPG0LV3V4oiRVWK0Q3vt+HXFSVqWbQMgKLyIorKi1hRtCIuWdPCaVVmnW749S+WpBvOMv3ZYzNTMqss15ckSZIkSYnhkUceCTpCoCw6pa0QCoUqiz+yfrq+tLSUNV+s4aDOW75XS0l5yUbl54ZF6a+VpBVfbzgztbi8OJYpUsrq4tWsLl4dl9ceDoWpl7p+lukvlaMbXvcrS/XrpdUjKzWLcMhz0SRJkiRJ0u9j0SkliPSUdNJT0mlIw7g8X1mkrLIY/Xk5umEhuqlidcMCdcPHAkSiEdaWrmVt6dq45ATISs3aqPysnE36s1mnG84u/aU9UNPCLteXJEmSJKmusehUwvrxjjtZ8cgjhLOzCWVnEc6uRzgri3B29vrPWYSyswlnZf90Xb3Y51DW+vtn/3T/yuuyMgml1P4Df1LDqeSm55Kb/usnkm2uSDRCUVnRLxamFeVolT1QN1i2v+FjKgrU8mg5AIVlhRSWFbK8aHlcsqaH0zeaUVpRlFaUqFXK0Q0L058v20+rR3o43eX6kiRJkiQlOItOJazIunVE8vOJ5OfH/blDmZk/labZWYQ2LEvXXxfOzo6VoxW3/cp1FY+jFpdh4VD4p+X6cRCNRikuL97kEvzNXapf5fbSAkoiJQCUREooKS5hZfHKuGRNDaVuvFR/c/Y2/YX7ZaVmWZxKkiRJkhRnFp1KWE2GnUPD444lUlBApLCQSEHh+q8LiBQUEP2l6/LX37/wp9ui668jGgUgWlREeVER5SvjU4RVSk2lc2oqc2+6mZTs7A1mo66febphubrhdfV+uq3KbNSK6zIzCYVr1z6WoVCIzNRMMlMzaZTZKC7PWRop3eRS/Z+Xo5tbrBaWFQJQFi1jbcla1pbEZ7l+iFCVmaVVlupveGjUL5SoP1+qn52aTWrYH+eSJEmSpLrNfxkrYaXk5ZGSlxe354tGo0SLijYoTfPXl6UFm76uYIPbKorUDa9b/zlaUEC0tDT2TcrKSCkri5WocUseE6qYObrh8v162VVno1Zet0G5WjHzNGuDcrXeT/cPbeGhSYksLZxGXkYeeRnxGTflkXIKywp/uRzdYGn+5izVLygrIBKNECUau66sgB8Lf4xL1syUzMrSs7Ik/aUDon6hMN1wD9QQzjiVJEmSJCUXi07VGaFQaP2y8yyIzwTCStHSUiKFhZSsXs0H77zDHv36kVJS8lMhusHM0yqzTivK0sKCja6rmKFa+T0KCigvKIh/gZqWFitAN1y6n5VVdSbqBvuibjTr9Od7oFbcJyMj6Zdnp4RTqJ9en/rp9ePyfNFolKLy2D6nhaWFVQ5+qrK36QYzT6vsgbqJpfxlkTIAisqLKCovYgUr4pI1NZxKWjSNO1++k/rp9Tdagr/h/qcb7oe6qRK1Xlo9MlMyk348SJIkSVIi2Hvvvdlhhx249dZbg46ScCw6pTgIpaWRkpZGalYWpU2bktmjB2lxmCkZjURis1A3nEVasMEy/Sqlaf7PlvP/bFl/QWGV56A8VplGS0uJrl5NZPXq3523inD4p8J0E3ugbnLW6a/sgRraYCZqsh4mFQrFlqxnpWZBVnyes7S8tMps0s3a23QTs1ErHlNUXgRAWaSMMsooLCiEgt+fMxwKx4rPn804rZdar3L/0w0Pjfqt2ahZqVmkhJNzHEiSJEmKr9GjR/Pvf/+bSZMmsWjRIl566SUGDx4MQGlpKZdffjlvvvkmc+bMIS8vj/33358bbriBVq1aVT7H119/zUUXXcSYMWMoKSlhu+2249prr2Wfffb51e/9zjvvcOWVVzJ9+nQyMzPZc889uemmm+jQoUM1vmL9EotObbXS8gghIDWldu0dmUhC4XDlbMt4ikajsYKzojj92X6n0V/bF/Vne6D+fDZqtLg49k0ikdhBUvn58Z+FmpHx0/6lmyhSf2lf1F/cAzUri3C9erHZrUk26zAtJY0GKQ1oQIO4PF9ZpIzCskJWF67mrVFv0W+3fhRHizcuRzcsVn/pgKj1X0eJEolGWFe6jnWl66Dwt3NsjqzUrE3ONP3VA6I23AP1Z8VqWrj2bOMgSZIk1SX5+flsv/32nHbaaRxxxBFVbisoKGDy5MlcccUVbL/99qxcuZK//OUvHHbYYUycOLHyfocccghdunThf//7H1lZWdx6660ccsghzJ49mxYtWmzy+86dO5c//vGPXHDBBTz55JOsXr2a888/nyOOOILJkydX62vWpll0aqu9/PkCLnp+KjkZqeRmpdEgO428DT7nZqXRICudvKyq1+dlpZGXnUZORmrSlUq1RSgUIpSeDunppDRoENfnjpaXEyks2uI9UKMbXbf+/htcRyQS+x7FxZQXF1O+alVcs5OSUnX5fr2fLd3f5B6oGxaoG++BWnFbshwmlRpOJSc9h8xQJs1SmtGzcc/fNTs5Eo1QVFb0mwc//bwwrbK36fpiteI+5dFYdV5YVkhhWSHLi5bH5bWnhdM2PiDqlwrTnxWrVZb0r79fRkryb98gSZIkJYNBgwYxaNCgTd6Wl5fHe++9V+W6O++8k/79+/P999/Trl07li1bxjfffMODDz7IdtttB8ANN9zA3XffzZdffvmLReekSZMoLy/nH//4B+H1/+a78MIL+eMf/0hpaekv/lvqww8/5OKLL2b69OmkpaXRs2dPnnrqKdq3b88pp5zCqlWrePnllyvvf9555zFlyhQ+/PDDyuvKysoYMWIEjz/+OGlpaZxzzjlcc801lf8Gufvuu7nllluYP38+eXl57LHHHjz//PNAbOl7r169iEajPP7446Snp2/0+Mcff5zbbruNWbNmUa9ePfbdd19uvfVWmjVrVplh+vTpXHLJJYwePZpoNMoOO+zAI488QufOnQF44IEHuOmmm5g7dy4dOnTg3HPPZdiwYZt8T+LFolNbbXVh7ACetcVlrC0uY8GqLZumFQ7xU/GZlUZedqwUbZBVtRD9eVHaICudzLSwBUKCCqWkkFK/Hin168X1eaPRKNHi4soDoKou5//ZHqibsS9qbDn/+j1SS0pi36S8nMi6dUTWrYtrdqByf9jf2gM19PPrKpbrb2Jf1HBWVqywTmDhULhyBmWTrCa/+/mi0SglkZKfDoDaxD6mG123vkDd1KFR+aX5lERi//1LI6WsKl7FquJVvzsnQEooZZP7mf58qf6vHhC1wR6oWalZhP+fvfsOj6O8+j7+ne1FvUvuvcuWreICxjXuhBZCyYMJnZgAIRVCCAl5E55QQw0t9PaQhAC2cWxs07GKmyz3XtW7drV93j9GWmtVbK0tWy7nk2suy6udnXuFImt/e+5zlLMjMBdCCCGEEOcAVQVvF/SxOhFGG5zC1/y1tbUoikJMU+FPfHw8Q4YM4Y033mDs2LGYzWZeeOEFkpKSGDduXIePM27cOHQ6Ha+++irXX389DQ0NvPnmm8yYMaPDkNPn83HJJZdw88038+677+LxeMjLyws743j99de58cYbycvLo6CggFtuuYXevXtz8803U1BQwJ133smbb77JxIkTqaqq4quvvmpz/g033MDKlSvZtm0bt912W/B80Lb8P/TQQwwZMoSysjLuuecerr/+epYuXQrA4cOHmTx5MlOmTGHVqlVERUXxzTff4PNpcyTefvttHnjgAZ555hkyMjJYv349N998M3a7nYULF4b1XMMhQac4YddP7MulGT2obfRS2+ilptFLXaOXGqf36G3Bjz0ht7l9AQIqVDu9VDu9YV/bpNeFVJE2B6RtbmtZRdpUXWoySFBwNlIUBcViQWexQGxslz626vOdVA/UYDVqO0Fq8BqNjfgbG/FXdc2woCCjMSRADQ1S7e1v3T9OD1SdzYZ6hvZBVRQFs96MWW8m1tI13wfegBen19luYNruVv12tue3DlkB/Kqfem899d76LlkncNyt+s2VqG0+385WfZvBhkEnvwYIIYQQQogOeJ3w57Tj3+9UuO8ImLq2eKaZy+Xi17/+NVdffTVRUVGA9jrjs88+45JLLiEyMhKdTkdSUhLLli0j9hivP/v168fy5cu58sorufXWW/H7/UyYMCEYBranrq6O2tpa5s+fH6x8HDZsWNjPo1evXjzxxBMoisKQIUPYtGkTTzzxBDfffDMHDhzAbrczf/58IiMj6dOnDxkZGW3Of/zxx6mvr2fcuHFs3rw5eD7ADTfcELxv//79eeqpp8jKyqKhoYGIiAieffZZoqOjee+994Kh7uDBg4Pn/P73v+exxx4LthLo168fW7Zs4YUXXpCgU5yZDHod8RFm4iPMYZ/r8vq1ULRNIOql1ukJCU+1245+3hdQ8fgDVDS4qWhwh31tm0kfWknaOhS1mULC0+aPo6xG9DqpIj0XKQYD+shI9JGRXfq4qqqGDpNyNPU07XQP1NCt+yHDpJreJcPrJeD1Eqir69K1oygMNBrZ+9dHmkJR2zH7ooZUo7aoRG2vL6piOLP+6THqjESbo4k2R3fJ4wXUQDA07WjwU3u9TVsHqw6vg0ZvIw6fg4CqtW1w+rRzKhorumStZr25c71NO9kD1aQzSbW9EEIIIYQ4Y3m9Xq688kpUVeX5558P3q6qKosWLSIpKYmvvvoKq9XKyy+/zIIFC8jPzyc1NZURI0awf/9+AC688EI+/fRTSkpKuPnmm1m4cCFXX3019fX1PPDAA1xxxRWsWLGCgwcPMnz48OB17rvvPu677z6uv/56Zs2axcyZM5kxYwZXXnklqampYT2X8ePHh/zuPWHCBB577DH8fj8zZ86kT58+9O/fn9mzZzN79mwuvfRSbC3mfxzrfL1ez9q1a3nwwQfZuHEj1dXVBJpayR04cIDhw4ezYcMGLrzwwnYrVx0OB7t37+bGG28MBqegVbNGR3fN666OnFmvNsV5w2LUYzHqSYqyhHWeqqo4PP5g+FnT6GlbRdr0Z8vba5we6t0+VBWcHj9Oj5/iWlfY6460GFpVkZpCqkgjTDp2VyrE7qkkLsJ69HbpR3peUhQluGW9q6keT6sK0nB7oDaFq60qVFVX0/8vVBWdx4O/shJ/Zdf0wGymmEyhPVBbVqTaWwSjTdv52/RAba5GDQav2m2K6cwI2XSKLlhV2RVUVcXtdx87HG3Z2/QYW/Wb7+cNaJX0br8bt99Ntbu6S9ZqUAztb9Vvp5q0o8C05d+tBusZ8d9UCCGEEOK8ZLRplZXdde0u1hxy7t+/P7jVutmqVatYvHgx1dXVwdufe+45VqxYweuvv85vfvMbli5diter/R5tbXqN11zV+Ne//jX4WG+99Ra9evUiNzeXzMxMNmzYEPxcXFwcAK+++ip33nkny5Yt4/333+f+++9nxYoVjB8/Hp1Oh6qqbdYejsjISNatW8fnn3/O8uXLeeCBB3jwwQfJz88Pbtc/FofDwaxZs5g1axZvv/02iYmJHDhwgFmzZuFpav1mPcbr3IamdnAvvfQSOTk5IZ/Tn+LdgxJ0irOKoihEmA1EmA30iAkvPPIHVBpcPmpabaMPVpK2CE+1v/uC1aUOjzb8pN7lo97l4+Axx0breXXH2tBbdMqxq0hDbgsd4GQxnplbiEX3Ukwm9CYT+i5+N0wNBFAbG3HX1bH600+ZnJ2NzuM9qR6owWFSfu3/R6rHg9/jgdraLl07Ol1o1WnLrfsd9UC1tVN1amvVF9VqQenGrfyKomAxWLAYLMRb47vkMb1+b5vBTy0rSFsOjmq9pb+9rfrN2/V9qo86Tx11nq6pMFZQQrbdt64gPV5vU5NiotxfTqmzlGhrNDaDDb1OfqYKIYQQQnSKopyy7eOnW3PIuXPnTlavXk18fOjv1c6mtmO6VkNkdTpdsJKxT58+bR7X6XS2Oac5yAsEAhgMBgYOHNjumjIyMsjIyODee+9lwoQJvPPOO4wfP57ExESKiopC7rthw4Y2lZO5ubkhf1+zZg2DBg0KXt9gMDBjxgxmzJjB73//e2JiYli1alVwK/mxzt+2bRuVlZU8/PDD9OrVCyBkQj1Aeno6r7/+ertDl5KTk0lLS2PPnj1ce+217T7/U0WCTnHe0OsUbbiRLfwJ0h5fgDrX0XC0rvFopWht49HwtMbhYe+RMvSWCOpcPmoavXh8AfwBlSqHhyqHJ+xrmwy6kC30MTZtG31zRWm01RAMR1v3KDXqpR+pCI+i06HY7RhMJrzx8ZiHDDmpqevNVFXVqlCdLcLSYEDaYut+p3qgtqhGbWxEdTe1sAgECDgcBByOk15va4rFErpNv72t++1VnR6jL2pzFWp3MOqNROu7bru+P+A/GoL6mgLTYw2Iam/7fqt+qGrT/5rPOeb7S8fxt//8LfixRW9p29u0qeq0ZYDafJ8Ot+03/WnUn/z/P4QQQgghxMlpaGhg165dwb/v3buXDRs2EBcXR2pqKldccQXr1q1j8eLF+P1+SkpKAK3C0mQyMWHCBGJjY1m4cCEPPPAAVquVl156ib179zJv3rwOrztv3jyeeOIJ/vjHPwa3rt93333t9sRsubYXX3yRiy++mLS0NLZv387OnTu57rrrAJg2bRqPPPIIb7zxBhMmTOCtt96iqKiozeMdOHCAe+65h1tvvZV169bx9NNP89hjjwGwePFi9uzZw+TJk4mNjWXp0qUEAgGGDBkScv7Pf/5zrrnmGnbs2BFyfu/evTGZTDz99NPcdtttFBUV8dBDD4Vc/4477uDpp5/mqquu4t577yU6Opo1a9aQnZ3NkCFD+MMf/sCdd95JdHQ0s2fPxu12U1BQQHV1Nffcc09n/9OGTYJOITrBZNCREGEm4Tj9SL1eL0uXLmXu3EnBcMjl9bca0OQJrSJtr7K06fAHVDy+AGX1bsrqw+9Ham/uR2rTAtFgOGrruLI0xmoi0mJAJ/1IRRdSFAXFbEZnNp+aYVIuV5g9UDvqi9oUrjbdRtOWEdXlwu9y4a/umi3eQQZDm2FSJ9YDtVUfVYsFRXf63ujQ6/REmiKJNHVNn1tVVWn0NbZbQdo6HG2vB2pw277XQW1jLV68+FStr63L78Lld1Hl6prBYEadsU3VachW/c70QG1xrkVvke36QgghhBBhKigoYOrUqcG/NwdpCxcu5MEHH+Tjjz8GYMyYMSHnrV69milTppCQkMCyZcv47W9/y7Rp0/B6vYwYMYKPPvqI0aNHd3jdadOm8c477/DXv/6Vv/71r9hsNiZMmMCyZcs63Npts9nYtm0br7/+OpWVlaSmprJo0SJuvfVWAGbNmsXvfvc7fvWrX+Fyubjhhhu47rrr2LRpU8jjXHfddTQ2NpKdnY1er+euu+7illtuASAmJoZ///vfPPjgg7hcLgYNGsS7777LiBEj2pw/ffp0DAZDyPmJiYm89tpr3HfffTz11FOMHTuWRx99lIsvvjh4fnx8PKtWreKXv/wlF110EXq9njFjxjBp0iQAbrrpJmw2G4888gi//OUvsdvtjBo1irvvvrvDr2dXUNTWG/9Fl6mrqyM6Opra2tqQ3g/i3HU06Jx70lVwqqrS4Pa1rSJtFY5qQ51Ct+PXu3wndW1FgUjz0UrR0CrSttWlLcNTu0kvL9LPAV35vXy2Cg6Taq8Haif6orbXAzXQ2IjqdKKG2WPnRCjNwWebHqgtQtGQvqjN1agttu8332Y/en/lLPp+aP4+njNnDujpcPBTRyFp899bbvNvrkx1+8N/86kzdIoOu6GpyrSjcLTlbS226rfe3m832rEarOgUqe4/28nPZHGukO9lca44V7+XXS4Xe/fupV+/flgs4c3TEGeXKVOmMGbMGB5//HHq6uqIiopqswX/dDvW9184+ZpUdApxhlIUhUiLkUiLkZ5hFsD5A2owGA2ZXt9iqn1HVaROjx9VhTqXj7oTCEwNLfuR2tqGo9GtepA2fz7KKv1IxZklZJhUXNc+tur1Hg0/nS0GRnW6B6ojuHU/pC9q49H93arTid/pxN+1S0cxGoMDoFoGqYqt1db9zvZAbb6P2XzK3iRRFAWj3ohJbyKWrqko9gV8R4PQVoOfOupt2hystrtt36f1hQqoAeq99dR767tknQBWg7VN+BmsJm1VddpeP9TWPVCNunPnBZ0QQgghhDi3SNApxDlIr1OItZuItYff/8/jCzSFnu0PbWquLm0Znjbf5vEH8AVUKh0eKk+gH6nZoGs1oMnUdmt9B9WlBulHKs4iitGI3mhE38XV/mogoFWhhoSmLbbph4SmrapRO+iB2nx+cJiU14taW0vgVAyTslhQ7K3D0hPrgapYbQRMRmhqHt/VDDoDUaYookxd898woAZw+VzH7m3aeqt+O9v2WwarflX7b9Y8MKrSVdklazXpTG0qSpuD0uYQNSQcbRmYtt62b7Rj0plkJ4AQQgghhOgSEnQKIUKYDDoSI80kRh67H2lrqqri8gaODmZqWTHqbKe6tFWv0oAKbl+A0jo3pXXhbwmNMBs6nmrfqgdpy4rTSLP0IxXnDkWnC1ZbdiVVVbWA09l+D1S1nds66oGqNjpDKlRDhkk5ndDFVaiDgd0P/uGYPVA76osaEqS27otqt2vVrV0U0OkUXTA4TCTxpB9PVVXcfnfHvU1bbdVv734tg1an14knoL2B5Ql48Lg9VLu7pmetQTG03arfmd6mHdzParBKcCqEEEIIcQyff/45QHCi/LlEgk4hRJdQFAWrSY/VZCU1uv2myx0JBFQaPL7QQDQkHPW0mHQfWkVa79a21ze4fTS4fRyuCW80s04hWCHaUVAaYzUdrSK1Hf3TapR+pOL8oCiKNiHeZEIfE9Olj636/QQaXWH3QFVb3xYMXY/e1lzNqbrd+N1u/DU1Xbp29PqQHqjBatQ2fVHb9kDtqC9q8+dOdpiUoihYDBYsBgtxlq7pveD1ezsVjrbX27T57y3PbfRpP699qo96Tz31nq7Zrq+ghFSWhmzVbzk0qoMQtfVWfZvBhkEnvzILIYQQQpwN5Lc2IUS30+kUoixGoixGeoV5rs8foM7VPLSp1UR7Z+sq0tDw1OUNEFChxqkFp+Ey6pV2wlFTx5WlLapLzQbpRyoEgKLXo4+wo4+wd+njqqqKp6GB5Z98wvRJk9B7PB33QO1UX9Sjlauqp6k1h99PoKGBQENDl64dCPaHPV4PVCWMvqg6q1ULrE+QUW8kWh9NtDm6S56jP+A/WjHaXjjaQQ/UjoZIOX1OAmoAFVW7zeekvLG8S9Zq0VuCoWcwJO0oHO0gMG3ZA9WkP/H/DkIIIYQQomMSdAohzmoGvY44u4k4uwkILyhx+/yhgWirLfZHq0g9bW7z+lW8fpWKBg8VDeH3I7Ua9W3Cz5Z9R0N6kbYIT6MsBulHKkQnKIqCzmIhYLdjTEvr0omoqs8XRg/U9qpOW/dFPRqkBq/R2Ii/sbHLh0lhNIZWnDZXo9qagtFO9EVt7oEasp3fYgm7wl2v0xNhiiDCFNElT01VVVx+rc9po7cxZPCTw9d0W6seqCFb+tupVvUFtF0DLr8Ll99FFVVdslaDznDMrfot+5/ajXbMOjPbPduJOhxFlDWqzTkWffhffyGEEEKIc5EEnUKI85bZoCcpUk9SpCWs81RVxenxt7PNvv0BTiFDm1xeVBUavX4avX5K6lxhrzvSbAgNR5s+jmrRgzR0qNPRfqTyQliIk6cYDOgjI9FHRnbp4waHSbUITdXG9vuitt8D1RGydb/ln/i0wA6vl4DXS6CurkvXjqI0Baa2NkFqRz1Qg9WoTVv8Q3qg2o7eXzF07tdVRdG2rFsNVgivg0qHvH5vSDVpe+Fouz1Q2xkU5fQ6cfm1n/m+gI9ady217vCGer3/xfvt3q5TdFrw2ari1G6wB/ufthwaFdLvtJ3t+1aDFb1Odh4IIYQQ4uwjQacQQoRJURTsZgN2s4G0mPD7kda7fW220bfsO9pRUNrQ1I+03u2j3u3jUHX4/UiPBp+mNlWkLatLI0wKRxxQXOsiMUqHxaiTkFSIU+xUDZMCUFtu3e+iHqgBpxPV1fRmjaqekmFSAIrJ1IkeqPZ2tu530AO16WusmI4/7d2oNxKjjyGGmC55Lr6AL2TAU4fhaItqVKfXSYOngYOlB7FEWYJVq83hqYpKQA3Q4G2gwdsA4f3T0CGrwdpupekxB0S17IHaVI3a/HejruuqqoUQQgghOiJBpxBCnEY63dG+nuHy+gNHg9CmP+taDWg6GoyGVpe6fVo/0mqnl2qnFyqdx78gBv638EsATHqdVjFqCw1Ho1pVjx79+GiQajLIVnshuptiMqE3mdBHd01/zWZqINBqiNTJ90ANDpPya5Gp6vHg93igNrzqx+PS6Vr1QLWH9EUNtwfq0cDVgqJvvxrSoDMQaYok0hReNbDX62Xp0qXMnTM3pA1DQA3g8rmOOfjpeD1QW1em+lXt697oa6TR10ilq/LEv8YtGHXGtgOiOhoG1aofans9UM16s7wBJ4QQQog2JOgUQoizhFGvIz7CTHyEOexzXV5/m230zf1HW4entY1eahweymsdNAZ0+AMqHn+AigY3FQ3usK9tM+lDt9G3GdDUtro0xmYk0mJEr5MXsUKcyRSdDsVuR2fv+mFSqscTGoAGA9IWW/c71QO16f5Nt6nupp9jgQABh4OAw9GlawdQLJajoandpvU0bb11v4MeqNogqtDb/EYjis+Hqqoh19EpumDFZII14aTXraoqnoDnaJWp1xGsQD3WVv1jbdv3BLQ+1t6Alxp3DTXumpNeJ4Be0bfbz7T1Vv1jDohqUXVqNVjRKfLGnBBCiLPDlClTGDNmDE8++WR3L+WMI0GnEEKcByxGPRajnuSozvUjba4emjPne3hUXchQppAq0nam3Dfft97tQ1XB6fHj9Pgprj2BfqQWQ2jFqNXUtoq0nerSCOlHKsRZTVEUFLMZndkMsbFd+tiqz0fA5QqzB2pHfVGbwtWm22gKIlWXC7/Lhb+6usvWPQjY/fsHO98DteWW/mP0QA0Ok9JpLUrMejNmvZk4S1yXrNsb8AaDz3Z7m7YIR1tXmrYMWpvPafRpe/P9qp96bz313vouWSdw3K36zZWobT7fYtt+y4pUg05eagkhxOny5Zdf8sgjj7B27VqKi4v58MMPueSSSwDttc3999/P0qVL2bNnD9HR0cyYMYOHH36YtLS04GPs2LGDX/7yl3zzzTd4PB7S09N56KGHmDp16jGvraoqjz32GC+++CL79+8nISGBn/zkJ/z2t789lU9ZdED+9RVCCNEhRVGIMBmIMBvoEWY/Un9Apd7Vtoo0tLI0dIt9c3Wpw6Ntnax3+ah3+TgYZtM5fYsWAR1Wkraaat/8OYtRBnAIcS5TDAb0ERHoI7pm2nszVVWPDpNq3QO1RTVqR31Rg1Wnrbbwq04nqterXcTnI1BfT6C+68K9ZsFBUm16oHbUF7VtD1SlnduMRiPR5miizV3TNiGgBkL6nLauIG359/a26rfsgdrobcThcxBQAwDafXxOKhorumStZr05GIimRaSRmZxJVkoW6YnpmPXh784QQgjRMYfDwejRo7nhhhu47LLLQj7ndDpZt24dv/vd7xg9ejTV1dXcddddXHzxxRQUFATvN3/+fAYNGsSqVauwWq08+eSTzJ8/n927d5OSktLhte+66y6WL1/Oo48+yqhRo6iqqqKqquqUPVdxbBJ0CiGEOCX0OoUYm4kYmynscz2+AHWuo+FoXfPQJqeX2kZfcIBT6FAn7U+PL4A/oFLl8FDl8IR9bZNBFzKgKcYWOtE+2moIhqOtq0uNetn2KMT5SlGUpm3nVuiaYsggj9PJfz/+mBmTJqHz+pqqS4/RF7VND1RHcOt+SF/UxqNvIqlOJ/5TMUzKaAwOgGoZpCq2Vlv3O+qBam17m9Vmw2ZNRLGdfOW+qqq4/K5jh6PH6oHqa1uZ6g1owbTb78btd1PtruZww2HyS/J5fuPzGHVG0hPTyUrJIjM5k9GJo7EYOrfjQgghRPvmzJnDnDlz2v1cdHQ0K1asCLntmWeeITs7mwMHDtC7d28qKirYuXMnr7zyCunp6QA8/PDDPPfccxQVFXUYdG7dupXnn3+eoqIihgwZAkC/fv2Ou97PP/+cX/3qV2zevBmj0ciIESN455136NOnD9dffz01NTX85z//Cd7/7rvvZsOGDXz++efB23w+H3fccQdvvvkmRqOR22+/nT/+8Y/BnW3PPfccTzzxBAcPHiQ6OpoLL7yQf/7zn4C29X3kyJGoqsqbb76JyWRqc/6bb77J3/72N7Zv347dbmfatGk8+eSTJCUlBdewefNmfv3rX/Pll1+iqipjxozhtddeY8CAAQC8/PLLPPbYY+zdu5e+ffty55138pOf/OS4X5+TIUGnEEKIM47JoCMhwkzCCfYjbdmDtHUVaW2rrfd1LW73B1Q8vgBl9W7K6sPvR2pv7kdqMxFtNQS320fbOq4sjbGaiLQY0Ek/UiFEBxSjkYDViiElJWQY0clSAwGtCjVkmFSLbfohoakjtBq1gx6ozecHh0l5vai1tQS6epiUomhVpPZWYWm7VafH7ouqt1qJstqIsceii07rcJhUZ3n93pBw1OF1sKtmF/kl+RSUFFDWWMba0rWsLV0LaIOaRiWM0oLPFC34tBrC20UhhBCngqqqwZYhp5vVYD2lrahqa2tRFIWYmBgA4uPjGTJkCG+88QZjx47FbDbzwgsvkJSUxLhx4zp8nE8++YT+/fuzePFiZs+ejaqqzJgxg7/+9a/ExbX/zqfP5+OSSy7h5ptv5t1338Xj8ZCXlxf283399de58cYbycvLo6CggFtuuYXevXtz8803U1BQwJ133smbb77JxIkTqaqq4quvvmpz/g033MDKlSvZtm0bt912W/B80Lb8P/TQQwwZMoSysjLuuecerr/+epYuXQrA4cOHmTx5MlOmTGHVqlVERUXxzTff4PP5AHj77bd54IEHeOaZZ8jIyGD9+vXcfPPN2O12Fi5cGNZzDYcEnUIIIc4pFqOelGg9KdHhVceoqkqD29eqirS9rfeeNrfVu7R/zB0ePw6PnyNh9iNVFIiytK0ibT2gKVhF2iI8tZv00o9UCHFCFJ0uWG3ZlVRV1QJOZ/s9UNWQ7fzH7oGqNjpDKlSDw6RUVQtUT0UVqtnctBW/bdWpITERW1YmtpwcjMnJ7Z5v1BuJ1odu1x+TNIYrBl+BqqocrD9Ifkk++aX55JfkU+YsY13ZOtaVreOFwhcw6AykJ6QzLnkcWSlZjEkaI8GnEKJbNPoayXknp1uunXtNLjZj1/771MzlcvHrX/+aq6++mqioKEDbmfHZZ59xySWXEBkZiU6nIykpiWXLlhF7jH7he/bsYf/+/XzwwQe88cYb+P1+fvazn3HFFVewatWqds+pq6ujtraW+fPnBysfhw0bFvbz6NWrF0888QSKojBkyBA2bdrEE088wc0338yBAwew2+3Mnz+fyMhI+vTpQ0ZGRpvzH3/8cerr6xk3bhybN28Ong9www03BO/bv39/nnrqKbKysmhoaCAiIoJnn32W6Oho3nvvveAbsYMHDw6e8/vf/57HHnss2EqgX79+bNmyhRdeeEGCTiGEEOJUUxSFSIs27b1XmOf6A2q70+trnW1D0ZZVpDVOL41eP6pK8H7hMjT3I21nQFPLqfYxLe4T0xSYSj9SIcSpoCgKiskEJhP6pkqZrqL6/QQaXWH3QFVb3xYMXY/eRkDr1am63fjdbqipaXcNNR98AICpb19sOTnYx+dgy87GEB9/3PUrikLvqN70jurN5YMvR1VVDtUfCoae+SX5lDpLg8HnS5tewqAzMCphFJnJmWSmZDImccwpe/EvhBDnOq/Xy5VXXomqqjz//PPB21VVZdGiRSQlJfHVV19htVp5+eWXWbBgAfn5+aSmpjJixAj2798PwIUXXsinn35KIBDA7XbzxhtvBEO+V155hXHjxrF9+3asVivDhw8PXue+++7jvvvu4/rrr2fWrFnMnDmTGTNmcOWVV5KamhrWcxk/fnxIwcOECRN47LHH8Pv9zJw5kz59+tC/f39mz57N7NmzufTSS7G1eHPzWOfr9XrWrl3Lgw8+yMaNG6muribQ9O/kgQMHGD58OBs2bODCCy9sd7eJw+Fg9+7d3HjjjcHgFLRq1ujorunb3REJOoUQQoiTpNcpxNpNxNrD70fq9vmpa/QFK0VrnG2rSFtWl2rb8X3UNXrx+AP4AiqVDg+VJ9CP1GzQBStFmyfat9laH+xRGrr93iD9SIUQ3UDR69FH2NFH2Lv0cVVVRXW7gwOgQrfzHw1GPfv24czNw7VlC559+/Ds20fN++8DYB40EFvOeGw52dizsjoV8iqKQq+oXvSK6sVlgy7Tgs+GQxSUFASrPkscJawvW8/6svVa8KkYGJkwksyUTLKStYpPCT6FEKeC1WAl95rcbrt2V2sOOffv3x/cat1s1apVLF68mOrq6uDtzz33HCtWrOD111/nN7/5DUuXLsXbNCDQatXWl5qaisFgCKlkbK7OPHDgAFOnTmXDhg3BzzVvZ3/11Ve58847WbZsGe+//z73338/K1asYPz48eh0OlRVbbP2cERGRrJu3To+//xzli9fzgMPPMCDDz5Ifn5+cLv+sTgcDmbNmsWsWbN4++23SUxM5MCBA8yaNQuPxxPyNWhPQ0MDAC+99BI5OaFVwfqTbBFzPBJ0CiGEEN3IbNCTGKknMTK8fqSqqtLo9bfdWh8yoEkLRVtWlzZXlAZUcPsClNa5Ka0Lvx9phNnQ8UT7Vj1IWwamkWbpRyqEOPMoioJisaCzWOAYWxSb+evqcBYU4FizBmduHu7t23Hv3IV75y6q33oLFAXzsKHYs3Owjc/BlpmJPiKiU+voFdmLXpG9uHTQpaiqGhxkVFCqhZ/FjmI2lG9gQ/kGXt70MgbFwPCE4WQlZ5GVkkVGUoYEn0KILqEoyjnz86Q55Ny5cyerV68mvlUVvtPpBECnC30zX6fTBSsZ+/Tp0+ZxJ02ahM/nY/fu3cFt6Dt27Aje32AwMHDgwHbXlJGRQUZGBvfeey8TJkzgnXfeYfz48SQmJlJUVBRy3w0bNrSpnMzNDQ2h16xZw6BBg4JBosFgYMaMGcyYMYPf//73xMTEsGrVquBW8mOdv23bNiorK3n44Yfp1Uvb79ZyQj1Aeno6r7/+Ol6vt83akpOTSUtLY8+ePVx77bXtPv9TRYJO0X12r4LiQrAnQkTS0T9tCWAIvypKCCHOJ4qiYDMZsJkMpEaH9453IKDS4PGFhqLOluFoUxVpq8/XNXqpd2v9SBvcPhrcPg7XhNegXqcQrBw91oCm1hPtY2xGrEbpRyqEODPoo6KInDaNyGnTAPBVVeHMy8eZl4sjNw/P7t24t2zFvWUrVa+9Bno9lhEjsOfkYMvJwTY2o1O9URVFoWdkT3pG9uTSQZcCBIPP/JJ81pau5XDDYQrLCyksL+SVolfQK3pGxI/QKj6bgk+7sWsrYIUQ4kzT0NDArl27gn/fu3cvGzZsIC4ujtTUVK644grWrVvH4sWL8fv9lJSUAFqFpclkYsKECcTGxrJw4UIeeOABrFYrL730Env37mXevHkdXnfGjBmMHTuWG264gSeffJJAIMCiRYuYOXNmSJVnS3v37uXFF1/k4osvJi0tje3bt7Nz506uu+46AKZNm8YjjzzCG2+8wYQJE3jrrbcoKipq02PzwIED3HPPPdx6662sW7eOp59+msceewyAxYsXs2fPHiZPnkxsbCxLly4lEAgEJ8M3n//zn/+ca665hh07doSc37t3b0wmE08//TS33XYbRUVFPPTQQyHXv+OOO3j66ae56qqruPfee4mOjmbNmjVkZ2czZMgQ/vCHP3DnnXcSHR3N7NmzcbvdFBQUUF1dzT333NPZ/7Rhk6BTdJ9tSyD/5fY/Z4lpCj+TwJ7Q/scRiVo4apJf3IQQIhw6nUKUxUjUCfQj9fkD1Ll87U+1d7buURoanrq8AQIq1Di14DRcRr3SKhw1tQlMW4amdqNCnUerXO3CQdVCCNGGIS6OqNmziJo9CwBvWZkWfOauwZGbh/fAAVyFhbgKC6l86SUwGrGmp2PPycaWMx7rmNHozJ2r7O8R0YMeA3twycBLAC34LCgpCFZ8Hm44TGFFIYUVhfyj6B/oFT3D44cHt7pnJGUQYTp+dakQQpxNCgoKmDp1avDvzUHawoULefDBB/n4448BGDNmTMh5q1evZsqUKSQkJLBs2TJ++9vfMm3aNLxeLyNGjOCjjz5i9OjRHV5Xp9PxySef8NOf/pTJkydjt9uZM2dOMDBsj81mY9u2bbz++utUVlaSmprKokWLuPXWWwGYNWsWv/vd7/jVr36Fy+Xihhtu4LrrrmPTpk0hj3PdddfR2NhIdnY2er2eu+66i1tuuQWAmJgY/v3vf/Pggw/icrkYNGgQ7777LiNGjGhz/vTp0zEYDCHnJyYm8tprr3Hffffx1FNPMXbsWB599FEuvvji4Pnx8fGsWrWKX/7yl1x00UXo9XrGjBnDpEmTALjpppuw2Ww88sgj/PKXv8RutzNq1CjuvvvuDr82XUFRW2/8F12mrq6O6OhoamtrQ3o/iCYb34M9n0NDGTjKoKEcHOWghjk702hvCj2bq0KbPo5oCkaDHyeCJVobb3yKeL1eli5dyty5c9ttyCvE2UK+l8Wp4PL6g4OYWg9oOlpF2mKAU4vKUl/g5H5dsRr1bbbWx1jb9iJtGaA2D23Sy1Z70c3kZ/LZz3vkCI7cPJy5uThyc/EVF4d8XjGZsGZkaIONcnKwjhypDXQ6AUcajlBQWhDs83mo4VDI53WKjuFxw8lKySIzJZOMpAwiTZEn/NzCId/L4lxxrn4vu1wu9u7dS79+/bBYLN29HHEKTZkyhTFjxvD4449TV1dHVFRUm237p9uxvv/CydekolN0n9FXaUdLgQC4ao6Gn47ypgC0rOm2ihahaBn4XOB1QLUDqvcd/5p6kxZ4ttwuH/y4VcWoLQ50MpFYCCG6isWox2LUkxQV3i/Oqqri9PjbDGtqb4BTbUiQ6qGu0YuKQqPXT6PXT0mdK+x1R5oNoX1Hgx+b2t163xymRpoNstVeCAGAMS2NmEsvIebSS1BVFe/Bgzhyc3GuycWRl4u/vAJnbi7Opn5pitWKbdw4bbDR+PFYhg1DMXTupVtaRBoXR1zMxQO0qpsSR0lIj8+D9QcpqiyiqLKIVze/ik7RMSxumBZ8JmcyNnnsaQs+hRBCiK4mQac4s+h0WsBoiwOGHvu+qgqehqYAtPzon8GPm4LR5tvddeD3QN1h7TgeRQe2+Bbb5I9RMWpPlL6iQghxiiiKgt1swG42kBbT+X6kXq+XxUuWcuG0mTi9hGyjb9l3NKQXaYvKUodH22FQ7/ZR7/ZxqDq8fqR6nUKUpWloU4tt9jHtVZGGVJqasBh1EpIKcY5SFAVT796Yevcm9gc/QFVVPHv3BgcbOfPy8FdX4/j6axxff005oIuIwJaZiW18DvacHMxDhqB0svImxZ7CggELWDBgAaAFny0rPg/UH2Bz5WY2V27mtc2voVN0DI0bSlayVvE5NnksUSbZnSaEEOLsIEFnmJxOJ8OGDeMHP/gBjz76aHcv5/ymKGCO1I74Ace/v9fVFIS2qAjtqGLUWQVq4GhwWtaJ9VhiMNgTmOTWo//3vyAyueOKUekrKoQQp4VOgWirkYSo8LeVef0BLfRsr+9oO5WlLW93+wL4AyrVTi/VTi9UOsO6tkmvazW9Xvuz5ZCm0CrSo0GqydC9246EEOFRFAVz//6Y+/cn7pprUAMB3Dt3atvc1+TizM8nUF9Pw+ef0/D55wDoo6OxZWdjy8nBPj4H04ABnX5zJMWewvz+85nffz4ApY7SYLVnQWkB++v2s6VyC1sqt/D6ltdRUBgaNzTY43Ns8liizdGn6sshhBDiNPi86d+T5ony5xIJOsP0//7f/2P8+PHdvQxxIowWiOmlHcfj94GzIrRStE3FaPnRI+ADVw2Kq4YEgK3bj7MWe/uDldqrHrXEnNK+okIIIdpn1OuIjzATH9G5ASEtubz+NuFny+30NY3eNpWlzRWl/oCKxx+gvN5Neb077GvbTPrQbfStQ9EWPUhbfj7SIv1IhTgTKDodliFDsAwZQtx116H6/bi2bmsabJRLY8Fa/LW11K9YQf2KFQDoExKwZ2dhyxmPPScbY58+nQ4+k+3JzOs/j3n9tanCZc4yrdqzNJ+CkgL21e1ja9VWtlZt5c0tbwaDz3HJ48hKyWJc8jgJPoUQQpwxJOgMw86dO9m2bRsLFiygqKiou5cjTiW9ASJTtON4mvuKOsrx1R5h/dfLGTu4J/rGqvarR32NWl/RGgfU7O/EWpr7irYarNSyz2jzn7Z46SsqhBBngOZ+pMkn0I/U4fGHTrRvtbW+7W0eap1e6t0+VBWcHj9Oj5/i2hPoR2oxHK0YbaoSjWoVlAYD0ha3RUg/UiFOGUWvxzpyBNaRI4i/8UZUr5fGoiKcuXk4ctfQuG49/ooK6pZ+St3STwEwJCdrg42ytYpPY48enb5eki2Juf3nMrf/XADKneUhFZ97a/cGg8+3tr6FgsLg2MHB4UaZyZkSfAohhOg2Z0TQefjwYX7961/z6aef4nQ6GThwIK+++iqZmZld8vhffvkljzzyCGvXrqW4uJgPP/yQSy65pM39nn32WR555BFKSkoYPXo0Tz/9NNnZ2cHP/+IXv+CRRx7h22+/7ZJ1iXNEi76iakx/jmyuY0zWXPTtTd8L6Sta0WLLfHk71aMV4K498b6ix60YTQBD+FVKQgghTh1FUYgwG4gwG+gZG965/oBKvaudKtLGFhWjztABTs3Vpc7mfqQuH/UuHwcJvx/pMatIW020b/l5i1HeoBMiHIrRiC0jA1tGBgm33UrA48G1caO2zT03l8aNG/GVllL70cfUfvQxAMaePYODjWzZORiTkzp9vURbInP6zWFOvzkAVDRWBPt7FpQWsKd2D9urt7O9ensw+BwUO4islCyykrWKzxhLzKn4UgghhBBtdHvQWV1dzaRJk5g6dSqffvopiYmJ7Ny5k9jY9n+7/+abb8jOzsbYKkTasmUL8fHxJCcntznH4XAwevRobrjhBi677LJ2H/f999/nnnvu4e9//zs5OTk8+eSTzJo1i+3bt5OUlMRHH33E4MGDGTx4sASd4sSdTF/R4GCl5irRVv1GW/cV7QxLdNvBSu1VjNoTwRxxcs9dCCHEKaXXKcTYTMTYTPSJD+9cjy9AnatlQOoJVo6226O0xd89fq0faZXDQ5XDE/a6TQZdyICmYBVpsO+o4WhA2io8NeqlH6kQOpMJW1YWtqws+OkdBBobaVy/HkduHs41a2gsKsJ76BC1hw5R+69/A2Dq2zc42MiWnY0hvvM/NBKsCczuN5vZ/WYDTcFn03CjgpICdtfuZkf1DnZU7+DtrW8DaMFnchYZiRk4Ao6u/yIIIYQQTbo96Pzf//1fevXqxauvvhq8rV+/fu3eNxAIsGjRIgYNGsR7772HXq9VAGzfvp1p06Zxzz338Ktf/arNeXPmzGHOnDnHXMfjjz/OzTffzI9//GMA/v73v7NkyRL+8Y9/8Jvf/IY1a9bw3nvv8cEHH9DQ0IDX6yUqKooHHnjgRJ+6EMcXdl/RyraDlTrqLRrwgatWOyp3dmIttrbhZ+shS80Vo9JXVAghziomg46ECDMJYfYjVVUVlzcQ0oO0bRVpaDha1+J+AVULWcvq3ZSdQD9Se3M/UptJC0St7QeirbfjR1oM6KQfqThH6axW7BMnYp84EQB/g4PGdWuDFZ+uLVvw7NuHZ98+at57HwDzoEHBwUa2rCz00Z3fep5gTWB239nM7ns0+FxbulYLPksL2FWzi53VO9lZvZN3tr0DwAdLPiArNYvM5EwyUzKJs8R18VdBCCHE+arbg86PP/6YWbNm8YMf/IAvvviCHj168JOf/ISbb765zX11Oh1Lly5l8uTJXHfddbz55pvs3buXadOmcckll7QbcnaGx+Nh7dq13HvvvSHXmjFjBt999x0Af/nLX/jLX/4CwGuvvUZRUVGHIeezzz7Ls88+i9/vP6H1CHFC9AZt0ntk26rmNlr0FW0bhLZTPeprBK9T6ynamb6iOuPRYUotByu1Vz0qfUWFEOKspSgKVpMeq0lPSnT4/Ugb3L5WE+w7nmrf8rZ6lw8Ah8ePw+PnSJj9SBUFoiyhQWhUOwOaQqpLbdrnbSa99CMVZxV9hJ2IyZOJmDwZAH9tLc6CAhy5uTjX5OLesQP3zp24d+6k+q23QFEwDxuKPWc8tpxsbJmZ6CM6v7MnwZrArL6zmNV3FgBVrirWlq4lvySf/OJ8dtXuCh7vbnsXgIExA4OhZ2ZyJvHWMMvShRBCiCbdHnTu2bOH559/nnvuuYf77ruP/Px87rzzTkwmEwsXLmxz/7S0NFatWsWFF17INddcw3fffceMGTN4/vnnT3gNFRUV+P3+Ntvek5OT2bZtW9iPt2jRIhYtWkRdXR3RYbwbKsRp06KvKIlDjn1fVQWPo/3BSm2qR8u1vqIBL9Qf0Y7jUrSws/VgpXYrRhOlr6gQQpwjFEUh0qJNe+/EvoUQPn+AepcvpFK0eaJ9Rz1Km29r9PpRVYKfD5ehuR9puwOaTKG32Y5+HCX9SMUZQh8dTeT06UROnw6Ar6oKZ14+jtw1OHPz8OzZg3vLVtxbtlL16qug12MZOQJ7dg628TnYxo5FZ7V2+npxljhm9pnJzD4z8Xq9fLD4A+JGx7G+fD35pfnsrN7Jrppd7KrZxXvb3wNgQPQALfRsCj4TrAmn5GshhBBnKkVROpwvI46t24POQCBAZmYmf/7znwHIyMigqKiIv//97+0GnQC9e/fmzTff5KKLLqJ///688sorp/Wd9euvv/60XUuIbqcoWn9OcwTE9T/+/b0ucFa0GqzUQcWosxJQtfs7Kzq3Hkt0O4OVOqgYlb6iQghxTjLodcTaTcTaTWGf6/b5g1voQ7bWt1NZenQ7vo/aRg9ev4ovoFLp8FB5Av1ILUZdiyDURFTrrfUhVaRHg1Rbt//GLs5lhrg4ombPImq2VoHpLSvDmZuHMy8Xx5pcvAcP4tpYiGtjIZUvvQRGI9b0dK2/Z04O1jGj0Zk7/0a0XWdneq/pzO6vbXWvdlWzrnQd+aX55Jfks6N6B7trd7O7djfvb9e21veP7k9mcmZwsrsEn0KI1o41hNrr9XL//fezdOlS9uzZQ3R0NDNmzODhhx8mLS0t+Bg7duzgl7/8Jd988w0ej4f09HQeeughpk6desxrq6rKY489xosvvsj+/ftJSEjgJz/5Cb/97W9P5VPutM8//5ypU6dSXV1NTExMdy/nlOv2X5tSU1MZPnx4yG3Dhg3jX//6V4fnlJaWcsstt7BgwQLy8/P52c9+xtNPP33Ca0hISECv11NaWtrmOikpKSf8uEKcl4wWiO6pHccT7Cta3omK0XKtUjTYV3RXJ9Zia3+wUnvVo9ZY6SsqhBDnAbNBT1KknqTI8LfaN3r9bUPRYA/SpgFOjb7Q6tKmUDWggssbwOV1U1oXfj9Ss17PX7d+SXRTn9GQqfatepC23I4faZZ+pCI8xqQkohfMJ3rBfAC8R44EBxs58vLwFRfTuHYtjWvXwnPPoZjNWDMysOdka8HnqFEorQbHHkusJZbpfaYzvY9WYVrjqmFt2drgZPcd1TvYU7uHPbV7+L8d/wdAv+h+R4PP5EwSbYld/4UQQpxVjjWE2ul0sm7dOn73u98xevRoqqurueuuu7j44ospKCgI3m/+/PkMGjSIVatWYbVaefLJJ5k/fz67d+8+ZjZ01113sXz5ch599FFGjRpFVVUVVVVVp+y5dhdVVfH7/RgM3R4lHlO3r27SpEls37495LYdO3bQp0+fdu9fUVHB9OnTGTZsGB988AE7duxgypQpmM1mHn300RNag8lkYty4caxcuTKY+AcCAVauXMkdd9xxQo8phOiEcPqKqio0Vh97yFLL27zOE+sr2nqwUnvVo/YE6SsqhBDnGUVRsJkM2EwGUqM7v20XIBBQqXf7Otha7wkJTFt+vq7RS71b60fq9iscrnFxuCa8fqQ6haOVo03Voq2D0ubq0pDbbEasRulHKsCYlkbMpZcQc+klqKqK9+BBHGu0be6O3Fz8FRU416zBuWYNAIrNhm3sWG2wUU4OluHDUfSd/70pxhLD9N7Tmd5bCz5r3bXBHp9rS9eyrWobe2v3srd2Lx/s+ACAvlF9yUzJJCtZq/hMsiV1/RdCCHFGO9YQ6ujoaFasWBFy2zPPPEN2djYHDhygd+/eVFRUsHPnTl555RXS09MBePjhh3nuuecoKirqMOjcunUrzz//PEVFRQwZorWF62jAdmv/+Mc/eOyxx9i1axdxcXFcfvnlPPPMM23u115F5oYNG8jIyGDv3r307duX/fv3c8cdd/D111/j8Xjo27cvjzzyCMOHDw9WpMbGxgKwcOFCXnvtNQKBAA8//DAvvPACZWVlDB48mN/97ndcccUVIdddunQp999/P5s2bWL58uVMmTKlU8+vu3R70Pmzn/2MiRMn8uc//5krr7ySvLw8XnzxRV588cU29w0EAsyZM4c+ffrw/vvvYzAYGD58OCtWrGDatGn06NGDn/3sZ23Oa2hoYNeuo9Vfe/fuZcOGDcTFxdG7d28A7rnnHhYuXEhmZibZ2dk8+eSTOByO4BR2IUQ3U5QWfUUHH//+7ob2Byu1Vz3qOpm+oi0GK3VUMSp9RYUQ4ryma+7raT2xfqSV9Y18vOwzxmRPxOFV2x3aVONssR2/KTx1eQMEVKhxap/vxNt+IYz6o+vWwk9TyN/bBKa2o0Gq2SBvCJ6LFEXB1Ls3pt69ib3ySlRVxbNnT3CwkTMvD39NDY6vv8bx9dcA6CIisGVlYcvJxjxunDYUMwzR5mim9Z7GtN7TAC34bN7qXlBSwLaqbeyr28e+un38c8c/AegT1Sek4jPZ3ok31YUQbaiqitrY2C3XVqzWU/pmW21tLYqiBIPD+Ph4hgwZwhtvvMHYsWMxm8288MILJCUlMW7cuA4f55NPPqF///4sXryY2bNno6oqM2bM4K9//StxcXEdntc8q+bhhx9mzpw51NbW8s0335zw81m0aBEej4cvv/wSu93Oli1biIiIoFevXvzrX//i8ssvZ/v27URFRWFt6rP8l7/8hbfeeovHH3+c0aNH8/XXX/OjH/2IxMRELrroouBj/+Y3v+HRRx+lf//+wbD0TNbtQWdWVhYffvgh9957L3/84x/p168fTz75JNdee22b++p0Ov785z9z4YUXYjId7ck0evRoPvvsMxIT29+yUFBQENJT4Z577gGOptgAP/zhDykvL+eBBx6gpKSEMWPGsGzZsjYDioQQZ4lw+or63C0qQo9TMXoifUXN0S0qQo9VMZoIpgjZQi+EECLIoNcRZzeRZIUxvWIwhrEl2OX1h2yhP7rNvnnbvadFZenRKtIapxdfQMXrV6lo8FDREH4/UqtR3yb8bD3VPqpVeNpccaqXrfZnDUVRMA8YgHnAAOKuuQY1EMC9c6e2zT03D2d+PoH6ehpWr6Zh9WoABthsFK9cScSECdhzcjANGBBWmBFtjmZq76lM7a29vqvz1LGudJ221b00n21V29hft5/9dfv5106tHVrvyN7B/p6ZyZmk2KU9mRCdoTY2sn1sxyHfqTRk3VoUm+2UPLbL5eLXv/41V199NVFRUYD28+yzzz7jkksuITIyEp1OR1JSEsuWLTtmuLdnzx7279/PBx98wBtvvIHf7+dnP/sZV1xxBatWrerwvD/96U/8/Oc/56677grelpWVdcLP6cCBA1x++eWMGjUKgP79j74Obg5ck5KSgsGu2+3mz3/+M8uXL2fEiBFERUUxcOBAvv76a1544YWQoPOPf/wjM2fOPOG1nW7dHnSC1gdh/vz5nbpvR1/cjIyMDs+ZMmUKqqoe97HvuOMO2aouxPnIYO58X9GAXws7Ww9WarditKmvqLtWOzrTV9RgPf6QpeZKUekrKoQQ4hgsRj0Wo56kqPD7kTo9/pBwtLbR06aStLn/aMh2e5dX6zbj9dPo9VNSF95We4BIs+Fo39GWvUhb9SANqS61af1IZat991J0OixDhmAZMoS4hQtR/X5cW7Zqg41yc3HmF6B3OnF8thLHZysB0CckYM/W+nvac7Ix9ukT1n/HKFMUU3pNYUqvKYAWfK4vXU9Bqdbjc2vVVg7UH+BA/YFg8Nkrslew2jMrJUuCTyHOI16vlyubKtKff/754O2qqrJo0SKSkpL46quvsFqtvPzyy8HZMKmpqYwYMYL9+7X9ERdeeCGffvopgUAAt9vNG2+8weDB2s7DV155hXHjxrF9+3asVmvIXJr77ruPm266iSNHjjB9+vQue1533nknt99+O8uXL2fGjBlcfvnlwS347dm1axdOp5NZs2aF3O7xeNrka5mZmV22ztPhjAg6hRDirKHTa0FjRCd6P6kquGraGazUQcWo1wm+Rqg5oB3HXYvhGMOWWlWP2uK1nqhCCCHEcSiKgt1swG420CPmBPqRunytBjSF9h2tCRng5Gu6zYPD4weg3u2j3u3jUHV42yX1OoUoi6Ep+DQRYzWS3jOaeempDEmOlBC0Gyh6PdZRI7GOGkn8jTficTpZ/fLLjNHrcRUU0LhuPf6KCuqWLqVu6VIADCkpTYONxmvBZ48eYV0zyhTFRb0u4qJeWjVSvaee9WXrg8ONtlRt4WD9QQ7WH+TfO/8NQM+InsGKz6zkLFIjUrv2CyHEWUqxWhmybm23XburNYec+/fvZ9WqVcFqToBVq1axePFiqqurg7c/99xzrFixgtdff53f/OY3LF26FK/XCxDc/p2amorBYAiGnKAN2AatynLq1Kls2LAh+Lm4uLiwdmeAtrsZCCnga15Hs5tuuolZs2axZMkSli9fzl/+8hcee+wxfvrTn7b7mA0NDYC29T46OpqIiIjgdczm0NZrdrs9rPV2N3nVK4QQp4qiaFWX1tgw+oqWtx2s1F71qKsWAj6oL9aO4y9G62/aerBSRxWj6E722QshhDgP6XSKVo1pM9Kb8LYcev0BLfRsNdG+zaT7pgC1ZWWpxxfAH1Cpdnqpdnqh0gnAFzvKeXrVLgYmRTBvVCoLRqcyMCnyVDx10QmK0YirTx/i5s7FuGgRAY+Hxg0bcObm4czNxblxI76SEmo/+pjajz4GwNizJ7bxOdhzcrBl52BMDm/QUKQpksk9JzO552QAGjwNrC9bH+zxuaVyC4caDnFo1yE+3PUhAD0ieoRUfKZFpHXtF0KIs4SiKKds+/jp1hxy7ty5k9WrVxMfHx/yeadT+3ejOexrptPpCDT1Fm5vaPakSZPw+Xzs3r2bAQMGANqA7eb7GwwGBg4c2Oa8vn37snLlypA2ix1pbtNYXFwc3EbfMjxt1qtXL2677TZuu+027r33Xl566SV++tOfBls/+v3+4H2HDx+O2WzmwIEDfP/73ycqKqrNcz9bSdAphBBnimBf0U5M6fO5W1SHthis1F71qLMS1ID2p7MSyrce9+EN5iimY0Vf8dzRCtaOKkalr6gQQoguYNTriI8wEx8R/hA/l9ffJhAtq3exels5X+4oZ1dZA39buZO/rdzJkORI5qWnMj89lf6JEafgmYjO0plM2LOzsWdnw0/vINDYSOP69TjW5OLMzaWxqAjvoUPU/vMQtf/Utp2b+vXDlpONffx4bNnZGI4x7KM9EaYILux5IRf2vBAAh9ehBZ8lWvC5uXIzhxsOc3jXYf6z6z+AFnyOSx5HVkoWWSlZ9IgIr8pUCHHqHWsIdWpqKldccQXr1q1j8eLF+P1+SkpKAK3C0mQyMWHCBGJjY1m4cCEPPPAAVquVl156ib179zJv3rwOrztjxgzGjh3LDTfcwJNPPkkgEGDRokXMnDkzpMqztQcffJDbbruNpKQk5syZQ319Pd988027FZgDBw6kV69ePPjgg/y///f/2LFjB4899ljIfe6++27mzJnD4MGDqa6uZvXq1cHK0j5NLUEWL17M3LlzsVqtREZG8otf/IKf//znOJ1OZsyYEVxDVFQUCxcuDOvrfyaRoFMIIc5GBjNE99CO42nuK9p6sFJHFaMBL4q7jgjq4GBpJ9ZibX+wUpvq0SSwxMA58k6hEEKIM0dzP9LkVv1Ir83pQ53Ly4rNpSzZVMxXO8vZXlrP9hX1PL5iB8NSo5ifnsq8Uan0TTi7tuadi3RWK/aJE7FPnAiAv8FB49oCbbBRbi6uLVvw7N2LZ+9eat57HwDzoEHYxmvb3G1ZWeijo8O6pt1o54IeF3BBjwsALfjcULaB/JJ88kvz2VKxRQs+Gw7z8W6tyjTNnhYcbNQcfEprBCG617GGUD/44IN8/LH2/98xY8aEnLd69WqmTJlCQkICy5Yt47e//S3Tpk3D6/UyYsQIPvroI0aPHt3hdXU6HZ988gk//elPmTx5Mna7nTlz5rQJIltbuHAhLpeLJ554gl/84hckJCRwxRVXtHtfo9HIu+++y+233056ejpZWVn86U9/4gc/+EHwPn6/n0WLFnHo0CGioqKYPXs2TzzxBAA9evTgD3/4A7/5zW/48Y9/zHXXXcdrr73GQw89REJCAk888QR33XUXMTExjB07lvvuu++Yaz/TKWpnpvSIE1JXV0d0dDS1tbUhvR/Eucvr9bJ06VLmzp0bdt8NIc4ITX1FvTXF5K76hPGjBmBorGpbMdr8sdcR3uMH+4q2GqzUss9o85+2BOkrKk6K/EwW5wr5Xu46tU4v/91SwpLCYr7ZVYEvcPSl0MgeUcwblcb89FR6xZ0bWzXPNCf7veyvrcVZUBCs+HQ3bQ8NUhQsw4Zpg43G52Adl4k+4uQCbKfXqQWfTVvdiyqK8Km+kPuk2lODoWdmSiY9I3pK8HmOO1d/LrtcLvbu3Uu/fv2wWMIbZCfOToFAgLq6ujNi6/qxvv/CydfkFaQQQoijmvuKGiKojBiKOmwuHOuXN4/j+EOWmv901Zx4X9GWg5U6qhg1yi9jQgghji3aZuTKzF5cmdmLaoeH/24uYcmmYr7dXUnR4TqKDtfxv8u2MbppiNG89LSwBzKJU0cfHU3k9OlENk0q9lVV4czL0ya65+bh2bMH15YtuLZsoerVV0GvxzJyBPac8dhysrGNHYsuzAEnNqONiT0mMrGHVmXq9DrZUL6BgpICCkoL2FSxiWJHMZ/s+YRP9nwCQLItObjNPSs5i56REnwKIcTpIkGnEEKIE2eyaz1FO9VX1NMUgrYarNRQfvT25orRNn1FO7EWc1SLitDjVIxKX1EhhDjvxdpNXJXdm6uye1PZ4GbZZq3Sc82eSjYeqmXjoVr+vHQbGb1jmDcqlXnpqaRGS+h5JjHExRE1ezZRs2cD4C0tawo+1+DMzcN78CCujYW4NhZS+eKLYDRiHZ2OPTsH2/gcrGPGoGsa0tFZNqONiWkTmZh2NPjcWL6R/JJ81paupbCikFJnKYv3LGbxnsUAJNmSgqFnVkoWvSJ7SfAphBCniASdQgghTg+DKcy+olVtByt1VDEa8IK7TjuqdndiLZZW1aHtDFlqDkqlr6gQQpzz4iPMXJvTh2tz+lBW7+K/RSUsLiwmb18V6w/UsP5ADX9aspXMPrHMS09l7qjUNv1ARfczJicRvWA+0QvmA+A9fDjY39ORm4uvpITGgrU0FqyF555DMZuxZmRgH69NdLeOGokS5jZkm9HGhLQJTEibAECjr5GN5RspKCkgvySfwopCypxlLNmzhCV7lgCQZE0iMyUzONm9T1QfCT6FEKKLSNAphBDizKPTayFkRCIkjzj2fVUVXLXtD1Zqr2LU6wCfC2oPaMdx12LQ+oWGDFZKbFEx2qJ6VPqKCiHEWS8p0sL/TOjL/0zoS2mdi083FbNkUzH5+6op2K8df1y8hay+ccxPT2XOyFQSI8OfFC9OPWOPHsRcdikxl12Kqqp4DxwIbnN35Obir6jAuWYNzjVrAFBsNmzjxmmDjXLGYxk+DEWvD+uaVoOV8anjGZ86HtCCz8LyQgpKm4LP8kLKGstYuncpS/cuBSDRmhgSfPaN6ivBpxBCnCB5NSaEEOLspihgjdGOhEHHv7/H0f5gpfaqR5v7ijaUaEdnWOPaDlYKqRhtEZRKX1EhhDijJUdZuH5SP66f1I/i2kaWbiphSeER1h2oIW9vFXl7q3jw483k9ItnXnoqc0amEB8hoeeZSFEUTH36YOrTh9grr0RVVTx79uBYo21zd+bl4a+pwfHVVzi++goAXWQktsxMreIzJwfz4MEoYe7ysBqs5KTmkJOaA4DL52oTfJY3lvPp3k/5dO+nACRYE8hK1gYbZaZk0i+qnwSf4rSSmdWiO3TV950EnUIIIc4vJrt2xPY9/n19HnBWtB2s1F71qLNC6yvaWKUd5duO//jmqBYVoR1VjDYFpeZI6SsqhBDdKDXayo0X9OPGC/pxuKaRpYXFLN5UzMaDNXy3p5Lv9lTy+483M6G/FnrOHpFCrD28/o/i9FEUBfOAAZgHDCDu2mtRAwHcO3Y0bXPPw5mfT6C+nobVq2lYvRoAfUwMtuxsbDnZ2MePx9S/f9gBpMVgITs1m+zUbADcfrcWfJYUkF+az8ayjVQ0VvDpvk/5dJ8WfMZb4oPVnlkpWfSLluBTnBr6pgpmj8eDNczBXUKcLI/HAxz9PjxREnQKIYQQHTGYICpNO44n2Fe01WCljipG/Z4WfUX3dGItllY9RNurGG0KSK2x0ldUCCFOoR4xVm6e3J+bJ/fnYJWTJZuKWVJYzKbDtXy9q4Kvd1Vw/3+KmDQwgfnpqcwankK0Lbzej+L0UnQ6LEOHYhk6lLiFC1H9flxbtuLMXaMFn2vX4q+poX75cuqXLwdAn5iAPSsb2/gc7Dk5GHv3DjuANOvNwQntt3M7br+bTeWbyC/Np6CkgI3lG6l0VbJs3zKW7VsGQJwlLhh6ZqVk0T86/MBViPYYDAZsNhvl5eUYjUZ08vvkOS8QCODxeHC5XN363zsQCFBeXo7NZsNgOLmoUoJOIYQQoiu07CvK8GPft2Vf0daDldqrGPU0nFhf0ZDq0MSjQWjL6lF7AujlxbcQQpyoXnE2brtoALddNID9lQ4WF2qh55biOr7cUc6XO8r5rX4TFwxMYH56GjNHJBNlkZ+7ZzpFr8c6aiTWUSOJv+kmVK+Xxk1FOPO0wUaN69bjL6+gbulS6pZqvTYNKSnYc7Rt7vacbIw9OjGAsRWz3hzcss5o8Pg9bKrYRH6JFnxuKN9AlauK5fuXs3y/FrjGWeIYlzwuONl9QMwACT7FCVEUhdTUVPbu3cv+/fu7ezniNFBVlcbGRqxWa7f/3NDpdPQ+gTeMWpOgUwghhDjdwu4r6jz+kKXm6tHG6tC+oqWdWE/LvqId9RZtDkqNso1JCCE60ifezqKpA1k0dSB7yhtYUqgNMtpWUs/q7eWs3l6O6d86Jg9OYF56KjOGJRMpoedZQTEasY3NwDY2g4TbbiPgdtO4cSPONbk48nJp3FiIr6SE2o8+ovajjwAw9uqlbXPPGY8tJxtjUlLY1zXpTYxLHse45HHB4LOookgLPksL2FCmBZ8r9q9gxf4VAMSaY7WwtKnqc0DMAHSKVOaJzjGZTAwaNCi4jVic27xeL19++SWTJ0/GaOzef49MJlOXVJVK0CmEEEKc6Uw2MPU9gb6iLQYrtVc9eiJ9RU2RLSpCE9oGoS17i0pfUSHEeax/YgQ/nT6In04fxK6yehYXFrO4sJhdZQ18trWMz7aWYTLomDI4MRh62s3y8uxsoTObsWdnY8/OJpGfEmhsxLlunTbYKDeXxqIivAcPUnvwILX//BcApn79gtvcbdnZGOLiwr6uSW9ibPJYxiaP5VZuxev3UlRZFFLxWe2uDgk+Y8wxZCZnBie7D4wZKMGnOCadTofFIkMzzwd6vR6fz4fFYun2oLOryL+kQgghxLkkrL6iTSFn68FKHVWM+j3gqYeq+jD6irasDE1oO2SpOSiVvqJCiHPYwKRI7p4Ryd0zBrO9pJ4lhUdYXFjMngoHy7eUsnxLKWaDjmlDk5iXnsq0oUnYTPJS7Wyis1qJmDSJiEmTAPA3NNC4di2ONbk4c3Nxbd2KZ+9ePHv3UvPuewCYBw/WtrmPz8GWmYk+Ojrs6xr1RjKSMshIyuCW9Fvw+r1srtwcrPhcX7aeGncNnx34jM8OfAZAtDk6WO2ZmZzJoNhBEnwKIc4Z8q+nEEIIcb7S6ZrCxwQ61VfUXddqsFJ5B71Fy1v0FT2oHcej6FtNoG8VhLbsNyp9RYUQZ7EhKZEMSRnCz2YOZmtxPUs2aaHn/konnxaV8GlRCVajnmnDkpg/KpWpQ5OwGE9uAq04/fQREURcdBERF10EgL+2Fmd+vjbYKDcX944dwaP6zTdBUbAMG4Zt/HjsOdlYx2Wij7CHfV2j3siYpDGMSRrDzdyMN+Blc8VmCkoLKCgpYF3ZOmrdtaw8sJKVB1YCWvA5Lknr8ZmZksng2MESfAohzloSdAohhBDi+BQFLNHakTDw+Pf3ONsZrNRyAn2LitHGalD90FCqHZ3qKxrbTnVo663zsegC0l9KCHFmUhSF4WlRDE+L4hffG8LmI3XaIKNNRzhY1aj19ywsxmbSM2NYMvPSU7locKKEnmcpfXQ0kTNmEDljBgC+qiqceXk41qzBmZuHZ+9eXFu24Nqyhap//AP0eqwjRwYrPq0ZGeis4ffJNuqOBp83jboJb8DLlsotFJQUkF+az/rS9dS6a1l1cBWrDq4CIMoUxbjkccGqz8Gxg9Hr5PtOCHF2kKBTCCGEEF3PZANTH4jtc/z7NvcVbRmEtplGX3F0e73q18LRxmqo2N7hwxqBBYC69e7QIUvtbZ1vDkrNUdJXVAhx2imKwsge0YzsEc2vZw+h8FAtSzZpQefhmkY+3niEjzceIcJsYObwZOaNSuXCwQmYDRI+na0McXFEzZ5N1OzZAHhLy4IT3Z25eXgPHqRx40YaN26k8sUXwWjEOjo9ONjIOmYMOpMp7OsadUZGJ45mdOJobhx1I76Aj62VW8kvzSe/JJ91peuo89Sx+uBqVh9cDUCkKTIk+BwSO0SCTyHEGUuCTiGEEEJ0rxPpK9omCG09gb4CtaEMxe9G8TRoW+mr9x7/8fXmVv1EE1sEoa0CUmuc9BUVQnQ5RVEY3SuG0b1iuHfOUNYfrAlWd5bUufhw/WE+XH+YSIuB7w1PYX56KpMGJmAyyM+js5kxOYnoBQuIXrAAAO/hw03b3NfgyM3DV1JCY8FaGgvWwrPPopjNWMdmaIONcnKwjhyJcgKDRAw6A6MSRzEqcRQ3jLwBX8DHtqpt5Jc0BZ9l66j31PP5wc/5/ODnAEQaIxmbPDa41X1o7FAJPoUQZwwJOoUQQghx9mjZVzRp2DHv6vN4WL74X3xv4hiM7ppjT6B3VGiDlvzuE+grmthxpWjLfqPSV1QIESZFURjbO5axvWP57dxhrDtQzeLCYpZuKqas3s2/1h3iX+sOEW01MmtEMvPT05gwIB6jXkLPs52xRw9iLruUmMsuRVVVvAcOBAcbOfLy8FdU4PxuDc7v1gCg2GzYxo3TBhtl52AZPgxFH374aNAZGJkwkpEJI/nxyB/jC/jYXrVdCz5LtYrPem89Xxz6gi8OfQFAhDFCCz6Ts7SKz7ghGHQSNQghuof89BFCCCHEuUlR8OltED8QOlPl0rKvaMveog3t3Na6r2hnNPcVDekn2hyEtuo3arKd3HMXQpxzdDqFzL5xZPaN44H5w8nfV8WSTcUs3VRCRYOb/ys4xP8VHCLWZmT2yBTmp6eR0y8Og4SeZz1FUTD16YOpTx9if3glqqri2b1b2+a+JhdnXh7+2locX32F46uvANBFRmLLysKek40tJwfz4MEoJ7ALwaAzMCJhBCMSRnD9yOvxB/xsq96m9fgsORp8fnnoS7489CWgBZ8ZSRlkpWjB59C4oRJ8CiFOG/lpI4QQQggB4fUV9Xtb9A0tbzWNvqLNNvrO9hU9upaIDqpE26kYlb6iQpx3dDqFnP7x5PSP5/cLRpC3t4rFhUdYVlRCpcPDu3kHeTfvIPF2UzD0zO4Xh14nPyvOBYqiYB44EPPAgcRdey1qIIB7x47gYCNnfj6B+noaVq2iYZU2YEgfE4MtOxvb+BzsOTmY+vdHOYF/O/Q6PSPiRzAifgQLRyzEH/CzvVqr+CwoLWBt6VrqPfV8dfgrvjqsha52o/1o8JmcxbD4YRJ8CiFOGfnpIoQQQggRLr0RolK143gCAS3gbHfrfFMQ2nIbvd+t9RQNp69oyyrRNhPoE45+LH1FhTjn6HUKEwbEM2FAPH+4eAS5rULPt3MP8HbuARIizMwdpYWemX1i0Unoec5QdDosQ4diGTqU+OuvR/X5cG3dqm1zz83DuXYt/poa6pcvp375cgD0iQnYs3Ow5WRjz8nB2Lv3CQefw+OHMzx+eDD43FG9Ixh8FpQWUO+p5+vDX/P14a8BsBlsZCRnkJWs9fgcHj8co07auwghuoYEnUIIIYQQp5JOB/Z47ThOX1FUFdz1rYLQY1SMNvcVrTukHcej6MCW0H6laHu3SV9RIc4qBr2OSQMTmDQwgT9+fyTf7q5kSeER/ru5lIoGN298t583vttPcpSZOSNTWTA6lYxeEnqeaxSDAeuoUVhHjSL+pptQvV4aNxUFBxs1rl+Pv7yCuiVLqFuyBABDair27Gxs48djz8nGmNaJAYHt0Ov0DIsfxrD4YVw34jr8AT87a3YGt7oXlBZQ56njm8Pf8M3hbwCwGqyMTRpLZkommcmZjEgYIcGnEOKESdAphBBCCHGmUBSwRGlH/IDj39/b2E4Q2mrYUvPHjVWgBprC07LOrccS085gpdbT6Js+lr6iQpxRjHodFw1O5KLBifzpkgDf7KpgcWExy7eUUFrn5rVv9/Hat/tIjbYwd1Qq89NTGdMr5oSq+sSZTTEasY3NwDY2g4TbbyfgdtO4YWPTYKNcGjcW4isupvajj6j96CMAjL16BQcb2XKyMSYlndC19To9Q+OGMjRuKD8a/iMCaoCd1TspKD0afNa6a/nmyDd8c+Ro8JmRlEFmciZZKVmMiB+BUd54E0J0kgSdQgghhBBnK6MVYnprx/EE+4qWt+ohWt7Ox+VaX1FXjXZU7Dj+45siQrfJH6ti1BItfUWFOI1MBh1ThyYxdWgSbt9IvtpRwZJNxazYUkpxrYtXvt7LK1/vpUeMlXnpWug5qke0hJ7nKJ3ZjD0nG3tONon8lIDTiXP9epxrtODTVbQZ78GD1Bw8SM0H/wTA1L9/0zb38dhysjHExp7YtRUdQ+KGMCRuCNcOu5aAGmBXzS4t9CzRtrrXuGv49si3fHvkW0ALPsckjiEzRQs+R8aPlOBTCNEhCTqFEEIIIc4HJ9RX9BiVoi230ftcLfqK7uvEWkxNFaEJx6gYbfrYFgc6/Uk/fSGExmzQM2N4MjOGJ+Py+vliRzlLCov5bGsph2saefHLPbz45R56xVmZNyqN+empjEiLktDzHKaz2YiYNImISZMA8Dc04Cwo0AYb5ebi2roVz549ePbsoebd9wAwDx4cHGxky8pCHxV1YtdWdAyOHczg2MHB4HN3ze6jPT5LCqh2V/Nd8Xd8V/wdABa9hTFJY4IVnyMTRmLSm7rmiyGEOOtJ0CmEEEIIIUK17CvK0GPft2Vf0Za9RZsrQ1tPoHfXgd8Tfl/RYw1Zaq4UtSeCQV7sCtFZFqOeWSNSmDUiBZfXz+ptZSzeVMyqrWUcrGrk71/s5u9f7KZvvK2p0jONoSmREnqe4/QREUROmULklCkA+GtqcBYU4FiTizM3F/fOnbh37MC9YwfVb7wJioJl+HBsOTnYx+dgHTsOfYT9hK6tU3QMih3EoNhBXDPsGgJqgD01e8gvzSe/JJ+1pWupclWxpngNa4rXAFrwOTpxdLDic1TCKAk+hTiPSdAphBBCCCFO3En1FT1OxWjrvqKdaS0a7Cvacut8B9WjphN7IS7Euchi1DNnVCpzRqXi9PhYta2MJYXFrNpWxr5KJ8+u3s2zq3fTP9HO/FGpzB+dxuDkyO5etjgN9DExRM6YQeSMGQD4Kitx5uXhyM3FmZuHZ+9eXJs349q8map//AP0eqwjRwYHG1kzMtBZrSd0bZ2iY2DsQAbGDuTqoVejqip7aveQX5IfrPqsclWRW5JLbkkuAGa9+WjwmZzFqMRRmPXmLvt6CCHObBJ0CiGEEEKI0yesvqI+cFa0mEBfcYyK0RPoK2q0tz9Yqb2KUekrKs4jNpOB+elpzE9Pw+H28dnWUpYUFvP5jnL2lDt4atUunlq1i0FJEcxPT2NeeioDkyK6e9niNDHExxM1Zw5Rc+YA4C0t1YLPNWtw5ubhPXSIxo0bady4kcoXXkAxGrGOHh2s+LSMHo3OdGIVl4qiMCBmAANiBnDV0KtQVZW9tXuDoWd+ST6VrkrySvLIK8kDwKQzMTppNFnJWWSmZJKemC7BpxDnMAk6hRBCCCHEmUlvgMgU7TieQEALOIOhaOtp9BWh2+h9LvA6oNoRRl/RxLaDldqrGJW+ouIcYjcb+P6YHnx/TA/qXd5g6PnFjnJ2ljXwxGc7eOKzHQxNiWTeqFTmpafSP1FCz/OJMTmZ6AULiF6wAADPocM4c3Nx5uXiWJOLr7RU6/lZUEDFs8+iWCxYM8YEBxtZR45EMZ7YcCFFUegf05/+Mf354dAfasFn3V5tsFFJAfml+VQ0VgQrQNmoBZ/pielkpWSRmawFnxaDpSu/JEKIbiRBpxBCCCGEOPvpdFrAaIujU31FPQ2tBiu16jEarB4tb9FX9LB2HI+iA1t8i23yrSpGW38sfUXFWSLSYuTSjJ5cmtGT2kYvK7aUsqTwCF/trGBbST3bSup5bMUOhqdGBae394mXFhHnG1PPHph6XkbM5Zehqire/ftx5ObhzF2DIzcPf2Ulzu/W4PxO67Gps9mwZo7TBhvljMcybCiK/sTeLFIUhf7R/ekf3Z8rh1yJqqrsq9sXrPYsKCmgvLFcG3RUWgCAUWcMCT5HJ46W4FOIs5gEnUIIIYQQ4vyiKGCO1I5O9RV1tTNYqYOKUWdzX9Gm4LRTfUWj2w5Wav44IgnFHIvNXQYeBxhjTvbZC9Eloq1GrhjXkyvG9aTG6WH55lIWbyrmm10VbCmuY0txHY/8dzujekQzLz2VeaNSSYk8sao9cfZSFAVT376Y+vYl9oda8OjZvTs42MiZl4e/thbHl1/h+PIrAHRRUdgyM7GPz8GWk4N50CAUne6Er98vuh/9ovvxg8E/QFVV9tftDwk+yxrLWFu6lrWlawEt+ByVMCo43Gh04mishhPrMSqEOP0k6BRCCCGEEOJYjBaI6aUdx9PcV7T1YKX2KkYd5RDwgatWOyp3tvuQBmAmwJZfaH1F2xus1F71qCVG+oqK0yLGZuLKrF5cmdWLKoeH/24uYUlhMd/urmDT4Vo2Ha7l4U+3kd4zin56hTE1jfRJlNDzfKQoCuaBAzEPHEjcj65FDQRwb98eHGzkzM8nUFdHw6pVNKxaBYA+NhZbdja2nGzs48dj6tcP5QR/timKQt/ovvSN7ssVg69AVVUO1h/UtrY3TXYvc5axrmwd68rW8WLhixh0BtIT0hmXPI6slCzGJI3BIFGKEGcsRVVVtbsXca6qq6sjOjqa2tpaoqKiuns54jTwer0sXbqUuXPnYjzBPjNCnAnke1mcC+T7WJzxQvqKthqs1KJ6VG0ox19XgkH1hPf4wb6iCcesGMWeqG21l76iootVNLhZVqSFnrl7Kwm0eOU5tncM89LTtErPaNkmLDSqz4dr69bgYCPn2rWojY0h99EnJmDPzsE2Pgd7Tg7GXr1OOPhsc31V5VD9oWDomV+ST6mzNOQ+Bp2BkfEjia6L5ocTf8i41HHYjLYuub4Qp9vZ8vtyOPmaBJ2nkASd55+z5YeEEMcj38viXCDfx+Jc4fV6WbpkCXNnTMborm4xWKl1pWj50W307trwLtKyr+gxK0abDoNMLBbhKat3sWTjYd76Yit7GhRavgrN6hvLvFGpzB2VSlKUhJ7iKNXjobGoCGduLo7cPBrXrUP1hL7pY0hNbervmYM9JxtjWlrXXV9VOdRwSBts1FT1WeIoCb2+YmBEwgiyUrLIStYqPiX4FGeLs+X35XDyNam3FkIIIYQQ4kzX3Fc0Ii78vqLBwUrtV4y26SvaGc19RVsOVopIar961CwTuAUkRVr4UU5v4iqLGHfBNFZsq2BJYTEF+6vJ36cdf1i8hey+ccxPT2X2yFQSIyVQP98pJhO2sWOxjR1Lwu23E3C7adywMTjYqLGwEF9xMbX/+Q+1//kPAMbevbHnZGPLGY89JxtDYuKJX19R6BXZi16Rvbh00KWoqsrhhsOsObyGj9Z9RImxhBJnCRvLN7KxfCMvb3oZg2JgeMJwspKzyErJIiMpQ4JPIU4jCTqFEEIIIYQ414TdV7Sy7WCl9ipGO9lXNHQtttBt8sGP26kelb6i54XkKAs/ntSPH0/qR3FtI0sKi1myqZj1B2rI3VtF7t4qfv/xZsb3j2deeiqzR6QQHyGhpwCd2Yw9Jxt7TjaJQMDpxLluvVbxmZeLa1MR3gMHqDlwgJoP/gmAqX9/bbBRdg62nGwMsbEnfH1FUegZ2ZPvD/g+xu1G5s6dS5m7LLjNfW3pWg43HKawvJDC8kJeKXoFvaJnRPyI4HCjjKQM7EZ7F31FhBCtSdAphBBCCCHE+UxvgMhk7Tie5r6irQcrdVQx6msErxNq9mvH8eiMoVWix6oYlb6i54TUaCs3Xdifmy7sz6FqJ0s3FbOksJiNh2r5dncl3+6u5IGPNjNxQDzz01P53vAUYu2m7l62OEPobDYiLphExAWTAPA3NOAsKMC5Rgs+3Vu34dmzB8+ePVS/8y4A5iFDgoONbJmZ6E+yzVyPiB70GNiDSwZeAsDhhsMUlBQEJ7sfbjhMYUUhhRWF/KPoH+gVPcPjh5OZkklmciZjk8YSYZLKdyG6igSdQgghhBBCiM7R6cAWpx2JQ459X1UFjyN0m7yj/OjHIdWj5Vpf0YAX6o9ox3EpWth5rCFLLT+WvqJnvJ6xNm6ZPIBbJg/gYJWTxYXFLNl0hKLDdXy1s4Kvdlbw2w+LmDQwIRh6RtvO3J5y4vTTR0QQOWUKkVOmAOCvqcGRn68NNsrNxb1zJ+7t23Fv3071G2+CTodl2LDgYCPr2HHoI06u2rI5+Pz+wO8DcKThCAWlBcE+n4caDrGpYhObKjbxatGr6BQdw+OGk5WSRWZKJhlJGUSaIk/2SyHEeUuCTiGEEEIIIUTXUxStP6c5AuL6H//+Xhc4K1oNVipvv3rUWQmo2v2dFZ1bjzm6RXVo6yFLrapHpa9ot+sVZ+P2KQO4fcoA9lU4WLKpmMWFxWwtruOLHeV8saOc+/SbuHBQIvPTU5kxPJkoi4SeIpQ+JoaomTOJmjkTAF9lJc68PBxrcnHm5uLZtw/X5s24Nm+m6pV/gMGAdeRIbbDR+BysGRnoLCc3ICstIo2LIy7m4gEXA1DiKCG/JD9Y8Xmw/iBFlUUUVRbx6mYt+BwWN0wLPpMzGZs8VoJPIcIgQacQQgghhBCi+xktEN1TO44n2Fe0vBMVo+Vapai7Vjsqd3ViLba2g5Vi+8KYazu3xV90qb4JdhZNHciiqQPZXd6g9fQsLGZ7aT2rtpWxalsZJr2OyYMTWTA6lenDkokwy0td0ZYhPp6oOXOImjMHAG9padNE91ycuXl4Dx2iccMGGjdsoPKFF1CMRqyjR2Mbrw02sowejc50cq0TUuwpLBiwgAUDFgBa8Nmy4vNA/QE2V25mc+VmXtv8GjpFx9C4oWQmaz0+xyaPJcp0ctvthTiXyU9/IYQQQgghxNklnL6iqgqN1W0HK3U0bMnrbOorekA7Wvr8YRhzNUy8E+IHnJrnJo5pQGIEd04fxJ3TB7GztJ7FhcUsLjzC7nIHn20t5bOtpZgMOqYOSWR+ehrThiZhl9BTdMCYnEz0xRcTfbFWbek5dLgp+FyDMzcPX2mp1vOzoICKZ0CxWLCNzcCcmYnF50f1esF4cpXEKfYU5vefz/z+8wEodZQGqz0LSgvYX7efLZVb2FK5hTe2vIGCogWfKZlkJWvBZ7Q5+qS/FkKcK+QnvhBCCCGEEOLcpSid7ysK4G5of8jS7pVwMBfWvgZrX4fhF8Oku6HH2FP9DEQHBiVH8rOZkdw9YxDbS+tZUqhtb99b4eC/m0v57+ZSLEYd04YmMT89jalDkrCaZICV6JipZw9MPS8j5vLLUFUV7/792jb3vFwcuXn4KytxfPsdjm+/ozew57XXsGWOw54zHltODpZhQ1H0J/c9lmxPZl7/eczrPw+AMmeZVu1Zmk9BSQH76vaxtWorW6u28uaWN1FQGBI3JFjxOS55nASf4rwmQacQQgghhBBCNOuor+iUX8P+7+DrJ2Dnf2HLR9rRb7IWeA6YpoWq4rRTFIWhKVEMTYninpmD2Vpcz+LCIyzZVMz+SidLN5WwdFMJVqOe6cO00HPKkEQsRgk9RccURcHUty+mvn2JveqHqKqKZ9cuHLl5NHz3HXXffove6cTx5Vc4vvwKAF1UFLasLOw52dhyxmMeNBBFpzupdSTZkpjbfy5z+88FoNxZHqz4zC/JZ1/dPrZVbWNb1Tbe2voWCgqDYwcHhxtlJmdK8CnOKxJ0CiGEEEIIIURn9JmgHaVb4Ju/QdE/Ye+X2pGSDpPuguGXaFvrRbdQFIXhaVEMT4vil7OGsPlIHZ8UHmFJYTGHqhubtroXYzfpmTE8mfnpaUwenIDZIKGnODZFUTAPGoR50CAif3gl6xcvZvrAgbgL1uLMzcVZUECgro6GlStpWLkSAH1sLLbsbOzjc7Dl5GDq1w/lJN8QSbQlMqffHOb00/qMVjRWBPt7FpQWsKd2D9urt7O9ejtvbX0LIBh8ZiVrFZ8xlpiTWoMQZzL5F1gIIYQQQgghwpE8HC57AabdD989C+teh5JC+NeNsOohmHAHZPwIjNbuXul5TVEURvaIZmSPaH4zeyiFh2q1Ss/CYo7UuvhowxE+2nCESLOBmcOTmT86lQsGJmIynFwFnjhP6HSYhw4lYtQo4n98ParPh2vLFm2w0ZpcnOvW4a+upv6//6X+v/8FwJCYiC0nB1tONvacHIy9ep108JlgTWB2v9nM7jcbaAo+m4YbFZQUsLt2Nzuqd7Cjegdvb30bgEGxg8hK1io+xyWPI84Sd3JfCyHOIBJ0CiGEEEIIIcSJiOkFcx6Gi34F+S9D7t+heh8s/YU2uCjnNsi6UesPKrqVoiiM7hXD6F4x3DtnGBsO1bB4YzFLNxVTUufi3+sP8+/1h4myGPjeiBTmp6cyaWACRr2EnqJzFIMBa3o61vR0uPlmVI+HxqIiHGu0wUaN69fjKy+nbvFi6hYvBsCQloo9Owfb+Bwt+ExNPel1JFgTmN13NrP7Hg0+15au1YLP0gJ21exiZ/VOdlbv5J1t7wAwMGagttU9OZPMlEwJPsVZTYJOIYQQQgghhDgZtjgt7JxwB2x4G759SpvYvvpPWk/PcdfDhJ9AdM/uXqkAdDqFsb1jGds7lvvnDWPdgWoWFxazZFMx5fVu/rn2EP9ce4gYm5FZw1OYPzqVCf3jMUjoKcKgmEzYxo7FNnYs/OQnBNxuGtdv0AYbrcmlsbAQ35Fiav/zH2r/8x8AjL17Y8/Rtrnbc7IxJCae9DoSrAnM6juLWX1nAVDZWKkFn019PnfV7Aoe7257F9CCz+bQMzM5k3hr/EmvQ4jTRYJOIYQQQgghhOgKJhtk3wzjfgybP9T6eJZugjXPQt4LMOpKrY9n0tDuXqlootMpZPaNI7NvHL+bP5yCfVUsLizm06JiKho8vF9wkPcLDhJnNzFrRAoL0lPJ7hcnoacIm85sxj4+B/v4HBLvhIDTiXPdepy5a3Dk5uEqKsJ74AA1Bw5Q88EHAJgGDAgONrJlZ2GIjT3pdcRb4/le3+/xvb7fA6DKVRWs+MwvzWdn9c5g8Pne9vcAGBA9QAs9m4LPBGvCSa9DiFNFgk4hhBBCCCGE6Ep6A6T/AEZdAbtWwjdPwr6vYOM72jF4DlxwN/Qe390rFS3odQo5/ePJ6R/PgxePIHdvJYsLi1lWVEKVw8O7eQd4N+8ACREmZo9MYX56Gll949DrTq7Hojg/6Ww2Ii6YRMQFkwDw19fjLCjAmZuHIy8X99ZteHbvxrN7N9XvaJWW5iFDgoONbJmZ6KOiTnodcZY4ZvaZycw+MwGodlWzrnQd+aXaVPcd1TvYXbub3bW7eX/7+wD0j+5PZnJmcLK7BJ/iTCJBpxBCCCGEEEKcCooCg2Zox6G1WuC59RPY8al29BqvBZ6DZoFOKgTPJHqdwsQBCUwckMAfLx7Bmj1VLC48wrLNJVQ0eHhrzQHeWnOAxEgzc0emMH90GuN6x6KT0FOcIH1kJJFTpxI5dSoA/poaHPn52mCjvFzcO3fh3r4d9/btVL3+Buh0WIYP1wYbjR+PbexYdHb7Sa8j1hLL9D7Tmd5nOgA1rhrWlq0NTnbfUb2DPbV72FO7h//b8X8A9I3qq011b+rzmWg7+S33QpwoCTqFEEIIIYQQ4lTrOQ5++CZU7NJ6eG58Fw6ugXevgsSh2pb2kVeAwdTdKxWtGPQ6LhiUwAWDEnjokpF8s6uCJYXF/HdzCeX1bl7/bj+vf7ef5Cgzc0elMj89jYxeMRJ6ipOij4khauZMomZqlZa+igqceXk4cvNw5ubi2bcPV1ERrqIiql75BxgMWEeODA42smZkoLNYTnodMZYYpveezvTeWvBZ665lbela8kvyKSgtYHvVdvbV7WNf3T4+2KFtue8b1ZfMlMzgZPckW9JJr0OIzpKgUwghhBBCCCFOl4SBcPFTMPU+WPM8FPwDyrfBf26HVX+CCYtg7EIwR3T3SkU7jHodU4YkMWVIEv/v0lF8vaucxYXFrNhcSmmdm1e/2cer3+wjLdqihZ6j0xjdMxpFkdBTnBxDQgJRc+cSNXcuAN7SUpy52mAjZ24u3sOHadywgcYNG6j8+wsoRiPWMWO0wUbjc7Cmp6OYTv6NlGhzNNN6T2Na72mAFnw2b3UvKClgW9W2YPD5zx3/BKBPVJ+jW92TM0m2J5/0OoToiASdQgghhBBCCHG6RabAzD/AhfdAwauw5jmoOwz/vQ+++Ctk3QQ5t0GEbAE9U5kMOqYNTWba0GTcPj9f7qhgSeERVmwp5Uiti5e/3svLX++lR4yV+elapefIHlESeoouYUxOJvrii4m++GIAPIcOacFnbi7ONbn4yspw5ufjzM+n4plnUCwWbGMzsOWMx56TjWXkSBTDyUdC0eZopvaeytTe2pb7Ok+dFnw2VXxuq9rG/rr97K/bz792/guA3pG9g/09M5MzSbGnnPQ6hGgmQacQQgghhBBCdBdLtNanc/ztsPE9bVt75S746lH47hnI+BFMuAPi+nX3SsUxmA16Zg5PZubwZFxeP59vL2fJpmJWbi3lcE0jL3y5hxe+3EPvOBvz0lOZn57K8FQJPUXXMfXsialnT2IuvxxVVfHs26cNNspdgzM3D39VFY5vv8Px7XeUow1DsmZlYs/OwTY+B8vQoSh6/UmvI8oUxZReU5jSawqgBZ/rS9dTUKr1+NxatZUD9Qc4UH8gGHz2iuwVrPbMSsmS4FOcFAk6hRBCCCGEEKK7GcwwbqEWbG5bog0uOrwW8l/WtrePuFTr45k6urtXKo7DYtQze2QKs0em0Ojx8/n2MhYXFrNyWykHqpw8//lunv98N/0S7Mwblcr80akMSY6U0FN0GUVRMPfrh7lfP2Kv+qEWfO7apW1zz8vFkZdPoLYWxxdf4vjiSwB0UVHYsrKw52hT3c2DBqJ0wZC0KFMUF/W6iIt6XQRAvaee9WXrg8ONtlRt4WD9QQ7WH+TfO/8NQM+InsGKz6zkLFIjUk96HeL8IUGnEEIIIYQQQpwpdHoYfjEMWwD7vtYCz12fQdG/tGPANJh0N/SbrE11F2c0q0nPnFGpzBmVitPjY9W2MhZvLGb19jL2Vjh4ZvUunlm9iwGJdualp7EgPZVByZHdvWxxjlEUBfOgQZgHDSLuf36EGgjg3rZNG2y0Zg3OggICdXU0rFxJw8qVAOhjY7X+njnZ2HLGY+rXt0vC+EhTJJN7TmZyz8kANHgaWFe2joLSAgpKCthSuYVDDYc4tOsQH+76EIAeET2C1Z5ZKVmkRaSd9DrEuUuCTiGEEEIIIYQ40ygK9LtQO0o2wTd/g6J/w+5V2pGWoVV4DrtYC0fFGc9mMjA/PY356Wk0uH2s3FrK4sJivthezu5yB0+t3MlTK3cyODmC+elpzEtPZUCiDKUSXU/R6bAMH45l+HDif3w9qs+Ha8uW4GAj57p1+KurqV+2jPplywAwJCYGBxvZcnIw9uzZJcFnhCkiJPh0eB2sL1uv9fgsKWBz5WYONxzmcMNhPtr9EaAFn+OSxwWDzx4RPU56HeLcIUGnEEIIIYQQQpzJUkbB5S/DtPvhu2dh3ZtwZD18cD3E9YeJP4XR14DR0t0rFZ0UYTbw/TE9+P6YHtS5vFroubGYL3eWs6O0gcdX7ODxFTsYmhLJgtFpzBuVSt8Ee3cvW5yjFIMBa3o61vR0uOVmVI+Hxk2btMFGuXk0rl+Pr7ycusWLqVu8GABDWir2nPHYcrKx5+RgTO2a7eV2o50LelzABT0uALTgc0PZBvJL8skvzWdLxZZg8Pnx7o8BSLOnBQcbNQef0gri/CVBpxBCCCGEEEKcDWL7wtxH4KJfQ96L2lG1Bxb/DFb/BcbfBpk3gjWmu1cqwhBlMXJpRk8uzehJbaOXFVtKWVx4hK93VrCtpJ5tJdt55L/bGZEWpVV6jkqld7ytu5ctzmGKyYRt3Dhs48bBT35CwO2mcf2G4GCjxsJCfEeKqf3wQ2o/1LaXG/v0Dg42smdnY0hM7JK12I12JvWYxKQekwBwep1a8FmaT35JPpsrNnPEcYSPd38cDD5T7ClkJWcFKz57RvbskrWIs4MEnUIIIYQQQghxNrEnwNT7YOKdsP5N+PYZqDsEK/8IXz0BmT+G8T+BKBngcbaJthq5YlxPrhjXkxqnh+WbS/mk8Ajf7q5k85E6Nh+p43+XbSO9ZzTz01OZOyqVnrESeopTS2c2Yx+vbVsHCDgcONet1wYb5ebhKirCu/8ANfsPUPPBBwCYBgwIDjayZWdhiI3tkrXYjDYm9pjIxB4Tgabgs3wDBSUFFJQWsKliEyWOEj7Z8wmf7PkEBYWHJj3E9wd+v0uuL858EnQKIYQQQgghxNnIHAHjb4esm7RBRd/8Dcq2wLdPwZrnYfQPYeJdkDi4u1cqTkCMzcSVWb24MqsXVQ4Py4pKWLLpCN/trqTwUC2Fh2r589JtjOkVEww902Ks3b1scR7Q2e1EXHgBERdq28v99fU4CwpwrsnFkZeHe9s2PLt349m9m+p33gHAPHRocLCRLSsTfWTXDN2yGW1MTJvIxLSjwefG8o3kl+Tz3ZHvKKos4pkNzzCv/zwMOonAzgfyX1kIIYQQQgghzmZ6I4y+CtJ/CDuXw9dPwoFvYf1bsP5tGDpPm9TeK6u7VypOUJzdxDU5vbkmpzcVDW4+LSphSeERcvdWseFgDRsO1vCnJVsZ1yc2GHomR0nPVnF66CMjiZw6lcipUwHwVVfjzM/HmZuHMy8X985duLdtw71tG1WvvwFNw5CaBxvZxo5FZ++aHrQ2o40JaROYkDaBW0ffyswPZlLiKOGLg18wvc/0LrmGOLNJ0CmEEEIIIYQQ5wJFgcGztONgnhZ4bl8C2xZrR59JWuA5aKZ2X3FWSogw8z/j+/A/4/tQVu/i000lLCksJn9/FWv3V7N2fzV/XLyFrD5xzB+dyuyRKSRFSugpTh9DbCxR3/seUd/7HgC+igqceXnBqe6e/ftxFRXhKiqi8uVXwGDAOmqUNtho/HisY8ags5z896xZb+bywZfz8qaXeXfbuxJ0nick6BRCCCGEEEKIc02vbLj6HSjfDt88BYXvw/5vtCNpBEy6C0ZeplWDirNWUqSFhRP7snBiX0pqXSzdVMySTcWs3V9N3r4q8vZV8fuPN5PTL4556WnMGZlCQoS5u5ctzjOGhASi5s4lau5cALwlJThztf6ezjVr8B45QuP69TSuX0/l319AMRqxjhmjDTbKycGano5iMp3Qta8cfCX/KPoHuSW57KrexcDYgV351MQZSIJOIYQQQgghhDhXJQ6BS57VhheteQ7WvgZlm+HDW2DVn2DCIhj7P2Dqmm2jovukRFu44YJ+3HBBP47UNLJ0UzGLC4vZcLCGNXuqWLOnit9/VMSEAfHMG5XG7JEpxNlPLDwS4mQYU1KI/v73if6+NiDIc+iQFnw2VXz6ysq0re/5+VQ8/QyK1YotIwNbTg6RM6ZjHjCg09dKjUhlWq9pfHbgM97b/h73j7//VD0tcYaQoFMIIYQQQgghznXRPWDW/4PJv4D8VyD371B7AJb9Gr74X8i+RTvs8d29UtEF0mKs3HRhf266sD8Hq5zBSs/CQ7V8s6uSb3ZV8ruPipg4IJ4F6Wl8b0QyMTYJPUX3MPXsialnT2IuvxxVVfHs29dU8ZmLMzcPf1UVjm+/xfHtt1Q8+yx933sXy/DhnX78q4dezWcHPuPj3R9z19i7iDR1zSAkcWaSoFMIIYQQQgghzhfWWC3snLAINrwD3z4N1Xvhi4e1ae0Z/wMT74CY3t29UtFFesXZuPWiAdx60QAOVDpZvOkISwqL2Xykjq92VvDVzgru+1DhgkEJzE9PY+bwZKKt0tJAdA9FUTD364e5Xz9ir7oKVVVx79yJMzePmg//jXvLVipf+Qc9Hnu004+ZlZLFgOgB7K7dzce7P+baYdeewmcgupuuuxcghBBCCCGEEOI0M1oh60b46Vq44lVIHQ1eJ+S9AH8bA/+6GUqKunuVoov1jrfxkykDWXLnhaz+xRR+OWsIQ1Mi8QVUPt9ezi8+2Ejmn1Zw42v5fLj+EPUub3cvWZznFEXBMngwcf/zI9L+9CcA6v77X7wlJWE9xtVDrwbg3W3vElADp2St4swgQacQQgghhBBCnK90em0o0S1fwP/8B/pPAdUPm/4P/j4J3roC9n0NqtrdKxVdrF+CnUVTB7Ls7sl8ds9F3DNzMIOTI/D6VVZuK+Nn729k3J8+4+Y3Cvhow2Ea3L7uXrI4z1mGD8eWnQ0+H9Vvvx3WuQsGLCDCGMH+uv18d+S7U7RCcSaQoFMIIYQQQgghzneKAgOmwnUfwS2fw4hLQdHBrhXw2jx4eQZs/QQCUgl1LhqYFMGd0wex/GcXsfxnk7lz+iD6J9rx+AKs2FLKXe9tYNxDK7jtzbUsLjyC0yOhp+gecdcvBKD6/f8j4HB0+jyb0cYlAy8BtKpOce6SoFMIIYQQQgghxFFpGfCD17Rt7Zk3gN4Mhwvg/R/Bs9mw7g3wubt7leIUGZwcyT0zB7PynotYdveF3DF1IH3jbbh9AZZtLuGOd9Yz9qEVLHp7HZ9uKqbR4+/uJYvzSMSUKRj79CZQV0fNf/4T1rk/HPJDAL489CUH6w+egtWJM4EEnUIIIYQQQggh2orrD/OfgJ8VwYU/B0s0VO6Ej38KfxsN3/wNXHXdvUpxiiiKwtCUKH4xawirfzGFJXdewO1TBtA7zobLG2DJpmJuf3sd4/60gp++u57/bi7B5ZXQU5xaik5H3HXXAVD1xhuoYVSZ943uy6S0SaiovL/t/VO1RNHNJOgUQgghhBBCCNGxiCSY/gD8bDN8708QmQb1xbDiAXhiJHz2INSXdvcqxSmkKAoj0qL59eyhfPHLKXxyxwXcOrk/PWKsOD1+Ptl4hFvfXEvmnz7j7vfW89mWUtw+CT3FqRFzySXooqLw7j9Aw+efh3XuNcOuAeDfu/5No6/xFKxOdDcJOoUQQgghhBBCHJ85Eib+FO7aCN9/FhIGg7sWvn4CnhwFn9wFlbu7e5XiFFMUhVE9o7l37jC+/vVU/rNoEjdd0I/UaAsNbh//2XCEm94oIPOhz7jn/zawelsZHp/0dhVdR2e3E3vlDwCoeu31sM6dlDaJnhE9qffUs3TP0lOxPNHNJOgUQgghhBBCCNF5BhNk/Ah+kgtXvQu9csDvhrWvwdPj4P+ug8PrunuV4jRQFIUxvWK4f/5wvvn1NP51+0R+PKkvyVFm6t0+/r3uMD9+LZ/MP63glx9s5Isd5Xj9EnqKkxf7ox+BwYAzLw/Xli2dPk+v03PV0KsAeGfbO6iqeqqWKLqJBJ1CCCGEEEIIIcKn08HQuXDjcvjxMhg0C1Bhy0fw0lR4fQHsWgkSJJwXdDqFcX1i+f2CEXz3m+l8cNsEFk7oQ2KkmTqXjw/WHmLhP/LI/n+fce+/C/l6ZwU+CT3FCTKmpBA1axYAVa+HV9V5ycBLsOgt7KjewboyeVPmXCNBpxBCCCGEEEKIk9NnAlz7f3D7d5B+FegMsPdLeOsyeGEybPonBHzdvUpxmuh0Cll94/jD90ey5t7pvHfLeH40vjfxdhPVTi/v5h3kR6/kkvPnlfz2w018u7sCf0ACcRGeuOuvB6B26ad4y8o6fV60OZp5/ecB8O62d0/F0kQ3kqBTCCGEEEIIIUTXSB4Ol70Ad26AnNvBaIOSQvjXjRiez6Fv+WfglQEg5xO9TmF8/3j+dMkocu+bzjs35XB1dm9ibUYqHR7ezj3ANS9poecDHxWRu6dSQk/RKdZRI7GOGwdeL9XvvBPWuVcPvRqAlftXUuqQYWrnEgk6hRBCCCGEEEJ0rZheMOdhbVL71N+CLR6lZj+jD72B4ZkM+OIRcFZ19yrFaWbQ65g4MIG/XDaK/N/O4M0bs7kqqxcxNiMVDW7e+G4/P3xxDRP+spIHP95Mwb4qAhJ6imOIW3gdADXvvkegsfNvogyJG8LYpLH4VB8f7PjgVC1PdAMJOoUQQgghhBBCnBq2OLjoV3B3Ef5Z/4vDlIDirIDVf4InRsKy+6D2UHevUnQDg17HhYMSefjydPJ/O4PXfpzFFeN6EmkxUFbv5rVv93HF379j0v+u4qHFW1h3oFoGx4g2IqdPx9izJ/7aWmo/+jisc68Zdg0AH+z4AI/fcyqWJ7qBBJ1CCCGEEEIIIU4tk41A5o2sHP4IvktegORR4HXAmmfhb6Phw9uhbFt3r1J0E6Nex5QhSTz6g9GsvX8m/7g+k8syehBpNlBc6+KVr/dy2XPfcsH/rubPS7ey8WCNhJ4CAEWvJ+66/wGg6o03UAOdH3A1rfc0kqxJVLmqWLF/xalaojjNJOgUQgghhBBCCHFaqIoedcTlcNtXcO2/oO+F2pCije/AcznwzlWw/7vuXqboRiaDjmlDk3n8h2PIv38GL12XyffHpGE36Tlc08iLX+7h+89+w4V/Xc1fPt1K0eFaCT3Pc9GXXY4uIgLPnj04vvqq0+cZdUZ+MOQHALyzLbwen+LMJUGnEEIIIYQQQojTS1Fg0Ay4fjHctAqGXQwosONTeHU2vDILtn8KYVRniXOPxahn5vBk/nZVBmt/N5O//2gc89NTsRr1HKpu5IUv9jD/6a+Z8ujn/HXZNrYcqZPQ8zykj7AT8wMtsKx6/fWwzr1i8BUYdAYKywvZXLH5VCxPnGYSdAohhBBCCCGE6D49x8EP34Q7CmDsQtCb4OAaePcqeH4CbHgHfNI/73xnMeqZPTKFZ64Zy7rfzeS5a8cyd1QKFqOO/ZVOnvt8N3Of+orpj33BY8u3s72kXkLP80jcj64FnQ7Ht9/h2r690+clWBOY1XcWAO9ue/dULU+cRhJ0CiGEEEIIIYTofgkD4eKn4O5NMOluMEdB+Tb4z+3w1Bj47llwN3T3KsUZwGrSM3dUKs9dO46198/k6aszmDUiGZNBx54KB0+v2sWsJ79k5hNf8uRnO9hVVt/dSxanmLFHDyK/9z0Aql5/I6xzrx56NQCf7v2Uald1l69NnF4SdAohhBBCCCGEOHNEpsDMP8DPimDGHyAiGeoOw3/vgydGwMqHoKG8u1cpzhB2s4EFo9N44X8yWfe7mfztqjHMGJaMSa9jV1kDT362kxmPf8msJ77k6ZU72VMuYfm5Kv76hQDUffIJvoqKTp+XnpDO8PjheAIe/rXzX6dqeeI0kaBTCCGEEEIIIcSZxxINF9ytVXgueAriB4KrBr56FJ4cCYvvgaq93b1KcQaJMBv4/pgevLwwk4LfzeDxK0czbWgSRr3C9tJ6Hluxg2mPffH/27vvKKnKw//j75mt9N57kSZFiiiioogiKipGVEBEY/QXg4nol2gSoyYmlm++xo4lRgULggULqCiioCBIR+lIE5G+9LJ1fn8MbCSi7sIOd2f2/Tpnz87euc/sZzz3bJhP7vM89HrkM4Z98jWrt+wJOrKKUKkTTqBUu3ZEsrPZNrLg09BDoRD9W/QH4NWlr5KTlxOriDoGLDolSZIkScVXchp0HASDZ8BlL0KdjpCzH2Y9C491gNeugfXzg06pYqZ8egqXdKjLc1efyKzbz+b/Lm1Lt2bVSA6HWLx+J//3wVLOeGASFzz2GU9NXsHajL1BR1YRqHzgrs5to0aRl5lZ4HHnNjqXimkVWb9nPZO/nRyreDoGLDolSZIkScVfOAlaXQi/mgiDxkHTHhDJg4Vj4OnT4cU+sHIyuAGN/kuF0in07VSPEb/szMzbe3D/JW047biqJIVDLFi3k/vfX8Jp//iEix6fwjOfrmTd9n1BR9YRKnf22STXrkVuRgY7x44t8Li0pDR+cdwvADclincWnZIkSZKk+BEKQaPT4Mo34NdToE1fCCXBio/hhQvhmTNh4ZuQlxt0UhVDlcqkckXn+rx47UnM+NNZ3NunDac0qUI4BPO/3cE97y2m6/0f0+eJqTw7ZRXrd1h6xpNQcjKVrxwIQMaIEUQK8X98XNb8MsKhMF+s/4IV21fEKqJizKJTkiRJkhSfaraBX/wbfjcHOl8PyaXgu7nw2tXweCeY9Rxk7w86pYqpKmXT6H9SfUZedzJf/KkHf7u4NSc1qkwoBHO/2c7fxi2iy30fc+mTnzN86io27vRaigcV+15KuHRpMpd/zZ6pnxd4XO2ytTmz3pmAd3XGM4tOSZIkSVJ8q9QQzvu/6E7t3W6DUpUgYyWMuxkebgOf/RP2bQ86pYqxauXSGHhyA0b/vy588cez+OuFx3Niw0oAzFqzjb+MXcTJ903ksqen8fIX37AzK+DA+lFJ5cpR4dLoNPSMESMKNbZfi34AvLPiHXZl7SrybIo9i05JkiRJUmIoUxXO/BMMWQDn3g/l68KeTTDxbnioNXz4Z9i5PuiUKuaql09n0CkNee3XpzD9j2dx5wWt6FC/IpEIzFiVwV/GLeHO2UkMfG4mL3+xhq27C77pjY6NygMHQijEns8+I/Prrws8rnPNzjSp0IR9Oft4Z8U7MUyoWLHolCRJkiQllrSycPINcNM86PM0VG8FWbvg88eid3i+PRg2Lws6peJAzQrp/PLURoz5TVem/qE7fz6/JW3rlidCiOmrtnH7mwvofO9Ervz3F4ya8Q3b9nirZ3GQWq8e5XqcBUDGiBcKPC4UCuXf1TlqySjyInkxyafYseiUJEmSJCWmpBRodwXc8Dn0fxXqnwJ52TD3JRjWGUYNgLUzg06pOFGnYil+dVpj3vh/J3Nn+xxu7XkcbepUIDcvwpSvt/CHMV9x4j0fMei5Gbw6ay079mYHHblEq3z11QDseOcdcjIyCjyud5PelE0py+qdq5n+3fQYpVOsWHRKkiRJkhJbKATNesIv34drJ0Dz84EILBkHz/aA58+DZR9CIXZoVslWJR2uO7URY397KpN/fwa3ntucVrXKk5MXYfKyzdz6+pd0umcCvxw+kzdmf8vO/Zaex1qpDh1Ib92aSGYm20aNKvC40imluajpRYCbEsUji05JkiRJUslRrzP0GwmDZ8AJV0I4BdZMhZF94cmuMH805FpKqeAaVCnDb85oyns3ncbH/9ONoec0o0XNcmTnRvh4ySb+57X5dPrbR/xqxCzemruOXZaex0QoFKLyoEEAbBv5CnlZBV9W4IrmVwAw+dvJrN21Nib5FBsWnZIkSZKkkqdac7h4GNw0H7rcCKllYdNCePN6eLQ9TH8KsvYEnVJxpnG1stzY/TjGDzmdj245nZt7NOO46mXJys3jo8UbGTJ6Hh3//hH/78VZvDP/O/Zk5gQdOaGVP7cnyTVqkLtlCzvffa/A4xpWaEjX2l2JEOHVpa/GMKGKmkWnJEmSJKnkqlAHet4DNy+A7ndAmWqwYy2Mvy26U/sn98GerUGnVBxqWr0cN/U4jgm3dOPDm0/nd92b0rhqGbJy8vhg4UZ+98pcOvxtAje8NJt3v1zP3ixLz6IWSkmh0pUDAMgYPpxIIZanOLgp0ZjlY9iXsy8m+VT0LDolSZIkSSpVCU4fCkO+gvMfhEqNYF8GTL4fHm4N790K278JOqXiVLMa5bjlnOZM/J9uvH/TaQw+swkNq5QmMyeP9xdsYPDIOXT820cMHjmH8QvWsz87N+jICaPSZZcRKlWKzKVL2fvFFwUed2qdU6lTtg47s3by/qr3Y5hQRcmiU5IkSZKkg1JKwYnXwm9nw6XPQ612kL0XZjwNj5wAb1wHGxYEnVJxKhQK0bJWeX7fswWfDD2Dcb89lV93a0K9yqXYl53Lu1+u59cvzaHj3ybwu1fm8uHCDZaeRympQgUq9ukDQMbwEQUfF07Kv6tz5OKRhbobVMGx6JQkSZIk6b+Fk6D1JXD9ZBj4FjQ+AyK58NWr8FRXeOlSWD3Fndp1xEKhEK3rVOAPvVrw6e/P5J0bu/L/Tm9MnYql2JOVyzvzv+P6F2fT6e8fcfPoeUxcvJHMHEvPI1H5qoEQCrF70iQyV64q8LiLm15MelI6S7ctZe6muTFMqKJi0SlJkiRJ0o8JhaDJmXDV23D9JDi+D4TC8PUEGH4+/LsHLB4LeXlBJ1UcC4VCtK1bkT+e15Ipt53Jm785hV+d2ohaFdLZnZnDm3PXce2IWXT6+0f8z6vz+WTpJrJyvOYKKrVhQ8qecQYAGS++UOBxFdIqcH7j8wF4ZckrsYimImbRKUmSJElSQdRuD32HR6e1d/olJKXBulkw+koY1hnmvAA5mUGnVJwLhUK0r1+JP1/Qiqm3deeNG7pwTdeG1Cifxq79Obwx51uueX4mJ97zEbe9/iWfLttMdq6l58+pfPXVAOx48y1yt28v8LiD09c/WvMRm/ZuikEyFSWLTkmSJEmSCqNyY7jgoehO7af9D6RXgK3L4Z3fwiPtYOojsH9n0CmVAMLhEB0bVOau3scz7Q9n8er/68KgLg2oWjaNHfuyGT1rLVc9N4PO93zEH8d8xdSvt5Bj6XlYpTufSFrLlkT272fb6FcLPK555eZ0qN6BnEgOry17LYYJVRQsOiVJkiRJOhJlq8NZd8LNC+Gcv0O52rBrPUy4Ex5qDR/9BXZtDDqlEkQ4HKJzo8r89aLWfPGns3jlupO58uT6VCmTyra92bwy4xsG/PsLTr5vIn9+6yumrdhKbp5ryB4UCoWoPOgqALa9/DKRrKwCj+3XMnpX52tLXyM7Nzsm+VQ0LDolSZIkSToaaeXglN/CTfPhomFQtRlk7oApD8HDbWDsTbB1RdAplUCSwiG6NKnC3y9uwxd/OouXf3US/TrXp1LpFLbszuKl6d/Q75npnHzfRO56ewEzVmWQZ+lJhfPOI7laNXI2bWLnBx8UeNxZ9c+iWqlqbN2/lQlrJsQwoY6WRackSZIkSUUhORXaXwm/+QKueAXqnQS5mTB7ODzWEV69CtbNCTqlEkxyUpiuTaty3yVtmHF7D174ZWcu61SXCqVS2LwrkxHT1nDZ09Pocv9E/jp2IbPXlNzSM5SaSqUB/QHIeH44kUjB/jukhFPo27wvACOXjIxZPh09i05JkiRJkopSOAwtzoNrP4RrxkOzc4EILHobnjkTRvSGrydCAUsWqaBSksKc3qwa/7i0HTNv78Hz15zILzrUpVx6Mht3ZvL81NX84slpnPq/H/P3cYuY+822Apd9iaLi5ZcTSktj/6JF7Js1q8Dj+jbrS3I4mfmb57Nw68IYJtTRsOiUJEmSJClWGnSB/qPhhmnQrh+Ek2HVp/DSJfD06fDV65CbE3RKJaDU5DBnNq/OPy9rx6w/9+DZQZ3o074OZdOS+W7Hfv49ZRV9nvicU//3E+57bzFffru9RJSeyZUqUeHiiwHYOmJEgcdVLVWVcxqcA8CoJaNiEU1FwKJTkiRJkqRYq9EK+jwFv5sHJ90AKaVhw5fwxrXwWAeY8Qxk7ws6pRJUWnISZ7WswUOXn8CsP/fgXwM7cmG72pROTWLd9n08/elKLnx8Kt3+bxL/O34JC9btSOjS8+CmRLsnfkzWmjUFHtevRXRTovdWvse2/dtikk1Hx6JTkiRJkqRjpWI96HV/dKf2M2+H0lVg+xp4b2h0p/bJ/4C9GUGnVAJLT0ninONr8mi/9sy542yeurID57etRamUJL7J2MuTk1ZwwWNT6P7PyTzwwVIWr9+ZcKVnWuPGlDn9NIhEyHjxpQKPa1etHa2qtCIrL4sxy8fEMKGOlEWnJEmSJEnHWunK0O1WGLIAznsAKtaHvVvgk3uihef4P8GOb4NOqQSXnpLEua1rMax/B2bf0YNh/TvQq3VN0pLDrNqyh8c/+Zpej3xGjwcn8+CEZSzbuCvoyEWmytVXA7B9zBhyd+4s0JhQKJR/V+fopaPJzcuNVTwdIYtOSZIkSZKCkloaOl8Hv50Lv3gWarSB7D0wfRg80g7evAE2LQk6pUqA0qnJnN+2Fk9e2ZE5d5zNo/3a0/P4GqQmh1mxeQ+PTlzOOQ99yjkPTeaRj5bz9abdQUc+KqW7dCGtWTMie/ey/bXXCjzu3IbnUjGtIuv3rGfyt5NjmFBHwqJTkiRJkqSgJSVDm0vh15/BgDeg4WmQlwPzR8ITJ8HIK2DNtKBTqoQok5bMhe1q8/TATsz+cw8evvwEerSsQWpSmGUbd/PQR8vo8eBkzn34Ux7/eDmrtuwJOnKhhUKh/LU6M156mUh2doHGpSenc8lxlwAwcsnImOXTkbHolCRJkiSpuAiF4LgecPU4+NXH0PJCIATL3ofnz4Vnz4El70FeXtBJVUKUS0/h4vZ1+PegTsz8cw/+2bcd3VtUJyUpxJINu3jgw2Wc+cAkLn96Gjv2FawsLC7KX3ABSVWqkLN+PTs//LDA4y5vfjnhUJgv1n/Byu0rY5hQhWXRKUmSJElScVS3I1z+Itw4CzoMgqRUWPsFjOoHT3aBuS9DTlbQKVWCVCiVwi861uW5q09k1u1n849L29KtWTWSwyG+WJXBqzPXBh2xUMJpaVTqF11zM2PECwXedKl22dqcUfcMAF5Z8kqs4ukIWHRKkiRJklScVW0KFz4KQ76CrkMgrTxsXgJv/wYePQE+fxwyE2eTGMWHCqVTuKxTPUb8sjN39W4FwNgvvws4VeFV6ncFodRU9n/5JfvmzivwuH4towXpOyveYXdWfK9XmkgsOiVJkiRJigflasLZf4WbF0CPv0LZGrBzHXx4e3Sn9ol/g92bg06pEqhXm1okhUN8+e0OVsfZep3JVapQ/sLeAGQMH17gcSfVPInGFRqzN2cvb694O0bpVFgWnZIkSZIkxZP0CnDqkOgdnr0fhSpNYf92+OwBeLg1jLsFMlYFnVIlSNWyaZzSpAoA4+Lwrs7KV0U3Jdr10UdkffttgcaEQiH6tYje1TlqySjyIq6bWxxYdEqSJEmSFI+S06DjIBg8Ay57Eep0hJz9MOtZeKwDvHYNrJ8fdEqVEL3b1gZg3JfrA05SeOnNmlHmlFMgL49tL75U4HG9m/SmTEoZVu9czfT102OYUAVl0SlJkiRJUjwLJ0GrC+FXE2HQOGjaAyJ5sHAMPH06vHAxrJwEBdxoRToSPY+vmb8T+7KN8bdmbOVrrgZg++uvk7u7YGtulkkpw0VNLgLglcVuSlQcWHRKkiRJkpQIQiFodBpc+Qb8egq06QuhJFj5CbxwEfzrDFj4JuTlBp1UCahC6RS6NasGwLj58Td9vcypp5LapAl5e/aw/fXXCzzuihZXADD528l8u6tg094VOxadkiRJkiQlmppt4Bf/ht/Nhc7XQ3IpWD8PXrsaHu8Es56D7P1Bp1SC6d0uOn197JfricTZHcShUCh/rc5tL75EJCenQOMaVWjEKbVPIUKEV5e+GsuIKgCLTkmSJEmSElWlBnDe/0V3au92G5SqBBkrYdzN8HAb+OyfsG970CmVIHq0rEF6SphVW/aw8LudQccptAoXXUhSxYpkr1vHro8mFnjcwU2J3lj+Bvty9sUqngrAolOSJEmSpERXpiqc+ScYsgDOvR/K14U9m2Di3fBQa/jwz7Az/qYbq3gpk5bMWS1qADA2Dqevh9PTqdgvOhU9Y8SIAo87rc5p1Clbh51ZO3l/1fuxiqcCsOiUJEmSJKmkSCsLJ98AN82DPk9D9VaQtQs+fwwebgtvD4bNy4JOqTjWu10tILr7el5efE1fB6jUrx+kpLBv7lz2zZ9foDFJ4SSuaB4tSF9Z8krcTdtPJBadkiRJkiSVNEkp0O4KuOFz6P8q1D8F8rJh7kswrDOMGgBrZwadUnHojObVKZuWzLrt+5i7dlvQcQotpXp1Kpx/PlC4uzr7HNeHtKQ0lmQsYd7meTFKp59j0SlJkiRJUkkVCkGznvDL9+HaCdD8fCACS8bBsz3g+fNg2YfgHWoqoPSUJM5pdXD6+vqA0xyZylcPAmDnBx+S/V3BpuBXSKvA+Y2jBenIxSNjlk0/zaJTkiRJkiRBvc7QbyQMngEnXAnhFFgzFUb2hSe7wvzRkJsddErFgQu+N309Nw6nr6e3aEHpk06C3FwyXn65wOMObkr00ZqP2LR3U6zi6SdYdEqSJEmSpP+o1hwuHgY3zYcuN0JqWdi0EN68Hh5tD9Ofgqw9QadUMXZq02pUKJXClt2ZfLFya9BxjsjBuzq3v/oaeXsKdr23qNyCDtU7kBPJ4fVlr8cynn6ERackSZIkSfqhCnWg5z1w8wLofgeUqQY71sL426I7tX9yH+yJzxJLsZWaHKZX65oAjP0yPqevl+3WjdSGDcnbtYvtY94s8LiDd3W+tuw1sr0D+piz6JQkSZIkST+uVCU4fSgM+QrOfxAqNYJ9GTD5fnjoeHjvVtj+TdApVcz0blcbgPcXrCc7Ny/gNIUXCoepdNVAADJefJFIbm6Bxp3V4CyqlarGln1bmLBmQiwj6jAsOiVJkiRJ0s9LKQUnXgu/nQ2XPg+12kHOPpjxNDxyArxxHWxYEHRKFRMnN65C1bJpbN+bzZSvtwQd54hUvPhiwhUqkP3NN+z+5JMCjUkJp9C3eV8AXlnySizj6TAsOiVJkiRJUsGFk6D1JXD9ZBj4FjQ+AyK58NWr8FRXeOlSWD3FndpLuKRwiPPbHJi+Pr9gO5cXN+HSpal02WUAZAwfUeBxfZv1JTmczLzN81i0dVGs4ukwLDolSZIkSVLhhULQ5Ey46m24fhIc3wdCYfh6Agw/H/7dAxaPhbz4m7asonFw+vqHCzeyP7tgU7+Lm0pXDoDkZPbOmsW+BQsLNKZqqaqc3eBsAEYtGRXLePovFp2SJEmSJOno1G4PfYdHp7V3+iUkpcG6WTD6ShjWGea8ADmZQafUMdahfiVqV0hnd2YOk5ZuDjrOEUmpUYPyvXoBkDGi4Hd19m/RH4D3Vr3H9v3bYxFNh2HRKUmSJEmSikblxnDBQ9Gd2k/7H0ivAFuXwzu/hYfbwtRHYP/OoFPqGAmHQ1xw4K7OsV/G5/R1gMqDBgGw8/33yd64sUBj2lVrR8vKLcnMzWTM12NiGU/fY9EpSZIkSZKKVtnqcNadcPNCOOfvUK427N4AE+6Eh1rDR3+BXQUrjBTfereNFp0TF29kT2ZOwGmOTKnWx1OqU0fIyWHbyyMLNCYUCtGvRT8ARi8ZTW5efE7djzcWnZIkSZIkKTbSysEpv4Wb5sNFw6BqM8jcAVMegofbwNibYOuKoFMqhlrXKU/DKqXZn53HR4vjt9yucvXVAGwbPZq8vXsLNKZXo15UTKvId3u+Y/K3k2OYTgdZdEqSJEmSpNhKToX2V8JvvoArXoF6J0FuJsweDo91hFevgnWzg06pGAiFQlxw4K7OsfPXB5zmyJU980xS6tUjb8cOdrz9doHGpCenc8lxlwDwypJXYhlPB1h0SpIkSZKkYyMchhbnwbUfwjXjodm5QAQWvQ3PdIcRveHriRCJBJ1URejg7uufLtvMjn3ZAac5MqGkJCoPHAhAxogXiOTlFWjcZc0vIxwKM339dFZuXxnLiMKiU5IkSZIkBaFBF+g/Gm6YBu36QTgZVn0KL10CT58OX70OufG5pqMO1bxmOZrVKEtWbh4fLtwQdJwjVuGSSwiXK0fW6tXsnlywqeh1ytahW91uAIxaOiqW8YRFpyRJkiRJClKNVtDnKfjdPDjpBkgpDRu+hDeuhcc6wIxnIHtf0Cl1lA5uSjT2y/idvp5UtgwV+/YFond1FtTBTYne/vptdmftjkk2RVl0SpIkSZKk4FWsB73uj+7UfubtULoKbF8D7w2N7tQ++R+wNyPolDpCFxyYvj716y1s3Z0ZcJojV/nKAZCUxN7p09m/eHGBxpxc62QaVWjE3py9vLPinRgnLNksOiVJkiRJUvFRujJ0uxWGLIDzHoCK9WHvFvjknmjhOf6PsOPboFOqkBpVLUObOhXIzYvw/oL4nb6eUrs25XueAxT8rs5QKJR/V+crS14h4hq0MWPRKUmSJEmSip/U0tD5OvjtXPjFs1CjDWTvgelPwCPt4M1fw6aC3VGn4qF3u1oAjJ3/XcBJjk7lQYMA2PHuu2Rv2lSgMRc2uZAyKWVYvXM109ZPi2W8Es2iU5IkSZIkFV9JydDmUvj1Z3DlG9DwNMjLgfmvwBMnw8grYI3FUTw4/8A6nTNWZ7Bhx/6A0xy5Uu3aUap9e8jOZtsrrxRoTJmUMlzU5CIgelenYsOiU5IkSZIkFX+hEDTtAVePg199DC0vBEKw7H14/lx49hxY8h7k5QWdVD+iTsVSdGpQiUgE3v0qfjclgv/c1bl91Gjy9hestL28xeUATF47mXW718UsW0lm0SlJkiRJkuJL3Y5w+Ytw4yzoMAiSUmHtFzCqHzzZBea+DDlZQafUYfQ+sClRvE9fL9fjLFJq1yZ32zZ2vFOwDYYaV2hMl1pdiBBh9NLRMU5YMll0SpIkSZKk+FS1KVz4KAz5CroOgbTysHkJvP0bePQE+PwR3pTyAABAwklEQVRxyNwVdEp9T682NQmHYN7a7azN2Bt0nCMWSk6m0lUDgeimRAXdYOjgpkRjlo9hf078Tt8vriw6C2nv3r00aNCAoUOHBh1FkiRJkiQBlKsJZ/8Vbl4APf4KZWvCznXw4e3w0PEw8W+we3PQKQVUL5fOyY2rADD2y/i+q7PipZcSLlOGrBUr2DNlSoHGnF73dOqUrcOOzB28v+r9GCcseSw6C+mee+7h5JNPDjqGJEmSJEn6b+kV4NQhMORL6P0oVGkK+3fAZw/Aw61h3C2QsSrolCXewenr4+bH9zqdSWXLUvHSXwCQMXxEwcaEk7i8eXStzpFLRhb4TlAVjEVnISxfvpwlS5bQq1evoKNIkiRJkqQfk5wGHQfB4Blw2YtQpyPk7IdZz8JjHeC1a2D9/KBTlljnHl+T5HCIRet38vWm3UHHOSqVBg6EcJg9U6eyf9myAo3p07QPaUlpLMlYwvzNXodFKfCi8y9/+QuhUOiQrxYtWhTp7/j000/p3bs3tWvXJhQK8dZbbx32vGHDhtGwYUPS09M56aSTmDFjxiHPDx06lPvuu69Is0mSJEmSpBgJJ0GrC+FXE2HQuOiu7ZE8WDgGnj4dXrgYVk4C76o7piqVSeW046oCMC7Op6+n1q1LuR49AMh44YUCjamYXpHzGp0HwMjFI2OWrSQKvOgEOP7441m/fn3+15SfWNdg6tSpZGdn/+D4okWL2Lhx42HH7Nmzh3bt2jFs2LAffd3Ro0dzyy23cNdddzFnzhzatWtHz5492bRpEwBvv/02zZo1o1mzZoV8d5IkSZIkKVChEDQ6Da58A349Bdr0hVASrPwEXrgI/nUGLHwT8nKDTlpifH/39Xifvl356kEA7HxnLDlbtxZozMFNiSasmcDmva4fW1SKRdGZnJxMzZo187+qVq162PPy8vIYPHgw/fv3Jzf3P398li5dSvfu3Rkx4vDrIfTq1Yu///3v9OnT50czPPjgg1x33XVcc801tGrViqeeeorSpUvz3HPPATB9+nRGjRpFw4YNGTp0KM888wx33333UbxrSZIkSZJ0zNVsA7/4N/xuLnS+HpJLwfp58NrV8HgnmPUcZLsbdqyd3aoGqclhVmzew+L1u4KOc1RKtW9Petu2RLKy2PbKqAKNaVmlJe2rtycnksPry16PccKSo1gUncuXL6d27do0btyYAQMG8M033xz2vHA4zHvvvcfcuXO56qqryMvLY8WKFXTv3p2LL76YW2+99Yh+f1ZWFrNnz6bHgVuND/6uHj16MG3aNADuu+8+1q5dy+rVq3nggQe47rrruPPOOw/7esOGDaNVq1aceOKJR5RHkiRJkiTFWKUGcN7/RXdq73YblKoEGSth3M3wcBv47J+wb3vQKRNWufQUujevDsT/7uuhUIjKg64CYNsrr5CXmVmgcQfv6nx12atk5/5w9rIKL/Ci86STTmL48OGMHz+eJ598klWrVnHaaaexa9fh2/zatWvz8ccfM2XKFPr370/37t3p0aMHTz755BFn2LJlC7m5udSoUeOQ4zVq1GDDhg2Ffr3BgwezaNEiZs6cecSZJEmSJEnSMVCmKpz5JxiyAM69H8rXhT2bYOLd8FBr+PDPsDO+i7jiKpGmr5c/5xySa9Ykd+tWdo57t0BjetTvQbVS1diybwsfffNRjBOWDIEXnb169aJv3760bduWnj178t5777F9+3ZeffXVHx1Tv359XnzxRUaPHk1ycjLPPvssoVDomGW++uqreeCBB47Z75MkSZIkSTGWVhZOvgFumgd9nobqrSBrF3z+GDzcFt4eDJsLtqu2CqZ7i+qUTk3i2237mLd2e9BxjkooJYXKA68EIGPEiAIVtylJKfRt1heAV5a8EtN8JUXgRed/q1ixIs2aNePrr7/+0XM2btzI9ddfT+/evdm7dy8333zzUf3OqlWrkpSU9IPNjDZu3EjNmjWP6rUlSZIkSVIcSUqBdlfADZ9D/1eh/imQlw1zX4JhnUl67Soq7fnxzkIFVyo1iR4to7Nrx85fH3Cao1exb19CpUuTuWwZew8shfhzLm12KcmhZOZumsvirYtjnDDxFbuic/fu3axYsYJatWod9vktW7Zw1lln0bJlS8aMGcPEiRMZPXo0Q4cOPeLfmZqaSseOHZk4cWL+sby8PCZOnEiXLl2O+HUlSZIkSVKcCoWgWU/45ftw7QRofj4QIbzsPU5fdjdJL/aGZR9CnE+5DtrB6evvfvUdeXnx/d8yqXx5Kh7YCHvrj2yY/d+qla7G2Q3OBryrsygEXnQOHTqUyZMns3r1aj7//HP69OlDUlIS/fr1+8G5eXl59OrViwYNGuRPW2/VqhUTJkzg+eef56GHHjrs79i9ezfz5s1j3rx5AKxatYp58+YdsunRLbfcwjPPPMOIESNYvHgxN9xwA3v27OGaa66JyfuWJEmSJElxol5n6DcSBs8gr21/8kJJhL+ZBiP7wpNdYf5ocDOZI3J6s6qUS09m485MZq7OCDrOUat81UAIhdgz+VMyV6wo0Jj+LfsD8N6q99i+f3sM0yW+wIvOb7/9ln79+tG8eXMuu+wyqlSpwvTp06lWrdoPzg2Hw9x777288cYbpKam5h9v164dH330EX379j3s75g1axbt27enffv2QLTUbN++/SG7pl9++eU88MAD3HnnnZxwwgnMmzeP8ePH/2CDIkmSJEmSVEJVa05u70eZ0Oqf5J70G0gtC5sWwpvXw6PtYfpTkLUn6JRxJS05iXOPjy4bGO+7rwOkNmhA2e7dAch44cUCjWlXrR0tK7ckMzeTN79+M5bxEl7gReeoUaP47rvvyMzM5Ntvv2XUqFE0adLkR88/++yzSU9P/8Hx9u3bU7du3cOOOeOMM4hEIj/4Gj58+CHn3XjjjaxZs4bMzEy++OILTjrppKN6b5IkSZIkKfHsT61MXo+74eYF0P0OKFMNdqyF8bdFd2r/5D7YszXomHHj4PT1977aQE5uXsBpjl7lQVcBsOPtt8nZtu1nzw+FQvRrEZ3ZPHrpaHLzcmOaL5EFXnRKkiRJkiTFpVKV4PShMOQrOP9BqNQI9mXA5PvhoePhvVth25qgUxZ7pzSpQuUyqWTsyeLzFfFfEJc+8UTSW7Uisn8/20ePLtCYXo16USGtAut2r+PTbz+NccLEZdEpSZIkSZJ0NFJKwYnXwm9nw6XPQ612kLMPZjwdndL+xnWwYUHQKYut5KQw57U5MH19fvxPXw+FQlS+ehAA214eSSQr62fHpCenc8lxlwBuSnQ0LDolSZIkSZKKQjgJWl8C10+GgW9B4zMgkgtfvQpPdYWXLoXVU9yp/TB6t41OXx+/cAOZOfE/dbv8ueeSXK0aOZs3s/P99ws05vLmlxMixLT101i5Y2WMEyYmi05JkiRJkqSiFApBkzPhqrfh+klwfB8IheHrCTD8fPh3D1g8FvLifz3KonJiw8rUKJ/Grv05fLpsS9BxjlooNZVKV14JwNbhI4gUoNyuU7YO3ep1A2DUklExzZeoLDolSZIkSZJipXZ76Ds8Oq290y8hKQ3WzYLRV8KwzjDnBcjJDDpl4MLhEBccuKszEaavA1S6/DJC6elkLl7M3hkzCzSmf4v+ALz99dvsztody3gJyaJTkiRJkiQp1io3hgseiu7UftpQSK8AW5fDO7+Fh9vClIdh/86gUwbqgra1AJiwaCN7s3ICTnP0kipWpMLFFwGQMWJEgcacXOtkGpZvyN6cvYxdOTaW8RKSRackSZIkSdKxUrY6nHUH3LwQzrkHytWG3Rvgo7uiO7V/9BfYtTHolIE4oV5F6lYqxb7sXD5esinoOEWi8lXRTYl2f/IJWatX/+z5oVCIfi36AdFNiQoy5V3/YdEpSZIkSZJ0rKWVg1NuhJvmw0XDoGozyNwJUx6Ch9vA2Jtg64qgUx5ToVCI3u0Sa/p6WuNGlO3WDSIRMl54sUBjLmxyIaWTS7Nqxyqmr58e44SJxaJTkiRJkiQpKMmp0P5K+M0XcMUrUO8kyM2E2cPhsY7w6lWwbnbQKY+Zg7uvf7J0M7v2ZwecpmhUvjp6V+f2N98kd/v2nz2/bGpZLmoanfL+ypJXYhkt4RRJ0blmzRoWLVpEnruFSZIkSZIkFV44DC3Og2s/hGvGQ7NzgQgsehue6Q7DL4CvP4IEn8rcslY5mlQrQ1ZOHhMWJcYU/tInn0xa8+ZE9u1j22uvFWjMFS2uAGDyt5NZt3tdLOMllEIVnc899xwPPvjgIceuv/56GjduTJs2bWjdujVr164t0oCSJEmSJEklSoMu0H803DAN2vWDcDKs/gxe+gU8fRp89Trkxv9mPYeTiNPXQ6EQlQdF7+rc9tLLRLJ//k7VxhUac3Ktk8mL5DF66ehYR0wYhSo6//Wvf1GpUqX8n8ePH8/zzz/PCy+8wMyZM6lYsSJ//etfizykJEmSJElSiVOjFfR5Cn43D07+DaSUgQ1fwRvXwmMdYMYzkLU36JRF7oID09c/W76FbXuyAk5TNMpfcD5JVauSs3EjO8d/UKAx/Vv0B2DM8jHsz9kfy3gJo1BF5/Lly+nUqVP+z2+//TYXXXQRAwYMoEOHDtx7771MnDixyENKkiRJkiSVWBXrwbn3wc0L4MzboXQV2L4G3hsa3bho8j9gb0bQKYtM0+plaVWrPDl5EcYv3BB0nCIRTk2lUv/obuoZI0YUaDf10+ueTu0ytdmRuYP3V70f64gJoVBF5759+yhfvnz+z59//jmnn356/s+NGzdmw4bEuAAlSZIkSZKKldKVodutMGQBnPcAVKwPe7fAJ/fAQ61h/B9hx7dBpywSiTZ9HaDSFVcQSk1l/4IF7Jsz52fPTwoncXmLy4HopkQFKUdLukIVnQ0aNGD27OhOX1u2bGHhwoV07do1//kNGzZQoUKFok0oSZIkSZKk/0gtDZ2vg9/OhV88CzXaQPYemP4EPNIO3vw1bFocdMqjckHbWgBMW7mVTTsTY9p2cuXKVLjoQgAyhg8v0JhLml5CWlIaizMWM3/z/BimSwyFKjoHDRrE4MGD+dvf/kbfvn1p0aIFHTt2zH/+888/p3Xr1kUeUpIkSZIkSf8lKRnaXAq//gyufAMangZ5OTD/FXjiZBh5OayZFnTKI1Kvcmna169IJALvfbU+6DhFpvJVVwGw66OJZBVgQ++K6RU5r9F5AIxcMjKm2RJBoYrOW2+9leuuu44xY8aQnp7Oa6+9dsjzU6dOpV+/fkUaUJIkSZIkST8hFIKmPeDqcfCrj6HlhUAIlo2H58+FZ8+BJe9BXl7QSQvl4KZEY79MnKIz7bjjKHPqqRCJkPHiiwUac0WLKwCYsHoCm/dujmW8uFeoojMcDnP33Xczd+5c3n//fVq2bHnI86+99hrXXnttkQaUJEmSJElSAdXtCJe/CDfOgg6DICkV1n4Bo/rBk11g7suQEx87mZ/fphahEMxes4112/cFHafIVB40CIAdr79B7q5dP3t+qyqtOKHaCeREcnh92euxjhfXClV0AowePZoBAwbQt29fnnrqqVhkkiRJkiRJ0tGo2hQufBSGfAVdh0Baedi8BN7+DTx6Anz+OGT+fMkWpJoV0uncsDIA736ZOJsSlTm1K6lNm5C3dy/bXytYcdm/ZX/qlq1LzTI1Y5wuvhWq6HzyySfp168fs2bNYvny5QwePJjf//73scomSZIkSZKko1GuJpz9V7h5AfT4K5StCTvXwYe3w0PHw8S/we7iOx36P7uvJ8709VAolH9XZ8ZLLxLJyfnZMec0OIdxfcbR57g+sY4X1wpVdD7++OPcddddLF26lHnz5jFixAieeOKJWGWTJEmSJElSUUivAKcOgSFfQu9HoUpT2L8DPnsAHm4N426BjFVBp/yBXq1rkhQO8dW6HazasifoOEWmQu/eJFWqRM5369k1YcLPnp8UTiIpnHQMksW3QhWdK1euZNCBxhmgf//+5OTksH594rTqkiRJkiRJCSs5DToOgsEz4LIXoU5HyNkPs56FxzrAa9fA+vlBp8xXpWwaXZtWBWDc/MSZvh5OT6fSgQ29M4aPCDhN4ihU0ZmZmUmZMmX+MzgcJjU1lX37EmdBWEmSJEmSpIQXToJWF8KvJsKgcdFd2yN5sHAMPH06vHAxrJwEkUjQSendthYAYxNonU6ASv37EUpJYd/8+exbsDDoOAkhubAD7rjjDkqXLp3/c1ZWFvfccw8VKlTIP/bggw8WTTpJkiRJkiTFTigEjU6Lfm34CqY+AgvGwMpPol+1TohOeW95YbQcDcA5x9fk9jcXsGzjbpZu2EXzmuUCyVHUkqtWpfrvf09a0yakH98q6DgJoVBF5+mnn87SpUsPOXbKKaewcuXK/J9DoVDRJJMkSZIkSdKxU7MN/OLf0P0OmPY4zHkR1s+D166Gyo3hlN9Cu/6Qkn5MY1UolUK35tWYsGgjY+d/R/OazY/p74+lylcNDDpCQilU0Tlp0qRDft6yZQupqamUL1++KDNJkiRJkiQpKJUawHn/B91ugxn/in5lrIRxN8Mn98HJv4ZO10KpiscsUu92taNF55ff8T/nNPNGOx1WodboBNi+fTuDBw+matWq1KhRg0qVKlGzZk3++Mc/snfv3lhklCRJkiRJ0rFWpiqc+ScYsgDOvR/K14U9m2Di3fBQa/jwz7Dz2KybeVaL6qSnhFmzdS9frdtxTH6n4k+h7ujMyMigS5curFu3jgEDBtCyZUsAFi1axGOPPcaECROYMmUKX375JdOnT+d3v/tdTEJLkiRJkiTpGEkrCyffACf+Cha8EV3Hc9Mi+PwxmP4UtLscTrkJqjWLWYQyacmc1bIG7365nrHzv6Nt3Yox+12KX4W6o/Puu+8mNTWVFStW8PTTTzNkyBCGDBnCv/71L77++muysrIYOHAgZ5999iGbE0mSJEmSJCnOJaVAuyvghs+h/6tQ/xTIy4a5L8GwzjBqAKydGbNf37ttbQDGfbmevLzgd4NX8VOoovOtt97igQceoEaNGj94rmbNmvzjH//gjTfe4JZbbmHQoEFFFlKSJEmSJEnFRCgEzXrCL9+HaydA8/OBCCwZB8/2gOfPg2UfQqRoy8gzmlejbFoy63fsZ84324r0tZUYClV0rl+/nuOPP/5Hn2/dujXhcJi77rrrqINJkiRJkiSpmKvXGfqNhMEz4IQrIZwCa6bCyL7wZFeYPxpys4vkV6WnJHHO8dGb78bOPzZrgyq+FKrorFq1KqtXr/7R51etWkX16tWPNpMkSZIkSZLiSbXmcPEwuGk+dLkRUsvCpoXw5vXwaPvoWp5Ze4761/RuF52+/u5X68nJzTvq11NiKVTR2bNnT26//XaysrJ+8FxmZiZ33HEH5557bpGFkyRJkiRJUhypUAd63gM3L4Dud0CZarBjLYy/LbpT+yf3wZ6tR/zypzatSsXSKWzZncUXqzKKMLgSQaF2Xb/77rvp1KkTxx13HIMHD6ZFixZEIhEWL17ME088QWZmJi+88EKsskqSJEmSJCkelKoEpw+FLoNh3sjoDu3bVsHk+6O7tne4KvpcpQaFetmUpDC9WtfilRnfMHb+d3RtWjVGb0DxqFB3dNatW5dp06bRqlUr/vjHP3LxxRfTp08fbr/9dlq1asXUqVOpX79+rLJKkiRJkiQpnqSUghOvhd/Ohkufh1rtIGcfzHg6OqX9jV/BhgWFesne7WoB8P6CDWTlOH1d/1GoOzoBGjVqxPvvv8+2bdtYvnw5AE2bNqVy5cpFHk6SJEmSJEkJIJwErS+B4/vAykkw9eHo969ei3417QFdh0DDU6O7uv+EkxpVoVq5NDbvymTK15vp3qLGMXgDigeFuqPz+ypVqkTnzp3p3LmzJackSZIkSZJ+XigETc6Eq96G6ydFi89QGL7+CEZcAP8+CxaPhbwfv1MzKRzi/DbRuzrHzl9/jIIrHhxx0SlJkiRJkiQdsdrtoe/w6LT2Tr+EpDRYNxtGXwnDOsOcFyAn87BDD05f/3DhBvZn5x7D0CrOLDolSZIkSZIUnMqN4YKHoju1nzYU0ivA1uXwzm/h4bYw5WHYv/OQIe3rVaJOxVLsycrlkyWbgsmtYseiU5IkSZIkScErWx3OugNuXgjn3APlasPuDfDRXfDQ8TDhLti1AYBwOMQFbaN3dY770unrirLolCRJkiRJUvGRVg5OuRFumg8XDYOqzSBzZ3QDo4fbwDu/g60r6N2uNgATl2xkd2ZOsJlVLFh0SpIkSZIkqfhJToX2V8JvvoArXoF6J0FuFswZAY915PgpN3Jupe/Yn53HxMUbg06rYsCiU5IkSZIkScVXOAwtzoNrP4RrxkOzc4EIocXv8NS+oYxM+Tsrp70NkUjQSRUwi05JkiRJkiTFhwZdoP9ouGEatOtHJJzMKUmLuHnjH8l98jT46nXIdRp7SWXRKUmSJEmSpPhSoxX0eYrQ7+YxJu1C9kTSSNr0FbxxLTzWAWY8A1l7g06pY8yiU5IkSZIkSfGpYj3Wn3wXXTMf5fXyV0HpKrB9Dbw3FB5pC5sWB51Qx5BFpyRJkiRJkuLWBW1rsZ1y3Lr5XLZcNxvOewAq1IM9m2H6E0HH0zFk0SlJkiRJkqS41aBKGdrWrUBeBN5fsgM6XwfnPxh9cuXkYMPpmLLolCRJkiRJUlzr3bY2AGPnr48eaHAKhJOj09gzVgWYTMeSRackSZIkSZLi2vltawEwY3UG63fsg7SyUPfE6JOrvKuzpLDolCRJkiRJUlyrXbEUJzasBMC7Xx64q7NRt+j3lZOCCaVjzqJTkiRJkiRJca93uwPT1w8WnY3PiH5f9Snk5QUTSseURackSZIkSZLiXq/WtQiHYP7a7XyzdS/U6QgpZWDvVti4IOh4OgYsOiVJkiRJkhT3qpVL45QmVQEY++V3kJwKDbtGn3SdzhLBolOSJEmSJEkJoXe76KZEY+d/Fz3gOp0likWnJEmSJEmSEkLP42uSkhRiyYZdLN+46z/rdK75HHKyAs2m2LPolCRJkiRJUkKoWDqV04+rBhzYlKh6KyhdFbL3wrczA06nWLPolCRJkiRJUsK44MD09XHzvyMSCkHjA9PXXacz4Vl0SpIkSZIkKWH0aFmDtOQwK7fsYeF3O7+3TqdFZ6Kz6JQkSZIkSVLCKJeeQvcW1YEDu68fXKdz3SzI3BVcMMWcRackSZIkSZISSu92tQEYN389kYr1oVJDyMuJbkqkhGXRKUmSJEmSpIRyZvPqlElNYt32fcxdu/0/d3WunBRgKsWaRackSZIkSZISSqnUJM5uVQOAsfO/c53OEsKiU5IkSZIkSQnn4PT1d79cT27D06MHNy2E3ZsCTKVYsuiUJEmSJElSwjntuGqUT09m065MZmwMQc020SdWfRpsMMWMRackSZIkSZISTmpymF6tawEHdl/Pn77+SYCpFEsWnZIkSZIkSUpIB6evv//VenIafm+dzkgkwFSKFYtOSZIkSZIkJaSTG1emSplUtu3N5vOcZhBOgR1rIWNl0NEUAxadkiRJkiRJSkjJSWHOaxOdvv72wh1Qr3P0iVXuvp6ILDolSZIkSZKUsA5OX/9w4QayG5wWPbhyUnCBFDMWnZIkSZIkSUpYnRpUomb5dHZl5jAnqV304KpPIS8v2GAqchadkiRJkiRJSljhcIgL2kanr49cVw1Sy8K+bbDhy4CTqahZdEqSJEmSJCmh5U9fX7yV3PpdowddpzPhWHRKkiRJkiQpobWtW4H6lUuzLzuXxaXaRw+6TmfCseiUJEmSJElSQguFQvRuF52+/vq2ptGDa6ZBTmaAqVTULDolSZIkSZKU8A5OXx+5qgx5ZapDzj74dmbAqVSULDolSZIkSZKU8JrXKMdx1cuSlRvh24onRg86fT2hWHRKkiRJkiQp4YVCIS5oe2BTon0togdXuiFRIrHolCRJkiRJUolwwYF1OodvaBg9sG427N8ZXCAVKYtOSZIkSZIklQhNqpXl+Nrl+TavCjtL14dILqyZGnQsFRGLTkmSJEmSJJUYBzcl+oI20QOu05kwLDolSZIkSZJUYpzfJjp9/c0dTaMHXKczYVh0SpIkSZIkqcSoV7k0HepX5PPcVkQIwebFsGtD0LFUBCw6JUmSJEmSVKL0bleb7ZRjZXKT6IFVnwYbSEXColOSJEmSJEklyvltahEKwYT9LaIHXKczIVh0SpIkSZIkqUSpXj6dkxtVYWpe6+iBlZMhEgk2lI6aRackSZIkSZJKnN7tajMzrznZJMPOb2HriqAj6ShZdEqSJEmSJKnEObd1TbLD6czKbRY9sGpSoHl09Cw6JUmSJEmSVOJULpPKqU2rMjXv+OgB1+mMexadkiRJkiRJKpF6t6udv05nZNVnkJcbcCIdDYtOSZIkSZIklUjnHF+DJeGm7IyUIrR/O6yfH3QkHQWLTkmSJEmSJJVI5dNTOK15Tb7IaxU9sGpysIF0VCw6JUmSJEmSVGJFp69H1+mMrLTojGcWnZIkSZIkSSqxzmpZnVnhdgBE1nwO2fsDTqQjZdEpSZIkSZKkEqt0ajKNWnZgY6Qi4dxM+HZG0JF0hCw6JUmSJEmSVKIdsvv6iknBhtERs+iUJEmSJElSidateTXmJLUFYM+SiQGn0ZGy6JQkSZIkSVKJlpacRFqzMwEoveVL2Lc92EA6IhadkiRJkiRJKvFO63gCK/JqESaP3FVTgo6jI2DRKUmSJEmSpBKva9OqzA63AWDjvPEBp9GRsOiUJEmSJElSiZeSFCa7QTcAktZ8GnAaHQmLTkmSJEmSJAlodlIv8iIhamSuISvj26DjqJAsOiVJkiRJkiSgQ4vGLAk3BmD5F+8GnEaFZdEpSZIkSZIkAUnhEBnVuwCwZ/HEgNOosCw6JUmSJEmSpANqnHAuAPV3zGRfZk7AaVQYFp2SJEmSJEnSAU07nkUmKdQMZTBj1hdBx1EhWHRKkiRJkiRJB4RSS7O+fDsA1s8dH3AaFYZFpyRJkiRJkvQ9pZp3B6DKpmns2p8dcBoVlEWnJEmSJEmS9D3V2/UEoHNoIR8t+i7gNCooi05JkiRJkiTpe0K127M/qSwVQnv5auanQcdRAVl0SpIkSZIkSd8XTiKnflcASq/9jO17swIOpIKw6JQkSZIkSZL+S9kWPQA4ObSA8Qs2BJxGBWHRKUmSJEmSJP23xmcAcGLSMnKz9gabRQVi0SlJkiRJkiT9t6rHESlbizSyGVDbOzrjgUWnJEmSJEmS9N9CIUJNzog+XjkpyCQqIItOSZIkSZIk6XAadYt+Xzk52BwqEItOSZIkSZIk6XAaHyg6v5sL+7YFm0U/y6JTkiRJkiRJOpzytaFqMyACq6cEnUY/w6JTkiRJkiRJ+jH509cnBRpDP8+iU5IkSZIkSfoxjc+IfnedzmLPolOSJEmSJEn6MQ1PhVAYti6HHeuCTqOfYNEpSZIkSZIk/ZhSFaF2++jjVd7VWZxZdEqSJEmSJEk/xXU644JFpyRJkiRJkvRTvr9OZyQSaBT9OItOSZIkSZIk6afUOwmS02H3Bti8NOg0+hEWnZIkSZIkSdJPSUmH+idHH7tOZ7Fl0SlJkiRJkiT9nPx1Oi06iyuLTkmSJEmSJOnnHFync/VnkJsTaBQdnkWnJEmSJEmS9HNqtYP0CpC5E9bPCzqNDsOiU5IkSZIkSfo54SRodHr08cpPgs2iw7LolCRJkiRJkgrCdTqLNYtOSZIkSZIkqSAanxn9vvYLyNobbBb9gEWnJEmSJEmSVBBVmkD5OpCbBWunB51G/8WiU5IkSZIkSSqIUOh709cnBRpFP2TRKUmSJEmSJBVU4zOi312ns9ix6JQkSZIkSZIK6uDO6+vnw96MYLPoEBadkiRJkiRJUkGVrwXVWgARWP1Z0Gn0PRadkiRJkiRJUmG4TmexZNEpSZIkSZIkFYbrdBZLFp2SJEmSJElSYTTsCqEwZKyA7WuDTqMDLDolSZIkSZKkwkivAHU6Rh+v8q7O4sKiU5IkSZIkSSos1+ksdiw6JUmSJEmSpML6/jqdkUigURRl0SlJkiRJkiQVVr3OkFwK9myCTYuDTiMsOiVJkiRJkqTCS06DBl2ij12ns1iw6JQkSZIkSZKORP46nRadxYFFpyRJkiRJknQkDq7TuXoK5OYEGkUWnZIkSZIkSdKRqdkWSlWCrF3w3Zyg05R4Fp2SJEmSJEnSkQiHodHp0ccrJwUaRRadkiRJkiRJ0pFznc5iw6JTkiRJkiRJOlIH1+lc+wVk7Qk0Skln0SlJkiRJkiQdqcqNoUI9yMuGb6YFnaZEs+iUJEmSJEmSjlQo9L3p65MCjVLSWXRKkiRJkiRJR+Pg9HXX6QyURackSZIkSZJ0NA7uvL7hS9izNdgsJZhFpyRJkiRJknQ0ytWA6q2ij1d/GmyWEsyiU5IkSZIkSTpartMZOItOSZIkSZIk6Wi5TmfgLDolSZIkSZKko9XgFAglwbZVsG1N0GlKJItOSZIkSZIk6Will4e6naKPV3lXZxAsOiVJkiRJkqSi4DqdgbLolCRJkiRJkorCwXU6V30KkUigUUoii05JkiRJkiSpKNQ9EVJKw57NsGlR0GlKHItOSZIkSZIkqSgkp0Y3JQKnrwfAolOSJEmSJEkqKvnrdLoh0bFm0SlJkiRJkiQVlYPrdK6ZCrnZgUYpaSw6JUmSJEmSpKJSozWUrgJZu2Hd7KDTlCgWnZIkSZIkSVJRCYeh4WnRx67TeUxZdEqSJEmSJElF6eD0ddfpPKYsOiVJkiRJkqSi1PjAhkTfzoDM3cFmKUEsOiVJkiRJkqSiVKkRVKwPeTnwzbSg05QYFp2SJEmSJElSUQqFoNGBuzpdp/OYseiUJEmSJEmSiprrdB5zFp2SJEmSJElSUTt4R+fGr2D35mCzlBAWnZIkSZIkSVJRK1sNarSOPl79abBZSgiLTkmSJEmSJCkWXKfzmLLolCRJkiRJkmLBdTqPKYtOSZIkSZIkKRYanALhZNi+BjJWBZ0m4Vl0SpIkSZIkSbGQVhbqnhh9vMq7OmPNolOSJEmSJEmKlfx1Oi06Y82iU5IkSZIkSYqVg+t0rpoMeXmBRkl0Fp2SJEmSJElSrNTpCCllYO9W2LQw6DQJzaJTkiRJkiRJipXkVGjYNfp45aRAoyQ6i05JkiRJkiQpllyn85iw6JQkSZIkSZJiqfGBonPNVMjJCjZLArPolCRJkiRJkmKp+vFQuipk74V1s4JOk7AsOiVJkiRJkqRYCoeh0enRx67TGTMWnZIkSZIkSVKsNT4j+t11OmPGolOSJEmSJEmKtYPrdK6bBZm7gs2SoCw6JUmSJEmSpFir1DD6lZcDaz4POk1CsuiUJEmSJEmSjoVGB+7qdJ3OmLDoLKS9e/fSoEEDhg4dGnQUSZIkSZIkxRPX6Ywpi85Cuueeezj55JODjiFJkiRJkqR4c3Dn9U0LYfemYLMkIIvOQli+fDlLliyhV69eQUeRJEmSJElSvClTFWq2iT5e9WmwWRJQsSo677//fkKhEEOGDCnS1/3000/p3bs3tWvXJhQK8dZbbx32vGHDhtGwYUPS09M56aSTmDFjxiHPDx06lPvuu69Is0mSJEmSJKkEyV+n85NgcySgYlN0zpw5k6effpq2bdv+5HlTp04lOzv7B8cXLVrExo0bDztmz549tGvXjmHDhv3o644ePZpbbrmFu+66izlz5tCuXTt69uzJpk3R24jffvttmjVrRrNmzQrxriRJkiRJkqTvaXxm9PvKTyESCTZLgikWRefu3bsZMGAAzzzzDJUqVfrR8/Ly8hg8eDD9+/cnNzc3//jSpUvp3r07I0aMOOy4Xr168fe//50+ffr86Gs/+OCDXHfddVxzzTW0atWKp556itKlS/Pcc88BMH36dEaNGkXDhg0ZOnQozzzzDHffffcRvmNJkiRJkiSVSA26QDgFdnwD21YFnSahFIuic/DgwZx//vn06NHjJ88Lh8O89957zJ07l6uuuoq8vDxWrFhB9+7dufjii7n11luP6PdnZWUxe/bsQ35/OBymR48eTJs2DYD77ruPtWvXsnr1ah544AGuu+467rzzzsO+3rBhw2jVqhUnnnjiEeWRJEmSJElSgkotA/U6Rx+vnBRolEQTeNE5atQo5syZU+C1L2vXrs3HH3/MlClT6N+/P927d6dHjx48+eSTR5xhy5Yt5ObmUqNGjUOO16hRgw0bNhT69QYPHsyiRYuYOXPmEWeSJEmSJElSgspfp3NysDkSTHKQv3zt2rXcdNNNTJgwgfT09AKPq1+/Pi+++CLdunWjcePGPPvss4RCoRgmPdTVV199zH6XJEmSJEmSEkzjM2DSvdGd1/PyIBz4vYgJIdD/irNnz2bTpk106NCB5ORkkpOTmTx5Mo8++ijJycmHrMP5fRs3buT666+nd+/e7N27l5tvvvmoclStWpWkpKQfbGa0ceNGataseVSvLUmSJEmSJB2iTgdILQv7MmDjV0GnSRiBFp1nnXUWX331FfPmzcv/6tSpEwMGDGDevHkkJSX9YMyWLVs466yzaNmyJWPGjGHixImMHj2aoUOHHnGO1NRUOnbsyMSJE/OP5eXlMXHiRLp06XLErytJkiRJkiT9QFIKNOgafew6nUUm0Knr5cqVo3Xr1occK1OmDFWqVPnBcYiWj7169aJBgwaMHj2a5ORkWrVqxYQJE+jevTt16tQ57N2du3fv5uuvv87/edWqVcybN4/KlStTv359AG655RYGDRpEp06d6Ny5Mw8//DB79uzhmmuuKeJ3LUmSJEmSpBKv8Rmw/IPoOp1dbwo6TUIItOgsrHA4zL333stpp51Gampq/vF27drx0UcfUa1atcOOmzVrFmeeeWb+z7fccgsAgwYNYvjw4QBcfvnlbN68mTvvvJMNGzZwwgknMH78+B9sUCRJkiRJkiQdtcYHNiRa8znkZEJyWrB5EkCxKzonTZr0k8+fffbZhz3evn37Hx1zxhlnEIlEfvZ333jjjdx4440/e54kSZIkSZJ0VKq3gjLVYM9m+HYmNDw16ERxzy2dJEmSJEmSpGMtFIJGB+7qdJ3OImHRKUmSJEmSJAWh8RnR7ysnBxojUVh0SpIkSZIkSUE4uE7nutmwf2ewWRKARackSZIkSZIUhIr1oXJjiOTCmqlBp4l7Fp2SJEmSJElSUFyns8hYdEqSJEmSJElBcZ3OImPRKUmSJEmSJAWl0elACDYvhl0bgk4T1yw6JUmSJEmSpKCUrgy12kYfr/o02CxxzqJTkiRJkiRJClL+Op1OXz8aFp2SJEmSJElSkPLX6ZwEkUiQSeKaRackSZIkSZIUpPpdICkVdn4LGSuDThO3LDolSZIkSZKkIKWWhnonRR+v/CTYLHHMolOSJEmSJEkKmut0HjWLTkmSJEmSJClojQ8Unas+hbzcYLPEKYtOSZIkSZIkKWi1O0BqOdi/HTZ8GXSauGTRKUmSJEmSJAUtKRkanhp9vHJSoFHilUWnJEmSJEmSVBw0PiP63XU6j4hFpyRJkiRJklQcHFyn85tpkL0/2CxxyKJTkiRJkiRJKg6qtYCyNSBnP3w7I+g0cceiU5IkSZIkSSoOQiFodOCuTtfpLDSLTkmSJEmSJKm4cJ3OI2bRKUmSJEmSJBUXB9fp/G4O7NseaJR4Y9EpSZIkSZIkFRcV6kKVphDJgzVTg04TVyw6JUmSJEmSpOLEdTqPiEWnJEmSJEmSVJy4TucRseiUJEmSJEmSipOGpwIh2LIUdq4POk3csOiUJEmSJEmSipPSlaH2CdHHq7yrs6AsOiVJkiRJkqTiJn+dTovOgrLolCRJkiRJkoqb/HU6J0EkEmSSuGHRKUmSJEmSJBU39U+GpDTY9R1s/TroNHHBolOSJEmSJEkqblJKQb3O0ccrJwUaJV5YdEqSJEmSJEnF0fenr+tnWXRKkiRJkiRJxdHBonP1Z5CXG2iUeGDRKUmSJEmSJBVHtU6AtAqwfwesnxd0mmLPolOSJEmSJEkqjpKSoeGp0cdOX/9ZFp2SJEmSJElScZW/TufkQGPEA4tOSZIkSZIkqbhq3C36/ZvpkL0v2CzFnEWnJEmSJEmSVFxVbQblakFuJqz9Iug0xZpFpyRJkiRJklRchULQ6MBdna7T+ZMsOiVJkiRJkqTizHU6C8SiU5IkSZIkSSrODq7T+d1c2Lct2CzFmEWnJEmSJEmSVJyVrx1dq5MIrJ4SdJpiy6JTkiRJkiRJKu7y1+l0+vqPseiUJEmSJEmSirv8dTonBZmiWLPolCRJkiRJkoq7hqdCKAxbl8OOdUGnKZYsOiVJkiRJkqTirlRFqN0++niV09cPx6JTkiRJkiRJigeu0/mTLDolSZIkSZKkeND4YNE5CSKRQKMURxadkiRJkiRJUjyodzIkp8PuDbBlWdBpih2LTkmSJEmSJCkepKRDvZOij919/QcsOiVJkiRJkqR40fiM6HfX6fwBi05JkiRJkiQpXhxcp3P1Z5CbE2yWYsaiU5IkSZIkSYoXtU6A9AqQuRPWzws6TbFi0SlJkiRJkiTFi3ASNDwt+njlJ8FmKWYsOiVJkiRJkqR44jqdh2XRKUmSJEmSJMWTg0Xn2i8ga2+gUYoTi05JkiRJkiQpnlRpCuXrQG4WrJ0edJpiw6JTkiRJkiRJiiehEDQ6sPv6ykmBRilOLDolSZIkSZKkeOM6nT9g0SlJkiRJkiTFm0anR7+vnw97M4LNUkxYdEqSJEmSJEnxpnwtqNYCiMDqz4JOUyxYdEqSJEmSJEnxKH+dTqevg0WnJEmSJEmSFJ/y1+mcFGSKYsOiU5IkSZIkSYpHDbtCKAx7t7pOJ5AcdABJkiRJkiRJRyC9AvxmOlRpCuGkoNMEzqJTkiRJkiRJilfVmgedoNhw6rokSZIkSZKkuGfRKUmSJEmSJCnuWXRKkiRJkiRJinsWnZIkSZIkSZLinkWnJEmSJEmSpLhn0SlJkiRJkiQp7ll0SpIkSZIkSYp7Fp2SJEmSJEmS4p5FpyRJkiRJkqS4Z9EpSZIkSZIkKe5ZdEqSJEmSJEmKexadkiRJkiRJkuKeRackSZIkSZKkuGfRKUmSJEmSJCnuWXRKkiRJkiRJinsWnZIkSZIkSZLinkWnJEmSJEmSpLhn0SlJkiRJkiQp7ll0SpIkSZIkSYp7Fp2SJEmSJEmS4p5FpyRJkiRJkqS4lxx0gEQWiUQA2LlzZ8BJdKxkZ2ezd+9edu7cSUpKStBxpCPmtaxE4HWsROG1rEThtaxE4bWsRBEv1/LBXu1gz/ZTLDpjaNeuXQDUq1cv4CSSJEmSJElS/Nq1axcVKlT4yXNCkYLUoToieXl5fPfdd5QrV45QKBR0HB0DO3fupF69eqxdu5by5csHHUc6Yl7LSgRex0oUXstKFF7LShRey0oU8XItRyIRdu3aRe3atQmHf3oVTu/ojKFwOEzdunWDjqEAlC9fvlj/kZAKymtZicDrWInCa1mJwmtZicJrWYkiHq7ln7uT8yA3I5IkSZIkSZIU9yw6JUmSJEmSJMU9i06pCKWlpXHXXXeRlpYWdBTpqHgtKxF4HStReC0rUXgtK1F4LStRJOK17GZEkiRJkiRJkuKed3RKkiRJkiRJinsWnZIkSZIkSZLinkWnJEmSJEmSpLhn0SlJkiRJkiQp7ll0Sj9h2LBhNGzYkPT0dE466SRmzJjxk+c//PDDNG/enFKlSlGvXj1uvvlm9u/fn/98bm4ud9xxB40aNaJUqVI0adKEv/3tb7gnmGKtMNdydnY2d999N02aNCE9PZ127doxfvz4o3pNqagU9bV83333ceKJJ1KuXDmqV6/OxRdfzNKlS2P9NqSY/F0+6P777ycUCjFkyJAYJJcOFYtred26dVx55ZVUqVKFUqVK0aZNG2bNmhXLt6ESrqivYz/3KQiffvopvXv3pnbt2oRCId56662fHTNp0iQ6dOhAWloaTZs2Zfjw4T84J+4+90UkHdaoUaMiqampkeeeey6ycOHCyHXXXRepWLFiZOPGjYc9/+WXX46kpaVFXn755ciqVasiH3zwQaRWrVqRm2++Of+ce+65J1KlSpXIuHHjIqtWrYq89tprkbJly0YeeeSRY/W2VAIV9lq+9dZbI7Vr1468++67kRUrVkSeeOKJSHp6emTOnDlH/JpSUYjFtdyzZ8/I888/H1mwYEFk3rx5kfPOOy9Sv379yO7du4/V21IJFItr+aAZM2ZEGjZsGGnbtm3kpptuivE7UUkXi2s5IyMj0qBBg8jVV18d+eKLLyIrV66MfPDBB5Gvv/76WL0tlTCxuI793KcgvPfee5Hbb789MmbMmAgQefPNN3/y/JUrV0ZKly4dueWWWyKLFi2KPPbYY5GkpKTI+PHj88+Jx899Fp3Sj+jcuXNk8ODB+T/n5uZGateuHbnvvvsOe/7gwYMj3bt3P+TYLbfcEunatWv+z+eff37kl7/85SHnXHLJJZEBAwYUYXLpUIW9lmvVqhV5/PHHDzn239dpYV9TKgqxuJb/26ZNmyJAZPLkyUUTWjqMWF3Lu3btihx33HGRCRMmRLp162bRqZiLxbV82223RU499dTYBJYOIxbXsZ/7FLSCFJ233npr5Pjjjz/k2OWXXx7p2bNn/s/x+LnPqevSYWRlZTF79mx69OiRfywcDtOjRw+mTZt22DGnnHIKs2fPzr+Ne+XKlbz33nucd955h5wzceJEli1bBsD8+fOZMmUKvXr1iuG7UUl2JNdyZmYm6enphxwrVaoUU6ZMOeLXlI5WLK7lw9mxYwcAlStXLoLU0g/F8loePHgw559//iGvLcVKrK7ld955h06dOtG3b1+qV69O+/bteeaZZ2LzJlTixeo69nOf4sG0adN+8G+Gnj175l/78fq5LznoAFJxtGXLFnJzc6lRo8Yhx2vUqMGSJUsOO6Z///5s2bKFU089lUgkQk5ODr/+9a/505/+lH/OH/7wB3bu3EmLFi1ISkoiNzeXe+65hwEDBsT0/ajkOpJruWfPnjz44IOcfvrpNGnShIkTJzJmzBhyc3OP+DWloxWLa/m/5eXlMWTIELp27Urr1q2L/D1IELtredSoUcyZM4eZM2fGNL90UKyu5ZUrV/Lkk09yyy238Kc//YmZM2fyu9/9jtTUVAYNGhTT96SSJ1bXsZ/7FA82bNhw2Gt/586d7Nu3j23btsXl5z7v6JSKyKRJk7j33nt54oknmDNnDmPGjOHdd9/lb3/7W/45r776Ki+//DIjR45kzpw5jBgxggceeIARI0YEmFw61COPPMJxxx1HixYtSE1N5cYbb+Saa64hHPZ/MhRfCnstDx48mAULFjBq1KhjnFT6aT93La9du5abbrqJl19++Qd3GUnFSUH+Lufl5dGhQwfuvfde2rdvz/XXX891113HU089FWBy6T8Kch37uU8Kjp9apcOoWrUqSUlJbNy48ZDjGzdupGbNmocdc8cddzBw4EB+9atf0aZNG/r06cO9997LfffdR15eHgC///3v+cMf/sAVV1xBmzZtGDhwIDfffDP33XdfzN+TSqYjuZarVavGW2+9xZ49e1izZg1LliyhbNmyNG7c+IhfUzpasbiWv+/GG29k3LhxfPLJJ9StWzcm70GC2FzLs2fPZtOmTXTo0IHk5GSSk5OZPHkyjz76KMnJyT96F7N0NGL1d7lWrVq0atXqkHEtW7bkm2++Kfo3oRIvVtexn/sUD2rWrHnYa798+fKUKlUqbj/3WXRKh5GamkrHjh2ZOHFi/rG8vDwmTpxIly5dDjtm7969P7hLKCkpCYBIJPKT5xwsQqWidiTX8kHp6enUqVOHnJwc3njjDS666KKjfk3pSMXiWobo3+cbb7yRN998k48//phGjRrF7D1IEJtr+ayzzuKrr75i3rx5+V+dOnViwIABzJs3L//fI1JRitXf5a5du7J06dJDzl+2bBkNGjQo2jcgEbvr2M99igddunQ55NoHmDBhQv61H7ef+wLeDEkqtkaNGhVJS0uLDB8+PLJo0aLI9ddfH6lYsWJkw4YNkUgkEhk4cGDkD3/4Q/75d911V6RcuXKRV155JbJy5crIhx9+GGnSpEnksssuyz9n0KBBkTp16kTGjRsXWbVqVWTMmDGRqlWrRm699dZj/v5UchT2Wp4+fXrkjTfeiKxYsSLy6aefRrp37x5p1KhRZNu2bQV+TSkWYnEt33DDDZEKFSpEJk2aFFm/fn3+1969e4/121MJEotr+b+567qOhVhcyzNmzIgkJydH7rnnnsjy5csjL7/8cqR06dKRl1566Vi/PZUQsbiO/dynIOzatSsyd+7cyNy5cyNA5MEHH4zMnTs3smbNmkgkEon84Q9/iAwcODD//JUrV0ZKly4d+f3vfx9ZvHhxZNiwYZGkpKTI+PHj88+Jx899Fp3ST3jsscci9evXj6SmpkY6d+4cmT59ev5z3bp1iwwaNCj/5+zs7Mhf/vKXSJMmTSLp6emRevXqRX7zm98c8j94O3fujNx0002R+vXrR9LT0yONGzeO3H777ZHMzMxj+K5UEhXmWp40aVKkZcuWkbS0tEiVKlUiAwcOjKxbt65QrynFSlFfy8Bhv55//vlj9I5UUsXi7/L3WXTqWInFtTx27NhI69atI2lpaZEWLVpE/vWvfx2Lt6ISrKivYz/3KQiffPLJYf9de/D6HTRoUKRbt24/GHPCCSdEUlNTI40bNz7sv4Hj7XNfKBI5MKdWkiRJkiRJkuKUa3RKkiRJkiRJinsWnZIkSZIkSZLinkWnJEmSJEmSpLhn0SlJkiRJkiQp7ll0SpIkSZIkSYp7Fp2SJEmSJEmS4p5FpyRJkiRJkqS4Z9EpSZIkFcJf/vIXTjjhhPyfr776ai6++OLA8kiSJCnKolOSJEmSJElS3LPolCRJUsLIysoKOoIkSZICYtEpSZKkuHXGGWdw4403MmTIEKpWrUrPnj1ZsGABvXr1omzZstSoUYOBAweyZcuW/DF5eXn84x//oGnTpqSlpVG/fn3uueee/Odvu+02mjVrRunSpWncuDF33HEH2dnZQbw9SZIkFYJFpyRJkuLaiBEjSE1NZerUqdx///10796d9u3bM2vWLMaPH8/GjRu57LLL8s//4x//yP33388dd9zBokWLGDlyJDVq1Mh/vly5cgwfPpxFixbxyCOP8Mwzz/DQQw8F8dYkSZJUCKFIJBIJOoQkSZJ0JM444wx27tzJnDlzAPj73//OZ599xgcffJB/zrfffku9evVYunQptWrVolq1ajz++OP86le/KtDveOCBBxg1ahSzZs0CopsRvfXWW8ybNw+Ibka0fft23nrrrSJ9b5IkSSqc5KADSJIkSUejY8eO+Y/nz5/PJ598QtmyZX9w3ooVK9i+fTuZmZmcddZZP/p6o0eP5tFHH2XFihXs3r2bnJwcypcvH5PskiRJKjoWnZIkSYprZcqUyX+8e/duevfuzf/+7//+4LxatWqxcuXKn3ytadOmMWDAAP7617/Ss2dPKlSowKhRo/jnP/9Z5LklSZJUtCw6JUmSlDA6dOjAG2+8QcOGDUlO/uE/dY877jhKlSrFxIkTDzt1/fPPP6dBgwbcfvvt+cfWrFkT08ySJEkqGm5GJEmSpIQxePBgMjIy6NevHzNnzmTFihV88MEHXHPNNeTm5pKens5tt93GrbfeygsvvMCKFSuYPn06zz77LBAtQr/55htGjRrFihUrePTRR3nzzTcDfleSJEkqCItOSZIkJYzatWszdepUcnNzOeecc2jTpg1DhgyhYsWKhMPRf/recccd/M///A933nknLVu25PLLL2fTpk0AXHjhhdx8883ceOONnHDCCXz++efccccdQb4lSZIkFZC7rkuSJEmSJEmKe97RKUmSJEmSJCnuWXRKkiRJkiRJinsWnZIkSZIkSZLinkWnJEmSJEmSpLhn0SlJkiRJkiQp7ll0SpIkSZIkSYp7Fp2SJEmSJEmS4p5FpyRJkiRJkqS4Z9EpSZIkSZIkKe5ZdEqSJEmSJEmKexadkiRJkiRJkuKeRackSZIkSZKkuPf/Ac/+u+pc7UL0AAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "fig, ax = plt.subplots(len(search_config_names), 1, figsize=(16, len(search_config_names)*8))\n", - "fig.suptitle(\n", - " f'Effects of index parameters on QPS/recall trade-off ({DATASET_FILENAME})\\n' + \\\n", - " f'k = {k}, n_lists = {n_lists}')\n", - "\n", - "for j, search_label in enumerate(search_config_names):\n", - " labels = []\n", - " for i, index_label in enumerate(build_configs.keys()):\n", - " ax[j].plot(bench_recall_ip[i, j, :], bench_qps_ip[i, j, :])\n", - " labels.append(index_label)\n", - "\n", - " ax[j].set_title(f\"search: {search_label}\")\n", - " ax[j].legend(labels)\n", - " ax[j].set_xlabel('recall')\n", - " ax[j].set_ylabel('QPS')\n", - " ax[j].set_yscale('log')\n", - " ax[j].grid()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Looks like `pq_dim = 128`, `pq_bits = 6` is the best parameter set for the `SIFT-128` dataset." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.16" - }, - "vscode": { - "interpreter": { - "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1" - } - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/notebooks/utils.py b/notebooks/utils.py deleted file mode 100644 index 311efc98bc..0000000000 --- a/notebooks/utils.py +++ /dev/null @@ -1,102 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import cupy as cp -import h5py -import os -import tempfile -import time -import urllib - -## Check the quality of the prediction (recall) -def calc_recall(found_indices, ground_truth): - found_indices = cp.asarray(found_indices) - bs, k = found_indices.shape - if bs != ground_truth.shape[0]: - raise RuntimeError( - "Batch sizes do not match {} vs {}".format( - bs, ground_truth.shape[0] - ) - ) - if k > ground_truth.shape[1]: - raise RuntimeError( - "Not enough indices in the ground truth ({} > {})".format( - k, ground_truth.shape[1] - ) - ) - n = 0 - # Go over the batch - for i in range(bs): - # Note, ivf-pq does not guarantee the ordered input, hence the use of intersect1d - n += cp.intersect1d(found_indices[i, :k], ground_truth[i, :k]).size - recall = n / found_indices.size - return recall - - -class BenchmarkTimer: - """Provides a context manager that runs a code block `reps` times - and records results to the instance variable `timings`. Use like: - .. code-block:: python - timer = BenchmarkTimer(rep=5) - for _ in timer.benchmark_runs(): - ... do something ... - print(np.min(timer.timings)) - - This class is borrowed from the rapids/cuml benchmark suite - """ - - def __init__(self, reps=1, warmup=0): - self.warmup = warmup - self.reps = reps - self.timings = [] - - def benchmark_runs(self): - for r in range(self.reps + self.warmup): - t0 = time.time() - yield r - t1 = time.time() - self.timings.append(t1 - t0) - if r >= self.warmup: - self.timings.append(t1 - t0) - - -def load_dataset(dataset_url="http://ann-benchmarks.com/sift-128-euclidean.hdf5", work_folder=None): - """Download dataset from url. It is expected that the dataset contains a hdf5 file in ann-benchmarks format - - Parameters - ---------- - dataset_url address of hdf5 file - work_folder name of the local folder to store the dataset - - """ - dataset_filename = dataset_url.split("/")[-1] - - # We'll need to load store some data in this tutorial - if work_folder is None: - work_folder = os.path.join(tempfile.gettempdir(), "raft_example") - - if not os.path.exists(work_folder): - os.makedirs(work_folder) - print("The index and data will be saved in", work_folder) - - ## download the dataset - dataset_path = os.path.join(work_folder, dataset_filename) - if not os.path.exists(dataset_path): - urllib.request.urlretrieve(dataset_url, dataset_path) - - f = h5py.File(dataset_path, "r") - - return f diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index 3e3cc15221..9bde613720 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -65,12 +65,14 @@ if(NOT raft_FOUND) add_subdirectory(../../cpp raft-cpp EXCLUDE_FROM_ALL) if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) - set_property(TARGET raft_lib PROPERTY INSTALL_RPATH - "$ORIGIN/../nvidia/cublas/lib" - "$ORIGIN/../nvidia/curand/lib" - "$ORIGIN/../nvidia/cusolver/lib" - "$ORIGIN/../nvidia/cusparse/lib" - "$ORIGIN/../nvidia/nvjitlink/lib" + set_property( + TARGET raft_lib + PROPERTY INSTALL_RPATH + "$ORIGIN/../nvidia/cublas/lib" + "$ORIGIN/../nvidia/curand/lib" + "$ORIGIN/../nvidia/cusolver/lib" + "$ORIGIN/../nvidia/cusparse/lib" + "$ORIGIN/../nvidia/nvjitlink/lib" ) endif() @@ -84,11 +86,7 @@ endif() rapids_cython_init() add_subdirectory(pylibraft/common) -add_subdirectory(pylibraft/distance) -add_subdirectory(pylibraft/matrix) -add_subdirectory(pylibraft/neighbors) add_subdirectory(pylibraft/random) -add_subdirectory(pylibraft/cluster) if(DEFINED cython_lib_dir) rapids_cython_add_rpath_entries(TARGET raft PATHS "${cython_lib_dir}") diff --git a/python/pylibraft/pylibraft/cluster/CMakeLists.txt b/python/pylibraft/pylibraft/cluster/CMakeLists.txt deleted file mode 100644 index 562cff5098..0000000000 --- a/python/pylibraft/pylibraft/cluster/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -# ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -# Set the list of Cython files to build -set(cython_sources kmeans.pyx) -set(linked_libraries raft::compiled) - -# Build all of the Cython targets -rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX cluster_ -) diff --git a/python/pylibraft/pylibraft/cluster/__init__.pxd b/python/pylibraft/pylibraft/cluster/__init__.pxd deleted file mode 100644 index 273b4497cc..0000000000 --- a/python/pylibraft/pylibraft/cluster/__init__.pxd +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/pylibraft/pylibraft/cluster/__init__.py b/python/pylibraft/pylibraft/cluster/__init__.py deleted file mode 100644 index 00b12eab9b..0000000000 --- a/python/pylibraft/pylibraft/cluster/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from .kmeans import ( - KMeansParams, - cluster_cost, - compute_new_centroids, - fit, - init_plus_plus, -) - -__all__ = [ - "KMeansParams", - "cluster_cost", - "compute_new_centroids", - "fit", - "init_plus_plus", -] diff --git a/python/pylibraft/pylibraft/cluster/cpp/__init__.pxd b/python/pylibraft/pylibraft/cluster/cpp/__init__.pxd deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/python/pylibraft/pylibraft/cluster/cpp/__init__.py b/python/pylibraft/pylibraft/cluster/cpp/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/python/pylibraft/pylibraft/cluster/cpp/kmeans.pxd b/python/pylibraft/pylibraft/cluster/cpp/kmeans.pxd deleted file mode 100644 index 4a5a47de68..0000000000 --- a/python/pylibraft/pylibraft/cluster/cpp/kmeans.pxd +++ /dev/null @@ -1,106 +0,0 @@ -# -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -from cython.operator cimport dereference as deref -from libc.stdint cimport uintptr_t -from libcpp cimport bool, nullptr - -from pylibraft.cluster.cpp.kmeans_types cimport KMeansParams -from pylibraft.common.cpp.mdspan cimport * -from pylibraft.common.cpp.optional cimport optional -from pylibraft.common.handle cimport device_resources - - -cdef extern from "raft_runtime/cluster/kmeans.hpp" \ - namespace "raft::runtime::cluster::kmeans" nogil: - - cdef void update_centroids( - const device_resources& handle, - const double *X, - int n_samples, - int n_features, - int n_clusters, - const double *sample_weights, - const double *centroids, - const int* labels, - double *new_centroids, - double *weight_per_cluster) except + - - cdef void update_centroids( - const device_resources& handle, - const float *X, - int n_samples, - int n_features, - int n_clusters, - const float *sample_weights, - const float *centroids, - const int* labels, - float *new_centroids, - float *weight_per_cluster) except + - - cdef void cluster_cost( - const device_resources& handle, - const float* X, - int n_samples, - int n_features, - int n_clusters, - const float * centroids, - float * cost) except + - - cdef void cluster_cost( - const device_resources& handle, - const double* X, - int n_samples, - int n_features, - int n_clusters, - const double * centroids, - double * cost) except + - - cdef void init_plus_plus( - const device_resources & handle, - const KMeansParams& params, - device_matrix_view[float, int, row_major] X, - device_matrix_view[float, int, row_major] centroids) except + - - cdef void init_plus_plus( - const device_resources & handle, - const KMeansParams& params, - device_matrix_view[double, int, row_major] X, - device_matrix_view[double, int, row_major] centroids) except + - - cdef void fit( - const device_resources & handle, - const KMeansParams& params, - device_matrix_view[float, int, row_major] X, - optional[device_vector_view[float, int]] sample_weight, - device_matrix_view[float, int, row_major] inertia, - host_scalar_view[float, int] inertia, - host_scalar_view[int, int] n_iter) except + - - cdef void fit( - const device_resources & handle, - const KMeansParams& params, - device_matrix_view[double, int, row_major] X, - optional[device_vector_view[double, int]] sample_weight, - device_matrix_view[double, int, row_major] inertia, - host_scalar_view[double, int] inertia, - host_scalar_view[int, int] n_iter) except + diff --git a/python/pylibraft/pylibraft/cluster/cpp/kmeans_types.pxd b/python/pylibraft/pylibraft/cluster/cpp/kmeans_types.pxd deleted file mode 100644 index 12cecd4336..0000000000 --- a/python/pylibraft/pylibraft/cluster/cpp/kmeans_types.pxd +++ /dev/null @@ -1,44 +0,0 @@ -# -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from libcpp cimport bool - -from pylibraft.distance.distance_type cimport DistanceType -from pylibraft.random.cpp.rng_state cimport RngState - - -cdef extern from "raft/cluster/kmeans_types.hpp" \ - namespace "raft::cluster::kmeans": - - ctypedef enum InitMethod 'raft::cluster::KMeansParams::InitMethod': - KMeansPlusPlus 'raft::cluster::kmeans::KMeansParams::InitMethod::KMeansPlusPlus' # noqa - Random 'raft::cluster::kmeans::KMeansParams::InitMethod::Random' - Array 'raft::cluster::kmeans::KMeansParams::InitMethod::Array' - - cdef cppclass KMeansParams: - KMeansParams() except + - int n_clusters - InitMethod init - int max_iter - double tol - int verbosity - RngState rng_state - DistanceType metric - int n_init - double oversampling_factor - int batch_samples - int batch_centroids - bool inertia_check diff --git a/python/pylibraft/pylibraft/cluster/kmeans.pyx b/python/pylibraft/pylibraft/cluster/kmeans.pyx deleted file mode 100644 index f4af519dc1..0000000000 --- a/python/pylibraft/pylibraft/cluster/kmeans.pyx +++ /dev/null @@ -1,589 +0,0 @@ -# -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -from cython.operator cimport dereference as deref -from libc.stdint cimport uintptr_t -from libcpp cimport nullptr - -from collections import namedtuple -from enum import IntEnum - -from pylibraft.common import Handle, cai_wrapper, device_ndarray -from pylibraft.common.handle import auto_sync_handle - -from pylibraft.common.handle cimport device_resources -from pylibraft.random.cpp.rng_state cimport RngState - -from pylibraft.common.input_validation import * -from pylibraft.distance import DISTANCE_TYPES - -from pylibraft.cluster.cpp cimport kmeans as cpp_kmeans, kmeans_types -from pylibraft.cluster.cpp.kmeans cimport ( - cluster_cost as cpp_cluster_cost, - init_plus_plus as cpp_init_plus_plus, - update_centroids, -) -from pylibraft.common.cpp.mdspan cimport * -from pylibraft.common.cpp.optional cimport optional -from pylibraft.common.handle cimport device_resources - -from pylibraft.common import auto_convert_output - - -@auto_sync_handle -@auto_convert_output -def compute_new_centroids(X, - centroids, - labels, - new_centroids, - sample_weights=None, - weight_per_cluster=None, - handle=None): - """ - Compute new centroids given an input matrix and existing centroids - - Parameters - ---------- - - X : Input CUDA array interface compliant matrix shape (m, k) - centroids : Input CUDA array interface compliant matrix shape - (n_clusters, k) - labels : Input CUDA array interface compliant matrix shape - (m, 1) - new_centroids : Writable CUDA array interface compliant matrix shape - (n_clusters, k) - sample_weights : Optional input CUDA array interface compliant matrix shape - (n_clusters, 1) default: None - weight_per_cluster : Optional writable CUDA array interface compliant - matrix shape (n_clusters, 1) default: None - batch_samples : Optional integer specifying the batch size for X to compute - distances in batches. default: m - batch_centroids : Optional integer specifying the batch size for centroids - to compute distances in batches. default: n_clusters - {handle_docstring} - - Examples - -------- - - >>> import cupy as cp - >>> from pylibraft.common import Handle - >>> from pylibraft.cluster.kmeans import compute_new_centroids - >>> # A single RAFT handle can optionally be reused across - >>> # pylibraft functions. - >>> handle = Handle() - >>> n_samples = 5000 - >>> n_features = 50 - >>> n_clusters = 3 - >>> X = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> centroids = cp.random.random_sample((n_clusters, n_features), - ... dtype=cp.float32) - ... - >>> labels = cp.random.randint(0, high=n_clusters, size=n_samples, - ... dtype=cp.int32) - >>> new_centroids = cp.empty((n_clusters, n_features), - ... dtype=cp.float32) - >>> compute_new_centroids( - ... X, centroids, labels, new_centroids, handle=handle - ... ) - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - """ - - x_cai = X.__cuda_array_interface__ - centroids_cai = centroids.__cuda_array_interface__ - new_centroids_cai = new_centroids.__cuda_array_interface__ - labels_cai = labels.__cuda_array_interface__ - - m = x_cai["shape"][0] - x_k = x_cai["shape"][1] - n_clusters = centroids_cai["shape"][0] - - centroids_k = centroids_cai["shape"][1] - new_centroids_k = centroids_cai["shape"][1] - - x_dt = np.dtype(x_cai["typestr"]) - centroids_dt = np.dtype(centroids_cai["typestr"]) - new_centroids_dt = np.dtype(new_centroids_cai["typestr"]) - labels_dt = np.dtype(labels_cai["typestr"]) - - if not do_cols_match(X, centroids): - raise ValueError("X and centroids must have same number of columns.") - - if not do_rows_match(X, labels): - raise ValueError("X and labels must have same number of rows") - - x_ptr = x_cai["data"][0] - centroids_ptr = centroids_cai["data"][0] - new_centroids_ptr = new_centroids_cai["data"][0] - labels_ptr = labels_cai["data"][0] - - if sample_weights is not None: - sample_weights_cai = sample_weights.__cuda_array_interface__ - sample_weights_ptr = sample_weights_cai["data"][0] - sample_weights_dt = np.dtype(sample_weights_cai["typestr"]) - else: - sample_weights_ptr = nullptr - - if weight_per_cluster is not None: - weight_per_cluster_cai = weight_per_cluster.__cuda_array_interface__ - weight_per_cluster_ptr = weight_per_cluster_cai["data"][0] - weight_per_cluster_dt = np.dtype(weight_per_cluster_cai["typestr"]) - else: - weight_per_cluster_ptr = nullptr - - handle = handle if handle is not None else Handle() - cdef device_resources *h = handle.getHandle() - - x_c_contiguous = is_c_contiguous(x_cai) - centroids_c_contiguous = is_c_contiguous(centroids_cai) - new_centroids_c_contiguous = is_c_contiguous(new_centroids_cai) - - if not x_c_contiguous or not centroids_c_contiguous \ - or not new_centroids_c_contiguous: - raise ValueError("Inputs must all be c contiguous") - - if not do_dtypes_match(X, centroids, new_centroids): - raise ValueError("Inputs must all have the same dtypes " - "(float32 or float64)") - - if x_dt == np.float32: - update_centroids(deref(h), - x_ptr, - m, - x_k, - n_clusters, - sample_weights_ptr, - centroids_ptr, - labels_ptr, - new_centroids_ptr, - weight_per_cluster_ptr) - elif x_dt == np.float64: - update_centroids(deref(h), - x_ptr, - m, - x_k, - n_clusters, - sample_weights_ptr, - centroids_ptr, - labels_ptr, - new_centroids_ptr, - weight_per_cluster_ptr) - else: - raise ValueError("dtype %s not supported" % x_dt) - - -@auto_sync_handle -@auto_convert_output -def init_plus_plus(X, n_clusters=None, seed=None, handle=None, centroids=None): - """ - Compute initial centroids using the "kmeans++" algorithm. - - Parameters - ---------- - - X : Input CUDA array interface compliant matrix shape (m, k) - n_clusters : Number of clusters to select - seed : Controls the random sampling of centroids - centroids : Optional writable CUDA array interface compliant matrix shape - (n_clusters, k). Use instead of passing `n_clusters`. - {handle_docstring} - - Examples - -------- - - >>> import cupy as cp - >>> from pylibraft.cluster.kmeans import init_plus_plus - >>> n_samples = 5000 - >>> n_features = 50 - >>> n_clusters = 3 - >>> X = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - - >>> centroids = init_plus_plus(X, n_clusters) - """ - if (n_clusters is not None and - centroids is not None and n_clusters != centroids.shape[0]): - msg = ("Parameters 'n_clusters' and 'centroids' " - "are exclusive. Only pass one at a time.") - raise RuntimeError(msg) - - cdef device_resources *h = handle.getHandle() - - X_cai = cai_wrapper(X) - X_cai.validate_shape_dtype(expected_dims=2) - dtype = X_cai.dtype - - if centroids is not None: - n_clusters = centroids.shape[0] - else: - centroids_shape = (n_clusters, X_cai.shape[1]) - centroids = device_ndarray.empty(centroids_shape, dtype=dtype) - - centroids_cai = cai_wrapper(centroids) - - # Can't set attributes of KMeansParameters after creating it, so taking - # a detour via a dict to collect the possible constructor arguments - params_ = dict(n_clusters=n_clusters) - if seed is not None: - params_["seed"] = seed - params = KMeansParams(**params_) - - if dtype == np.float64: - cpp_init_plus_plus( - deref(h), params.c_obj, - make_device_matrix_view[double, int, row_major]( - X_cai.data, - X_cai.shape[0], X_cai.shape[1]), - make_device_matrix_view[double, int, row_major]( - centroids_cai.data, - centroids_cai.shape[0], centroids_cai.shape[1]), - ) - elif dtype == np.float32: - cpp_init_plus_plus( - deref(h), params.c_obj, - make_device_matrix_view[float, int, row_major]( - X_cai.data, - X_cai.shape[0], X_cai.shape[1]), - make_device_matrix_view[float, int, row_major]( - centroids_cai.data, - centroids_cai.shape[0], centroids_cai.shape[1]), - ) - else: - raise ValueError(f"Unhandled dtype ({dtype}) for X.") - - return centroids - - -@auto_sync_handle -@auto_convert_output -def cluster_cost(X, centroids, handle=None): - """ - Compute cluster cost given an input matrix and existing centroids - - Parameters - ---------- - X : Input CUDA array interface compliant matrix shape (m, k) - centroids : Input CUDA array interface compliant matrix shape - (n_clusters, k) - {handle_docstring} - - Examples - -------- - - >>> import cupy as cp - >>> from pylibraft.cluster.kmeans import cluster_cost - >>> n_samples = 5000 - >>> n_features = 50 - >>> n_clusters = 3 - >>> X = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> centroids = cp.random.random_sample((n_clusters, n_features), - ... dtype=cp.float32) - >>> inertia = cluster_cost(X, centroids) - """ - x_cai = X.__cuda_array_interface__ - centroids_cai = centroids.__cuda_array_interface__ - - m = x_cai["shape"][0] - x_k = x_cai["shape"][1] - n_clusters = centroids_cai["shape"][0] - - centroids_k = centroids_cai["shape"][1] - - x_dt = np.dtype(x_cai["typestr"]) - centroids_dt = np.dtype(centroids_cai["typestr"]) - - if not do_cols_match(X, centroids): - raise ValueError("X and centroids must have same number of columns.") - - x_ptr = x_cai["data"][0] - centroids_ptr = centroids_cai["data"][0] - - handle = handle if handle is not None else Handle() - cdef device_resources *h = handle.getHandle() - - x_c_contiguous = is_c_contiguous(x_cai) - centroids_c_contiguous = is_c_contiguous(centroids_cai) - - if not x_c_contiguous or not centroids_c_contiguous: - raise ValueError("Inputs must all be c contiguous") - - if not do_dtypes_match(X, centroids): - raise ValueError("Inputs must all have the same dtypes " - "(float32 or float64)") - - cdef float f_cost = 0 - cdef double d_cost = 0 - - if x_dt == np.float32: - cpp_cluster_cost(deref(h), - x_ptr, - m, - x_k, - n_clusters, - centroids_ptr, - &f_cost) - return f_cost - elif x_dt == np.float64: - cpp_cluster_cost(deref(h), - x_ptr, - m, - x_k, - n_clusters, - centroids_ptr, - &d_cost) - return d_cost - else: - raise ValueError("dtype %s not supported" % x_dt) - - -class InitMethod(IntEnum): - """ Method for initializing kmeans """ - KMeansPlusPlus = kmeans_types.InitMethod.KMeansPlusPlus - Random = kmeans_types.InitMethod.Random - Array = kmeans_types.InitMethod.Array - - -cdef class KMeansParams: - """ Specifies hyper-parameters for the kmeans algorithm. - - Parameters - ---------- - n_clusters : int, optional - The number of clusters to form as well as the number of centroids - to generate - max_iter : int, optional - Maximum number of iterations of the k-means algorithm for a single run - tol : float, optional - Relative tolerance with regards to inertia to declare convergence - verbosity : int, optional - seed: int, optional - Seed to the random number generator. - metric : str, optional - Metric names to use for distance computation, see - :func:`pylibraft.distance.pairwise_distance` for valid values. - init : InitMethod, optional - n_init : int, optional - Number of instance k-means algorithm will be run with different seeds. - oversampling_factor : float, optional - Oversampling factor for use in the k-means algorithm - """ - cdef kmeans_types.KMeansParams c_obj - - def __init__(self, - n_clusters: Optional[int] = None, - max_iter: Optional[int] = None, - tol: Optional[float] = None, - verbosity: Optional[int] = None, - seed: Optional[int] = None, - metric: Optional[str] = None, - init: Optional[InitMethod] = None, - n_init: Optional[int] = None, - oversampling_factor: Optional[float] = None, - batch_samples: Optional[int] = None, - batch_centroids: Optional[int] = None, - inertia_check: Optional[bool] = None): - if n_clusters is not None: - self.c_obj.n_clusters = n_clusters - if max_iter is not None: - self.c_obj.max_iter = max_iter - if tol is not None: - self.c_obj.tol = tol - if verbosity is not None: - self.c_obj.verbosity = verbosity - if seed is not None: - self.c_obj.rng_state.seed = seed - if metric is not None: - distance = DISTANCE_TYPES.get(metric) - if distance is None: - valid_metrics = list(DISTANCE_TYPES.keys()) - raise ValueError(f"Unknown metric '{metric}'. Valid values " - f"are: {valid_metrics}") - self.c_obj.metric = distance - if init is not None: - self.c_obj.init = init - if n_init is not None: - self.c_obj.n_init = n_init - if oversampling_factor is not None: - self.c_obj.oversampling_factor = oversampling_factor - if batch_samples is not None: - self.c_obj.batch_samples = batch_samples - if batch_centroids is not None: - self.c_obj.batch_centroids = batch_centroids - if inertia_check is not None: - self.c_obj.inertia_check = inertia_check - - @property - def n_clusters(self): - return self.c_obj.n_clusters - - @property - def max_iter(self): - return self.c_obj.max_iter - - @property - def tol(self): - return self.c_obj.tol - - @property - def verbosity(self): - return self.c_obj.verbosity - - @property - def seed(self): - return self.c_obj.rng_state.seed - - @property - def init(self): - return InitMethod(self.c_obj.init) - - @property - def oversampling_factor(self): - return self.c_obj.oversampling_factor - - @property - def batch_samples(self): - return self.c_obj.batch_samples - - @property - def batch_centroids(self): - return self.c_obj.batch_centroids - - @property - def inertia_check(self): - return self.c_obj.inertia_check - -FitOutput = namedtuple("FitOutput", "centroids inertia n_iter") - - -@auto_sync_handle -@auto_convert_output -def fit( - KMeansParams params, X, centroids=None, sample_weights=None, handle=None -): - """ - Find clusters with the k-means algorithm - - Parameters - ---------- - - params : KMeansParams - Parameters to use to fit KMeans model - X : Input CUDA array interface compliant matrix shape (m, k) - centroids : Optional writable CUDA array interface compliant matrix - shape (n_clusters, k) - sample_weights : Optional input CUDA array interface compliant matrix shape - (n_clusters, 1) default: None - {handle_docstring} - - Returns - ------- - centroids : raft.device_ndarray - The computed centroids for each cluster - inertia : float - Sum of squared distances of samples to their closest cluster center - n_iter : int - The number of iterations used to fit the model - - Examples - -------- - - >>> import cupy as cp - >>> from pylibraft.cluster.kmeans import fit, KMeansParams - >>> n_samples = 5000 - >>> n_features = 50 - >>> n_clusters = 3 - >>> X = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - - >>> params = KMeansParams(n_clusters=n_clusters) - >>> centroids, inertia, n_iter = fit(params, X) - """ - cdef device_resources *h = handle.getHandle() - - cdef float f_inertia = 0.0 - cdef double d_inertia = 0.0 - cdef int n_iter = 0 - - cdef optional[device_vector_view[const double, int]] d_sample_weights - cdef optional[device_vector_view[const float, int]] f_sample_weights - - X_cai = cai_wrapper(X) - dtype = X_cai.dtype - - if centroids is None: - centroids_shape = (params.n_clusters, X_cai.shape[1]) - centroids = device_ndarray.empty(centroids_shape, dtype=dtype) - centroids_cai = cai_wrapper(centroids) - - # validate inputs have are all c-contiguous, and have a consistent dtype - # and expected shape - X_cai.validate_shape_dtype(2) - centroids_cai.validate_shape_dtype(2, dtype) - if sample_weights is not None: - sample_weights_cai = cai_wrapper(sample_weights) - sample_weights_cai.validate_shape_dtype(1, dtype) - - if dtype == np.float64: - if sample_weights is not None: - d_sample_weights = make_device_vector_view( - sample_weights_cai.data, - sample_weights_cai.shape[0]) - - cpp_kmeans.fit( - deref(h), - params.c_obj, - make_device_matrix_view[double, int, row_major]( - X_cai.data, - X_cai.shape[0], X_cai.shape[1]), - d_sample_weights, - make_device_matrix_view[double, int, row_major]( - centroids_cai.data, - centroids_cai.shape[0], centroids_cai.shape[1]), - make_host_scalar_view[double, int](&d_inertia), - make_host_scalar_view[int, int](&n_iter)) - return FitOutput(centroids, d_inertia, n_iter) - - elif dtype == np.float32: - if sample_weights is not None: - f_sample_weights = make_device_vector_view( - sample_weights_cai.data, - sample_weights_cai.shape[0]) - - cpp_kmeans.fit( - deref(h), - params.c_obj, - make_device_matrix_view[float, int, row_major]( - X_cai.data, - X_cai.shape[0], X_cai.shape[1]), - f_sample_weights, - make_device_matrix_view[float, int, row_major]( - centroids_cai.data, - centroids_cai.shape[0], centroids_cai.shape[1]), - make_host_scalar_view[float, int](&f_inertia), - make_host_scalar_view[int, int](&n_iter)) - return FitOutput(centroids, f_inertia, n_iter) - - else: - raise ValueError(f"unhandled dtype {dtype}") diff --git a/python/pylibraft/pylibraft/distance/CMakeLists.txt b/python/pylibraft/pylibraft/distance/CMakeLists.txt deleted file mode 100644 index 2530e07a98..0000000000 --- a/python/pylibraft/pylibraft/distance/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -# ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -# Set the list of Cython files to build -set(cython_sources pairwise_distance.pyx fused_l2_nn.pyx fused_distance_nn.pyx) -set(linked_libraries raft::raft raft::compiled) - -# Build all of the Cython targets -rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX distance_ -) diff --git a/python/pylibraft/pylibraft/distance/__init__.pxd b/python/pylibraft/pylibraft/distance/__init__.pxd deleted file mode 100644 index 273b4497cc..0000000000 --- a/python/pylibraft/pylibraft/distance/__init__.pxd +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/pylibraft/pylibraft/distance/__init__.py b/python/pylibraft/pylibraft/distance/__init__.py deleted file mode 100644 index d16ab30b2f..0000000000 --- a/python/pylibraft/pylibraft/distance/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from .fused_distance_nn import fused_distance_nn_argmin -from .fused_l2_nn import fused_l2_nn_argmin -from .pairwise_distance import DISTANCE_TYPES, distance as pairwise_distance - -__all__ = [ - "fused_distance_nn_argmin", - "fused_l2_nn_argmin", - "pairwise_distance", -] diff --git a/python/pylibraft/pylibraft/distance/distance_type.pxd b/python/pylibraft/pylibraft/distance/distance_type.pxd deleted file mode 100644 index e058238d45..0000000000 --- a/python/pylibraft/pylibraft/distance/distance_type.pxd +++ /dev/null @@ -1,44 +0,0 @@ -# -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -cdef extern from "raft/distance/distance_types.hpp" namespace "raft::distance": - - ctypedef enum DistanceType: - L2Expanded "raft::distance::DistanceType::L2Expanded" - L2SqrtExpanded "raft::distance::DistanceType::L2SqrtExpanded" - CosineExpanded "raft::distance::DistanceType::CosineExpanded" - L1 "raft::distance::DistanceType::L1" - L2Unexpanded "raft::distance::DistanceType::L2Unexpanded" - L2SqrtUnexpanded "raft::distance::DistanceType::L2SqrtUnexpanded" - InnerProduct "raft::distance::DistanceType::InnerProduct" - Linf "raft::distance::DistanceType::Linf" - Canberra "raft::distance::DistanceType::Canberra" - LpUnexpanded "raft::distance::DistanceType::LpUnexpanded" - CorrelationExpanded "raft::distance::DistanceType::CorrelationExpanded" - JaccardExpanded "raft::distance::DistanceType::JaccardExpanded" - HellingerExpanded "raft::distance::DistanceType::HellingerExpanded" - Haversine "raft::distance::DistanceType::Haversine" - BrayCurtis "raft::distance::DistanceType::BrayCurtis" - JensenShannon "raft::distance::DistanceType::JensenShannon" - HammingUnexpanded "raft::distance::DistanceType::HammingUnexpanded" - KLDivergence "raft::distance::DistanceType::KLDivergence" - RusselRaoExpanded "raft::distance::DistanceType::RusselRaoExpanded" - DiceExpanded "raft::distance::DistanceType::DiceExpanded" - Precomputed "raft::distance::DistanceType::Precomputed" diff --git a/python/pylibraft/pylibraft/distance/fused_distance_nn.pyx b/python/pylibraft/pylibraft/distance/fused_distance_nn.pyx deleted file mode 100644 index 0e9fa4b366..0000000000 --- a/python/pylibraft/pylibraft/distance/fused_distance_nn.pyx +++ /dev/null @@ -1,200 +0,0 @@ -# -# Copyright (c) 2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -from cython.operator cimport dereference as deref -from libc.stdint cimport uintptr_t -from libcpp cimport bool - -from .distance_type cimport DistanceType - -from pylibraft.common import ( - Handle, - auto_convert_output, - cai_wrapper, - device_ndarray, -) -from pylibraft.common.handle import auto_sync_handle - -from pylibraft.common.handle cimport device_resources - - -cdef extern from "raft_runtime/distance/fused_distance_nn.hpp" \ - namespace "raft::runtime::distance" nogil: - - void fused_distance_nn_min_arg( - const device_resources &handle, - int* min, - const float* x, - const float* y, - int m, - int n, - int k, - bool sqrt, - DistanceType metric, - bool isRowMajor, - float metric_arg) except + - - -from pylibraft.distance.pairwise_distance import DISTANCE_TYPES - -SUPPORTED_DISTANCES = ["euclidean", "l2", "cosine", "sqeuclidean"] - - -@auto_sync_handle -@auto_convert_output -def fused_distance_nn_argmin(X, Y, out=None, sqrt=True, metric="euclidean", - handle=None): - """ - Compute the 1-nearest neighbors between X and Y using the distance metrics - - Valid values for metric: - ["euclidean", "l2", "cosine", "sqeuclidean"] - - Parameters - ---------- - - X : CUDA array interface compliant matrix shape (m, k) - Y : CUDA array interface compliant matrix shape (n, k) - out : Writable CUDA array interface matrix shape (m, 1) - metric : string denoting the metric type (default="euclidean") - - {handle_docstring} - - Examples - -------- - To compute the 1-nearest neighbors argmin: - - >>> import cupy as cp - >>> from pylibraft.common import Handle - >>> from pylibraft.distance import fused_distance_nn_argmin - >>> n_samples = 5000 - >>> n_clusters = 5 - >>> n_features = 50 - >>> in1 = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> in2 = cp.random.random_sample((n_clusters, n_features), - ... dtype=cp.float32) - >>> # A single RAFT handle can optionally be reused across - >>> # pylibraft functions. - >>> handle = Handle() - - >>> output = fused_distance_nn_argmin(in1, in2, handle=handle) - - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - - The output can also be computed in-place on a preallocated - array: - - >>> import cupy as cp - >>> from pylibraft.common import Handle - >>> from pylibraft.distance import fused_distance_nn_argmin - >>> n_samples = 5000 - >>> n_clusters = 5 - >>> n_features = 50 - >>> in1 = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> in2 = cp.random.random_sample((n_clusters, n_features), - ... dtype=cp.float32) - >>> output = cp.empty((n_samples, 1), dtype=cp.int32) - >>> # A single RAFT handle can optionally be reused across - >>> # pylibraft functions. - >>> handle = Handle() - - >>> fused_distance_nn_argmin(in1, in2, out=output, handle=handle) - array(...) - - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - """ - - x_cai = cai_wrapper(X) - y_cai = cai_wrapper(Y) - - x_dt = x_cai.dtype - y_dt = y_cai.dtype - - m = x_cai.shape[0] - n = y_cai.shape[0] - - if out is None: - output = device_ndarray.empty((m,), dtype="int32") - else: - output = out - - output_cai = cai_wrapper(output) - - x_k = x_cai.shape[1] - y_k = y_cai.shape[1] - - if x_k != y_k: - raise ValueError("Inputs must have same number of columns. " - "a=%s, b=%s" % (x_k, y_k)) - - if metric not in SUPPORTED_DISTANCES: - raise ValueError("metric %s is not supported" % metric) - - cdef DistanceType distance_type = DISTANCE_TYPES[metric] - - x_ptr = x_cai.data - y_ptr = y_cai.data - - d_ptr = output_cai.data - - handle = handle if handle is not None else Handle() - cdef device_resources *h = handle.getHandle() - - d_dt = output_cai.dtype - - x_c_contiguous = x_cai.c_contiguous - y_c_contiguous = y_cai.c_contiguous - - if x_c_contiguous != y_c_contiguous: - raise ValueError("Inputs must have matching strides") - - if not x_c_contiguous: - raise ValueError("Inputs must be C contiguous") - - if x_dt != y_dt: - raise ValueError("Inputs must have the same dtypes") - if d_dt != np.int32: - raise ValueError("Output array must be int32") - # unused arg for now. - metric_arg = 0.0 - if x_dt == np.float32: - fused_distance_nn_min_arg(deref(h), - d_ptr, - x_ptr, - y_ptr, - m, - n, - x_k, - sqrt, - distance_type, - x_c_contiguous, - metric_arg) - else: - raise ValueError("dtype %s not supported" % x_dt) - - return output diff --git a/python/pylibraft/pylibraft/distance/fused_l2_nn.pyx b/python/pylibraft/pylibraft/distance/fused_l2_nn.pyx deleted file mode 100644 index c8e7101ee0..0000000000 --- a/python/pylibraft/pylibraft/distance/fused_l2_nn.pyx +++ /dev/null @@ -1,193 +0,0 @@ -# -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -from cython.operator cimport dereference as deref -from libc.stdint cimport uintptr_t -from libcpp cimport bool - -from .distance_type cimport DistanceType - -from pylibraft.common import ( - Handle, - auto_convert_output, - cai_wrapper, - device_ndarray, -) -from pylibraft.common.handle import auto_sync_handle - -from pylibraft.common.handle cimport device_resources - - -cdef extern from "raft_runtime/distance/fused_l2_nn.hpp" \ - namespace "raft::runtime::distance" nogil: - - void fused_l2_nn_min_arg( - const device_resources &handle, - int* min, - const float* x, - const float* y, - int m, - int n, - int k, - bool sqrt) except + - - void fused_l2_nn_min_arg( - const device_resources &handle, - int* min, - const double* x, - const double* y, - int m, - int n, - int k, - bool sqrt) except + - - -@auto_sync_handle -@auto_convert_output -def fused_l2_nn_argmin(X, Y, out=None, sqrt=True, handle=None): - """ - Compute the 1-nearest neighbors between X and Y using the L2 distance - - Parameters - ---------- - - X : CUDA array interface compliant matrix shape (m, k) - Y : CUDA array interface compliant matrix shape (n, k) - output : Writable CUDA array interface matrix shape (m, 1) - {handle_docstring} - - Examples - -------- - To compute the 1-nearest neighbors argmin: - - >>> import cupy as cp - >>> from pylibraft.common import Handle - >>> from pylibraft.distance import fused_l2_nn_argmin - >>> n_samples = 5000 - >>> n_clusters = 5 - >>> n_features = 50 - >>> in1 = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> in2 = cp.random.random_sample((n_clusters, n_features), - ... dtype=cp.float32) - >>> # A single RAFT handle can optionally be reused across - >>> # pylibraft functions. - >>> handle = Handle() - - >>> output = fused_l2_nn_argmin(in1, in2, handle=handle) - - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - - The output can also be computed in-place on a preallocated - array: - - >>> import cupy as cp - >>> from pylibraft.common import Handle - >>> from pylibraft.distance import fused_l2_nn_argmin - >>> n_samples = 5000 - >>> n_clusters = 5 - >>> n_features = 50 - >>> in1 = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> in2 = cp.random.random_sample((n_clusters, n_features), - ... dtype=cp.float32) - >>> output = cp.empty((n_samples, 1), dtype=cp.int32) - >>> # A single RAFT handle can optionally be reused across - >>> # pylibraft functions. - >>> handle = Handle() - - >>> fused_l2_nn_argmin(in1, in2, out=output, handle=handle) - array(...) - - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - """ - - x_cai = cai_wrapper(X) - y_cai = cai_wrapper(Y) - - x_dt = x_cai.dtype - y_dt = y_cai.dtype - - m = x_cai.shape[0] - n = y_cai.shape[0] - - if out is None: - output = device_ndarray.empty((m,), dtype="int32") - else: - output = out - - output_cai = cai_wrapper(output) - - x_k = x_cai.shape[1] - y_k = y_cai.shape[1] - - if x_k != y_k: - raise ValueError("Inputs must have same number of columns. " - "a=%s, b=%s" % (x_k, y_k)) - - x_ptr = x_cai.data - y_ptr = y_cai.data - - d_ptr = output_cai.data - - handle = handle if handle is not None else Handle() - cdef device_resources *h = handle.getHandle() - - d_dt = output_cai.dtype - - x_c_contiguous = x_cai.c_contiguous - y_c_contiguous = y_cai.c_contiguous - - if x_c_contiguous != y_c_contiguous: - raise ValueError("Inputs must have matching strides") - - if x_dt != y_dt: - raise ValueError("Inputs must have the same dtypes") - if d_dt != np.int32: - raise ValueError("Output array must be int32") - - if x_dt == np.float32: - fused_l2_nn_min_arg(deref(h), - d_ptr, - x_ptr, - y_ptr, - m, - n, - x_k, - sqrt) - elif x_dt == np.float64: - fused_l2_nn_min_arg(deref(h), - d_ptr, - x_ptr, - y_ptr, - m, - n, - x_k, - sqrt) - else: - raise ValueError("dtype %s not supported" % x_dt) - - return output diff --git a/python/pylibraft/pylibraft/distance/pairwise_distance.pyx b/python/pylibraft/pylibraft/distance/pairwise_distance.pyx deleted file mode 100644 index 20dadf0275..0000000000 --- a/python/pylibraft/pylibraft/distance/pairwise_distance.pyx +++ /dev/null @@ -1,242 +0,0 @@ -# -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -from cython.operator cimport dereference as deref -from libc.stdint cimport uintptr_t -from libcpp cimport bool - -from .distance_type cimport DistanceType - -from pylibraft.common import Handle -from pylibraft.common.handle import auto_sync_handle - -from pylibraft.common.handle cimport device_resources - -from pylibraft.common import auto_convert_output, cai_wrapper, device_ndarray - - -cdef extern from "raft_runtime/distance/pairwise_distance.hpp" \ - namespace "raft::runtime::distance" nogil: - - cdef void pairwise_distance(const device_resources &handle, - float *x, - float *y, - float *dists, - int m, - int n, - int k, - DistanceType metric, - bool isRowMajor, - float metric_arg) except + - - cdef void pairwise_distance(const device_resources &handle, - double *x, - double *y, - double *dists, - int m, - int n, - int k, - DistanceType metric, - bool isRowMajor, - float metric_arg) except + - -DISTANCE_TYPES = { - "l2": DistanceType.L2SqrtExpanded, - "sqeuclidean": DistanceType.L2Expanded, - "euclidean": DistanceType.L2SqrtExpanded, - "l1": DistanceType.L1, - "cityblock": DistanceType.L1, - "inner_product": DistanceType.InnerProduct, - "chebyshev": DistanceType.Linf, - "canberra": DistanceType.Canberra, - "cosine": DistanceType.CosineExpanded, - "lp": DistanceType.LpUnexpanded, - "correlation": DistanceType.CorrelationExpanded, - "jaccard": DistanceType.JaccardExpanded, - "hellinger": DistanceType.HellingerExpanded, - "braycurtis": DistanceType.BrayCurtis, - "jensenshannon": DistanceType.JensenShannon, - "hamming": DistanceType.HammingUnexpanded, - "kl_divergence": DistanceType.KLDivergence, - "minkowski": DistanceType.LpUnexpanded, - "russellrao": DistanceType.RusselRaoExpanded, - "dice": DistanceType.DiceExpanded, -} - -SUPPORTED_DISTANCES = ["euclidean", "l1", "cityblock", "l2", "inner_product", - "chebyshev", "minkowski", "canberra", "kl_divergence", - "correlation", "russellrao", "hellinger", "lp", - "hamming", "jensenshannon", "cosine", "sqeuclidean"] - - -@auto_sync_handle -@auto_convert_output -def distance(X, Y, out=None, metric="euclidean", p=2.0, handle=None): - """ - Compute pairwise distances between X and Y - - Valid values for metric: - ["euclidean", "l2", "l1", "cityblock", "inner_product", - "chebyshev", "canberra", "lp", "hellinger", "jensenshannon", - "kl_divergence", "russellrao", "minkowski", "correlation", - "cosine"] - - Parameters - ---------- - - X : CUDA array interface compliant matrix shape (m, k) - Y : CUDA array interface compliant matrix shape (n, k) - out : Optional writable CUDA array interface matrix shape (m, n) - metric : string denoting the metric type (default="euclidean") - p : metric parameter (currently used only for "minkowski") - {handle_docstring} - - Returns - ------- - - raft.device_ndarray containing pairwise distances - - Examples - -------- - To compute pairwise distances on cupy arrays: - - >>> import cupy as cp - >>> from pylibraft.common import Handle - >>> from pylibraft.distance import pairwise_distance - >>> n_samples = 5000 - >>> n_features = 50 - >>> in1 = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> in2 = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - - A single RAFT handle can optionally be reused across - pylibraft functions. - - >>> handle = Handle() - >>> output = pairwise_distance(in1, in2, metric="euclidean", handle=handle) - - pylibraft functions are often asynchronous so the - handle needs to be explicitly synchronized - - >>> handle.sync() - - It's also possible to write to a pre-allocated output array: - - >>> import cupy as cp - >>> from pylibraft.common import Handle - >>> from pylibraft.distance import pairwise_distance - >>> n_samples = 5000 - >>> n_features = 50 - >>> in1 = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> in2 = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> output = cp.empty((n_samples, n_samples), dtype=cp.float32) - - A single RAFT handle can optionally be reused across - pylibraft functions. - - >>> - >>> handle = Handle() - >>> pairwise_distance(in1, in2, out=output, - ... metric="euclidean", handle=handle) - array(...) - - pylibraft functions are often asynchronous so the - handle needs to be explicitly synchronized - - >>> handle.sync() - """ - - x_cai = cai_wrapper(X) - y_cai = cai_wrapper(Y) - - m = x_cai.shape[0] - n = y_cai.shape[0] - - x_dt = x_cai.dtype - y_dt = y_cai.dtype - - if out is None: - dists = device_ndarray.empty((m, n), dtype=y_dt) - else: - dists = out - - x_k = x_cai.shape[1] - y_k = y_cai.shape[1] - - dists_cai = cai_wrapper(dists) - - if x_k != y_k: - raise ValueError("Inputs must have same number of columns. " - "a=%s, b=%s" % (x_k, y_k)) - - x_ptr = x_cai.data - y_ptr = y_cai.data - d_ptr = dists_cai.data - - handle = handle if handle is not None else Handle() - cdef device_resources *h = handle.getHandle() - - d_dt = dists_cai.dtype - - x_c_contiguous = x_cai.c_contiguous - y_c_contiguous = y_cai.c_contiguous - - if x_c_contiguous != y_c_contiguous: - raise ValueError("Inputs must have matching strides") - - if metric not in SUPPORTED_DISTANCES: - raise ValueError("metric %s is not supported" % metric) - - cdef DistanceType distance_type = DISTANCE_TYPES[metric] - - if x_dt != y_dt or x_dt != d_dt: - raise ValueError("Inputs must have the same dtypes") - - if x_dt == np.float32: - pairwise_distance(deref(h), - x_ptr, - y_ptr, - d_ptr, - m, - n, - x_k, - distance_type, - x_c_contiguous, - p) - elif x_dt == np.float64: - pairwise_distance(deref(h), - x_ptr, - y_ptr, - d_ptr, - m, - n, - x_k, - distance_type, - x_c_contiguous, - p) - else: - raise ValueError("dtype %s not supported" % x_dt) - - return dists diff --git a/python/pylibraft/pylibraft/matrix/CMakeLists.txt b/python/pylibraft/pylibraft/matrix/CMakeLists.txt deleted file mode 100644 index 5b7803db00..0000000000 --- a/python/pylibraft/pylibraft/matrix/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -# ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -# Set the list of Cython files to build -set(cython_sources select_k.pyx) -set(linked_libraries raft::raft raft::compiled) - -# Build all of the Cython targets -rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX matrix_ -) diff --git a/python/pylibraft/pylibraft/matrix/__init__.pxd b/python/pylibraft/pylibraft/matrix/__init__.pxd deleted file mode 100644 index a7e7b75096..0000000000 --- a/python/pylibraft/pylibraft/matrix/__init__.pxd +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/pylibraft/pylibraft/matrix/__init__.py b/python/pylibraft/pylibraft/matrix/__init__.py deleted file mode 100644 index 5eb35795ed..0000000000 --- a/python/pylibraft/pylibraft/matrix/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from .select_k import select_k - -__all__ = ["select_k"] diff --git a/python/pylibraft/pylibraft/matrix/cpp/__init__.pxd b/python/pylibraft/pylibraft/matrix/cpp/__init__.pxd deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/python/pylibraft/pylibraft/matrix/cpp/__init__.py b/python/pylibraft/pylibraft/matrix/cpp/__init__.py deleted file mode 100644 index 8f2cc34855..0000000000 --- a/python/pylibraft/pylibraft/matrix/cpp/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/pylibraft/pylibraft/matrix/cpp/select_k.pxd b/python/pylibraft/pylibraft/matrix/cpp/select_k.pxd deleted file mode 100644 index ab466fdce6..0000000000 --- a/python/pylibraft/pylibraft/matrix/cpp/select_k.pxd +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -from libc.stdint cimport int64_t -from libcpp cimport bool - -from pylibraft.common.cpp.mdspan cimport device_matrix_view, row_major -from pylibraft.common.cpp.optional cimport optional -from pylibraft.common.handle cimport device_resources - - -cdef extern from "raft_runtime/matrix/select_k.hpp" \ - namespace "raft::runtime::matrix" nogil: - - cdef void select_k(const device_resources & handle, - device_matrix_view[float, int64_t, row_major], - optional[device_matrix_view[int64_t, - int64_t, - row_major]], - device_matrix_view[float, int64_t, row_major], - device_matrix_view[int64_t, int64_t, row_major], - bool) except + diff --git a/python/pylibraft/pylibraft/matrix/select_k.pyx b/python/pylibraft/pylibraft/matrix/select_k.pyx deleted file mode 100644 index fbb1e2e5d3..0000000000 --- a/python/pylibraft/pylibraft/matrix/select_k.pyx +++ /dev/null @@ -1,133 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -from cython.operator cimport dereference as deref -from libc.stdint cimport int64_t -from libcpp cimport bool - -import numpy as np - -from pylibraft.common import auto_convert_output, cai_wrapper, device_ndarray -from pylibraft.common.handle import auto_sync_handle -from pylibraft.common.input_validation import is_c_contiguous - -from pylibraft.common.cpp.mdspan cimport ( - device_matrix_view, - host_matrix_view, - make_device_matrix_view, - make_host_matrix_view, - row_major, -) -from pylibraft.common.cpp.optional cimport optional -from pylibraft.common.handle cimport device_resources -from pylibraft.common.mdspan cimport get_dmv_float, get_dmv_int64 -from pylibraft.matrix.cpp.select_k cimport select_k as c_select_k - - -@auto_sync_handle -@auto_convert_output -def select_k(dataset, k=None, distances=None, indices=None, select_min=True, - handle=None): - """ - Selects the top k items from each row in a matrix - - - Parameters - ---------- - dataset : array interface compliant matrix, row-major layout, - shape (n_rows, dim). Supported dtype [float] - k : int - Number of items to return for each row. Optional if indices or - distances arrays are given (in which case their second dimension - is k). - distances : Optional array interface compliant matrix shape - (n_rows, k), dtype float. If supplied, - distances will be written here in-place. (default None) - indices : Optional array interface compliant matrix shape - (n_rows, k), dtype int64_t. If supplied, neighbor - indices will be written here in-place. (default None) - select_min: : bool - Whether to select the minimum or maximum K items - - {handle_docstring} - - Returns - ------- - distances: array interface compliant object containing resulting distances - shape (n_rows, k) - - indices: array interface compliant object containing resulting indices - shape (n_rows, k) - - Examples - -------- - - >>> import cupy as cp - - >>> from pylibraft.matrix import select_k - - >>> n_features = 50 - >>> n_rows = 1000 - - >>> queries = cp.random.random_sample((n_rows, n_features), - ... dtype=cp.float32) - >>> k = 40 - >>> distances, ids = select_k(queries, k) - >>> distances = cp.asarray(distances) - >>> ids = cp.asarray(ids) - """ - - dataset_cai = cai_wrapper(dataset) - - if k is None: - if indices is not None: - k = cai_wrapper(indices).shape[1] - elif distances is not None: - k = cai_wrapper(distances).shape[1] - else: - raise ValueError("Argument k must be specified if both indices " - "and distances arg is None") - - n_rows = dataset.shape[0] - if indices is None: - indices = device_ndarray.empty((n_rows, k), dtype='int64') - - if distances is None: - distances = device_ndarray.empty((n_rows, k), dtype='float32') - - distances_cai = cai_wrapper(distances) - indices_cai = cai_wrapper(indices) - - cdef device_resources* handle_ = \ - handle.getHandle() - - cdef optional[device_matrix_view[int64_t, int64_t, row_major]] in_idx - - if dataset_cai.dtype == np.float32: - c_select_k(deref(handle_), - get_dmv_float(dataset_cai, check_shape=True), - in_idx, - get_dmv_float(distances_cai, check_shape=True), - get_dmv_int64(indices_cai, check_shape=True), - select_min) - else: - raise TypeError("dtype %s not supported" % dataset_cai.dtype) - - return distances, indices diff --git a/python/pylibraft/pylibraft/neighbors/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/CMakeLists.txt deleted file mode 100644 index 069038a0e8..0000000000 --- a/python/pylibraft/pylibraft/neighbors/CMakeLists.txt +++ /dev/null @@ -1,28 +0,0 @@ -# ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -# Set the list of Cython files to build -set(cython_sources common.pyx refine.pyx brute_force.pyx hnsw.pyx rbc.pyx) -set(linked_libraries raft::raft raft::compiled) - -# Build all of the Cython targets -rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ -) - -add_subdirectory(cagra) -add_subdirectory(ivf_flat) -add_subdirectory(ivf_pq) diff --git a/python/pylibraft/pylibraft/neighbors/__init__.pxd b/python/pylibraft/pylibraft/neighbors/__init__.pxd deleted file mode 100644 index 273b4497cc..0000000000 --- a/python/pylibraft/pylibraft/neighbors/__init__.pxd +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/pylibraft/pylibraft/neighbors/__init__.py b/python/pylibraft/pylibraft/neighbors/__init__.py deleted file mode 100644 index 86612b2fbb..0000000000 --- a/python/pylibraft/pylibraft/neighbors/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from pylibraft.neighbors import brute_force # type: ignore -from pylibraft.neighbors import hnsw # type: ignore -from pylibraft.neighbors import rbc # type: ignore -from pylibraft.neighbors import cagra, ivf_flat, ivf_pq - -from .refine import refine - -__all__ = [ - "common", - "refine", - "brute_force", - "ivf_flat", - "ivf_pq", - "cagra", - "hnsw", - "rbc", -] diff --git a/python/pylibraft/pylibraft/neighbors/brute_force.pyx b/python/pylibraft/pylibraft/neighbors/brute_force.pyx deleted file mode 100644 index 19d20fb75d..0000000000 --- a/python/pylibraft/pylibraft/neighbors/brute_force.pyx +++ /dev/null @@ -1,269 +0,0 @@ -# -# Copyright (c) 2022-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -from cython.operator cimport dereference as deref -from libcpp cimport bool, nullptr -from libcpp.vector cimport vector - -from pylibraft.distance.distance_type cimport DistanceType - -from pylibraft.common import ( - DeviceResources, - auto_convert_output, - cai_wrapper, - device_ndarray, -) - -from libc.stdint cimport int64_t, uintptr_t - -from pylibraft.common.cpp.optional cimport optional -from pylibraft.common.handle cimport device_resources -from pylibraft.common.mdspan cimport get_dmv_bool, get_dmv_float, get_dmv_int64 - -from pylibraft.common.handle import auto_sync_handle -from pylibraft.common.interruptible import cuda_interruptible - -from pylibraft.distance.distance_type cimport DistanceType - -# TODO: Centralize this - -from pylibraft.distance.pairwise_distance import DISTANCE_TYPES -from pylibraft.neighbors.common import _check_input_array - -from pylibraft.common.cpp.mdspan cimport ( - device_matrix_view, - device_vector_view, - host_matrix_view, - make_device_matrix_view, - make_device_vector_view, - make_host_matrix_view, - row_major, -) -from pylibraft.neighbors.cpp.brute_force cimport ( - eps_neighbors as c_eps_neighbors, - knn as c_knn, -) - - -def _get_array_params(array_interface, check_dtype=None): - dtype = np.dtype(array_interface["typestr"]) - if check_dtype is None and dtype != check_dtype: - raise TypeError("dtype %s not supported" % dtype) - shape = array_interface["shape"] - if len(shape) != 2: - raise ValueError("Expected a 2D array, got %d D" % len(shape)) - data = array_interface["data"][0] - return (shape, dtype, data) - - -@auto_sync_handle -@auto_convert_output -def knn(dataset, queries, k=None, indices=None, distances=None, - metric="sqeuclidean", metric_arg=2.0, - global_id_offset=0, handle=None): - """ - Perform a brute-force nearest neighbors search. - - Parameters - ---------- - dataset : array interface compliant matrix, row-major layout, - shape (n_samples, dim). Supported dtype [float] - queries : array interface compliant matrix, row-major layout, - shape (n_queries, dim) Supported dtype [float] - k : int - Number of neighbors to search (k <= 2048). Optional if indices or - distances arrays are given (in which case their second dimension - is k). - indices : Optional array interface compliant matrix shape - (n_queries, k), dtype int64_t. If supplied, neighbor - indices will be written here in-place. (default None) - Supported dtype uint64 - distances : Optional array interface compliant matrix shape - (n_queries, k), dtype float. If supplied, neighbor - indices will be written here in-place. (default None) - {handle_docstring} - - Returns - ------- - indices: array interface compliant object containing resulting indices - shape (n_queries, k) - - distances: array interface compliant object containing resulting distances - shape (n_queries, k) - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors.brute_force import knn - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Search using the built index - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> k = 40 - >>> distances, neighbors = knn(dataset, queries, k) - >>> distances = cp.asarray(distances) - >>> neighbors = cp.asarray(neighbors) - """ - - if handle is None: - handle = DeviceResources() - - dataset_cai = cai_wrapper(dataset) - queries_cai = cai_wrapper(queries) - - if k is None: - if indices is not None: - k = cai_wrapper(indices).shape[1] - elif distances is not None: - k = cai_wrapper(distances).shape[1] - else: - raise ValueError("Argument k must be specified if both indices " - "and distances arg is None") - - # we require c-contiguous (rowmajor) inputs here - _check_input_array(dataset_cai, [np.dtype("float32")]) - _check_input_array(queries_cai, [np.dtype("float32")], - exp_cols=dataset_cai.shape[1]) - - n_queries = queries_cai.shape[0] - - if indices is None: - indices = device_ndarray.empty((n_queries, k), dtype='int64') - - if distances is None: - distances = device_ndarray.empty((n_queries, k), dtype='float32') - - cdef DistanceType c_metric = DISTANCE_TYPES[metric] - - distances_cai = cai_wrapper(distances) - indices_cai = cai_wrapper(indices) - - cdef optional[float] c_metric_arg = metric_arg - cdef optional[int64_t] c_global_offset = global_id_offset - - cdef device_resources* handle_ = \ - handle.getHandle() - - if dataset_cai.dtype == np.float32: - with cuda_interruptible(): - c_knn(deref(handle_), - get_dmv_float(dataset_cai, check_shape=True), - get_dmv_float(queries_cai, check_shape=True), - get_dmv_int64(indices_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True), - c_metric, - c_metric_arg, - c_global_offset) - else: - raise TypeError("dtype %s not supported" % dataset_cai.dtype) - - return (distances, indices) - - -@auto_sync_handle -@auto_convert_output -def eps_neighbors(dataset, queries, eps, handle=None): - """ - Perform an epsilon neighborhood search using the L2-norm. - - Parameters - ---------- - dataset : array interface compliant matrix, row-major layout, - shape (n_samples, dim). Supported dtype [float] - queries : array interface compliant matrix, row-major layout, - shape (n_queries, dim) Supported dtype [float] - eps : threshold - {handle_docstring} - - Returns - ------- - adj: array interface compliant object containing bool adjacency mask - shape (n_queries, n_samples) - - vd: array interface compliant object containing row sums of adj - shape (n_queries + 1). vd[n_queries] contains the total sum - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors.brute_force import eps_neighbors - >>> handle = DeviceResources() - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> eps = 0.1 - >>> adj, vd = eps_neighbors(dataset, queries, eps, handle=handle) - >>> adj = cp.asarray(adj) - >>> vd = cp.asarray(vd) - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - """ - - if handle is None: - handle = DeviceResources() - - dataset_cai = cai_wrapper(dataset) - queries_cai = cai_wrapper(queries) - - # we require c-contiguous (rowmajor) inputs here - _check_input_array(dataset_cai, [np.dtype("float32")]) - _check_input_array(queries_cai, [np.dtype("float32")], - exp_cols=dataset_cai.shape[1]) - - n_queries = queries_cai.shape[0] - n_samples = dataset_cai.shape[0] - - adj = device_ndarray.empty((n_queries, n_samples), dtype='bool') - vd = device_ndarray.empty((n_queries + 1, ), dtype='int64') - adj_cai = cai_wrapper(adj) - vd_cai = cai_wrapper(vd) - - cdef device_resources* handle_ = \ - handle.getHandle() - - vd_vector_view = make_device_vector_view( - vd_cai.data, vd_cai.shape[0]) - - if dataset_cai.dtype == np.float32: - with cuda_interruptible(): - c_eps_neighbors( - deref(handle_), - get_dmv_float(dataset_cai, check_shape=True), - get_dmv_float(queries_cai, check_shape=True), - get_dmv_bool(adj_cai, check_shape=True), - vd_vector_view, - eps) - else: - raise TypeError("dtype %s not supported" % dataset_cai.dtype) - - return (adj, vd) diff --git a/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt deleted file mode 100644 index 0939d7c5b3..0000000000 --- a/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -# ============================================================================= -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -# Set the list of Cython files to build -set(cython_sources cagra.pyx) -set(linked_libraries raft::raft raft::compiled) - -# Build all of the Cython targets -rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_cagra_ -) diff --git a/python/pylibraft/pylibraft/neighbors/cagra/__init__.pxd b/python/pylibraft/pylibraft/neighbors/cagra/__init__.pxd deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/python/pylibraft/pylibraft/neighbors/cagra/__init__.py b/python/pylibraft/pylibraft/neighbors/cagra/__init__.py deleted file mode 100644 index b2a872fc89..0000000000 --- a/python/pylibraft/pylibraft/neighbors/cagra/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from .cagra import Index, IndexParams, SearchParams, build, load, save, search - -__all__ = [ - "Index", - "IndexParams", - "SearchParams", - "build", - "load", - "save", - "search", -] diff --git a/python/pylibraft/pylibraft/neighbors/cagra/cagra.pxd b/python/pylibraft/pylibraft/neighbors/cagra/cagra.pxd deleted file mode 100644 index 98537f8357..0000000000 --- a/python/pylibraft/pylibraft/neighbors/cagra/cagra.pxd +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -from libc.stdint cimport int8_t, uint8_t, uint32_t -from libcpp cimport bool -from libcpp.string cimport string - -cimport pylibraft.neighbors.cagra.cpp.c_cagra as c_cagra - - -cdef class Index: - cdef readonly bool trained - cdef str active_index_type - -cdef class IndexFloat(Index): - cdef c_cagra.index[float, uint32_t] * index - -cdef class IndexInt8(Index): - cdef c_cagra.index[int8_t, uint32_t] * index - -cdef class IndexUint8(Index): - cdef c_cagra.index[uint8_t, uint32_t] * index diff --git a/python/pylibraft/pylibraft/neighbors/cagra/cagra.pyx b/python/pylibraft/pylibraft/neighbors/cagra/cagra.pyx deleted file mode 100644 index 9b376f5f0a..0000000000 --- a/python/pylibraft/pylibraft/neighbors/cagra/cagra.pyx +++ /dev/null @@ -1,900 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import warnings - -import numpy as np - -from cython.operator cimport dereference as deref -from libc.stdint cimport ( - int8_t, - int32_t, - int64_t, - uint8_t, - uint32_t, - uint64_t, - uintptr_t, -) -from libcpp cimport bool, nullptr -from libcpp.string cimport string - -from pylibraft.distance.distance_type cimport DistanceType - -from pylibraft.common import ( - DeviceResources, - ai_wrapper, - auto_convert_output, - cai_wrapper, - device_ndarray, -) -from pylibraft.common.cai_wrapper import wrap_array -from pylibraft.common.interruptible import cuda_interruptible - -from pylibraft.common.handle cimport device_resources - -from pylibraft.common.handle import auto_sync_handle -from pylibraft.common.input_validation import is_c_contiguous - -from rmm.librmm.memory_resource cimport device_memory_resource -from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource - -cimport pylibraft.neighbors.cagra.cpp.c_cagra as c_cagra -from pylibraft.common.optional cimport make_optional, optional - -from pylibraft.neighbors.common import _check_input_array, _get_metric - -from pylibraft.common.cpp.mdspan cimport ( - device_matrix_view, - device_vector_view, - make_device_vector_view, - row_major, -) -from pylibraft.common.mdspan cimport ( - get_const_dmv_float, - get_const_dmv_int8, - get_const_dmv_uint8, - get_const_hmv_float, - get_const_hmv_int8, - get_const_hmv_uint8, - get_dmv_float, - get_dmv_int8, - get_dmv_int64, - get_dmv_uint8, - get_dmv_uint32, - get_hmv_float, - get_hmv_int8, - get_hmv_int64, - get_hmv_uint8, - get_hmv_uint32, - make_optional_view_int64, -) -from pylibraft.neighbors.common cimport _get_metric_string - - -cdef class IndexParams: - """ - Parameters to build index for CAGRA nearest neighbor search - - Parameters - ---------- - metric : string denoting the metric type, default="sqeuclidean" - Valid values for metric: ["sqeuclidean", "inner_product"], where - - sqeuclidean is the euclidean distance without the square root - operation, i.e.: distance(a,b) = \\sum_i (a_i - b_i)^2 - - inner_product is the dot product between two vectors i.e.: - distance(a, b) = \\sum_i (a_i * b_i) - intermediate_graph_degree : int, default = 128 - - graph_degree : int, default = 64 - - build_algo: string denoting the graph building algorithm to use, \ - default = "ivf_pq" - Valid values for algo: ["ivf_pq", "nn_descent"], where - - ivf_pq will use the IVF-PQ algorithm for building the knn graph - - nn_descent (experimental) will use the NN-Descent algorithm for - building the knn graph. It is expected to be generally - faster than ivf_pq. - """ - cdef c_cagra.index_params params - - def __init__(self, *, - metric="sqeuclidean", - intermediate_graph_degree=128, - graph_degree=64, - build_algo="ivf_pq"): - self.params.metric = _get_metric(metric) - self.params.metric_arg = 0 - self.params.intermediate_graph_degree = intermediate_graph_degree - self.params.graph_degree = graph_degree - if build_algo == "ivf_pq": - self.params.build_algo = c_cagra.graph_build_algo.IVF_PQ - elif build_algo == "nn_descent": - self.params.build_algo = c_cagra.graph_build_algo.NN_DESCENT - - @property - def metric(self): - return self.params.metric - - @property - def intermediate_graph_degree(self): - return self.params.intermediate_graph_degree - - @property - def graph_degree(self): - return self.params.graph_degree - - -cdef class Index: - - def __cinit__(self): - self.trained = False - self.active_index_type = None - - -cdef class IndexFloat(Index): - - def __cinit__(self, handle=None): - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - self.index = new c_cagra.index[float, uint32_t]( - deref(handle_)) - - def __repr__(self): - m_str = "metric=" + _get_metric_string(self.index.metric()) - attr_str = [attr + "=" + str(getattr(self, attr)) - for attr in ["metric", "dim", "graph_degree"]] - attr_str = [m_str] + attr_str - return "Index(type=CAGRA, " + (", ".join(attr_str)) + ")" - - @auto_sync_handle - def update_dataset(self, dataset, handle=None): - """ Replace the dataset with a new dataset. - - Parameters - ---------- - dataset : array interface compliant matrix shape (n_samples, dim) - {handle_docstring} - """ - cdef device_resources* handle_ = \ - handle.getHandle() - - dataset_ai = wrap_array(dataset) - dataset_dt = dataset_ai.dtype - _check_input_array(dataset_ai, [np.dtype("float32")]) - - if dataset_ai.from_cai: - self.index[0].update_dataset(deref(handle_), - get_const_dmv_float(dataset_ai, - check_shape=True)) - else: - self.index[0].update_dataset(deref(handle_), - get_const_hmv_float(dataset_ai, - check_shape=True)) - - @property - def metric(self): - return self.index[0].metric() - - @property - def size(self): - return self.index[0].size() - - @property - def dim(self): - return self.index[0].dim() - - @property - def graph_degree(self): - return self.index[0].graph_degree() - - def __dealloc__(self): - if self.index is not NULL: - del self.index - - -cdef class IndexInt8(Index): - - def __cinit__(self, handle=None): - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - self.index = new c_cagra.index[int8_t, uint32_t]( - deref(handle_)) - - @auto_sync_handle - def update_dataset(self, dataset, handle=None): - """ Replace the dataset with a new dataset. - - Parameters - ---------- - dataset : array interface compliant matrix shape (n_samples, dim) - {handle_docstring} - """ - cdef device_resources* handle_ = \ - handle.getHandle() - - dataset_ai = wrap_array(dataset) - dataset_dt = dataset_ai.dtype - _check_input_array(dataset_ai, [np.dtype("byte")]) - - if dataset_ai.from_cai: - self.index[0].update_dataset(deref(handle_), - get_const_dmv_int8(dataset_ai, - check_shape=True)) - else: - self.index[0].update_dataset(deref(handle_), - get_const_hmv_int8(dataset_ai, - check_shape=True)) - - def __repr__(self): - m_str = "metric=" + _get_metric_string(self.index.metric()) - attr_str = [attr + "=" + str(getattr(self, attr)) - for attr in ["metric", "dim", "graph_degree"]] - attr_str = [m_str] + attr_str - return "Index(type=CAGRA, " + (", ".join(attr_str)) + ")" - - @property - def metric(self): - return self.index[0].metric() - - @property - def size(self): - return self.index[0].size() - - @property - def dim(self): - return self.index[0].dim() - - @property - def graph_degree(self): - return self.index[0].graph_degree() - - def __dealloc__(self): - if self.index is not NULL: - del self.index - - -cdef class IndexUint8(Index): - - def __cinit__(self, handle=None): - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - self.index = new c_cagra.index[uint8_t, uint32_t]( - deref(handle_)) - - @auto_sync_handle - def update_dataset(self, dataset, handle=None): - """ Replace the dataset with a new dataset. - - Parameters - ---------- - dataset : array interface compliant matrix shape (n_samples, dim) - {handle_docstring} - """ - cdef device_resources* handle_ = \ - handle.getHandle() - - dataset_ai = wrap_array(dataset) - dataset_dt = dataset_ai.dtype - _check_input_array(dataset_ai, [np.dtype("ubyte")]) - - if dataset_ai.from_cai: - self.index[0].update_dataset(deref(handle_), - get_const_dmv_uint8(dataset_ai, - check_shape=True)) - else: - self.index[0].update_dataset(deref(handle_), - get_const_hmv_uint8(dataset_ai, - check_shape=True)) - - def __repr__(self): - m_str = "metric=" + _get_metric_string(self.index.metric()) - attr_str = [attr + "=" + str(getattr(self, attr)) - for attr in ["metric", "dim", "graph_degree"]] - attr_str = [m_str] + attr_str - return "Index(type=CAGRA, " + (", ".join(attr_str)) + ")" - - @property - def metric(self): - return self.index[0].metric() - - @property - def size(self): - return self.index[0].size() - - @property - def dim(self): - return self.index[0].dim() - - @property - def graph_degree(self): - return self.index[0].graph_degree() - - def __dealloc__(self): - if self.index is not NULL: - del self.index - - -@auto_sync_handle -@auto_convert_output -def build(IndexParams index_params, dataset, handle=None): - """ - Build the CAGRA index from the dataset for efficient search. - - The build performs two different steps- first an intermediate knn-graph is - constructed, then it's optimized it to create the final graph. The - index_params object controls the node degree of these graphs. - - It is required that both the dataset and the optimized graph fit the - GPU memory. - - The following distance metrics are supported: - - L2 - - inner_product - - Parameters - ---------- - index_params : IndexParams object - dataset : CUDA array interface compliant matrix shape (n_samples, dim) - Supported dtype [float, int8, uint8] - {handle_docstring} - - Returns - ------- - index: cagra.Index - - Examples - -------- - - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import cagra - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> k = 10 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> handle = DeviceResources() - >>> build_params = cagra.IndexParams(metric="sqeuclidean") - >>> index = cagra.build(build_params, dataset, handle=handle) - >>> distances, neighbors = cagra.search(cagra.SearchParams(), - ... index, dataset, - ... k, handle=handle) - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - >>> distances = cp.asarray(distances) - >>> neighbors = cp.asarray(neighbors) - """ - dataset_ai = wrap_array(dataset) - dataset_dt = dataset_ai.dtype - _check_input_array(dataset_ai, [np.dtype('float32'), np.dtype('byte'), - np.dtype('ubyte')]) - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - cdef IndexFloat idx_float - cdef IndexInt8 idx_int8 - cdef IndexUint8 idx_uint8 - - if dataset_ai.from_cai: - if dataset_dt == np.float32: - idx_float = IndexFloat(handle) - idx_float.active_index_type = "float32" - with cuda_interruptible(): - c_cagra.build_device( - deref(handle_), - index_params.params, - get_dmv_float(dataset_ai, check_shape=True), - deref(idx_float.index)) - idx_float.trained = True - return idx_float - elif dataset_dt == np.byte: - idx_int8 = IndexInt8(handle) - idx_int8.active_index_type = "byte" - with cuda_interruptible(): - c_cagra.build_device( - deref(handle_), - index_params.params, - get_dmv_int8(dataset_ai, check_shape=True), - deref(idx_int8.index)) - idx_int8.trained = True - return idx_int8 - elif dataset_dt == np.ubyte: - idx_uint8 = IndexUint8(handle) - idx_uint8.active_index_type = "ubyte" - with cuda_interruptible(): - c_cagra.build_device( - deref(handle_), - index_params.params, - get_dmv_uint8(dataset_ai, check_shape=True), - deref(idx_uint8.index)) - idx_uint8.trained = True - return idx_uint8 - else: - raise TypeError("dtype %s not supported" % dataset_dt) - else: - if dataset_dt == np.float32: - idx_float = IndexFloat(handle) - idx_float.active_index_type = "float32" - with cuda_interruptible(): - c_cagra.build_host( - deref(handle_), - index_params.params, - get_hmv_float(dataset_ai, check_shape=True), - deref(idx_float.index)) - idx_float.trained = True - return idx_float - elif dataset_dt == np.byte: - idx_int8 = IndexInt8(handle) - idx_int8.active_index_type = "byte" - with cuda_interruptible(): - c_cagra.build_host( - deref(handle_), - index_params.params, - get_hmv_int8(dataset_ai, check_shape=True), - deref(idx_int8.index)) - idx_int8.trained = True - return idx_int8 - elif dataset_dt == np.ubyte: - idx_uint8 = IndexUint8(handle) - idx_uint8.active_index_type = "ubyte" - with cuda_interruptible(): - c_cagra.build_host( - deref(handle_), - index_params.params, - get_hmv_uint8(dataset_ai, check_shape=True), - deref(idx_uint8.index)) - idx_uint8.trained = True - return idx_uint8 - else: - raise TypeError("dtype %s not supported" % dataset_dt) - - -cdef class SearchParams: - """ - CAGRA search parameters - - Parameters - ---------- - max_queries: int, default = 0 - Maximum number of queries to search at the same time (batch size). - Auto select when 0. - itopk_size: int, default = 64 - Number of intermediate search results retained during the search. - This is the main knob to adjust trade off between accuracy and - search speed. Higher values improve the search accuracy. - max_iterations: int, default = 0 - Upper limit of search iterations. Auto select when 0. - algo: string denoting the search algorithm to use, default = "auto" - Valid values for algo: ["auto", "single_cta", "multi_cta"], where - - auto will automatically select the best value based on query size - - single_cta is better when query contains larger number of - vectors (e.g >10) - - multi_cta is better when query contains only a few vectors - team_size: int, default = 0 - Number of threads used to calculate a single distance. 4, 8, 16, - or 32. - search_width: int, default = 1 - Number of graph nodes to select as the starting point for the - search in each iteration. - min_iterations: int, default = 0 - Lower limit of search iterations. - thread_block_size: int, default = 0 - Thread block size. 0, 64, 128, 256, 512, 1024. - Auto selection when 0. - hashmap_mode: string denoting the type of hash map to use. - It's usually better to allow the algorithm to select this value, - default = "auto". - Valid values for hashmap_mode: ["auto", "small", "hash"], where - - auto will automatically select the best value based on algo - - small will use the small shared memory hash table with resetting. - - hash will use a single hash table in global memory. - hashmap_min_bitlen: int, default = 0 - Upper limit of hashmap fill rate. More than 0.1, less than 0.9. - hashmap_max_fill_rate: float, default = 0.5 - Upper limit of hashmap fill rate. More than 0.1, less than 0.9. - num_random_samplings: int, default = 1 - Number of iterations of initial random seed node selection. 1 or - more. - rand_xor_mask: int, default = 0x128394 - Bit mask used for initial random seed node selection. - """ - cdef c_cagra.search_params params - - def __init__(self, *, - max_queries=0, - itopk_size=64, - max_iterations=0, - algo="auto", - team_size=0, - search_width=1, - min_iterations=0, - thread_block_size=0, - hashmap_mode="auto", - hashmap_min_bitlen=0, - hashmap_max_fill_rate=0.5, - num_random_samplings=1, - rand_xor_mask=0x128394): - self.params.max_queries = max_queries - self.params.itopk_size = itopk_size - self.params.max_iterations = max_iterations - if algo == "single_cta": - self.params.algo = c_cagra.search_algo.SINGLE_CTA - elif algo == "multi_cta": - self.params.algo = c_cagra.search_algo.MULTI_CTA - elif algo == "multi_kernel": - self.params.algo = c_cagra.search_algo.MULTI_KERNEL - elif algo == "auto": - self.params.algo = c_cagra.search_algo.AUTO - else: - raise ValueError("`algo` value not supported.") - - self.params.team_size = team_size - self.params.search_width = search_width - self.params.min_iterations = min_iterations - self.params.thread_block_size = thread_block_size - if hashmap_mode == "hash": - self.params.hashmap_mode = c_cagra.hash_mode.HASH - elif hashmap_mode == "small": - self.params.hashmap_mode = c_cagra.hash_mode.SMALL - elif hashmap_mode == "auto": - self.params.hashmap_mode = c_cagra.hash_mode.AUTO - else: - raise ValueError("`hashmap_mode` value not supported.") - - self.params.hashmap_min_bitlen = hashmap_min_bitlen - self.params.hashmap_max_fill_rate = hashmap_max_fill_rate - self.params.num_random_samplings = num_random_samplings - self.params.rand_xor_mask = rand_xor_mask - - def __repr__(self): - attr_str = [attr + "=" + str(getattr(self, attr)) - for attr in [ - "max_queries", "itopk_size", "max_iterations", "algo", - "team_size", "search_width", "min_iterations", - "thread_block_size", "hashmap_mode", - "hashmap_min_bitlen", "hashmap_max_fill_rate", - "num_random_samplings", "rand_xor_mask"]] - return "SearchParams(type=CAGRA, " + (", ".join(attr_str)) + ")" - - @property - def max_queries(self): - return self.params.max_queries - - @property - def itopk_size(self): - return self.params.itopk_size - - @property - def max_iterations(self): - return self.params.max_iterations - - @property - def algo(self): - return self.params.algo - - @property - def team_size(self): - return self.params.team_size - - @property - def search_width(self): - return self.params.search_width - - @property - def min_iterations(self): - return self.params.min_iterations - - @property - def thread_block_size(self): - return self.params.thread_block_size - - @property - def hashmap_mode(self): - return self.params.hashmap_mode - - @property - def hashmap_min_bitlen(self): - return self.params.hashmap_min_bitlen - - @property - def hashmap_max_fill_rate(self): - return self.params.hashmap_max_fill_rate - - @property - def num_random_samplings(self): - return self.params.num_random_samplings - - @property - def rand_xor_mask(self): - return self.params.rand_xor_mask - - -@auto_sync_handle -@auto_convert_output -def search(SearchParams search_params, - Index index, - queries, - k, - neighbors=None, - distances=None, - handle=None): - """ - Find the k nearest neighbors for each query. - - Parameters - ---------- - search_params : SearchParams - index : Index - Trained CAGRA index. - queries : CUDA array interface compliant matrix shape (n_samples, dim) - Supported dtype [float, int8, uint8] - k : int - The number of neighbors. - neighbors : Optional CUDA array interface compliant matrix shape - (n_queries, k), dtype int64_t. If supplied, neighbor - indices will be written here in-place. (default None) - distances : Optional CUDA array interface compliant matrix shape - (n_queries, k) If supplied, the distances to the - neighbors will be written here in-place. (default None) - {handle_docstring} - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import cagra - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Build index - >>> handle = DeviceResources() - >>> index = cagra.build(cagra.IndexParams(), dataset, handle=handle) - >>> # Search using the built index - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> k = 10 - >>> search_params = cagra.SearchParams( - ... max_queries=100, - ... itopk_size=64 - ... ) - >>> # Using a pooling allocator reduces overhead of temporary array - >>> # creation during search. This is useful if multiple searches - >>> # are performad with same query size. - >>> distances, neighbors = cagra.search(search_params, index, queries, - ... k, handle=handle) - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - >>> neighbors = cp.asarray(neighbors) - >>> distances = cp.asarray(distances) - """ - - if not index.trained: - raise ValueError("Index need to be built before calling search.") - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - queries_cai = cai_wrapper(queries) - queries_dt = queries_cai.dtype - cdef uint32_t n_queries = queries_cai.shape[0] - - _check_input_array(queries_cai, [np.dtype('float32'), np.dtype('byte'), - np.dtype('ubyte')], - exp_cols=index.dim) - - if neighbors is None: - neighbors = device_ndarray.empty((n_queries, k), dtype='uint32') - - neighbors_cai = cai_wrapper(neighbors) - _check_input_array(neighbors_cai, [np.dtype('uint32')], - exp_rows=n_queries, exp_cols=k) - - if distances is None: - distances = device_ndarray.empty((n_queries, k), dtype='float32') - - distances_cai = cai_wrapper(distances) - _check_input_array(distances_cai, [np.dtype('float32')], - exp_rows=n_queries, exp_cols=k) - - cdef c_cagra.search_params params = search_params.params - cdef IndexFloat idx_float - cdef IndexInt8 idx_int8 - cdef IndexUint8 idx_uint8 - - if queries_dt == np.float32: - idx_float = index - with cuda_interruptible(): - c_cagra.search(deref(handle_), - params, - deref(idx_float.index), - get_dmv_float(queries_cai, check_shape=True), - get_dmv_uint32(neighbors_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True)) - elif queries_dt == np.byte: - idx_int8 = index - with cuda_interruptible(): - c_cagra.search(deref(handle_), - params, - deref(idx_int8.index), - get_dmv_int8(queries_cai, check_shape=True), - get_dmv_uint32(neighbors_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True)) - elif queries_dt == np.ubyte: - idx_uint8 = index - with cuda_interruptible(): - c_cagra.search(deref(handle_), - params, - deref(idx_uint8.index), - get_dmv_uint8(queries_cai, check_shape=True), - get_dmv_uint32(neighbors_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True)) - else: - raise ValueError("query dtype %s not supported" % queries_dt) - - return (distances, neighbors) - - -@auto_sync_handle -def save(filename, Index index, bool include_dataset=True, handle=None): - """ - Saves the index to a file. - - Saving / loading the index is experimental. The serialization format is - subject to change. - - Parameters - ---------- - filename : string - Name of the file. - index : Index - Trained CAGRA index. - include_dataset : bool - Whether or not to write out the dataset along with the index. Including - the dataset in the serialized index will use extra disk space, and - might not be desired if you already have a copy of the dataset on - disk. If this option is set to false, you will have to call - `index.update_dataset(dataset)` after loading the index. - {handle_docstring} - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import cagra - >>> n_samples = 50000 - >>> n_features = 50 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Build index - >>> handle = DeviceResources() - >>> index = cagra.build(cagra.IndexParams(), dataset, handle=handle) - >>> # Serialize and deserialize the cagra index built - >>> cagra.save("my_index.bin", index, handle=handle) - >>> index_loaded = cagra.load("my_index.bin", handle=handle) - """ - if not index.trained: - raise ValueError("Index need to be built before saving it.") - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - cdef string c_filename = filename.encode('utf-8') - - cdef IndexFloat idx_float - cdef IndexInt8 idx_int8 - cdef IndexUint8 idx_uint8 - - if index.active_index_type == "float32": - idx_float = index - c_cagra.serialize_file( - deref(handle_), c_filename, deref(idx_float.index), - include_dataset) - elif index.active_index_type == "byte": - idx_int8 = index - c_cagra.serialize_file( - deref(handle_), c_filename, deref(idx_int8.index), include_dataset) - elif index.active_index_type == "ubyte": - idx_uint8 = index - c_cagra.serialize_file( - deref(handle_), c_filename, deref(idx_uint8.index), - include_dataset) - else: - raise ValueError( - "Index dtype %s not supported" % index.active_index_type) - - -@auto_sync_handle -def load(filename, handle=None): - """ - Loads index from file. - - Saving / loading the index is experimental. The serialization format is - subject to change, therefore loading an index saved with a previous - version of raft is not guaranteed to work. - - Parameters - ---------- - filename : string - Name of the file. - {handle_docstring} - - Returns - ------- - index : Index - - """ - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - cdef string c_filename = filename.encode('utf-8') - cdef IndexFloat idx_float - cdef IndexInt8 idx_int8 - cdef IndexUint8 idx_uint8 - - with open(filename, "rb") as f: - type_str = f.read(3).decode("utf8") - dataset_dt = np.dtype(type_str) - - if dataset_dt == np.float32: - idx_float = IndexFloat(handle) - c_cagra.deserialize_file( - deref(handle_), c_filename, idx_float.index) - idx_float.trained = True - idx_float.active_index_type = 'float32' - return idx_float - elif dataset_dt == np.byte: - idx_int8 = IndexInt8(handle) - c_cagra.deserialize_file( - deref(handle_), c_filename, idx_int8.index) - idx_int8.trained = True - idx_int8.active_index_type = 'byte' - return idx_int8 - elif dataset_dt == np.ubyte: - idx_uint8 = IndexUint8(handle) - c_cagra.deserialize_file( - deref(handle_), c_filename, idx_uint8.index) - idx_uint8.trained = True - idx_uint8.active_index_type = 'ubyte' - return idx_uint8 - else: - raise ValueError("Dataset dtype %s not supported" % dataset_dt) diff --git a/python/pylibraft/pylibraft/neighbors/cagra/cpp/__init__.pxd b/python/pylibraft/pylibraft/neighbors/cagra/cpp/__init__.pxd deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/python/pylibraft/pylibraft/neighbors/cagra/cpp/__init__.py b/python/pylibraft/pylibraft/neighbors/cagra/cpp/__init__.py deleted file mode 100644 index 8f2cc34855..0000000000 --- a/python/pylibraft/pylibraft/neighbors/cagra/cpp/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/pylibraft/pylibraft/neighbors/cagra/cpp/c_cagra.pxd b/python/pylibraft/pylibraft/neighbors/cagra/cpp/c_cagra.pxd deleted file mode 100644 index 75ace7f1a8..0000000000 --- a/python/pylibraft/pylibraft/neighbors/cagra/cpp/c_cagra.pxd +++ /dev/null @@ -1,255 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -import pylibraft.common.handle - -from cython.operator cimport dereference as deref -from libc.stdint cimport int8_t, int64_t, uint8_t, uint32_t, uint64_t -from libcpp cimport bool, nullptr -from libcpp.string cimport string - -from rmm.librmm.memory_resource cimport device_memory_resource - -from pylibraft.common.cpp.mdspan cimport ( - device_matrix_view, - device_vector_view, - host_matrix_view, - row_major, -) -from pylibraft.common.handle cimport device_resources -from pylibraft.common.mdspan cimport const_float, const_int8_t, const_uint8_t -from pylibraft.common.optional cimport optional -from pylibraft.distance.distance_type cimport DistanceType -from pylibraft.neighbors.ivf_pq.cpp.c_ivf_pq cimport ( - ann_index, - ann_index_params, - ann_search_params, - index_params as ivfpq_ip, - search_params as ivfpq_sp, -) - - -cdef extern from "raft/neighbors/cagra_types.hpp" \ - namespace "raft::neighbors::cagra" nogil: - - ctypedef enum graph_build_algo: - IVF_PQ "raft::neighbors::cagra::graph_build_algo::IVF_PQ", - NN_DESCENT "raft::neighbors::cagra::graph_build_algo::NN_DESCENT" - - cpdef cppclass index_params(ann_index_params): - size_t intermediate_graph_degree - size_t graph_degree - graph_build_algo build_algo - - ctypedef enum search_algo: - SINGLE_CTA "raft::neighbors::cagra::search_algo::SINGLE_CTA", - MULTI_CTA "raft::neighbors::cagra::search_algo::MULTI_CTA", - MULTI_KERNEL "raft::neighbors::cagra::search_algo::MULTI_KERNEL", - AUTO "raft::neighbors::cagra::search_algo::AUTO" - - ctypedef enum hash_mode: - HASH "raft::neighbors::cagra::hash_mode::HASH", - SMALL "raft::neighbors::cagra::hash_mode::SMALL", - AUTO "raft::neighbors::cagra::hash_mode::AUTO" - - cpdef cppclass search_params(ann_search_params): - size_t max_queries - size_t itopk_size - size_t max_iterations - search_algo algo - size_t team_size - size_t search_width - size_t min_iterations - size_t thread_block_size - hash_mode hashmap_mode - size_t hashmap_min_bitlen - float hashmap_max_fill_rate - uint32_t num_random_samplings - uint64_t rand_xor_mask - - cdef cppclass index[T, IdxT](ann_index): - index(const device_resources&) - - DistanceType metric() - IdxT size() - uint32_t dim() - uint32_t graph_degree() - device_matrix_view[T, IdxT, row_major] dataset() - device_matrix_view[T, IdxT, row_major] graph() - - # hack: can't use the T template param here because of issues handling - # const w/ cython. introduce a new template param to get around this - void update_dataset[ValueT](const device_resources & handle, - host_matrix_view[ValueT, - int64_t, - row_major] dataset) - void update_dataset[ValueT](const device_resources & handle, - device_matrix_view[ValueT, - int64_t, - row_major] dataset) - -cdef extern from "raft_runtime/neighbors/cagra.hpp" \ - namespace "raft::runtime::neighbors::cagra" nogil: - - cdef void build_device( - const device_resources& handle, - const index_params& params, - device_matrix_view[float, int64_t, row_major] dataset, - index[float, uint32_t]& index) except + - - cdef void build_device( - const device_resources& handle, - const index_params& params, - device_matrix_view[int8_t, int64_t, row_major] dataset, - index[int8_t, uint32_t]& index) except + - - cdef void build_device( - const device_resources& handle, - const index_params& params, - device_matrix_view[uint8_t, int64_t, row_major] dataset, - index[uint8_t, uint32_t]& index) except + - - cdef void build_host( - const device_resources& handle, - const index_params& params, - host_matrix_view[float, int64_t, row_major] dataset, - index[float, uint32_t]& index) except + - - cdef void build_host( - const device_resources& handle, - const index_params& params, - host_matrix_view[int8_t, int64_t, row_major] dataset, - index[int8_t, uint32_t]& index) except + - - cdef void build_host( - const device_resources& handle, - const index_params& params, - host_matrix_view[uint8_t, int64_t, row_major] dataset, - index[uint8_t, uint32_t]& index) except + - - cdef void search( - const device_resources& handle, - const search_params& params, - const index[float, uint32_t]& index, - device_matrix_view[float, int64_t, row_major] queries, - device_matrix_view[uint32_t, int64_t, row_major] neighbors, - device_matrix_view[float, int64_t, row_major] distances) except + - - cdef void search( - const device_resources& handle, - const search_params& params, - const index[int8_t, uint32_t]& index, - device_matrix_view[int8_t, int64_t, row_major] queries, - device_matrix_view[uint32_t, int64_t, row_major] neighbors, - device_matrix_view[float, int64_t, row_major] distances) except + - - cdef void search( - const device_resources& handle, - const search_params& params, - const index[uint8_t, uint32_t]& index, - device_matrix_view[uint8_t, int64_t, row_major] queries, - device_matrix_view[uint32_t, int64_t, row_major] neighbors, - device_matrix_view[float, int64_t, row_major] distances) except + - - cdef void serialize(const device_resources& handle, - string& str, - const index[float, uint32_t]& index, - bool include_dataset) except + - - cdef void deserialize(const device_resources& handle, - const string& str, - index[float, uint32_t]* index) except + - - cdef void serialize(const device_resources& handle, - string& str, - const index[uint8_t, uint32_t]& index, - bool include_dataset) except + - - cdef void deserialize(const device_resources& handle, - const string& str, - index[uint8_t, uint32_t]* index) except + - - cdef void serialize(const device_resources& handle, - string& str, - const index[int8_t, uint32_t]& index, - bool include_dataset) except + - - cdef void deserialize(const device_resources& handle, - const string& str, - index[int8_t, uint32_t]* index) except + - - cdef void serialize_file(const device_resources& handle, - const string& filename, - const index[float, uint32_t]& index, - bool include_dataset) except + - - cdef void deserialize_file(const device_resources& handle, - const string& filename, - index[float, uint32_t]* index) except + - - cdef void serialize_file(const device_resources& handle, - const string& filename, - const index[uint8_t, uint32_t]& index, - bool include_dataset) except + - - cdef void serialize_to_hnswlib( - const device_resources& handle, - string& str, - const index[float, uint32_t]& index) except + - - cdef void serialize_to_hnswlib( - const device_resources& handle, - string& str, - const index[uint8_t, uint32_t]& index) except + - - cdef void serialize_to_hnswlib( - const device_resources& handle, - string& str, - const index[int8_t, uint32_t]& index) except + - - cdef void serialize_to_hnswlib_file( - const device_resources& handle, - const string& filename, - const index[float, uint32_t]& index) except + - - cdef void serialize_to_hnswlib_file( - const device_resources& handle, - const string& filename, - const index[uint8_t, uint32_t]& index) except + - - cdef void serialize_to_hnswlib_file( - const device_resources& handle, - const string& filename, - const index[int8_t, uint32_t]& index) except + - - cdef void deserialize_file(const device_resources& handle, - const string& filename, - index[uint8_t, uint32_t]* index) except + - - cdef void serialize_file(const device_resources& handle, - const string& filename, - const index[int8_t, uint32_t]& index, - bool include_dataset) except + - - cdef void deserialize_file(const device_resources& handle, - const string& filename, - index[int8_t, uint32_t]* index) except + diff --git a/python/pylibraft/pylibraft/neighbors/common.pxd b/python/pylibraft/pylibraft/neighbors/common.pxd deleted file mode 100644 index b11ef3176e..0000000000 --- a/python/pylibraft/pylibraft/neighbors/common.pxd +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -from pylibraft.distance.distance_type cimport DistanceType - - -cdef _get_metric_string(DistanceType metric) diff --git a/python/pylibraft/pylibraft/neighbors/common.pyx b/python/pylibraft/pylibraft/neighbors/common.pyx deleted file mode 100644 index 24c1abcf18..0000000000 --- a/python/pylibraft/pylibraft/neighbors/common.pyx +++ /dev/null @@ -1,63 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import warnings - -from pylibraft.distance.distance_type cimport DistanceType - -SUPPORTED_DISTANCES = { - "sqeuclidean": DistanceType.L2Expanded, - "euclidean": DistanceType.L2SqrtExpanded, - "inner_product": DistanceType.InnerProduct, - -} - - -def _get_metric(metric): - if metric not in SUPPORTED_DISTANCES: - if metric == "l2_expanded": - warnings.warn("Using l2_expanded as a metric name is deprecated," - " use sqeuclidean instead", FutureWarning) - return DistanceType.L2Expanded - - raise ValueError("metric %s is not supported" % metric) - return SUPPORTED_DISTANCES[metric] - - -cdef _get_metric_string(DistanceType metric): - return {DistanceType.L2Expanded : "sqeuclidean", - DistanceType.InnerProduct: "inner_product", - DistanceType.L2SqrtExpanded: "euclidean"}[metric] - - -def _check_input_array(cai, exp_dt, exp_rows=None, exp_cols=None): - if cai.dtype not in exp_dt: - raise TypeError("dtype %s not supported" % cai.dtype) - - if not cai.c_contiguous: - raise ValueError("Row major input is expected") - - if exp_cols is not None and cai.shape[1] != exp_cols: - raise ValueError("Incorrect number of columns, expected {} got {}" - .format(exp_cols, cai.shape[1])) - - if exp_rows is not None and cai.shape[0] != exp_rows: - raise ValueError("Incorrect number of rows, expected {} , got {}" - .format(exp_rows, cai.shape[0])) diff --git a/python/pylibraft/pylibraft/neighbors/cpp/__init__.pxd b/python/pylibraft/pylibraft/neighbors/cpp/__init__.pxd deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/python/pylibraft/pylibraft/neighbors/cpp/__init__.py b/python/pylibraft/pylibraft/neighbors/cpp/__init__.py deleted file mode 100644 index a7e7b75096..0000000000 --- a/python/pylibraft/pylibraft/neighbors/cpp/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/pylibraft/pylibraft/neighbors/cpp/brute_force.pxd b/python/pylibraft/pylibraft/neighbors/cpp/brute_force.pxd deleted file mode 100644 index f513517868..0000000000 --- a/python/pylibraft/pylibraft/neighbors/cpp/brute_force.pxd +++ /dev/null @@ -1,68 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -import pylibraft.common.handle - -from cython.operator cimport dereference as deref -from libc.stdint cimport int8_t, int64_t, uint8_t, uint64_t, uintptr_t -from libcpp cimport bool, nullptr -from libcpp.string cimport string -from libcpp.vector cimport vector - -from rmm.librmm.memory_resource cimport device_memory_resource - -from pylibraft.common.cpp.mdspan cimport ( - device_matrix_view, - device_vector_view, - host_matrix_view, - make_device_matrix_view, - make_device_vector_view, - make_host_matrix_view, - row_major, -) -from pylibraft.common.cpp.optional cimport optional -from pylibraft.common.handle cimport device_resources -from pylibraft.distance.distance_type cimport DistanceType - - -cdef extern from "raft_runtime/neighbors/brute_force.hpp" \ - namespace "raft::runtime::neighbors::brute_force" nogil: - - cdef void knn(const device_resources & handle, - device_matrix_view[float, int64_t, row_major] index, - device_matrix_view[float, int64_t, row_major] search, - device_matrix_view[int64_t, int64_t, row_major] indices, - device_matrix_view[float, int64_t, row_major] distances, - DistanceType metric, - optional[float] metric_arg, - optional[int64_t] global_id_offset) except + - -cdef extern from "raft_runtime/neighbors/eps_neighborhood.hpp" \ - namespace "raft::runtime::neighbors::epsilon_neighborhood" nogil: - - cdef void eps_neighbors( - const device_resources & handle, - device_matrix_view[float, int64_t, row_major] index, - device_matrix_view[float, int64_t, row_major] search, - device_matrix_view[bool, int64_t, row_major] adj, - device_vector_view[int64_t, int64_t] vd, - float eps) except + diff --git a/python/pylibraft/pylibraft/neighbors/cpp/hnsw.pxd b/python/pylibraft/pylibraft/neighbors/cpp/hnsw.pxd deleted file mode 100644 index 7b2cf59c81..0000000000 --- a/python/pylibraft/pylibraft/neighbors/cpp/hnsw.pxd +++ /dev/null @@ -1,82 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -from libc.stdint cimport int8_t, int64_t, uint8_t, uint32_t, uint64_t -from libcpp.memory cimport unique_ptr -from libcpp.string cimport string - -from pylibraft.common.cpp.mdspan cimport ( - device_vector_view, - host_matrix_view, - row_major, -) -from pylibraft.common.handle cimport device_resources -from pylibraft.distance.distance_type cimport DistanceType -from pylibraft.neighbors.ivf_pq.cpp.c_ivf_pq cimport ( - ann_index, - ann_search_params, -) - - -cdef extern from "raft/neighbors/hnsw.hpp" \ - namespace "raft::neighbors::hnsw" nogil: - - cpdef cppclass search_params(ann_search_params): - int ef - int num_threads - - cdef cppclass index[T](ann_index): - index(int dim, DistanceType metric) - - int dim() - DistanceType metric() - - -cdef extern from "raft_runtime/neighbors/hnsw.hpp" \ - namespace "raft::runtime::neighbors::hnsw" nogil: - cdef void search( - const device_resources& handle, - const search_params& params, - const index[float]& index, - host_matrix_view[float, int64_t, row_major] queries, - host_matrix_view[uint64_t, int64_t, row_major] neighbors, - host_matrix_view[float, int64_t, row_major] distances) except + - - cdef void search( - const device_resources& handle, - const search_params& params, - const index[int8_t]& index, - host_matrix_view[int8_t, int64_t, row_major] queries, - host_matrix_view[uint64_t, int64_t, row_major] neighbors, - host_matrix_view[float, int64_t, row_major] distances) except + - - cdef void search( - const device_resources& handle, - const search_params& params, - const index[uint8_t]& index, - host_matrix_view[uint8_t, int64_t, row_major] queries, - host_matrix_view[uint64_t, int64_t, row_major] neighbors, - host_matrix_view[float, int64_t, row_major] distances) except + - - cdef unique_ptr[index[T]] deserialize_file[T]( - const device_resources& handle, - const string& filename, - int dim, - DistanceType metric) except + diff --git a/python/pylibraft/pylibraft/neighbors/cpp/rbc.pxd b/python/pylibraft/pylibraft/neighbors/cpp/rbc.pxd deleted file mode 100644 index d544797119..0000000000 --- a/python/pylibraft/pylibraft/neighbors/cpp/rbc.pxd +++ /dev/null @@ -1,84 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -import pylibraft.common.handle - -from cython.operator cimport dereference as deref -from libc.stdint cimport int8_t, int64_t, uint8_t, uint64_t, uintptr_t -from libcpp cimport bool, nullptr -from libcpp.string cimport string -from libcpp.vector cimport vector - -from rmm.librmm.memory_resource cimport device_memory_resource - -from pylibraft.common.cpp.mdspan cimport ( - device_matrix_view, - device_vector_view, - host_matrix_view, - make_device_matrix_view, - make_host_matrix_view, - row_major, -) -from pylibraft.common.handle cimport device_resources -from pylibraft.distance.distance_type cimport DistanceType - - -cdef extern from "raft/neighbors/ball_cover_types.hpp" \ - namespace "raft::neighbors::ball_cover" nogil: - - cdef cppclass BallCoverIndex[IdxT, T, IntT, MatIdxT]: - BallCoverIndex(const device_resources& handle, - device_matrix_view[T, MatIdxT, row_major] dataset, - DistanceType metric) - - -cdef extern from "raft_runtime/neighbors/eps_neighborhood.hpp" \ - namespace "raft::runtime::neighbors::epsilon_neighborhood" nogil: - - cdef void eps_neighbors_rbc( - const device_resources & handle, - device_matrix_view[float, int64_t, row_major] index, - device_matrix_view[float, int64_t, row_major] search, - device_matrix_view[bool, int64_t, row_major] adj, - device_vector_view[int64_t, int64_t] vd, - float eps) except + - - cdef void build_rbc_index( - const device_resources & handle, - BallCoverIndex[int64_t, float, int64_t, int64_t] rbc_index) except + - - cdef void eps_neighbors_rbc_pass1( - const device_resources & handle, - BallCoverIndex[int64_t, float, int64_t, int64_t] rbc_index, - device_matrix_view[float, int64_t, row_major] search, - device_vector_view[int64_t, int64_t] adj_ia, - device_vector_view[int64_t, int64_t] vd, - float eps) except + - - cdef void eps_neighbors_rbc_pass2( - const device_resources & handle, - BallCoverIndex[int64_t, float, int64_t, int64_t] rbc_index, - device_matrix_view[float, int64_t, row_major] search, - device_vector_view[int64_t, int64_t] adj_ia, - device_vector_view[int64_t, int64_t] adj_ja, - device_vector_view[int64_t, int64_t] vd, - float eps) except + diff --git a/python/pylibraft/pylibraft/neighbors/hnsw.pyx b/python/pylibraft/pylibraft/neighbors/hnsw.pyx deleted file mode 100644 index e6f2d69eb8..0000000000 --- a/python/pylibraft/pylibraft/neighbors/hnsw.pyx +++ /dev/null @@ -1,490 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -from cython.operator cimport dereference as deref -from libc.stdint cimport int8_t, uint8_t, uint32_t -from libcpp cimport bool -from libcpp.memory cimport unique_ptr -from libcpp.string cimport string - -cimport pylibraft.neighbors.cagra.cpp.c_cagra as c_cagra -from pylibraft.distance.distance_type cimport DistanceType -from pylibraft.neighbors.cagra.cagra cimport ( - Index, - IndexFloat, - IndexInt8, - IndexUint8, -) - -from pylibraft.common.handle import auto_sync_handle - -from pylibraft.common.handle cimport device_resources - -from pylibraft.common import DeviceResources, ai_wrapper, auto_convert_output - -cimport pylibraft.neighbors.cpp.hnsw as c_hnsw - -from pylibraft.neighbors.common import _check_input_array, _get_metric - -from pylibraft.common.mdspan cimport ( - get_hmv_float, - get_hmv_int8, - get_hmv_uint8, - get_hmv_uint64, -) -from pylibraft.neighbors.common cimport _get_metric_string - -import os -import uuid - -import numpy as np - - -cdef class HnswIndex: - cdef readonly bool trained - cdef str active_index_type - - def __cinit__(self): - self.trained = False - self.active_index_type = None - -cdef class HnswIndexFloat(HnswIndex): - cdef unique_ptr[c_hnsw.index[float]] index - - def __cinit__(self): - pass - - def __repr__(self): - m_str = "metric=" + _get_metric_string(self.metric) - attr_str = [attr + "=" + str(getattr(self, attr)) - for attr in ["dim"]] - attr_str = [m_str] + attr_str - return "Index(type=hnsw, " + (", ".join(attr_str)) + ")" - - @property - def dim(self): - return self.index.get()[0].dim() - - @property - def metric(self): - return self.index.get()[0].metric() - -cdef class HnswIndexInt8(HnswIndex): - cdef unique_ptr[c_hnsw.index[int8_t]] index - - def __cinit__(self): - pass - - def __repr__(self): - m_str = "metric=" + _get_metric_string(self.metric) - attr_str = [attr + "=" + str(getattr(self, attr)) - for attr in ["dim"]] - attr_str = [m_str] + attr_str - return "Index(type=hnsw, " + (", ".join(attr_str)) + ")" - - @property - def dim(self): - return self.index.get()[0].dim() - - @property - def metric(self): - return self.index.get()[0].metric() - -cdef class HnswIndexUint8(HnswIndex): - cdef unique_ptr[c_hnsw.index[uint8_t]] index - - def __cinit__(self): - pass - - def __repr__(self): - m_str = "metric=" + _get_metric_string(self.metric) - attr_str = [attr + "=" + str(getattr(self, attr)) - for attr in ["dim"]] - attr_str = [m_str] + attr_str - return "Index(type=hnsw, " + (", ".join(attr_str)) + ")" - - @property - def dim(self): - return self.index.get()[0].dim() - - @property - def metric(self): - return self.index.get()[0].metric() - - -@auto_sync_handle -def save(filename, Index index, handle=None): - """ - Saves the CAGRA index as an hnswlib base-layer-only index to a file. - - Saving / loading the index is experimental. The serialization format is - subject to change. - - Parameters - ---------- - filename : string - Name of the file. - index : Index - Trained CAGRA index. - {handle_docstring} - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import cagra - >>> from pylibraft.neighbors import hnsw - >>> n_samples = 50000 - >>> n_features = 50 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Build index - >>> handle = DeviceResources() - >>> index = cagra.build(cagra.IndexParams(), dataset, handle=handle) - >>> # Serialize the CAGRA index to hnswlib base layer only index format - >>> hnsw.save("my_index.bin", index, handle=handle) - """ - if not index.trained: - raise ValueError("Index need to be built before saving it.") - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - cdef string c_filename = filename.encode('utf-8') - - cdef IndexFloat idx_float - cdef IndexInt8 idx_int8 - cdef IndexUint8 idx_uint8 - - cdef c_cagra.index[float, uint32_t] * c_index_float - cdef c_cagra.index[int8_t, uint32_t] * c_index_int8 - cdef c_cagra.index[uint8_t, uint32_t] * c_index_uint8 - - if index.active_index_type == "float32": - idx_float = index - c_index_float = \ - idx_float.index - c_cagra.serialize_to_hnswlib_file( - deref(handle_), c_filename, deref(c_index_float)) - elif index.active_index_type == "byte": - idx_int8 = index - c_index_int8 = \ - idx_int8.index - c_cagra.serialize_to_hnswlib_file( - deref(handle_), c_filename, deref(c_index_int8)) - elif index.active_index_type == "ubyte": - idx_uint8 = index - c_index_uint8 = \ - idx_uint8.index - c_cagra.serialize_to_hnswlib_file( - deref(handle_), c_filename, deref(c_index_uint8)) - else: - raise ValueError( - "Index dtype %s not supported" % index.active_index_type) - - -@auto_sync_handle -def load(filename, dim, dtype, metric="sqeuclidean", handle=None): - """ - Loads base-layer-only hnswlib index from file, which was originally - saved as a built CAGRA index. - - Saving / loading the index is experimental. The serialization format is - subject to change, therefore loading an index saved with a previous - version of raft is not guaranteed to work. - - Parameters - ---------- - filename : string - Name of the file. - dim : int - Dimensions of the training dataest - dtype : np.dtype of the saved index - Valid values for dtype: [np.float32, np.byte, np.ubyte] - metric : string denoting the metric type, default="sqeuclidean" - Valid values for metric: ["sqeuclidean", "inner_product"], where - - sqeuclidean is the euclidean distance without the square root - operation, i.e.: distance(a,b) = \\sum_i (a_i - b_i)^2, - - inner product distance is defined as - distance(a, b) = \\sum_i a_i * b_i. - {handle_docstring} - - Returns - ------- - index : HnswIndex - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import cagra - >>> from pylibraft.neighbors import hnsw - >>> n_samples = 50000 - >>> n_features = 50 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Build index - >>> handle = DeviceResources() - >>> index = cagra.build(cagra.IndexParams(), dataset, handle=handle) - >>> # Serialize the CAGRA index to hnswlib base layer only index format - >>> hnsw.save("my_index.bin", index, handle=handle) - >>> index = hnsw.load("my_index.bin", n_features, np.float32, - ... "sqeuclidean") - """ - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - cdef string c_filename = filename.encode('utf-8') - cdef HnswIndexFloat idx_float - cdef HnswIndexInt8 idx_int8 - cdef HnswIndexUint8 idx_uint8 - - cdef DistanceType c_metric = _get_metric(metric) - - if dtype == np.float32: - idx_float = HnswIndexFloat() - idx_float.index = c_hnsw.deserialize_file[float]( - deref(handle_), c_filename, dim, c_metric) - idx_float.trained = True - idx_float.active_index_type = 'float32' - return idx_float - elif dtype == np.byte: - idx_int8 = HnswIndexInt8(dim, metric) - idx_int8.index = c_hnsw.deserialize_file[int8_t]( - deref(handle_), c_filename, dim, c_metric) - idx_int8.trained = True - idx_int8.active_index_type = 'byte' - return idx_int8 - elif dtype == np.ubyte: - idx_uint8 = HnswIndexUint8(dim, metric) - idx_uint8.index = c_hnsw.deserialize_file[uint8_t]( - deref(handle_), c_filename, dim, c_metric) - idx_uint8.trained = True - idx_uint8.active_index_type = 'ubyte' - return idx_uint8 - else: - raise ValueError("Dataset dtype %s not supported" % dtype) - - -@auto_sync_handle -def from_cagra(Index index, handle=None): - """ - Returns an hnswlib base-layer-only index from a CAGRA index. - - NOTE: This method uses the filesystem to write the CAGRA index in - `/tmp/.bin` before reading it as an hnswlib index, - then deleting the temporary file. - - Saving / loading the index is experimental. The serialization format is - subject to change. - - Parameters - ---------- - index : Index - Trained CAGRA index. - {handle_docstring} - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import cagra - >>> from pylibraft.neighbors import hnsw - >>> n_samples = 50000 - >>> n_features = 50 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Build index - >>> handle = DeviceResources() - >>> index = cagra.build(cagra.IndexParams(), dataset, handle=handle) - >>> # Serialize the CAGRA index to hnswlib base layer only index format - >>> hnsw_index = hnsw.from_cagra(index, handle=handle) - """ - uuid_num = uuid.uuid4() - filename = f"/tmp/{uuid_num}.bin" - save(filename, index, handle=handle) - hnsw_index = load(filename, index.dim, np.dtype(index.active_index_type), - _get_metric_string(index.metric), handle=handle) - os.remove(filename) - return hnsw_index - - -cdef class SearchParams: - """ - Hnswlib search parameters - - Parameters - ---------- - ef: int, default=200 - Size of list from which final neighbors k will be selected. - ef should be greater than or equal to k. - num_threads: int, default=1 - Number of host threads to use to search the hnswlib index - and increase concurrency - """ - cdef c_hnsw.search_params params - - def __init__(self, ef=200, num_threads=1): - self.params.ef = ef - self.params.num_threads = num_threads - - def __repr__(self): - attr_str = [attr + "=" + str(getattr(self, attr)) - for attr in [ - "ef", "num_threads"]] - return "SearchParams(type=hnsw, " + ( - ", ".join(attr_str)) + ")" - - @property - def ef(self): - return self.params.ef - - @property - def num_threads(self): - return self.params.num_threads - - -@auto_sync_handle -@auto_convert_output -def search(SearchParams search_params, - HnswIndex index, - queries, - k, - neighbors=None, - distances=None, - handle=None): - """ - Find the k nearest neighbors for each query. - - Parameters - ---------- - search_params : SearchParams - index : HnswIndex - Trained CAGRA index saved as base-layer-only hnswlib index. - queries : array interface compliant matrix shape (n_samples, dim) - Supported dtype [float, int8, uint8] - k : int - The number of neighbors. - neighbors : Optional array interface compliant matrix shape - (n_queries, k), dtype int64_t. If supplied, neighbor - indices will be written here in-place. (default None) - distances : Optional array interface compliant matrix shape - (n_queries, k) If supplied, the distances to the - neighbors will be written here in-place. (default None) - {handle_docstring} - - Examples - -------- - >>> import cupy as cp - >>> import numpy as np - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import cagra - >>> from pylibraft.neighbors import hnsw - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Build index - >>> handle = DeviceResources() - >>> index = cagra.build(cagra.IndexParams(), dataset, handle=handle) - >>> - >>> # Load saved base-layer-only hnswlib index from CAGRA index - >>> hnsw_index = hnsw.from_cagra(index, handle=handle) - >>> - >>> # Search hnswlib using the loaded index - >>> queries = np.random.random_sample((n_queries, n_features)).astype( - ... np.float32) - >>> k = 10 - >>> search_params = hnsw.SearchParams( - ... ef=20, - ... num_threads=5 - ... ) - >>> distances, neighbors = hnsw.search(search_params, hnsw_index, - ... queries, k, handle=handle) - """ - - if not index.trained: - raise ValueError("Index need to be built before calling search.") - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - queries_ai = ai_wrapper(queries) - queries_dt = queries_ai.dtype - cdef uint32_t n_queries = queries_ai.shape[0] - - _check_input_array(queries_ai, [np.dtype('float32'), np.dtype('byte'), - np.dtype('ubyte')], - exp_cols=index.dim) - - if neighbors is None: - neighbors = np.empty((n_queries, k), dtype='uint64') - - neighbors_ai = ai_wrapper(neighbors) - _check_input_array(neighbors_ai, [np.dtype('uint64')], - exp_rows=n_queries, exp_cols=k) - - if distances is None: - distances = np.empty((n_queries, k), dtype='float32') - - distances_ai = ai_wrapper(distances) - _check_input_array(distances_ai, [np.dtype('float32')], - exp_rows=n_queries, exp_cols=k) - - cdef c_hnsw.search_params params = search_params.params - cdef HnswIndexFloat idx_float - cdef HnswIndexInt8 idx_int8 - cdef HnswIndexUint8 idx_uint8 - - if queries_dt == np.float32: - idx_float = index - c_hnsw.search(deref(handle_), - params, - deref(idx_float.index), - get_hmv_float(queries_ai, check_shape=True), - get_hmv_uint64(neighbors_ai, check_shape=True), - get_hmv_float(distances_ai, check_shape=True)) - elif queries_dt == np.byte: - idx_int8 = index - c_hnsw.search(deref(handle_), - params, - deref(idx_int8.index), - get_hmv_int8(queries_ai, check_shape=True), - get_hmv_uint64(neighbors_ai, check_shape=True), - get_hmv_float(distances_ai, check_shape=True)) - elif queries_dt == np.ubyte: - idx_uint8 = index - c_hnsw.search(deref(handle_), - params, - deref(idx_uint8.index), - get_hmv_uint8(queries_ai, check_shape=True), - get_hmv_uint64(neighbors_ai, check_shape=True), - get_hmv_float(distances_ai, check_shape=True)) - else: - raise ValueError("query dtype %s not supported" % queries_dt) - - return (distances, neighbors) diff --git a/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt deleted file mode 100644 index 37c57c45db..0000000000 --- a/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -# ============================================================================= -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -# Set the list of Cython files to build -set(cython_sources ivf_flat.pyx) -set(linked_libraries raft::raft raft::compiled) - -# Build all of the Cython targets -rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ivfflat_ -) diff --git a/python/pylibraft/pylibraft/neighbors/ivf_flat/__init__.pxd b/python/pylibraft/pylibraft/neighbors/ivf_flat/__init__.pxd deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/python/pylibraft/pylibraft/neighbors/ivf_flat/__init__.py b/python/pylibraft/pylibraft/neighbors/ivf_flat/__init__.py deleted file mode 100644 index 057cb98f17..0000000000 --- a/python/pylibraft/pylibraft/neighbors/ivf_flat/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from .ivf_flat import ( - Index, - IndexParams, - SearchParams, - build, - extend, - load, - save, - search, -) - -__all__ = [ - "Index", - "IndexParams", - "SearchParams", - "build", - "extend", - "search", - "save", - "load", -] diff --git a/python/pylibraft/pylibraft/neighbors/ivf_flat/cpp/__init__.pxd b/python/pylibraft/pylibraft/neighbors/ivf_flat/cpp/__init__.pxd deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/python/pylibraft/pylibraft/neighbors/ivf_flat/cpp/__init__.py b/python/pylibraft/pylibraft/neighbors/ivf_flat/cpp/__init__.py deleted file mode 100644 index 8f2cc34855..0000000000 --- a/python/pylibraft/pylibraft/neighbors/ivf_flat/cpp/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/pylibraft/pylibraft/neighbors/ivf_flat/cpp/c_ivf_flat.pxd b/python/pylibraft/pylibraft/neighbors/ivf_flat/cpp/c_ivf_flat.pxd deleted file mode 100644 index 22b08b3f19..0000000000 --- a/python/pylibraft/pylibraft/neighbors/ivf_flat/cpp/c_ivf_flat.pxd +++ /dev/null @@ -1,183 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -import pylibraft.common.handle - -from cython.operator cimport dereference as deref -from libc.stdint cimport int8_t, int64_t, uint8_t, uint32_t, uintptr_t -from libcpp cimport bool, nullptr -from libcpp.string cimport string - -from rmm.librmm.memory_resource cimport device_memory_resource - -from pylibraft.common.cpp.mdspan cimport ( - device_matrix_view, - device_vector_view, - host_matrix_view, - make_device_matrix_view, - make_host_matrix_view, - row_major, -) -from pylibraft.common.cpp.optional cimport optional -from pylibraft.common.handle cimport device_resources -from pylibraft.distance.distance_type cimport DistanceType -from pylibraft.neighbors.ivf_pq.cpp.c_ivf_pq cimport ( - ann_index, - ann_index_params, - ann_search_params, -) - - -cdef extern from "raft/neighbors/ivf_flat_types.hpp" \ - namespace "raft::neighbors::ivf_flat" nogil: - - cpdef cppclass index_params(ann_index_params): - uint32_t n_lists - uint32_t kmeans_n_iters - double kmeans_trainset_fraction - bool adaptive_centers - bool conservative_memory_allocation - - cdef cppclass index[T, IdxT](ann_index): - index(const device_resources& handle, - DistanceType metric, - uint32_t n_lists, - bool adaptive_centers, - bool conservative_memory_allocation, - uint32_t dim) - IdxT size() - uint32_t dim() - DistanceType metric() - uint32_t n_lists() - bool adaptive_centers() - - cpdef cppclass search_params(ann_search_params): - uint32_t n_probes - - -cdef extern from "raft_runtime/neighbors/ivf_flat.hpp" \ - namespace "raft::runtime::neighbors::ivf_flat" nogil: - - cdef void build(const device_resources&, - const index_params& params, - device_matrix_view[float, int64_t, row_major] dataset, - index[float, int64_t]& index) except + - - cdef void build(const device_resources& handle, - const index_params& params, - device_matrix_view[int8_t, int64_t, row_major] dataset, - index[int8_t, int64_t]& index) except + - - cdef void build(const device_resources& handle, - const index_params& params, - device_matrix_view[uint8_t, int64_t, row_major] dataset, - index[uint8_t, int64_t]& index) except + - - cdef void extend( - const device_resources& handle, - device_matrix_view[float, int64_t, row_major] new_vectors, - optional[device_vector_view[int64_t, int64_t]] new_indices, - index[float, int64_t]* index) except + - - cdef void extend( - const device_resources& handle, - device_matrix_view[int8_t, int64_t, row_major] new_vectors, - optional[device_vector_view[int64_t, int64_t]] new_indices, - index[int8_t, int64_t]* index) except + - - cdef void extend( - const device_resources& handle, - device_matrix_view[uint8_t, int64_t, row_major] new_vectors, - optional[device_vector_view[int64_t, int64_t]] new_indices, - index[uint8_t, int64_t]* index) except + - - cdef void search( - const device_resources& handle, - const search_params& params, - const index[float, int64_t]& index, - device_matrix_view[float, int64_t, row_major] queries, - device_matrix_view[int64_t, int64_t, row_major] neighbors, - device_matrix_view[float, int64_t, row_major] distances) except + - - cdef void search( - const device_resources& handle, - const search_params& params, - const index[int8_t, int64_t]& index, - device_matrix_view[int8_t, int64_t, row_major] queries, - device_matrix_view[int64_t, int64_t, row_major] neighbors, - device_matrix_view[float, int64_t, row_major] distances) except + - - cdef void search( - const device_resources& handle, - const search_params& params, - const index[uint8_t, int64_t]& index, - device_matrix_view[uint8_t, int64_t, row_major] queries, - device_matrix_view[int64_t, int64_t, row_major] neighbors, - device_matrix_view[float, int64_t, row_major] distances) except + - - cdef void serialize(const device_resources& handle, - string& str, - const index[float, int64_t]& index) except + - - cdef void deserialize(const device_resources& handle, - const string& str, - index[float, int64_t]* index) except + - - cdef void serialize(const device_resources& handle, - string& str, - const index[uint8_t, int64_t]& index) except + - - cdef void deserialize(const device_resources& handle, - const string& str, - index[uint8_t, int64_t]* index) except + - - cdef void serialize(const device_resources& handle, - string& str, - const index[int8_t, int64_t]& index) except + - - cdef void deserialize(const device_resources& handle, - const string& str, - index[int8_t, int64_t]* index) except + - - cdef void serialize_file(const device_resources& handle, - const string& filename, - const index[float, int64_t]& index) except + - - cdef void deserialize_file(const device_resources& handle, - const string& filename, - index[float, int64_t]* index) except + - - cdef void serialize_file(const device_resources& handle, - const string& filename, - const index[uint8_t, int64_t]& index) except + - - cdef void deserialize_file(const device_resources& handle, - const string& filename, - index[uint8_t, int64_t]* index) except + - - cdef void serialize_file(const device_resources& handle, - const string& filename, - const index[int8_t, int64_t]& index) except + - - cdef void deserialize_file(const device_resources& handle, - const string& filename, - index[int8_t, int64_t]* index) except + diff --git a/python/pylibraft/pylibraft/neighbors/ivf_flat/ivf_flat.pyx b/python/pylibraft/pylibraft/neighbors/ivf_flat/ivf_flat.pyx deleted file mode 100644 index 6826b2bc59..0000000000 --- a/python/pylibraft/pylibraft/neighbors/ivf_flat/ivf_flat.pyx +++ /dev/null @@ -1,821 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import warnings - -import numpy as np - -from cython.operator cimport dereference as deref -from libc.stdint cimport int8_t, int64_t, uint8_t, uint32_t, uintptr_t -from libcpp cimport bool, nullptr -from libcpp.string cimport string - -from pylibraft.distance.distance_type cimport DistanceType - -from pylibraft.common import ( - DeviceResources, - ai_wrapper, - auto_convert_output, - device_ndarray, -) -from pylibraft.common.cai_wrapper import cai_wrapper - -from pylibraft.common.cpp.mdspan cimport ( - device_matrix_view, - device_vector_view, - make_device_vector_view, - row_major, -) - -from pylibraft.common.interruptible import cuda_interruptible - -from pylibraft.common.handle cimport device_resources - -from pylibraft.common.handle import auto_sync_handle -from pylibraft.common.input_validation import is_c_contiguous - -from rmm.librmm.memory_resource cimport device_memory_resource -from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource - -cimport pylibraft.neighbors.ivf_flat.cpp.c_ivf_flat as c_ivf_flat -from pylibraft.common.cpp.optional cimport optional - -from pylibraft.neighbors.common import _check_input_array, _get_metric - -from pylibraft.common.mdspan cimport ( - get_dmv_float, - get_dmv_int8, - get_dmv_int64, - get_dmv_uint8, -) -from pylibraft.neighbors.common cimport _get_metric_string -from pylibraft.neighbors.ivf_flat.cpp.c_ivf_flat cimport ( - index_params, - search_params, -) - - -cdef class IndexParams: - """ - Parameters to build index for IVF-FLAT nearest neighbor search - - Parameters - ---------- - n_list : int, default = 1024 - The number of clusters used in the coarse quantizer. - metric : string denoting the metric type, default="sqeuclidean" - Valid values for metric: ["sqeuclidean", "inner_product", - "euclidean"], where - - sqeuclidean is the euclidean distance without the square root - operation, i.e.: distance(a,b) = \\sum_i (a_i - b_i)^2, - - euclidean is the euclidean distance - - inner product distance is defined as - distance(a, b) = \\sum_i a_i * b_i. - kmeans_n_iters : int, default = 20 - The number of iterations searching for kmeans centers during index - building. - kmeans_trainset_fraction : int, default = 0.5 - If kmeans_trainset_fraction is less than 1, then the dataset is - subsampled, and only n_samples * kmeans_trainset_fraction rows - are used for training. - add_data_on_build : bool, default = True - After training the coarse and fine quantizers, we will populate - the index with the dataset if add_data_on_build == True, otherwise - the index is left empty, and the extend method can be used - to add new vectors to the index. - adaptive_centers : bool, default = False - By default (adaptive_centers = False), the cluster centers are - trained in `ivf_flat::build`, and and never modified in - `ivf_flat::extend`. The alternative behavior (adaptive_centers - = true) is to update the cluster centers for new data when it is - added. In this case, `index.centers()` are always exactly the - centroids of the data in the corresponding clusters. The drawback - of this behavior is that the centroids depend on the order of - adding new data (through the classification of the added data); - that is, `index.centers()` "drift" together with the changing - distribution of the newly added data. - """ - cdef c_ivf_flat.index_params params - - def __init__(self, *, - n_lists=1024, - metric="sqeuclidean", - kmeans_n_iters=20, - kmeans_trainset_fraction=0.5, - add_data_on_build=True, - bool adaptive_centers=False): - self.params.n_lists = n_lists - self.params.metric = _get_metric(metric) - self.params.metric_arg = 0 - self.params.kmeans_n_iters = kmeans_n_iters - self.params.kmeans_trainset_fraction = kmeans_trainset_fraction - self.params.add_data_on_build = add_data_on_build - self.params.adaptive_centers = adaptive_centers - - @property - def n_lists(self): - return self.params.n_lists - - @property - def metric(self): - return self.params.metric - - @property - def kmeans_n_iters(self): - return self.params.kmeans_n_iters - - @property - def kmeans_trainset_fraction(self): - return self.params.kmeans_trainset_fraction - - @property - def add_data_on_build(self): - return self.params.add_data_on_build - - @property - def adaptive_centers(self): - return self.params.adaptive_centers - - -cdef class Index: - cdef readonly bool trained - cdef str active_index_type - - def __cinit__(self): - self.trained = False - self.active_index_type = None - - -cdef class IndexFloat(Index): - cdef c_ivf_flat.index[float, int64_t] * index - - def __cinit__(self, handle=None): - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - # this is to keep track of which index type is being used - # We create a placeholder object. The actual parameter values do - # not matter, it will be replaced with a built index object later. - self.index = new c_ivf_flat.index[float, int64_t]( - deref(handle_), _get_metric("sqeuclidean"), - 1, - False, - False, - 4) - - def __repr__(self): - m_str = "metric=" + _get_metric_string(self.index.metric()) - attr_str = [ - attr + "=" + str(getattr(self, attr)) - for attr in ["size", "dim", "n_lists", "adaptive_centers"] - ] - attr_str = [m_str] + attr_str - return "Index(type=IVF-FLAT, " + (", ".join(attr_str)) + ")" - - @property - def dim(self): - return self.index[0].dim() - - @property - def size(self): - return self.index[0].size() - - @property - def metric(self): - return self.index[0].metric() - - @property - def n_lists(self): - return self.index[0].n_lists() - - @property - def adaptive_centers(self): - return self.index[0].adaptive_centers() - - -cdef class IndexInt8(Index): - cdef c_ivf_flat.index[int8_t, int64_t] * index - - def __cinit__(self, handle=None): - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - # this is to keep track of which index type is being used - # We create a placeholder object. The actual parameter values do - # not matter, it will be replaced with a built index object later. - self.index = new c_ivf_flat.index[int8_t, int64_t]( - deref(handle_), _get_metric("sqeuclidean"), - 1, - False, - False, - 4) - - def __repr__(self): - m_str = "metric=" + _get_metric_string(self.index.metric()) - attr_str = [ - attr + "=" + str(getattr(self, attr)) - for attr in ["size", "dim", "n_lists", "adaptive_centers"] - ] - attr_str = [m_str] + attr_str - return "Index(type=IVF-FLAT, " + (", ".join(attr_str)) + ")" - - @property - def dim(self): - return self.index[0].dim() - - @property - def size(self): - return self.index[0].size() - - @property - def metric(self): - return self.index[0].metric() - - @property - def n_lists(self): - return self.index[0].n_lists() - - @property - def adaptive_centers(self): - return self.index[0].adaptive_centers() - - -cdef class IndexUint8(Index): - cdef c_ivf_flat.index[uint8_t, int64_t] * index - - def __cinit__(self, handle=None): - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - # this is to keep track of which index type is being used - # We create a placeholder object. The actual parameter values do - # not matter, it will be replaced with a built index object later. - self.index = new c_ivf_flat.index[uint8_t, int64_t]( - deref(handle_), _get_metric("sqeuclidean"), - 1, - False, - False, - 4) - - def __repr__(self): - m_str = "metric=" + _get_metric_string(self.index.metric()) - attr_str = [ - attr + "=" + str(getattr(self, attr)) - for attr in ["size", "dim", "n_lists", "adaptive_centers"] - ] - attr_str = [m_str] + attr_str - return "Index(type=IVF-FLAT, " + (", ".join(attr_str)) + ")" - - @property - def dim(self): - return self.index[0].dim() - - @property - def size(self): - return self.index[0].size() - - @property - def metric(self): - return self.index[0].metric() - - @property - def n_lists(self): - return self.index[0].n_lists() - - @property - def adaptive_centers(self): - return self.index[0].adaptive_centers() - - -@auto_sync_handle -@auto_convert_output -def build(IndexParams index_params, dataset, handle=None): - """ - Builds an IVF-FLAT index that can be used for nearest neighbor search. - - Parameters - ---------- - index_params : IndexParams object - dataset : CUDA array interface compliant matrix shape (n_samples, dim) - Supported dtype [float, int8, uint8] - {handle_docstring} - - Returns - ------- - index: ivf_flat.Index - - Examples - -------- - - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import ivf_flat - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> handle = DeviceResources() - >>> index_params = ivf_flat.IndexParams( - ... n_lists=1024, - ... metric="sqeuclidean") - >>> index = ivf_flat.build(index_params, dataset, handle=handle) - >>> # Search using the built index - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> k = 10 - >>> distances, neighbors = ivf_flat.search(ivf_flat.SearchParams(), - ... index, queries, k, - ... handle=handle) - >>> distances = cp.asarray(distances) - >>> neighbors = cp.asarray(neighbors) - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - """ - dataset_cai = cai_wrapper(dataset) - dataset_dt = dataset_cai.dtype - _check_input_array(dataset_cai, [np.dtype('float32'), np.dtype('byte'), - np.dtype('ubyte')]) - - cdef int64_t n_rows = dataset_cai.shape[0] - cdef uint32_t dim = dataset_cai.shape[1] - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - cdef IndexFloat idx_float - cdef IndexInt8 idx_int8 - cdef IndexUint8 idx_uint8 - - if dataset_dt == np.float32: - idx_float = IndexFloat(handle) - idx_float.active_index_type = "float32" - with cuda_interruptible(): - c_ivf_flat.build(deref(handle_), - index_params.params, - get_dmv_float(dataset_cai, check_shape=True), - deref(idx_float.index)) - idx_float.trained = True - return idx_float - elif dataset_dt == np.byte: - idx_int8 = IndexInt8(handle) - idx_int8.active_index_type = "byte" - with cuda_interruptible(): - c_ivf_flat.build(deref(handle_), - index_params.params, - get_dmv_int8(dataset_cai, check_shape=True), - deref(idx_int8.index)) - idx_int8.trained = True - return idx_int8 - elif dataset_dt == np.ubyte: - idx_uint8 = IndexUint8(handle) - idx_uint8.active_index_type = "ubyte" - with cuda_interruptible(): - c_ivf_flat.build(deref(handle_), - index_params.params, - get_dmv_uint8(dataset_cai, check_shape=True), - deref(idx_uint8.index)) - idx_uint8.trained = True - return idx_uint8 - else: - raise TypeError("dtype %s not supported" % dataset_dt) - - -@auto_sync_handle -@auto_convert_output -def extend(Index index, new_vectors, new_indices, handle=None): - """ - Extend an existing index with new vectors. - - Parameters - ---------- - index : ivf_flat.Index - Trained ivf_flat object. - new_vectors : CUDA array interface compliant matrix shape (n_samples, dim) - Supported dtype [float, int8, uint8] - new_indices : CUDA array interface compliant vector shape (n_samples) - Supported dtype [int64] - {handle_docstring} - - Returns - ------- - index: ivf_flat.Index - - Examples - -------- - - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import ivf_flat - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> handle = DeviceResources() - >>> index = ivf_flat.build(ivf_flat.IndexParams(), dataset, - ... handle=handle) - >>> n_rows = 100 - >>> more_data = cp.random.random_sample((n_rows, n_features), - ... dtype=cp.float32) - >>> indices = index.size + cp.arange(n_rows, dtype=cp.int64) - >>> index = ivf_flat.extend(index, more_data, indices) - >>> # Search using the built index - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> k = 10 - >>> distances, neighbors = ivf_flat.search(ivf_flat.SearchParams(), - ... index, queries, - ... k, handle=handle) - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - - >>> distances = cp.asarray(distances) - >>> neighbors = cp.asarray(neighbors) - """ - if not index.trained: - raise ValueError("Index need to be built before calling extend.") - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - vecs_cai = cai_wrapper(new_vectors) - vecs_dt = vecs_cai.dtype - cdef int64_t n_rows = vecs_cai.shape[0] - cdef uint32_t dim = vecs_cai.shape[1] - - _check_input_array(vecs_cai, [np.dtype(index.active_index_type)], - exp_cols=index.dim) - - idx_cai = cai_wrapper(new_indices) - _check_input_array(idx_cai, [np.dtype('int64')], exp_rows=n_rows) - if len(idx_cai.shape)!=1: - raise ValueError("Indices array is expected to be 1D") - - cdef optional[device_vector_view[int64_t, int64_t]] new_indices_opt - - cdef IndexFloat idx_float - cdef IndexInt8 idx_int8 - cdef IndexUint8 idx_uint8 - - if vecs_dt == np.float32: - idx_float = index - if idx_float.index.size() > 0: - new_indices_opt = make_device_vector_view( - idx_cai.data, - idx_cai.shape[0]) - with cuda_interruptible(): - c_ivf_flat.extend(deref(handle_), - get_dmv_float(vecs_cai, check_shape=True), - new_indices_opt, - idx_float.index) - elif vecs_dt == np.int8: - idx_int8 = index - if idx_int8.index[0].size() > 0: - new_indices_opt = make_device_vector_view( - idx_cai.data, - idx_cai.shape[0]) - with cuda_interruptible(): - c_ivf_flat.extend(deref(handle_), - get_dmv_int8(vecs_cai, check_shape=True), - new_indices_opt, - idx_int8.index) - elif vecs_dt == np.uint8: - idx_uint8 = index - if idx_uint8.index[0].size() > 0: - new_indices_opt = make_device_vector_view( - idx_cai.data, - idx_cai.shape[0]) - with cuda_interruptible(): - c_ivf_flat.extend(deref(handle_), - get_dmv_uint8(vecs_cai, check_shape=True), - new_indices_opt, - idx_uint8.index) - else: - raise TypeError("query dtype %s not supported" % vecs_dt) - - return index - - -cdef class SearchParams: - """ - IVF-FLAT search parameters - - Parameters - ---------- - n_probes: int, default = 1024 - The number of course clusters to select for the fine search. - """ - cdef c_ivf_flat.search_params params - - def __init__(self, *, n_probes=20): - self.params.n_probes = n_probes - - def __repr__(self): - attr_str = [attr + "=" + str(getattr(self, attr)) - for attr in ["n_probes"]] - return "SearchParams(type=IVF-FLAT, " + (", ".join(attr_str)) + ")" - - @property - def n_probes(self): - return self.params.n_probes - - -@auto_sync_handle -@auto_convert_output -def search(SearchParams search_params, - Index index, - queries, - k, - neighbors=None, - distances=None, - handle=None): - """ - Find the k nearest neighbors for each query. - - Parameters - ---------- - search_params : SearchParams - index : Index - Trained IVF-FLAT index. - queries : CUDA array interface compliant matrix shape (n_samples, dim) - Supported dtype [float, int8, uint8] - k : int - The number of neighbors. - neighbors : Optional CUDA array interface compliant matrix shape - (n_queries, k), dtype int64_t. If supplied, neighbor - indices will be written here in-place. (default None) - distances : Optional CUDA array interface compliant matrix shape - (n_queries, k) If supplied, the distances to the - neighbors will be written here in-place. (default None) - {handle_docstring} - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import ivf_flat - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Build index - >>> handle = DeviceResources() - >>> index = ivf_flat.build(ivf_flat.IndexParams(), dataset, - ... handle=handle) - >>> # Search using the built index - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> k = 10 - >>> search_params = ivf_flat.SearchParams( - ... n_probes=20 - ... ) - >>> distances, neighbors = ivf_flat.search(search_params, index, - ... queries, k, handle=handle) - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - >>> neighbors = cp.asarray(neighbors) - >>> distances = cp.asarray(distances) - """ - - if not index.trained: - raise ValueError("Index need to be built before calling search.") - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - queries_cai = cai_wrapper(queries) - queries_dt = queries_cai.dtype - cdef uint32_t n_queries = queries_cai.shape[0] - - _check_input_array(queries_cai, [np.dtype(index.active_index_type)], - exp_cols=index.dim) - - if neighbors is None: - neighbors = device_ndarray.empty((n_queries, k), dtype='int64') - - neighbors_cai = cai_wrapper(neighbors) - _check_input_array(neighbors_cai, [np.dtype('int64')], - exp_rows=n_queries, exp_cols=k) - - if distances is None: - distances = device_ndarray.empty((n_queries, k), dtype='float32') - - distances_cai = cai_wrapper(distances) - _check_input_array(distances_cai, [np.dtype('float32')], - exp_rows=n_queries, exp_cols=k) - - cdef c_ivf_flat.search_params params = search_params.params - cdef IndexFloat idx_float - cdef IndexInt8 idx_int8 - cdef IndexUint8 idx_uint8 - - if queries_dt == np.float32: - idx_float = index - with cuda_interruptible(): - c_ivf_flat.search(deref(handle_), - params, - deref(idx_float.index), - get_dmv_float(queries_cai, check_shape=True), - get_dmv_int64(neighbors_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True)) - elif queries_dt == np.byte: - idx_int8 = index - with cuda_interruptible(): - c_ivf_flat.search(deref(handle_), - params, - deref(idx_int8.index), - get_dmv_int8(queries_cai, check_shape=True), - get_dmv_int64(neighbors_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True)) - elif queries_dt == np.ubyte: - idx_uint8 = index - with cuda_interruptible(): - c_ivf_flat.search(deref(handle_), - params, - deref(idx_uint8.index), - get_dmv_uint8(queries_cai, check_shape=True), - get_dmv_int64(neighbors_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True)) - else: - raise ValueError("query dtype %s not supported" % queries_dt) - - return (distances, neighbors) - - -@auto_sync_handle -def save(filename, Index index, handle=None): - """ - Saves the index to a file. - - Saving / loading the index is experimental. The serialization format is - subject to change. - - Parameters - ---------- - filename : string - Name of the file. - index : Index - Trained IVF-Flat index. - {handle_docstring} - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import ivf_flat - >>> n_samples = 50000 - >>> n_features = 50 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Build index - >>> handle = DeviceResources() - >>> index = ivf_flat.build(ivf_flat.IndexParams(), dataset, - ... handle=handle) - >>> ivf_flat.save("my_index.bin", index, handle=handle) - """ - if not index.trained: - raise ValueError("Index need to be built before saving it.") - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - cdef string c_filename = filename.encode('utf-8') - - cdef IndexFloat idx_float - cdef IndexInt8 idx_int8 - cdef IndexUint8 idx_uint8 - - if index.active_index_type == "float32": - idx_float = index - c_ivf_flat.serialize_file( - deref(handle_), c_filename, deref(idx_float.index)) - elif index.active_index_type == "byte": - idx_int8 = index - c_ivf_flat.serialize_file( - deref(handle_), c_filename, deref(idx_int8.index)) - elif index.active_index_type == "ubyte": - idx_uint8 = index - c_ivf_flat.serialize_file( - deref(handle_), c_filename, deref(idx_uint8.index)) - else: - raise ValueError( - "Index dtype %s not supported" % index.active_index_type) - - -@auto_sync_handle -def load(filename, handle=None): - """ - Loads index from a file. - - Saving / loading the index is experimental. The serialization format is - subject to change, therefore loading an index saved with a previous - version of raft is not guaranteed to work. - - Parameters - ---------- - filename : string - Name of the file. - {handle_docstring} - - Returns - ------- - index : Index - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import ivf_flat - >>> n_samples = 50000 - >>> n_features = 50 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Build and save index - >>> handle = DeviceResources() - >>> index = ivf_flat.build(ivf_flat.IndexParams(), dataset, - ... handle=handle) - >>> ivf_flat.save("my_index.bin", index, handle=handle) - >>> del index - >>> n_queries = 100 - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> handle = DeviceResources() - >>> index = ivf_flat.load("my_index.bin", handle=handle) - >>> distances, neighbors = ivf_flat.search(ivf_flat.SearchParams(), - ... index, queries, k=10, - ... handle=handle) - """ - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - cdef string c_filename = filename.encode('utf-8') - cdef IndexFloat idx_float - cdef IndexInt8 idx_int8 - cdef IndexUint8 idx_uint8 - - with open(filename, 'rb') as f: - type_str = f.read(3).decode('utf-8') - - dataset_dt = np.dtype(type_str) - - if dataset_dt == np.float32: - idx_float = IndexFloat(handle) - c_ivf_flat.deserialize_file( - deref(handle_), c_filename, idx_float.index) - idx_float.trained = True - idx_float.active_index_type = 'float32' - return idx_float - elif dataset_dt == np.byte: - idx_int8 = IndexInt8(handle) - c_ivf_flat.deserialize_file( - deref(handle_), c_filename, idx_int8.index) - idx_int8.trained = True - idx_int8.active_index_type = 'byte' - return idx_int8 - elif dataset_dt == np.ubyte: - idx_uint8 = IndexUint8(handle) - c_ivf_flat.deserialize_file( - deref(handle_), c_filename, idx_uint8.index) - idx_uint8.trained = True - idx_uint8.active_index_type = 'ubyte' - return idx_uint8 - else: - raise ValueError("Index dtype %s not supported" % dataset_dt) diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt deleted file mode 100644 index af431adb16..0000000000 --- a/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -# ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -# Set the list of Cython files to build -set(cython_sources ivf_pq.pyx) -set(linked_libraries raft::raft raft::compiled) - -# Build all of the Cython targets -rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ivfpq_ -) diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/__init__.pxd b/python/pylibraft/pylibraft/neighbors/ivf_pq/__init__.pxd deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/__init__.py b/python/pylibraft/pylibraft/neighbors/ivf_pq/__init__.py deleted file mode 100644 index 3d604f829d..0000000000 --- a/python/pylibraft/pylibraft/neighbors/ivf_pq/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from .ivf_pq import ( - Index, - IndexParams, - SearchParams, - build, - extend, - load, - save, - search, -) - -__all__ = [ - "Index", - "IndexParams", - "SearchParams", - "build", - "extend", - "load", - "save", - "search", -] diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/cpp/__init__.pxd b/python/pylibraft/pylibraft/neighbors/ivf_pq/cpp/__init__.pxd deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/cpp/__init__.py b/python/pylibraft/pylibraft/neighbors/ivf_pq/cpp/__init__.py deleted file mode 100644 index 273b4497cc..0000000000 --- a/python/pylibraft/pylibraft/neighbors/ivf_pq/cpp/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/cpp/c_ivf_pq.pxd b/python/pylibraft/pylibraft/neighbors/ivf_pq/cpp/c_ivf_pq.pxd deleted file mode 100644 index 18319bf452..0000000000 --- a/python/pylibraft/pylibraft/neighbors/ivf_pq/cpp/c_ivf_pq.pxd +++ /dev/null @@ -1,178 +0,0 @@ -# -# Copyright (c) 2022-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -import pylibraft.common.handle - -from cython.operator cimport dereference as deref -from libc.stdint cimport int8_t, int64_t, uint8_t, uint32_t, uintptr_t -from libcpp cimport bool, nullptr -from libcpp.string cimport string - -from rmm.librmm.memory_resource cimport device_memory_resource - -from pylibraft.common.cpp.mdspan cimport ( - device_matrix_view, - device_vector_view, - row_major, -) -from pylibraft.common.handle cimport device_resources -from pylibraft.common.optional cimport optional -from pylibraft.distance.distance_type cimport DistanceType - - -cdef extern from "library_types.h": - ctypedef enum cudaDataType_t: - CUDA_R_32F "CUDA_R_32F" # float - CUDA_R_16F "CUDA_R_16F" # half - - # uint8 - used to refer to IVF-PQ's fp8 storage type - CUDA_R_8U "CUDA_R_8U" - -cdef extern from "raft/neighbors/ann_types.hpp" \ - namespace "raft::neighbors::ann" nogil: - - cdef cppclass ann_index "raft::neighbors::index": - pass - - cdef cppclass ann_index_params "raft::spatial::knn::index_params": - DistanceType metric - float metric_arg - bool add_data_on_build - - cdef cppclass ann_search_params "raft::spatial::knn::search_params": - pass - - -cdef extern from "raft/neighbors/ivf_pq_types.hpp" \ - namespace "raft::neighbors::ivf_pq" nogil: - - ctypedef enum codebook_gen: - PER_SUBSPACE "raft::neighbors::ivf_pq::codebook_gen::PER_SUBSPACE", - PER_CLUSTER "raft::neighbors::ivf_pq::codebook_gen::PER_CLUSTER" - - cpdef cppclass index_params(ann_index_params): - uint32_t n_lists - uint32_t kmeans_n_iters - double kmeans_trainset_fraction - uint32_t pq_bits - uint32_t pq_dim - codebook_gen codebook_kind - bool force_random_rotation - bool conservative_memory_allocation - - cdef cppclass index[IdxT](ann_index): - index(const device_resources& handle, - DistanceType metric, - codebook_gen codebook_kind, - uint32_t n_lists, - uint32_t dim, - uint32_t pq_bits, - uint32_t pq_dim, - bool conservative_memory_allocation) - - IdxT size() - uint32_t dim() - uint32_t pq_dim() - uint32_t pq_len() - uint32_t pq_bits() - DistanceType metric() - uint32_t n_lists() - uint32_t rot_dim() - codebook_gen codebook_kind() - bool conservative_memory_allocation() - - cpdef cppclass search_params(ann_search_params): - uint32_t n_probes - cudaDataType_t lut_dtype - cudaDataType_t internal_distance_dtype - - -cdef extern from "raft_runtime/neighbors/ivf_pq.hpp" \ - namespace "raft::runtime::neighbors::ivf_pq" nogil: - - cdef void build( - const device_resources& handle, - const index_params& params, - device_matrix_view[float, int64_t, row_major] dataset, - index[int64_t]* index) except + - - cdef void build( - const device_resources& handle, - const index_params& params, - device_matrix_view[int8_t, int64_t, row_major] dataset, - index[int64_t]* index) except + - - cdef void build( - const device_resources& handle, - const index_params& params, - device_matrix_view[uint8_t, int64_t, row_major] dataset, - index[int64_t]* index) except + - - cdef void extend( - const device_resources& handle, - device_matrix_view[float, int64_t, row_major] new_vectors, - optional[device_vector_view[int64_t, int64_t]] new_indices, - index[int64_t]* index) except + - - cdef void extend( - const device_resources& handle, - device_matrix_view[int8_t, int64_t, row_major] new_vectors, - optional[device_vector_view[int64_t, int64_t]] new_indices, - index[int64_t]* index) except + - - cdef void extend( - const device_resources& handle, - device_matrix_view[uint8_t, int64_t, row_major] new_vectors, - optional[device_vector_view[int64_t, int64_t]] new_indices, - index[int64_t]* index) except + - - cdef void search( - const device_resources& handle, - const search_params& params, - const index[int64_t]& index, - device_matrix_view[float, int64_t, row_major] queries, - device_matrix_view[int64_t, int64_t, row_major] neighbors, - device_matrix_view[float, int64_t, row_major] distances) except + - - cdef void search( - const device_resources& handle, - const search_params& params, - const index[int64_t]& index, - device_matrix_view[int8_t, int64_t, row_major] queries, - device_matrix_view[int64_t, int64_t, row_major] neighbors, - device_matrix_view[float, int64_t, row_major] distances) except + - - cdef void search( - const device_resources& handle, - const search_params& params, - const index[int64_t]& index, - device_matrix_view[uint8_t, int64_t, row_major] queries, - device_matrix_view[int64_t, int64_t, row_major] neighbors, - device_matrix_view[float, int64_t, row_major] distances) except + - - cdef void serialize(const device_resources& handle, - const string& filename, - const index[int64_t]& index) except + - - cdef void deserialize(const device_resources& handle, - const string& filename, - index[int64_t]* index) except + diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/ivf_pq.pxd b/python/pylibraft/pylibraft/neighbors/ivf_pq/ivf_pq.pxd deleted file mode 100644 index 1b99da1fd7..0000000000 --- a/python/pylibraft/pylibraft/neighbors/ivf_pq/ivf_pq.pxd +++ /dev/null @@ -1,25 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# distutils: language = c++ - -cimport pylibraft.neighbors.ivf_pq.cpp.c_ivf_pq as c_ivf_pq - - -cdef class IndexParams: - cdef c_ivf_pq.index_params params - -cdef class SearchParams: - cdef c_ivf_pq.search_params params diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/ivf_pq.pyx b/python/pylibraft/pylibraft/neighbors/ivf_pq/ivf_pq.pyx deleted file mode 100644 index f467957fd6..0000000000 --- a/python/pylibraft/pylibraft/neighbors/ivf_pq/ivf_pq.pyx +++ /dev/null @@ -1,797 +0,0 @@ -# -# Copyright (c) 2022-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import warnings - -import numpy as np - -from cython.operator cimport dereference as deref -from libc.stdint cimport int32_t, int64_t, uint32_t, uintptr_t -from libcpp cimport bool, nullptr -from libcpp.string cimport string - -from pylibraft.distance.distance_type cimport DistanceType - -from pylibraft.common import ( - DeviceResources, - ai_wrapper, - auto_convert_output, - cai_wrapper, - device_ndarray, -) -from pylibraft.common.cai_wrapper import wrap_array -from pylibraft.common.interruptible import cuda_interruptible - -from pylibraft.common.handle cimport device_resources - -from pylibraft.common.handle import auto_sync_handle -from pylibraft.common.input_validation import is_c_contiguous - -from rmm.librmm.memory_resource cimport device_memory_resource -from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource - -cimport pylibraft.neighbors.ivf_flat.cpp.c_ivf_flat as c_ivf_flat -cimport pylibraft.neighbors.ivf_pq.cpp.c_ivf_pq as c_ivf_pq -from pylibraft.common.optional cimport make_optional, optional - -from pylibraft.neighbors.common import _check_input_array, _get_metric - -from pylibraft.common.cpp.mdspan cimport ( - device_matrix_view, - device_vector_view, - make_device_vector_view, - row_major, -) -from pylibraft.common.mdspan cimport ( - get_dmv_float, - get_dmv_int8, - get_dmv_int64, - get_dmv_uint8, - make_optional_view_int64, -) -from pylibraft.neighbors.common cimport _get_metric_string -from pylibraft.neighbors.ivf_pq.cpp.c_ivf_pq cimport ( - index_params, - search_params, -) - - -cdef _get_codebook_string(c_ivf_pq.codebook_gen codebook): - return {c_ivf_pq.codebook_gen.PER_SUBSPACE: "subspace", - c_ivf_pq.codebook_gen.PER_CLUSTER: "cluster"}[codebook] - - -cdef _map_dtype_np_to_cuda(dtype, supported_dtypes=None): - if supported_dtypes is not None and dtype not in supported_dtypes: - raise TypeError("Type %s is not supported" % str(dtype)) - return {np.float32: c_ivf_pq.cudaDataType_t.CUDA_R_32F, - np.float16: c_ivf_pq.cudaDataType_t.CUDA_R_16F, - np.uint8: c_ivf_pq.cudaDataType_t.CUDA_R_8U}[dtype] - - -cdef _get_dtype_string(dtype): - return str({c_ivf_pq.cudaDataType_t.CUDA_R_32F: np.float32, - c_ivf_pq.cudaDataType_t.CUDA_R_16F: np.float16, - c_ivf_pq.cudaDataType_t.CUDA_R_8U: np.uint8}[dtype]) - - -cdef class IndexParams: - """ - Parameters to build index for IVF-PQ nearest neighbor search - - Parameters - ---------- - n_list : int, default = 1024 - The number of clusters used in the coarse quantizer. - metric : string denoting the metric type, default="sqeuclidean" - Valid values for metric: ["sqeuclidean", "inner_product", - "euclidean"], where - - sqeuclidean is the euclidean distance without the square root - operation, i.e.: distance(a,b) = \\sum_i (a_i - b_i)^2, - - euclidean is the euclidean distance - - inner product distance is defined as - distance(a, b) = \\sum_i a_i * b_i. - kmeans_n_iters : int, default = 20 - The number of iterations searching for kmeans centers during index - building. - kmeans_trainset_fraction : int, default = 0.5 - If kmeans_trainset_fraction is less than 1, then the dataset is - subsampled, and only n_samples * kmeans_trainset_fraction rows - are used for training. - pq_bits : int, default = 8 - The bit length of the vector element after quantization. - pq_dim : int, default = 0 - The dimensionality of a the vector after product quantization. - When zero, an optimal value is selected using a heuristic. Note - pq_dim * pq_bits must be a multiple of 8. Hint: a smaller 'pq_dim' - results in a smaller index size and better search performance, but - lower recall. If 'pq_bits' is 8, 'pq_dim' can be set to any number, - but multiple of 8 are desirable for good performance. If 'pq_bits' - is not 8, 'pq_dim' should be a multiple of 8. For good performance, - it is desirable that 'pq_dim' is a multiple of 32. Ideally, - 'pq_dim' should be also a divisor of the dataset dim. - codebook_kind : string, default = "subspace" - Valid values ["subspace", "cluster"] - force_random_rotation : bool, default = False - Apply a random rotation matrix on the input data and queries even - if `dim % pq_dim == 0`. Note: if `dim` is not multiple of `pq_dim`, - a random rotation is always applied to the input data and queries - to transform the working space from `dim` to `rot_dim`, which may - be slightly larger than the original space and and is a multiple - of `pq_dim` (`rot_dim % pq_dim == 0`). However, this transform is - not necessary when `dim` is multiple of `pq_dim` (`dim == rot_dim`, - hence no need in adding "extra" data columns / features). By - default, if `dim == rot_dim`, the rotation transform is - initialized with the identity matrix. When - `force_random_rotation == True`, a random orthogonal transform - matrix is generated regardless of the values of `dim` and `pq_dim`. - add_data_on_build : bool, default = True - After training the coarse and fine quantizers, we will populate - the index with the dataset if add_data_on_build == True, otherwise - the index is left empty, and the extend method can be used - to add new vectors to the index. - conservative_memory_allocation : bool, default = True - By default, the algorithm allocates more space than necessary for - individual clusters (`list_data`). This allows to amortize the cost - of memory allocation and reduce the number of data copies during - repeated calls to `extend` (extending the database). - To disable this behavior and use as little GPU memory for the - database as possible, set this flat to `True`. - """ - def __init__(self, *, - n_lists=1024, - metric="sqeuclidean", - kmeans_n_iters=20, - kmeans_trainset_fraction=0.5, - pq_bits=8, - pq_dim=0, - codebook_kind="subspace", - force_random_rotation=False, - add_data_on_build=True, - conservative_memory_allocation=False): - self.params.n_lists = n_lists - self.params.metric = _get_metric(metric) - self.params.metric_arg = 0 - self.params.kmeans_n_iters = kmeans_n_iters - self.params.kmeans_trainset_fraction = kmeans_trainset_fraction - self.params.pq_bits = pq_bits - self.params.pq_dim = pq_dim - if codebook_kind == "subspace": - self.params.codebook_kind = c_ivf_pq.codebook_gen.PER_SUBSPACE - elif codebook_kind == "cluster": - self.params.codebook_kind = c_ivf_pq.codebook_gen.PER_CLUSTER - else: - raise ValueError("Incorrect codebook kind %s" % codebook_kind) - self.params.force_random_rotation = force_random_rotation - self.params.add_data_on_build = add_data_on_build - self.params.conservative_memory_allocation = \ - conservative_memory_allocation - - @property - def n_lists(self): - return self.params.n_lists - - @property - def metric(self): - return self.params.metric - - @property - def kmeans_n_iters(self): - return self.params.kmeans_n_iters - - @property - def kmeans_trainset_fraction(self): - return self.params.kmeans_trainset_fraction - - @property - def pq_bits(self): - return self.params.pq_bits - - @property - def pq_dim(self): - return self.params.pq_dim - - @property - def codebook_kind(self): - return self.params.codebook_kind - - @property - def force_random_rotation(self): - return self.params.force_random_rotation - - @property - def add_data_on_build(self): - return self.params.add_data_on_build - - @property - def conservative_memory_allocation(self): - return self.params.conservative_memory_allocation - - -cdef class Index: - # We store a pointer to the index because it dose not have a trivial - # constructor. - cdef c_ivf_pq.index[int64_t] * index - cdef readonly bool trained - - def __cinit__(self, handle=None): - self.trained = False - self.index = NULL - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - # We create a placeholder object. The actual parameter values do - # not matter, it will be replaced with a built index object later. - self.index = new c_ivf_pq.index[int64_t]( - deref(handle_), _get_metric("sqeuclidean"), - c_ivf_pq.codebook_gen.PER_SUBSPACE, - 1, - 4, - 8, - 0, - False) - - def __dealloc__(self): - if self.index is not NULL: - del self.index - - def __repr__(self): - m_str = "metric=" + _get_metric_string(self.index.metric()) - code_str = "codebook=" + _get_codebook_string( - self.index.codebook_kind()) - attr_str = [attr + "=" + str(getattr(self, attr)) - for attr in ["size", "dim", "pq_dim", "pq_bits", - "n_lists", "rot_dim"]] - attr_str = [m_str, code_str] + attr_str - return "Index(type=IVF-PQ, " + (", ".join(attr_str)) + ")" - - @property - def dim(self): - return self.index[0].dim() - - @property - def size(self): - return self.index[0].size() - - @property - def pq_dim(self): - return self.index[0].pq_dim() - - @property - def pq_len(self): - return self.index[0].pq_len() - - @property - def pq_bits(self): - return self.index[0].pq_bits() - - @property - def metric(self): - return self.index[0].metric() - - @property - def n_lists(self): - return self.index[0].n_lists() - - @property - def rot_dim(self): - return self.index[0].rot_dim() - - @property - def codebook_kind(self): - return self.index[0].codebook_kind() - - @property - def conservative_memory_allocation(self): - return self.index[0].conservative_memory_allocation() - - -@auto_sync_handle -@auto_convert_output -def build(IndexParams index_params, dataset, handle=None): - """ - Builds an IVF-PQ index that can be later used for nearest neighbor search. - - The input array can be either CUDA array interface compliant matrix or - array interface compliant matrix in host memory. - - Parameters - ---------- - index_params : IndexParams object - dataset : array interface compliant matrix shape (n_samples, dim) - Supported dtype [float, int8, uint8] - {handle_docstring} - - Returns - ------- - index: ivf_pq.Index - - Examples - -------- - - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import ivf_pq - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> handle = DeviceResources() - >>> index_params = ivf_pq.IndexParams( - ... n_lists=1024, - ... metric="sqeuclidean", - ... pq_dim=10) - >>> index = ivf_pq.build(index_params, dataset, handle=handle) - >>> # Search using the built index - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> k = 10 - >>> distances, neighbors = ivf_pq.search(ivf_pq.SearchParams(), index, - ... queries, k, handle=handle) - >>> distances = cp.asarray(distances) - >>> neighbors = cp.asarray(neighbors) - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - """ - dataset_cai = wrap_array(dataset) - dataset_dt = dataset_cai.dtype - _check_input_array(dataset_cai, [np.dtype('float32'), np.dtype('byte'), - np.dtype('ubyte')]) - - cdef int64_t n_rows = dataset_cai.shape[0] - cdef uint32_t dim = dataset_cai.shape[1] - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - idx = Index() - - if dataset_dt == np.float32: - with cuda_interruptible(): - c_ivf_pq.build(deref(handle_), - index_params.params, - get_dmv_float(dataset_cai, check_shape=True), - idx.index) - idx.trained = True - elif dataset_dt == np.byte: - with cuda_interruptible(): - c_ivf_pq.build(deref(handle_), - index_params.params, - get_dmv_int8(dataset_cai, check_shape=True), - idx.index) - idx.trained = True - elif dataset_dt == np.ubyte: - with cuda_interruptible(): - c_ivf_pq.build(deref(handle_), - index_params.params, - get_dmv_uint8(dataset_cai, check_shape=True), - idx.index) - idx.trained = True - else: - raise TypeError("dtype %s not supported" % dataset_dt) - - return idx - - -@auto_sync_handle -@auto_convert_output -def extend(Index index, new_vectors, new_indices, handle=None): - """ - Extend an existing index with new vectors. - - The input array can be either CUDA array interface compliant matrix or - array interface compliant matrix in host memory. - - Parameters - ---------- - index : ivf_pq.Index - Trained ivf_pq object. - new_vectors : array interface compliant matrix shape (n_samples, dim) - Supported dtype [float, int8, uint8] - new_indices : array interface compliant vector shape (n_samples) - Supported dtype [int64] - {handle_docstring} - - Returns - ------- - index: ivf_pq.Index - - Examples - -------- - - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import ivf_pq - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> handle = DeviceResources() - >>> index = ivf_pq.build(ivf_pq.IndexParams(), dataset, handle=handle) - >>> n_rows = 100 - >>> more_data = cp.random.random_sample((n_rows, n_features), - ... dtype=cp.float32) - >>> indices = index.size + cp.arange(n_rows, dtype=cp.int64) - >>> index = ivf_pq.extend(index, more_data, indices) - >>> # Search using the built index - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> k = 10 - >>> distances, neighbors = ivf_pq.search(ivf_pq.SearchParams(), - ... index, queries, - ... k, handle=handle) - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - >>> distances = cp.asarray(distances) - >>> neighbors = cp.asarray(neighbors) - """ - if not index.trained: - raise ValueError("Index need to be built before calling extend.") - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - vecs_cai = wrap_array(new_vectors) - vecs_dt = vecs_cai.dtype - cdef optional[device_vector_view[int64_t, int64_t]] new_indices_opt - cdef int64_t n_rows = vecs_cai.shape[0] - cdef uint32_t dim = vecs_cai.shape[1] - - _check_input_array(vecs_cai, [np.dtype('float32'), np.dtype('byte'), - np.dtype('ubyte')], - exp_cols=index.dim) - - idx_cai = wrap_array(new_indices) - _check_input_array(idx_cai, [np.dtype('int64')], exp_rows=n_rows) - if len(idx_cai.shape)!=1: - raise ValueError("Indices array is expected to be 1D") - - if index.index.size() > 0: - new_indices_opt = make_device_vector_view( - idx_cai.data, - idx_cai.shape[0]) - - if vecs_dt == np.float32: - with cuda_interruptible(): - c_ivf_pq.extend(deref(handle_), - get_dmv_float(vecs_cai, check_shape=True), - new_indices_opt, - index.index) - elif vecs_dt == np.int8: - with cuda_interruptible(): - c_ivf_pq.extend(deref(handle_), - get_dmv_int8(vecs_cai, check_shape=True), - new_indices_opt, - index.index) - elif vecs_dt == np.uint8: - with cuda_interruptible(): - c_ivf_pq.extend(deref(handle_), - get_dmv_uint8(vecs_cai, check_shape=True), - new_indices_opt, - index.index) - else: - raise TypeError("query dtype %s not supported" % vecs_dt) - - return index - - -cdef class SearchParams: - """ - IVF-PQ search parameters - - Parameters - ---------- - n_probes: int, default = 1024 - The number of course clusters to select for the fine search. - lut_dtype: default = np.float32 - Data type of look up table to be created dynamically at search - time. The use of low-precision types reduces the amount of shared - memory required at search time, so fast shared memory kernels can - be used even for datasets with large dimansionality. Note that - the recall is slightly degraded when low-precision type is - selected. Possible values [np.float32, np.float16, np.uint8] - internal_distance_dtype: default = np.float32 - Storage data type for distance/similarity computation. - Possible values [np.float32, np.float16] - """ - def __init__(self, *, n_probes=20, - lut_dtype=np.float32, - internal_distance_dtype=np.float32): - self.params.n_probes = n_probes - self.params.lut_dtype = _map_dtype_np_to_cuda(lut_dtype) - self.params.internal_distance_dtype = \ - _map_dtype_np_to_cuda(internal_distance_dtype) - # TODO(tfeher): enable if #926 adds this - # self.params.shmem_carveout = self.shmem_carveout - - def __repr__(self): - lut_str = "lut_dtype=" + _get_dtype_string(self.params.lut_dtype) - idt_str = "internal_distance_dtype=" + \ - _get_dtype_string(self.params.internal_distance_dtype) - attr_str = [attr + "=" + str(getattr(self, attr)) - for attr in ["n_probes"]] - # TODO (tfeher) add "shmem_carveout" - attr_str = attr_str + [lut_str, idt_str] - return "SearchParams(type=IVF-PQ, " + (", ".join(attr_str)) + ")" - - @property - def n_probes(self): - return self.params.n_probes - - @property - def lut_dtype(self): - return self.params.lut_dtype - - @property - def internal_distance_dtype(self): - return self.params.internal_distance_dtype - - -@auto_sync_handle -@auto_convert_output -def search(SearchParams search_params, - Index index, - queries, - k, - neighbors=None, - distances=None, - DeviceMemoryResource memory_resource=None, - handle=None): - """ - Find the k nearest neighbors for each query. - - Parameters - ---------- - search_params : SearchParams - index : Index - Trained IVF-PQ index. - queries : CUDA array interface compliant matrix shape (n_samples, dim) - Supported dtype [float, int8, uint8] - k : int - The number of neighbors. - neighbors : Optional CUDA array interface compliant matrix shape - (n_queries, k), dtype int64_t. If supplied, neighbor - indices will be written here in-place. (default None) - distances : Optional CUDA array interface compliant matrix shape - (n_queries, k) If supplied, the distances to the - neighbors will be written here in-place. (default None) - memory_resource : RMM DeviceMemoryResource object, optional - This can be used to explicitly manage the temporary memory - allocation during search. Passing a pooling allocator can reduce - memory allocation overhead. If not specified, then the memory - resource from the raft handle is used. - {handle_docstring} - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import ivf_pq - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Build index - >>> handle = DeviceResources() - >>> index = ivf_pq.build(ivf_pq.IndexParams(), dataset, handle=handle) - >>> # Search using the built index - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> k = 10 - >>> search_params = ivf_pq.SearchParams( - ... n_probes=20, - ... lut_dtype=cp.float16, - ... internal_distance_dtype=cp.float32 - ... ) - >>> # Using a pooling allocator reduces overhead of temporary array - >>> # creation during search. This is useful if multiple searches - >>> # are performad with same query size. - >>> import rmm - >>> mr = rmm.mr.PoolMemoryResource( - ... rmm.mr.CudaMemoryResource(), - ... initial_pool_size=2**29, - ... maximum_pool_size=2**31 - ... ) - >>> distances, neighbors = ivf_pq.search(search_params, index, queries, - ... k, memory_resource=mr, - ... handle=handle) - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - >>> neighbors = cp.asarray(neighbors) - >>> distances = cp.asarray(distances) - """ - - if not index.trained: - raise ValueError("Index need to be built before calling search.") - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - queries_cai = cai_wrapper(queries) - queries_dt = queries_cai.dtype - cdef uint32_t n_queries = queries_cai.shape[0] - - _check_input_array(queries_cai, [np.dtype('float32'), np.dtype('byte'), - np.dtype('ubyte')], - exp_cols=index.dim) - - if neighbors is None: - neighbors = device_ndarray.empty((n_queries, k), dtype='int64') - - neighbors_cai = cai_wrapper(neighbors) - _check_input_array(neighbors_cai, [np.dtype('int64')], - exp_rows=n_queries, exp_cols=k) - - if distances is None: - distances = device_ndarray.empty((n_queries, k), dtype='float32') - - distances_cai = cai_wrapper(distances) - _check_input_array(distances_cai, [np.dtype('float32')], - exp_rows=n_queries, exp_cols=k) - - cdef c_ivf_pq.search_params params = search_params.params - - cdef uintptr_t neighbors_ptr = neighbors_cai.data - cdef uintptr_t distances_ptr = distances_cai.data - # TODO(tfeher) pass mr_ptr arg - cdef device_memory_resource* mr_ptr = nullptr - if memory_resource is not None: - mr_ptr = memory_resource.get_mr() - - if queries_dt == np.float32: - with cuda_interruptible(): - c_ivf_pq.search(deref(handle_), - params, - deref(index.index), - get_dmv_float(queries_cai, check_shape=True), - get_dmv_int64(neighbors_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True)) - elif queries_dt == np.byte: - with cuda_interruptible(): - c_ivf_pq.search(deref(handle_), - params, - deref(index.index), - get_dmv_int8(queries_cai, check_shape=True), - get_dmv_int64(neighbors_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True)) - elif queries_dt == np.ubyte: - with cuda_interruptible(): - c_ivf_pq.search(deref(handle_), - params, - deref(index.index), - get_dmv_uint8(queries_cai, check_shape=True), - get_dmv_int64(neighbors_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True)) - else: - raise ValueError("query dtype %s not supported" % queries_dt) - - return (distances, neighbors) - - -@auto_sync_handle -def save(filename, Index index, handle=None): - """ - Saves the index to a file. - - Saving / loading the index is experimental. The serialization format is - subject to change. - - Parameters - ---------- - filename : string - Name of the file. - index : Index - Trained IVF-PQ index. - {handle_docstring} - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import ivf_pq - >>> n_samples = 50000 - >>> n_features = 50 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Build index - >>> handle = DeviceResources() - >>> index = ivf_pq.build(ivf_pq.IndexParams(), dataset, handle=handle) - >>> ivf_pq.save("my_index.bin", index, handle=handle) - """ - if not index.trained: - raise ValueError("Index need to be built before saving it.") - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - cdef string c_filename = filename.encode('utf-8') - - c_ivf_pq.serialize(deref(handle_), c_filename, deref(index.index)) - - -@auto_sync_handle -def load(filename, handle=None): - """ - Loads index from a file. - - Saving / loading the index is experimental. The serialization format is - subject to change, therefore loading an index saved with a previous - version of raft is not guaranteed to work. - - Parameters - ---------- - filename : string - Name of the file. - {handle_docstring} - - Returns - ------- - index : Index - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import ivf_pq - >>> n_samples = 50000 - >>> n_features = 50 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Build and save index - >>> handle = DeviceResources() - >>> index = ivf_pq.build(ivf_pq.IndexParams(), dataset, handle=handle) - >>> ivf_pq.save("my_index.bin", index, handle=handle) - >>> del index - >>> n_queries = 100 - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> handle = DeviceResources() - >>> index = ivf_pq.load("my_index.bin", handle=handle) - >>> distances, neighbors = ivf_pq.search(ivf_pq.SearchParams(), index, - ... queries, k=10, handle=handle) - """ - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - cdef string c_filename = filename.encode('utf-8') - index = Index() - - c_ivf_pq.deserialize(deref(handle_), c_filename, index.index) - index.trained = True - - return index diff --git a/python/pylibraft/pylibraft/neighbors/rbc.pyx b/python/pylibraft/pylibraft/neighbors/rbc.pyx deleted file mode 100644 index a703dc1745..0000000000 --- a/python/pylibraft/pylibraft/neighbors/rbc.pyx +++ /dev/null @@ -1,241 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -from cython.operator cimport dereference as deref -from libcpp cimport bool, nullptr -from libcpp.vector cimport vector - -from pylibraft.common import ( - DeviceResources, - auto_convert_output, - cai_wrapper, - device_ndarray, -) - -from libc.stdint cimport int64_t, uintptr_t - -from pylibraft.common.cpp.optional cimport optional -from pylibraft.common.handle cimport device_resources -from pylibraft.common.mdspan cimport get_dmv_bool, get_dmv_float, get_dmv_int64 - -from pylibraft.common.handle import auto_sync_handle -from pylibraft.common.interruptible import cuda_interruptible -from pylibraft.neighbors.common import _check_input_array, _get_metric - -from pylibraft.common.cpp.mdspan cimport ( - device_matrix_view, - device_vector_view, - host_matrix_view, - make_device_matrix_view, - make_device_vector_view, - make_host_matrix_view, - row_major, -) -from pylibraft.neighbors.cpp.rbc cimport ( - BallCoverIndex as c_BallCoverIndex, - build_rbc_index as c_build_rbc_index, - eps_neighbors_rbc as c_eps_neighbors_rbc, - eps_neighbors_rbc_pass1 as c_eps_neighbors_rbc_pass1, - eps_neighbors_rbc_pass2 as c_eps_neighbors_rbc_pass2, -) - - -cdef class RbcIndex: - cdef readonly bool trained - cdef str data_type - - def __cinit__(self): - self.trained = False - self.data_type = None - - -cdef class RbcIndexFloat(RbcIndex): - cdef c_BallCoverIndex[int64_t, float, int64_t, int64_t]* index - - def __cinit__(self, dataset, handle): - cdef device_resources* handle_ = \ - handle.getHandle() - self.index = new c_BallCoverIndex[int64_t, float, int64_t, int64_t]( - deref(handle_), - get_dmv_float(dataset, check_shape=True), - _get_metric("euclidean")) - - -@auto_sync_handle -@auto_convert_output -def build_rbc_index(dataset, handle=None): - """ - Builds a random ball cover index from dataset using the L2-norm. - - Parameters - ---------- - dataset : array interface compliant matrix, row-major layout, - shape (n_samples, dim). Supported dtype [float] - {handle_docstring} - - Returns - ------- - index : Index - - Examples - -------- - see 'eps_neighbors_sparse' - - """ - if handle is None: - handle = DeviceResources() - - dataset_cai = cai_wrapper(dataset) - - # we require c-contiguous (rowmajor) inputs here - _check_input_array(dataset_cai, [np.dtype("float32")]) - - cdef device_resources* handle_ = \ - handle.getHandle() - - cdef RbcIndexFloat rbc_index_float - - if dataset_cai.dtype == np.float32: - rbc_index_float = RbcIndexFloat(dataset=dataset_cai, handle=handle) - rbc_index_float.data_type = "float32" - with cuda_interruptible(): - c_build_rbc_index( - deref(handle_), - deref(rbc_index_float.index)) - rbc_index_float.trained = True - return rbc_index_float - else: - raise TypeError("dtype %s not supported" % dataset_cai.dtype) - - -@auto_sync_handle -@auto_convert_output -def eps_neighbors(RbcIndex rbc_index, queries, eps, handle=None): - """ - Perform an epsilon neighborhood search with random ball cover (rbc) - using the L2-norm. - - Parameters - ---------- - rbc_index : RbcIndex created via 'build_rbc_index'. - Supported dtype [float] - queries : array interface compliant matrix, row-major layout, - shape (n_queries, dim) Supported dtype [float] - eps : threshold - {handle_docstring} - - Returns - ------- - adj_ia: array interface compliant object containing row indices for - adj_ja - - adj_ja: array interface compliant object containing adjacency mask - column indices - - vd: array interface compliant object containing row sums of adj - shape (n_queries + 1). vd[n_queries] contains the total sum - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors.rbc import eps_neighbors - >>> from pylibraft.neighbors.rbc import build_rbc_index - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> eps = 0.1 - >>> handle = DeviceResources() - >>> rbc_index = build_rbc_index(dataset) - >>> adj_ia, adj_ja, vd = eps_neighbors(rbc_index, queries, eps) - >>> adj_ia = cp.asarray(adj_ia) - >>> adj_ja = cp.asarray(adj_ja) - >>> vd = cp.asarray(vd) - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - """ - if not rbc_index.trained: - raise ValueError("Index need to be built before calling extend.") - - if handle is None: - handle = DeviceResources() - - queries_cai = cai_wrapper(queries) - - _check_input_array(queries_cai, [np.dtype(rbc_index.data_type)]) - - n_queries = queries_cai.shape[0] - - adj_ia = device_ndarray.empty((n_queries + 1, ), dtype='int64') - vd = device_ndarray.empty((n_queries + 1, ), dtype='int64') - adj_ia_cai = cai_wrapper(adj_ia) - vd_cai = cai_wrapper(vd) - - cdef device_resources* handle_ = \ - handle.getHandle() - - vd_vector_view = make_device_vector_view( - vd_cai.data, vd_cai.shape[0]) - adj_ia_vector_view = make_device_vector_view( - adj_ia_cai.data, adj_ia_cai.shape[0]) - - cdef RbcIndexFloat rbc_index_float - - if queries_cai.dtype == np.float32: - rbc_index_float = rbc_index - with cuda_interruptible(): - c_eps_neighbors_rbc_pass1( - deref(handle_), - deref(rbc_index_float.index), - get_dmv_float(queries_cai, check_shape=True), - adj_ia_vector_view, - vd_vector_view, - eps) - else: - raise TypeError("dtype %s not supported" % queries_cai.dtype) - - handle.sync() - n_nnz = adj_ia.copy_to_host()[n_queries] - adj_ja = device_ndarray.empty((n_nnz, ), dtype='int64') - adj_ja_cai = cai_wrapper(adj_ja) - adj_ja_vector_view = make_device_vector_view( - adj_ja_cai.data, adj_ja_cai.shape[0]) - - if queries_cai.dtype == np.float32: - with cuda_interruptible(): - c_eps_neighbors_rbc_pass2( - deref(handle_), - deref(rbc_index_float.index), - get_dmv_float(queries_cai, check_shape=True), - adj_ia_vector_view, - adj_ja_vector_view, - vd_vector_view, - eps) - else: - raise TypeError("dtype %s not supported" % queries_cai.dtype) - - return (adj_ia, adj_ja, vd) diff --git a/python/pylibraft/pylibraft/neighbors/refine.pyx b/python/pylibraft/pylibraft/neighbors/refine.pyx deleted file mode 100644 index a9bf811c9f..0000000000 --- a/python/pylibraft/pylibraft/neighbors/refine.pyx +++ /dev/null @@ -1,375 +0,0 @@ -# -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -from cython.operator cimport dereference as deref -from libc.stdint cimport int8_t, int64_t, uint8_t, uintptr_t -from libcpp cimport bool, nullptr - -from pylibraft.distance.distance_type cimport DistanceType - -from pylibraft.common import ( - DeviceResources, - auto_convert_output, - cai_wrapper, - device_ndarray, -) - -from pylibraft.common.handle cimport device_resources - -from pylibraft.common.handle import auto_sync_handle -from pylibraft.common.input_validation import is_c_contiguous -from pylibraft.common.interruptible import cuda_interruptible - -from pylibraft.distance.distance_type cimport DistanceType - -import pylibraft.neighbors.ivf_pq as ivf_pq -from pylibraft.neighbors.common import _get_metric - -cimport pylibraft.neighbors.ivf_pq.cpp.c_ivf_pq as c_ivf_pq -from pylibraft.common.cpp.mdspan cimport ( - device_matrix_view, - host_matrix_view, - make_host_matrix_view, - row_major, -) -from pylibraft.common.mdspan cimport ( - get_dmv_float, - get_dmv_int8, - get_dmv_int64, - get_dmv_uint8, -) -from pylibraft.neighbors.common cimport _get_metric_string -from pylibraft.neighbors.ivf_pq.cpp.c_ivf_pq cimport ( - index_params, - search_params, -) - - -# We omit the const qualifiers in the interface for refine, because cython -# has an issue parsing it (https://github.com/cython/cython/issues/4180). -cdef extern from "raft_runtime/neighbors/refine.hpp" \ - namespace "raft::runtime::neighbors" nogil: - - cdef void c_refine "raft::runtime::neighbors::refine" ( - const device_resources& handle, - device_matrix_view[float, int64_t, row_major] dataset, - device_matrix_view[float, int64_t, row_major] queries, - device_matrix_view[int64_t, int64_t, row_major] candidates, - device_matrix_view[int64_t, int64_t, row_major] indices, - device_matrix_view[float, int64_t, row_major] distances, - DistanceType metric) except + - - cdef void c_refine "raft::runtime::neighbors::refine" ( - const device_resources& handle, - device_matrix_view[uint8_t, int64_t, row_major] dataset, - device_matrix_view[uint8_t, int64_t, row_major] queries, - device_matrix_view[int64_t, int64_t, row_major] candidates, - device_matrix_view[int64_t, int64_t, row_major] indices, - device_matrix_view[float, int64_t, row_major] distances, - DistanceType metric) except + - - cdef void c_refine "raft::runtime::neighbors::refine" ( - const device_resources& handle, - device_matrix_view[int8_t, int64_t, row_major] dataset, - device_matrix_view[int8_t, int64_t, row_major] queries, - device_matrix_view[int64_t, int64_t, row_major] candidates, - device_matrix_view[int64_t, int64_t, row_major] indices, - device_matrix_view[float, int64_t, row_major] distances, - DistanceType metric) except + - - cdef void c_refine "raft::runtime::neighbors::refine" ( - const device_resources& handle, - host_matrix_view[float, int64_t, row_major] dataset, - host_matrix_view[float, int64_t, row_major] queries, - host_matrix_view[int64_t, int64_t, row_major] candidates, - host_matrix_view[int64_t, int64_t, row_major] indices, - host_matrix_view[float, int64_t, row_major] distances, - DistanceType metric) except + - - cdef void c_refine "raft::runtime::neighbors::refine" ( - const device_resources& handle, - host_matrix_view[uint8_t, int64_t, row_major] dataset, - host_matrix_view[uint8_t, int64_t, row_major] queries, - host_matrix_view[int64_t, int64_t, row_major] candidates, - host_matrix_view[int64_t, int64_t, row_major] indices, - host_matrix_view[float, int64_t, row_major] distances, - DistanceType metric) except + - - cdef void c_refine "raft::runtime::neighbors::refine" ( - const device_resources& handle, - host_matrix_view[int8_t, int64_t, row_major] dataset, - host_matrix_view[int8_t, int64_t, row_major] queries, - host_matrix_view[int64_t, int64_t, row_major] candidates, - host_matrix_view[int64_t, int64_t, row_major] indices, - host_matrix_view[float, int64_t, row_major] distances, - DistanceType metric) except + - - -def _get_array_params(array_interface, check_dtype=None): - dtype = np.dtype(array_interface["typestr"]) - if check_dtype is None and dtype != check_dtype: - raise TypeError("dtype %s not supported" % dtype) - shape = array_interface["shape"] - if len(shape) != 2: - raise ValueError("Expected a 2D array, got %d D" % len(shape)) - data = array_interface["data"][0] - return (shape, dtype, data) - - -cdef host_matrix_view[float, int64_t, row_major] \ - get_host_matrix_view_float(array) except *: - shape, dtype, data = _get_array_params( - array.__array_interface__, check_dtype=np.float32) - return make_host_matrix_view[float, int64_t, row_major]( - data, shape[0], shape[1]) - - -cdef host_matrix_view[int64_t, int64_t, row_major] \ - get_host_matrix_view_int64_t(array) except *: - shape, dtype, data = _get_array_params( - array.__array_interface__, check_dtype=np.int64) - return make_host_matrix_view[int64_t, int64_t, row_major]( - data, shape[0], shape[1]) - - -cdef host_matrix_view[uint8_t, int64_t, row_major] \ - get_host_matrix_view_uint8(array) except *: - shape, dtype, data = _get_array_params( - array.__array_interface__, check_dtype=np.uint8) - return make_host_matrix_view[uint8_t, int64_t, row_major]( - data, shape[0], shape[1]) - - -cdef host_matrix_view[int8_t, int64_t, row_major] \ - get_host_matrix_view_int8(array) except *: - shape, dtype, data = _get_array_params( - array.__array_interface__, check_dtype=np.int8) - return make_host_matrix_view[int8_t, int64_t, row_major]( - data, shape[0], shape[1]) - - -@auto_sync_handle -@auto_convert_output -def refine(dataset, queries, candidates, k=None, indices=None, distances=None, - metric="sqeuclidean", handle=None): - """ - Refine nearest neighbor search. - - Refinement is an operation that follows an approximate NN search. The - approximate search has already selected n_candidates neighbor candidates - for each query. We narrow it down to k neighbors. For each query, we - calculate the exact distance between the query and its n_candidates - neighbor candidate, and select the k nearest ones. - - Input arrays can be either CUDA array interface compliant matrices or - array interface compliant matrices in host memory. All array must be in - the same memory space. - - Parameters - ---------- - index_params : IndexParams object - dataset : array interface compliant matrix, shape (n_samples, dim) - Supported dtype [float, int8, uint8] - queries : array interface compliant matrix, shape (n_queries, dim) - Supported dtype [float, int8, uint8] - candidates : array interface compliant matrix, shape (n_queries, k0) - Supported dtype int64 - k : int - Number of neighbors to search (k <= k0). Optional if indices or - distances arrays are given (in which case their second dimension - is k). - indices : Optional array interface compliant matrix shape \ - (n_queries, k). - If supplied, neighbor indices will be written here in-place. - (default None). Supported dtype int64. - distances : Optional array interface compliant matrix shape \ - (n_queries, k). - If supplied, neighbor indices will be written here in-place. - (default None) Supported dtype float. - {handle_docstring} - - Returns - ------- - index: ivf_pq.Index - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import ivf_pq, refine - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> handle = DeviceResources() - >>> index_params = ivf_pq.IndexParams(n_lists=1024, - ... metric="sqeuclidean", - ... pq_dim=10) - >>> index = ivf_pq.build(index_params, dataset, handle=handle) - >>> # Search using the built index - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> k = 40 - >>> _, candidates = ivf_pq.search(ivf_pq.SearchParams(), index, - ... queries, k, handle=handle) - >>> k = 10 - >>> distances, neighbors = refine(dataset, queries, candidates, k, - ... handle=handle) - >>> distances = cp.asarray(distances) - >>> neighbors = cp.asarray(neighbors) - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - """ - - if handle is None: - handle = DeviceResources() - - if hasattr(dataset, "__cuda_array_interface__"): - return _refine_device(dataset, queries, candidates, k, indices, - distances, metric, handle) - else: - return _refine_host(dataset, queries, candidates, k, indices, - distances, metric, handle) - - -def _refine_device(dataset, queries, candidates, k, indices, distances, - metric, handle): - cdef device_resources* handle_ = \ - handle.getHandle() - - if k is None: - if indices is not None: - k = cai_wrapper(indices).shape[1] - elif distances is not None: - k = cai_wrapper(distances).shape[1] - else: - raise ValueError("Argument k must be specified if both indices " - "and distances arg is None") - - queries_cai = cai_wrapper(queries) - dataset_cai = cai_wrapper(dataset) - candidates_cai = cai_wrapper(candidates) - n_queries = cai_wrapper(queries).shape[0] - - if indices is None: - indices = device_ndarray.empty((n_queries, k), dtype='int64') - - if distances is None: - distances = device_ndarray.empty((n_queries, k), dtype='float32') - - indices_cai = cai_wrapper(indices) - distances_cai = cai_wrapper(distances) - - cdef DistanceType c_metric = _get_metric(metric) - - if dataset_cai.dtype == np.float32: - with cuda_interruptible(): - c_refine(deref(handle_), - get_dmv_float(dataset_cai, check_shape=True), - get_dmv_float(queries_cai, check_shape=True), - get_dmv_int64(candidates_cai, check_shape=True), - get_dmv_int64(indices_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True), - c_metric) - elif dataset_cai.dtype == np.int8: - with cuda_interruptible(): - c_refine(deref(handle_), - get_dmv_int8(dataset_cai, check_shape=True), - get_dmv_int8(queries_cai, check_shape=True), - get_dmv_int64(candidates_cai, check_shape=True), - get_dmv_int64(indices_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True), - c_metric) - elif dataset_cai.dtype == np.uint8: - with cuda_interruptible(): - c_refine(deref(handle_), - get_dmv_uint8(dataset_cai, check_shape=True), - get_dmv_uint8(queries_cai, check_shape=True), - get_dmv_int64(candidates_cai, check_shape=True), - get_dmv_int64(indices_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True), - c_metric) - else: - raise TypeError("dtype %s not supported" % dataset_cai.dtype) - - return (distances, indices) - - -def _refine_host(dataset, queries, candidates, k, indices, distances, - metric, handle): - cdef device_resources* handle_ = \ - handle.getHandle() - - if k is None: - if indices is not None: - k = indices.__array_interface__["shape"][1] - elif distances is not None: - k = distances.__array_interface__["shape"][1] - else: - raise ValueError("Argument k must be specified if both indices " - "and distances arg is None") - - n_queries = queries.__array_interface__["shape"][0] - - if indices is None: - indices = np.empty((n_queries, k), dtype='int64') - - if distances is None: - distances = np.empty((n_queries, k), dtype='float32') - - cdef DistanceType c_metric = _get_metric(metric) - - dtype = np.dtype(dataset.__array_interface__["typestr"]) - - if dtype == np.float32: - with cuda_interruptible(): - c_refine(deref(handle_), - get_host_matrix_view_float(dataset), - get_host_matrix_view_float(queries), - get_host_matrix_view_int64_t(candidates), - get_host_matrix_view_int64_t(indices), - get_host_matrix_view_float(distances), - c_metric) - elif dtype == np.int8: - with cuda_interruptible(): - c_refine(deref(handle_), - get_host_matrix_view_int8(dataset), - get_host_matrix_view_int8(queries), - get_host_matrix_view_int64_t(candidates), - get_host_matrix_view_int64_t(indices), - get_host_matrix_view_float(distances), - c_metric) - elif dtype == np.uint8: - with cuda_interruptible(): - c_refine(deref(handle_), - get_host_matrix_view_uint8(dataset), - get_host_matrix_view_uint8(queries), - get_host_matrix_view_int64_t(candidates), - get_host_matrix_view_int64_t(indices), - get_host_matrix_view_float(distances), - c_metric) - else: - raise TypeError("dtype %s not supported" % dtype) - - return (distances, indices) diff --git a/python/pylibraft/pylibraft/test/ann_utils.py b/python/pylibraft/pylibraft/test/ann_utils.py deleted file mode 100644 index 60db7f3273..0000000000 --- a/python/pylibraft/pylibraft/test/ann_utils.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# h ttp://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import numpy as np - - -def generate_data(shape, dtype): - if dtype == np.byte: - x = np.random.randint(-127, 128, size=shape, dtype=np.byte) - elif dtype == np.ubyte: - x = np.random.randint(0, 255, size=shape, dtype=np.ubyte) - else: - x = np.random.random_sample(shape).astype(dtype) - - return x - - -def calc_recall(ann_idx, true_nn_idx): - assert ann_idx.shape == true_nn_idx.shape - n = 0 - for i in range(ann_idx.shape[0]): - n += np.intersect1d(ann_idx[i, :], true_nn_idx[i, :]).size - recall = n / ann_idx.size - return recall diff --git a/python/pylibraft/pylibraft/test/test_brute_force.py b/python/pylibraft/pylibraft/test/test_brute_force.py deleted file mode 100644 index 42095c3b9f..0000000000 --- a/python/pylibraft/pylibraft/test/test_brute_force.py +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest -from scipy.spatial.distance import cdist - -from pylibraft.common import DeviceResources, Stream, device_ndarray -from pylibraft.neighbors.brute_force import knn - - -@pytest.mark.parametrize("n_index_rows", [32, 100]) -@pytest.mark.parametrize("n_query_rows", [32, 100]) -@pytest.mark.parametrize("n_cols", [40, 100]) -@pytest.mark.parametrize("k", [1, 5, 32]) -@pytest.mark.parametrize( - "metric", - [ - "euclidean", - "cityblock", - "chebyshev", - "canberra", - "correlation", - "russellrao", - "cosine", - "sqeuclidean", - # "inner_product", - ], -) -@pytest.mark.parametrize("inplace", [True, False]) -@pytest.mark.parametrize("dtype", [np.float32]) -def test_knn(n_index_rows, n_query_rows, n_cols, k, inplace, metric, dtype): - index = np.random.random_sample((n_index_rows, n_cols)).astype(dtype) - queries = np.random.random_sample((n_query_rows, n_cols)).astype(dtype) - - # RussellRao expects boolean arrays - if metric == "russellrao": - index[index < 0.5] = 0.0 - index[index >= 0.5] = 1.0 - queries[queries < 0.5] = 0.0 - queries[queries >= 0.5] = 1.0 - - indices = np.zeros((n_query_rows, k), dtype="int64") - distances = np.zeros((n_query_rows, k), dtype=dtype) - - index_device = device_ndarray(index) - - queries_device = device_ndarray(queries) - indices_device = device_ndarray(indices) - distances_device = device_ndarray(distances) - - s2 = Stream() - handle = DeviceResources(stream=s2) - ret_distances, ret_indices = knn( - index_device, - queries_device, - k, - indices=indices_device, - distances=distances_device, - metric=metric, - handle=handle, - ) - handle.sync() - - pw_dists = cdist(queries, index, metric=metric) - - distances_device = ret_distances if not inplace else distances_device - - actual_distances = distances_device.copy_to_host() - - actual_distances[actual_distances <= 1e-5] = 0.0 - argsort = np.argsort(pw_dists, axis=1) - - for i in range(pw_dists.shape[0]): - expected_indices = argsort[i] - gpu_dists = actual_distances[i] - - cpu_ordered = pw_dists[i, expected_indices] - np.testing.assert_allclose( - cpu_ordered[:k], gpu_dists, atol=1e-3, rtol=1e-3 - ) - - -def test_knn_check_col_major_inputs(): - # make sure that we get an exception if passed col-major inputs, - # instead of returning incorrect results - cp = pytest.importorskip("cupy") - n_index_rows, n_query_rows, n_cols = 128, 16, 32 - index = cp.random.random_sample((n_index_rows, n_cols), dtype="float32") - queries = cp.random.random_sample((n_query_rows, n_cols), dtype="float32") - - with pytest.raises(ValueError): - knn(cp.asarray(index, order="F"), queries, k=4) - - with pytest.raises(ValueError): - knn(index, cp.asarray(queries, order="F"), k=4) - - # shouldn't throw an exception with c-contiguous inputs - knn(index, queries, k=4) diff --git a/python/pylibraft/pylibraft/test/test_cagra.py b/python/pylibraft/pylibraft/test/test_cagra.py deleted file mode 100644 index ef8e54917a..0000000000 --- a/python/pylibraft/pylibraft/test/test_cagra.py +++ /dev/null @@ -1,292 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# h ttp://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest -from sklearn.neighbors import NearestNeighbors -from sklearn.preprocessing import normalize - -from pylibraft.common import device_ndarray -from pylibraft.neighbors import cagra -from pylibraft.test.ann_utils import calc_recall, generate_data - - -def run_cagra_build_search_test( - n_rows=10000, - n_cols=10, - n_queries=100, - k=10, - dtype=np.float32, - metric="sqeuclidean", - intermediate_graph_degree=128, - graph_degree=64, - build_algo="ivf_pq", - array_type="device", - compare=True, - inplace=True, - add_data_on_build=True, - search_params={}, -): - dataset = generate_data((n_rows, n_cols), dtype) - if metric == "inner_product": - dataset = normalize(dataset, norm="l2", axis=1) - dataset_device = device_ndarray(dataset) - - build_params = cagra.IndexParams( - metric=metric, - intermediate_graph_degree=intermediate_graph_degree, - graph_degree=graph_degree, - build_algo=build_algo, - ) - - if array_type == "device": - index = cagra.build(build_params, dataset_device) - else: - index = cagra.build(build_params, dataset) - - assert index.trained - - if not add_data_on_build: - dataset_1 = dataset[: n_rows // 2, :] - dataset_2 = dataset[n_rows // 2 :, :] - indices_1 = np.arange(n_rows // 2, dtype=np.uint32) - indices_2 = np.arange(n_rows // 2, n_rows, dtype=np.uint32) - if array_type == "device": - dataset_1_device = device_ndarray(dataset_1) - dataset_2_device = device_ndarray(dataset_2) - indices_1_device = device_ndarray(indices_1) - indices_2_device = device_ndarray(indices_2) - index = cagra.extend(index, dataset_1_device, indices_1_device) - index = cagra.extend(index, dataset_2_device, indices_2_device) - else: - index = cagra.extend(index, dataset_1, indices_1) - index = cagra.extend(index, dataset_2, indices_2) - - queries = generate_data((n_queries, n_cols), dtype) - out_idx = np.zeros((n_queries, k), dtype=np.uint32) - out_dist = np.zeros((n_queries, k), dtype=np.float32) - - queries_device = device_ndarray(queries) - out_idx_device = device_ndarray(out_idx) if inplace else None - out_dist_device = device_ndarray(out_dist) if inplace else None - - search_params = cagra.SearchParams(**search_params) - - ret_output = cagra.search( - search_params, - index, - queries_device, - k, - neighbors=out_idx_device, - distances=out_dist_device, - ) - - if not inplace: - out_dist_device, out_idx_device = ret_output - - if not compare: - return - - out_idx = out_idx_device.copy_to_host() - out_dist = out_dist_device.copy_to_host() - - # Calculate reference values with sklearn - skl_metric = { - "sqeuclidean": "sqeuclidean", - "inner_product": "cosine", - "euclidean": "euclidean", - }[metric] - nn_skl = NearestNeighbors( - n_neighbors=k, algorithm="brute", metric=skl_metric - ) - nn_skl.fit(dataset) - skl_idx = nn_skl.kneighbors(queries, return_distance=False) - - recall = calc_recall(out_idx, skl_idx) - assert recall > 0.7 - - -@pytest.mark.parametrize("inplace", [True, False]) -@pytest.mark.parametrize("dtype", [np.float32, np.int8, np.uint8]) -@pytest.mark.parametrize("array_type", ["device", "host"]) -@pytest.mark.parametrize("build_algo", ["ivf_pq", "nn_descent"]) -def test_cagra_dataset_dtype_host_device( - dtype, array_type, inplace, build_algo -): - # Note that inner_product tests use normalized input which we cannot - # represent in int8, therefore we test only sqeuclidean metric here. - run_cagra_build_search_test( - dtype=dtype, - inplace=inplace, - array_type=array_type, - build_algo=build_algo, - ) - - -@pytest.mark.parametrize( - "params", - [ - { - "intermediate_graph_degree": 64, - "graph_degree": 32, - "add_data_on_build": True, - "k": 1, - "metric": "sqeuclidean", - "build_algo": "ivf_pq", - }, - { - "intermediate_graph_degree": 32, - "graph_degree": 16, - "add_data_on_build": False, - "k": 5, - "metric": "sqeuclidean", - "build_algo": "ivf_pq", - }, - { - "intermediate_graph_degree": 128, - "graph_degree": 32, - "add_data_on_build": True, - "k": 10, - "metric": "sqeuclidean", - "build_algo": "nn_descent", - }, - ], -) -def test_cagra_index_params(params): - # Note that inner_product tests use normalized input which we cannot - # represent in int8, therefore we test only sqeuclidean metric here. - run_cagra_build_search_test( - k=params["k"], - metric=params["metric"], - graph_degree=params["graph_degree"], - intermediate_graph_degree=params["intermediate_graph_degree"], - compare=False, - build_algo=params["build_algo"], - ) - - -@pytest.mark.parametrize( - "params", - [ - { - "max_queries": 100, - "itopk_size": 32, - "max_iterations": 100, - "algo": "single_cta", - "team_size": 0, - "search_width": 1, - "min_iterations": 1, - "thread_block_size": 64, - "hashmap_mode": "hash", - "hashmap_min_bitlen": 0.2, - "hashmap_max_fill_rate": 0.5, - "num_random_samplings": 1, - }, - { - "max_queries": 10, - "itopk_size": 128, - "max_iterations": 0, - "algo": "multi_cta", - "team_size": 8, - "search_width": 2, - "min_iterations": 10, - "thread_block_size": 0, - "hashmap_mode": "auto", - "hashmap_min_bitlen": 0.9, - "hashmap_max_fill_rate": 0.5, - "num_random_samplings": 10, - }, - { - "max_queries": 0, - "itopk_size": 64, - "max_iterations": 0, - "algo": "multi_kernel", - "team_size": 16, - "search_width": 1, - "min_iterations": 0, - "thread_block_size": 0, - "hashmap_mode": "auto", - "hashmap_min_bitlen": 0, - "hashmap_max_fill_rate": 0.5, - "num_random_samplings": 1, - }, - { - "max_queries": 0, - "itopk_size": 64, - "max_iterations": 0, - "algo": "auto", - "team_size": 32, - "search_width": 4, - "min_iterations": 0, - "thread_block_size": 0, - "hashmap_mode": "auto", - "hashmap_min_bitlen": 0, - "hashmap_max_fill_rate": 0.5, - "num_random_samplings": 1, - }, - ], -) -def test_cagra_search_params(params): - # Note that inner_product tests use normalized input which we cannot - # represent in int8, therefore we test only sqeuclidean metric here. - run_cagra_build_search_test(search_params=params) - - -@pytest.mark.parametrize("dtype", [np.float32, np.int8, np.ubyte]) -@pytest.mark.parametrize("include_dataset", [True, False]) -def test_save_load(dtype, include_dataset): - n_rows = 10000 - n_cols = 50 - n_queries = 1000 - - dataset = generate_data((n_rows, n_cols), dtype) - dataset_device = device_ndarray(dataset) - - build_params = cagra.IndexParams() - index = cagra.build(build_params, dataset_device) - - assert index.trained - filename = "my_index.bin" - cagra.save(filename, index, include_dataset=include_dataset) - loaded_index = cagra.load(filename) - - # if we didn't save the dataset with the index, we need to update the - # index with an already loaded copy - if not include_dataset: - loaded_index.update_dataset(dataset) - - queries = generate_data((n_queries, n_cols), dtype) - - queries_device = device_ndarray(queries) - search_params = cagra.SearchParams() - k = 10 - - distance_dev, neighbors_dev = cagra.search( - search_params, index, queries_device, k - ) - - neighbors = neighbors_dev.copy_to_host() - dist = distance_dev.copy_to_host() - del index - - distance_dev, neighbors_dev = cagra.search( - search_params, loaded_index, queries_device, k - ) - - neighbors2 = neighbors_dev.copy_to_host() - dist2 = distance_dev.copy_to_host() - - assert np.all(neighbors == neighbors2) - assert np.allclose(dist, dist2, rtol=1e-6) diff --git a/python/pylibraft/pylibraft/test/test_distance.py b/python/pylibraft/pylibraft/test/test_distance.py deleted file mode 100644 index 34ed86db01..0000000000 --- a/python/pylibraft/pylibraft/test/test_distance.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest -from scipy.spatial.distance import cdist - -from pylibraft.common import DeviceResources, Stream, device_ndarray -from pylibraft.distance import pairwise_distance - - -@pytest.mark.parametrize("n_rows", [50, 100]) -@pytest.mark.parametrize("n_cols", [10, 50]) -@pytest.mark.parametrize( - "metric", - [ - "euclidean", - "cityblock", - "chebyshev", - "canberra", - "correlation", - "hamming", - "jensenshannon", - "russellrao", - "cosine", - "sqeuclidean", - "inner_product", - ], -) -@pytest.mark.parametrize("inplace", [True, False]) -@pytest.mark.parametrize("order", ["F", "C"]) -@pytest.mark.parametrize("dtype", [np.float32, np.float64]) -def test_distance(n_rows, n_cols, inplace, metric, order, dtype): - input1 = np.random.random_sample((n_rows, n_cols)) - input1 = np.asarray(input1, order=order).astype(dtype) - - # RussellRao expects boolean arrays - if metric == "russellrao": - input1[input1 < 0.5] = 0 - input1[input1 >= 0.5] = 1 - - # JensenShannon expects probability arrays - elif metric == "jensenshannon": - norm = np.sum(input1, axis=1) - input1 = (input1.T / norm).T - - output = np.zeros((n_rows, n_rows), dtype=dtype) - - if metric == "inner_product": - expected = np.matmul(input1, input1.T) - else: - expected = cdist(input1, input1, metric) - - input1_device = device_ndarray(input1) - output_device = device_ndarray(output) if inplace else None - - s2 = Stream() - handle = DeviceResources(stream=s2) - ret_output = pairwise_distance( - input1_device, input1_device, output_device, metric, handle=handle - ) - handle.sync() - - output_device = ret_output if not inplace else output_device - - actual = output_device.copy_to_host() - - assert np.allclose(expected, actual, atol=1e-3, rtol=1e-3) diff --git a/python/pylibraft/pylibraft/test/test_doctests.py b/python/pylibraft/pylibraft/test/test_doctests.py index c75f565236..b1de79dfcd 100644 --- a/python/pylibraft/pylibraft/test/test_doctests.py +++ b/python/pylibraft/pylibraft/test/test_doctests.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,10 +20,6 @@ import pytest -import pylibraft.cluster -import pylibraft.distance -import pylibraft.matrix -import pylibraft.neighbors import pylibraft.random # Code adapted from https://github.com/rapidsai/cudf/blob/branch-23.02/python/cudf/cudf/tests/test_doctests.py # noqa @@ -92,16 +88,7 @@ def _find_doctests_in_obj(obj, finder=None, criteria=None): # since the root pylibraft module doesn't import submodules (or define an # __all__) we are explicitly adding all the submodules we want to run # doctests for here -DOC_STRINGS = list(_find_doctests_in_obj(pylibraft.cluster)) -DOC_STRINGS.extend(_find_doctests_in_obj(pylibraft.common)) -DOC_STRINGS.extend(_find_doctests_in_obj(pylibraft.distance)) -DOC_STRINGS.extend(_find_doctests_in_obj(pylibraft.matrix.select_k)) -DOC_STRINGS.extend(_find_doctests_in_obj(pylibraft.neighbors)) -DOC_STRINGS.extend(_find_doctests_in_obj(pylibraft.neighbors.brute_force)) -DOC_STRINGS.extend(_find_doctests_in_obj(pylibraft.neighbors.cagra)) -DOC_STRINGS.extend(_find_doctests_in_obj(pylibraft.neighbors.ivf_flat)) -DOC_STRINGS.extend(_find_doctests_in_obj(pylibraft.neighbors.ivf_pq)) -DOC_STRINGS.extend(_find_doctests_in_obj(pylibraft.neighbors.refine)) +DOC_STRINGS = list(_find_doctests_in_obj(pylibraft.common)) DOC_STRINGS.extend(_find_doctests_in_obj(pylibraft.random)) diff --git a/python/pylibraft/pylibraft/test/test_eps_neighborhood.py b/python/pylibraft/pylibraft/test/test_eps_neighborhood.py deleted file mode 100644 index f2643de904..0000000000 --- a/python/pylibraft/pylibraft/test/test_eps_neighborhood.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest -from scipy.sparse import csr_array - -from pylibraft.common import DeviceResources, Stream -from pylibraft.neighbors.brute_force import eps_neighbors as eps_neighbors_bf -from pylibraft.neighbors.rbc import ( - build_rbc_index, - eps_neighbors as eps_neighbors_rbc, -) - - -def test_bf_eps_neighbors_check_col_major_inputs(): - # make sure that we get an exception if passed col-major inputs, - # instead of returning incorrect results - cp = pytest.importorskip("cupy") - n_index_rows, n_query_rows, n_cols = 128, 16, 32 - eps = 0.02 - index = cp.random.random_sample((n_index_rows, n_cols), dtype="float32") - queries = cp.random.random_sample((n_query_rows, n_cols), dtype="float32") - - with pytest.raises(ValueError): - eps_neighbors_bf(cp.asarray(index, order="F"), queries, eps) - - with pytest.raises(ValueError): - eps_neighbors_bf(index, cp.asarray(queries, order="F"), eps) - - # shouldn't throw an exception with c-contiguous inputs - eps_neighbors_bf(index, queries, eps) - - -def test_rbc_eps_neighbors_check_col_major_inputs(): - # make sure that we get an exception if passed col-major inputs, - # instead of returning incorrect results - cp = pytest.importorskip("cupy") - n_index_rows, n_query_rows, n_cols = 128, 16, 32 - eps = 0.02 - index = cp.random.random_sample((n_index_rows, n_cols), dtype="float32") - queries = cp.random.random_sample((n_query_rows, n_cols), dtype="float32") - - with pytest.raises(ValueError): - build_rbc_index(cp.asarray(index, order="F")) - - rbc_index = build_rbc_index(index) - - with pytest.raises(ValueError): - eps_neighbors_rbc(rbc_index, cp.asarray(queries, order="F"), eps) - - eps_neighbors_rbc(rbc_index, queries, eps) - - -@pytest.mark.parametrize("n_index_rows", [32, 100, 1000]) -@pytest.mark.parametrize("n_query_rows", [32, 100, 1000]) -@pytest.mark.parametrize("n_cols", [2, 3, 40, 100]) -def test_eps_neighbors(n_index_rows, n_query_rows, n_cols): - s2 = Stream() - handle = DeviceResources(stream=s2) - - cp = pytest.importorskip("cupy") - eps = 0.02 - index = cp.random.random_sample((n_index_rows, n_cols), dtype="float32") - queries = cp.random.random_sample((n_query_rows, n_cols), dtype="float32") - - # brute force - adj_bf, vd_bf = eps_neighbors_bf(index, queries, eps, handle=handle) - adj_bf = cp.asarray(adj_bf) - vd_bf = cp.asarray(vd_bf) - - rbc_index = build_rbc_index(index, handle=handle) - adj_rbc_ia, adj_rbc_ja, vd_rbc = eps_neighbors_rbc( - rbc_index, queries, eps, handle=handle - ) - adj_rbc_ia = cp.asarray(adj_rbc_ia) - adj_rbc_ja = cp.asarray(adj_rbc_ja) - vd_rbc = cp.asarray(vd_rbc) - - np.testing.assert_array_equal(vd_bf.get(), vd_rbc.get()) - - adj_rbc = csr_array( - ( - np.ones(adj_rbc_ia.get()[n_query_rows]), - adj_rbc_ja.get(), - adj_rbc_ia.get(), - ), - shape=(n_query_rows, n_index_rows), - ).toarray() - np.testing.assert_array_equal(adj_bf.get(), adj_rbc) diff --git a/python/pylibraft/pylibraft/test/test_fused_distance_argmin.py b/python/pylibraft/pylibraft/test/test_fused_distance_argmin.py deleted file mode 100755 index 6736128242..0000000000 --- a/python/pylibraft/pylibraft/test/test_fused_distance_argmin.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest -from scipy.spatial.distance import cdist - -from pylibraft.common import DeviceResources, device_ndarray -from pylibraft.distance import fused_distance_nn_argmin - - -@pytest.mark.parametrize("inplace", [True, False]) -@pytest.mark.parametrize("n_rows", [10, 100]) -@pytest.mark.parametrize("n_clusters", [50, 100]) -@pytest.mark.parametrize("n_cols", [128, 31]) -@pytest.mark.parametrize("dtype", [np.float32]) -@pytest.mark.parametrize( - "metric", - [ - "euclidean", - "cosine", - "sqeuclidean", - ], -) -def test_fused_distance_nn_minarg( - n_rows, n_cols, n_clusters, dtype, inplace, metric -): - input1 = np.random.random_sample((n_rows, n_cols)) - input1 = np.asarray(input1, order="C").astype(dtype) - - input2 = np.random.random_sample((n_clusters, n_cols)) - input2 = np.asarray(input2, order="C").astype(dtype) - - output = np.zeros((n_rows), dtype="int32") - expected = cdist(input1, input2, metric) - - expected = expected.argmin(axis=1) - - input1_device = device_ndarray(input1) - input2_device = device_ndarray(input2) - output_device = device_ndarray(output) if inplace else None - - is_sqrt = True if metric == "sqeuclidean" else False - handle = DeviceResources() - ret_output = fused_distance_nn_argmin( - input1_device, - input2_device, - output_device, - is_sqrt, - metric, - handle=handle, - ) - handle.sync() - output_device = ret_output if not inplace else output_device - actual = output_device.copy_to_host() - - assert np.allclose(expected, actual, rtol=1e-4) diff --git a/python/pylibraft/pylibraft/test/test_fused_l2_argmin.py b/python/pylibraft/pylibraft/test/test_fused_l2_argmin.py deleted file mode 100644 index 086bb26f17..0000000000 --- a/python/pylibraft/pylibraft/test/test_fused_l2_argmin.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest -from scipy.spatial.distance import cdist - -from pylibraft.common import DeviceResources, device_ndarray -from pylibraft.distance import fused_l2_nn_argmin - - -@pytest.mark.parametrize("inplace", [True, False]) -@pytest.mark.parametrize("n_rows", [10, 100]) -@pytest.mark.parametrize("n_clusters", [5, 10]) -@pytest.mark.parametrize("n_cols", [3, 5]) -@pytest.mark.parametrize("dtype", [np.float32, np.float64]) -def test_fused_l2_nn_minarg(n_rows, n_cols, n_clusters, dtype, inplace): - input1 = np.random.random_sample((n_rows, n_cols)) - input1 = np.asarray(input1, order="C").astype(dtype) - - input2 = np.random.random_sample((n_clusters, n_cols)) - input2 = np.asarray(input2, order="C").astype(dtype) - - output = np.zeros((n_rows), dtype="int32") - expected = cdist(input1, input2, metric="euclidean") - - expected = expected.argmin(axis=1) - - input1_device = device_ndarray(input1) - input2_device = device_ndarray(input2) - output_device = device_ndarray(output) if inplace else None - - handle = DeviceResources() - ret_output = fused_l2_nn_argmin( - input1_device, input2_device, output_device, True, handle=handle - ) - handle.sync() - output_device = ret_output if not inplace else output_device - actual = output_device.copy_to_host() - - assert np.allclose(expected, actual, rtol=1e-4) diff --git a/python/pylibraft/pylibraft/test/test_handle.py b/python/pylibraft/pylibraft/test/test_handle.py index bb07df1000..d77a6820f1 100644 --- a/python/pylibraft/pylibraft/test/test_handle.py +++ b/python/pylibraft/pylibraft/test/test_handle.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,27 +17,37 @@ import pytest from pylibraft.common import DeviceResources, Stream, device_ndarray -from pylibraft.distance import pairwise_distance +from pylibraft.random import rmat cupy = pytest.importorskip("cupy") -@pytest.mark.parametrize("stream", [cupy.cuda.Stream().ptr, Stream()]) -def test_handle_external_stream(stream): +def generate_theta(r_scale, c_scale): + max_scale = max(r_scale, c_scale) + theta = np.random.random_sample(max_scale * 4) + for i in range(max_scale): + a = theta[4 * i] + b = theta[4 * i + 1] + c = theta[4 * i + 2] + d = theta[4 * i + 3] + total = a + b + c + d + theta[4 * i] = a / total + theta[4 * i + 1] = b / total + theta[4 * i + 2] = c / total + theta[4 * i + 3] = d / total + theta_device = device_ndarray(theta) + return theta, theta_device - input1 = np.random.random_sample((50, 3)) - input1 = np.asarray(input1, order="F").astype("float") - output = np.zeros((50, 50), dtype="float") +@pytest.mark.parametrize("stream", [cupy.cuda.Stream().ptr, Stream()]) +def test_handle_external_stream(stream): - input1_device = device_ndarray(input1) - output_device = device_ndarray(output) + theta, theta_device = generate_theta(16, 16) + out_buff = np.empty((1000, 2), dtype=np.int32) + output_device = device_ndarray(out_buff) - # We are just testing that this doesn't segfault - handle = DeviceResources(stream) - pairwise_distance( - input1_device, input1_device, output_device, "euclidean", handle=handle - ) + handle = DeviceResources() + rmat(output_device, theta_device, 16, 16, 12345, handle=handle) handle.sync() with pytest.raises(ValueError): diff --git a/python/pylibraft/pylibraft/test/test_hnsw.py b/python/pylibraft/pylibraft/test/test_hnsw.py deleted file mode 100644 index 8cdf8c904f..0000000000 --- a/python/pylibraft/pylibraft/test/test_hnsw.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# h ttp://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest -from sklearn.neighbors import NearestNeighbors -from sklearn.preprocessing import normalize - -from pylibraft.neighbors import cagra, hnsw -from pylibraft.test.ann_utils import calc_recall, generate_data - - -def run_hnsw_build_search_test( - n_rows=10000, - n_cols=10, - n_queries=100, - k=10, - dtype=np.float32, - metric="sqeuclidean", - build_algo="ivf_pq", - intermediate_graph_degree=128, - graph_degree=64, - search_params={}, -): - dataset = generate_data((n_rows, n_cols), dtype) - if metric == "inner_product": - dataset = normalize(dataset, norm="l2", axis=1) - if dtype in [np.int8, np.uint8]: - pytest.skip( - "inner_product metric is not supported for int8/uint8 data" - ) - if build_algo == "nn_descent": - pytest.skip("inner_product metric is not supported for nn_descent") - - build_params = cagra.IndexParams( - metric=metric, - intermediate_graph_degree=intermediate_graph_degree, - graph_degree=graph_degree, - build_algo=build_algo, - ) - - index = cagra.build(build_params, dataset) - - assert index.trained - - hnsw_index = hnsw.from_cagra(index) - - queries = generate_data((n_queries, n_cols), dtype) - out_idx = np.zeros((n_queries, k), dtype=np.uint32) - - search_params = hnsw.SearchParams(**search_params) - - out_dist, out_idx = hnsw.search(search_params, hnsw_index, queries, k) - - # Calculate reference values with sklearn - skl_metric = { - "sqeuclidean": "sqeuclidean", - "inner_product": "cosine", - "euclidean": "euclidean", - }[metric] - nn_skl = NearestNeighbors( - n_neighbors=k, algorithm="brute", metric=skl_metric - ) - nn_skl.fit(dataset) - skl_idx = nn_skl.kneighbors(queries, return_distance=False) - - recall = calc_recall(out_idx, skl_idx) - assert recall > 0.95 - - -@pytest.mark.parametrize("dtype", [np.float32, np.int8, np.uint8]) -@pytest.mark.parametrize("k", [10, 20]) -@pytest.mark.parametrize("ef", [30, 40]) -@pytest.mark.parametrize("num_threads", [2, 4]) -@pytest.mark.parametrize("metric", ["sqeuclidean", "inner_product"]) -@pytest.mark.parametrize("build_algo", ["ivf_pq", "nn_descent"]) -def test_hnsw(dtype, k, ef, num_threads, metric, build_algo): - # Note that inner_product tests use normalized input which we cannot - # represent in int8, therefore we test only sqeuclidean metric here. - run_hnsw_build_search_test( - dtype=dtype, - k=k, - metric=metric, - build_algo=build_algo, - search_params={"ef": ef, "num_threads": num_threads}, - ) diff --git a/python/pylibraft/pylibraft/test/test_ivf_flat.py b/python/pylibraft/pylibraft/test/test_ivf_flat.py deleted file mode 100644 index 2e38dab7bc..0000000000 --- a/python/pylibraft/pylibraft/test/test_ivf_flat.py +++ /dev/null @@ -1,518 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# h ttp://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest -from sklearn.metrics import pairwise_distances -from sklearn.neighbors import NearestNeighbors -from sklearn.preprocessing import normalize - -from pylibraft.common import device_ndarray -from pylibraft.neighbors import ivf_flat - - -def generate_data(shape, dtype): - if dtype == np.byte: - x = np.random.randint(-127, 128, size=shape, dtype=np.byte) - elif dtype == np.ubyte: - x = np.random.randint(0, 255, size=shape, dtype=np.ubyte) - else: - x = np.random.random_sample(shape).astype(dtype) - - return x - - -def calc_recall(ann_idx, true_nn_idx): - assert ann_idx.shape == true_nn_idx.shape - n = 0 - for i in range(ann_idx.shape[0]): - n += np.intersect1d(ann_idx[i, :], true_nn_idx[i, :]).size - recall = n / ann_idx.size - return recall - - -def check_distances(dataset, queries, metric, out_idx, out_dist, eps=None): - """ - Calculate the real distance between queries and dataset[out_idx], - and compare it to out_dist. - """ - if eps is None: - # Quantization leads to errors in the distance calculation. - # The aim of this test is not to test precision, but to catch obvious - # errors. - eps = 0.1 - - dist = np.empty(out_dist.shape, out_dist.dtype) - for i in range(queries.shape[0]): - X = queries[np.newaxis, i, :] - Y = dataset[out_idx[i, :], :] - if metric == "sqeuclidean": - dist[i, :] = pairwise_distances(X, Y, "sqeuclidean") - elif metric == "euclidean": - dist[i, :] = pairwise_distances(X, Y, "euclidean") - elif metric == "inner_product": - dist[i, :] = np.matmul(X, Y.T) - else: - raise ValueError("Invalid metric") - - dist_eps = abs(dist) - dist_eps[dist < 1e-3] = 1e-3 - diff = abs(out_dist - dist) / dist_eps - - assert np.mean(diff) < eps - - -def run_ivf_flat_build_search_test( - n_rows, - n_cols, - n_queries, - k, - n_lists, - metric, - dtype, - add_data_on_build=True, - n_probes=100, - kmeans_trainset_fraction=1, - kmeans_n_iters=20, - compare=True, - inplace=True, - array_type="device", -): - dataset = generate_data((n_rows, n_cols), dtype) - if metric == "inner_product": - dataset = normalize(dataset, norm="l2", axis=1) - dataset_device = device_ndarray(dataset) - - build_params = ivf_flat.IndexParams( - n_lists=n_lists, - metric=metric, - kmeans_n_iters=kmeans_n_iters, - kmeans_trainset_fraction=kmeans_trainset_fraction, - add_data_on_build=add_data_on_build, - ) - - if array_type == "device": - index = ivf_flat.build(build_params, dataset_device) - else: - index = ivf_flat.build(build_params, dataset) - - assert index.trained - - assert index.metric == build_params.metric - assert index.n_lists == build_params.n_lists - - if not add_data_on_build: - dataset_1 = dataset[: n_rows // 2, :] - dataset_2 = dataset[n_rows // 2 :, :] - indices_1 = np.arange(n_rows // 2, dtype=np.int64) - indices_2 = np.arange(n_rows // 2, n_rows, dtype=np.int64) - if array_type == "device": - dataset_1_device = device_ndarray(dataset_1) - dataset_2_device = device_ndarray(dataset_2) - indices_1_device = device_ndarray(indices_1) - indices_2_device = device_ndarray(indices_2) - index = ivf_flat.extend(index, dataset_1_device, indices_1_device) - index = ivf_flat.extend(index, dataset_2_device, indices_2_device) - else: - index = ivf_flat.extend(index, dataset_1, indices_1) - index = ivf_flat.extend(index, dataset_2, indices_2) - - assert index.size >= n_rows - - queries = generate_data((n_queries, n_cols), dtype) - out_idx = np.zeros((n_queries, k), dtype=np.int64) - out_dist = np.zeros((n_queries, k), dtype=np.float32) - - queries_device = device_ndarray(queries) - out_idx_device = device_ndarray(out_idx) if inplace else None - out_dist_device = device_ndarray(out_dist) if inplace else None - - search_params = ivf_flat.SearchParams(n_probes=n_probes) - - ret_output = ivf_flat.search( - search_params, - index, - queries_device, - k, - neighbors=out_idx_device, - distances=out_dist_device, - ) - - if not inplace: - out_dist_device, out_idx_device = ret_output - - if not compare: - return - - out_idx = out_idx_device.copy_to_host() - out_dist = out_dist_device.copy_to_host() - - # Calculate reference values with sklearn - skl_metric = { - "sqeuclidean": "sqeuclidean", - "inner_product": "cosine", - "euclidean": "euclidean", - }[metric] - nn_skl = NearestNeighbors( - n_neighbors=k, algorithm="brute", metric=skl_metric - ) - nn_skl.fit(dataset) - skl_idx = nn_skl.kneighbors(queries, return_distance=False) - - recall = calc_recall(out_idx, skl_idx) - assert recall > 0.7 - - check_distances(dataset, queries, metric, out_idx, out_dist) - - -@pytest.mark.parametrize("inplace", [True, False]) -@pytest.mark.parametrize("n_rows", [10000]) -@pytest.mark.parametrize("n_cols", [10]) -@pytest.mark.parametrize("n_queries", [100]) -@pytest.mark.parametrize("n_lists", [100]) -@pytest.mark.parametrize("dtype", [np.float32, np.int8, np.uint8]) -@pytest.mark.parametrize("array_type", ["device"]) -def test_ivf_pq_dtypes( - n_rows, n_cols, n_queries, n_lists, dtype, inplace, array_type -): - # Note that inner_product tests use normalized input which we cannot - # represent in int8, therefore we test only sqeuclidean metric here. - run_ivf_flat_build_search_test( - n_rows=n_rows, - n_cols=n_cols, - n_queries=n_queries, - k=10, - n_lists=n_lists, - metric="sqeuclidean", - dtype=dtype, - inplace=inplace, - array_type=array_type, - ) - - -@pytest.mark.parametrize( - "params", - [ - pytest.param( - { - "n_rows": 0, - "n_cols": 10, - "n_queries": 10, - "k": 1, - "n_lists": 10, - }, - marks=pytest.mark.xfail(reason="empty dataset"), - ), - {"n_rows": 1, "n_cols": 10, "n_queries": 10, "k": 1, "n_lists": 1}, - {"n_rows": 10, "n_cols": 1, "n_queries": 10, "k": 10, "n_lists": 10}, - # {"n_rows": 999, "n_cols": 42, "n_queries": 453, "k": 137, - # "n_lists": 53}, - ], -) -def test_ivf_flat_n(params): - # We do not test recall, just confirm that we can handle edge cases for - # certain parameters - run_ivf_flat_build_search_test( - n_rows=params["n_rows"], - n_cols=params["n_cols"], - n_queries=params["n_queries"], - k=params["k"], - n_lists=params["n_lists"], - metric="sqeuclidean", - dtype=np.float32, - compare=False, - ) - - -@pytest.mark.parametrize( - "metric", ["sqeuclidean", "inner_product", "euclidean"] -) -@pytest.mark.parametrize("dtype", [np.float32]) -def test_ivf_flat_build_params(metric, dtype): - run_ivf_flat_build_search_test( - n_rows=10000, - n_cols=10, - n_queries=1000, - k=10, - n_lists=100, - metric=metric, - dtype=dtype, - add_data_on_build=True, - n_probes=100, - ) - - -@pytest.mark.parametrize( - "params", - [ - { - "n_lists": 100, - "trainset_fraction": 0.9, - "n_iters": 30, - }, - ], -) -def test_ivf_flat_params(params): - run_ivf_flat_build_search_test( - n_rows=10000, - n_cols=16, - n_queries=1000, - k=10, - n_lists=params["n_lists"], - metric="sqeuclidean", - dtype=np.float32, - kmeans_trainset_fraction=params.get("trainset_fraction", 1.0), - kmeans_n_iters=params.get("n_iters", 20), - ) - - -@pytest.mark.parametrize( - "params", - [ - { - "k": 10, - "n_probes": 100, - }, - { - "k": 10, - "n_probes": 99, - }, - { - "k": 10, - "n_probes": 100, - }, - { - "k": 129, - "n_probes": 100, - }, - { - "k": 257, - "n_probes": 100, - }, - { - "k": 4096, - "n_probes": 100, - }, - ], -) -def test_ivf_pq_search_params(params): - run_ivf_flat_build_search_test( - n_rows=10000, - n_cols=16, - n_queries=1000, - k=params["k"], - n_lists=100, - n_probes=params["n_probes"], - metric="sqeuclidean", - dtype=np.float32, - ) - - -@pytest.mark.parametrize("dtype", [np.float32, np.int8, np.uint8]) -@pytest.mark.parametrize("array_type", ["device"]) -def test_extend(dtype, array_type): - run_ivf_flat_build_search_test( - n_rows=10000, - n_cols=10, - n_queries=100, - k=10, - n_lists=100, - metric="sqeuclidean", - dtype=dtype, - add_data_on_build=False, - array_type=array_type, - ) - - -def test_build_assertions(): - with pytest.raises(TypeError): - run_ivf_flat_build_search_test( - n_rows=1000, - n_cols=10, - n_queries=100, - k=10, - n_lists=100, - metric="sqeuclidean", - dtype=np.float64, - ) - - n_rows = 1000 - n_cols = 100 - n_queries = 212 - k = 10 - dataset = generate_data((n_rows, n_cols), np.float32) - dataset_device = device_ndarray(dataset) - - index_params = ivf_flat.IndexParams( - n_lists=50, - metric="sqeuclidean", - kmeans_n_iters=20, - kmeans_trainset_fraction=1, - add_data_on_build=False, - ) - - index = ivf_flat.Index() - - queries = generate_data((n_queries, n_cols), np.float32) - out_idx = np.zeros((n_queries, k), dtype=np.int64) - out_dist = np.zeros((n_queries, k), dtype=np.float32) - - queries_device = device_ndarray(queries) - out_idx_device = device_ndarray(out_idx) - out_dist_device = device_ndarray(out_dist) - - search_params = ivf_flat.SearchParams(n_probes=50) - - with pytest.raises(ValueError): - # Index must be built before search - ivf_flat.search( - search_params, - index, - queries_device, - k, - out_idx_device, - out_dist_device, - ) - - index = ivf_flat.build(index_params, dataset_device) - assert index.trained - - indices = np.arange(n_rows + 1, dtype=np.int64) - indices_device = device_ndarray(indices) - - with pytest.raises(ValueError): - # Dataset dimension mismatch - ivf_flat.extend(index, queries_device, indices_device) - - with pytest.raises(ValueError): - # indices dimension mismatch - ivf_flat.extend(index, dataset_device, indices_device) - - -@pytest.mark.parametrize( - "params", - [ - {"q_dt": np.float64}, - {"q_order": "F"}, - {"q_cols": 101}, - {"idx_dt": np.uint32}, - {"idx_order": "F"}, - {"idx_rows": 42}, - {"idx_cols": 137}, - {"dist_dt": np.float64}, - {"dist_order": "F"}, - {"dist_rows": 42}, - {"dist_cols": 137}, - ], -) -def test_search_inputs(params): - """Test with invalid input dtype, order, or dimension.""" - n_rows = 1000 - n_cols = 100 - n_queries = 256 - k = 10 - dtype = np.float32 - - q_dt = params.get("q_dt", np.float32) - q_order = params.get("q_order", "C") - queries = generate_data( - (n_queries, params.get("q_cols", n_cols)), q_dt - ).astype(q_dt, order=q_order) - queries_device = device_ndarray(queries) - - idx_dt = params.get("idx_dt", np.int64) - idx_order = params.get("idx_order", "C") - out_idx = np.zeros( - (params.get("idx_rows", n_queries), params.get("idx_cols", k)), - dtype=idx_dt, - order=idx_order, - ) - out_idx_device = device_ndarray(out_idx) - - dist_dt = params.get("dist_dt", np.float32) - dist_order = params.get("dist_order", "C") - out_dist = np.zeros( - (params.get("dist_rows", n_queries), params.get("dist_cols", k)), - dtype=dist_dt, - order=dist_order, - ) - out_dist_device = device_ndarray(out_dist) - - index_params = ivf_flat.IndexParams( - n_lists=50, metric="sqeuclidean", add_data_on_build=True - ) - - dataset = generate_data((n_rows, n_cols), dtype) - dataset_device = device_ndarray(dataset) - index = ivf_flat.build(index_params, dataset_device) - assert index.trained - - with pytest.raises(Exception): - search_params = ivf_flat.SearchParams(n_probes=50) - ivf_flat.search( - search_params, - index, - queries_device, - k, - out_idx_device, - out_dist_device, - ) - - -@pytest.mark.parametrize("dtype", [np.float32, np.int8, np.ubyte]) -def test_save_load(dtype): - n_rows = 10000 - n_cols = 50 - n_queries = 1000 - - dataset = generate_data((n_rows, n_cols), dtype) - dataset_device = device_ndarray(dataset) - - build_params = ivf_flat.IndexParams(n_lists=100, metric="sqeuclidean") - index = ivf_flat.build(build_params, dataset_device) - - assert index.trained - filename = "my_index.bin" - ivf_flat.save(filename, index) - loaded_index = ivf_flat.load(filename) - - assert index.metric == loaded_index.metric - assert index.n_lists == loaded_index.n_lists - assert index.dim == loaded_index.dim - assert index.adaptive_centers == loaded_index.adaptive_centers - - queries = generate_data((n_queries, n_cols), dtype) - - queries_device = device_ndarray(queries) - search_params = ivf_flat.SearchParams(n_probes=100) - k = 10 - - distance_dev, neighbors_dev = ivf_flat.search( - search_params, index, queries_device, k - ) - - neighbors = neighbors_dev.copy_to_host() - dist = distance_dev.copy_to_host() - del index - - distance_dev, neighbors_dev = ivf_flat.search( - search_params, loaded_index, queries_device, k - ) - - neighbors2 = neighbors_dev.copy_to_host() - dist2 = distance_dev.copy_to_host() - - assert np.all(neighbors == neighbors2) - assert np.allclose(dist, dist2, rtol=1e-6) diff --git a/python/pylibraft/pylibraft/test/test_ivf_pq.py b/python/pylibraft/pylibraft/test/test_ivf_pq.py deleted file mode 100644 index aa58e2a8fc..0000000000 --- a/python/pylibraft/pylibraft/test/test_ivf_pq.py +++ /dev/null @@ -1,550 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# h ttp://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest -from sklearn.metrics import pairwise_distances -from sklearn.neighbors import NearestNeighbors -from sklearn.preprocessing import normalize - -from pylibraft.common import device_ndarray -from pylibraft.neighbors import ivf_pq - - -def generate_data(shape, dtype): - if dtype == np.byte: - x = np.random.randint(-127, 128, size=shape, dtype=np.byte) - elif dtype == np.ubyte: - x = np.random.randint(0, 255, size=shape, dtype=np.ubyte) - else: - x = np.random.random_sample(shape).astype(dtype) - - return x - - -def calc_recall(ann_idx, true_nn_idx): - assert ann_idx.shape == true_nn_idx.shape - n = 0 - for i in range(ann_idx.shape[0]): - n += np.intersect1d(ann_idx[i, :], true_nn_idx[i, :]).size - recall = n / ann_idx.size - return recall - - -def check_distances(dataset, queries, metric, out_idx, out_dist, eps=None): - """ - Calculate the real distance between queries and dataset[out_idx], - and compare it to out_dist. - """ - if eps is None: - # Quantization leads to errors in the distance calculation. - # The aim of this test is not to test precision, but to catch obvious - # errors. - eps = 0.1 - - dist = np.empty(out_dist.shape, out_dist.dtype) - for i in range(queries.shape[0]): - X = queries[np.newaxis, i, :] - Y = dataset[out_idx[i, :], :] - if metric == "sqeuclidean": - dist[i, :] = pairwise_distances(X, Y, "sqeuclidean") - elif metric == "euclidean": - dist[i, :] = pairwise_distances(X, Y, "euclidean") - elif metric == "inner_product": - dist[i, :] = np.matmul(X, Y.T) - else: - raise ValueError("Invalid metric") - - dist_eps = abs(dist) - dist_eps[dist < 1e-3] = 1e-3 - diff = abs(out_dist - dist) / dist_eps - - assert np.mean(diff) < eps - - -def run_ivf_pq_build_search_test( - n_rows, - n_cols, - n_queries, - k, - n_lists, - metric, - dtype, - pq_bits=8, - pq_dim=0, - codebook_kind="subspace", - add_data_on_build="True", - n_probes=100, - lut_dtype=np.float32, - internal_distance_dtype=np.float32, - force_random_rotation=False, - kmeans_trainset_fraction=1, - kmeans_n_iters=20, - compare=True, - inplace=True, - array_type="device", -): - dataset = generate_data((n_rows, n_cols), dtype) - if metric == "inner_product": - dataset = normalize(dataset, norm="l2", axis=1) - dataset_device = device_ndarray(dataset) - - build_params = ivf_pq.IndexParams( - n_lists=n_lists, - metric=metric, - kmeans_n_iters=kmeans_n_iters, - kmeans_trainset_fraction=kmeans_trainset_fraction, - pq_bits=pq_bits, - pq_dim=pq_dim, - codebook_kind=codebook_kind, - force_random_rotation=force_random_rotation, - add_data_on_build=add_data_on_build, - ) - - if array_type == "device": - index = ivf_pq.build(build_params, dataset_device) - else: - index = ivf_pq.build(build_params, dataset) - - assert index.trained - if pq_dim != 0: - assert index.pq_dim == build_params.pq_dim - assert index.pq_bits == build_params.pq_bits - assert index.metric == build_params.metric - assert index.n_lists == build_params.n_lists - - if not add_data_on_build: - dataset_1 = dataset[: n_rows // 2, :] - dataset_2 = dataset[n_rows // 2 :, :] - indices_1 = np.arange(n_rows // 2, dtype=np.int64) - indices_2 = np.arange(n_rows // 2, n_rows, dtype=np.int64) - if array_type == "device": - dataset_1_device = device_ndarray(dataset_1) - dataset_2_device = device_ndarray(dataset_2) - indices_1_device = device_ndarray(indices_1) - indices_2_device = device_ndarray(indices_2) - index = ivf_pq.extend(index, dataset_1_device, indices_1_device) - index = ivf_pq.extend(index, dataset_2_device, indices_2_device) - else: - index = ivf_pq.extend(index, dataset_1, indices_1) - index = ivf_pq.extend(index, dataset_2, indices_2) - - assert index.size >= n_rows - - queries = generate_data((n_queries, n_cols), dtype) - out_idx = np.zeros((n_queries, k), dtype=np.int64) - out_dist = np.zeros((n_queries, k), dtype=np.float32) - - queries_device = device_ndarray(queries) - out_idx_device = device_ndarray(out_idx) if inplace else None - out_dist_device = device_ndarray(out_dist) if inplace else None - - search_params = ivf_pq.SearchParams( - n_probes=n_probes, - lut_dtype=lut_dtype, - internal_distance_dtype=internal_distance_dtype, - ) - - ret_output = ivf_pq.search( - search_params, - index, - queries_device, - k, - neighbors=out_idx_device, - distances=out_dist_device, - ) - - if not inplace: - out_dist_device, out_idx_device = ret_output - - if not compare: - return - - out_idx = out_idx_device.copy_to_host() - out_dist = out_dist_device.copy_to_host() - - # Calculate reference values with sklearn - skl_metric = { - "sqeuclidean": "sqeuclidean", - "inner_product": "cosine", - "euclidean": "euclidean", - }[metric] - nn_skl = NearestNeighbors( - n_neighbors=k, algorithm="brute", metric=skl_metric - ) - nn_skl.fit(dataset) - skl_idx = nn_skl.kneighbors(queries, return_distance=False) - - recall = calc_recall(out_idx, skl_idx) - assert recall > 0.7 - - check_distances(dataset, queries, metric, out_idx, out_dist) - - -@pytest.mark.parametrize("inplace", [True, False]) -@pytest.mark.parametrize("n_rows", [10000]) -@pytest.mark.parametrize("n_cols", [10]) -@pytest.mark.parametrize("n_queries", [100]) -@pytest.mark.parametrize("n_lists", [100]) -@pytest.mark.parametrize("dtype", [np.float32, np.int8, np.uint8]) -@pytest.mark.parametrize("array_type", ["host", "device"]) -def test_ivf_pq_dtypes( - n_rows, n_cols, n_queries, n_lists, dtype, inplace, array_type -): - # Note that inner_product tests use normalized input which we cannot - # represent in int8, therefore we test only sqeuclidean metric here. - run_ivf_pq_build_search_test( - n_rows=n_rows, - n_cols=n_cols, - n_queries=n_queries, - k=10, - n_lists=n_lists, - metric="sqeuclidean", - dtype=dtype, - inplace=inplace, - array_type=array_type, - ) - - -@pytest.mark.parametrize( - "params", - [ - pytest.param( - { - "n_rows": 0, - "n_cols": 10, - "n_queries": 10, - "k": 1, - "n_lists": 10, - }, - marks=pytest.mark.xfail(reason="empty dataset"), - ), - {"n_rows": 1, "n_cols": 10, "n_queries": 10, "k": 1, "n_lists": 1}, - {"n_rows": 10, "n_cols": 1, "n_queries": 10, "k": 10, "n_lists": 10}, - # {"n_rows": 999, "n_cols": 42, "n_queries": 453, "k": 137, - # "n_lists": 53}, - ], -) -def test_ivf_pq_n(params): - # We do not test recall, just confirm that we can handle edge cases for - # certain parameters - run_ivf_pq_build_search_test( - n_rows=params["n_rows"], - n_cols=params["n_cols"], - n_queries=params["n_queries"], - k=params["k"], - n_lists=params["n_lists"], - metric="sqeuclidean", - dtype=np.float32, - compare=False, - ) - - -@pytest.mark.parametrize( - "metric", ["sqeuclidean", "inner_product", "euclidean"] -) -@pytest.mark.parametrize("dtype", [np.float32]) -@pytest.mark.parametrize("codebook_kind", ["subspace", "cluster"]) -@pytest.mark.parametrize("rotation", [True, False]) -def test_ivf_pq_build_params(metric, dtype, codebook_kind, rotation): - run_ivf_pq_build_search_test( - n_rows=10000, - n_cols=10, - n_queries=1000, - k=10, - n_lists=100, - metric=metric, - dtype=dtype, - pq_bits=8, - pq_dim=0, - codebook_kind=codebook_kind, - add_data_on_build=True, - n_probes=100, - force_random_rotation=rotation, - ) - - -@pytest.mark.parametrize( - "params", - [ - {"pq_dims": 10, "pq_bits": 8, "n_lists": 100}, - {"pq_dims": 16, "pq_bits": 7, "n_lists": 100}, - {"pq_dims": 0, "pq_bits": 8, "n_lists": 90}, - { - "pq_dims": 0, - "pq_bits": 8, - "n_lists": 100, - "trainset_fraction": 0.9, - "n_iters": 30, - }, - ], -) -def test_ivf_pq_params(params): - run_ivf_pq_build_search_test( - n_rows=10000, - n_cols=16, - n_queries=1000, - k=10, - n_lists=params["n_lists"], - metric="sqeuclidean", - dtype=np.float32, - pq_bits=params["pq_bits"], - pq_dim=params["pq_dims"], - kmeans_trainset_fraction=params.get("trainset_fraction", 1.0), - kmeans_n_iters=params.get("n_iters", 20), - ) - - -@pytest.mark.parametrize( - "params", - [ - { - "k": 10, - "n_probes": 100, - "lut": np.float16, - "idd": np.float32, - }, - { - "k": 10, - "n_probes": 99, - "lut": np.uint8, - "idd": np.float32, - }, - { - "k": 10, - "n_probes": 100, - "lut": np.float16, - "idd": np.float16, - }, - { - "k": 129, - "n_probes": 100, - "lut": np.float32, - "idd": np.float32, - }, - ], -) -def test_ivf_pq_search_params(params): - run_ivf_pq_build_search_test( - n_rows=10000, - n_cols=16, - n_queries=1000, - k=params["k"], - n_lists=100, - n_probes=params["n_probes"], - metric="sqeuclidean", - dtype=np.float32, - lut_dtype=params["lut"], - internal_distance_dtype=params["idd"], - ) - - -@pytest.mark.parametrize("dtype", [np.float32, np.int8, np.uint8]) -@pytest.mark.parametrize("array_type", ["host", "device"]) -def test_extend(dtype, array_type): - run_ivf_pq_build_search_test( - n_rows=10000, - n_cols=10, - n_queries=100, - k=10, - n_lists=100, - metric="sqeuclidean", - dtype=dtype, - add_data_on_build=False, - array_type=array_type, - ) - - -def test_build_assertions(): - with pytest.raises(TypeError): - run_ivf_pq_build_search_test( - n_rows=1000, - n_cols=10, - n_queries=100, - k=10, - n_lists=100, - metric="sqeuclidean", - dtype=np.float64, - ) - - n_rows = 1000 - n_cols = 100 - n_queries = 212 - k = 10 - dataset = generate_data((n_rows, n_cols), np.float32) - dataset_device = device_ndarray(dataset) - - index_params = ivf_pq.IndexParams( - n_lists=50, - metric="sqeuclidean", - kmeans_n_iters=20, - kmeans_trainset_fraction=1, - add_data_on_build=False, - ) - - index = ivf_pq.Index() - - queries = generate_data((n_queries, n_cols), np.float32) - out_idx = np.zeros((n_queries, k), dtype=np.int64) - out_dist = np.zeros((n_queries, k), dtype=np.float32) - - queries_device = device_ndarray(queries) - out_idx_device = device_ndarray(out_idx) - out_dist_device = device_ndarray(out_dist) - - search_params = ivf_pq.SearchParams(n_probes=50) - - with pytest.raises(ValueError): - # Index must be built before search - ivf_pq.search( - search_params, - index, - queries_device, - k, - out_idx_device, - out_dist_device, - ) - - index = ivf_pq.build(index_params, dataset_device) - assert index.trained - - indices = np.arange(n_rows + 1, dtype=np.int64) - indices_device = device_ndarray(indices) - - with pytest.raises(ValueError): - # Dataset dimension mismatch - ivf_pq.extend(index, queries_device, indices_device) - - with pytest.raises(ValueError): - # indices dimension mismatch - ivf_pq.extend(index, dataset_device, indices_device) - - -@pytest.mark.parametrize( - "params", - [ - {"q_dt": np.float64}, - {"q_order": "F"}, - {"q_cols": 101}, - {"idx_dt": np.uint32}, - {"idx_order": "F"}, - {"idx_rows": 42}, - {"idx_cols": 137}, - {"dist_dt": np.float64}, - {"dist_order": "F"}, - {"dist_rows": 42}, - {"dist_cols": 137}, - ], -) -def test_search_inputs(params): - """Test with invalid input dtype, order, or dimension.""" - n_rows = 1000 - n_cols = 100 - n_queries = 256 - k = 10 - dtype = np.float32 - - q_dt = params.get("q_dt", np.float32) - q_order = params.get("q_order", "C") - queries = generate_data( - (n_queries, params.get("q_cols", n_cols)), q_dt - ).astype(q_dt, order=q_order) - queries_device = device_ndarray(queries) - - idx_dt = params.get("idx_dt", np.int64) - idx_order = params.get("idx_order", "C") - out_idx = np.zeros( - (params.get("idx_rows", n_queries), params.get("idx_cols", k)), - dtype=idx_dt, - order=idx_order, - ) - out_idx_device = device_ndarray(out_idx) - - dist_dt = params.get("dist_dt", np.float32) - dist_order = params.get("dist_order", "C") - out_dist = np.zeros( - (params.get("dist_rows", n_queries), params.get("dist_cols", k)), - dtype=dist_dt, - order=dist_order, - ) - out_dist_device = device_ndarray(out_dist) - - index_params = ivf_pq.IndexParams( - n_lists=50, metric="sqeuclidean", add_data_on_build=True - ) - - dataset = generate_data((n_rows, n_cols), dtype) - dataset_device = device_ndarray(dataset) - index = ivf_pq.build(index_params, dataset_device) - assert index.trained - - with pytest.raises(Exception): - search_params = ivf_pq.SearchParams(n_probes=50) - ivf_pq.search( - search_params, - index, - queries_device, - k, - out_idx_device, - out_dist_device, - ) - - -def test_save_load(): - n_rows = 10000 - n_cols = 50 - n_queries = 1000 - dtype = np.float32 - - dataset = generate_data((n_rows, n_cols), dtype) - dataset_device = device_ndarray(dataset) - - build_params = ivf_pq.IndexParams(n_lists=100, metric="sqeuclidean") - index = ivf_pq.build(build_params, dataset_device) - - assert index.trained - filename = "my_index.bin" - ivf_pq.save(filename, index) - loaded_index = ivf_pq.load(filename) - - assert index.pq_dim == loaded_index.pq_dim - assert index.pq_bits == loaded_index.pq_bits - assert index.metric == loaded_index.metric - assert index.n_lists == loaded_index.n_lists - assert index.size == loaded_index.size - - queries = generate_data((n_queries, n_cols), dtype) - - queries_device = device_ndarray(queries) - search_params = ivf_pq.SearchParams(n_probes=100) - k = 10 - - distance_dev, neighbors_dev = ivf_pq.search( - search_params, index, queries_device, k - ) - - neighbors = neighbors_dev.copy_to_host() - dist = distance_dev.copy_to_host() - del index - - distance_dev, neighbors_dev = ivf_pq.search( - search_params, loaded_index, queries_device, k - ) - - neighbors2 = neighbors_dev.copy_to_host() - dist2 = distance_dev.copy_to_host() - - assert np.all(neighbors == neighbors2) - assert np.allclose(dist, dist2, rtol=1e-6) diff --git a/python/pylibraft/pylibraft/test/test_kmeans.py b/python/pylibraft/pylibraft/test/test_kmeans.py deleted file mode 100644 index 8736c6ee7a..0000000000 --- a/python/pylibraft/pylibraft/test/test_kmeans.py +++ /dev/null @@ -1,206 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest - -from pylibraft.cluster.kmeans import ( - KMeansParams, - cluster_cost, - compute_new_centroids, - fit, - init_plus_plus, -) -from pylibraft.common import DeviceResources, device_ndarray -from pylibraft.distance import pairwise_distance - - -@pytest.mark.parametrize("n_rows", [100]) -@pytest.mark.parametrize("n_cols", [5, 25]) -@pytest.mark.parametrize("n_clusters", [5, 15]) -@pytest.mark.parametrize("dtype", [np.float32, np.float64]) -def test_kmeans_fit(n_rows, n_cols, n_clusters, dtype): - # generate some random input points / centroids - X_host = np.random.random_sample((n_rows, n_cols)).astype(dtype) - centroids = device_ndarray(X_host[:n_clusters]) - X = device_ndarray(X_host) - - # compute the inertia, before fitting centroids - original_inertia = cluster_cost(X, centroids) - - params = KMeansParams(n_clusters=n_clusters, seed=42) - - # fit the centroids, make sure inertia has gone down - # TODO: once we have make_blobs exposed to python - # (https://github.com/rapidsai/raft/issues/1059) - # we should use that to test out the kmeans fit, like the C++ - # tests do right now - centroids, inertia, n_iter = fit(params, X, centroids) - assert inertia < original_inertia - assert n_iter >= 1 - assert np.allclose(cluster_cost(X, centroids), inertia, rtol=1e-6) - - -@pytest.mark.parametrize("n_rows", [100]) -@pytest.mark.parametrize("n_cols", [5, 25]) -@pytest.mark.parametrize("n_clusters", [5, 15]) -@pytest.mark.parametrize("metric", ["euclidean", "sqeuclidean"]) -@pytest.mark.parametrize("dtype", [np.float32, np.float64]) -@pytest.mark.parametrize("additional_args", [True, False]) -def test_compute_new_centroids( - n_rows, n_cols, metric, n_clusters, dtype, additional_args -): - - # A single RAFT handle can optionally be reused across - # pylibraft functions. - handle = DeviceResources() - - X = np.random.random_sample((n_rows, n_cols)).astype(dtype) - X_device = device_ndarray(X) - - centroids = X[:n_clusters] - centroids_device = device_ndarray(centroids) - - weight_per_cluster = np.zeros((n_clusters,), dtype=dtype) - weight_per_cluster_device = ( - device_ndarray(weight_per_cluster) if additional_args else None - ) - - new_centroids = np.zeros((n_clusters, n_cols), dtype=dtype) - new_centroids_device = device_ndarray(new_centroids) - - sample_weights = np.ones((n_rows,)).astype(dtype) / n_rows - sample_weights_device = ( - device_ndarray(sample_weights) if additional_args else None - ) - - # Compute new centroids naively - dists = np.zeros((n_rows, n_clusters), dtype=dtype) - dists_device = device_ndarray(dists) - pairwise_distance(X_device, centroids_device, dists_device, metric=metric) - handle.sync() - - labels = np.argmin(dists_device.copy_to_host(), axis=1).astype(np.int32) - labels_device = device_ndarray(labels) - - expected_centers = np.empty((n_clusters, n_cols), dtype=dtype) - expected_wX = X * sample_weights.reshape((-1, 1)) - for i in range(n_clusters): - j = expected_wX[labels == i] - j = j.sum(axis=0) - g = sample_weights[labels == i].sum() - expected_centers[i, :] = j / g - - compute_new_centroids( - X_device, - centroids_device, - labels_device, - new_centroids_device, - sample_weights=sample_weights_device, - weight_per_cluster=weight_per_cluster_device, - handle=handle, - ) - - # pylibraft functions are often asynchronous so the - # handle needs to be explicitly synchronized - handle.sync() - - actual_centers = new_centroids_device.copy_to_host() - - assert np.allclose(expected_centers, actual_centers, rtol=1e-6) - - -@pytest.mark.parametrize("n_rows", [100]) -@pytest.mark.parametrize("n_cols", [5, 25]) -@pytest.mark.parametrize("n_clusters", [4, 15]) -@pytest.mark.parametrize("dtype", [np.float32, np.float64]) -def test_cluster_cost(n_rows, n_cols, n_clusters, dtype): - X = np.random.random_sample((n_rows, n_cols)).astype(dtype) - X_device = device_ndarray(X) - - centroids = X[:n_clusters] - centroids_device = device_ndarray(centroids) - - inertia = cluster_cost(X_device, centroids_device) - - # compute the nearest centroid to each sample - distances = pairwise_distance( - X_device, centroids_device, metric="sqeuclidean" - ).copy_to_host() - cluster_ids = np.argmin(distances, axis=1) - - cluster_distances = np.take_along_axis( - distances, cluster_ids[:, None], axis=1 - ) - - # need reduced tolerance for float32 - tol = 1e-3 if dtype == np.float32 else 1e-6 - assert np.allclose(inertia, sum(cluster_distances), rtol=tol, atol=tol) - - -@pytest.mark.parametrize("n_rows", [100]) -@pytest.mark.parametrize("n_cols", [5, 25]) -@pytest.mark.parametrize("n_clusters", [4, 15]) -@pytest.mark.parametrize("dtype", [np.float32, np.float64]) -def test_init_plus_plus(n_rows, n_cols, n_clusters, dtype): - X = np.random.random_sample((n_rows, n_cols)).astype(dtype) - X_device = device_ndarray(X) - - centroids = init_plus_plus(X_device, n_clusters, seed=1) - centroids_ = centroids.copy_to_host() - - assert centroids_.shape == (n_clusters, X.shape[1]) - - # Centroids are selected from the existing points - for centroid in centroids_: - assert (centroid == X).all(axis=1).any() - - -@pytest.mark.parametrize("n_rows", [100]) -@pytest.mark.parametrize("n_cols", [5, 25]) -@pytest.mark.parametrize("n_clusters", [4, 15]) -@pytest.mark.parametrize("dtype", [np.float32, np.float64]) -def test_init_plus_plus_preallocated_output(n_rows, n_cols, n_clusters, dtype): - X = np.random.random_sample((n_rows, n_cols)).astype(dtype) - X_device = device_ndarray(X) - - centroids = device_ndarray.empty((n_clusters, n_cols), dtype=dtype) - - new_centroids = init_plus_plus(X_device, centroids=centroids, seed=1) - new_centroids_ = new_centroids.copy_to_host() - - # The shape should not have changed - assert new_centroids_.shape == centroids.shape - - # Centroids are selected from the existing points - for centroid in new_centroids_: - assert (centroid == X).all(axis=1).any() - - -def test_init_plus_plus_exclusive_arguments(): - # Check an exception is raised when n_clusters and centroids shape - # are inconsistent. - X = np.random.random_sample((10, 5)).astype(np.float64) - X = device_ndarray(X) - - n_clusters = 3 - - centroids = np.random.random_sample((n_clusters + 1, 5)).astype(np.float64) - centroids = device_ndarray(centroids) - - with pytest.raises( - RuntimeError, match="Parameters 'n_clusters' and 'centroids'" - ): - init_plus_plus(X, n_clusters, centroids=centroids) diff --git a/python/pylibraft/pylibraft/test/test_refine.py b/python/pylibraft/pylibraft/test/test_refine.py deleted file mode 100644 index 397ea70ec7..0000000000 --- a/python/pylibraft/pylibraft/test/test_refine.py +++ /dev/null @@ -1,233 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# h ttp://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest -from sklearn.neighbors import NearestNeighbors -from sklearn.preprocessing import normalize -from test_ivf_pq import calc_recall, check_distances, generate_data - -from pylibraft.common import device_ndarray -from pylibraft.neighbors import refine - - -def run_refine( - n_rows=500, - n_cols=50, - n_queries=100, - metric="sqeuclidean", - k0=40, - k=10, - inplace=False, - dtype=np.float32, - memory_type="device", -): - - dataset = generate_data((n_rows, n_cols), dtype) - queries = generate_data((n_queries, n_cols), dtype) - - if metric == "inner_product": - if dtype != np.float32: - pytest.skip("Normalized input cannot be represented in int8") - return - dataset = normalize(dataset, norm="l2", axis=1) - queries = normalize(queries, norm="l2", axis=1) - - dataset_device = device_ndarray(dataset) - queries_device = device_ndarray(queries) - - # Calculate reference values with sklearn - skl_metric = {"sqeuclidean": "euclidean", "inner_product": "cosine"}[ - metric - ] - nn_skl = NearestNeighbors( - n_neighbors=k0, algorithm="brute", metric=skl_metric - ) - nn_skl.fit(dataset) - skl_dist, candidates = nn_skl.kneighbors(queries) - candidates = candidates.astype(np.int64) - candidates_device = device_ndarray(candidates) - - out_idx = np.zeros((n_queries, k), dtype=np.int64) - out_dist = np.zeros((n_queries, k), dtype=np.float32) - out_idx_device = device_ndarray(out_idx) if inplace else None - out_dist_device = device_ndarray(out_dist) if inplace else None - - if memory_type == "device": - if inplace: - refine( - dataset_device, - queries_device, - candidates_device, - indices=out_idx_device, - distances=out_dist_device, - metric=metric, - ) - else: - out_dist_device, out_idx_device = refine( - dataset_device, - queries_device, - candidates_device, - k=k, - metric=metric, - ) - out_idx = out_idx_device.copy_to_host() - out_dist = out_dist_device.copy_to_host() - elif memory_type == "host": - if inplace: - refine( - dataset, - queries, - candidates, - indices=out_idx, - distances=out_dist, - metric=metric, - ) - else: - out_dist, out_idx = refine( - dataset, queries, candidates, k=k, metric=metric - ) - - skl_idx = candidates[:, :k] - - recall = calc_recall(out_idx, skl_idx) - if recall <= 0.999: - # We did not find the same neighbor indices. - # We could have found other neighbor with same distance. - if metric == "sqeuclidean": - skl_dist = np.power(skl_dist[:, :k], 2) - elif metric == "inner_product": - skl_dist = 1 - skl_dist[:, :k] - else: - raise ValueError("Invalid metric") - mask = out_idx != skl_idx - assert np.all(out_dist[mask] <= skl_dist[mask] + 1.0e-6) - - check_distances(dataset, queries, metric, out_idx, out_dist, 0.001) - - -@pytest.mark.parametrize("n_queries", [100, 1024, 37]) -@pytest.mark.parametrize("inplace", [True, False]) -@pytest.mark.parametrize("metric", ["sqeuclidean", "inner_product"]) -@pytest.mark.parametrize("dtype", [np.float32, np.int8, np.uint8]) -@pytest.mark.parametrize("memory_type", ["device", "host"]) -def test_refine_dtypes(n_queries, dtype, inplace, metric, memory_type): - run_refine( - n_rows=2000, - n_queries=n_queries, - n_cols=50, - k0=40, - k=10, - dtype=dtype, - inplace=inplace, - metric=metric, - memory_type=memory_type, - ) - - -@pytest.mark.parametrize( - "params", - [ - pytest.param( - { - "n_rows": 0, - "n_cols": 10, - "n_queries": 10, - "k0": 10, - "k": 1, - }, - marks=pytest.mark.xfail(reason="empty dataset"), - ), - {"n_rows": 1, "n_cols": 10, "n_queries": 10, "k": 1, "k0": 1}, - {"n_rows": 10, "n_cols": 1, "n_queries": 10, "k": 10, "k0": 10}, - {"n_rows": 999, "n_cols": 42, "n_queries": 453, "k0": 137, "k": 53}, - ], -) -@pytest.mark.parametrize("memory_type", ["device", "host"]) -def test_refine_row_col(params, memory_type): - run_refine( - n_rows=params["n_rows"], - n_queries=params["n_queries"], - n_cols=params["n_cols"], - k0=params["k0"], - k=params["k"], - memory_type=memory_type, - ) - - -@pytest.mark.parametrize("memory_type", ["device", "host"]) -def test_input_dtype(memory_type): - with pytest.raises(Exception): - run_refine(dtype=np.float64, memory_type=memory_type) - - -@pytest.mark.parametrize( - "params", - [ - {"idx_shape": None, "dist_shape": None, "k": None}, - {"idx_shape": [100, 9], "dist_shape": None, "k": 10}, - {"idx_shape": [101, 10], "dist_shape": None, "k": None}, - {"idx_shape": None, "dist_shape": [100, 11], "k": 10}, - {"idx_shape": None, "dist_shape": [99, 10], "k": None}, - ], -) -@pytest.mark.parametrize("memory_type", ["device", "host"]) -def test_input_assertions(params, memory_type): - n_cols = 5 - n_queries = 100 - k0 = 40 - dtype = np.float32 - dataset = generate_data((500, n_cols), dtype) - dataset_device = device_ndarray(dataset) - - queries = generate_data((n_queries, n_cols), dtype) - queries_device = device_ndarray(queries) - - candidates = np.random.randint( - 0, 500, size=(n_queries, k0), dtype=np.int64 - ) - candidates_device = device_ndarray(candidates) - - if params["idx_shape"] is not None: - out_idx = np.zeros(params["idx_shape"], dtype=np.int64) - out_idx_device = device_ndarray(out_idx) - else: - out_idx_device = None - if params["dist_shape"] is not None: - out_dist = np.zeros(params["dist_shape"], dtype=np.float32) - out_dist_device = device_ndarray(out_dist) - else: - out_dist_device = None - - if memory_type == "device": - with pytest.raises(Exception): - distances, indices = refine( - dataset_device, - queries_device, - candidates_device, - k=params["k"], - indices=out_idx_device, - distances=out_dist_device, - ) - else: - with pytest.raises(Exception): - distances, indices = refine( - dataset, - queries, - candidates, - k=params["k"], - indices=out_idx, - distances=out_dist, - ) diff --git a/python/pylibraft/pylibraft/test/test_select_k.py b/python/pylibraft/pylibraft/test/test_select_k.py deleted file mode 100644 index 203e735b9c..0000000000 --- a/python/pylibraft/pylibraft/test/test_select_k.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest - -from pylibraft.common import device_ndarray -from pylibraft.matrix import select_k - - -@pytest.mark.parametrize("n_rows", [32, 100]) -@pytest.mark.parametrize("n_cols", [40, 100]) -@pytest.mark.parametrize("k", [1, 5, 16, 35]) -@pytest.mark.parametrize("inplace", [True, False]) -def test_select_k(n_rows, n_cols, k, inplace): - dataset = np.random.random_sample((n_rows, n_cols)).astype("float32") - dataset_device = device_ndarray(dataset) - - indices = np.zeros((n_rows, k), dtype="int64") - distances = np.zeros((n_rows, k), dtype="float32") - indices_device = device_ndarray(indices) - distances_device = device_ndarray(distances) - - ret_distances, ret_indices = select_k( - dataset_device, - k=k, - distances=distances_device, - indices=indices_device, - ) - - distances_device = ret_distances if not inplace else distances_device - actual_distances = distances_device.copy_to_host() - argsort = np.argsort(dataset, axis=1) - - for i in range(dataset.shape[0]): - expected_indices = argsort[i] - gpu_dists = actual_distances[i] - - cpu_ordered = dataset[i, expected_indices] - np.testing.assert_allclose( - cpu_ordered[:k], gpu_dists, atol=1e-4, rtol=1e-4 - ) diff --git a/setup.cfg b/setup.cfg index e64641d05b..94140d4d00 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,7 @@ per-file-ignores = # unlike the match option above this match-dir will have no effect when # pydocstyle is invoked from pre-commit. Therefore this exclusion list must # also be maintained in the pre-commit config file. -match-dir = ^(?!(ci|cpp|conda|docs|java|notebooks)).*$ +match-dir = ^(?!(ci|cpp|conda|docs)).*$ # Allow missing docstrings for docutils ignore-decorators = .*(docutils|doc_apply|copy_docstring).* select = From 7ab2b3d3f4e4efa7c7d34bf92f8dd7c822549600 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Mon, 18 Nov 2024 16:24:26 -0500 Subject: [PATCH 54/79] Removing some left over places where implicit instantiations were being ignored in headers (#2501) Authors: - Corey J. Nolet (https://github.com/cjnolet) - Bradley Dice (https://github.com/bdice) Approvers: - Divye Gala (https://github.com/divyegala) URL: https://github.com/rapidsai/raft/pull/2501 --- .../raft/distance/fused_distance_nn-ext.cuh | 84 --- .../raft/distance/fused_distance_nn.cuh | 8 +- .../linalg/detail/coalesced_reduction-ext.cuh | 73 --- .../linalg/detail/coalesced_reduction.cuh | 7 - .../cagra/search_multi_cta_kernel-ext.cuh | 418 ------------ .../detail/cagra/search_multi_cta_kernel.cuh | 6 - .../cagra/search_single_cta_kernel-ext.cuh | 602 ------------------ .../detail/cagra/search_single_cta_kernel.cuh | 6 - .../detail/ivf_flat_interleaved_scan-ext.cuh | 87 --- .../detail/ivf_flat_interleaved_scan.cuh | 6 - .../neighbors/detail/ivf_flat_search-ext.cuh | 71 --- .../raft/neighbors/detail/ivf_flat_search.cuh | 6 - .../detail/ivf_pq_compute_similarity-ext.cuh | 220 ------- .../detail/ivf_pq_compute_similarity.cuh | 6 - .../sparse/matrix/detail/select_k-ext.cuh | 64 -- .../raft/sparse/matrix/detail/select_k.cuh | 7 - 16 files changed, 1 insertion(+), 1670 deletions(-) delete mode 100644 cpp/include/raft/distance/fused_distance_nn-ext.cuh delete mode 100644 cpp/include/raft/linalg/detail/coalesced_reduction-ext.cuh delete mode 100644 cpp/include/raft/neighbors/detail/cagra/search_multi_cta_kernel-ext.cuh delete mode 100644 cpp/include/raft/neighbors/detail/cagra/search_single_cta_kernel-ext.cuh delete mode 100644 cpp/include/raft/neighbors/detail/ivf_flat_interleaved_scan-ext.cuh delete mode 100644 cpp/include/raft/neighbors/detail/ivf_flat_search-ext.cuh delete mode 100644 cpp/include/raft/neighbors/detail/ivf_pq_compute_similarity-ext.cuh delete mode 100644 cpp/include/raft/sparse/matrix/detail/select_k-ext.cuh diff --git a/cpp/include/raft/distance/fused_distance_nn-ext.cuh b/cpp/include/raft/distance/fused_distance_nn-ext.cuh deleted file mode 100644 index 263bbcea81..0000000000 --- a/cpp/include/raft/distance/fused_distance_nn-ext.cuh +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // raft::KeyValuePair -#include // raft::resources -#include // include initialize and reduce operations -#include // RAFT_EXPLICIT - -#include // int64_t - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft { -namespace distance { - -template -void fusedDistanceNNMinReduce(OutT* min, - const DataT* x, - const DataT* y, - const DataT* xn, - const DataT* yn, - IdxT m, - IdxT n, - IdxT k, - void* workspace, - bool sqrt, - bool initOutBuffer, - bool isRowMajor, - raft::distance::DistanceType metric, - float metric_arg, - cudaStream_t stream) RAFT_EXPLICIT; - -} // namespace distance -} // namespace raft - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_distance_fusedDistanceNNMinReduce(DataT, OutT, IdxT) \ - extern template void raft::distance::fusedDistanceNNMinReduce( \ - OutT * min, \ - const DataT* x, \ - const DataT* y, \ - const DataT* xn, \ - const DataT* yn, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - void* workspace, \ - bool sqrt, \ - bool initOutBuffer, \ - bool isRowMajor, \ - raft::distance::DistanceType metric, \ - float metric_arg, \ - cudaStream_t stream) - -instantiate_raft_distance_fusedDistanceNNMinReduce(float, float, int); -instantiate_raft_distance_fusedDistanceNNMinReduce(float, float, int64_t); - -// We can't have comma's in the macro expansion, so we use the COMMA macro: -#define COMMA , - -instantiate_raft_distance_fusedDistanceNNMinReduce(float, raft::KeyValuePair, int); -instantiate_raft_distance_fusedDistanceNNMinReduce(float, - raft::KeyValuePair, - int64_t); - -#undef COMMA - -#undef instantiate_raft_distance_fusedDistanceNNMinReduce diff --git a/cpp/include/raft/distance/fused_distance_nn.cuh b/cpp/include/raft/distance/fused_distance_nn.cuh index 04c42e49a1..25b1ae01ea 100755 --- a/cpp/include/raft/distance/fused_distance_nn.cuh +++ b/cpp/include/raft/distance/fused_distance_nn.cuh @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY -#include "fused_distance_nn-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "fused_distance_nn-ext.cuh" -#endif +#include "fused_distance_nn-inl.cuh" \ No newline at end of file diff --git a/cpp/include/raft/linalg/detail/coalesced_reduction-ext.cuh b/cpp/include/raft/linalg/detail/coalesced_reduction-ext.cuh deleted file mode 100644 index 4800f2e3cf..0000000000 --- a/cpp/include/raft/linalg/detail/coalesced_reduction-ext.cuh +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -// The explicit instantiation of raft::linalg::detail::coalescedReduction is not -// forced because there would be too many instances. Instead, we cover the most -// common instantiations with extern template instantiations below. - -#define instantiate_raft_linalg_detail_coalescedReduction( \ - InType, OutType, IdxType, MainLambda, ReduceLambda, FinalLambda) \ - extern template void raft::linalg::detail::coalescedReduction(OutType* dots, \ - const InType* data, \ - IdxType D, \ - IdxType N, \ - OutType init, \ - cudaStream_t stream, \ - bool inplace, \ - MainLambda main_op, \ - ReduceLambda reduce_op, \ - FinalLambda final_op) - -instantiate_raft_linalg_detail_coalescedReduction( - double, double, int, raft::identity_op, raft::min_op, raft::identity_op); -instantiate_raft_linalg_detail_coalescedReduction( - double, double, int, raft::sq_op, raft::add_op, raft::identity_op); -instantiate_raft_linalg_detail_coalescedReduction( - double, double, int, raft::sq_op, raft::add_op, raft::sqrt_op); -instantiate_raft_linalg_detail_coalescedReduction( - double, double, int, raft::abs_op, raft::add_op, raft::identity_op); -instantiate_raft_linalg_detail_coalescedReduction( - double, double, int, raft::abs_op, raft::max_op, raft::identity_op); -instantiate_raft_linalg_detail_coalescedReduction( - float, float, size_t, raft::abs_op, raft::add_op, raft::sqrt_op); -instantiate_raft_linalg_detail_coalescedReduction( - float, float, int, raft::abs_op, raft::add_op, raft::identity_op); -instantiate_raft_linalg_detail_coalescedReduction( - float, float, int, raft::identity_op, raft::add_op, raft::identity_op); -instantiate_raft_linalg_detail_coalescedReduction( - float, float, int, raft::identity_op, raft::min_op, raft::identity_op); -instantiate_raft_linalg_detail_coalescedReduction( - float, float, int, raft::sq_op, raft::add_op, raft::identity_op); -instantiate_raft_linalg_detail_coalescedReduction( - float, float, int, raft::sq_op, raft::add_op, raft::sqrt_op); -instantiate_raft_linalg_detail_coalescedReduction( - float, float, long, raft::sq_op, raft::add_op, raft::identity_op); -instantiate_raft_linalg_detail_coalescedReduction( - float, float, size_t, raft::identity_op, raft::add_op, raft::identity_op); -instantiate_raft_linalg_detail_coalescedReduction( - float, float, size_t, raft::sq_op, raft::add_op, raft::identity_op); -instantiate_raft_linalg_detail_coalescedReduction( - float, float, size_t, raft::abs_op, raft::max_op, raft::sqrt_op); -instantiate_raft_linalg_detail_coalescedReduction( - float, float, size_t, raft::sq_op, raft::add_op, raft::sqrt_op); -instantiate_raft_linalg_detail_coalescedReduction( - float, float, unsigned int, raft::sq_op, raft::add_op, raft::identity_op); - -#undef instantiate_raft_linalg_detail_coalescedReduction diff --git a/cpp/include/raft/linalg/detail/coalesced_reduction.cuh b/cpp/include/raft/linalg/detail/coalesced_reduction.cuh index 3e6b17978b..d24c2a7444 100644 --- a/cpp/include/raft/linalg/detail/coalesced_reduction.cuh +++ b/cpp/include/raft/linalg/detail/coalesced_reduction.cuh @@ -16,11 +16,4 @@ #pragma once -// Always include inline definitions of coalesced reduction, because we do not -// force explicit instantion. #include "coalesced_reduction-inl.cuh" - -// Do include the extern template instantiations when possible. -#ifdef RAFT_COMPILED -#include "coalesced_reduction-ext.cuh" -#endif diff --git a/cpp/include/raft/neighbors/detail/cagra/search_multi_cta_kernel-ext.cuh b/cpp/include/raft/neighbors/detail/cagra/search_multi_cta_kernel-ext.cuh deleted file mode 100644 index 35f4f0e1c9..0000000000 --- a/cpp/include/raft/neighbors/detail/cagra/search_multi_cta_kernel-ext.cuh +++ /dev/null @@ -1,418 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include -#include // none_cagra_sample_filter -#include // RAFT_EXPLICIT - -#include - -namespace raft::neighbors::cagra::detail { -namespace multi_cta_search { - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -template -void select_and_run( - DATASET_DESCRIPTOR_T dataset_desc, - raft::device_matrix_view graph, - typename DATASET_DESCRIPTOR_T::INDEX_T* const topk_indices_ptr, - typename DATASET_DESCRIPTOR_T::DISTANCE_T* const topk_distances_ptr, - const typename DATASET_DESCRIPTOR_T::DATA_T* const queries_ptr, - const uint32_t num_queries, - const typename DATASET_DESCRIPTOR_T::INDEX_T* dev_seed_ptr, - uint32_t* const num_executed_iterations, - uint32_t topk, - uint32_t block_size, - uint32_t result_buffer_size, - uint32_t smem_size, - int64_t hash_bitlen, - typename DATASET_DESCRIPTOR_T::INDEX_T* hashmap_ptr, - uint32_t num_cta_per_query, - uint32_t num_random_samplings, - uint64_t rand_xor_mask, - uint32_t num_seeds, - size_t itopk_size, - size_t search_width, - size_t min_iterations, - size_t max_iterations, - SAMPLE_FILTER_T sample_filter, - raft::distance::DistanceType metric, - cudaStream_t stream) RAFT_EXPLICIT; -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_kernel_selection( \ - TEAM_SIZE, MAX_DATASET_DIM, DATA_T, INDEX_T, DISTANCE_T, SAMPLE_FILTER_T) \ - extern template void select_and_run< \ - TEAM_SIZE, \ - MAX_DATASET_DIM, \ - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, \ - SAMPLE_FILTER_T>( \ - raft::neighbors::cagra::detail::standard_dataset_descriptor_t \ - dataset_desc, \ - raft::device_matrix_view graph, \ - INDEX_T* const topk_indices_ptr, \ - DISTANCE_T* const topk_distances_ptr, \ - const DATA_T* const queries_ptr, \ - const uint32_t num_queries, \ - const INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ - uint32_t topk, \ - uint32_t block_size, \ - uint32_t result_buffer_size, \ - uint32_t smem_size, \ - int64_t hash_bitlen, \ - INDEX_T* hashmap_ptr, \ - uint32_t num_cta_per_query, \ - uint32_t num_random_samplings, \ - uint64_t rand_xor_mask, \ - uint32_t num_seeds, \ - size_t itopk_size, \ - size_t search_width, \ - size_t min_iterations, \ - size_t max_iterations, \ - SAMPLE_FILTER_T sample_filter, \ - raft::distance::DistanceType metric, \ - cudaStream_t stream); - -instantiate_kernel_selection( - 32, 1024, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 8, 128, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 16, 256, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 32, 512, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 32, 1024, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 8, 128, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 16, 256, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 32, 512, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 32, 1024, int8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 8, 128, int8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 16, 256, int8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 32, 512, int8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 32, 1024, uint8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 8, 128, uint8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 16, 256, uint8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 32, 512, uint8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); - -#undef instantiate_kernel_selection - -#define instantiate_q_kernel_selection(TEAM_SIZE, \ - MAX_DATASET_DIM, \ - CODE_BOOK_T, \ - PQ_BITS, \ - PQ_CODE_BOOK_DIM, \ - DATA_T, \ - INDEX_T, \ - DISTANCE_T, \ - SAMPLE_FILTER_T) \ - extern template void \ - select_and_run, \ - SAMPLE_FILTER_T>( \ - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t dataset_desc, \ - raft::device_matrix_view graph, \ - INDEX_T* const topk_indices_ptr, \ - DISTANCE_T* const topk_distances_ptr, \ - const DATA_T* const queries_ptr, \ - const uint32_t num_queries, \ - const INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ - uint32_t topk, \ - uint32_t block_size, \ - uint32_t result_buffer_size, \ - uint32_t smem_size, \ - int64_t hash_bitlen, \ - INDEX_T* hashmap_ptr, \ - uint32_t num_cta_per_query, \ - uint32_t num_random_samplings, \ - uint64_t rand_xor_mask, \ - uint32_t num_seeds, \ - size_t itopk_size, \ - size_t search_width, \ - size_t min_iterations, \ - size_t max_iterations, \ - SAMPLE_FILTER_T sample_filter, \ - raft::distance::DistanceType metric, \ - cudaStream_t stream); - -instantiate_q_kernel_selection( - 8, 128, half, 8, 2, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection( - 16, 256, half, 8, 2, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection( - 32, 512, half, 8, 2, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 1024, - half, - 8, - 2, - half, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection( - 8, 128, half, 8, 4, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection( - 16, 256, half, 8, 4, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection( - 32, 512, half, 8, 4, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 1024, - half, - 8, - 4, - half, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -instantiate_q_kernel_selection( - 8, 128, half, 8, 2, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(16, - 256, - half, - 8, - 2, - float, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 512, - half, - 8, - 2, - float, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 1024, - half, - 8, - 2, - float, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection( - 8, 128, half, 8, 4, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(16, - 256, - half, - 8, - 4, - float, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 512, - half, - 8, - 4, - float, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 1024, - half, - 8, - 4, - float, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -instantiate_q_kernel_selection(8, - 128, - half, - 8, - 2, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(16, - 256, - half, - 8, - 2, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 512, - half, - 8, - 2, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 1024, - half, - 8, - 2, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(8, - 128, - half, - 8, - 4, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(16, - 256, - half, - 8, - 4, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 512, - half, - 8, - 4, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 1024, - half, - 8, - 4, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -instantiate_q_kernel_selection(8, - 128, - half, - 8, - 2, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(16, - 256, - half, - 8, - 2, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 512, - half, - 8, - 2, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 1024, - half, - 8, - 2, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(8, - 128, - half, - 8, - 4, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(16, - 256, - half, - 8, - 4, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 512, - half, - 8, - 4, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 1024, - half, - 8, - 4, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -#undef instantiate_q_kernel_selection -} // namespace multi_cta_search -} // namespace raft::neighbors::cagra::detail diff --git a/cpp/include/raft/neighbors/detail/cagra/search_multi_cta_kernel.cuh b/cpp/include/raft/neighbors/detail/cagra/search_multi_cta_kernel.cuh index e003907292..3dc0745e6d 100644 --- a/cpp/include/raft/neighbors/detail/cagra/search_multi_cta_kernel.cuh +++ b/cpp/include/raft/neighbors/detail/cagra/search_multi_cta_kernel.cuh @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "search_multi_cta_kernel-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "search_multi_cta_kernel-ext.cuh" -#endif diff --git a/cpp/include/raft/neighbors/detail/cagra/search_single_cta_kernel-ext.cuh b/cpp/include/raft/neighbors/detail/cagra/search_single_cta_kernel-ext.cuh deleted file mode 100644 index 510219ab5d..0000000000 --- a/cpp/include/raft/neighbors/detail/cagra/search_single_cta_kernel-ext.cuh +++ /dev/null @@ -1,602 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include -#include // RAFT_EXPLICIT - -#include - -namespace raft::neighbors::cagra::detail { -namespace single_cta_search { - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -template -void select_and_run( // raft::resources const& res, - DATASET_DESCRIPTOR_T dataset_desc, - raft::device_matrix_view graph, - typename DATASET_DESCRIPTOR_T::INDEX_T* const topk_indices_ptr, // [num_queries, topk] - typename DATASET_DESCRIPTOR_T::DISTANCE_T* const topk_distances_ptr, // [num_queries, topk] - const typename DATASET_DESCRIPTOR_T::DATA_T* const queries_ptr, // [num_queries, dataset_dim] - const uint32_t num_queries, - const typename DATASET_DESCRIPTOR_T::INDEX_T* dev_seed_ptr, // [num_queries, num_seeds] - uint32_t* const num_executed_iterations, // [num_queries,] - uint32_t topk, - uint32_t num_itopk_candidates, - uint32_t block_size, - uint32_t smem_size, - int64_t hash_bitlen, - typename DATASET_DESCRIPTOR_T::INDEX_T* hashmap_ptr, - size_t small_hash_bitlen, - size_t small_hash_reset_interval, - uint32_t num_random_samplings, - uint64_t rand_xor_mask, - uint32_t num_seeds, - size_t itopk_size, - size_t search_width, - size_t min_iterations, - size_t max_iterations, - SAMPLE_FILTER_T sample_filter, - raft::distance::DistanceType metric, - cudaStream_t stream) RAFT_EXPLICIT; - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_single_cta_select_and_run( \ - TEAM_SIZE, MAX_DATASET_DIM, DATA_T, INDEX_T, DISTANCE_T, SAMPLE_FILTER_T) \ - extern template void select_and_run< \ - TEAM_SIZE, \ - MAX_DATASET_DIM, \ - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, \ - SAMPLE_FILTER_T>( \ - raft::neighbors::cagra::detail::standard_dataset_descriptor_t \ - dataset, \ - raft::device_matrix_view graph, \ - INDEX_T* const topk_indices_ptr, \ - DISTANCE_T* const topk_distances_ptr, \ - const DATA_T* const queries_ptr, \ - const uint32_t num_queries, \ - const INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ - uint32_t topk, \ - uint32_t num_itopk_candidates, \ - uint32_t block_size, \ - uint32_t smem_size, \ - int64_t hash_bitlen, \ - INDEX_T* hashmap_ptr, \ - size_t small_hash_bitlen, \ - size_t small_hash_reset_interval, \ - uint32_t num_random_samplings, \ - uint64_t rand_xor_mask, \ - uint32_t num_seeds, \ - size_t itopk_size, \ - size_t search_width, \ - size_t min_iterations, \ - size_t max_iterations, \ - SAMPLE_FILTER_T sample_filter, \ - raft::distance::DistanceType metric, \ - cudaStream_t stream); - -instantiate_single_cta_select_and_run( - 32, 1024, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 8, 128, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 16, 256, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 32, 512, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 32, 1024, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 8, 128, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 16, 256, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 32, 512, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 32, 1024, int8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 8, 128, int8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 16, 256, int8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 32, 512, int8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 32, 1024, uint8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 8, 128, uint8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 16, 256, uint8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 32, 512, uint8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); - -#undef instantiate_single_cta_select_and_run - -#define instantiate_q_single_cta_select_and_run(TEAM_SIZE, \ - MAX_DATASET_DIM, \ - CODE_BOOK_T, \ - PQ_BITS, \ - PQ_CODE_BOOK_DIM, \ - DATA_T, \ - INDEX_T, \ - DISTANCE_T, \ - SAMPLE_FILTER_T) \ - extern template void \ - select_and_run, \ - SAMPLE_FILTER_T>( \ - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t dataset, \ - raft::device_matrix_view graph, \ - INDEX_T* const topk_indices_ptr, \ - DISTANCE_T* const topk_distances_ptr, \ - const DATA_T* const queries_ptr, \ - const uint32_t num_queries, \ - const INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ - uint32_t topk, \ - uint32_t num_itopk_candidates, \ - uint32_t block_size, \ - uint32_t smem_size, \ - int64_t hash_bitlen, \ - INDEX_T* hashmap_ptr, \ - size_t small_hash_bitlen, \ - size_t small_hash_reset_interval, \ - uint32_t num_random_samplings, \ - uint64_t rand_xor_mask, \ - uint32_t num_seeds, \ - size_t itopk_size, \ - size_t search_width, \ - size_t min_iterations, \ - size_t max_iterations, \ - SAMPLE_FILTER_T sample_filter, \ - raft::distance::DistanceType metric, \ - cudaStream_t stream); - -instantiate_q_single_cta_select_and_run( - 8, 128, half, 8, 2, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 16, 256, half, 8, 2, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 32, 512, half, 8, 2, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 2, - half, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 8, 128, half, 8, 4, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 16, 256, half, 8, 4, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 32, 512, half, 8, 4, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 4, - half, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -instantiate_q_single_cta_select_and_run( - 8, 128, half, 8, 2, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(16, - 256, - half, - 8, - 2, - float, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 512, - half, - 8, - 2, - float, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 2, - float, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 8, 128, half, 8, 4, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(16, - 256, - half, - 8, - 4, - float, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 512, - half, - 8, - 4, - float, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 4, - float, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -instantiate_q_single_cta_select_and_run( - 8, 128, half, 8, 2, half, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 16, 256, half, 8, 2, half, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 32, 512, half, 8, 2, half, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 32, 1024, half, 8, 2, half, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 8, 128, half, 8, 4, half, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 16, 256, half, 8, 4, half, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 32, 512, half, 8, 4, half, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 32, 1024, half, 8, 4, half, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); - -instantiate_q_single_cta_select_and_run( - 8, 128, half, 8, 2, float, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 16, 256, half, 8, 2, float, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 32, 512, half, 8, 2, float, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 2, - float, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 8, 128, half, 8, 4, float, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 16, 256, half, 8, 4, float, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 32, 512, half, 8, 4, float, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 4, - float, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -instantiate_q_single_cta_select_and_run(8, - 128, - half, - 8, - 2, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(16, - 256, - half, - 8, - 2, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 512, - half, - 8, - 2, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 2, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(8, - 128, - half, - 8, - 4, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(16, - 256, - half, - 8, - 4, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 512, - half, - 8, - 4, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 4, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -instantiate_q_single_cta_select_and_run(8, - 128, - half, - 8, - 2, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(16, - 256, - half, - 8, - 2, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 512, - half, - 8, - 2, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 2, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(8, - 128, - half, - 8, - 4, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(16, - 256, - half, - 8, - 4, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 512, - half, - 8, - 4, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 4, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -instantiate_q_single_cta_select_and_run(8, - 128, - half, - 8, - 2, - uint8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(16, - 256, - half, - 8, - 2, - uint8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 512, - half, - 8, - 2, - uint8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 2, - uint8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(8, - 128, - half, - 8, - 4, - uint8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(16, - 256, - half, - 8, - 4, - uint8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 512, - half, - 8, - 4, - uint8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 4, - uint8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -instantiate_q_single_cta_select_and_run( - 8, 128, half, 8, 2, int8_t, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(16, - 256, - half, - 8, - 2, - int8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 512, - half, - 8, - 2, - int8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 2, - int8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 8, 128, half, 8, 4, int8_t, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(16, - 256, - half, - 8, - 4, - int8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 512, - half, - 8, - 4, - int8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 4, - int8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -#undef instantiate_q_single_cta_select_and_run - -} // namespace single_cta_search -} // namespace raft::neighbors::cagra::detail diff --git a/cpp/include/raft/neighbors/detail/cagra/search_single_cta_kernel.cuh b/cpp/include/raft/neighbors/detail/cagra/search_single_cta_kernel.cuh index 1d8fd8e30a..3e72fbf8e8 100644 --- a/cpp/include/raft/neighbors/detail/cagra/search_single_cta_kernel.cuh +++ b/cpp/include/raft/neighbors/detail/cagra/search_single_cta_kernel.cuh @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "search_single_cta_kernel-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "search_single_cta_kernel-ext.cuh" -#endif diff --git a/cpp/include/raft/neighbors/detail/ivf_flat_interleaved_scan-ext.cuh b/cpp/include/raft/neighbors/detail/ivf_flat_interleaved_scan-ext.cuh deleted file mode 100644 index 140a9f17c8..0000000000 --- a/cpp/include/raft/neighbors/detail/ivf_flat_interleaved_scan-ext.cuh +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // raft::neighbors::ivf_flat::index -#include // none_ivf_sample_filter -#include // RAFT_EXPLICIT - -#include // rmm:cuda_stream_view - -#include - -#include // uintX_t - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft::neighbors::ivf_flat::detail { - -auto RAFT_WEAK_FUNCTION is_local_topk_feasible(uint32_t k) -> bool; - -template -void ivfflat_interleaved_scan(const raft::neighbors::ivf_flat::index& index, - const T* queries, - const uint32_t* coarse_query_results, - const uint32_t n_queries, - const uint32_t queries_offset, - const raft::distance::DistanceType metric, - const uint32_t n_probes, - const uint32_t k, - const uint32_t max_samples, - const uint32_t* chunk_indices, - const bool select_min, - IvfSampleFilterT sample_filter, - uint32_t* neighbors, - float* distances, - uint32_t& grid_dim_x, - rmm::cuda_stream_view stream) RAFT_EXPLICIT; - -} // namespace raft::neighbors::ivf_flat::detail - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( \ - T, AccT, IdxT, IvfSampleFilterT) \ - extern template void \ - raft::neighbors::ivf_flat::detail::ivfflat_interleaved_scan( \ - const raft::neighbors::ivf_flat::index& index, \ - const T* queries, \ - const uint32_t* coarse_query_results, \ - const uint32_t n_queries, \ - const uint32_t queries_offset, \ - const raft::distance::DistanceType metric, \ - const uint32_t n_probes, \ - const uint32_t k, \ - const uint32_t max_samples, \ - const uint32_t* chunk_indices, \ - const bool select_min, \ - IvfSampleFilterT sample_filter, \ - uint32_t* neighbors, \ - float* distances, \ - uint32_t& grid_dim_x, \ - rmm::cuda_stream_view stream) - -instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( - float, float, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); -instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( - half, half, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); -instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( - int8_t, int32_t, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); -instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( - uint8_t, uint32_t, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); - -#undef instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan diff --git a/cpp/include/raft/neighbors/detail/ivf_flat_interleaved_scan.cuh b/cpp/include/raft/neighbors/detail/ivf_flat_interleaved_scan.cuh index 63f341dd9a..11d7da851a 100644 --- a/cpp/include/raft/neighbors/detail/ivf_flat_interleaved_scan.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_flat_interleaved_scan.cuh @@ -16,10 +16,4 @@ #pragma once -#if !defined(RAFT_EXPLICIT_INSTANTIATE_ONLY) #include "ivf_flat_interleaved_scan-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "ivf_flat_interleaved_scan-ext.cuh" -#endif diff --git a/cpp/include/raft/neighbors/detail/ivf_flat_search-ext.cuh b/cpp/include/raft/neighbors/detail/ivf_flat_search-ext.cuh deleted file mode 100644 index c14b0e810f..0000000000 --- a/cpp/include/raft/neighbors/detail/ivf_flat_search-ext.cuh +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // raft::neighbors::ivf_flat::index -#include // none_ivf_sample_filter -#include // RAFT_EXPLICIT - -#include - -#include - -#include // uintX_t - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft::neighbors::ivf_flat::detail { - -template -void search(raft::resources const& handle, - const search_params& params, - const raft::neighbors::ivf_flat::index& index, - const T* queries, - uint32_t n_queries, - uint32_t k, - IdxT* neighbors, - float* distances, - rmm::device_async_resource_ref mr, - IvfSampleFilterT sample_filter = IvfSampleFilterT()) RAFT_EXPLICIT; - -} // namespace raft::neighbors::ivf_flat::detail - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_neighbors_ivf_flat_detail_search(T, IdxT, IvfSampleFilterT) \ - extern template void raft::neighbors::ivf_flat::detail::search( \ - raft::resources const& handle, \ - const search_params& params, \ - const raft::neighbors::ivf_flat::index& index, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances, \ - rmm::device_async_resource_ref mr, \ - IvfSampleFilterT sample_filter) - -instantiate_raft_neighbors_ivf_flat_detail_search( - float, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); -instantiate_raft_neighbors_ivf_flat_detail_search( - half, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); -instantiate_raft_neighbors_ivf_flat_detail_search( - int8_t, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); -instantiate_raft_neighbors_ivf_flat_detail_search( - uint8_t, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); - -#undef instantiate_raft_neighbors_ivf_flat_detail_search diff --git a/cpp/include/raft/neighbors/detail/ivf_flat_search.cuh b/cpp/include/raft/neighbors/detail/ivf_flat_search.cuh index 7b03ebeab6..56e58bac27 100644 --- a/cpp/include/raft/neighbors/detail/ivf_flat_search.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_flat_search.cuh @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "ivf_flat_search-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "ivf_flat_search-ext.cuh" -#endif diff --git a/cpp/include/raft/neighbors/detail/ivf_pq_compute_similarity-ext.cuh b/cpp/include/raft/neighbors/detail/ivf_pq_compute_similarity-ext.cuh deleted file mode 100644 index 5e1a9b46d6..0000000000 --- a/cpp/include/raft/neighbors/detail/ivf_pq_compute_similarity-ext.cuh +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // RAFT_WEAK_FUNCTION -#include // raft::distance::DistanceType -#include // raft::neighbors::ivf_pq::detail::fp_8bit -#include // raft::neighbors::ivf_pq::codebook_gen -#include // none_ivf_sample_filter -#include // RAFT_EXPLICIT - -#include // rmm::cuda_stream_view - -#include // __half - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft::neighbors::ivf_pq::detail { - -// is_local_topk_feasible is not inline here, because we would have to define it -// here as well. That would run the risk of the definitions here and in the -// -inl.cuh header diverging. -auto RAFT_WEAK_FUNCTION is_local_topk_feasible(uint32_t k, uint32_t n_probes, uint32_t n_queries) - -> bool; - -template -RAFT_KERNEL compute_similarity_kernel(uint32_t dim, - uint32_t n_probes, - uint32_t pq_dim, - uint32_t n_queries, - uint32_t queries_offset, - distance::DistanceType metric, - codebook_gen codebook_kind, - uint32_t topk, - uint32_t max_samples, - const float* cluster_centers, - const float* pq_centers, - const uint8_t* const* pq_dataset, - const uint32_t* cluster_labels, - const uint32_t* _chunk_indices, - const float* queries, - const uint32_t* index_list, - float* query_kths, - IvfSampleFilterT sample_filter, - LutT* lut_scores, - OutT* _out_scores, - uint32_t* _out_indices) RAFT_EXPLICIT; - -// The signature of the kernel defined by a minimal set of template parameters -template -using compute_similarity_kernel_t = - decltype(&compute_similarity_kernel); - -template -struct selected { - compute_similarity_kernel_t kernel; - dim3 grid_dim; - dim3 block_dim; - size_t smem_size; - size_t device_lut_size; -}; - -template -void compute_similarity_run(selected s, - rmm::cuda_stream_view stream, - uint32_t dim, - uint32_t n_probes, - uint32_t pq_dim, - uint32_t n_queries, - uint32_t queries_offset, - distance::DistanceType metric, - codebook_gen codebook_kind, - uint32_t topk, - uint32_t max_samples, - const float* cluster_centers, - const float* pq_centers, - const uint8_t* const* pq_dataset, - const uint32_t* cluster_labels, - const uint32_t* _chunk_indices, - const float* queries, - const uint32_t* index_list, - float* query_kths, - IvfSampleFilterT sample_filter, - LutT* lut_scores, - OutT* _out_scores, - uint32_t* _out_indices) RAFT_EXPLICIT; - -/** - * Use heuristics to choose an optimal instance of the search kernel. - * It selects among a few kernel variants (with/out using shared mem for - * lookup tables / precomputed distances) and tries to choose the block size - * to maximize kernel occupancy. - * - * @param manage_local_topk - * whether use the fused calculate+select or just calculate the distances for each - * query and probed cluster. - * - * @param locality_hint - * beyond this limit do not consider increasing the number of active blocks per SM - * would improve locality anymore. - */ -template -auto compute_similarity_select(const cudaDeviceProp& dev_props, - bool manage_local_topk, - int locality_hint, - double preferred_shmem_carveout, - uint32_t pq_bits, - uint32_t pq_dim, - uint32_t precomp_data_count, - uint32_t n_queries, - uint32_t n_probes, - uint32_t topk) - -> selected RAFT_EXPLICIT; - -} // namespace raft::neighbors::ivf_pq::detail - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( \ - OutT, LutT, IvfSampleFilterT) \ - extern template auto \ - raft::neighbors::ivf_pq::detail::compute_similarity_select( \ - const cudaDeviceProp& dev_props, \ - bool manage_local_topk, \ - int locality_hint, \ - double preferred_shmem_carveout, \ - uint32_t pq_bits, \ - uint32_t pq_dim, \ - uint32_t precomp_data_count, \ - uint32_t n_queries, \ - uint32_t n_probes, \ - uint32_t topk) \ - ->raft::neighbors::ivf_pq::detail::selected; \ - \ - extern template void \ - raft::neighbors::ivf_pq::detail::compute_similarity_run( \ - raft::neighbors::ivf_pq::detail::selected s, \ - rmm::cuda_stream_view stream, \ - uint32_t dim, \ - uint32_t n_probes, \ - uint32_t pq_dim, \ - uint32_t n_queries, \ - uint32_t queries_offset, \ - raft::distance::DistanceType metric, \ - raft::neighbors::ivf_pq::codebook_gen codebook_kind, \ - uint32_t topk, \ - uint32_t max_samples, \ - const float* cluster_centers, \ - const float* pq_centers, \ - const uint8_t* const* pq_dataset, \ - const uint32_t* cluster_labels, \ - const uint32_t* _chunk_indices, \ - const float* queries, \ - const uint32_t* index_list, \ - float* query_kths, \ - IvfSampleFilterT sample_filter, \ - LutT* lut_scores, \ - OutT* _out_scores, \ - uint32_t* _out_indices); - -#define COMMA , -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - float, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); - -#undef COMMA - -#undef instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select diff --git a/cpp/include/raft/neighbors/detail/ivf_pq_compute_similarity.cuh b/cpp/include/raft/neighbors/detail/ivf_pq_compute_similarity.cuh index d987c0d4ed..467d389d38 100644 --- a/cpp/include/raft/neighbors/detail/ivf_pq_compute_similarity.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_pq_compute_similarity.cuh @@ -16,10 +16,4 @@ #pragma once -#if !defined(RAFT_EXPLICIT_INSTANTIATE_ONLY) #include "ivf_pq_compute_similarity-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "ivf_pq_compute_similarity-ext.cuh" -#endif diff --git a/cpp/include/raft/sparse/matrix/detail/select_k-ext.cuh b/cpp/include/raft/sparse/matrix/detail/select_k-ext.cuh deleted file mode 100644 index 01625a0ce8..0000000000 --- a/cpp/include/raft/sparse/matrix/detail/select_k-ext.cuh +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include // RAFT_EXPLICIT - -#include // __half - -#include // uint32_t - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft::sparse::matrix::detail { - -template -void select_k(raft::resources const& handle, - raft::device_csr_matrix_view in_val, - std::optional> in_idx, - raft::device_matrix_view out_val, - raft::device_matrix_view out_idx, - bool select_min, - bool sorted = false, - raft::matrix::SelectAlgo algo = raft::matrix::SelectAlgo::kAuto) RAFT_EXPLICIT; -} // namespace raft::sparse::matrix::detail - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_sparse_matrix_detail_select_k(T, IdxT) \ - extern template void raft::sparse::matrix::detail::select_k( \ - raft::resources const& handle, \ - raft::device_csr_matrix_view in_val, \ - std::optional> in_idx, \ - raft::device_matrix_view out_val, \ - raft::device_matrix_view out_idx, \ - bool select_min, \ - bool sorted, \ - raft::matrix::SelectAlgo algo) - -instantiate_raft_sparse_matrix_detail_select_k(__half, uint32_t); -instantiate_raft_sparse_matrix_detail_select_k(__half, int64_t); -instantiate_raft_sparse_matrix_detail_select_k(float, int64_t); -instantiate_raft_sparse_matrix_detail_select_k(float, uint32_t); -instantiate_raft_sparse_matrix_detail_select_k(float, int); -instantiate_raft_sparse_matrix_detail_select_k(double, int64_t); -instantiate_raft_sparse_matrix_detail_select_k(double, uint32_t); - -#undef instantiate_raft_sparse_matrix_detail_select_k diff --git a/cpp/include/raft/sparse/matrix/detail/select_k.cuh b/cpp/include/raft/sparse/matrix/detail/select_k.cuh index 5d52b94b2f..31a4b54a94 100644 --- a/cpp/include/raft/sparse/matrix/detail/select_k.cuh +++ b/cpp/include/raft/sparse/matrix/detail/select_k.cuh @@ -15,11 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "select_k-inl.cuh" - -#endif - -#ifdef RAFT_COMPILED -#include "select_k-ext.cuh" -#endif From d8dc72cae177f600aa4ea4430c352e3b10286b4a Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 19 Nov 2024 10:45:04 -0600 Subject: [PATCH 55/79] Remove leftover template project code. (#2500) Follow-up to https://github.com/rapidsai/raft/pull/2498#issuecomment-2480666296. This removes an extraneous file in `cpp/template` that resulted from an incorrect merge conflict resolution and a leftover reference to the template project in `update-version.sh`. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - Corey J. Nolet (https://github.com/cjnolet) - Kyle Edwards (https://github.com/KyleFromNVIDIA) URL: https://github.com/rapidsai/raft/pull/2500 --- ci/release/update-version.sh | 1 - cpp/template/cmake/thirdparty/get_raft.cmake | 67 -------------------- 2 files changed, 68 deletions(-) delete mode 100644 cpp/template/cmake/thirdparty/get_raft.cmake diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index a118becb4b..a70fed9ec8 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -35,7 +35,6 @@ function sed_runner() { sed -i.bak ''"$1"'' $2 && rm -f ${2}.bak } -sed_runner "s/set(RAPIDS_VERSION .*)/set(RAPIDS_VERSION \"${NEXT_SHORT_TAG}\")/g" cpp/template/cmake/thirdparty/fetch_rapids.cmake sed_runner 's/'"find_and_configure_ucxx(VERSION .*"'/'"find_and_configure_ucxx(VERSION ${NEXT_UCXX_SHORT_TAG_PEP440}"'/g' python/raft-dask/cmake/thirdparty/get_ucxx.cmake sed_runner 's/'"branch-.*"'/'"branch-${NEXT_UCXX_SHORT_TAG_PEP440}"'/g' python/raft-dask/cmake/thirdparty/get_ucxx.cmake diff --git a/cpp/template/cmake/thirdparty/get_raft.cmake b/cpp/template/cmake/thirdparty/get_raft.cmake deleted file mode 100644 index 4474fd2875..0000000000 --- a/cpp/template/cmake/thirdparty/get_raft.cmake +++ /dev/null @@ -1,67 +0,0 @@ -# ============================================================================= -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. - -# Use RAPIDS_VERSION from cmake/thirdparty/fetch_rapids.cmake -set(RAFT_VERSION "${RAPIDS_VERSION}") -set(RAFT_FORK "rapidsai") -set(RAFT_PINNED_TAG "branch-${RAPIDS_VERSION}") - -function(find_and_configure_raft) - set(oneValueArgs VERSION FORK PINNED_TAG COMPILE_LIBRARY ENABLE_NVTX ENABLE_MNMG_DEPENDENCIES) - cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" - "${multiValueArgs}" ${ARGN} ) - - set(RAFT_COMPONENTS "") - if(PKG_COMPILE_LIBRARY) - string(APPEND RAFT_COMPONENTS " compiled") - endif() - - if(PKG_ENABLE_MNMG_DEPENDENCIES) - string(APPEND RAFT_COMPONENTS " distributed") - endif() - - #----------------------------------------------------- - # Invoke CPM find_package() - #----------------------------------------------------- - # Since the RAFT_NVTX option is used by targets generated by - # find_package(RAFT_NVTX) and when building from source we want to - # make `RAFT_NVTX` a cache variable so we get consistent - # behavior - # - set(RAFT_NVTX ${PKG_ENABLE_NVTX} CACHE BOOL "Enable raft nvtx logging" FORCE) - rapids_cpm_find(raft ${PKG_VERSION} - GLOBAL_TARGETS raft::raft - BUILD_EXPORT_SET raft-template-exports - INSTALL_EXPORT_SET raft-template-exports - COMPONENTS ${RAFT_COMPONENTS} - CPM_ARGS - GIT_REPOSITORY https://github.com/${PKG_FORK}/raft.git - GIT_TAG ${PKG_PINNED_TAG} - SOURCE_SUBDIR cpp - OPTIONS - "BUILD_TESTS OFF" - "BUILD_PRIMS_BENCH OFF" - "RAFT_COMPILE_LIBRARY ${PKG_COMPILE_LIBRARY}" - ) -endfunction() - -# Change pinned tag here to test a commit in CI -# To use a different RAFT locally, set the CMake variable -# CPM_raft_SOURCE=/path/to/local/raft -find_and_configure_raft(VERSION ${RAFT_VERSION}.00 - FORK ${RAFT_FORK} - PINNED_TAG ${RAFT_PINNED_TAG} - COMPILE_LIBRARY ON - ENABLE_MNMG_DEPENDENCIES OFF - ENABLE_NVTX OFF -) From 64c09d7b7012aa3e0194d16e60ef1d2e680f3df9 Mon Sep 17 00:00:00 2001 From: Micka Date: Wed, 20 Nov 2024 23:27:49 +0100 Subject: [PATCH 56/79] [FEA] Lanczos solver v2 (#2481) I unfortunately don't have permissions to push on @aamijar branch for the previous Lanczos solver PR (#2416) so I kept his commits and continued it here. ## Lanczos Solver for Sparse Eigen Decomposition We propose a new lanczos solver in raft that fixes the issues present in the previous solver `raft::sparse::solver::detail::computeSmallestEigenvectors`. Specifically we address the following issues: 1. Numerical Stability for both float32 and float64 datatypes 2. Efficiency and Speed of Convergence This new implementation is taken from the cupy library `cupyx.scipy.sparse.linalg.eigsh` where the thick-restart and full reorthogonalzation methods are used. Additionally this PR exposes a python api for raft lanczos solver with an interface similar to `scipy.sparse.linalg.eigsh` and `cupyx.scipy.sparse.linalg.eigsh`. ```py3 from pylibraft.solver import eigsh ``` Authors: - Micka (https://github.com/lowener) - Anupam (https://github.com/aamijar) - Corey J. Nolet (https://github.com/cjnolet) Approvers: - Kyle Edwards (https://github.com/KyleFromNVIDIA) - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/raft/pull/2481 --- cpp/CMakeLists.txt | 4 + .../sparse/linalg/detail/cusparse_utils.hpp | 20 +- .../raft/sparse/solver/detail/lanczos.cuh | 711 ++++++++++++++++++ cpp/include/raft/sparse/solver/lanczos.cuh | 75 +- .../raft/sparse/solver/lanczos_types.hpp | 37 + cpp/include/raft/spectral/eigen_solvers.cuh | 39 +- cpp/include/raft_runtime/solver/lanczos.hpp | 52 ++ .../raft_runtime/solver/lanczos_solver.cuh | 32 + .../solver/lanczos_solver_int64_double.cu | 23 + .../solver/lanczos_solver_int64_float.cu | 23 + .../solver/lanczos_solver_int_double.cu | 23 + .../solver/lanczos_solver_int_float.cu | 23 + cpp/test/CMakeLists.txt | 4 +- cpp/test/sparse/solver/lanczos.cu | 445 +++++++++++ docs/source/pylibraft_api.rst | 1 + docs/source/pylibraft_api/sparse.rst | 11 + python/pylibraft/CMakeLists.txt | 1 + .../pylibraft/pylibraft/sparse/CMakeLists.txt | 15 + python/pylibraft/pylibraft/sparse/__init__.py | 18 + .../pylibraft/sparse/linalg/CMakeLists.txt | 27 + .../pylibraft/sparse/linalg/__init__.pxd | 0 .../pylibraft/sparse/linalg/__init__.py | 18 + .../pylibraft/sparse/linalg/cpp/__init__.pxd | 0 .../pylibraft/sparse/linalg/cpp/__init__.py | 0 .../pylibraft/sparse/linalg/lanczos.pyx | 277 +++++++ .../pylibraft/pylibraft/test/test_sparse.py | 142 ++++ 26 files changed, 2005 insertions(+), 16 deletions(-) create mode 100644 cpp/include/raft/sparse/solver/lanczos_types.hpp create mode 100644 cpp/include/raft_runtime/solver/lanczos.hpp create mode 100644 cpp/src/raft_runtime/solver/lanczos_solver.cuh create mode 100644 cpp/src/raft_runtime/solver/lanczos_solver_int64_double.cu create mode 100644 cpp/src/raft_runtime/solver/lanczos_solver_int64_float.cu create mode 100644 cpp/src/raft_runtime/solver/lanczos_solver_int_double.cu create mode 100644 cpp/src/raft_runtime/solver/lanczos_solver_int_float.cu create mode 100644 cpp/test/sparse/solver/lanczos.cu create mode 100644 docs/source/pylibraft_api/sparse.rst create mode 100644 python/pylibraft/pylibraft/sparse/CMakeLists.txt create mode 100644 python/pylibraft/pylibraft/sparse/__init__.py create mode 100644 python/pylibraft/pylibraft/sparse/linalg/CMakeLists.txt create mode 100644 python/pylibraft/pylibraft/sparse/linalg/__init__.pxd create mode 100644 python/pylibraft/pylibraft/sparse/linalg/__init__.py create mode 100644 python/pylibraft/pylibraft/sparse/linalg/cpp/__init__.pxd create mode 100644 python/pylibraft/pylibraft/sparse/linalg/cpp/__init__.py create mode 100644 python/pylibraft/pylibraft/sparse/linalg/lanczos.pyx create mode 100644 python/pylibraft/pylibraft/test/test_sparse.py diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 4ed9529a36..780f6f8581 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -268,6 +268,10 @@ if(RAFT_COMPILE_LIBRARY) src/raft_runtime/random/rmat_rectangular_generator_int64_float.cu src/raft_runtime/random/rmat_rectangular_generator_int_double.cu src/raft_runtime/random/rmat_rectangular_generator_int_float.cu + src/raft_runtime/solver/lanczos_solver_int64_double.cu + src/raft_runtime/solver/lanczos_solver_int64_float.cu + src/raft_runtime/solver/lanczos_solver_int_double.cu + src/raft_runtime/solver/lanczos_solver_int_float.cu ) set_target_properties( raft_objs diff --git a/cpp/include/raft/sparse/linalg/detail/cusparse_utils.hpp b/cpp/include/raft/sparse/linalg/detail/cusparse_utils.hpp index c10c0de426..97ac7c45f4 100644 --- a/cpp/include/raft/sparse/linalg/detail/cusparse_utils.hpp +++ b/cpp/include/raft/sparse/linalg/detail/cusparse_utils.hpp @@ -30,7 +30,25 @@ namespace linalg { namespace detail { /** - * @brief create a cuSparse dense descriptor + * @brief create a cuSparse dense descriptor for a vector + * @tparam ValueType Data type of vector_view (float/double) + * @tparam IndexType Type of vector_view + * @param[in] vector_view input raft::device_vector_view + * @returns dense vector descriptor to be used by cuSparse API + */ +template +cusparseDnVecDescr_t create_descriptor(raft::device_vector_view vector_view) +{ + cusparseDnVecDescr_t descr; + RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsecreatednvec( + &descr, + vector_view.extent(0), + const_cast*>(vector_view.data_handle()))); + return descr; +} + +/** + * @brief create a cuSparse dense descriptor for a matrix * @tparam ValueType Data type of dense_view (float/double) * @tparam IndexType Type of dense_view * @tparam LayoutPolicy layout of dense_view diff --git a/cpp/include/raft/sparse/solver/detail/lanczos.cuh b/cpp/include/raft/sparse/solver/detail/lanczos.cuh index 9ecb4b729f..02a77a0d99 100644 --- a/cpp/include/raft/sparse/solver/detail/lanczos.cuh +++ b/cpp/include/raft/sparse/solver/detail/lanczos.cuh @@ -19,10 +19,43 @@ // for cmath: #define _USE_MATH_DEFINES +#include +#include +#include +#include +#include +#include +#include #include #include #include +#include +#include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include @@ -30,9 +63,17 @@ #include +#include #include +#include +#include +#include #include +#include +#include +#include +#include #include namespace raft::sparse::solver::detail { @@ -1396,4 +1437,674 @@ int computeLargestEigenvectors( return status; } +template +RAFT_KERNEL kernel_triangular_populate(T* M, const T* beta, int n) +{ + int row = blockIdx.x * blockDim.x + threadIdx.x; + + if (row < n) { + // Upper diagonal: M[row + 1, row] in column-major + if (row < n - 1) { M[(row + 1) * n + row] = beta[row]; } + + // Lower diagonal: M[row - 1, row] in column-major + if (row > 0) { M[(row - 1) * n + row] = beta[row - 1]; } + } +} + +template +RAFT_KERNEL kernel_triangular_beta_k(T* t, const T* beta_k, int k, int n) +{ + int tid = threadIdx.x + blockIdx.x * blockDim.x; + + if (tid < k) { + // Update the k-th column: t[i, k] -> t[k * n + i] in column-major + t[tid * n + k] = beta_k[tid]; + + // Update the k-th row: t[k, j] -> t[j * n + k] in column-major + t[k * n + tid] = beta_k[tid]; + } +} + +template +RAFT_KERNEL kernel_normalize(const T* u, const T* beta, int j, int n, T* v, T* V, int size) +{ + int i = blockIdx.x * blockDim.x + threadIdx.x; + + if (i < size) { + if (beta[j] == 0) { + v[i] = u[i] / 1; + } else { + v[i] = u[i] / beta[j]; + } + V[i + (j + 1) * n] = v[i]; + } +} + +template +RAFT_KERNEL kernel_clamp_down(T* value, T threshold) +{ + *value = (fabs(*value) < threshold) ? 0 : *value; +} + +template +RAFT_KERNEL kernel_clamp_down_vector(T* vec, T threshold, int size) +{ + int idx = threadIdx.x + blockIdx.x * blockDim.x; + if (idx < size) { vec[idx] = (fabs(vec[idx]) < threshold) ? 0 : vec[idx]; } +} + +template +void lanczos_solve_ritz( + raft::resources const& handle, + raft::device_matrix_view alpha, + raft::device_matrix_view beta, + std::optional> beta_k, + IndexTypeT k, + int which, + int ncv, + raft::device_matrix_view eigenvectors, + raft::device_vector_view eigenvalues) +{ + auto stream = resource::get_cuda_stream(handle); + + ValueTypeT zero = 0; + auto triangular_matrix = + raft::make_device_matrix(handle, ncv, ncv); + raft::matrix::fill(handle, triangular_matrix.view(), zero); + + raft::device_vector_view alphaVec = + raft::make_device_vector_view(alpha.data_handle(), ncv); + raft::matrix::set_diagonal(handle, alphaVec, triangular_matrix.view()); + + // raft::matrix::initializeDiagonalMatrix( + // alpha.data_handle(), triangular_matrix.data_handle(), ncv, ncv, stream); + + int blockSize = 256; + int numBlocks = (ncv + blockSize - 1) / blockSize; + kernel_triangular_populate + <<>>(triangular_matrix.data_handle(), beta.data_handle(), ncv); + + if (beta_k) { + int threadsPerBlock = 256; + int blocksPerGrid = (k + threadsPerBlock - 1) / threadsPerBlock; + kernel_triangular_beta_k<<>>( + triangular_matrix.data_handle(), beta_k.value().data_handle(), (int)k, ncv); + } + + auto triangular_matrix_view = + raft::make_device_matrix_view( + triangular_matrix.data_handle(), ncv, ncv); + + raft::linalg::eig_dc(handle, triangular_matrix_view, eigenvectors, eigenvalues); +} + +template +void lanczos_aux(raft::resources const& handle, + raft::device_csr_matrix_view A, + raft::device_matrix_view V, + raft::device_matrix_view u, + raft::device_matrix_view alpha, + raft::device_matrix_view beta, + int start_idx, + int end_idx, + int ncv, + raft::device_matrix_view v, + raft::device_matrix_view uu, + raft::device_matrix_view vv) +{ + auto stream = resource::get_cuda_stream(handle); + + IndexTypeT n = A.structure_view().get_n_rows(); + auto v_vector = raft::make_device_vector_view(v.data_handle(), n); + auto u_vector = raft::make_device_vector_view(u.data_handle(), n); + + raft::copy( + v.data_handle(), V.data_handle() + start_idx * V.stride(0), n, stream); // V(start_idx, 0) + + auto cusparse_h = resource::get_cusparse_handle(handle); + cusparseSpMatDescr_t cusparse_A = raft::sparse::linalg::detail::create_descriptor(A); + + cusparseDnVecDescr_t cusparse_v = raft::sparse::linalg::detail::create_descriptor(v_vector); + cusparseDnVecDescr_t cusparse_u = raft::sparse::linalg::detail::create_descriptor(u_vector); + + ValueTypeT one = 1; + ValueTypeT zero = 0; + size_t bufferSize; + raft::sparse::detail::cusparsespmv_buffersize(cusparse_h, + CUSPARSE_OPERATION_NON_TRANSPOSE, + &one, + cusparse_A, + cusparse_v, + &zero, + cusparse_u, + CUSPARSE_SPMV_ALG_DEFAULT, + &bufferSize, + stream); + auto cusparse_spmv_buffer = raft::make_device_vector(handle, bufferSize); + + for (int i = start_idx; i < end_idx; i++) { + raft::sparse::detail::cusparsespmv(cusparse_h, + CUSPARSE_OPERATION_NON_TRANSPOSE, + &one, + cusparse_A, + cusparse_v, + &zero, + cusparse_u, + CUSPARSE_SPMV_ALG_DEFAULT, + cusparse_spmv_buffer.data_handle(), + stream); + + auto alpha_i = + raft::make_device_scalar_view(alpha.data_handle() + i * alpha.stride(1)); // alpha(0, i) + raft::linalg::dot(handle, v_vector, u_vector, alpha_i); + + raft::matrix::fill(handle, vv, zero); + + auto cublas_h = resource::get_cublas_handle(handle); + + ValueTypeT alpha_i_host = 0; + ValueTypeT b = 0; + ValueTypeT mone = -1; + + raft::copy( + &b, beta.data_handle() + ((i - 1 + ncv) % ncv) * beta.stride(1), 1, stream); + raft::copy( + &alpha_i_host, alpha.data_handle() + i * alpha.stride(1), 1, stream); // alpha(0, i) + + raft::linalg::axpy(handle, n, &alpha_i_host, v.data_handle(), 1, vv.data_handle(), 1, stream); + raft::linalg::axpy(handle, + n, + &b, + V.data_handle() + (((i - 1 + ncv) % ncv) * V.stride(0)), + 1, + vv.data_handle(), + 1, + stream); + raft::linalg::axpy(handle, n, &mone, vv.data_handle(), 1, u.data_handle(), 1, stream); + + raft::linalg::gemv(handle, + CUBLAS_OP_T, + n, + i + 1, + &one, + V.data_handle(), + n, + u.data_handle(), + 1, + &zero, + uu.data_handle(), + 1, + stream); + + raft::linalg::gemv(handle, + CUBLAS_OP_N, + n, + i + 1, + &mone, + V.data_handle(), + n, + uu.data_handle(), + 1, + &one, + u.data_handle(), + 1, + stream); + + auto uu_i = raft::make_device_scalar_view(uu.data_handle() + uu.stride(1) * i); // uu(0, i) + raft::linalg::add(handle, make_const_mdspan(alpha_i), make_const_mdspan(uu_i), alpha_i); + + kernel_clamp_down<<<1, 1, 0, stream>>>(alpha_i.data_handle(), static_cast(1e-9)); + + auto output = raft::make_device_vector_view( + beta.data_handle() + beta.stride(1) * i, 1); + auto input = raft::make_device_matrix_view(u.data_handle(), 1, n); + raft::linalg::norm(handle, + input, + output, + raft::linalg::L2Norm, + raft::linalg::Apply::ALONG_ROWS, + raft::sqrt_op()); + + int blockSize = 256; + int numBlocks = (n + blockSize - 1) / blockSize; + + kernel_clamp_down_vector<<>>( + u.data_handle(), static_cast(1e-7), n); + + kernel_clamp_down<<<1, 1, 0, stream>>>(beta.data_handle() + beta.stride(1) * i, + static_cast(1e-6)); + + if (i >= end_idx - 1) { break; } + + int threadsPerBlock = 256; + int blocksPerGrid = (n + threadsPerBlock - 1) / threadsPerBlock; + + kernel_normalize<<>>( + u.data_handle(), beta.data_handle(), i, n, v.data_handle(), V.data_handle(), n); + } +} + +template +auto lanczos_smallest( + raft::resources const& handle, + raft::device_csr_matrix_view A, + int nEigVecs, + int maxIter, + int restartIter, + ValueTypeT tol, + ValueTypeT* eigVals_dev, + ValueTypeT* eigVecs_dev, + ValueTypeT* v0, + uint64_t seed) -> int +{ + int n = A.structure_view().get_n_rows(); + int ncv = restartIter; + auto stream = resource::get_cuda_stream(handle); + + auto V = raft::make_device_matrix(handle, ncv, n); + auto V_0_view = + raft::make_device_matrix_view(V.data_handle(), 1, n); // First Row V[0] + auto v0_view = raft::make_device_matrix_view(v0, 1, n); + + auto u = raft::make_device_matrix(handle, 1, n); + auto u_vector = raft::make_device_vector_view(u.data_handle(), n); + raft::copy(u.data_handle(), v0, n, stream); + + auto cublas_h = resource::get_cublas_handle(handle); + auto v0nrm = raft::make_device_vector(handle, 1); + raft::linalg::norm(handle, + v0_view, + v0nrm.view(), + raft::linalg::L2Norm, + raft::linalg::Apply::ALONG_ROWS, + raft::sqrt_op()); + + auto v0_vector_const = raft::make_device_vector_view(v0, n); + + raft::linalg::unary_op( + handle, v0_vector_const, V_0_view, [device_scalar = v0nrm.data_handle()] __device__(auto y) { + return y / *device_scalar; + }); + + auto alpha = raft::make_device_matrix(handle, 1, ncv); + auto beta = raft::make_device_matrix(handle, 1, ncv); + ValueTypeT zero = 0; + raft::matrix::fill(handle, alpha.view(), zero); + raft::matrix::fill(handle, beta.view(), zero); + + auto v = raft::make_device_matrix(handle, 1, n); + auto aux_uu = raft::make_device_matrix(handle, 1, ncv); + auto vv = raft::make_device_matrix(handle, 1, n); + + lanczos_aux(handle, + A, + V.view(), + u.view(), + alpha.view(), + beta.view(), + 0, + ncv, + ncv, + v.view(), + aux_uu.view(), + vv.view()); + + auto eigenvectors = + raft::make_device_matrix(handle, ncv, ncv); + auto eigenvalues = raft::make_device_vector(handle, ncv); + + lanczos_solve_ritz(handle, + alpha.view(), + beta.view(), + std::nullopt, + nEigVecs, + 0, + ncv, + eigenvectors.view(), + eigenvalues.view()); + + auto eigenvectors_k = raft::make_device_matrix_view( + eigenvectors.data_handle(), ncv, nEigVecs); + auto eigenvalues_k = + raft::make_device_vector_view(eigenvalues.data_handle(), nEigVecs); + + auto ritz_eigenvectors = + raft::make_device_matrix_view(eigVecs_dev, n, nEigVecs); + + auto V_T = + raft::make_device_matrix_view(V.data_handle(), n, ncv); + raft::linalg::gemm( + handle, V_T, eigenvectors_k, ritz_eigenvectors); + + auto s = raft::make_device_vector(handle, nEigVecs); + + auto eigenvectors_k_slice = + raft::make_device_matrix_view( + eigenvectors.data_handle(), ncv, nEigVecs); + auto S_matrix = raft::make_device_matrix_view( + s.data_handle(), 1, nEigVecs); + + raft::matrix::slice_coordinates coords(ncv - 1, 0, ncv, nEigVecs); + raft::matrix::slice(handle, make_const_mdspan(eigenvectors_k_slice), S_matrix, coords); + + auto beta_k = raft::make_device_vector(handle, nEigVecs); + raft::matrix::fill(handle, beta_k.view(), zero); + auto beta_scalar = raft::make_device_scalar_view(beta.data_handle() + + (ncv - 1) * beta.stride(1)); + + raft::linalg::axpy(handle, beta_scalar, raft::make_const_mdspan(s.view()), beta_k.view()); + + ValueTypeT res = 0; + + raft::device_vector output = + raft::make_device_vector(handle, 1); + raft::device_matrix_view input = + raft::make_device_matrix_view(beta_k.data_handle(), 1, nEigVecs); + raft::linalg::norm(handle, + input, + output.view(), + raft::linalg::L2Norm, + raft::linalg::Apply::ALONG_ROWS, + raft::sqrt_op()); + raft::copy(&res, output.data_handle(), 1, stream); + resource::sync_stream(handle, stream); + + auto uu = raft::make_device_matrix(handle, 0, nEigVecs); + int iter = ncv; + while (res > tol && iter < maxIter) { + auto beta_view = raft::make_device_matrix_view( + beta.data_handle(), 1, nEigVecs); + raft::matrix::fill(handle, beta_view, zero); + + raft::copy(alpha.data_handle(), eigenvalues_k.data_handle(), nEigVecs, stream); + + auto x_T = + raft::make_device_matrix_view(ritz_eigenvectors.data_handle(), nEigVecs, n); + + raft::copy(V.data_handle(), x_T.data_handle(), nEigVecs * n, stream); + + ValueTypeT one = 1; + ValueTypeT mone = -1; + + // Using raft::linalg::gemv leads to Reason=7:CUBLAS_STATUS_INVALID_VALUE (issue raft#2484) + raft::linalg::detail::cublasgemv(cublas_h, + CUBLAS_OP_T, + nEigVecs, + n, + &one, + V.data_handle(), + nEigVecs, + u.data_handle(), + 1, + &zero, + uu.data_handle(), + 1, + stream); + + raft::linalg::detail::cublasgemv(cublas_h, + CUBLAS_OP_N, + nEigVecs, + n, + &mone, + V.data_handle(), + nEigVecs, + uu.data_handle(), + 1, + &one, + u.data_handle(), + 1, + stream); + + auto V_0_view = + raft::make_device_matrix_view(V.data_handle() + (nEigVecs * n), 1, n); + auto V_0_view_vector = + raft::make_device_vector_view(V_0_view.data_handle(), n); + auto unrm = raft::make_device_vector(handle, 1); + raft::linalg::norm(handle, + raft::make_const_mdspan(u.view()), + unrm.view(), + raft::linalg::L2Norm, + raft::linalg::Apply::ALONG_ROWS, + raft::sqrt_op()); + + raft::linalg::unary_op( + handle, + raft::make_const_mdspan(u_vector), + V_0_view, + [device_scalar = unrm.data_handle()] __device__(auto y) { return y / *device_scalar; }); + + auto cusparse_h = resource::get_cusparse_handle(handle); + cusparseSpMatDescr_t cusparse_A = raft::sparse::linalg::detail::create_descriptor(A); + + cusparseDnVecDescr_t cusparse_v = + raft::sparse::linalg::detail::create_descriptor(V_0_view_vector); + cusparseDnVecDescr_t cusparse_u = raft::sparse::linalg::detail::create_descriptor(u_vector); + + ValueTypeT zero = 0; + size_t bufferSize; + raft::sparse::detail::cusparsespmv_buffersize(cusparse_h, + CUSPARSE_OPERATION_NON_TRANSPOSE, + &one, + cusparse_A, + cusparse_v, + &zero, + cusparse_u, + CUSPARSE_SPMV_ALG_DEFAULT, + &bufferSize, + stream); + auto cusparse_spmv_buffer = raft::make_device_vector(handle, bufferSize); + + raft::sparse::detail::cusparsespmv(cusparse_h, + CUSPARSE_OPERATION_NON_TRANSPOSE, + &one, + cusparse_A, + cusparse_v, + &zero, + cusparse_u, + CUSPARSE_SPMV_ALG_DEFAULT, + cusparse_spmv_buffer.data_handle(), + stream); + + auto alpha_k = raft::make_device_scalar_view(alpha.data_handle() + nEigVecs); + + raft::linalg::dot( + handle, make_const_mdspan(V_0_view_vector), make_const_mdspan(u_vector), alpha_k); + + raft::linalg::binary_op(handle, + make_const_mdspan(u_vector), + make_const_mdspan(V_0_view_vector), + u_vector, + [device_scalar_ptr = alpha_k.data_handle()] __device__( + ValueTypeT u_element, ValueTypeT V_0_element) { + return u_element - (*device_scalar_ptr) * V_0_element; + }); + + auto temp = raft::make_device_vector(handle, n); + + auto V_k = raft::make_device_matrix_view( + V.data_handle(), nEigVecs, n); + auto V_k_T = + raft::make_device_matrix(handle, n, nEigVecs); + + raft::linalg::transpose(handle, V_k, V_k_T.view()); + + ValueTypeT three = 3; + ValueTypeT two = 2; + + std::vector M = {1, 2, 3, 4, 5, 6}; + std::vector vec = {1, 1}; + + auto M_dev = raft::make_device_matrix(handle, 2, 3); + auto vec_dev = raft::make_device_vector(handle, 2); + auto out = raft::make_device_vector(handle, 3); + raft::copy(M_dev.data_handle(), M.data(), 6, stream); + raft::copy(vec_dev.data_handle(), vec.data(), 2, stream); + + raft::linalg::gemv(handle, + CUBLAS_OP_N, + three, + two, + &one, + M_dev.data_handle(), + three, + vec_dev.data_handle(), + 1, + &zero, + out.data_handle(), + 1, + stream); + + raft::linalg::gemv(handle, + CUBLAS_OP_N, + n, + nEigVecs, + &one, + V_k.data_handle(), + n, + beta_k.data_handle(), + 1, + &zero, + temp.data_handle(), + 1, + stream); + + auto one_scalar = raft::make_device_scalar(handle, 1); + raft::linalg::binary_op(handle, + make_const_mdspan(u_vector), + make_const_mdspan(temp.view()), + u_vector, + [device_scalar_ptr = one_scalar.data_handle()] __device__( + ValueTypeT u_element, ValueTypeT temp_element) { + return u_element - (*device_scalar_ptr) * temp_element; + }); + + auto output1 = raft::make_device_vector_view( + beta.data_handle() + beta.stride(1) * nEigVecs, 1); + raft::linalg::norm(handle, + raft::make_const_mdspan(u.view()), + output1, + raft::linalg::L2Norm, + raft::linalg::Apply::ALONG_ROWS, + raft::sqrt_op()); + + auto V_kplus1 = + raft::make_device_vector_view(V.data_handle() + V.stride(0) * (nEigVecs + 1), n); + + raft::linalg::unary_op( + handle, + make_const_mdspan(u_vector), + V_kplus1, + [device_scalar = (beta.data_handle() + beta.stride(1) * nEigVecs)] __device__(auto y) { + return y / *device_scalar; + }); + + lanczos_aux(handle, + A, + V.view(), + u.view(), + alpha.view(), + beta.view(), + nEigVecs + 1, + ncv, + ncv, + v.view(), + aux_uu.view(), + vv.view()); + iter += ncv - nEigVecs; + lanczos_solve_ritz(handle, + alpha.view(), + beta.view(), + beta_k.view(), + nEigVecs, + 0, + ncv, + eigenvectors.view(), + eigenvalues.view()); + auto eigenvectors_k = raft::make_device_matrix_view( + eigenvectors.data_handle(), ncv, nEigVecs); + + auto ritz_eigenvectors = raft::make_device_matrix_view( + eigVecs_dev, n, nEigVecs); + + auto V_T = + raft::make_device_matrix_view(V.data_handle(), n, ncv); + raft::linalg::gemm( + handle, V_T, eigenvectors_k, ritz_eigenvectors); + + auto eigenvectors_k_slice = + raft::make_device_matrix_view( + eigenvectors.data_handle(), ncv, nEigVecs); + auto S_matrix = raft::make_device_matrix_view( + s.data_handle(), 1, nEigVecs); + + raft::matrix::slice_coordinates coords(ncv - 1, 0, ncv, nEigVecs); + raft::matrix::slice(handle, make_const_mdspan(eigenvectors_k_slice), S_matrix, coords); + + raft::matrix::fill(handle, beta_k.view(), zero); + + auto beta_scalar = raft::make_device_scalar_view( + beta.data_handle() + beta.stride(1) * (ncv - 1)); // &((beta.view())(0, ncv - 1)) + + raft::linalg::axpy(handle, beta_scalar, raft::make_const_mdspan(s.view()), beta_k.view()); + + raft::device_vector output2 = + raft::make_device_vector(handle, 1); + raft::device_matrix_view input2 = + raft::make_device_matrix_view(beta_k.data_handle(), 1, nEigVecs); + raft::linalg::norm(handle, + input2, + output2.view(), + raft::linalg::L2Norm, + raft::linalg::Apply::ALONG_ROWS, + raft::sqrt_op()); + raft::copy(&res, output2.data_handle(), 1, stream); + resource::sync_stream(handle, stream); + RAFT_LOG_TRACE("Iteration %f: residual (tolerance) %d", iter, res); + } + + raft::copy(eigVals_dev, eigenvalues_k.data_handle(), nEigVecs, stream); + raft::copy(eigVecs_dev, ritz_eigenvectors.data_handle(), n * nEigVecs, stream); + + return 0; +} + +template +auto lanczos_compute_smallest_eigenvectors( + raft::resources const& handle, + lanczos_solver_config const& config, + raft::device_csr_matrix_view A, + std::optional> v0, + raft::device_vector_view eigenvalues, + raft::device_matrix_view eigenvectors) -> int +{ + if (v0.has_value()) { + return lanczos_smallest(handle, + A, + config.n_components, + config.max_iterations, + config.ncv, + config.tolerance, + eigenvalues.data_handle(), + eigenvectors.data_handle(), + v0->data_handle(), + config.seed); + } else { + // Handle the optional v0 initial Lanczos vector if nullopt is used + auto n = A.structure_view().get_n_rows(); + auto temp_v0 = raft::make_device_vector(handle, n); + raft::random::RngState rng_state(config.seed); + raft::random::uniform(handle, rng_state, temp_v0.view(), ValueTypeT{0.0}, ValueTypeT{1.0}); + return lanczos_smallest(handle, + A, + config.n_components, + config.max_iterations, + config.ncv, + config.tolerance, + eigenvalues.data_handle(), + eigenvectors.data_handle(), + temp_v0.data_handle(), + config.seed); + } +} + } // namespace raft::sparse::solver::detail diff --git a/cpp/include/raft/sparse/solver/lanczos.cuh b/cpp/include/raft/sparse/solver/lanczos.cuh index 1aa56d6ba2..fed31e6a9c 100644 --- a/cpp/include/raft/sparse/solver/lanczos.cuh +++ b/cpp/include/raft/sparse/solver/lanczos.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ #pragma once #include +#include #include namespace raft::sparse::solver { @@ -27,6 +28,78 @@ namespace raft::sparse::solver { // Eigensolver // ========================================================= +/** + * @brief Find the smallest eigenpairs using lanczos solver + * @tparam index_type_t the type of data used for indexing. + * @tparam value_type_t the type of data used for weights, distances. + * @param handle the raft handle. + * @param config lanczos config used to set hyperparameters + * @param A Sparse matrix in CSR format. + * @param v0 Optional Initial lanczos vector + * @param eigenvalues output eigenvalues + * @param eigenvectors output eigenvectors + * @todo Add largest eigenvalues computation (issue #2483) + * @return Zero if successful. Otherwise non-zero. + */ +template +auto lanczos_compute_smallest_eigenvectors( + raft::resources const& handle, + lanczos_solver_config const& config, + raft::device_csr_matrix_view A, + std::optional> v0, + raft::device_vector_view eigenvalues, + raft::device_matrix_view eigenvectors) -> int +{ + return detail::lanczos_compute_smallest_eigenvectors( + handle, config, A, v0, eigenvalues, eigenvectors); +} + +/** + * @brief Find the smallest eigenpairs using lanczos solver + * @tparam index_type_t the type of data used for indexing. + * @tparam value_type_t the type of data used for weights, distances. + * @param handle the raft handle. + * @param config lanczos config used to set hyperparameters + * @param rows Vector view of the rows of the sparse matrix. + * @param cols Vector view of the cols of the sparse matrix. + * @param vals Vector view of the vals of the sparse matrix. + * @param v0 Optional Initial lanczos vector + * @param eigenvalues output eigenvalues + * @param eigenvectors output eigenvectors + * @todo Add largest eigenvalues computation (issue #2483) + * @return Zero if successful. Otherwise non-zero. + */ +template +auto lanczos_compute_smallest_eigenvectors( + raft::resources const& handle, + lanczos_solver_config const& config, + raft::device_vector_view rows, + raft::device_vector_view cols, + raft::device_vector_view vals, + std::optional> v0, + raft::device_vector_view eigenvalues, + raft::device_matrix_view eigenvectors) -> int +{ + IndexTypeT ncols = rows.extent(0) - 1; + IndexTypeT nrows = rows.extent(0) - 1; + IndexTypeT nnz = cols.extent(0); + + auto csr_structure = + raft::make_device_compressed_structure_view( + const_cast(rows.data_handle()), + const_cast(cols.data_handle()), + ncols, + nrows, + nnz); + + auto csr_matrix = + raft::make_device_csr_matrix_view( + const_cast(vals.data_handle()), csr_structure); + + return lanczos_compute_smallest_eigenvectors( + handle, config, csr_matrix, v0, eigenvalues, eigenvectors); +} + /** * @brief Compute smallest eigenvectors of symmetric matrix * Computes eigenvalues and eigenvectors that are least diff --git a/cpp/include/raft/sparse/solver/lanczos_types.hpp b/cpp/include/raft/sparse/solver/lanczos_types.hpp new file mode 100644 index 0000000000..edd5548079 --- /dev/null +++ b/cpp/include/raft/sparse/solver/lanczos_types.hpp @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace raft::sparse::solver { + +template +struct lanczos_solver_config { + /** The number of eigenvalues and eigenvectors to compute. Must be 1 <= k < n.*/ + int n_components; + /** Maximum number of iteration. */ + int max_iterations; + /** The number of Lanczos vectors generated. Must be k + 1 < ncv < n. */ + int ncv; + /** Tolerance for residuals ``||Ax - wx||`` */ + ValueTypeT tolerance; + /** random seed */ + uint64_t seed; +}; + +} // namespace raft::sparse::solver diff --git a/cpp/include/raft/spectral/eigen_solvers.cuh b/cpp/include/raft/spectral/eigen_solvers.cuh index 4774d8b8ae..d98e90532e 100644 --- a/cpp/include/raft/spectral/eigen_solvers.cuh +++ b/cpp/include/raft/spectral/eigen_solvers.cuh @@ -18,6 +18,7 @@ #pragma once +#include #include #include @@ -57,18 +58,32 @@ struct lanczos_solver_t { { RAFT_EXPECTS(eigVals != nullptr, "Null eigVals buffer."); RAFT_EXPECTS(eigVecs != nullptr, "Null eigVecs buffer."); - index_type_t iters{}; - sparse::solver::computeSmallestEigenvectors(handle, - A, - config_.n_eigVecs, - config_.maxIter, - config_.restartIter, - config_.tol, - config_.reorthogonalize, - iters, - eigVals, - eigVecs, - config_.seed); + index_type_t iters{0}; // TODO: return total number of iter + auto lanczos_config = raft::sparse::solver::lanczos_solver_config{ + config_.n_eigVecs, config_.maxIter, config_.restartIter, config_.tol, config_.seed}; + auto csr_structure = + raft::make_device_compressed_structure_view( + const_cast(A.row_offsets_), + const_cast(A.col_indices_), + A.nrows_, + A.ncols_, + A.nnz_); + + auto csr_matrix = + raft::make_device_csr_matrix_view( + const_cast(A.values_), csr_structure); + std::optional> v0_opt; + + sparse::solver::lanczos_compute_smallest_eigenvectors( + handle, + lanczos_config, + csr_matrix, + v0_opt, + raft::make_device_vector_view(eigVals, + config_.n_eigVecs), + raft::make_device_matrix_view( + eigVecs, A.nrows_, config_.n_eigVecs)); + return iters; } diff --git a/cpp/include/raft_runtime/solver/lanczos.hpp b/cpp/include/raft_runtime/solver/lanczos.hpp new file mode 100644 index 0000000000..6c9d901bf1 --- /dev/null +++ b/cpp/include/raft_runtime/solver/lanczos.hpp @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include + +namespace raft::runtime::solver { + +/** + * @defgroup lanczos_runtime lanczos Runtime API + * @{ + */ + +#define FUNC_DECL(IndexType, ValueType) \ + void lanczos_solver( \ + const raft::resources& handle, \ + raft::sparse::solver::lanczos_solver_config config, \ + raft::device_vector_view rows, \ + raft::device_vector_view cols, \ + raft::device_vector_view vals, \ + std::optional> v0, \ + raft::device_vector_view eigenvalues, \ + raft::device_matrix_view eigenvectors) + +FUNC_DECL(int, float); +FUNC_DECL(int64_t, float); +FUNC_DECL(int, double); +FUNC_DECL(int64_t, double); + +#undef FUNC_DECL + +/** @} */ // end group lanczos_runtime + +} // namespace raft::runtime::solver diff --git a/cpp/src/raft_runtime/solver/lanczos_solver.cuh b/cpp/src/raft_runtime/solver/lanczos_solver.cuh new file mode 100644 index 0000000000..0c851ef13a --- /dev/null +++ b/cpp/src/raft_runtime/solver/lanczos_solver.cuh @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#define FUNC_DEF(IndexType, ValueType) \ + void lanczos_solver( \ + const raft::resources& handle, \ + raft::sparse::solver::lanczos_solver_config config, \ + raft::device_vector_view rows, \ + raft::device_vector_view cols, \ + raft::device_vector_view vals, \ + std::optional> v0, \ + raft::device_vector_view eigenvalues, \ + raft::device_matrix_view eigenvectors) \ + { \ + raft::sparse::solver::lanczos_compute_smallest_eigenvectors( \ + handle, config, rows, cols, vals, v0, eigenvalues, eigenvectors); \ + } diff --git a/cpp/src/raft_runtime/solver/lanczos_solver_int64_double.cu b/cpp/src/raft_runtime/solver/lanczos_solver_int64_double.cu new file mode 100644 index 0000000000..f772a8a0d1 --- /dev/null +++ b/cpp/src/raft_runtime/solver/lanczos_solver_int64_double.cu @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "lanczos_solver.cuh" + +namespace raft::runtime::solver { + +FUNC_DEF(int64_t, double); + +} // namespace raft::runtime::solver diff --git a/cpp/src/raft_runtime/solver/lanczos_solver_int64_float.cu b/cpp/src/raft_runtime/solver/lanczos_solver_int64_float.cu new file mode 100644 index 0000000000..efaf3be565 --- /dev/null +++ b/cpp/src/raft_runtime/solver/lanczos_solver_int64_float.cu @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "lanczos_solver.cuh" + +namespace raft::runtime::solver { + +FUNC_DEF(int64_t, float); + +} // namespace raft::runtime::solver diff --git a/cpp/src/raft_runtime/solver/lanczos_solver_int_double.cu b/cpp/src/raft_runtime/solver/lanczos_solver_int_double.cu new file mode 100644 index 0000000000..9bbc00e78a --- /dev/null +++ b/cpp/src/raft_runtime/solver/lanczos_solver_int_double.cu @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "lanczos_solver.cuh" + +namespace raft::runtime::solver { + +FUNC_DEF(int, double); + +} // namespace raft::runtime::solver diff --git a/cpp/src/raft_runtime/solver/lanczos_solver_int_float.cu b/cpp/src/raft_runtime/solver/lanczos_solver_int_float.cu new file mode 100644 index 0000000000..316a9fb7e1 --- /dev/null +++ b/cpp/src/raft_runtime/solver/lanczos_solver_int_float.cu @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "lanczos_solver.cuh" + +namespace raft::runtime::solver { + +FUNC_DEF(int, float); + +} // namespace raft::runtime::solver diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index a387e9ce09..621ee6c160 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -232,8 +232,8 @@ if(BUILD_TESTS) ) ConfigureTest( - NAME SOLVERS_TEST PATH linalg/eigen_solvers.cu lap/lap.cu sparse/mst.cu LIB - EXPLICIT_INSTANTIATE_ONLY + NAME SOLVERS_TEST PATH linalg/eigen_solvers.cu lap/lap.cu sparse/mst.cu + sparse/solver/lanczos.cu LIB EXPLICIT_INSTANTIATE_ONLY ) ConfigureTest( diff --git a/cpp/test/sparse/solver/lanczos.cu b/cpp/test/sparse/solver/lanczos.cu new file mode 100644 index 0000000000..74611a1fd8 --- /dev/null +++ b/cpp/test/sparse/solver/lanczos.cu @@ -0,0 +1,445 @@ +/* + * Copyright (c) 2019-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../../test_utils.cuh" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include +#include +#include +#include + +namespace raft::sparse { + +template +struct lanczos_inputs { + int n_components; + int restartiter; + int maxiter; + int conv_n_iters; + float conv_eps; + float tol; + uint64_t seed; + std::vector rows; // indptr + std::vector cols; // indices + std::vector vals; // data + std::vector expected_eigenvalues; +}; + +template +struct rmat_lanczos_inputs { + int n_components; + int restartiter; + int maxiter; + int conv_n_iters; + float conv_eps; + float tol; + uint64_t seed; + int r_scale; + int c_scale; + float sparsity; + std::vector expected_eigenvalues; +}; + +template +class rmat_lanczos_tests + : public ::testing::TestWithParam> { + public: + rmat_lanczos_tests() + : params(::testing::TestWithParam>::GetParam()), + stream(resource::get_cuda_stream(handle)), + rng(params.seed), + expected_eigenvalues(raft::make_device_vector( + handle, params.n_components)), + r_scale(params.r_scale), + c_scale(params.c_scale), + sparsity(params.sparsity) + { + } + + protected: + void SetUp() override + { + raft::copy(expected_eigenvalues.data_handle(), + params.expected_eigenvalues.data(), + params.n_components, + stream); + } + + void TearDown() override {} + + void Run() + { + uint64_t n_edges = sparsity * ((long long)(1 << r_scale) * (long long)(1 << c_scale)); + uint64_t n_nodes = 1 << std::max(r_scale, c_scale); + uint64_t theta_len = std::max(r_scale, c_scale) * 4; + + auto theta = raft::make_device_vector(handle, theta_len); + raft::random::uniform(handle, rng, theta.view(), 0, 1); + + auto out = + raft::make_device_matrix(handle, n_edges * 2, 2); + auto out_src = raft::make_device_vector(handle, n_edges); + auto out_dst = raft::make_device_vector(handle, n_edges); + + raft::random::RngState rng1{params.seed}; + + raft::random::rmat_rectangular_gen(handle, + rng1, + make_const_mdspan(theta.view()), + out.view(), + out_src.view(), + out_dst.view(), + r_scale, + c_scale); + + raft::device_vector out_data = + raft::make_device_vector(handle, n_edges); + raft::matrix::fill(handle, out_data.view(), 1.0); + raft::sparse::COO coo(stream); + + raft::sparse::op::coo_sort(n_nodes, + n_nodes, + n_edges, + out_src.data_handle(), + out_dst.data_handle(), + out_data.data_handle(), + stream); + raft::sparse::op::max_duplicates(handle, + coo, + out_src.data_handle(), + out_dst.data_handle(), + out_data.data_handle(), + n_edges, + n_nodes, + n_nodes); + + raft::sparse::COO symmetric_coo(stream); + raft::sparse::linalg::symmetrize( + handle, coo.rows(), coo.cols(), coo.vals(), coo.n_rows, coo.n_cols, coo.nnz, symmetric_coo); + + raft::device_vector row_indices = + raft::make_device_vector(handle, + symmetric_coo.n_rows + 1); + raft::sparse::convert::sorted_coo_to_csr(symmetric_coo.rows(), + symmetric_coo.nnz, + row_indices.data_handle(), + symmetric_coo.n_rows + 1, + stream); + + int n_components = params.n_components; + + raft::device_vector v0 = + raft::make_device_vector(handle, symmetric_coo.n_rows); + + raft::random::uniform(handle, rng, v0.view(), 0, 1); + std::tuple stats; + + raft::device_vector eigenvalues = + raft::make_device_vector(handle, n_components); + raft::device_matrix eigenvectors = + raft::make_device_matrix( + handle, symmetric_coo.n_rows, n_components); + + raft::spectral::matrix::sparse_matrix_t const csr_m{ + handle, + row_indices.data_handle(), + symmetric_coo.cols(), + symmetric_coo.vals(), + symmetric_coo.n_rows, + symmetric_coo.nnz}; + raft::sparse::solver::lanczos_solver_config config{ + n_components, params.maxiter, params.restartiter, params.tol, rng.seed}; + + auto csr_structure = + raft::make_device_compressed_structure_view( + const_cast(row_indices.data_handle()), + const_cast(symmetric_coo.cols()), + symmetric_coo.n_rows, + symmetric_coo.n_rows, + symmetric_coo.nnz); + + auto csr_matrix = raft::make_device_csr_matrix_view( + const_cast(symmetric_coo.vals()), csr_structure); + + std::get<0>(stats) = + raft::sparse::solver::lanczos_compute_smallest_eigenvectors( + handle, + config, + csr_matrix, + std::make_optional(v0.view()), + eigenvalues.view(), + eigenvectors.view()); + + ASSERT_TRUE(raft::devArrMatch(eigenvalues.data_handle(), + expected_eigenvalues.data_handle(), + n_components, + raft::CompareApprox(1e-5), + stream)); + } + + protected: + rmat_lanczos_inputs params; + raft::resources handle; + cudaStream_t stream; + raft::random::RngState rng; + int r_scale; + int c_scale; + float sparsity; + raft::device_vector expected_eigenvalues; +}; + +template +class lanczos_tests : public ::testing::TestWithParam> { + public: + lanczos_tests() + : params(::testing::TestWithParam>::GetParam()), + stream(resource::get_cuda_stream(handle)), + n(params.rows.size() - 1), + nnz(params.vals.size()), + rng(params.seed), + rows(raft::make_device_vector(handle, n + 1)), + cols(raft::make_device_vector(handle, nnz)), + vals(raft::make_device_vector(handle, nnz)), + v0(raft::make_device_vector(handle, n)), + eigenvalues(raft::make_device_vector( + handle, params.n_components)), + eigenvectors(raft::make_device_matrix( + handle, n, params.n_components)), + expected_eigenvalues( + raft::make_device_vector(handle, params.n_components)) + { + } + + protected: + void SetUp() override + { + raft::copy(rows.data_handle(), params.rows.data(), n + 1, stream); + raft::copy(cols.data_handle(), params.cols.data(), nnz, stream); + raft::copy(vals.data_handle(), params.vals.data(), nnz, stream); + raft::copy(expected_eigenvalues.data_handle(), + params.expected_eigenvalues.data(), + params.n_components, + stream); + } + + void TearDown() override {} + + void Run() + { + raft::random::uniform(handle, rng, v0.view(), 0, 1); + std::tuple stats; + + raft::sparse::solver::lanczos_solver_config config{ + params.n_components, params.maxiter, params.restartiter, params.tol, rng.seed}; + auto csr_structure = + raft::make_device_compressed_structure_view( + const_cast(rows.data_handle()), + const_cast(cols.data_handle()), + n, + n, + nnz); + + auto csr_matrix = raft::make_device_csr_matrix_view( + const_cast(vals.data_handle()), csr_structure); + + std::get<0>(stats) = + raft::sparse::solver::lanczos_compute_smallest_eigenvectors( + handle, + config, + csr_matrix, + std::make_optional(v0.view()), + eigenvalues.view(), + eigenvectors.view()); + + ASSERT_TRUE(raft::devArrMatch(eigenvalues.data_handle(), + expected_eigenvalues.data_handle(), + params.n_components, + raft::CompareApprox(1e-5), + stream)); + } + + protected: + lanczos_inputs params; + raft::resources handle; + cudaStream_t stream; + int n; + int nnz; + raft::random::RngState rng; + raft::device_vector rows; + raft::device_vector cols; + raft::device_vector vals; + raft::device_vector v0; + raft::device_vector eigenvalues; + raft::device_matrix eigenvectors; + raft::device_vector expected_eigenvalues; +}; + +// TODO: Find a way to generate and validate test data without hardcoding them (issue #2485) +const std::vector> inputsf = { + {2, + 34, + 10000, + 0, + 0, + 1e-15, + 42, + {0, 0, 0, 0, 3, 5, 6, 8, 9, 11, 16, 16, 18, 20, 23, 24, 27, + 30, 31, 33, 37, 37, 39, 41, 43, 44, 46, 46, 47, 49, 50, 50, 51, 53, + 57, 58, 59, 66, 67, 68, 69, 71, 72, 75, 78, 83, 86, 90, 93, 94, 96, + 98, 99, 101, 101, 104, 106, 108, 109, 109, 109, 109, 111, 113, 118, 120, 121, 123, + 124, 128, 132, 134, 136, 138, 139, 141, 145, 148, 151, 152, 154, 155, 157, 160, 164, + 167, 170, 170, 170, 173, 178, 179, 182, 184, 186, 191, 192, 196, 198, 198, 198}, + {44, 68, 74, 16, 36, 85, 34, 75, 61, 51, 83, 15, 33, 55, 69, 71, 18, 84, 70, 95, 71, 83, + 97, 83, 9, 36, 54, 4, 42, 46, 52, 11, 89, 31, 37, 74, 96, 36, 88, 56, 64, 68, 94, 82, + 35, 90, 50, 82, 85, 83, 19, 47, 94, 9, 44, 56, 79, 6, 25, 4, 15, 21, 52, 75, 79, 92, + 19, 72, 94, 94, 96, 80, 16, 54, 89, 46, 48, 63, 3, 33, 67, 73, 77, 46, 47, 75, 16, 43, + 45, 81, 32, 45, 68, 43, 55, 63, 27, 89, 8, 17, 36, 15, 42, 96, 9, 49, 22, 33, 77, 7, + 75, 78, 88, 43, 49, 66, 76, 91, 22, 82, 69, 63, 84, 44, 3, 23, 47, 81, 9, 65, 76, 92, + 12, 96, 9, 13, 38, 93, 44, 3, 19, 6, 36, 45, 61, 63, 69, 89, 44, 57, 94, 62, 33, 36, + 41, 46, 68, 24, 28, 64, 8, 13, 14, 29, 11, 66, 88, 5, 28, 93, 21, 62, 84, 18, 42, 50, + 76, 91, 25, 63, 89, 97, 36, 69, 72, 85, 23, 32, 39, 40, 77, 12, 19, 40, 54, 70, 13, 91}, + {0.4734894, 0.1402491, 0.7686475, 0.0416142, 0.2559651, 0.9360436, 0.7486080, 0.5206724, + 0.0374126, 0.8082515, 0.5993828, 0.4866583, 0.8907925, 0.9251201, 0.8566143, 0.9528994, + 0.4557763, 0.4907070, 0.4158074, 0.8311127, 0.9026024, 0.3103237, 0.5876446, 0.7585195, + 0.4866583, 0.4493615, 0.5909155, 0.0416142, 0.0963910, 0.6722401, 0.3468698, 0.4557763, + 0.1445242, 0.7720124, 0.9923756, 0.1227579, 0.7194629, 0.8916773, 0.4320931, 0.5840980, + 0.0216121, 0.3709223, 0.1705930, 0.8297898, 0.2409706, 0.9585592, 0.3171389, 0.0228039, + 0.4350971, 0.4939908, 0.7720124, 0.2722416, 0.1792683, 0.8907925, 0.1085757, 0.8745620, + 0.3298612, 0.7486080, 0.2409706, 0.2559651, 0.4493615, 0.8916773, 0.5540361, 0.5150571, + 0.9160119, 0.1767728, 0.9923756, 0.5717281, 0.1077409, 0.9368132, 0.6273088, 0.6616613, + 0.0963910, 0.9378265, 0.3059566, 0.3159291, 0.0449106, 0.9085807, 0.4734894, 0.1085757, + 0.2909013, 0.7787509, 0.7168902, 0.9691764, 0.2669757, 0.4389115, 0.6722401, 0.3159291, + 0.9691764, 0.7467896, 0.2722416, 0.2669757, 0.1532843, 0.0449106, 0.2023634, 0.8934466, + 0.3171389, 0.6594226, 0.8082515, 0.3468698, 0.5540361, 0.5909155, 0.9378265, 0.2909178, + 0.9251201, 0.2023634, 0.5840980, 0.8745620, 0.2624605, 0.0374126, 0.1034030, 0.3736577, + 0.3315690, 0.9085807, 0.8934466, 0.5548525, 0.2302140, 0.7827352, 0.0216121, 0.8262919, + 0.1646078, 0.5548525, 0.2658700, 0.2909013, 0.1402491, 0.3709223, 0.1532843, 0.5792196, + 0.8566143, 0.1646078, 0.0827300, 0.5810611, 0.4158074, 0.5188584, 0.9528994, 0.9026024, + 0.5717281, 0.7269946, 0.7787509, 0.7686475, 0.1227579, 0.5206724, 0.5150571, 0.4389115, + 0.1034030, 0.2302140, 0.0827300, 0.8961608, 0.7168902, 0.2624605, 0.4823034, 0.3736577, + 0.3298612, 0.9160119, 0.6616613, 0.7467896, 0.5792196, 0.8297898, 0.0228039, 0.8262919, + 0.5993828, 0.3103237, 0.7585195, 0.4939908, 0.4907070, 0.2658700, 0.0844443, 0.9360436, + 0.4350971, 0.6997072, 0.4320931, 0.3315690, 0.0844443, 0.1445242, 0.3059566, 0.6594226, + 0.8961608, 0.6498466, 0.9585592, 0.7827352, 0.6498466, 0.2812338, 0.1767728, 0.5810611, + 0.7269946, 0.6997072, 0.1705930, 0.1792683, 0.1077409, 0.9368132, 0.4823034, 0.8311127, + 0.7194629, 0.6273088, 0.2909178, 0.5188584, 0.5876446, 0.2812338}, + {-2.0369630, -1.7673520}}}; + +const std::vector> inputsd = { + {2, + 34, + 10000, + 0, + 0, + 1e-15, + 42, + {0, 0, 0, 0, 3, 5, 6, 8, 9, 11, 16, 16, 18, 20, 23, 24, 27, + 30, 31, 33, 37, 37, 39, 41, 43, 44, 46, 46, 47, 49, 50, 50, 51, 53, + 57, 58, 59, 66, 67, 68, 69, 71, 72, 75, 78, 83, 86, 90, 93, 94, 96, + 98, 99, 101, 101, 104, 106, 108, 109, 109, 109, 109, 111, 113, 118, 120, 121, 123, + 124, 128, 132, 134, 136, 138, 139, 141, 145, 148, 151, 152, 154, 155, 157, 160, 164, + 167, 170, 170, 170, 173, 178, 179, 182, 184, 186, 191, 192, 196, 198, 198, 198}, + {44, 68, 74, 16, 36, 85, 34, 75, 61, 51, 83, 15, 33, 55, 69, 71, 18, 84, 70, 95, 71, 83, + 97, 83, 9, 36, 54, 4, 42, 46, 52, 11, 89, 31, 37, 74, 96, 36, 88, 56, 64, 68, 94, 82, + 35, 90, 50, 82, 85, 83, 19, 47, 94, 9, 44, 56, 79, 6, 25, 4, 15, 21, 52, 75, 79, 92, + 19, 72, 94, 94, 96, 80, 16, 54, 89, 46, 48, 63, 3, 33, 67, 73, 77, 46, 47, 75, 16, 43, + 45, 81, 32, 45, 68, 43, 55, 63, 27, 89, 8, 17, 36, 15, 42, 96, 9, 49, 22, 33, 77, 7, + 75, 78, 88, 43, 49, 66, 76, 91, 22, 82, 69, 63, 84, 44, 3, 23, 47, 81, 9, 65, 76, 92, + 12, 96, 9, 13, 38, 93, 44, 3, 19, 6, 36, 45, 61, 63, 69, 89, 44, 57, 94, 62, 33, 36, + 41, 46, 68, 24, 28, 64, 8, 13, 14, 29, 11, 66, 88, 5, 28, 93, 21, 62, 84, 18, 42, 50, + 76, 91, 25, 63, 89, 97, 36, 69, 72, 85, 23, 32, 39, 40, 77, 12, 19, 40, 54, 70, 13, 91}, + {0.4734894, 0.1402491, 0.7686475, 0.0416142, 0.2559651, 0.9360436, 0.7486080, 0.5206724, + 0.0374126, 0.8082515, 0.5993828, 0.4866583, 0.8907925, 0.9251201, 0.8566143, 0.9528994, + 0.4557763, 0.4907070, 0.4158074, 0.8311127, 0.9026024, 0.3103237, 0.5876446, 0.7585195, + 0.4866583, 0.4493615, 0.5909155, 0.0416142, 0.0963910, 0.6722401, 0.3468698, 0.4557763, + 0.1445242, 0.7720124, 0.9923756, 0.1227579, 0.7194629, 0.8916773, 0.4320931, 0.5840980, + 0.0216121, 0.3709223, 0.1705930, 0.8297898, 0.2409706, 0.9585592, 0.3171389, 0.0228039, + 0.4350971, 0.4939908, 0.7720124, 0.2722416, 0.1792683, 0.8907925, 0.1085757, 0.8745620, + 0.3298612, 0.7486080, 0.2409706, 0.2559651, 0.4493615, 0.8916773, 0.5540361, 0.5150571, + 0.9160119, 0.1767728, 0.9923756, 0.5717281, 0.1077409, 0.9368132, 0.6273088, 0.6616613, + 0.0963910, 0.9378265, 0.3059566, 0.3159291, 0.0449106, 0.9085807, 0.4734894, 0.1085757, + 0.2909013, 0.7787509, 0.7168902, 0.9691764, 0.2669757, 0.4389115, 0.6722401, 0.3159291, + 0.9691764, 0.7467896, 0.2722416, 0.2669757, 0.1532843, 0.0449106, 0.2023634, 0.8934466, + 0.3171389, 0.6594226, 0.8082515, 0.3468698, 0.5540361, 0.5909155, 0.9378265, 0.2909178, + 0.9251201, 0.2023634, 0.5840980, 0.8745620, 0.2624605, 0.0374126, 0.1034030, 0.3736577, + 0.3315690, 0.9085807, 0.8934466, 0.5548525, 0.2302140, 0.7827352, 0.0216121, 0.8262919, + 0.1646078, 0.5548525, 0.2658700, 0.2909013, 0.1402491, 0.3709223, 0.1532843, 0.5792196, + 0.8566143, 0.1646078, 0.0827300, 0.5810611, 0.4158074, 0.5188584, 0.9528994, 0.9026024, + 0.5717281, 0.7269946, 0.7787509, 0.7686475, 0.1227579, 0.5206724, 0.5150571, 0.4389115, + 0.1034030, 0.2302140, 0.0827300, 0.8961608, 0.7168902, 0.2624605, 0.4823034, 0.3736577, + 0.3298612, 0.9160119, 0.6616613, 0.7467896, 0.5792196, 0.8297898, 0.0228039, 0.8262919, + 0.5993828, 0.3103237, 0.7585195, 0.4939908, 0.4907070, 0.2658700, 0.0844443, 0.9360436, + 0.4350971, 0.6997072, 0.4320931, 0.3315690, 0.0844443, 0.1445242, 0.3059566, 0.6594226, + 0.8961608, 0.6498466, 0.9585592, 0.7827352, 0.6498466, 0.2812338, 0.1767728, 0.5810611, + 0.7269946, 0.6997072, 0.1705930, 0.1792683, 0.1077409, 0.9368132, 0.4823034, 0.8311127, + 0.7194629, 0.6273088, 0.2909178, 0.5188584, 0.5876446, 0.2812338}, + {-2.0369630, -1.7673520}}}; + +const std::vector> rmat_inputsf = { + {50, 100, 10000, 0, 0, 1e-9, 42, 12, 12, 1, {-122.526794, -74.00686, -59.698284, -54.68617, + -49.686813, -34.02644, -32.130703, -31.26906, + -30.32097, -22.946098, -20.497862, -20.23817, + -19.269697, -18.42496, -17.675667, -17.013401, + -16.734581, -15.820215, -15.73925, -15.448187, + -15.044634, -14.692028, -14.127425, -13.967386, + -13.6237755, -13.469393, -13.181225, -12.777589, + -12.623185, -12.55508, -12.2874565, -12.053391, + -11.677346, -11.558279, -11.163732, -10.922034, + -10.7936945, -10.558049, -10.205776, -10.005316, + -9.559181, -9.491834, -9.242631, -8.883637, + -8.765364, -8.688508, -8.458255, -8.385196, + -8.217982, -8.0442095}}}; + +using LanczosTestF = lanczos_tests; +TEST_P(LanczosTestF, Result) { Run(); } + +using LanczosTestD = lanczos_tests; +TEST_P(LanczosTestD, Result) { Run(); } + +using RmatLanczosTestF = rmat_lanczos_tests; +TEST_P(RmatLanczosTestF, Result) { Run(); } + +INSTANTIATE_TEST_CASE_P(LanczosTests, LanczosTestF, ::testing::ValuesIn(inputsf)); +INSTANTIATE_TEST_CASE_P(LanczosTests, LanczosTestD, ::testing::ValuesIn(inputsd)); +INSTANTIATE_TEST_CASE_P(LanczosTests, RmatLanczosTestF, ::testing::ValuesIn(rmat_inputsf)); + +} // namespace raft::sparse diff --git a/docs/source/pylibraft_api.rst b/docs/source/pylibraft_api.rst index aaa359e646..ad7d2873d7 100644 --- a/docs/source/pylibraft_api.rst +++ b/docs/source/pylibraft_api.rst @@ -9,3 +9,4 @@ Python API pylibraft_api/common.rst pylibraft_api/random.rst + pylibraft_api/sparse.rst diff --git a/docs/source/pylibraft_api/sparse.rst b/docs/source/pylibraft_api/sparse.rst new file mode 100644 index 0000000000..b2c3f7a2b1 --- /dev/null +++ b/docs/source/pylibraft_api/sparse.rst @@ -0,0 +1,11 @@ +Sparse +====== + +This page provides pylibraft class references for the publicly-exposed elements of the `pylibraft.sparse.linalg.eigsh` package. + + +.. role:: py(code) + :language: python + :class: highlight + +.. autofunction:: pylibraft.sparse.linalg.eigsh \ No newline at end of file diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index 9bde613720..758c1e4711 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -87,6 +87,7 @@ rapids_cython_init() add_subdirectory(pylibraft/common) add_subdirectory(pylibraft/random) +add_subdirectory(pylibraft/sparse) if(DEFINED cython_lib_dir) rapids_cython_add_rpath_entries(TARGET raft PATHS "${cython_lib_dir}") diff --git a/python/pylibraft/pylibraft/sparse/CMakeLists.txt b/python/pylibraft/pylibraft/sparse/CMakeLists.txt new file mode 100644 index 0000000000..3779fd2715 --- /dev/null +++ b/python/pylibraft/pylibraft/sparse/CMakeLists.txt @@ -0,0 +1,15 @@ +# ============================================================================= +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +add_subdirectory(linalg) diff --git a/python/pylibraft/pylibraft/sparse/__init__.py b/python/pylibraft/pylibraft/sparse/__init__.py new file mode 100644 index 0000000000..c77def5bb0 --- /dev/null +++ b/python/pylibraft/pylibraft/sparse/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from pylibraft.sparse import linalg + +__all__ = ["linalg"] diff --git a/python/pylibraft/pylibraft/sparse/linalg/CMakeLists.txt b/python/pylibraft/pylibraft/sparse/linalg/CMakeLists.txt new file mode 100644 index 0000000000..ef16981644 --- /dev/null +++ b/python/pylibraft/pylibraft/sparse/linalg/CMakeLists.txt @@ -0,0 +1,27 @@ +# ============================================================================= +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +# Set the list of Cython files to build +set(cython_sources lanczos.pyx) + +# TODO: should finally be replaced with 'compiled' library to be more generic, when that is +# available +set(linked_libraries raft::raft raft::compiled) + +# Build all of the Cython targets +rapids_cython_create_modules( + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX sparse_ +) diff --git a/python/pylibraft/pylibraft/sparse/linalg/__init__.pxd b/python/pylibraft/pylibraft/sparse/linalg/__init__.pxd new file mode 100644 index 0000000000..e69de29bb2 diff --git a/python/pylibraft/pylibraft/sparse/linalg/__init__.py b/python/pylibraft/pylibraft/sparse/linalg/__init__.py new file mode 100644 index 0000000000..04a8106496 --- /dev/null +++ b/python/pylibraft/pylibraft/sparse/linalg/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from .lanczos import eigsh + +__all__ = ["eigsh"] diff --git a/python/pylibraft/pylibraft/sparse/linalg/cpp/__init__.pxd b/python/pylibraft/pylibraft/sparse/linalg/cpp/__init__.pxd new file mode 100644 index 0000000000..e69de29bb2 diff --git a/python/pylibraft/pylibraft/sparse/linalg/cpp/__init__.py b/python/pylibraft/pylibraft/sparse/linalg/cpp/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/python/pylibraft/pylibraft/sparse/linalg/lanczos.pyx b/python/pylibraft/pylibraft/sparse/linalg/lanczos.pyx new file mode 100644 index 0000000000..dc2a84b428 --- /dev/null +++ b/python/pylibraft/pylibraft/sparse/linalg/lanczos.pyx @@ -0,0 +1,277 @@ +# +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# cython: profile=False +# distutils: language = c++ +# cython: embedsignature = True +# cython: language_level = 3 + +import cupy as cp +import numpy as np + +from cython.operator cimport dereference as deref +from libc.stdint cimport int64_t, uint32_t, uint64_t, uintptr_t + +from pylibraft.common import Handle, cai_wrapper, device_ndarray +from pylibraft.common.handle import auto_sync_handle + +from libcpp cimport bool + +from pylibraft.common.cpp.mdspan cimport ( + col_major, + device_matrix_view, + device_vector_view, + make_device_matrix_view, + make_device_vector_view, + row_major, +) +from pylibraft.common.cpp.optional cimport optional +from pylibraft.common.handle cimport device_resources +from pylibraft.random.cpp.rng_state cimport RngState + + +cdef extern from "raft/sparse/solver/lanczos_types.hpp" \ + namespace "raft::sparse::solver" nogil: + + cdef cppclass lanczos_solver_config[ValueTypeT]: + int n_components + int max_iterations + int ncv + ValueTypeT tolerance + uint64_t seed + +cdef lanczos_solver_config[float] config_float +cdef lanczos_solver_config[double] config_double + +cdef extern from "raft_runtime/solver/lanczos.hpp" \ + namespace "raft::runtime::solver" nogil: + + cdef void lanczos_solver( + const device_resources &handle, + lanczos_solver_config[double] config, + device_vector_view[int64_t, uint32_t] rows, + device_vector_view[int64_t, uint32_t] cols, + device_vector_view[double, uint32_t] vals, + optional[device_vector_view[double, uint32_t]] v0, + device_vector_view[double, uint32_t] eigenvalues, + device_matrix_view[double, uint32_t, col_major] eigenvectors) except + + + cdef void lanczos_solver( + const device_resources &handle, + lanczos_solver_config[float] config, + device_vector_view[int64_t, uint32_t] rows, + device_vector_view[int64_t, uint32_t] cols, + device_vector_view[float, uint32_t] vals, + optional[device_vector_view[float, uint32_t]] v0, + device_vector_view[float, uint32_t] eigenvalues, + device_matrix_view[float, uint32_t, col_major] eigenvectors) except + + + cdef void lanczos_solver( + const device_resources &handle, + lanczos_solver_config[double] config, + device_vector_view[int, uint32_t] rows, + device_vector_view[int, uint32_t] cols, + device_vector_view[double, uint32_t] vals, + optional[device_vector_view[double, uint32_t]] v0, + device_vector_view[double, uint32_t] eigenvalues, + device_matrix_view[double, uint32_t, col_major] eigenvectors) except + + + cdef void lanczos_solver( + const device_resources &handle, + lanczos_solver_config[float] config, + device_vector_view[int, uint32_t] rows, + device_vector_view[int, uint32_t] cols, + device_vector_view[float, uint32_t] vals, + optional[device_vector_view[float, uint32_t]] v0, + device_vector_view[float, uint32_t] eigenvalues, + device_matrix_view[float, uint32_t, col_major] eigenvectors) except + + + +@auto_sync_handle +def eigsh(A, k=6, v0=None, ncv=None, maxiter=None, + tol=0, seed=None, handle=None): + """ + Find ``k`` eigenvalues and eigenvectors of the real symmetric square + matrix or complex Hermitian matrix ``A``. + + Solves ``Ax = wx``, the standard eigenvalue problem for ``w`` eigenvalues + with corresponding eigenvectors ``x``. + + Args: + a (spmatrix): A symmetric square sparse CSR matrix with + dimension ``(n, n)``. ``a`` must be of type + :class:`cupyx.scipy.sparse._csr.csr_matrix` + k (int): The number of eigenvalues and eigenvectors to compute. Must be + ``1 <= k < n``. + v0 (ndarray): Starting vector for iteration. If ``None``, a random + unit vector is used. + ncv (int): The number of Lanczos vectors generated. Must be + ``k + 1 < ncv < n``. If ``None``, default value is used. + maxiter (int): Maximum number of Lanczos update iterations. + If ``None``, default value is used. + tol (float): Tolerance for residuals ``||Ax - wx||``. If ``0``, machine + precision is used. + + Returns: + tuple: + It returns ``w`` and ``x`` + where ``w`` is eigenvalues and ``x`` is eigenvectors. + + .. seealso:: + :func:`scipy.sparse.linalg.eigsh` + :func:`cupyx.scipy.sparse.linalg.eigsh` + + .. note:: + This function uses the thick-restart Lanczos methods + (https://sdm.lbl.gov/~kewu/ps/trlan.html). + + """ + + if A is None: + raise Exception("'A' cannot be None!") + + rows = A.indptr + cols = A.indices + vals = A.data + + rows = cai_wrapper(rows) + cols = cai_wrapper(cols) + vals = cai_wrapper(vals) + + IndexType = rows.dtype + ValueType = vals.dtype + + N = A.shape[0] + n = N + nnz = A.nnz + + rows_ptr = rows.data + cols_ptr = cols.data + vals_ptr = vals.data + cdef optional[device_vector_view[double, uint32_t]] d_v0 + cdef optional[device_vector_view[float, uint32_t]] f_v0 + + if ncv is None: + ncv = min(n, max(2*k + 1, 20)) + else: + ncv = min(max(ncv, k + 2), n - 1) + + seed = seed if seed is not None else 42 + if maxiter is None: + maxiter = 10 * n + if tol == 0: + tol = np.finfo(ValueType).eps + + eigenvectors = device_ndarray.empty((N, k), dtype=ValueType, order='F') + eigenvalues = device_ndarray.empty((k), dtype=ValueType, order='F') + + eigenvectors_cai = cai_wrapper(eigenvectors) + eigenvalues_cai = cai_wrapper(eigenvalues) + + eigenvectors_ptr = eigenvectors_cai.data + eigenvalues_ptr = eigenvalues_cai.data + + handle = handle if handle is not None else Handle() + cdef device_resources *h = handle.getHandle() + + if IndexType == np.int32 and ValueType == np.float32: + config_float.n_components = k + config_float.max_iterations = maxiter + config_float.ncv = ncv + config_float.tolerance = tol + config_float.seed = seed + if v0 is not None: + v0 = cai_wrapper(v0) + v0_ptr = v0.data + f_v0 = make_device_vector_view(v0_ptr, N) + lanczos_solver( + deref(h), + config_float, + make_device_vector_view(rows_ptr, (N + 1)), + make_device_vector_view(cols_ptr, nnz), + make_device_vector_view(vals_ptr, nnz), + f_v0, + make_device_vector_view(eigenvalues_ptr, k), + make_device_matrix_view[float, uint32_t, col_major]( + eigenvectors_ptr, N, k), + ) + elif IndexType == np.int64 and ValueType == np.float32: + config_float.n_components = k + config_float.max_iterations = maxiter + config_float.ncv = ncv + config_float.tolerance = tol + config_float.seed = seed + if v0 is not None: + v0 = cai_wrapper(v0) + v0_ptr = v0.data + f_v0 = make_device_vector_view(v0_ptr, N) + lanczos_solver( + deref(h), + config_float, + make_device_vector_view(rows_ptr, (N + 1)), + make_device_vector_view(cols_ptr, nnz), + make_device_vector_view(vals_ptr, nnz), + f_v0, + make_device_vector_view(eigenvalues_ptr, k), + make_device_matrix_view[float, uint32_t, col_major]( + eigenvectors_ptr, N, k), + ) + elif IndexType == np.int32 and ValueType == np.float64: + config_double.n_components = k + config_double.max_iterations = maxiter + config_double.ncv = ncv + config_double.tolerance = tol + config_double.seed = seed + if v0 is not None: + v0 = cai_wrapper(v0) + v0_ptr = v0.data + d_v0 = make_device_vector_view(v0_ptr, N) + lanczos_solver( + deref(h), + config_double, + make_device_vector_view(rows_ptr, (N + 1)), + make_device_vector_view(cols_ptr, nnz), + make_device_vector_view(vals_ptr, nnz), + d_v0, + make_device_vector_view(eigenvalues_ptr, k), + make_device_matrix_view[double, uint32_t, col_major]( + eigenvectors_ptr, N, k), + ) + elif IndexType == np.int64 and ValueType == np.float64: + config_double.n_components = k + config_double.max_iterations = maxiter + config_double.ncv = ncv + config_double.tolerance = tol + config_double.seed = seed + if v0 is not None: + v0 = cai_wrapper(v0) + v0_ptr = v0.data + d_v0 = make_device_vector_view(v0_ptr, N) + lanczos_solver( + deref(h), + config_double, + make_device_vector_view(rows_ptr, (N + 1)), + make_device_vector_view(cols_ptr, nnz), + make_device_vector_view(vals_ptr, nnz), + d_v0, + make_device_vector_view(eigenvalues_ptr, k), + make_device_matrix_view[double, uint32_t, col_major]( + eigenvectors_ptr, N, k), + ) + else: + raise ValueError("dtype IndexType=%s and ValueType=%s not supported" % + (IndexType, ValueType)) + + return (cp.asarray(eigenvalues), cp.asarray(eigenvectors)) diff --git a/python/pylibraft/pylibraft/test/test_sparse.py b/python/pylibraft/pylibraft/test/test_sparse.py new file mode 100644 index 0000000000..10b261d322 --- /dev/null +++ b/python/pylibraft/pylibraft/test/test_sparse.py @@ -0,0 +1,142 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import cupy +import cupyx.scipy.sparse.linalg # NOQA +import numpy +import pytest +from cupyx.scipy import sparse + +from pylibraft.sparse.linalg import eigsh + + +def shaped_random( + shape, xp=cupy, dtype=numpy.float32, scale=10, seed=0, order="C" +): + """ + Returns an array filled with random values. + + Args + ---- + shape(tuple): Shape of returned ndarray. + xp(numpy or cupy): Array module to use. + dtype(dtype): Dtype of returned ndarray. + scale(float): Scaling factor of elements. + seed(int): Random seed. + + Returns + ------- + numpy.ndarray or cupy.ndarray: The array with + given shape, array module, + + If ``dtype`` is ``numpy.bool_``, the elements are + independently drawn from ``True`` and ``False`` + with same probabilities. + Otherwise, the array is filled with samples + independently and identically drawn + from uniform distribution over :math:`[0, scale)` + with specified dtype. + """ + numpy.random.seed(seed) + dtype = numpy.dtype(dtype) + if dtype == "?": + a = numpy.random.randint(2, size=shape) + elif dtype.kind == "c": + a = numpy.random.rand(*shape) + 1j * numpy.random.rand(*shape) + a *= scale + else: + a = numpy.random.rand(*shape) * scale + return xp.asarray(a, dtype=dtype, order=order) + + +class TestEigsh: + n = 30 + density = 0.33 + tol = {numpy.float32: 1e-5, numpy.complex64: 1e-5, "default": 1e-12} + res_tol = {"f": 1e-5, "d": 1e-12} + return_eigenvectors = True + + def _make_matrix(self, dtype, xp): + shape = (self.n, self.n) + a = shaped_random(shape, xp, dtype=dtype) + mask = shaped_random(shape, xp, dtype="f", scale=1) + a[mask > self.density] = 0 + a = a * a.conj().T + return a + + def _test_eigsh(self, a, k, xp, sp): + expected_ret = sp.linalg.eigsh( + a, k=k, return_eigenvectors=self.return_eigenvectors + ) + actual_ret = eigsh(a, k=k) + if self.return_eigenvectors: + w, x = actual_ret + exp_w, _ = expected_ret + # Check the residuals to see if eigenvectors are correct. + ax_xw = a @ x - xp.multiply(x, w.reshape(1, k)) + res = xp.linalg.norm(ax_xw) / xp.linalg.norm(w) + tol = self.res_tol[numpy.dtype(a.dtype).char.lower()] + assert res < tol + else: + w = actual_ret + exp_w = expected_ret + w = xp.sort(w) + cupy.allclose(w, exp_w, rtol=tol, atol=tol) + + @pytest.mark.parametrize("format", ["csr"]) # , 'csc', 'coo']) + @pytest.mark.parametrize("k", [3, 6, 12]) + @pytest.mark.parametrize("dtype", ["f", "d"]) + def test_sparse(self, format, k, dtype, xp=cupy, sp=sparse): + if format == "csc": + pytest.xfail("may be buggy") # trans=True + + a = self._make_matrix(dtype, xp) + a = sp.coo_matrix(a).asformat(format) + return self._test_eigsh(a, k, xp, sp) + + def test_invalid(self): + xp, sp = cupy, sparse + a = xp.diag(xp.ones((self.n,), dtype="f")) + with pytest.raises(ValueError): + sp.linalg.eigsh(xp.ones((2, 1), dtype="f")) + with pytest.raises(ValueError): + sp.linalg.eigsh(a, k=0) + a = xp.diag(xp.ones((self.n,), dtype="f")) + with pytest.raises(ValueError): + sp.linalg.eigsh(xp.ones((1,), dtype="f")) + with pytest.raises(TypeError): + sp.linalg.eigsh(xp.ones((2, 2), dtype="i")) + with pytest.raises(ValueError): + sp.linalg.eigsh(a, k=self.n) + + def test_starting_vector(self): + # Make symmetric matrix + aux = self._make_matrix("f", cupy) + aux = sparse.coo_matrix(aux).asformat("csr") + matrix = (aux + aux.T) / 2.0 + + # Find reference eigenvector + ew, ev = eigsh(matrix, k=1) + v = ev[:, 0] + + # Obtain non-converged eigenvector from random initial guess. + ew_aux, ev_aux = eigsh(matrix, k=1, ncv=1, maxiter=0) + v_aux = cupy.copysign(ev_aux[:, 0], v) + + # Obtain eigenvector using known eigenvector as initial guess. + ew_v0, ev_v0 = eigsh(matrix, k=1, v0=v.copy(), ncv=1, maxiter=0) + v_v0 = cupy.copysign(ev_v0[:, 0], v) + + assert cupy.linalg.norm(v - v_v0) < cupy.linalg.norm(v - v_aux) From fbdf2df7f4e7efc4ff8fab7f7767356a7ee1edc7 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 21 Nov 2024 07:06:29 -0800 Subject: [PATCH 57/79] Upgrade to latest cutlass version (#2503) This upgrade is important because cutlass 3.4 fixed issues with kernel visibility that would otherwise lead to global kernel symbols from raft kernels using cutlass. Authors: - Vyas Ramasubramani (https://github.com/vyasr) - Kyle Edwards (https://github.com/KyleFromNVIDIA) Approvers: - Robert Maynard (https://github.com/robertmaynard) - Corey J. Nolet (https://github.com/cjnolet) - Kyle Edwards (https://github.com/KyleFromNVIDIA) URL: https://github.com/rapidsai/raft/pull/2503 --- .github/workflows/pr.yaml | 1 - .github/workflows/test.yaml | 1 - .pre-commit-config.yaml | 5 ++- cpp/cmake/patches/cutlass/build-export.patch | 27 +++++++++++++++ cpp/cmake/patches/cutlass_override.json | 16 +++++++++ cpp/cmake/thirdparty/get_cutlass.cmake | 33 ++++++++++--------- .../pairwise_distance_epilogue_elementwise.h | 1 + 7 files changed, 65 insertions(+), 19 deletions(-) create mode 100644 cpp/cmake/patches/cutlass/build-export.patch create mode 100644 cpp/cmake/patches/cutlass_override.json diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 47951783ba..9c22edf74c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -80,7 +80,6 @@ jobs: with: build_type: pull-request enable_check_symbols: true - symbol_exclusions: raft_cutlass conda-python-build: needs: conda-cpp-build secrets: inherit diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2bee8a3d1d..92020f6a76 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -23,7 +23,6 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} enable_check_symbols: true - symbol_exclusions: raft_cutlass conda-cpp-tests: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.12 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d8ccf92ce5..e3b3c8c440 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -93,7 +93,10 @@ repos: - id: codespell additional_dependencies: [tomli] args: ["--toml", "pyproject.toml"] - exclude: (?x)^(^CHANGELOG.md$) + exclude: | + (?x) + ^CHANGELOG[.]md$| + ^cpp/cmake/patches/cutlass/build-export[.]patch$ - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: diff --git a/cpp/cmake/patches/cutlass/build-export.patch b/cpp/cmake/patches/cutlass/build-export.patch new file mode 100644 index 0000000000..a6423e9c08 --- /dev/null +++ b/cpp/cmake/patches/cutlass/build-export.patch @@ -0,0 +1,27 @@ +From e0a9597946257a01ae8444200f836ee51d5597ba Mon Sep 17 00:00:00 2001 +From: Kyle Edwards +Date: Wed, 20 Nov 2024 16:37:38 -0500 +Subject: [PATCH] Remove erroneous include directories + +These directories are left over from when CuTe was a separate +CMake project. Remove them. +--- + CMakeLists.txt | 2 -- + 1 file changed, 2 deletions(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 7419bdf5e..545384d82 100755 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -665,8 +665,6 @@ target_include_directories( + $ + $ + $ +- $ +- $ + ) + + # Mark CTK headers as system to supress warnings from them +-- +2.34.1 + diff --git a/cpp/cmake/patches/cutlass_override.json b/cpp/cmake/patches/cutlass_override.json new file mode 100644 index 0000000000..7bf818987f --- /dev/null +++ b/cpp/cmake/patches/cutlass_override.json @@ -0,0 +1,16 @@ +{ + "packages" : { + "cutlass" : { + "version": "3.5.1", + "git_url": "https://github.com/NVIDIA/cutlass.git", + "git_tag": "v${version}", + "patches" : [ + { + "file" : "${current_json_dir}/cutlass/build-export.patch", + "issue" : "Fix build directory export", + "fixed_in" : "" + } + ] + } + } +} diff --git a/cpp/cmake/thirdparty/get_cutlass.cmake b/cpp/cmake/thirdparty/get_cutlass.cmake index 0123c4b07a..d5bdd4632f 100644 --- a/cpp/cmake/thirdparty/get_cutlass.cmake +++ b/cpp/cmake/thirdparty/get_cutlass.cmake @@ -13,7 +13,9 @@ # ============================================================================= function(find_and_configure_cutlass) - set(oneValueArgs VERSION REPOSITORY PINNED_TAG) + set(options) + set(oneValueArgs) + set(multiValueArgs) cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) # if(RAFT_ENABLE_DIST_DEPENDENCIES OR RAFT_COMPILE_LIBRARIES) @@ -34,13 +36,22 @@ function(find_and_configure_cutlass) set(CUDART_LIBRARY "${CUDA_cudart_static_LIBRARY}" CACHE FILEPATH "fixing cutlass cmake code" FORCE) endif() + include("${rapids-cmake-dir}/cpm/package_override.cmake") + rapids_cpm_package_override("${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../patches/cutlass_override.json") + + include("${rapids-cmake-dir}/cpm/detail/package_details.cmake") + rapids_cpm_package_details(cutlass version repository tag shallow exclude) + + include("${rapids-cmake-dir}/cpm/detail/generate_patch_command.cmake") + rapids_cpm_generate_patch_command(cutlass ${version} patch_command) + rapids_cpm_find( - NvidiaCutlass ${PKG_VERSION} + NvidiaCutlass ${version} GLOBAL_TARGETS nvidia::cutlass::cutlass CPM_ARGS - GIT_REPOSITORY ${PKG_REPOSITORY} - GIT_TAG ${PKG_PINNED_TAG} - GIT_SHALLOW TRUE + GIT_REPOSITORY ${repository} + GIT_TAG ${tag} + GIT_SHALLOW ${shallow} ${patch_command} OPTIONS "CUDAToolkit_ROOT ${CUDAToolkit_LIBRARY_DIR}" ) @@ -79,14 +90,4 @@ function(find_and_configure_cutlass) ) endfunction() -if(NOT RAFT_CUTLASS_GIT_TAG) - set(RAFT_CUTLASS_GIT_TAG v2.10.0) -endif() - -if(NOT RAFT_CUTLASS_GIT_REPOSITORY) - set(RAFT_CUTLASS_GIT_REPOSITORY https://github.com/NVIDIA/cutlass.git) -endif() - -find_and_configure_cutlass( - VERSION 2.10.0 REPOSITORY ${RAFT_CUTLASS_GIT_REPOSITORY} PINNED_TAG ${RAFT_CUTLASS_GIT_TAG} -) +find_and_configure_cutlass() diff --git a/cpp/include/raft/distance/detail/pairwise_distance_epilogue_elementwise.h b/cpp/include/raft/distance/detail/pairwise_distance_epilogue_elementwise.h index 2b2c04b9d3..f6dea987e5 100644 --- a/cpp/include/raft/distance/detail/pairwise_distance_epilogue_elementwise.h +++ b/cpp/include/raft/distance/detail/pairwise_distance_epilogue_elementwise.h @@ -61,6 +61,7 @@ class PairwiseDistanceEpilogueElementwise { using ElementT = ElementT_; static int const kElementsPerAccess = ElementsPerAccess; static int const kCount = kElementsPerAccess; + static bool const kIsSingleSource = true; using DistanceOp = DistanceOp_; using FinalOp = FinalOp_; From deff99577e1a5e74d6601431eea207713a13efaa Mon Sep 17 00:00:00 2001 From: Micka Date: Mon, 25 Nov 2024 15:28:32 +0100 Subject: [PATCH 58/79] Revert use of new Lanczos solver in spectral clustering (#2507) Revert partly #2481 --- cpp/include/raft/spectral/eigen_solvers.cuh | 38 +++++++-------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/cpp/include/raft/spectral/eigen_solvers.cuh b/cpp/include/raft/spectral/eigen_solvers.cuh index d98e90532e..324f16ac7b 100644 --- a/cpp/include/raft/spectral/eigen_solvers.cuh +++ b/cpp/include/raft/spectral/eigen_solvers.cuh @@ -18,7 +18,6 @@ #pragma once -#include #include #include @@ -58,31 +57,18 @@ struct lanczos_solver_t { { RAFT_EXPECTS(eigVals != nullptr, "Null eigVals buffer."); RAFT_EXPECTS(eigVecs != nullptr, "Null eigVecs buffer."); - index_type_t iters{0}; // TODO: return total number of iter - auto lanczos_config = raft::sparse::solver::lanczos_solver_config{ - config_.n_eigVecs, config_.maxIter, config_.restartIter, config_.tol, config_.seed}; - auto csr_structure = - raft::make_device_compressed_structure_view( - const_cast(A.row_offsets_), - const_cast(A.col_indices_), - A.nrows_, - A.ncols_, - A.nnz_); - - auto csr_matrix = - raft::make_device_csr_matrix_view( - const_cast(A.values_), csr_structure); - std::optional> v0_opt; - - sparse::solver::lanczos_compute_smallest_eigenvectors( - handle, - lanczos_config, - csr_matrix, - v0_opt, - raft::make_device_vector_view(eigVals, - config_.n_eigVecs), - raft::make_device_matrix_view( - eigVecs, A.nrows_, config_.n_eigVecs)); + index_type_t iters{}; + sparse::solver::computeSmallestEigenvectors(handle, + A, + config_.n_eigVecs, + config_.maxIter, + config_.restartIter, + config_.tol, + config_.reorthogonalize, + iters, + eigVals, + eigVecs, + config_.seed); return iters; } From 4ecd92516dffecaa1a6101b8d2c4b44b3f13a89b Mon Sep 17 00:00:00 2001 From: Divye Gala Date: Tue, 26 Nov 2024 09:10:28 -0500 Subject: [PATCH 59/79] Switch `assert` to `static_assert` (#2510) This problem came up in cuVS CI here https://github.com/rapidsai/cuvs/actions/runs/11982382629/job/33410385067?pr=465#step:10:558 as an `unused-variable` error. --- cpp/include/raft/core/device_mdspan.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/raft/core/device_mdspan.hpp b/cpp/include/raft/core/device_mdspan.hpp index b4e9f8d1d7..c5241e831b 100644 --- a/cpp/include/raft/core/device_mdspan.hpp +++ b/cpp/include/raft/core/device_mdspan.hpp @@ -210,7 +210,7 @@ auto constexpr make_device_strided_matrix_view(ElementType* ptr, constexpr auto is_row_major = std::is_same_v; constexpr auto is_col_major = std::is_same_v; - assert(is_row_major || is_col_major); + static_assert(is_row_major || is_col_major, "Unsupported layout policy for strided matrix view"); IndexType stride0 = is_row_major ? (stride > 0 ? stride : n_cols) : 1; IndexType stride1 = is_row_major ? 1 : (stride > 0 ? stride : n_rows); From 4cdc1d80aa01c147a94eed9fbc68a38fba29eaf4 Mon Sep 17 00:00:00 2001 From: Jake Awe <50372925+AyodeAwe@users.noreply.github.com> Date: Tue, 26 Nov 2024 15:27:54 -0600 Subject: [PATCH 60/79] Add breaking change workflow trigger (#2482) Adds a workflow that triggers a second workflow which sends a notification to a designated Slack channel on every PR labelled with breaking, whenever any of the following events are triggered on the PR: - closed - reopened - labeled - unlabeled Depends on https://github.com/rapidsai/shared-workflows/pull/257 --- .../trigger-breaking-change-alert.yaml | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/trigger-breaking-change-alert.yaml diff --git a/.github/workflows/trigger-breaking-change-alert.yaml b/.github/workflows/trigger-breaking-change-alert.yaml new file mode 100644 index 0000000000..3b972f31ca --- /dev/null +++ b/.github/workflows/trigger-breaking-change-alert.yaml @@ -0,0 +1,26 @@ +name: Trigger Breaking Change Notifications + +on: + pull_request_target: + types: + - closed + - reopened + - labeled + - unlabeled + +jobs: + trigger-notifier: + if: contains(github.event.pull_request.labels.*.name, 'breaking') + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/breaking-change-alert.yaml@branch-24.12 + with: + sender_login: ${{ github.event.sender.login }} + sender_avatar: ${{ github.event.sender.avatar_url }} + repo: ${{ github.repository }} + pr_number: ${{ github.event.pull_request.number }} + pr_title: "${{ github.event.pull_request.title }}" + pr_body: "${{ github.event.pull_request.body || '_Empty PR description_' }}" + pr_base_ref: ${{ github.event.pull_request.base.ref }} + pr_author: ${{ github.event.pull_request.user.login }} + event_action: ${{ github.event.action }} + pr_merged: ${{ github.event.pull_request.merged }} From adfd2f6f765b5a979742399e10581b75ba5a2834 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 26 Nov 2024 16:38:40 -0600 Subject: [PATCH 61/79] Require approval to run CI on draft PRs (#2512) By default, CI runs on draft PRs. This leads to many CI runs that may be unnecessary. With this PR's change to `.github/copy-pr-bot.yaml`, an `/ok to test` comment from a trusted user is required to trigger CI on draft PRs. Non-draft PRs will run CI by default, assuming that all commits are signed by trusted users. Otherwise an `/ok to test` is required (as before) -- see the `copy-pr-bot` docs at https://docs.gha-runners.nvidia.com/apps/copy-pr-bot/ for more information. Part of https://github.com/rapidsai/build-planning/issues/123. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - James Lamb (https://github.com/jameslamb) - Jake Awe (https://github.com/AyodeAwe) URL: https://github.com/rapidsai/raft/pull/2512 --- .github/copy-pr-bot.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/copy-pr-bot.yaml b/.github/copy-pr-bot.yaml index 895ba83ee5..e0ea775aad 100644 --- a/.github/copy-pr-bot.yaml +++ b/.github/copy-pr-bot.yaml @@ -2,3 +2,4 @@ # https://docs.gha-runners.nvidia.com/apps/copy-pr-bot/ enabled: true +auto_sync_draft: false From c943181c4d48e5050f2b8c40f17e40155bfd9d61 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 26 Nov 2024 19:29:03 -0600 Subject: [PATCH 62/79] Shrink wheel size limit following removal of vector search APIs. (#2509) Following #2498, we can apply this feedback from #2490: https://github.com/rapidsai/raft/pull/2490#discussion_r1841357165 These changes are inspired by https://github.com/rapidsai/cuvs/pull/469. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - Kyle Edwards (https://github.com/KyleFromNVIDIA) URL: https://github.com/rapidsai/raft/pull/2509 --- ci/build_wheel_pylibraft.sh | 2 +- ci/build_wheel_raft_dask.sh | 2 +- ci/validate_wheel.sh | 24 ++++++++++++++++++++++++ python/pylibraft/pyproject.toml | 4 +--- 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/ci/build_wheel_pylibraft.sh b/ci/build_wheel_pylibraft.sh index dacaa1190e..dd62ab5399 100755 --- a/ci/build_wheel_pylibraft.sh +++ b/ci/build_wheel_pylibraft.sh @@ -18,4 +18,4 @@ esac export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF${EXTRA_CMAKE_ARGS}" ci/build_wheel.sh pylibraft ${package_dir} -ci/validate_wheel.sh ${package_dir} final_dist +ci/validate_wheel.sh ${package_dir} final_dist pylibraft diff --git a/ci/build_wheel_raft_dask.sh b/ci/build_wheel_raft_dask.sh index e4f3f0a833..d49d131abf 100755 --- a/ci/build_wheel_raft_dask.sh +++ b/ci/build_wheel_raft_dask.sh @@ -9,4 +9,4 @@ package_dir="python/raft-dask" export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF" ci/build_wheel.sh raft-dask ${package_dir} -ci/validate_wheel.sh ${package_dir} final_dist +ci/validate_wheel.sh ${package_dir} final_dist raft-dask diff --git a/ci/validate_wheel.sh b/ci/validate_wheel.sh index 5910a5c59f..5ef72ad895 100755 --- a/ci/validate_wheel.sh +++ b/ci/validate_wheel.sh @@ -5,6 +5,29 @@ set -euo pipefail package_dir=$1 wheel_dir_relative_path=$2 +package_name=$3 + +RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" + +# some packages are much larger on CUDA 11 than on CUDA 12 +if [[ "${package_name}" == "raft-dask" ]]; then + PYDISTCHECK_ARGS=( + --max-allowed-size-compressed '200M' + ) +elif [[ "${package_name}" == "pylibraft" ]]; then + if [[ "${RAPIDS_CUDA_MAJOR}" == "11" ]]; then + PYDISTCHECK_ARGS=( + --max-allowed-size-compressed '600M' + ) + else + PYDISTCHECK_ARGS=( + --max-allowed-size-compressed '100M' + ) + fi +else + echo "Unsupported package name: ${package_name}" + exit 1 +fi cd "${package_dir}" @@ -12,6 +35,7 @@ rapids-logger "validate packages with 'pydistcheck'" pydistcheck \ --inspect \ + "${PYDISTCHECK_ARGS[@]}" \ "$(echo ${wheel_dir_relative_path}/*.whl)" rapids-logger "validate packages with 'twine'" diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index 3502d82fd4..ba454af591 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -132,12 +132,10 @@ matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" [tool.pydistcheck] select = [ + # NOTE: size threshold is managed via CLI args in CI scripts "distro-too-large-compressed", ] -# detect when package size grows significantly -max_allowed_size_compressed = '825M' - [tool.pytest.ini_options] filterwarnings = [ "error", From 0e6d35f7cbb4354641d69868d6cb10dcee21fbca Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 27 Nov 2024 12:36:38 -0800 Subject: [PATCH 63/79] Adapt to rmm logger changes (#2513) This PR adapts to breaking changes in rmm in https://github.com/rapidsai/rmm/pull/1722. This PR is a breaking change because consumers of raft that use any functionality that touches rmm logging will need to link to the rmm::rmm_logger_impl target as well now. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/raft/pull/2513 --- cpp/CMakeLists.txt | 13 ++++++++++--- cpp/bench/prims/CMakeLists.txt | 4 ++++ cpp/cmake/thirdparty/get_spdlog.cmake | 6 +++--- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 780f6f8581..78a4dbb913 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -180,7 +180,10 @@ target_include_directories( ) # Keep RAFT as lightweight as possible. Only CUDA libs and rmm should be used in global target. -target_link_libraries(raft INTERFACE rmm::rmm cuco::cuco nvidia::cutlass::cutlass CCCL::CCCL) +target_link_libraries( + raft INTERFACE rmm::rmm rmm::rmm_logger spdlog::spdlog_header_only cuco::cuco + nvidia::cutlass::cutlass CCCL::CCCL +) target_compile_features(raft INTERFACE cxx_std_17 $) target_compile_options( @@ -288,8 +291,10 @@ if(RAFT_COMPILE_LIBRARY) "$<$:${RAFT_CUDA_FLAGS}>" ) - add_library(raft_lib SHARED $) - add_library(raft_lib_static STATIC $) + # Make sure not to add the rmm logger twice since it will be brought in as an interface source by + # the rmm::rmm_logger_impl target. + add_library(raft_lib SHARED $,EXCLUDE,rmm.*logger>) + add_library(raft_lib_static STATIC $,EXCLUDE,rmm.*logger>) set_target_properties( raft_lib raft_lib_static @@ -313,6 +318,8 @@ if(RAFT_COMPILE_LIBRARY) # ensure CUDA symbols aren't relocated to the middle of the debug build binaries target_link_options(${target} PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/fatbin.ld") endforeach() + target_link_libraries(raft_lib PRIVATE rmm::rmm_logger_impl) + target_link_libraries(raft_lib_static PRIVATE rmm::rmm_logger_impl) endif() if(TARGET raft_lib AND (NOT TARGET raft::raft_lib)) diff --git a/cpp/bench/prims/CMakeLists.txt b/cpp/bench/prims/CMakeLists.txt index cf03a36612..edc1af4e02 100644 --- a/cpp/bench/prims/CMakeLists.txt +++ b/cpp/bench/prims/CMakeLists.txt @@ -32,6 +32,7 @@ function(ConfigureBench) PRIVATE raft::raft raft_internal $<$:raft::compiled> + $<$>:bench_rmm_logger> ${RAFT_CTK_MATH_DEPENDENCIES} benchmark::benchmark Threads::Threads @@ -73,6 +74,9 @@ function(ConfigureBench) endfunction() +add_library(bench_rmm_logger OBJECT) +target_link_libraries(bench_rmm_logger PRIVATE rmm::rmm_logger_impl) + if(BUILD_PRIMS_BENCH) ConfigureBench(NAME CORE_BENCH PATH core/bitset.cu core/copy.cu main.cpp) diff --git a/cpp/cmake/thirdparty/get_spdlog.cmake b/cpp/cmake/thirdparty/get_spdlog.cmake index 57e38c2638..b1ffbe246f 100644 --- a/cpp/cmake/thirdparty/get_spdlog.cmake +++ b/cpp/cmake/thirdparty/get_spdlog.cmake @@ -16,9 +16,9 @@ function(find_and_configure_spdlog) include(${rapids-cmake-dir}/cpm/spdlog.cmake) - rapids_cpm_spdlog(FMT_OPTION "EXTERNAL_FMT_HO" INSTALL_EXPORT_SET rmm-exports) - rapids_export_package(BUILD spdlog rmm-exports) + rapids_cpm_spdlog(FMT_OPTION "EXTERNAL_FMT_HO" INSTALL_EXPORT_SET raft-exports) + rapids_export_package(BUILD spdlog raft-exports) endfunction() -find_and_configure_spdlog() \ No newline at end of file +find_and_configure_spdlog() From fc7818f078a69393e8a0cb27c117b19208c76aaf Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 4 Dec 2024 10:48:49 -0600 Subject: [PATCH 64/79] prefer system install of UCX in devcontainers, update outdated RAPIDS references (#2514) Contributes to https://github.com/rapidsai/build-planning/issues/118 Proposes the following changes for pip devcontainers: * prefer system installation of ucx to the one provided by the `libucx-cu{11,12}` wheels (ref: https://github.com/rapidsai/devcontainers/pull/421#issuecomment-2502324982) And some other related changes noticed while doing that: * update lingering `24.*` references to `25.02` ## Notes for Reviewers ### How I tested this Relying on CI for most things. Double-checked that `update-version.sh` would have caught the one lingering `24.12` reference like this: ```shell ./ci/release/update-version.sh '25.02.00' git grep -E '24\.' ``` Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/raft/pull/2514 --- .devcontainer/Dockerfile | 1 + .github/workflows/trigger-breaking-change-alert.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index dc12ab2ade..0f6a8b46af 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -13,6 +13,7 @@ RUN apt update -y \ && rm -rf /tmp/* /var/tmp/* /var/cache/apt/* /var/lib/apt/lists/*; ENV DEFAULT_VIRTUAL_ENV=rapids +ENV RAPIDS_LIBUCX_PREFER_SYSTEM_LIBRARY=true FROM ${BASE} as conda-base diff --git a/.github/workflows/trigger-breaking-change-alert.yaml b/.github/workflows/trigger-breaking-change-alert.yaml index 3b972f31ca..01dd2436be 100644 --- a/.github/workflows/trigger-breaking-change-alert.yaml +++ b/.github/workflows/trigger-breaking-change-alert.yaml @@ -12,7 +12,7 @@ jobs: trigger-notifier: if: contains(github.event.pull_request.labels.*.name, 'breaking') secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/breaking-change-alert.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/breaking-change-alert.yaml@branch-25.02 with: sender_login: ${{ github.event.sender.login }} sender_avatar: ${{ github.event.sender.avatar_url }} From 3ce5b6ad45946a9c790711addb7b5d358534d8d9 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 4 Dec 2024 18:14:06 -0600 Subject: [PATCH 65/79] Remove upper bounds on cuda-python to allow 12.6.2 and 11.8.5 (#2517) Now that some upstream bugs have been fixed, we can allow cuda-python 12.6.2 and 11.8.5. See https://github.com/NVIDIA/cuda-python/issues/226#issuecomment-2472355738 for more information. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - James Lamb (https://github.com/jameslamb) - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/raft/pull/2517 --- conda/environments/all_cuda-118_arch-aarch64.yaml | 3 ++- conda/environments/all_cuda-118_arch-x86_64.yaml | 3 ++- conda/environments/all_cuda-125_arch-aarch64.yaml | 3 ++- conda/environments/all_cuda-125_arch-x86_64.yaml | 3 ++- conda/recipes/pylibraft/meta.yaml | 8 ++++---- conda/recipes/raft-dask/meta.yaml | 8 ++++---- cpp/cmake/thirdparty/get_rmm.cmake | 2 +- dependencies.yaml | 7 ++++--- 8 files changed, 21 insertions(+), 16 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index 269af03e9f..f8201cbccf 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -14,7 +14,7 @@ dependencies: - cmake>=3.26.4,!=3.30.0 - cuda-nvtx=11.8 - cuda-profiler-api=11.8.86 -- cuda-python>=11.7.1,<12.0a0,<=11.8.3 +- cuda-python>=11.7.1,<12.0a0 - cuda-version=11.8 - cudatoolkit - cupy>=12.0.0 @@ -54,6 +54,7 @@ dependencies: - scikit-build-core>=0.10.0 - scikit-learn - scipy +- spdlog>=1.14.1,<1.15 - sphinx-copybutton - sphinx-markdown-tables - sysroot_linux-aarch64==2.17 diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 4c7150264b..66b97854ab 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -14,7 +14,7 @@ dependencies: - cmake>=3.26.4,!=3.30.0 - cuda-nvtx=11.8 - cuda-profiler-api=11.8.86 -- cuda-python>=11.7.1,<12.0a0,<=11.8.3 +- cuda-python>=11.7.1,<12.0a0 - cuda-version=11.8 - cudatoolkit - cupy>=12.0.0 @@ -54,6 +54,7 @@ dependencies: - scikit-build-core>=0.10.0 - scikit-learn - scipy +- spdlog>=1.14.1,<1.15 - sphinx-copybutton - sphinx-markdown-tables - sysroot_linux-64==2.17 diff --git a/conda/environments/all_cuda-125_arch-aarch64.yaml b/conda/environments/all_cuda-125_arch-aarch64.yaml index 648a5a00f0..1fd6edfb6f 100644 --- a/conda/environments/all_cuda-125_arch-aarch64.yaml +++ b/conda/environments/all_cuda-125_arch-aarch64.yaml @@ -16,7 +16,7 @@ dependencies: - cuda-nvcc - cuda-nvtx-dev - cuda-profiler-api -- cuda-python>=12.0,<13.0a0,<=12.6.0 +- cuda-python>=12.0,<13.0a0 - cuda-version=12.5 - cupy>=12.0.0 - cxx-compiler @@ -50,6 +50,7 @@ dependencies: - scikit-build-core>=0.10.0 - scikit-learn - scipy +- spdlog>=1.14.1,<1.15 - sphinx-copybutton - sphinx-markdown-tables - sysroot_linux-aarch64==2.17 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 7d7b9c4454..72108fed48 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -16,7 +16,7 @@ dependencies: - cuda-nvcc - cuda-nvtx-dev - cuda-profiler-api -- cuda-python>=12.0,<13.0a0,<=12.6.0 +- cuda-python>=12.0,<13.0a0 - cuda-version=12.5 - cupy>=12.0.0 - cxx-compiler @@ -50,6 +50,7 @@ dependencies: - scikit-build-core>=0.10.0 - scikit-learn - scipy +- spdlog>=1.14.1,<1.15 - sphinx-copybutton - sphinx-markdown-tables - sysroot_linux-64==2.17 diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 01a9d61f0f..f1edf5d767 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -43,10 +43,10 @@ requirements: - {{ stdlib("c") }} host: {% if cuda_major == "11" %} - - cuda-python >=11.7.1,<12.0a0,<=11.8.3 + - cuda-python >=11.7.1,<12.0a0 - cudatoolkit {% else %} - - cuda-python >=12.0,<13.0a0,<=12.6.0 + - cuda-python >=12.0,<13.0a0 - cuda-cudart-dev {% endif %} - cuda-version ={{ cuda_version }} @@ -61,10 +61,10 @@ requirements: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} - cudatoolkit - - cuda-python >=11.7.1,<12.0a0,<=11.8.3 + - cuda-python >=11.7.1,<12.0a0 {% else %} - cuda-cudart - - cuda-python >=12.0,<13.0a0,<=12.6.0 + - cuda-python >=12.0,<13.0a0 {% endif %} - libraft {{ version }} - libraft-headers {{ version }} diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index 02a8957b06..14ffa5c092 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -43,10 +43,10 @@ requirements: - {{ stdlib("c") }} host: {% if cuda_major == "11" %} - - cuda-python >=11.7.1,<12.0a0,<=11.8.3 + - cuda-python >=11.7.1,<12.0a0 - cudatoolkit {% else %} - - cuda-python >=12.0,<13.0a0,<=12.6.0 + - cuda-python >=12.0,<13.0a0 - cuda-cudart-dev {% endif %} - cuda-version ={{ cuda_version }} @@ -62,10 +62,10 @@ requirements: run: {% if cuda_major == "11" %} - cudatoolkit - - cuda-python >=11.7.1,<12.0a0,<=11.8.3 + - cuda-python >=11.7.1,<12.0a0 {% else %} - cuda-cudart - - cuda-python >=12.0,<13.0a0,<=12.6.0 + - cuda-python >=12.0,<13.0a0 {% endif %} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - dask-cuda ={{ minor_version }} diff --git a/cpp/cmake/thirdparty/get_rmm.cmake b/cpp/cmake/thirdparty/get_rmm.cmake index 5a7d54ea4a..0e93363039 100644 --- a/cpp/cmake/thirdparty/get_rmm.cmake +++ b/cpp/cmake/thirdparty/get_rmm.cmake @@ -17,7 +17,7 @@ function(find_and_configure_rmm) include(${rapids-cmake-dir}/cpm/rmm.cmake) rapids_cpm_rmm(BUILD_EXPORT_SET raft-exports - INSTALL_EXPORT_SET raft-exports) + INSTALL_EXPORT_SET raft-exports) endfunction() find_and_configure_rmm() diff --git a/dependencies.yaml b/dependencies.yaml index daef3ad2ea..80c7f29447 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -143,8 +143,9 @@ dependencies: packages: - c-compiler - cxx-compiler - - nccl>=2.19 - libucxx==0.42.*,>=0.0.0a0 + - nccl>=2.19 + - spdlog>=1.14.1,<1.15 specific: - output_types: conda matrices: @@ -196,11 +197,11 @@ dependencies: - matrix: cuda: "12.*" packages: - - &cuda_python12 cuda-python>=12.0,<13.0a0,<=12.6.0 + - &cuda_python12 cuda-python>=12.0,<13.0a0 - matrix: cuda: "11.*" packages: - - &cuda_python11 cuda-python>=11.7.1,<12.0a0,<=11.8.3 + - &cuda_python11 cuda-python>=11.7.1,<12.0a0 - matrix: packages: - &cuda_python cuda-python From 0af5afb045a907c15d2fc8901fbc2cb4f2761ccb Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Fri, 6 Dec 2024 12:36:02 -0500 Subject: [PATCH 66/79] Skip gtests for new lanczos solver when CUDA version is 11.4 or below. (#2520) Medium- to longer term, we plan to fix this issue (ref: https://github.com/rapidsai/raft/issues/2519). --- cpp/test/sparse/solver/lanczos.cu | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cpp/test/sparse/solver/lanczos.cu b/cpp/test/sparse/solver/lanczos.cu index 74611a1fd8..6e349627b2 100644 --- a/cpp/test/sparse/solver/lanczos.cu +++ b/cpp/test/sparse/solver/lanczos.cu @@ -36,6 +36,7 @@ #include #include +#include #include #include @@ -265,6 +266,18 @@ class lanczos_tests : public ::testing::TestWithParam(handle, rng, v0.view(), 0, 1); std::tuple stats; From ee45ce786686b54d1972408b927d7fcd8ce0cf20 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Sat, 7 Dec 2024 00:35:26 -0600 Subject: [PATCH 67/79] Update cuda-python lower bounds to 12.6.2 / 11.8.5 (#2522) We require a newer cuda-python lower bound for new features and to use the new layout. This will fix a number of errors observed when the runtime version of cuda-python is older than the version used to build packages using Cython features from cuda-python. See https://github.com/rapidsai/build-planning/issues/117#issuecomment-2524250915 for details. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/raft/pull/2522 --- conda/environments/all_cuda-118_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-125_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-125_arch-x86_64.yaml | 2 +- conda/recipes/pylibraft/meta.yaml | 8 ++++---- conda/recipes/raft-dask/meta.yaml | 8 ++++---- dependencies.yaml | 4 ++-- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index f8201cbccf..e145aeb92e 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -14,7 +14,7 @@ dependencies: - cmake>=3.26.4,!=3.30.0 - cuda-nvtx=11.8 - cuda-profiler-api=11.8.86 -- cuda-python>=11.7.1,<12.0a0 +- cuda-python>=11.8.5,<12.0a0 - cuda-version=11.8 - cudatoolkit - cupy>=12.0.0 diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 66b97854ab..75dcffa95d 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -14,7 +14,7 @@ dependencies: - cmake>=3.26.4,!=3.30.0 - cuda-nvtx=11.8 - cuda-profiler-api=11.8.86 -- cuda-python>=11.7.1,<12.0a0 +- cuda-python>=11.8.5,<12.0a0 - cuda-version=11.8 - cudatoolkit - cupy>=12.0.0 diff --git a/conda/environments/all_cuda-125_arch-aarch64.yaml b/conda/environments/all_cuda-125_arch-aarch64.yaml index 1fd6edfb6f..bfa32c80d1 100644 --- a/conda/environments/all_cuda-125_arch-aarch64.yaml +++ b/conda/environments/all_cuda-125_arch-aarch64.yaml @@ -16,7 +16,7 @@ dependencies: - cuda-nvcc - cuda-nvtx-dev - cuda-profiler-api -- cuda-python>=12.0,<13.0a0 +- cuda-python>=12.6.2,<13.0a0 - cuda-version=12.5 - cupy>=12.0.0 - cxx-compiler diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 72108fed48..98ec334635 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -16,7 +16,7 @@ dependencies: - cuda-nvcc - cuda-nvtx-dev - cuda-profiler-api -- cuda-python>=12.0,<13.0a0 +- cuda-python>=12.6.2,<13.0a0 - cuda-version=12.5 - cupy>=12.0.0 - cxx-compiler diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index f1edf5d767..4a8ed29c85 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -43,10 +43,10 @@ requirements: - {{ stdlib("c") }} host: {% if cuda_major == "11" %} - - cuda-python >=11.7.1,<12.0a0 + - cuda-python >=11.8.5,<12.0a0 - cudatoolkit {% else %} - - cuda-python >=12.0,<13.0a0 + - cuda-python >=12.6.2,<13.0a0 - cuda-cudart-dev {% endif %} - cuda-version ={{ cuda_version }} @@ -61,10 +61,10 @@ requirements: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} - cudatoolkit - - cuda-python >=11.7.1,<12.0a0 + - cuda-python >=11.8.5,<12.0a0 {% else %} - cuda-cudart - - cuda-python >=12.0,<13.0a0 + - cuda-python >=12.6.2,<13.0a0 {% endif %} - libraft {{ version }} - libraft-headers {{ version }} diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index 14ffa5c092..a8be273f82 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -43,10 +43,10 @@ requirements: - {{ stdlib("c") }} host: {% if cuda_major == "11" %} - - cuda-python >=11.7.1,<12.0a0 + - cuda-python >=11.8.5,<12.0a0 - cudatoolkit {% else %} - - cuda-python >=12.0,<13.0a0 + - cuda-python >=12.6.2,<13.0a0 - cuda-cudart-dev {% endif %} - cuda-version ={{ cuda_version }} @@ -62,10 +62,10 @@ requirements: run: {% if cuda_major == "11" %} - cudatoolkit - - cuda-python >=11.7.1,<12.0a0 + - cuda-python >=11.8.5,<12.0a0 {% else %} - cuda-cudart - - cuda-python >=12.0,<13.0a0 + - cuda-python >=12.6.2,<13.0a0 {% endif %} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - dask-cuda ={{ minor_version }} diff --git a/dependencies.yaml b/dependencies.yaml index 80c7f29447..37ea223a01 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -197,11 +197,11 @@ dependencies: - matrix: cuda: "12.*" packages: - - &cuda_python12 cuda-python>=12.0,<13.0a0 + - &cuda_python12 cuda-python>=12.6.2,<13.0a0 - matrix: cuda: "11.*" packages: - - &cuda_python11 cuda-python>=11.7.1,<12.0a0 + - &cuda_python11 cuda-python>=11.8.5,<12.0a0 - matrix: packages: - &cuda_python cuda-python From d02ab5c618c9798d085a9223bc37041a465d587f Mon Sep 17 00:00:00 2001 From: Ben Frederickson Date: Mon, 9 Dec 2024 13:45:47 -0800 Subject: [PATCH 68/79] Skip gtests for Rmat Lanczos tests with cuda <= 11.4 (#2525) Similar to #2520, also skip gtests for cuda 11.4 on the RmatLanczosTest as well --- cpp/test/sparse/solver/lanczos.cu | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cpp/test/sparse/solver/lanczos.cu b/cpp/test/sparse/solver/lanczos.cu index 6e349627b2..128ab73747 100644 --- a/cpp/test/sparse/solver/lanczos.cu +++ b/cpp/test/sparse/solver/lanczos.cu @@ -109,6 +109,18 @@ class rmat_lanczos_tests void Run() { + int runtimeVersion; + cudaError_t result = cudaRuntimeGetVersion(&runtimeVersion); + + if (result == cudaSuccess) { + int major = runtimeVersion / 1000; + int minor = (runtimeVersion % 1000) / 10; + + // Skip gtests for CUDA 11.4.x and below because hard-coded results are causing issues. + // See https://github.com/rapidsai/raft/issues/2519 for more information. + if (major == 11 && minor <= 4) { GTEST_SKIP(); } + } + uint64_t n_edges = sparsity * ((long long)(1 << r_scale) * (long long)(1 << c_scale)); uint64_t n_nodes = 1 << std::max(r_scale, c_scale); uint64_t theta_len = std::max(r_scale, c_scale) * 4; From e8dca7cdb560fc5d6f9c9e0650fbd9185ffcfb68 Mon Sep 17 00:00:00 2001 From: Ray Douglass Date: Wed, 11 Dec 2024 13:12:10 -0500 Subject: [PATCH 69/79] Update Changelog [skip ci] --- CHANGELOG.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9caa5ef571..1d7c641b21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,42 @@ +# raft 24.12.00 (11 Dec 2024) + +## 🚨 Breaking Changes + +- Do not initialize the pinned mdarray at construction time ([#2478](https://github.com/rapidsai/raft/pull/2478)) [@achirkin](https://github.com/achirkin) + +## 🐛 Bug Fixes + +- Skip gtests for new lanczos solver when CUDA version is 11.4 or below. ([#2520](https://github.com/rapidsai/raft/pull/2520)) [@cjnolet](https://github.com/cjnolet) +- Switch `assert` to `static_assert` ([#2510](https://github.com/rapidsai/raft/pull/2510)) [@divyegala](https://github.com/divyegala) +- Revert use of new Lanczos solver in spectral clustering ([#2507](https://github.com/rapidsai/raft/pull/2507)) [@lowener](https://github.com/lowener) +- Put a ceiling on cuda-python ([#2486](https://github.com/rapidsai/raft/pull/2486)) [@bdice](https://github.com/bdice) +- Don't presume pointers location infers usability. ([#2480](https://github.com/rapidsai/raft/pull/2480)) [@robertmaynard](https://github.com/robertmaynard) +- Use Python for sccache hit rate computation. ([#2474](https://github.com/rapidsai/raft/pull/2474)) [@bdice](https://github.com/bdice) +- Allow compilation with CUDA 12.6.1 ([#2469](https://github.com/rapidsai/raft/pull/2469)) [@robertmaynard](https://github.com/robertmaynard) + +## 🚀 New Features + +- [FEA] Lanczos solver v2 ([#2481](https://github.com/rapidsai/raft/pull/2481)) [@lowener](https://github.com/lowener) + +## 🛠️ Improvements + +- Skip gtests for Rmat Lanczos tests with cuda <= 11.4 ([#2525](https://github.com/rapidsai/raft/pull/2525)) [@benfred](https://github.com/benfred) +- Upgrade to latest cutlass version ([#2503](https://github.com/rapidsai/raft/pull/2503)) [@vyasr](https://github.com/vyasr) +- Removing some left over places where implicit instantiations were being ignored in headers ([#2501](https://github.com/rapidsai/raft/pull/2501)) [@cjnolet](https://github.com/cjnolet) +- Remove leftover template project code. ([#2500](https://github.com/rapidsai/raft/pull/2500)) [@bdice](https://github.com/bdice) +- 2412 remove libraft vss instantiations ([#2498](https://github.com/rapidsai/raft/pull/2498)) [@cjnolet](https://github.com/cjnolet) +- Remove raft-ann-bench ([#2497](https://github.com/rapidsai/raft/pull/2497)) [@cjnolet](https://github.com/cjnolet) +- Pin FAISS Version for raft-ann-bench ([#2496](https://github.com/rapidsai/raft/pull/2496)) [@tarang-jain](https://github.com/tarang-jain) +- enforce wheel size limits and README formatting in CI, put a ceiling on Cython dependency ([#2490](https://github.com/rapidsai/raft/pull/2490)) [@jameslamb](https://github.com/jameslamb) +- Do not initialize the pinned mdarray at construction time ([#2478](https://github.com/rapidsai/raft/pull/2478)) [@achirkin](https://github.com/achirkin) +- Use environment variables in cache hit rate computation. ([#2475](https://github.com/rapidsai/raft/pull/2475)) [@bdice](https://github.com/bdice) +- devcontainer: replace `VAULT_HOST` with `AWS_ROLE_ARN` ([#2472](https://github.com/rapidsai/raft/pull/2472)) [@jjacobelli](https://github.com/jjacobelli) +- print sccache stats in builds ([#2470](https://github.com/rapidsai/raft/pull/2470)) [@jameslamb](https://github.com/jameslamb) +- make package installations in CI stricter ([#2467](https://github.com/rapidsai/raft/pull/2467)) [@jameslamb](https://github.com/jameslamb) +- Prune workflows based on changed files ([#2466](https://github.com/rapidsai/raft/pull/2466)) [@KyleFromNVIDIA](https://github.com/KyleFromNVIDIA) +- Merge branch-24.10 into branch-24.12 ([#2461](https://github.com/rapidsai/raft/pull/2461)) [@jameslamb](https://github.com/jameslamb) +- Update all rmm imports to use pylibrmm/librmm ([#2451](https://github.com/rapidsai/raft/pull/2451)) [@Matt711](https://github.com/Matt711) + # raft 24.10.00 (9 Oct 2024) ## 🚨 Breaking Changes From 1e5030d1b4f85a9f306c36f8a030494fa59aaaa4 Mon Sep 17 00:00:00 2001 From: Tamas Bela Feher Date: Wed, 11 Dec 2024 23:39:15 +0100 Subject: [PATCH 70/79] Fix rnd bit generation in rmat_rectangular_kernel (#2524) For certain architectures, the compiler always generates zero destination bit in the following loop https://github.com/rapidsai/raft/blob/ee45ce786686b54d1972408b927d7fcd8ce0cf20/cpp/include/raft/random/detail/rmat_rectangular_generator.cuh#L160-L162 irrespective of the random value that shall determine which bit to use for `dst_id`. This PR refactors the loop. This way the `dst_id` number has the desired random distribution for all bits. Authors: - Tamas Bela Feher (https://github.com/tfeher) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/raft/pull/2524 --- .../detail/rmat_rectangular_generator.cuh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/cpp/include/raft/random/detail/rmat_rectangular_generator.cuh b/cpp/include/raft/random/detail/rmat_rectangular_generator.cuh index 9ad7c68f87..24207ba6db 100644 --- a/cpp/include/raft/random/detail/rmat_rectangular_generator.cuh +++ b/cpp/include/raft/random/detail/rmat_rectangular_generator.cuh @@ -151,15 +151,16 @@ RAFT_KERNEL rmat_gen_kernel(IdxT* out, raft::random::PCGenerator gen{r.seed, r.base_subsequence + idx, 0}; auto min_scale = min(r_scale, c_scale); IdxT i = 0; - for (; i < min_scale; ++i) { - gen_and_update_bits(src_id, dst_id, a, a + b, a + b + c, r_scale, c_scale, i, gen); - } - for (; i < r_scale; ++i) { - gen_and_update_bits(src_id, dst_id, a + b, a + b, ProbT(1), r_scale, c_scale, i, gen); - } - for (; i < c_scale; ++i) { - gen_and_update_bits(src_id, dst_id, a + c, ProbT(1), ProbT(1), r_scale, c_scale, i, gen); + // Whether we have more rows than columns. + const bool more_rows = r_scale > c_scale; + + for (; i < max_scale; ++i) { + ProbT A = (i < min_scale) ? a : (more_rows ? a + b : a + c); + ProbT AB = (i < min_scale) ? a + b : (more_rows ? a + b : ProbT(1)); + ProbT ABC = (i < min_scale) ? a + b + c : ProbT(1); + gen_and_update_bits(src_id, dst_id, A, AB, ABC, r_scale, c_scale, i, gen); } + store_ids(out, out_src, out_dst, src_id, dst_id, idx, n_edges); } From 3720d8e91c21ec95d3dbe8e0d1a4515eb60fa7fa Mon Sep 17 00:00:00 2001 From: rhdong Date: Wed, 11 Dec 2024 14:41:14 -0800 Subject: [PATCH 71/79] [Opt] Optimizing the performance of `bitmap_to_csr` (#2516) This PR optimizes the performance of `bitmap_to_csr` related kernels by 14~1000 times. It could also benefit the `bitset_to_csr` in the future. #### After (Updated Dec 08) ```shell --------------------------------------------------------------------------------------------------- Benchmark Time CPU Iterations --------------------------------------------------------------------------------------------------- BitmapToCsrBench/0/manual_time 0.161 ms 0.197 ms 4350 rows*cols=1*100000000 sparsity=0.95 BitmapToCsrBench/1/manual_time 0.110 ms 0.147 ms 6363 rows*cols=1*100000000 sparsity=0.99 BitmapToCsrBench/2/manual_time 14.2 ms 14.2 ms 50 rows*cols=100*100000000 sparsity=0.95 BitmapToCsrBench/3/manual_time 8.76 ms 8.80 ms 80 rows*cols=100*100000000 sparsity=0.99 ``` #### Before ```shell --------------------------------------------------------------------------------------------------- Benchmark Time CPU Iterations --------------------------------------------------------------------------------------------------- BitmapToCsrBench/0/manual_time 176 ms 176 ms 4 rows*cols=1*100000000 sparsity=0.95 BitmapToCsrBench/1/manual_time 146 ms 146 ms 5 rows*cols=1*100000000 sparsity=0.99 BitmapToCsrBench/2/manual_time 180 ms 180 ms 4 rows*cols=100*100000000 sparsity=0.95 BitmapToCsrBench/3/manual_time 148 ms 148 ms 5 rows*cols=100*100000000 sparsity=0.99 ``` Authors: - rhdong (https://github.com/rhdong) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/raft/pull/2516 --- cpp/bench/prims/sparse/bitmap_to_csr.cu | 26 +- .../sparse/convert/detail/bitmap_to_csr.cuh | 358 ++++++++++-------- cpp/include/raft/util/device_loads_stores.cuh | 42 ++ cpp/test/sparse/convert_csr.cu | 78 ++-- 4 files changed, 319 insertions(+), 185 deletions(-) diff --git a/cpp/bench/prims/sparse/bitmap_to_csr.cu b/cpp/bench/prims/sparse/bitmap_to_csr.cu index ed53df3265..71aabb1bf9 100644 --- a/cpp/bench/prims/sparse/bitmap_to_csr.cu +++ b/cpp/bench/prims/sparse/bitmap_to_csr.cu @@ -71,7 +71,7 @@ struct BitmapToCsrBench : public fixture { index_t create_sparse_matrix(index_t m, index_t n, float sparsity, std::vector& bitmap) { index_t total = static_cast(m * n); - index_t num_ones = static_cast((total * 1.0f) * sparsity); + index_t num_ones = static_cast((total * 1.0f) * (1.0f - sparsity)); index_t res = num_ones; for (auto& item : bitmap) { @@ -141,7 +141,27 @@ const std::vector> getInputs() }; const std::vector params_group = raft::util::itertools::product( - {index_t(10), index_t(1024)}, {index_t(1024 * 1024)}, {0.01f, 0.1f, 0.2f, 0.5f}); + {index_t(10), index_t(1024)}, {index_t(1024 * 1024)}, {0.99f, 0.9f, 0.8f, 0.5f}); + + param_vec.reserve(params_group.size()); + for (TestParams params : params_group) { + param_vec.push_back(bench_param({params.m, params.n, params.sparsity})); + } + return param_vec; +} + +template +const std::vector> getLargeInputs() +{ + std::vector> param_vec; + struct TestParams { + index_t m; + index_t n; + float sparsity; + }; + + const std::vector params_group = raft::util::itertools::product( + {index_t(1), index_t(100)}, {index_t(100 * 1000000)}, {0.95f, 0.99f}); param_vec.reserve(params_group.size()); for (TestParams params : params_group) { @@ -153,4 +173,6 @@ const std::vector> getInputs() RAFT_BENCH_REGISTER((BitmapToCsrBench), "", getInputs()); RAFT_BENCH_REGISTER((BitmapToCsrBench), "", getInputs()); +RAFT_BENCH_REGISTER((BitmapToCsrBench), "", getLargeInputs()); + } // namespace raft::bench::sparse diff --git a/cpp/include/raft/sparse/convert/detail/bitmap_to_csr.cuh b/cpp/include/raft/sparse/convert/detail/bitmap_to_csr.cuh index 769d5de9be..866923d647 100644 --- a/cpp/include/raft/sparse/convert/detail/bitmap_to_csr.cuh +++ b/cpp/include/raft/sparse/convert/detail/bitmap_to_csr.cuh @@ -21,6 +21,7 @@ #include #include #include +#include #include @@ -41,61 +42,68 @@ namespace sparse { namespace convert { namespace detail { -// Threads per block in calc_nnz_by_rows_kernel. -static const constexpr int calc_nnz_by_rows_tpb = 32; +// Threads per block in bitmap_to_csr. +static const constexpr int bitmap_to_csr_tpb = 256; template -RAFT_KERNEL __launch_bounds__(calc_nnz_by_rows_tpb) calc_nnz_by_rows_kernel(const bitmap_t* bitmap, - index_t num_rows, - index_t num_cols, - index_t bitmap_num, - nnz_t* nnz_per_row) +RAFT_KERNEL __launch_bounds__(bitmap_to_csr_tpb) calc_nnz_by_rows_kernel(const bitmap_t* bitmap, + index_t num_rows, + index_t num_cols, + index_t bitmap_num, + nnz_t* sub_col_nnz, + index_t bits_per_sub_col) { - constexpr bitmap_t FULL_MASK = ~bitmap_t(0u); - constexpr bitmap_t ONE = bitmap_t(1u); + using mutable_bitmap_t = typename std::remove_const_t; + using BlockReduce = cub::BlockReduce; + + __shared__ typename BlockReduce::TempStorage reduce_storage; + constexpr index_t BITS_PER_BITMAP = sizeof(bitmap_t) * 8; - auto block = cg::this_thread_block(); - auto tile = cg::tiled_partition<32>(block); + const auto tid = threadIdx.x; + const auto row = blockIdx.x; - int lane_id = threadIdx.x & 0x1f; + const auto num_sub_cols = gridDim.y; + const auto sub_col = blockIdx.y; - for (index_t row = blockIdx.x; row < num_rows; row += gridDim.x) { - index_t offset = 0; - index_t s_bit = row * num_cols; - index_t e_bit = s_bit + num_cols; - index_t l_sum = 0; + size_t s_bit = size_t(row) * num_cols + sub_col * bits_per_sub_col; + size_t e_bit = min(s_bit + bits_per_sub_col, size_t(num_cols) * (row + 1)); - int s_gap = 0; - int e_gap = 0; + nnz_t l_sum = 0; + nnz_t g_sum = 0; - while (offset < num_cols) { - index_t bitmap_idx = lane_id + (s_bit + offset) / BITS_PER_BITMAP; - std::remove_const_t l_bitmap = 0; + index_t s_offset = s_bit % BITS_PER_BITMAP; + size_t bitmap_idx = s_bit / BITS_PER_BITMAP; - if (bitmap_idx * BITS_PER_BITMAP < e_bit) { l_bitmap = bitmap[bitmap_idx]; } + if (tid == 0 && s_offset != 0) { + mutable_bitmap_t l_bitmap = bitmap[bitmap_idx]; - offset += BITS_PER_BITMAP * warpSize; + l_bitmap >>= s_offset; - s_gap = s_bit - bitmap_idx * BITS_PER_BITMAP; - if (s_gap > 0) { - l_bitmap >>= s_gap; - l_bitmap <<= s_gap; - offset -= s_gap; - } + size_t remaining_bits = min(size_t(BITS_PER_BITMAP - s_offset), e_bit - s_bit); - e_gap = (bitmap_idx + 1) * BITS_PER_BITMAP - e_bit; - if (e_gap > 0) { - l_bitmap <<= e_gap; - l_bitmap >>= e_gap; - } - l_sum += static_cast(raft::detail::popc(l_bitmap)); + if (remaining_bits < BITS_PER_BITMAP) { + l_bitmap &= ((mutable_bitmap_t(1) << remaining_bits) - 1); } + l_sum += static_cast(raft::detail::popc(l_bitmap)); + } + if (s_offset != 0) { s_bit += (BITS_PER_BITMAP - s_offset); } - l_sum = cg::reduce(tile, l_sum, cg::plus()); + for (size_t bit_idx = s_bit; bit_idx < e_bit; bit_idx += BITS_PER_BITMAP * blockDim.x) { + mutable_bitmap_t l_bitmap = 0; + bitmap_idx = bit_idx / BITS_PER_BITMAP + tid; - if (lane_id == 0) { *(nnz_per_row + row) += static_cast(l_sum); } + index_t remaining_bits = min(BITS_PER_BITMAP, index_t(e_bit - bitmap_idx * BITS_PER_BITMAP)); + + if (bitmap_idx * BITS_PER_BITMAP < e_bit) { l_bitmap = bitmap[bitmap_idx]; } + + if (remaining_bits < BITS_PER_BITMAP) { + l_bitmap &= ((mutable_bitmap_t(1) << remaining_bits) - 1); + } + l_sum += static_cast(raft::detail::popc(l_bitmap)); } + g_sum = BlockReduce(reduce_storage).Reduce(l_sum, cub::Sum()); + stg(g_sum, sub_col_nnz + sub_col + row * num_sub_cols, tid == 0); } template @@ -103,144 +111,164 @@ void calc_nnz_by_rows(raft::resources const& handle, const bitmap_t* bitmap, index_t num_rows, index_t num_cols, - nnz_t* nnz_per_row) + nnz_t* sub_col_nnz, + size_t& sub_nnz_size, + index_t& bits_per_sub_col) { - auto stream = resource::get_cuda_stream(handle); - const index_t total = num_rows * num_cols; - const index_t bitmap_num = raft::ceildiv(total, index_t(sizeof(bitmap_t) * 8)); - - int dev_id, sm_count, blocks_per_sm; + if (sub_nnz_size == 0) { + bits_per_sub_col = bitmap_to_csr_tpb * sizeof(index_t) * 8 * 8; + auto grid_dim_y = (num_cols + bits_per_sub_col - 1) / bits_per_sub_col; + sub_nnz_size = num_rows * ((num_cols + bits_per_sub_col - 1) / bits_per_sub_col); + return; + } + auto stream = resource::get_cuda_stream(handle); + const size_t total = num_rows * num_cols; + const size_t bitmap_num = + (total + index_t(sizeof(bitmap_t) * 8) - 1) / index_t(sizeof(bitmap_t) * 8); - cudaGetDevice(&dev_id); - cudaDeviceGetAttribute(&sm_count, cudaDevAttrMultiProcessorCount, dev_id); - cudaOccupancyMaxActiveBlocksPerMultiprocessor( - &blocks_per_sm, calc_nnz_by_rows_kernel, calc_nnz_by_rows_tpb, 0); + auto block_x = num_rows; + auto block_y = sub_nnz_size / num_rows; + dim3 grid(block_x, block_y, 1); - index_t max_active_blocks = sm_count * blocks_per_sm; - auto grid = std::min(max_active_blocks, raft::ceildiv(bitmap_num, index_t(calc_nnz_by_rows_tpb))); - auto block = calc_nnz_by_rows_tpb; + auto block = bitmap_to_csr_tpb; - calc_nnz_by_rows_kernel - <<>>(bitmap, num_rows, num_cols, bitmap_num, nnz_per_row); + calc_nnz_by_rows_kernel<<>>( + bitmap, num_rows, num_cols, bitmap_num, sub_col_nnz, bits_per_sub_col); RAFT_CUDA_TRY(cudaPeekAtLastError()); } -/* - Execute the exclusive_scan within one warp with no inter-warp communication. - This function calculates the exclusive prefix sum of `value` across threads within the same warp. - Each thread in the warp will end up with the sum of all the values of the threads with lower IDs - in the same warp, with the first thread always getting a sum of 0. -*/ -template -RAFT_DEVICE_INLINE_FUNCTION value_t warp_exclusive_scan(value_t value) -{ - int lane_id = threadIdx.x & 0x1f; - value_t shifted_value = __shfl_up_sync(0xffffffff, value, 1, warpSize); - if (lane_id == 0) shifted_value = 0; - - value_t sum = shifted_value; - - for (int i = 1; i < warpSize; i *= 2) { - value_t n = __shfl_up_sync(0xffffffff, sum, i, warpSize); - if (lane_id >= i) { sum += n; } - } - return sum; -} - -// Threads per block in fill_indices_by_rows_kernel. -static const constexpr int fill_indices_by_rows_tpb = 32; - template -RAFT_KERNEL __launch_bounds__(fill_indices_by_rows_tpb) +RAFT_KERNEL __launch_bounds__(bitmap_to_csr_tpb) fill_indices_by_rows_kernel(const bitmap_t* bitmap, - const index_t* indptr, - index_t num_rows, - index_t num_cols, + index_t* indptr, + size_t num_rows, + size_t num_cols, nnz_t nnz, - index_t bitmap_num, - index_t* indices) + index_t* indices, + nnz_t* sub_col_nnz, + index_t bits_per_sub_col) { - constexpr bitmap_t FULL_MASK = ~bitmap_t(0u); constexpr bitmap_t ONE = bitmap_t(1u); constexpr index_t BITS_PER_BITMAP = sizeof(bitmap_t) * 8; - int lane_id = threadIdx.x & 0x1f; + using mutable_bitmap_t = typename std::remove_const_t; + using BlockScan = cub::BlockScan; + + __shared__ typename BlockScan::TempStorage scan_storage; + + const auto tid = threadIdx.x; + const auto row = blockIdx.x; + + const auto num_sub_cols = gridDim.y; + const auto sub_col = blockIdx.y; // Ensure the HBM allocated for CSR values is sufficient to handle all non-zero bitmap bits. // An assert will trigger if the allocated HBM is insufficient when `NDEBUG` isn't defined. // Note: Assertion is active only if `NDEBUG` is undefined. if constexpr (check_nnz) { - if (lane_id == 0) { assert(nnz < indptr[num_rows]); } + if (tid == 0) { assert(nnz < sub_col_nnz[num_rows * num_sub_cols]); } } + size_t s_bit = size_t(row) * num_cols + sub_col * bits_per_sub_col; + size_t e_bit = min(s_bit + bits_per_sub_col, size_t(num_cols) * (row + 1)); + + size_t l_sum = 0; + __shared__ size_t g_sum; + + index_t s_offset = s_bit % BITS_PER_BITMAP; + size_t bitmap_idx = s_bit / BITS_PER_BITMAP; + + if (tid == 0 && row == 0 && sub_col == 0) { indptr[0] = 0; } + if (tid == 0 && sub_col == 0) { indptr[row + 1] = sub_col_nnz[(row + 1) * num_sub_cols]; } + + size_t g_nnz = sub_col_nnz[sub_col + row * num_sub_cols]; + index_t* sub_cols_indices_addr = indices + g_nnz; + + bool guard[BITS_PER_BITMAP]; + + index_t g_bits = sub_col * bits_per_sub_col + tid * BITS_PER_BITMAP; + + if (tid == 0 && s_offset != 0) { + mutable_bitmap_t l_bitmap = bitmap[bitmap_idx]; + l_bitmap >>= s_offset; + + size_t remaining_bits = min(size_t(BITS_PER_BITMAP - s_offset), e_bit - s_bit); + if (remaining_bits < BITS_PER_BITMAP) { + l_bitmap &= ((mutable_bitmap_t(1) << remaining_bits) - 1); + } + +#pragma unroll + for (int i = 0; i < BITS_PER_BITMAP; i++) { + guard[i] = l_bitmap & (ONE << i); + } #pragma unroll - for (index_t row = blockIdx.x; row < num_rows; row += gridDim.x) { - index_t g_sum = 0; - index_t s_bit = row * num_cols; - index_t e_bit = s_bit + num_cols; - index_t indptr_row = indptr[row]; + for (int i = 0; i < BITS_PER_BITMAP; i++) { + stg(index_t(i + g_bits), sub_cols_indices_addr + l_sum, guard[i]); + l_sum += guard[i]; + } + } + + if (tid == 0) { g_sum = l_sum; } + __syncthreads(); + + if (s_offset != 0) { + s_bit += (BITS_PER_BITMAP - s_offset); + g_bits += (BITS_PER_BITMAP - s_offset); + } + + for (size_t bit_idx = s_bit; bit_idx < e_bit; bit_idx += BITS_PER_BITMAP * blockDim.x) { + mutable_bitmap_t l_bitmap = 0; + bitmap_idx = bit_idx / BITS_PER_BITMAP + tid; + + if (bitmap_idx * BITS_PER_BITMAP < e_bit) { l_bitmap = bitmap[bitmap_idx]; } + + index_t remaining_bits = min(BITS_PER_BITMAP, index_t(e_bit - bitmap_idx * BITS_PER_BITMAP)); + if (remaining_bits < BITS_PER_BITMAP) { + l_bitmap &= ((mutable_bitmap_t(1) << remaining_bits) - 1); + } + + int l_bits = raft::detail::popc(l_bitmap); + int l_sum_32b = 0; + BlockScan(scan_storage).InclusiveSum(l_bits, l_sum_32b); + l_sum = l_sum_32b + g_sum - l_bits; + __syncthreads(); #pragma unroll - for (index_t offset = 0; offset < num_cols; offset += BITS_PER_BITMAP * warpSize) { - index_t bitmap_idx = lane_id + (s_bit + offset) / BITS_PER_BITMAP; - std::remove_const_t l_bitmap = 0; - index_t l_offset = offset + lane_id * BITS_PER_BITMAP - (s_bit % BITS_PER_BITMAP); - - if (bitmap_idx * BITS_PER_BITMAP < e_bit) { l_bitmap = bitmap[bitmap_idx]; } - - if (s_bit > bitmap_idx * BITS_PER_BITMAP) { - l_bitmap >>= (s_bit - bitmap_idx * BITS_PER_BITMAP); - l_bitmap <<= (s_bit - bitmap_idx * BITS_PER_BITMAP); - } - - if ((bitmap_idx + 1) * BITS_PER_BITMAP > e_bit) { - l_bitmap <<= ((bitmap_idx + 1) * BITS_PER_BITMAP - e_bit); - l_bitmap >>= ((bitmap_idx + 1) * BITS_PER_BITMAP - e_bit); - } - - index_t l_sum = - g_sum + warp_exclusive_scan(static_cast(raft::detail::popc(l_bitmap))); - - for (int i = 0; i < BITS_PER_BITMAP; i++) { - if (l_bitmap & (ONE << i)) { - indices[indptr_row + l_sum] = l_offset + i; - l_sum++; - } - } - g_sum = __shfl_sync(0xffffffff, l_sum, warpSize - 1); + for (int i = 0; i < BITS_PER_BITMAP; i++) { + guard[i] = l_bitmap & (ONE << i); } +#pragma unroll + for (int i = 0; i < BITS_PER_BITMAP; i++) { + stg(index_t(i + g_bits), sub_cols_indices_addr + l_sum, guard[i]); + l_sum += guard[i]; + } + + if (threadIdx.x == (bitmap_to_csr_tpb - 1)) { g_sum += (l_sum_32b); } + g_bits += BITS_PER_BITMAP * blockDim.x; } } template void fill_indices_by_rows(raft::resources const& handle, const bitmap_t* bitmap, - const index_t* indptr, + index_t* indptr, index_t num_rows, index_t num_cols, nnz_t nnz, - index_t* indices) + index_t* indices, + nnz_t* sub_col_nnz, + index_t bits_per_sub_col, + size_t sub_nnz_size) { - auto stream = resource::get_cuda_stream(handle); - const index_t total = num_rows * num_cols; - const index_t bitmap_num = raft::ceildiv(total, index_t(sizeof(bitmap_t) * 8)); - - int dev_id, sm_count, blocks_per_sm; - - cudaGetDevice(&dev_id); - cudaDeviceGetAttribute(&sm_count, cudaDevAttrMultiProcessorCount, dev_id); - cudaOccupancyMaxActiveBlocksPerMultiprocessor( - &blocks_per_sm, - fill_indices_by_rows_kernel, - fill_indices_by_rows_tpb, - 0); - - index_t max_active_blocks = sm_count * blocks_per_sm; - auto grid = std::min(max_active_blocks, num_rows); - auto block = fill_indices_by_rows_tpb; - - fill_indices_by_rows_kernel - <<>>(bitmap, indptr, num_rows, num_cols, nnz, bitmap_num, indices); + auto stream = resource::get_cuda_stream(handle); + auto block_x = num_rows; + auto block_y = sub_nnz_size / num_rows; + dim3 grid(block_x, block_y, 1); + + auto block = bitmap_to_csr_tpb; + + fill_indices_by_rows_kernel<<>>( + bitmap, indptr, num_rows, num_cols, nnz, indices, sub_col_nnz, bits_per_sub_col); RAFT_CUDA_TRY(cudaPeekAtLastError()); } @@ -252,6 +280,7 @@ void bitmap_to_csr(raft::resources const& handle, raft::core::bitmap_view bitmap, csr_matrix_t& csr) { + using nnz_t = typename csr_matrix_t::nnz_type; auto csr_view = csr.structure_view(); if (csr_view.get_n_rows() == 0 || csr_view.get_n_cols() == 0 || csr_view.get_nnz() == 0) { @@ -274,25 +303,50 @@ void bitmap_to_csr(raft::resources const& handle, RAFT_CUDA_TRY(cudaMemsetAsync(indptr, 0, (csr_view.get_n_rows() + 1) * sizeof(index_t), stream)); - calc_nnz_by_rows(handle, bitmap.data(), csr_view.get_n_rows(), csr_view.get_n_cols(), indptr); - thrust::exclusive_scan(thrust_policy, indptr, indptr + csr_view.get_n_rows() + 1, indptr); + size_t sub_nnz_size = 0; + index_t bits_per_sub_col = 0; + + // Get buffer size and number of bits per each sub-columns + calc_nnz_by_rows(handle, + bitmap.data(), + csr_view.get_n_rows(), + csr_view.get_n_cols(), + static_cast(nullptr), + sub_nnz_size, + bits_per_sub_col); + + rmm::device_async_resource_ref device_memory = resource::get_workspace_resource(handle); + rmm::device_uvector sub_nnz(sub_nnz_size + 1, stream, device_memory); + + calc_nnz_by_rows(handle, + bitmap.data(), + csr_view.get_n_rows(), + csr_view.get_n_cols(), + sub_nnz.data(), + sub_nnz_size, + bits_per_sub_col); + + thrust::exclusive_scan( + thrust_policy, sub_nnz.data(), sub_nnz.data() + sub_nnz_size + 1, sub_nnz.data()); if constexpr (is_device_csr_sparsity_owning_v) { index_t nnz = 0; RAFT_CUDA_TRY(cudaMemcpyAsync( - &nnz, indptr + csr_view.get_n_rows(), sizeof(index_t), cudaMemcpyDeviceToHost, stream)); + &nnz, sub_nnz.data() + sub_nnz_size, sizeof(index_t), cudaMemcpyDeviceToHost, stream)); resource::sync_stream(handle); csr.initialize_sparsity(nnz); } constexpr bool check_nnz = is_device_csr_sparsity_preserving_v; - fill_indices_by_rows( - handle, - bitmap.data(), - indptr, - csr_view.get_n_rows(), - csr_view.get_n_cols(), - csr_view.get_nnz(), - indices); + fill_indices_by_rows(handle, + bitmap.data(), + indptr, + csr_view.get_n_rows(), + csr_view.get_n_cols(), + csr_view.get_nnz(), + indices, + sub_nnz.data(), + bits_per_sub_col, + sub_nnz_size); thrust::fill_n(thrust_policy, csr.get_elements().data(), diff --git a/cpp/include/raft/util/device_loads_stores.cuh b/cpp/include/raft/util/device_loads_stores.cuh index 2c954ec99a..c1b668fed6 100644 --- a/cpp/include/raft/util/device_loads_stores.cuh +++ b/cpp/include/raft/util/device_loads_stores.cuh @@ -739,4 +739,46 @@ DI void block_copy(raft::device_span dst, const raft::device_span src) /** @} */ +/** + * @defgroup GlobalStores Global Store Operations + * @{ + * @brief Perform conditional stores to global memory. + * + * These functions store data to a specified global memory address, + * controlled by a guard flag to enable conditional execution. + * + * @param[in] reg The data to store in global memory. + * The type of `reg` determines the size of the store. + * @param[in] addr The global memory address where the data will be stored. + * @param[in] guard A flag to conditionally enable the store operation. + * If `true`, the store is performed; otherwise, it is skipped + */ +DI void stg(const int& reg, void* addr, bool guard) +{ + asm volatile( + "{\n" + ".reg .pred p;\n" + "setp.ne.b32 p, %2, 0;\n" + "@p st.global.b32 [%0], %1;\n" + "}\n" + : + : "l"(addr), "r"(reg), "r"((int)guard) + : "memory"); +} + +DI void stg(const int64_t& reg, void* addr, bool guard) +{ + asm volatile( + "{\n" + ".reg .pred p;\n" + "setp.ne.b32 p, %2, 0;\n" + "@p st.global.b64 [%0], %1;\n" + "}\n" + : + : "l"(addr), "l"(reg), "r"((int)guard) + : "memory"); +} + +/** @} */ + } // namespace raft diff --git a/cpp/test/sparse/convert_csr.cu b/cpp/test/sparse/convert_csr.cu index 1cd49b0bbd..c1a495ea3d 100644 --- a/cpp/test/sparse/convert_csr.cu +++ b/cpp/test/sparse/convert_csr.cu @@ -249,7 +249,7 @@ class BitmapToCSRTest : public ::testing::TestWithParam& bitmap) { index_t total = static_cast(m * n); - index_t num_ones = static_cast((total * 1.0f) * sparsity); + index_t num_ones = static_cast((total * 1.0f) * (1.0f - sparsity)); index_t res = num_ones; for (auto& item : bitmap) { @@ -257,7 +257,7 @@ class BitmapToCSRTest : public ::testing::TestWithParam dis(0, total - 1); while (num_ones > 0) { @@ -318,8 +318,8 @@ class BitmapToCSRTest : public ::testing::TestWithParam cols1(col_indices1.begin() + start_idx, col_indices1.begin() + end_idx); - std::vector cols2(col_indices2.begin() + start_idx, col_indices2.begin() + end_idx); + std::vector cols1(col_indices1.begin() + start_idx, col_indices1.begin() + end_idx); + std::vector cols2(col_indices2.begin() + start_idx, col_indices2.begin() + end_idx); std::sort(cols1.begin(), cols1.end()); std::sort(cols2.begin(), cols2.end()); @@ -396,9 +396,13 @@ class BitmapToCSRTest : public ::testing::TestWithParam( - values_expected_d.data(), values_d.data(), nnz, raft::Compare(), stream)); + EXPECT_TRUE(csr_compare(indptr_h, indices_h, indptr_expected_h, indices_expected_h)) + << " n_row: " << params.n_rows << ", n_cols: " << params.n_cols << ", nnz: " << nnz + << ", random_number: " << random_number; + EXPECT_TRUE(raft::devArrMatch( + values_expected_d.data(), values_d.data(), nnz, raft::Compare(), stream)) + << " n_row: " << params.n_rows << ", n_cols: " << params.n_cols << ", nnz: " << nnz + << ", random_number: " << random_number; } protected: @@ -418,6 +422,8 @@ class BitmapToCSRTest : public ::testing::TestWithParam indptr_expected_d; rmm::device_uvector indices_expected_d; rmm::device_uvector values_expected_d; + + unsigned int random_number; }; using BitmapToCSRTestI = BitmapToCSRTest; @@ -426,40 +432,50 @@ TEST_P(BitmapToCSRTestI, Result) { Run(); } using BitmapToCSRTestL = BitmapToCSRTest; TEST_P(BitmapToCSRTestL, Result) { Run(); } +using BitmapToCSRTestLOnLargeSize = BitmapToCSRTest; +TEST_P(BitmapToCSRTestLOnLargeSize, Result) { Run(); } + template const std::vector> bitmaptocsr_inputs = { - {0, 0, 0.2, false}, - {10, 32, 0.4, false}, - {10, 3, 0.2, false}, - {32, 1024, 0.4, false}, - {1024, 1048576, 0.01, false}, - {1024, 1024, 0.4, false}, - {64 * 1024 + 10, 2, 0.3, false}, // 64K + 10 is slightly over maximum of blockDim.y - {16, 16, 0.3, false}, // No peeling-remainder - {17, 16, 0.3, false}, // Check peeling-remainder - {18, 16, 0.3, false}, // Check peeling-remainder - {32 + 9, 33, 0.2, false}, // Check peeling-remainder - {2, 33, 0.2, false}, // Check peeling-remainder - {0, 0, 0.2, true}, - {10, 32, 0.4, true}, - {10, 3, 0.2, true}, - {32, 1024, 0.4, true}, - {1024, 1048576, 0.01, true}, - {1024, 1024, 0.4, true}, - {64 * 1024 + 10, 2, 0.3, true}, // 64K + 10 is slightly over maximum of blockDim.y - {16, 16, 0.3, true}, // No peeling-remainder - {17, 16, 0.3, true}, // Check peeling-remainder - {18, 16, 0.3, true}, // Check peeling-remainder - {32 + 9, 33, 0.2, true}, // Check peeling-remainder - {2, 33, 0.2, true}, // Check peeling-remainder + {0, 0, 0.8, false}, + {10, 32, 0.6, false}, + {10, 3, 0.8, false}, + {32, 1024, 0.6, false}, + {1024, 1048576, 0.99, false}, + {1024, 1024, 0.6, false}, + {64 * 1024 + 10, 2, 0.7, false}, // 64K + 10 is slightly over maximum of blockDim.y + {16, 16, 0.7, false}, // No peeling-remainder + {17, 16, 0.7, false}, // Check peeling-remainder + {18, 16, 0.7, false}, // Check peeling-remainder + {32 + 9, 33, 0.8, false}, // Check peeling-remainder + {2, 33, 0.8, false}, // Check peeling-remainder + {0, 0, 0.8, true}, + {10, 32, 0.6, true}, + {10, 3, 0.8, true}, + {32, 1024, 0.6, true}, + {1024, 1048576, 0.99, true}, + {1024, 1024, 0.6, true}, + {64 * 1024 + 10, 2, 0.7, true}, // 64K + 10 is slightly over maximum of blockDim.y + {16, 16, 0.7, true}, // No peeling-remainder + {17, 16, 0.7, true}, // Check peeling-remainder + {18, 16, 0.7, true}, // Check peeling-remainder + {32 + 9, 33, 0.8, true}, // Check peeling-remainder + {2, 33, 0.8, true}, // Check peeling-remainder }; +template +const std::vector> bitmaptocsr_large_inputs = { + {100, 100000000, 0.99, true}, {100, 100000000, 0.95, false}, {100, 100000000 + 17, 0.95, false}}; + INSTANTIATE_TEST_CASE_P(SparseConvertCSRTest, BitmapToCSRTestI, ::testing::ValuesIn(bitmaptocsr_inputs)); INSTANTIATE_TEST_CASE_P(SparseConvertCSRTest, BitmapToCSRTestL, ::testing::ValuesIn(bitmaptocsr_inputs)); +INSTANTIATE_TEST_CASE_P(SparseConvertCSRTest, + BitmapToCSRTestLOnLargeSize, + ::testing::ValuesIn(bitmaptocsr_large_inputs)); } // namespace sparse } // namespace raft From bfd190687ee396374b7106d9ac26add73b57b22a Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 17 Dec 2024 12:08:26 -0600 Subject: [PATCH 72/79] reduce duplication, removed unused things in dependencies.yaml (#2529) Proposes some small cleanup for `dependencies.yaml` * removes `rapids_build_setuptools` dependency group - *#2497 removed the last use of `setuptools` here* * breaks `cuda-python` and `rmm` out into `depends_on_*` groups to reduce duplication, and for consistency with other RAPIDS projects ([docs explaining this](https://github.com/rapidsai/build-planning/blob/d9e3c606d95c835ee384ac6480a4af0ac6cb024a/docs/docs/packaging.md#L181)) * alphabetizes lists Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/raft/pull/2529 --- dependencies.yaml | 156 ++++++++++++++++++---------------------------- 1 file changed, 60 insertions(+), 96 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 37ea223a01..dc1807fbf9 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -6,22 +6,22 @@ files: cuda: ["11.8", "12.5"] arch: [x86_64, aarch64] includes: - - rapids_build - - build_pylibraft + - checks - cuda - cuda_version + - depends_on_cuda_python - depends_on_cupy - depends_on_distributed_ucxx + - depends_on_rmm - develop - - checks - - test_libraft - docs - - rapids_build_setuptools + - rapids_build - rapids_build_skbuild - - run_raft_dask - run_pylibraft - - test_python_common + - run_raft_dask + - test_libraft - test_pylibraft + - test_python_common test_cpp: output: none includes: @@ -31,10 +31,10 @@ files: output: none includes: - cuda_version + - depends_on_cupy - py_version - - test_python_common - test_pylibraft - - depends_on_cupy + - test_python_common checks: output: none includes: @@ -62,8 +62,9 @@ files: table: tool.rapids-build-backend key: requires includes: + - depends_on_cuda_python + - depends_on_rmm - rapids_build - - build_pylibraft py_run_pylibraft: output: pyproject pyproject_dir: python/pylibraft @@ -71,6 +72,8 @@ files: table: project includes: - cuda_wheels + - depends_on_cuda_python + - depends_on_rmm - run_pylibraft py_test_pylibraft: output: pyproject @@ -79,9 +82,9 @@ files: table: project.optional-dependencies key: test includes: - - test_python_common - - test_pylibraft - depends_on_cupy + - test_pylibraft + - test_python_common py_build_raft_dask: output: pyproject pyproject_dir: python/raft-dask @@ -96,16 +99,16 @@ files: table: tool.rapids-build-backend key: requires includes: - - rapids_build - depends_on_ucx_build + - rapids_build py_run_raft_dask: output: pyproject pyproject_dir: python/raft-dask extras: table: project includes: - - run_raft_dask - depends_on_distributed_ucxx + - run_raft_dask py_test_raft_dask: output: pyproject pyproject_dir: python/raft-dask @@ -125,7 +128,7 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: - - &rapids_build_backend rapids-build-backend>=0.3.0,<0.4.0.dev0 + - rapids-build-backend>=0.3.0,<0.4.0.dev0 - output_types: [conda] packages: - scikit-build-core>=0.10.0 @@ -180,44 +183,6 @@ dependencies: - matrix: {cuda: "11.2", arch: aarch64} packages: [nvcc_linux-aarch64=11.2] - build_pylibraft: - common: - - output_types: [conda] - packages: - - &rmm_unsuffixed rmm==25.2.*,>=0.0.0a0 - - output_types: requirements - packages: - # pip recognizes the index as a global option for the requirements.txt file - # This index is needed for rmm-cu{11,12}. - - --extra-index-url=https://pypi.nvidia.com - - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple - specific: - - output_types: [conda, requirements, pyproject] - matrices: - - matrix: - cuda: "12.*" - packages: - - &cuda_python12 cuda-python>=12.6.2,<13.0a0 - - matrix: - cuda: "11.*" - packages: - - &cuda_python11 cuda-python>=11.8.5,<12.0a0 - - matrix: - packages: - - &cuda_python cuda-python - - output_types: [requirements, pyproject] - matrices: - - matrix: - cuda: "12.*" - cuda_suffixed: "true" - packages: - - &rmm_cu12 rmm-cu12==25.2.*,>=0.0.0a0 - - matrix: - cuda: "11.*" - cuda_suffixed: "true" - packages: - - &rmm_cu11 rmm-cu11==25.2.*,>=0.0.0a0 - - {matrix: null, packages: [*rmm_unsuffixed] } checks: common: - output_types: [conda, requirements] @@ -398,13 +363,6 @@ dependencies: - recommonmark - sphinx-copybutton - sphinx-markdown-tables - rapids_build_setuptools: - common: - - output_types: [requirements, pyproject] - packages: - - wheel - - setuptools - - *rapids_build_backend py_version: specific: - output_types: conda @@ -429,42 +387,6 @@ dependencies: - output_types: [conda, pyproject] packages: - numpy>=1.23,<3.0a0 - - output_types: [conda] - packages: - - *rmm_unsuffixed - - output_types: requirements - packages: - # pip recognizes the index as a global option for the requirements.txt file - # This index is needed for cudf and rmm. - - --extra-index-url=https://pypi.nvidia.com - - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple - specific: - - output_types: [conda, requirements, pyproject] - matrices: - - matrix: - cuda: "12.*" - packages: - - *cuda_python12 - - matrix: - cuda: "11.*" - packages: - - *cuda_python11 - - matrix: - packages: - - *cuda_python - - output_types: [requirements, pyproject] - matrices: - - matrix: - cuda: "12.*" - cuda_suffixed: "true" - packages: - - *rmm_cu12 - - matrix: - cuda: "11.*" - cuda_suffixed: "true" - packages: - - *rmm_cu11 - - {matrix: null, packages: [*rmm_unsuffixed]} run_raft_dask: common: - output_types: [conda, pyproject] @@ -511,6 +433,21 @@ dependencies: packages: - scikit-learn - scipy + depends_on_cuda_python: + specific: + - output_types: [conda, requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + packages: + - cuda-python>=12.6.2,<13.0a0 + - matrix: + cuda: "11.*" + packages: + - cuda-python>=11.8.5,<12.0a0 + - matrix: + packages: + - cuda-python depends_on_distributed_ucxx: common: - output_types: conda @@ -537,6 +474,33 @@ dependencies: packages: - distributed-ucxx-cu11==0.42.*,>=0.0.0a0 - {matrix: null, packages: [*distributed_ucxx_unsuffixed]} + depends_on_rmm: + common: + - output_types: conda + packages: + - &rmm_unsuffixed rmm==25.2.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + # This index is needed for rmm-cu{11,12}. + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - rmm-cu12==25.2.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - rmm-cu11==25.2.*,>=0.0.0a0 + - matrix: + packages: + - *rmm_unsuffixed depends_on_ucx_build: common: - output_types: conda From d7e68f55c58493bc9cf1fbe4eb775a9593891c58 Mon Sep 17 00:00:00 2001 From: tsuki <12711693+enp1s0@users.noreply.github.com> Date: Wed, 18 Dec 2024 14:30:22 +0900 Subject: [PATCH 73/79] [DOC] Fix sample codes (#2518) `raft::raft::resources` -> `raft::resources` Authors: - tsuki (https://github.com/enp1s0) - Corey J. Nolet (https://github.com/cjnolet) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/raft/pull/2518 --- cpp/include/raft/cluster/kmeans.cuh | 8 ++++---- cpp/include/raft/comms/std_comms.hpp | 4 ++-- cpp/include/raft/distance/distance-inl.cuh | 2 +- cpp/include/raft/neighbors/epsilon_neighborhood.cuh | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cpp/include/raft/cluster/kmeans.cuh b/cpp/include/raft/cluster/kmeans.cuh index 38318e8ec8..ee1fc83a9b 100644 --- a/cpp/include/raft/cluster/kmeans.cuh +++ b/cpp/include/raft/cluster/kmeans.cuh @@ -52,7 +52,7 @@ using KeyValueIndexOp = detail::KeyValueIndexOp; * #include * using namespace raft::cluster; * ... - * raft::raft::resources handle; + * raft::resources handle; * raft::cluster::KMeansParams params; * int n_features = 15, inertia, n_iter; * auto centroids = raft::make_device_matrix(handle, params.n_clusters, n_features); @@ -61,7 +61,7 @@ using KeyValueIndexOp = detail::KeyValueIndexOp; * params, * X, * std::nullopt, - * centroids, + * centroids.view(), * raft::make_scalar_view(&inertia), * raft::make_scalar_view(&n_iter)); * @endcode @@ -107,7 +107,7 @@ template * #include * using namespace raft::cluster; * ... - * raft::raft::resources handle; + * raft::resources handle; * raft::cluster::KMeansParams params; * int n_features = 15, inertia, n_iter; * auto centroids = raft::make_device_matrix(handle, params.n_clusters, n_features); @@ -175,7 +175,7 @@ template * #include * using namespace raft::cluster; * ... - * raft::raft::resources handle; + * raft::resources handle; * raft::cluster::KMeansParams params; * int n_features = 15, inertia, n_iter; * auto centroids = raft::make_device_matrix(handle, params.n_clusters, n_features); diff --git a/cpp/include/raft/comms/std_comms.hpp b/cpp/include/raft/comms/std_comms.hpp index 667c8be285..8481360897 100644 --- a/cpp/include/raft/comms/std_comms.hpp +++ b/cpp/include/raft/comms/std_comms.hpp @@ -52,7 +52,7 @@ using std_comms = detail::std_comms; * #include * * ncclComm_t nccl_comm; - * raft::raft::resources handle; + * raft::resources handle; * * build_comms_nccl_only(&handle, nccl_comm, 5, 0); * ... @@ -98,7 +98,7 @@ void build_comms_nccl_only(resources* handle, ncclComm_t nccl_comm, int num_rank * #include * * ncclComm_t nccl_comm; - * raft::raft::resources handle; + * raft::resources handle; * ucp_worker_h ucp_worker; * ucp_ep_h *ucp_endpoints_arr; * diff --git a/cpp/include/raft/distance/distance-inl.cuh b/cpp/include/raft/distance/distance-inl.cuh index 13c9d57efd..d5f8d1cfe1 100644 --- a/cpp/include/raft/distance/distance-inl.cuh +++ b/cpp/include/raft/distance/distance-inl.cuh @@ -366,7 +366,7 @@ void pairwise_distance(raft::resources const& handle, * #include * #include * - * raft::raft::resources handle; + * raft::resources handle; * int n_samples = 5000; * int n_features = 50; * diff --git a/cpp/include/raft/neighbors/epsilon_neighborhood.cuh b/cpp/include/raft/neighbors/epsilon_neighborhood.cuh index bade4385fb..c2f531263d 100644 --- a/cpp/include/raft/neighbors/epsilon_neighborhood.cuh +++ b/cpp/include/raft/neighbors/epsilon_neighborhood.cuh @@ -76,7 +76,7 @@ void epsUnexpL2SqNeighborhood(bool* adj, * #include * #include * using namespace raft::neighbors; - * raft::raft::resources handle; + * raft::resources handle; * ... * auto adj = raft::make_device_matrix(handle, m * n); * auto vd = raft::make_device_vector(handle, m+1); @@ -120,4 +120,4 @@ void eps_neighbors_l2sq(raft::resources const& handle, } // namespace raft::neighbors::epsilon_neighborhood -#endif \ No newline at end of file +#endif From dee71f89e4bdf5ce3ff982e53da9c6c81d883608 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 30 Dec 2024 11:44:33 -0800 Subject: [PATCH 74/79] Check if nightlies have succeeded recently enough (#2533) Contributes to https://github.com/rapidsai/build-planning/issues/127 This PR cannot be merged unless nightly CI has passed within the past 7 days, so if it remains unmerged that will itself be an indication that nightly CI needs fixing. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/raft/pull/2533 --- .github/workflows/pr.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 965943e726..a270df1dfa 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,6 +12,7 @@ concurrency: jobs: pr-builder: needs: + - check-nightly-ci - changed-files - checks - conda-cpp-build @@ -30,6 +31,18 @@ jobs: if: always() with: needs: ${{ toJSON(needs) }} + check-nightly-ci: + # Switch to ubuntu-latest once it defaults to a version of Ubuntu that + # provides at least Python 3.11 (see + # https://docs.python.org/3/library/datetime.html#datetime.date.fromisoformat) + runs-on: ubuntu-24.04 + env: + RAPIDS_GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Check if nightly CI is passing + uses: rapidsai/shared-actions/check_nightly_success/dispatch@main + with: + repo: raft changed-files: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-25.02 From eef9a4fa9a39d4349ed699b097a3e3ff6c78cbc4 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 30 Dec 2024 11:48:07 -0800 Subject: [PATCH 75/79] Switch over to rapids-logger (#2530) This PR removes raft's implementation of a logger in favor of the centralized one in [rapids-logger](https://github.com/rapidsai/rapids-logger). Consumers still get the benefits of a PImpl idiom, but now that is primarily handled by using the appropriate targets (if necessary the impl header is of course still available for direct inclusion). This change paves the way for ensuring consistent fmt/spdlog (lack of) linkage throughout RAPIDS conda and wheel packages. This PR requires https://github.com/rapidsai/rapids-logger/pull/1 Contributes to https://github.com/rapidsai/build-planning/issues/104 Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Bradley Dice (https://github.com/bdice) - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/raft/pull/2530 --- cpp/CMakeLists.txt | 28 +++- cpp/include/raft/cluster/detail/kmeans.cuh | 8 +- .../raft/cluster/detail/kmeans_balanced.cuh | 1 + cpp/include/raft/cluster/kmeans_types.hpp | 2 +- cpp/include/raft/common/logger.hpp | 24 --- cpp/include/raft/core/cublas_macros.hpp | 3 - cpp/include/raft/core/cusolver_macros.hpp | 7 +- cpp/include/raft/core/cusparse_macros.hpp | 2 - .../raft/core/detail/callback_sink.hpp | 71 -------- .../core/detail/fail_container_policy.hpp | 2 +- cpp/include/raft/core/detail/logger.hpp | 24 --- cpp/include/raft/core/logger-ext.hpp | 152 ----------------- cpp/include/raft/core/logger-inl.hpp | 153 ------------------ cpp/include/raft/core/logger-macros.hpp | 95 ++--------- cpp/include/raft/core/logger.hpp | 23 --- .../raft/neighbors/detail/ivf_flat_build.cuh | 1 + .../neighbors/detail/ivf_flat_search-inl.cuh | 3 +- .../raft/solver/detail/lap_kernels.cuh | 3 +- .../raft/sparse/solver/detail/lanczos.cuh | 2 +- cpp/src/core/logger.cpp | 16 -- cpp/test/CMakeLists.txt | 5 + cpp/test/core/device_resources_manager.cpp | 2 +- cpp/test/core/logger.cpp | 57 +++---- docs/source/developer_guide.md | 4 +- 24 files changed, 87 insertions(+), 601 deletions(-) delete mode 100644 cpp/include/raft/common/logger.hpp delete mode 100644 cpp/include/raft/core/detail/callback_sink.hpp delete mode 100644 cpp/include/raft/core/detail/logger.hpp delete mode 100644 cpp/include/raft/core/logger-ext.hpp delete mode 100644 cpp/include/raft/core/logger-inl.hpp delete mode 100644 cpp/include/raft/core/logger.hpp delete mode 100644 cpp/src/core/logger.cpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 78a4dbb913..06531941aa 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -100,6 +100,17 @@ set_property( ) message(VERBOSE "RAFT: RMM_LOGGING_LEVEL = '${RMM_LOGGING_LEVEL}'.") +# Set logging level +set(LIBRAFT_LOGGING_LEVEL + "INFO" + CACHE STRING "Choose the logging level." +) +set_property( + CACHE LIBRAFT_LOGGING_LEVEL PROPERTY STRINGS "TRACE" "DEBUG" "INFO" "WARN" "ERROR" "CRITICAL" + "OFF" +) +message(VERBOSE "RAFT: LIBRAFT_LOGGING_LEVEL = '${LIBRAFT_LOGGING_LEVEL}'.") + # ################################################################################################## # * Conda environment detection ---------------------------------------------- @@ -152,6 +163,13 @@ include(cmake/modules/ConfigureCUDA.cmake) # add third party dependencies using CPM rapids_cpm_init() +# Not using rapids-cmake since we never want to find, always download. +CPMAddPackage( + NAME rapids_logger GITHUB_REPOSITORY rapidsai/rapids-logger GIT_SHALLOW FALSE GIT_TAG + 4df3ee70c6746fd1b6c0dc14209dae2e2d4378c6 VERSION 4df3ee70c6746fd1b6c0dc14209dae2e2d4378c6 +) +rapids_make_logger(raft LOGGER_HEADER_DIR include/raft/core EXPORT_SET raft-exports) + # CCCL before rmm/cuco so we get the right version of CCCL include(cmake/thirdparty/get_cccl.cmake) include(cmake/thirdparty/get_rmm.cmake) @@ -182,7 +200,7 @@ target_include_directories( # Keep RAFT as lightweight as possible. Only CUDA libs and rmm should be used in global target. target_link_libraries( raft INTERFACE rmm::rmm rmm::rmm_logger spdlog::spdlog_header_only cuco::cuco - nvidia::cutlass::cutlass CCCL::CCCL + nvidia::cutlass::cutlass CCCL::CCCL raft_logger ) target_compile_features(raft INTERFACE cxx_std_17 $) @@ -190,6 +208,9 @@ target_compile_options( raft INTERFACE $<$:--expt-extended-lambda --expt-relaxed-constexpr> ) +target_compile_definitions( + raft INTERFACE "RAFT_LOG_ACTIVE_LEVEL=RAFT_LOG_LEVEL_${LIBRAFT_LOGGING_LEVEL}" +) set(RAFT_CUSOLVER_DEPENDENCY CUDA::cusolver${_ctk_static_suffix}) set(RAFT_CUBLAS_DEPENDENCY CUDA::cublas${_ctk_static_suffix}) @@ -265,7 +286,6 @@ set_target_properties(raft_compiled PROPERTIES EXPORT_NAME compiled) if(RAFT_COMPILE_LIBRARY) add_library( raft_objs OBJECT - src/core/logger.cpp src/linalg/detail/coalesced_reduction.cu src/raft_runtime/random/rmat_rectangular_generator_int64_double.cu src/raft_runtime/random/rmat_rectangular_generator_int64_float.cu @@ -318,8 +338,8 @@ if(RAFT_COMPILE_LIBRARY) # ensure CUDA symbols aren't relocated to the middle of the debug build binaries target_link_options(${target} PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/fatbin.ld") endforeach() - target_link_libraries(raft_lib PRIVATE rmm::rmm_logger_impl) - target_link_libraries(raft_lib_static PRIVATE rmm::rmm_logger_impl) + target_link_libraries(raft_lib PRIVATE rmm::rmm_logger_impl raft_logger_impl) + target_link_libraries(raft_lib_static PRIVATE rmm::rmm_logger_impl raft_logger_impl) endif() if(TARGET raft_lib AND (NOT TARGET raft::raft_lib)) diff --git a/cpp/include/raft/cluster/detail/kmeans.cuh b/cpp/include/raft/cluster/detail/kmeans.cuh index 4efeedcbaa..4203f0969b 100644 --- a/cpp/include/raft/cluster/detail/kmeans.cuh +++ b/cpp/include/raft/cluster/detail/kmeans.cuh @@ -369,7 +369,7 @@ void kmeans_fit_main(raft::resources const& handle, rmm::device_uvector& workspace) { common::nvtx::range fun_scope("kmeans_fit_main"); - logger::get(RAFT_NAME).set_level(params.verbosity); + default_logger().set_level(params.verbosity); cudaStream_t stream = resource::get_cuda_stream(handle); auto n_samples = X.extent(0); auto n_features = X.extent(1); @@ -865,7 +865,7 @@ void kmeans_fit(raft::resources const& handle, params.n_clusters); } - logger::get(RAFT_NAME).set_level(params.verbosity); + default_logger().set_level(params.verbosity); // Allocate memory rmm::device_uvector workspace(0, stream); @@ -1010,7 +1010,7 @@ void kmeans_predict(raft::resources const& handle, RAFT_EXPECTS(centroids.extent(1) == n_features, "invalid parameter (centroids.extent(1) != n_features)"); - logger::get(RAFT_NAME).set_level(params.verbosity); + default_logger().set_level(params.verbosity); auto metric = params.metric; // Allocate memory @@ -1201,7 +1201,7 @@ void kmeans_transform(raft::resources const& handle, raft::device_matrix_view X_new) { common::nvtx::range fun_scope("kmeans_transform"); - logger::get(RAFT_NAME).set_level(params.verbosity); + default_logger().set_level(params.verbosity); cudaStream_t stream = resource::get_cuda_stream(handle); auto n_samples = X.extent(0); auto n_features = X.extent(1); diff --git a/cpp/include/raft/cluster/detail/kmeans_balanced.cuh b/cpp/include/raft/cluster/detail/kmeans_balanced.cuh index 0a5a3ba5aa..5dcd679bd5 100644 --- a/cpp/include/raft/cluster/detail/kmeans_balanced.cuh +++ b/cpp/include/raft/cluster/detail/kmeans_balanced.cuh @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include diff --git a/cpp/include/raft/cluster/kmeans_types.hpp b/cpp/include/raft/cluster/kmeans_types.hpp index 4d956ad7a0..fbedd58417 100644 --- a/cpp/include/raft/cluster/kmeans_types.hpp +++ b/cpp/include/raft/cluster/kmeans_types.hpp @@ -82,7 +82,7 @@ struct KMeansParams : kmeans_base_params { /** * verbosity level. */ - int verbosity = RAFT_LEVEL_INFO; + level_enum verbosity = level_enum::info; /** * Seed to the random number generator. diff --git a/cpp/include/raft/common/logger.hpp b/cpp/include/raft/common/logger.hpp deleted file mode 100644 index 77483e577d..0000000000 --- a/cpp/include/raft/common/logger.hpp +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2022, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * This file is deprecated and will be removed in release 22.08. - * Please use the include/core/logger.hpp instead. - */ - -#pragma once - -#include \ No newline at end of file diff --git a/cpp/include/raft/core/cublas_macros.hpp b/cpp/include/raft/core/cublas_macros.hpp index b69b121161..6c195d8a6f 100644 --- a/cpp/include/raft/core/cublas_macros.hpp +++ b/cpp/include/raft/core/cublas_macros.hpp @@ -23,9 +23,6 @@ #include -///@todo: enable this once we have logger enabled -// #include - #include #define _CUBLAS_ERR_TO_STR(err) \ diff --git a/cpp/include/raft/core/cusolver_macros.hpp b/cpp/include/raft/core/cusolver_macros.hpp index 74a8b7c36c..beaf2d74dc 100644 --- a/cpp/include/raft/core/cusolver_macros.hpp +++ b/cpp/include/raft/core/cusolver_macros.hpp @@ -19,11 +19,10 @@ #pragma once +#include + #include #include -///@todo: enable this once logging is enabled -// #include -#include #include @@ -135,4 +134,4 @@ inline const char* cusolver_error_to_string(cusolverStatus_t err) #define CUSOLVER_CHECK_NO_THROW(call) CUSOLVER_TRY_NO_THROW(call) #endif -#endif \ No newline at end of file +#endif diff --git a/cpp/include/raft/core/cusparse_macros.hpp b/cpp/include/raft/core/cusparse_macros.hpp index 5a1968b529..2a1df14345 100644 --- a/cpp/include/raft/core/cusparse_macros.hpp +++ b/cpp/include/raft/core/cusparse_macros.hpp @@ -19,8 +19,6 @@ #include #include -///@todo: enable this once logging is enabled -// #include #define _CUSPARSE_ERR_TO_STR(err) \ case err: return #err; diff --git a/cpp/include/raft/core/detail/callback_sink.hpp b/cpp/include/raft/core/detail/callback_sink.hpp deleted file mode 100644 index a110af5c76..0000000000 --- a/cpp/include/raft/core/detail/callback_sink.hpp +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include -#include - -#define SPDLOG_HEADER_ONLY -#include -#include -#include - -namespace spdlog::sinks { - -typedef void (*LogCallback)(int lvl, const char* msg); - -template -class CallbackSink : public base_sink { - public: - explicit CallbackSink(std::string tag = "spdlog", - LogCallback callback = nullptr, - void (*flush)() = nullptr) - : _callback{callback}, _flush{flush} {}; - - void set_callback(LogCallback callback) { _callback = callback; } - void set_flush(void (*flush)()) { _flush = flush; } - - protected: - void sink_it_(const details::log_msg& msg) override - { - spdlog::memory_buf_t formatted; - base_sink::formatter_->format(msg, formatted); - std::string msg_string = fmt::to_string(formatted); - - if (_callback) { - _callback(static_cast(msg.level), msg_string.c_str()); - } else { - std::cout << msg_string; - } - } - - void flush_() override - { - if (_flush) { - _flush(); - } else { - std::cout << std::flush; - } - } - - LogCallback _callback; - void (*_flush)(); -}; - -using callback_sink_mt = CallbackSink; -using callback_sink_st = CallbackSink; - -} // end namespace spdlog::sinks diff --git a/cpp/include/raft/core/detail/fail_container_policy.hpp b/cpp/include/raft/core/detail/fail_container_policy.hpp index cf9d0887dd..f5f1bfb377 100644 --- a/cpp/include/raft/core/detail/fail_container_policy.hpp +++ b/cpp/include/raft/core/detail/fail_container_policy.hpp @@ -16,7 +16,7 @@ #pragma once #include -#include +#include #include #include diff --git a/cpp/include/raft/core/detail/logger.hpp b/cpp/include/raft/core/detail/logger.hpp deleted file mode 100644 index f3f52b46ae..0000000000 --- a/cpp/include/raft/core/detail/logger.hpp +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message(__FILE__ \ - " is deprecated and will be removed in future releases." \ - " Please use the version instead.") -#endif - -#include diff --git a/cpp/include/raft/core/logger-ext.hpp b/cpp/include/raft/core/logger-ext.hpp deleted file mode 100644 index 73fe463aba..0000000000 --- a/cpp/include/raft/core/logger-ext.hpp +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include // RAFT_INLINE_CONDITIONAL - -#include // std::unique_ptr -#include // std::string -#include // std::unordered_map - -namespace raft { - -static const std::string RAFT_NAME = "raft"; -static const std::string default_log_pattern("[%L] [%H:%M:%S.%f] %v"); - -namespace detail { -RAFT_INLINE_CONDITIONAL std::string format(const char* fmt, ...); -} -/** - * @brief The main Logging class for raft library. - * - * This class acts as a thin wrapper over the underlying `spdlog` interface. The - * design is done in this way in order to avoid us having to also ship `spdlog` - * header files in our installation. - * - * @todo This currently only supports logging to stdout. Need to add support in - * future to add custom loggers as well [Issue #2046] - */ -class logger { - public: - // @todo setting the logger once per process with - logger(std::string const& name_ = ""); - /** - * @brief Singleton method to get the underlying logger object - * - * @return the singleton logger object - */ - static logger& get(std::string const& name = ""); - - /** - * @brief Set the logging level. - * - * Only messages with level equal or above this will be printed - * - * @param[in] level logging level - * - * @note The log level will actually be set only if the input is within the - * range [RAFT_LEVEL_TRACE, RAFT_LEVEL_OFF]. If it is not, then it'll - * be ignored. See documentation of decisiontree for how this gets used - */ - void set_level(int level); - - /** - * @brief Set the logging pattern - * - * @param[in] pattern the pattern to be set. Refer this link - * https://github.com/gabime/spdlog/wiki/3.-Custom-formatting - * to know the right syntax of this pattern - */ - void set_pattern(const std::string& pattern); - - /** - * @brief Register a callback function to be run in place of usual log call - * - * @param[in] callback the function to be run on all logged messages - */ - void set_callback(void (*callback)(int lvl, const char* msg)); - - /** - * @brief Register a flush function compatible with the registered callback - * - * @param[in] flush the function to use when flushing logs - */ - void set_flush(void (*flush)()); - - /** - * @brief Tells whether messages will be logged for the given log level - * - * @param[in] level log level to be checked for - * @return true if messages will be logged for this level, else false - */ - bool should_log_for(int level) const; - /** - * @brief Query for the current log level - * - * @return the current log level - */ - int get_level() const; - - /** - * @brief Get the current logging pattern - * @return the pattern - */ - std::string get_pattern() const; - - /** - * @brief Main logging method - * - * @param[in] level logging level of this message - * @param[in] fmt C-like format string, followed by respective params - */ - void log(int level, const char* fmt, ...); - - /** - * @brief Flush logs by calling flush on underlying logger - */ - void flush(); - - ~logger(); - - private: - logger(); - // pimpl pattern: - // https://learn.microsoft.com/en-us/cpp/cpp/pimpl-for-compile-time-encapsulation-modern-cpp?view=msvc-170 - class impl; - std::unique_ptr pimpl; - static inline std::unordered_map> log_map; -}; // class logger - -/** - * @brief An object used for scoped log level setting - * - * Instances of `raft::log_level_setter` will set RAFT logging to the level - * indicated on construction and will revert to the previous set level on - * destruction. - */ -struct log_level_setter { - explicit log_level_setter(int level) - { - prev_level_ = logger::get(RAFT_NAME).get_level(); - logger::get(RAFT_NAME).set_level(level); - } - ~log_level_setter() { logger::get(RAFT_NAME).set_level(prev_level_); } - - private: - int prev_level_; -}; // class log_level_setter - -}; // namespace raft diff --git a/cpp/include/raft/core/logger-inl.hpp b/cpp/include/raft/core/logger-inl.hpp deleted file mode 100644 index ea5f4ea26e..0000000000 --- a/cpp/include/raft/core/logger-inl.hpp +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "logger-macros.hpp" - -#include - -#include -#include -#include -#include -#include -#include -// The logger-ext.hpp file contains the class declaration of the logger class. -// In this case, it is okay to include the logger-ext.hpp file because it -// contains no RAFT_EXPLICIT template instantiations. -#include "logger-ext.hpp" - -#define SPDLOG_HEADER_ONLY -#include -#include // RAFT_INLINE_CONDITIONAL - -#include // NOLINT -#include // NOLINT - -namespace raft { - -namespace detail { - -inline std::string format(const char* fmt, va_list& vl) -{ - va_list vl_copy; - va_copy(vl_copy, vl); - int length = std::vsnprintf(nullptr, 0, fmt, vl_copy); - assert(length >= 0); - std::vector buf(length + 1); - std::vsnprintf(buf.data(), length + 1, fmt, vl); - return std::string(buf.data()); -} - -RAFT_INLINE_CONDITIONAL std::string format(const char* fmt, ...) -{ - va_list vl; - va_start(vl, fmt); - std::string str = format(fmt, vl); - va_end(vl); - return str; -} - -inline int convert_level_to_spdlog(int level) -{ - level = std::max(RAFT_LEVEL_OFF, std::min(RAFT_LEVEL_TRACE, level)); - return RAFT_LEVEL_TRACE - level; -} - -} // namespace detail - -class logger::impl { // defined privately here - // ... all private data and functions: all of these - // can now change without recompiling callers ... - public: - std::shared_ptr sink; - std::shared_ptr spdlogger; - std::string cur_pattern; - int cur_level; - - impl(std::string const& name_ = "") - : sink{std::make_shared()}, - spdlogger{std::make_shared(name_, sink)}, - cur_pattern() - { - } -}; // class logger::impl - -RAFT_INLINE_CONDITIONAL logger::logger(std::string const& name_) : pimpl(new impl(name_)) -{ - set_pattern(default_log_pattern); - set_level(RAFT_ACTIVE_LEVEL); -} - -RAFT_INLINE_CONDITIONAL logger& logger::get(std::string const& name) -{ - if (log_map.find(name) == log_map.end()) { log_map[name] = std::make_shared(name); } - return *log_map[name]; -} - -RAFT_INLINE_CONDITIONAL void logger::set_level(int level) -{ - level = raft::detail::convert_level_to_spdlog(level); - pimpl->spdlogger->set_level(static_cast(level)); -} - -RAFT_INLINE_CONDITIONAL void logger::set_pattern(const std::string& pattern) -{ - pimpl->cur_pattern = pattern; - pimpl->spdlogger->set_pattern(pattern); -} - -RAFT_INLINE_CONDITIONAL void logger::set_callback(void (*callback)(int lvl, const char* msg)) -{ - pimpl->sink->set_callback(callback); -} - -RAFT_INLINE_CONDITIONAL void logger::set_flush(void (*flush)()) { pimpl->sink->set_flush(flush); } - -RAFT_INLINE_CONDITIONAL bool logger::should_log_for(int level) const -{ - level = raft::detail::convert_level_to_spdlog(level); - auto level_e = static_cast(level); - return pimpl->spdlogger->should_log(level_e); -} - -RAFT_INLINE_CONDITIONAL int logger::get_level() const -{ - auto level_e = pimpl->spdlogger->level(); - return RAFT_LEVEL_TRACE - static_cast(level_e); -} - -RAFT_INLINE_CONDITIONAL std::string logger::get_pattern() const { return pimpl->cur_pattern; } - -RAFT_INLINE_CONDITIONAL void logger::log(int level, const char* fmt, ...) -{ - level = raft::detail::convert_level_to_spdlog(level); - auto level_e = static_cast(level); - // explicit check to make sure that we only expand messages when required - if (pimpl->spdlogger->should_log(level_e)) { - va_list vl; - va_start(vl, fmt); - auto msg = raft::detail::format(fmt, vl); - va_end(vl); - pimpl->spdlogger->log(level_e, msg); - } -} - -RAFT_INLINE_CONDITIONAL void logger::flush() { pimpl->spdlogger->flush(); } - -RAFT_INLINE_CONDITIONAL logger::~logger() {} - -}; // namespace raft diff --git a/cpp/include/raft/core/logger-macros.hpp b/cpp/include/raft/core/logger-macros.hpp index 5ddb072067..e32440dcce 100644 --- a/cpp/include/raft/core/logger-macros.hpp +++ b/cpp/include/raft/core/logger-macros.hpp @@ -15,92 +15,17 @@ */ #pragma once -/** - * @defgroup logging levels used in raft - * - * @note exactly match the corresponding ones (but reverse in terms of value) - * in spdlog for wrapping purposes - * - * @{ - */ -#define RAFT_LEVEL_TRACE 6 -#define RAFT_LEVEL_DEBUG 5 -#define RAFT_LEVEL_INFO 4 -#define RAFT_LEVEL_WARN 3 -#define RAFT_LEVEL_ERROR 2 -#define RAFT_LEVEL_CRITICAL 1 -#define RAFT_LEVEL_OFF 0 -/** @} */ - -#if !defined(RAFT_ACTIVE_LEVEL) -#define RAFT_ACTIVE_LEVEL RAFT_LEVEL_INFO -#endif - -/** - * @defgroup loggerMacros Helper macros for dealing with logging - * @{ - */ -#if (RAFT_ACTIVE_LEVEL >= RAFT_LEVEL_TRACE) -#define RAFT_LOG_TRACE(fmt, ...) \ - do { \ - std::stringstream ss; \ - ss << raft::detail::format("%s:%d ", __FILE__, __LINE__); \ - ss << raft::detail::format(fmt, ##__VA_ARGS__); \ - raft::logger::get(RAFT_NAME).log(RAFT_LEVEL_TRACE, ss.str().c_str()); \ - } while (0) -#else -#define RAFT_LOG_TRACE(fmt, ...) void(0) -#endif - -#if (RAFT_ACTIVE_LEVEL >= RAFT_LEVEL_TRACE) -#define RAFT_LOG_TRACE_VEC(ptr, len) \ - do { \ - std::stringstream ss; \ - ss << raft::detail::format("%s:%d ", __FILE__, __LINE__); \ - print_vector(#ptr, ptr, len, ss); \ - raft::logger::get(RAFT_NAME).log(RAFT_LEVEL_TRACE, ss.str().c_str()); \ +#include + +#if (RAFT_LOG_ACTIVE_LEVEL <= RAFT_LOG_LEVEL_TRACE) +#define RAFT_LOG_TRACE_VEC(ptr, len) \ + do { \ + std::stringstream ss; \ + ss << raft::detail::format("%s:%d ", __FILE__, __LINE__); \ + print_vector(#ptr, ptr, len, ss); \ + raft::logger::get(RAFT_NAME).log(RAFT_LEVEL_TRACE, ss.str().c_str()); \ + RAFT_LOGGER_CALL(raft::default_logger(), raft::level_enum::trace, __VA_ARGS__) \ } while (0) #else #define RAFT_LOG_TRACE_VEC(ptr, len) void(0) #endif - -#if (RAFT_ACTIVE_LEVEL >= RAFT_LEVEL_DEBUG) -#define RAFT_LOG_DEBUG(fmt, ...) \ - do { \ - std::stringstream ss; \ - ss << raft::detail::format("%s:%d ", __FILE__, __LINE__); \ - ss << raft::detail::format(fmt, ##__VA_ARGS__); \ - raft::logger::get(RAFT_NAME).log(RAFT_LEVEL_DEBUG, ss.str().c_str()); \ - } while (0) -#else -#define RAFT_LOG_DEBUG(fmt, ...) void(0) -#endif - -#if (RAFT_ACTIVE_LEVEL >= RAFT_LEVEL_INFO) -#define RAFT_LOG_INFO(fmt, ...) \ - raft::logger::get(RAFT_NAME).log(RAFT_LEVEL_INFO, fmt, ##__VA_ARGS__) -#else -#define RAFT_LOG_INFO(fmt, ...) void(0) -#endif - -#if (RAFT_ACTIVE_LEVEL >= RAFT_LEVEL_WARN) -#define RAFT_LOG_WARN(fmt, ...) \ - raft::logger::get(RAFT_NAME).log(RAFT_LEVEL_WARN, fmt, ##__VA_ARGS__) -#else -#define RAFT_LOG_WARN(fmt, ...) void(0) -#endif - -#if (RAFT_ACTIVE_LEVEL >= RAFT_LEVEL_ERROR) -#define RAFT_LOG_ERROR(fmt, ...) \ - raft::logger::get(RAFT_NAME).log(RAFT_LEVEL_ERROR, fmt, ##__VA_ARGS__) -#else -#define RAFT_LOG_ERROR(fmt, ...) void(0) -#endif - -#if (RAFT_ACTIVE_LEVEL >= RAFT_LEVEL_CRITICAL) -#define RAFT_LOG_CRITICAL(fmt, ...) \ - raft::logger::get(RAFT_NAME).log(RAFT_LEVEL_CRITICAL, fmt, ##__VA_ARGS__) -#else -#define RAFT_LOG_CRITICAL(fmt, ...) void(0) -#endif -/** @} */ diff --git a/cpp/include/raft/core/logger.hpp b/cpp/include/raft/core/logger.hpp deleted file mode 100644 index e64a0db257..0000000000 --- a/cpp/include/raft/core/logger.hpp +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "logger-ext.hpp" -#include "logger-macros.hpp" - -#if !defined(RAFT_COMPILED) -#include "logger-inl.hpp" -#endif diff --git a/cpp/include/raft/neighbors/detail/ivf_flat_build.cuh b/cpp/include/raft/neighbors/detail/ivf_flat_build.cuh index 55184cc615..0e00ef571f 100644 --- a/cpp/include/raft/neighbors/detail/ivf_flat_build.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_flat_build.cuh @@ -17,6 +17,7 @@ #pragma once #include +#include #include #include #include diff --git a/cpp/include/raft/neighbors/detail/ivf_flat_search-inl.cuh b/cpp/include/raft/neighbors/detail/ivf_flat_search-inl.cuh index 388dd60f14..44d55c36de 100644 --- a/cpp/include/raft/neighbors/detail/ivf_flat_search-inl.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_flat_search-inl.cuh @@ -16,7 +16,8 @@ #pragma once -#include // RAFT_LOG_TRACE +#include +#include #include #include // raft::resources #include // is_min_close, DistanceType diff --git a/cpp/include/raft/solver/detail/lap_kernels.cuh b/cpp/include/raft/solver/detail/lap_kernels.cuh index 383c3ab713..3c25852240 100644 --- a/cpp/include/raft/solver/detail/lap_kernels.cuh +++ b/cpp/include/raft/solver/detail/lap_kernels.cuh @@ -26,6 +26,7 @@ #include "../linear_assignment_types.hpp" +#include #include #include @@ -552,4 +553,4 @@ RAFT_KERNEL kernel_calcObjValPrimal(weight_t* d_obj_val_primal, } } -} // namespace raft::solver::detail \ No newline at end of file +} // namespace raft::solver::detail diff --git a/cpp/include/raft/sparse/solver/detail/lanczos.cuh b/cpp/include/raft/sparse/solver/detail/lanczos.cuh index 02a77a0d99..6f03f77bc0 100644 --- a/cpp/include/raft/sparse/solver/detail/lanczos.cuh +++ b/cpp/include/raft/sparse/solver/detail/lanczos.cuh @@ -24,7 +24,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/cpp/src/core/logger.cpp b/cpp/src/core/logger.cpp deleted file mode 100644 index 8f81cf2926..0000000000 --- a/cpp/src/core/logger.cpp +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 621ee6c160..4cd0a32f51 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -55,6 +55,7 @@ function(ConfigureTest) ${RAFT_CTK_MATH_DEPENDENCIES} $ $ + raft_test_logger ) set_target_properties( ${TEST_NAME} @@ -87,6 +88,10 @@ function(ConfigureTest) ) endfunction() +# Create an object library for the logger so that we don't have to recompile it. +add_library(raft_test_logger OBJECT) +target_link_libraries(raft_test_logger PRIVATE raft_logger_impl) + # ################################################################################################## # test sources ################################################################################## # ################################################################################################## diff --git a/cpp/test/core/device_resources_manager.cpp b/cpp/test/core/device_resources_manager.cpp index c63d5896e5..007b57378f 100644 --- a/cpp/test/core/device_resources_manager.cpp +++ b/cpp/test/core/device_resources_manager.cpp @@ -89,7 +89,7 @@ TEST(DeviceResourcesManager, ObeysSetters) // Suppress the many warnings from testing use of setters after initial // get_device_resources call - auto scoped_log_level = log_level_setter{RAFT_LEVEL_ERROR}; + auto scoped_log_level = log_level_setter{level_enum::error}; omp_set_dynamic(0); #pragma omp parallel for num_threads(5) diff --git a/cpp/test/core/logger.cpp b/cpp/test/core/logger.cpp index 7f31beed71..10adb71dda 100644 --- a/cpp/test/core/logger.cpp +++ b/cpp/test/core/logger.cpp @@ -14,10 +14,10 @@ * limitations under the License. */ -// We set RAFT_ACTIVE_LEVEL to a value that would enable testing trace and debug logs +// We set RAFT_LOG_ACTIVE_LEVEL to a value that would enable testing trace and debug logs // (otherwise trace and debug logs are desabled by default). -#undef RAFT_ACTIVE_LEVEL -#define RAFT_ACTIVE_LEVEL 6 +#undef RAFT_LOG_ACTIVE_LEVEL +#define RAFT_LOG_ACTIVE_LEVEL RAFT_LOG_LEVEL_TRACE #include @@ -34,15 +34,15 @@ TEST(logger, Test) RAFT_LOG_WARN("This is a warning message"); RAFT_LOG_INFO("This is an info message"); - logger::get(RAFT_NAME).set_level(RAFT_LEVEL_WARN); - ASSERT_EQ(RAFT_LEVEL_WARN, logger::get(RAFT_NAME).get_level()); - logger::get(RAFT_NAME).set_level(RAFT_LEVEL_INFO); - ASSERT_EQ(RAFT_LEVEL_INFO, logger::get(RAFT_NAME).get_level()); + default_logger().set_level(raft::level_enum::warn); + ASSERT_EQ(raft::level_enum::warn, default_logger().level()); + default_logger().set_level(raft::level_enum::info); + ASSERT_EQ(raft::level_enum::info, default_logger().level()); - ASSERT_FALSE(logger::get(RAFT_NAME).should_log_for(RAFT_LEVEL_TRACE)); - ASSERT_FALSE(logger::get(RAFT_NAME).should_log_for(RAFT_LEVEL_DEBUG)); - ASSERT_TRUE(logger::get(RAFT_NAME).should_log_for(RAFT_LEVEL_INFO)); - ASSERT_TRUE(logger::get(RAFT_NAME).should_log_for(RAFT_LEVEL_WARN)); + ASSERT_FALSE(default_logger().should_log(raft::level_enum::trace)); + ASSERT_FALSE(default_logger().should_log(raft::level_enum::debug)); + ASSERT_TRUE(default_logger().should_log(raft::level_enum::info)); + ASSERT_TRUE(default_logger().should_log(raft::level_enum::warn)); } std::string logged = ""; @@ -57,60 +57,61 @@ class loggerTest : public ::testing::Test { { flushCount = 0; logged = ""; - logger::get(RAFT_NAME).set_level(RAFT_LEVEL_TRACE); + default_logger().set_level(raft::level_enum::trace); } void TearDown() override { - logger::get(RAFT_NAME).set_callback(nullptr); - logger::get(RAFT_NAME).set_flush(nullptr); - logger::get(RAFT_NAME).set_level(RAFT_LEVEL_INFO); + default_logger().sinks().pop_back(); + default_logger().set_level(raft::level_enum::info); } }; -// The logging macros depend on `RAFT_ACTIVE_LEVEL` as well as the logger verbosity; -// The verbosity is set to `RAFT_LEVEL_TRACE`, but `RAFT_ACTIVE_LEVEL` is set outside of here. -auto check_if_logged(const std::string& msg, int log_level_def) -> bool +// The logging macros depend on `RAFT_LOG_ACTIVE_LEVEL` as well as the logger verbosity; +// The verbosity is set to `RAFT_LOG_LEVEL_TRACE`, but `RAFT_LOG_ACTIVE_LEVEL` is set outside of +// here. +auto check_if_logged(const std::string& msg, raft::level_enum log_level_def) -> bool { bool actually_logged = logged.find(msg) != std::string::npos; - bool should_be_logged = RAFT_ACTIVE_LEVEL >= log_level_def; + bool should_be_logged = RAFT_LOG_ACTIVE_LEVEL <= static_cast(log_level_def); return actually_logged == should_be_logged; } TEST_F(loggerTest, callback) { std::string testMsg; - logger::get(RAFT_NAME).set_callback(exampleCallback); + default_logger().sinks().push_back(std::make_shared(exampleCallback)); testMsg = "This is a critical message"; RAFT_LOG_CRITICAL(testMsg.c_str()); - ASSERT_TRUE(check_if_logged(testMsg, RAFT_LEVEL_CRITICAL)); + ASSERT_TRUE(check_if_logged(testMsg, raft::level_enum::critical)); testMsg = "This is an error message"; RAFT_LOG_ERROR(testMsg.c_str()); - ASSERT_TRUE(check_if_logged(testMsg, RAFT_LEVEL_ERROR)); + ASSERT_TRUE(check_if_logged(testMsg, raft::level_enum::error)); testMsg = "This is a warning message"; RAFT_LOG_WARN(testMsg.c_str()); - ASSERT_TRUE(check_if_logged(testMsg, RAFT_LEVEL_WARN)); + ASSERT_TRUE(check_if_logged(testMsg, raft::level_enum::warn)); testMsg = "This is an info message"; RAFT_LOG_INFO(testMsg.c_str()); - ASSERT_TRUE(check_if_logged(testMsg, RAFT_LEVEL_INFO)); + ASSERT_TRUE(check_if_logged(testMsg, raft::level_enum::info)); testMsg = "This is a debug message"; RAFT_LOG_DEBUG(testMsg.c_str()); - ASSERT_TRUE(check_if_logged(testMsg, RAFT_LEVEL_DEBUG)); + ASSERT_TRUE(check_if_logged(testMsg, raft::level_enum::debug)); testMsg = "This is a trace message"; RAFT_LOG_TRACE(testMsg.c_str()); - ASSERT_TRUE(check_if_logged(testMsg, RAFT_LEVEL_TRACE)); + ASSERT_TRUE(check_if_logged(testMsg, raft::level_enum::trace)); } TEST_F(loggerTest, flush) { - logger::get(RAFT_NAME).set_flush(exampleFlush); - logger::get(RAFT_NAME).flush(); + default_logger().sinks().push_back( + std::make_shared(exampleCallback, exampleFlush)); + default_logger().flush(); ASSERT_EQ(1, flushCount); } diff --git a/docs/source/developer_guide.md b/docs/source/developer_guide.md index 5cc694dc8f..6240b2638b 100644 --- a/docs/source/developer_guide.md +++ b/docs/source/developer_guide.md @@ -256,14 +256,14 @@ There are 7 logging levels with each successive level becoming quieter: 7. RAFT_LEVEL_OFF Pass one of these as per your needs into the `set_level()` method as follows: ```cpp -raft::logger::get().set_level(RAFT_LEVEL_WARN); +raft::default_logger().set_level(RAFT_LEVEL_WARN); // From now onwards, this will print only WARN and above kind of messages ``` ### Changing logging pattern Pass the [format string](https://github.com/gabime/spdlog/wiki/3.-Custom-formatting) as follows in order use a different logging pattern than the default. ```cpp -raft::logger::get.set_pattern(YourFavoriteFormat); +raft::default_logger().set_pattern(YourFavoriteFormat); ``` One can also use the corresponding `get_pattern()` method to know the current format as well. From 26f8d06c6699e50fa085763fc8f0d7a6d02c5ceb Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 3 Jan 2025 11:29:49 -0800 Subject: [PATCH 76/79] Use rapids-cmake for the logger (#2534) This PR switches raft to use rapids-cmake to fetch rapids-logger so that it uses a consistent version with the rest of RAPIDS to avoid any cases where transitive CPM loads result in multiple packages being built from source that require a different version of rapids-logger. Depends on https://github.com/rapidsai/rapids-cmake/pull/737 and https://github.com/rapidsai/rmm/pull/1776. Contributes to rapidsai/build-planning#104. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/raft/pull/2534 --- cpp/CMakeLists.txt | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 06531941aa..621f9fcef2 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -163,11 +163,8 @@ include(cmake/modules/ConfigureCUDA.cmake) # add third party dependencies using CPM rapids_cpm_init() -# Not using rapids-cmake since we never want to find, always download. -CPMAddPackage( - NAME rapids_logger GITHUB_REPOSITORY rapidsai/rapids-logger GIT_SHALLOW FALSE GIT_TAG - 4df3ee70c6746fd1b6c0dc14209dae2e2d4378c6 VERSION 4df3ee70c6746fd1b6c0dc14209dae2e2d4378c6 -) +include(${rapids-cmake-dir}/cpm/rapids_logger.cmake) +rapids_cpm_rapids_logger() rapids_make_logger(raft LOGGER_HEADER_DIR include/raft/core EXPORT_SET raft-exports) # CCCL before rmm/cuco so we get the right version of CCCL From 8fc988e11d82404ef7b52f4c810d4a4ed07cd2a2 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 7 Jan 2025 15:49:56 -0600 Subject: [PATCH 77/79] remove unused 'joblib' and 'numba' dependencies, other packaging cleanup (#2532) Proposes some cleanup of packaging details, noticed while I was working on #2531 * removes runtime dependencies on `joblib` and `numba` for `raft-dask` - *`raft-dask` doesn't directly import from these libraries, and the git blame didn't suggest any other reason that they were being pinned here* - *checked with `git grep -E 'joblib|numba'` * removes `setup.cfg` files - *these are currently being ignored by tools, in favor of identical configuration in `pyproject.toml` and `.flake8` files* - e.g. https://github.com/rapidsai/raft/blob/bfd190687ee396374b7106d9ac26add73b57b22a/.pre-commit-config.yaml#L16-L19 * packages license files in conda packages - *think these were just missed in the round of PRs like this: https://github.com/rapidsai/cuml/pull/6061* * removes some outdated / inaccurate comments in packaging configs Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Bradley Dice (https://github.com/bdice) - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/raft/pull/2532 --- .pre-commit-config.yaml | 3 +- .../all_cuda-118_arch-aarch64.yaml | 2 - .../all_cuda-118_arch-x86_64.yaml | 2 - .../all_cuda-125_arch-aarch64.yaml | 2 - .../all_cuda-125_arch-x86_64.yaml | 2 - conda/recipes/pylibraft/meta.yaml | 4 +- conda/recipes/raft-dask/meta.yaml | 5 +- dependencies.yaml | 4 -- pyproject.toml | 2 +- python/pylibraft/setup.cfg | 38 ------------- python/raft-dask/pyproject.toml | 2 - setup.cfg | 55 ------------------- 12 files changed, 4 insertions(+), 117 deletions(-) delete mode 100644 python/pylibraft/setup.cfg delete mode 100644 setup.cfg diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e3b3c8c440..d5456ba30b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -110,8 +110,7 @@ repos: [.](cmake|cpp|cu|cuh|h|hpp|sh|pxd|py|pyx)$| CMakeLists[.]txt$| CMakeLists_standalone[.]txt$| - meta[.]yaml$| - setup[.]cfg$ + meta[.]yaml$ exclude: | (?x) cpp/include/raft/neighbors/detail/faiss_select/| diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index e145aeb92e..793ca8dc67 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -26,7 +26,6 @@ dependencies: - gcc_linux-aarch64=11.* - graphviz - ipython -- joblib>=0.11 - libcublas-dev=11.11.3.6 - libcublas=11.11.3.6 - libcurand-dev=10.3.0.86 @@ -38,7 +37,6 @@ dependencies: - libucxx==0.42.*,>=0.0.0a0 - nccl>=2.19 - ninja -- numba>=0.57 - numpy>=1.23,<3.0a0 - numpydoc - nvcc_linux-aarch64=11.8 diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 75dcffa95d..a9f839bd03 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -26,7 +26,6 @@ dependencies: - gcc_linux-64=11.* - graphviz - ipython -- joblib>=0.11 - libcublas-dev=11.11.3.6 - libcublas=11.11.3.6 - libcurand-dev=10.3.0.86 @@ -38,7 +37,6 @@ dependencies: - libucxx==0.42.*,>=0.0.0a0 - nccl>=2.19 - ninja -- numba>=0.57 - numpy>=1.23,<3.0a0 - numpydoc - nvcc_linux-64=11.8 diff --git a/conda/environments/all_cuda-125_arch-aarch64.yaml b/conda/environments/all_cuda-125_arch-aarch64.yaml index bfa32c80d1..9d7286bb8e 100644 --- a/conda/environments/all_cuda-125_arch-aarch64.yaml +++ b/conda/environments/all_cuda-125_arch-aarch64.yaml @@ -27,7 +27,6 @@ dependencies: - gcc_linux-aarch64=11.* - graphviz - ipython -- joblib>=0.11 - libcublas-dev - libcurand-dev - libcusolver-dev @@ -35,7 +34,6 @@ dependencies: - libucxx==0.42.*,>=0.0.0a0 - nccl>=2.19 - ninja -- numba>=0.57 - numpy>=1.23,<3.0a0 - numpydoc - pre-commit diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 98ec334635..e4ec074ae5 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -27,7 +27,6 @@ dependencies: - gcc_linux-64=11.* - graphviz - ipython -- joblib>=0.11 - libcublas-dev - libcurand-dev - libcusolver-dev @@ -35,7 +34,6 @@ dependencies: - libucxx==0.42.*,>=0.0.0a0 - nccl>=2.19 - ninja -- numba>=0.57 - numpy>=1.23,<3.0a0 - numpydoc - pre-commit diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 4a8ed29c85..0b57432402 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -1,7 +1,5 @@ # Copyright (c) 2022-2024, NVIDIA CORPORATION. -# Usage: -# conda build . -c conda-forge -c numba -c rapidsai -c pytorch {% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') + environ.get('VERSION_SUFFIX', '') %} {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} {% set py_version = environ['CONDA_PY'] %} @@ -81,5 +79,5 @@ tests: about: home: https://rapids.ai/ license: Apache-2.0 - # license_file: LICENSE + license_file: LICENSE summary: pylibraft library diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index a8be273f82..19155166af 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -1,7 +1,5 @@ # Copyright (c) 2022-2024, NVIDIA CORPORATION. -# Usage: -# conda build . -c conda-forge -c numba -c rapidsai -c pytorch {% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') + environ.get('VERSION_SUFFIX', '') %} {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} {% set py_version = environ['CONDA_PY'] %} @@ -70,7 +68,6 @@ requirements: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - dask-cuda ={{ minor_version }} - rapids-dask-dependency ={{ minor_version }} - - joblib >=0.11 - nccl {{ nccl_version }} - pylibraft {{ version }} - python x.x @@ -87,5 +84,5 @@ tests: about: home: https://rapids.ai/ license: Apache-2.0 - # license_file: LICENSE + license_file: LICENSE summary: raft-dask library diff --git a/dependencies.yaml b/dependencies.yaml index dc1807fbf9..689cf8414c 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -392,8 +392,6 @@ dependencies: - output_types: [conda, pyproject] packages: - dask-cuda==25.2.*,>=0.0.0a0 - - joblib>=0.11 - - numba>=0.57 - rapids-dask-dependency==25.2.*,>=0.0.0a0 - output_types: conda packages: @@ -402,7 +400,6 @@ dependencies: - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file - # This index is needed for cudf and rmm. - --extra-index-url=https://pypi.nvidia.com - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple specific: @@ -482,7 +479,6 @@ dependencies: - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file - # This index is needed for rmm-cu{11,12}. - --extra-index-url=https://pypi.nvidia.com - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple specific: diff --git a/pyproject.toml b/pyproject.toml index 5042113388..2f23debfbe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,6 @@ exclude = [ skip = "./.git,./.github,./cpp/build,.*egg-info.*,./.mypy_cache,.*_skbuild" # ignore short words, and typename parameters like OffsetT ignore-regex = "\\b(.{1,4}|[A-Z]\\w*T)\\b" -ignore-words-list = "inout,numer" +ignore-words-list = "inout,unparseable,numer" builtin = "clear" quiet-level = 3 diff --git a/python/pylibraft/setup.cfg b/python/pylibraft/setup.cfg deleted file mode 100644 index 7d1a0c9065..0000000000 --- a/python/pylibraft/setup.cfg +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. - -[isort] -line_length=79 -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -combine_as_imports=True -order_by_type=True -known_dask= - dask - distributed - dask_cuda -known_rapids= - nvtext - cudf - cuml - cugraph - dask_cudf - rmm -known_first_party= - raft - pylibraft -default_section=THIRDPARTY -sections=FUTURE,STDLIB,THIRDPARTY,DASK,RAPIDS,FIRSTPARTY,LOCALFOLDER -skip= - thirdparty - .eggs - .git - .hg - .mypy_cache - .tox - .venv - _build - buck-out - build - dist - __init__.py diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index 33643c481e..cabe8e72a6 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -33,8 +33,6 @@ requires-python = ">=3.10" dependencies = [ "dask-cuda==25.2.*,>=0.0.0a0", "distributed-ucxx==0.42.*,>=0.0.0a0", - "joblib>=0.11", - "numba>=0.57", "pylibraft==25.2.*,>=0.0.0a0", "rapids-dask-dependency==25.2.*,>=0.0.0a0", "ucx-py==0.42.*,>=0.0.0a0", diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 94140d4d00..0000000000 --- a/setup.cfg +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. - -[flake8] -filename = *.py, *.pyx, *.pxd, *.pxi -exclude = __init__.py, *.egg, build, docs, .git -force-check = True -ignore = - # line break before binary operator - W503, - # whitespace before : - E203 -per-file-ignores = - # Rules ignored only in Cython: - # E211: whitespace before '(' (used in multi-line imports) - # E225: Missing whitespace around operators (breaks cython casting syntax like ) - # E226: Missing whitespace around arithmetic operators (breaks cython pointer syntax like int*) - # E227: Missing whitespace around bitwise or shift operator (Can also break casting syntax) - # E275: Missing whitespace after keyword (Doesn't work with Cython except?) - # E402: invalid syntax (works for Python, not Cython) - # E999: invalid syntax (works for Python, not Cython) - # W504: line break after binary operator (breaks lines that end with a pointer) - *.pyx: E211, E225, E226, E227, E275, E402, E999, W504 - *.pxd: E211, E225, E226, E227, E275, E402, E999, W504 - *.pxi: E211, E225, E226, E227, E275, E402, E999, W504 - -[pydocstyle] -# Due to https://github.com/PyCQA/pydocstyle/issues/363, we must exclude rather -# than include using match-dir. Note that as discussed in -# https://stackoverflow.com/questions/65478393/how-to-filter-directories-using-the-match-dir-flag-for-pydocstyle, -# unlike the match option above this match-dir will have no effect when -# pydocstyle is invoked from pre-commit. Therefore this exclusion list must -# also be maintained in the pre-commit config file. -match-dir = ^(?!(ci|cpp|conda|docs)).*$ -# Allow missing docstrings for docutils -ignore-decorators = .*(docutils|doc_apply|copy_docstring).* -select = - D201, D204, D206, D207, D208, D209, D210, D211, D214, D215, D300, D301, D302, D403, D405, D406, D407, D408, D409, D410, D411, D412, D414, D418 - # Would like to enable the following rules in the future: - # D200, D202, D205, D400 - -[mypy] -ignore_missing_imports = True -# If we don't specify this, then mypy will check excluded files if -# they are imported by a checked file. -follow_imports = skip - -[codespell] -# note: pre-commit passes explicit lists of files here, which this skip file list doesn't override - -# this is only to allow you to run codespell interactively -skip = ./.git,./.github,./cpp/build,.*egg-info.*,./.mypy_cache,.*_skbuild -# ignore short words, and typename parameters like OffsetT -ignore-regex = \b(.{1,4}|[A-Z]\w*T)\b -ignore-words-list = inout,unparseable,numer -builtin = clear -quiet-level = 3 From 1b62c4117a35b11ce3c830daae248e32ebf75e3f Mon Sep 17 00:00:00 2001 From: Victor Lafargue Date: Fri, 10 Jan 2025 22:29:03 +0100 Subject: [PATCH 78/79] Fix lanczos solver integer overflow (#2536) Partially answers https://github.com/rapidsai/cuml/issues/6204 Authors: - Victor Lafargue (https://github.com/viclafargue) - Corey J. Nolet (https://github.com/cjnolet) Approvers: - Corey J. Nolet (https://github.com/cjnolet) - Micka (https://github.com/lowener) URL: https://github.com/rapidsai/raft/pull/2536 --- cpp/include/raft/sparse/detail/coo.cuh | 14 ++++++++----- .../raft/sparse/solver/detail/lanczos.cuh | 21 +++++++++++-------- .../raft/spectral/detail/matrix_wrappers.hpp | 8 +++---- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/cpp/include/raft/sparse/detail/coo.cuh b/cpp/include/raft/sparse/detail/coo.cuh index 91ba363168..9a38c11a07 100644 --- a/cpp/include/raft/sparse/detail/coo.cuh +++ b/cpp/include/raft/sparse/detail/coo.cuh @@ -182,7 +182,7 @@ class COO { * @param n_rows: number of rows in the dense matrix * @param n_cols: number of columns in the dense matrix */ - void setSize(int n_rows, int n_cols) + void setSize(Index_Type n_rows, Index_Type n_cols) { this->n_rows = n_rows; this->n_cols = n_cols; @@ -192,7 +192,7 @@ class COO { * @brief Set the number of rows and cols for a square dense matrix * @param n: number of rows and cols */ - void setSize(int n) + void setSize(Index_Type n) { this->n_rows = n; this->n_cols = n; @@ -204,7 +204,10 @@ class COO { * @param init: should values be initialized to 0? * @param stream: CUDA stream to use */ - void allocate(int nnz, bool init, cudaStream_t stream) { this->allocate(nnz, 0, init, stream); } + void allocate(Index_Type nnz, bool init, cudaStream_t stream) + { + this->allocate(nnz, 0, init, stream); + } /** * @brief Allocate the underlying arrays @@ -213,7 +216,7 @@ class COO { * @param init: should values be initialized to 0? * @param stream: CUDA stream to use */ - void allocate(int nnz, int size, bool init, cudaStream_t stream) + void allocate(Index_Type nnz, Index_Type size, bool init, cudaStream_t stream) { this->allocate(nnz, size, size, init, stream); } @@ -226,7 +229,8 @@ class COO { * @param init: should values be initialized to 0? * @param stream: stream to use for init */ - void allocate(int nnz, int n_rows, int n_cols, bool init, cudaStream_t stream) + void allocate( + Index_Type nnz, Index_Type n_rows, Index_Type n_cols, bool init, cudaStream_t stream) { this->n_rows = n_rows; this->n_cols = n_cols; diff --git a/cpp/include/raft/sparse/solver/detail/lanczos.cuh b/cpp/include/raft/sparse/solver/detail/lanczos.cuh index 6f03f77bc0..ddfa01731a 100644 --- a/cpp/include/raft/sparse/solver/detail/lanczos.cuh +++ b/cpp/include/raft/sparse/solver/detail/lanczos.cuh @@ -624,7 +624,7 @@ static int lanczosRestart(raft::resources const& handle, value_type_t* shifts_host; // Orthonormal matrix for similarity transform - value_type_t* V_dev = work_dev + n * iter; + value_type_t* V_dev = work_dev + (size_t)n * (size_t)iter; // ------------------------------------------------------- // Implementation @@ -641,7 +641,7 @@ static int lanczosRestart(raft::resources const& handle, // std::cout < 0 && nEigVecs <= n, "Invalid number of eigenvectors."); + RAFT_EXPECTS(nEigVecs > 0 && (size_t)nEigVecs <= n, "Invalid number of eigenvectors."); RAFT_EXPECTS(restartIter > 0, "Invalid restartIter."); RAFT_EXPECTS(tol > 0, "Invalid tolerance."); RAFT_EXPECTS(maxIter >= nEigVecs, "Invalid maxIter."); @@ -1395,10 +1398,10 @@ int computeLargestEigenvectors( unsigned long long seed = 123456) { // Matrix dimension - index_type_t n = A.nrows_; + size_t n = A.nrows_; // Check that parameters are valid - RAFT_EXPECTS(nEigVecs > 0 && nEigVecs <= n, "Invalid number of eigenvectors."); + RAFT_EXPECTS(nEigVecs > 0 && (size_t)nEigVecs <= n, "Invalid number of eigenvectors."); RAFT_EXPECTS(restartIter > 0, "Invalid restartIter."); RAFT_EXPECTS(tol > 0, "Invalid tolerance."); RAFT_EXPECTS(maxIter >= nEigVecs, "Invalid maxIter."); diff --git a/cpp/include/raft/spectral/detail/matrix_wrappers.hpp b/cpp/include/raft/spectral/detail/matrix_wrappers.hpp index 1fe078bd32..db8a5dc9ef 100644 --- a/cpp/include/raft/spectral/detail/matrix_wrappers.hpp +++ b/cpp/include/raft/spectral/detail/matrix_wrappers.hpp @@ -39,14 +39,14 @@ // ========================================================= // Get index of matrix entry -#define IDX(i, j, lda) ((i) + (j) * (lda)) +#define IDX(i, j, lda) ((size_t)(i) + (j) * (lda)) namespace raft { namespace spectral { namespace matrix { namespace detail { -using size_type = int; // for now; TODO: move it in appropriate header +using size_type = size_t; // for now; TODO: move it in appropriate header // Apply diagonal matrix to vector: // @@ -326,7 +326,7 @@ struct laplacian_matrix_t : sparse_matrix_t { raft_handle, row_offsets, col_indices, values, nrows, nnz), diagonal_(raft_handle, nrows) { - vector_t ones{raft_handle, nrows}; + vector_t ones{raft_handle, (size_t)nrows}; ones.fill(1.0); sparse_matrix_t::mv(1, ones.raw(), 0, diagonal_.raw()); } @@ -341,7 +341,7 @@ struct laplacian_matrix_t : sparse_matrix_t { csr_m.nnz_), diagonal_(raft_handle, csr_m.nrows_) { - vector_t ones{raft_handle, csr_m.nrows_}; + vector_t ones{raft_handle, (size_t)csr_m.nrows_}; ones.fill(1.0); sparse_matrix_t::mv(1, ones.raw(), 0, diagonal_.raw()); } From 5c826d7320486852c30a18f6e039d0cda83c5c62 Mon Sep 17 00:00:00 2001 From: Micka Date: Tue, 14 Jan 2025 00:34:20 +0100 Subject: [PATCH 79/79] Add support for different data type of bitset (#2535) This PR is useful for Milvus. Previously the `bitset_view` object only supported the data type used to create the bitset. With the proposed modifications, a `bitset_view` object can support any data type used to create the bitset by specifying the `original_nbits` parameter in the class constructor. Authors: - Micka (https://github.com/lowener) Approvers: - Corey J. Nolet (https://github.com/cjnolet) - rhdong (https://github.com/rhdong) URL: https://github.com/rapidsai/raft/pull/2535 --- cpp/include/raft/core/bitmap.hpp | 24 ++++++-- cpp/include/raft/core/bitset.cuh | 53 ++++++++++++++--- cpp/include/raft/core/bitset.hpp | 34 +++++++++-- cpp/test/core/bitset.cu | 98 ++++++++++++++++++++++++++++++-- 4 files changed, 188 insertions(+), 21 deletions(-) diff --git a/cpp/include/raft/core/bitmap.hpp b/cpp/include/raft/core/bitmap.hpp index 86b2d77478..5a6656f572 100644 --- a/cpp/include/raft/core/bitmap.hpp +++ b/cpp/include/raft/core/bitmap.hpp @@ -53,9 +53,18 @@ struct bitmap_view : public bitset_view { * @param bitmap_ptr Device raw pointer * @param rows Number of row in the matrix. * @param cols Number of col in the matrix. + * @param original_nbits Original number of bits used when the bitmap was created, to handle + * potential mismatches of data types. This is useful for using ANN indexes when a bitmap was + * originally created with a different data type than the ones currently supported in cuVS ANN + * indexes. */ - _RAFT_HOST_DEVICE bitmap_view(bitmap_t* bitmap_ptr, index_t rows, index_t cols) - : bitset_view(bitmap_ptr, rows * cols), rows_(rows), cols_(cols) + _RAFT_HOST_DEVICE bitmap_view(bitmap_t* bitmap_ptr, + index_t rows, + index_t cols, + index_t original_nbits = 0) + : bitset_view(bitmap_ptr, rows * cols, original_nbits), + rows_(rows), + cols_(cols) { } @@ -65,11 +74,18 @@ struct bitmap_view : public bitset_view { * @param bitmap_span Device vector view of the bitmap * @param rows Number of row in the matrix. * @param cols Number of col in the matrix. + * @param original_nbits Original number of bits used when the bitmap was created, to handle + * potential mismatches of data types. This is useful for using ANN indexes when a bitmap was + * originally created with a different data type than the ones currently supported in cuVS ANN + * indexes. */ _RAFT_HOST_DEVICE bitmap_view(raft::device_vector_view bitmap_span, index_t rows, - index_t cols) - : bitset_view(bitmap_span, rows * cols), rows_(rows), cols_(cols) + index_t cols, + index_t original_nbits = 0) + : bitset_view(bitmap_span, rows * cols, original_nbits), + rows_(rows), + cols_(cols) { } diff --git a/cpp/include/raft/core/bitset.cuh b/cpp/include/raft/core/bitset.cuh index d1bffdb81e..feaef1a172 100644 --- a/cpp/include/raft/core/bitset.cuh +++ b/cpp/include/raft/core/bitset.cuh @@ -32,12 +32,41 @@ namespace raft::core { +template +_RAFT_HOST_DEVICE void inline compute_original_nbits_position(const index_t original_nbits, + const index_t nbits, + const index_t sample_index, + index_t& new_bit_index, + index_t& new_bit_offset) +{ + const index_t original_bit_index = sample_index / original_nbits; + const index_t original_bit_offset = sample_index % original_nbits; + new_bit_index = original_bit_index * original_nbits / nbits; + new_bit_offset = 0; + if (original_nbits > nbits) { + new_bit_index += original_bit_offset / nbits; + new_bit_offset = original_bit_offset % nbits; + } else { + index_t ratio = nbits / original_nbits; + new_bit_offset += (original_bit_index % ratio) * original_nbits; + new_bit_offset += original_bit_offset % nbits; + } +} + template _RAFT_HOST_DEVICE inline bool bitset_view::test(const index_t sample_index) const { - const bitset_t bit_element = bitset_ptr_[sample_index / bitset_element_size]; - const index_t bit_index = sample_index % bitset_element_size; - const bool is_bit_set = (bit_element & (bitset_t{1} << bit_index)) != 0; + const index_t nbits = sizeof(bitset_t) * 8; + index_t bit_index = 0; + index_t bit_offset = 0; + if (original_nbits_ == 0 || nbits == original_nbits_) { + bit_index = sample_index / bitset_element_size; + bit_offset = sample_index % bitset_element_size; + } else { + compute_original_nbits_position(original_nbits_, nbits, sample_index, bit_index, bit_offset); + } + const bitset_t bit_element = bitset_ptr_[bit_index]; + const bool is_bit_set = (bit_element & (bitset_t{1} << bit_offset)) != 0; return is_bit_set; } @@ -51,14 +80,22 @@ template _RAFT_DEVICE void bitset_view::set(const index_t sample_index, bool set_value) const { - const index_t bit_element = sample_index / bitset_element_size; - const index_t bit_index = sample_index % bitset_element_size; - const bitset_t bitmask = bitset_t{1} << bit_index; + const index_t nbits = sizeof(bitset_t) * 8; + index_t bit_index = 0; + index_t bit_offset = 0; + + if (original_nbits_ == 0 || nbits == original_nbits_) { + bit_index = sample_index / bitset_element_size; + bit_offset = sample_index % bitset_element_size; + } else { + compute_original_nbits_position(original_nbits_, nbits, sample_index, bit_index, bit_offset); + } + const bitset_t bitmask = bitset_t{1} << bit_offset; if (set_value) { - atomicOr(bitset_ptr_ + bit_element, bitmask); + atomicOr(bitset_ptr_ + bit_index, bitmask); } else { const bitset_t bitmask2 = ~bitmask; - atomicAnd(bitset_ptr_ + bit_element, bitmask2); + atomicAnd(bitset_ptr_ + bit_index, bitmask2); } } diff --git a/cpp/include/raft/core/bitset.hpp b/cpp/include/raft/core/bitset.hpp index be828def87..e4bea2c0c5 100644 --- a/cpp/include/raft/core/bitset.hpp +++ b/cpp/include/raft/core/bitset.hpp @@ -42,8 +42,20 @@ template struct bitset_view { static constexpr index_t bitset_element_size = sizeof(bitset_t) * 8; - _RAFT_HOST_DEVICE bitset_view(bitset_t* bitset_ptr, index_t bitset_len) - : bitset_ptr_{bitset_ptr}, bitset_len_{bitset_len} + /** + * @brief Create a bitset view from a device pointer to the bitset. + * + * @param bitset_ptr Device pointer to the bitset + * @param bitset_len Number of bits in the bitset + * @param original_nbits Original number of bits used when the bitset was created, to handle + * potential mismatches of data types. This is useful for using ANN indexes when a bitset was + * originally created with a different data type than the ones currently supported in cuVS ANN + * indexes. + */ + _RAFT_HOST_DEVICE bitset_view(bitset_t* bitset_ptr, + index_t bitset_len, + index_t original_nbits = 0) + : bitset_ptr_{bitset_ptr}, bitset_len_{bitset_len}, original_nbits_{original_nbits} { } /** @@ -51,10 +63,17 @@ struct bitset_view { * * @param bitset_span Device vector view of the bitset * @param bitset_len Number of bits in the bitset + * @param original_nbits Original number of bits used when the bitset was created, to handle + * potential mismatches of data types. This is useful for using ANN indexes when a bitset was + * originally created with a different data type than the ones currently supported in cuVS ANN + * indexes. */ _RAFT_HOST_DEVICE bitset_view(raft::device_vector_view bitset_span, - index_t bitset_len) - : bitset_ptr_{bitset_span.data_handle()}, bitset_len_{bitset_len} + index_t bitset_len, + index_t original_nbits = 0) + : bitset_ptr_{bitset_span.data_handle()}, + bitset_len_{bitset_len}, + original_nbits_{original_nbits} { } /** @@ -180,9 +199,16 @@ struct bitset_view { return (bitset_len + bits_per_element - 1) / bits_per_element; } + /** + * @brief Get the original number of bits of the bitset. + */ + auto get_original_nbits() const -> index_t { return original_nbits_; } + void set_original_nbits(index_t original_nbits) { original_nbits_ = original_nbits; } + private: bitset_t* bitset_ptr_; index_t bitset_len_; + index_t original_nbits_; }; /** diff --git a/cpp/test/core/bitset.cu b/cpp/test/core/bitset.cu index ac601274c1..f094f60ded 100644 --- a/cpp/test/core/bitset.cu +++ b/cpp/test/core/bitset.cu @@ -24,6 +24,8 @@ #include #include +#include +#include #include namespace raft::core { @@ -73,6 +75,40 @@ void test_cpu_bitset(const std::vector& bitset, } } +template +void test_cpu_bitset_nbits(const bitset_t* bitset, + const std::vector& queries, + std::vector& result, + unsigned original_nbits_) +{ + constexpr size_t nbits = sizeof(bitset_t) * 8; + if (original_nbits_ == nbits) { + for (size_t i = 0; i < queries.size(); i++) { + result[i] = + uint8_t((bitset[queries[i] / nbits] & (bitset_t{1} << (queries[i] % nbits))) != 0); + } + } + for (size_t i = 0; i < queries.size(); i++) { + const index_t sample_index = queries[i]; + const index_t original_bit_index = sample_index / original_nbits_; + const index_t original_bit_offset = sample_index % original_nbits_; + index_t new_bit_index = original_bit_index * original_nbits_ / nbits; + index_t new_bit_offset = 0; + if (original_nbits_ > nbits) { + new_bit_index += original_bit_offset / nbits; + new_bit_offset = original_bit_offset % nbits; + } else { + index_t ratio = nbits / original_nbits_; + new_bit_offset += (original_bit_index % ratio) * original_nbits_; + new_bit_offset += original_bit_offset % nbits; + } + const bitset_t bit_element = bitset[new_bit_index]; + const bool is_bit_set = (bit_element & (bitset_t{1} << new_bit_offset)) != 0; + + result[i] = uint8_t(is_bit_set); + } +} + template void flip_cpu_bitset(std::vector& bitset) { @@ -168,11 +204,12 @@ class BitsetTest : public testing::TestWithParam { resource::sync_stream(res, stream); ASSERT_TRUE(hostVecMatch(bitset_ref, bitset_result, raft::Compare())); - auto query_device = raft::make_device_vector(res, spec.query_len); - auto result_device = raft::make_device_vector(res, spec.query_len); - auto query_cpu = std::vector(spec.query_len); - auto result_cpu = std::vector(spec.query_len); - auto result_ref = std::vector(spec.query_len); + auto query_device = raft::make_device_vector(res, spec.query_len); + auto result_device = raft::make_device_vector(res, spec.query_len); + auto query_cpu = std::vector(spec.query_len); + auto result_cpu = std::vector(spec.query_len); + auto result_ref_nbits = std::vector(spec.query_len); + auto result_ref = std::vector(spec.query_len); // Create queries and verify the test results raft::random::uniformInt(res, rng, query_device.view(), index_t(0), index_t(spec.bitset_len)); @@ -194,6 +231,57 @@ class BitsetTest : public testing::TestWithParam { resource::sync_stream(res, stream); ASSERT_TRUE(hostVecMatch(bitset_ref, bitset_result, raft::Compare())); + // Reinterpret the bitset as uint8_t, uint32 then uint64_t + { + // Test CPU logic + test_cpu_bitset(bitset_ref, query_cpu, result_ref); + uint8_t* bitset_cpu_uint8 = (uint8_t*)std::malloc(sizeof(bitset_t) * bitset_ref.size()); + std::memcpy(bitset_cpu_uint8, bitset_ref.data(), sizeof(bitset_t) * bitset_ref.size()); + test_cpu_bitset_nbits(bitset_cpu_uint8, query_cpu, result_ref_nbits, sizeof(bitset_t) * 8); + ASSERT_TRUE(hostVecMatch(result_ref, result_ref_nbits, raft::Compare())); + std::free(bitset_cpu_uint8); + + // Test GPU uint8_t, uint32_t, uint64_t + auto my_bitset_view_uint8_t = raft::core::bitset_view( + reinterpret_cast(my_bitset.data()), my_bitset.size(), sizeof(bitset_t) * 8); + raft::linalg::map( + res, + result_device.view(), + [my_bitset_view_uint8_t] __device__(index_t query) { + return my_bitset_view_uint8_t.test(query); + }, + raft::make_const_mdspan(query_device.view())); + update_host(result_cpu.data(), result_device.data_handle(), result_device.extent(0), stream); + resource::sync_stream(res, stream); + ASSERT_TRUE(hostVecMatch(result_ref, result_cpu, Compare())); + + auto my_bitset_view_uint32_t = raft::core::bitset_view( + reinterpret_cast(my_bitset.data()), my_bitset.size(), sizeof(bitset_t) * 8); + raft::linalg::map( + res, + result_device.view(), + [my_bitset_view_uint32_t] __device__(index_t query) { + return my_bitset_view_uint32_t.test(query); + }, + raft::make_const_mdspan(query_device.view())); + update_host(result_cpu.data(), result_device.data_handle(), result_device.extent(0), stream); + resource::sync_stream(res, stream); + ASSERT_TRUE(hostVecMatch(result_ref, result_cpu, Compare())); + + auto my_bitset_view_uint64_t = raft::core::bitset_view( + reinterpret_cast(my_bitset.data()), my_bitset.size(), sizeof(bitset_t) * 8); + raft::linalg::map( + res, + result_device.view(), + [my_bitset_view_uint64_t] __device__(index_t query) { + return my_bitset_view_uint64_t.test(query); + }, + raft::make_const_mdspan(query_device.view())); + update_host(result_cpu.data(), result_device.data_handle(), result_device.extent(0), stream); + resource::sync_stream(res, stream); + ASSERT_TRUE(hostVecMatch(result_ref, result_cpu, Compare())); + } + // test sparsity, repeat and eval_n_elements { auto my_bitset_view = my_bitset.view();