[WIP] new CUDA CI+development Docker container

scripts/docker/Dockerfile.nvcc

@@ -1,23 +1,28 @@
-FROM nvidia/cuda:11.6.1-devel-ubuntu20.04
+FROM nvidia/cuda:12.0.0-devel-ubuntu22.04
 
 RUN apt-get clean && apt-get update -y && \
-    DEBIAN_FRONTEND="noninteractive" TZ=America/New_York apt-get install -y --no-install-recommends git python3-minimal libpython3-stdlib bc hwloc wget openssh-client python3-numpy python3-h5py python3-matplotlib python3-scipy python3-pip lcov curl cuda-nsight-systems-11-6 cmake ninja-build
+    DEBIAN_FRONTEND="noninteractive" TZ=America/New_York apt-get install -y --no-install-recommends git python3-minimal libpython3-stdlib bc hwloc wget openssh-client python3-numpy python3-h5py python3-matplotlib python3-scipy python3-pip lcov curl cuda-nsight-systems-12-6 cmake ninja-build libpython3-dev gcc-11 g++-11 && \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 && \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 10
+
+RUN g++ --version
 
 RUN pip3 install unyt
 
+RUN pip3 install blosc2
+
 RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key| apt-key add - && \
-    echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-15 main" > /etc/apt/sources.list.d/llvm.list
+    echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-19 main" > /etc/apt/sources.list.d/llvm.list
 
 RUN apt-get clean && apt-get update -y && \
-    DEBIAN_FRONTEND="noninteractive" TZ=America/New_York apt-get install -y --no-install-recommends clang-15 llvm-15 libomp-15-dev && \
+    DEBIAN_FRONTEND="noninteractive" TZ=America/New_York apt-get install -y --no-install-recommends clang-19 llvm-19 libomp-19-dev && \
     rm -rf /var/lib/apt/lists/*
 
-
 RUN cd /tmp && \
     wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.4.tar.bz2 && \
     tar xjf openmpi-4.1.4.tar.bz2 && \
     cd openmpi-4.1.4 && \
-    ./configure --prefix=/opt/openmpi --enable-mpi-cxx --with-cuda && \
+    ./configure --prefix=/opt/openmpi --disable-mpi-fortran --disable-oshmem --with-cuda && \
     make -j16 && \
     make install && \
     cd / && \
@@ -36,19 +41,28 @@ RUN cd /tmp && \
     cd / && \
     rm -rf /tmp/hdf5-1.12.2*
 
-RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+RUN mkdir /tmp/build-adios2 && cd /tmp/build-adios2 && \
+    git clone https://github.com/ornladios/ADIOS2.git ADIOS2 && \
+    mkdir adios2-build && cd adios2-build && \
+    cmake ../ADIOS2 -DADIOS2_USE_Blosc2=ON -DADIOS2_USE_Fortran=OFF && \
+    make -j8 && make install && \
+    cd / && \
+    rm -rf /tmp/build-adios2
 
-RUN curl -L https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-linux-x86_64.tar.gz -o cmake-3.23.2-linux-x86_64.tar.gz && \
-    tar -xzf cmake-3.23.2-linux-x86_64.tar.gz -C /opt
+RUN mkdir /tmp/build-openpmd && cd /tmp/build-openpmd && \
+    git clone https://github.com/openPMD/openPMD-api.git && \
+    mkdir openPMD-api-build && cd openPMD-api-build && \
+    cmake ../openPMD-api -DopenPMD_USE_PYTHON=ON -DPython_EXECUTABLE=$(which python3) -DopenPMD_USE_ADIOS2=ON && \
+    cmake --build . && \
+    cmake --build . --target install && \
+    cd / && \
+    rm -rf /tmp/build-openpmd
 
-ENV PATH=/opt/cmake-3.23.2-linux-x86_64/bin:$PATH
+COPY ascent_build /tmp/ascent_build
 
-COPY build_ascent_cuda.sh /tmp/build-ascent/build_ascent_cuda.sh
+## NOTE: with enable_cuda=ON, arm64 Dockerfile builds fail for some reason
 
-RUN cd /tmp/build-ascent && \
-    bash build_ascent_cuda.sh && \
+RUN cd /tmp/ascent_build && \
+    env enable_cuda=ON bash build_ascent.sh && \
     cd / && \
-    rm -rf /tmp/build-ascent
-
-# manually downgrade numpy as deprecated `typeDict` is still used by h5py
-RUN pip install numpy==1.21
+    rm -rf /tmp/ascent_build

scripts/docker/ascent_build/2023_01_30_raja.patch

@@ -0,0 +1,36 @@
+From 9a50702cf835996f96cb33e2cb4c0aa1a7a86df5 Mon Sep 17 00:00:00 2001
+From: Cyrus Harrison <[email protected]>
+Date: Fri, 27 Jan 2023 15:49:35 -0800
+Subject: [PATCH] try new logic for windows shared exports
+
+---
+ azure-pipelines.yml        | 2 +-
+ include/RAJA/config.hpp.in | 2 +-
+ 2 files changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/azure-pipelines.yml b/azure-pipelines.yml
+index c84a71eb18..268ba4a660 100644
+--- a/azure-pipelines.yml
++++ b/azure-pipelines.yml
+@@ -3,7 +3,7 @@ jobs:
+   strategy:
+     matrix:
+       shared:
+-        SHARED_ARGS: '-DBUILD_SHARED_LIBS=On -DCMAKE_CXX_FLAGS="/DRAJASHAREDDLL_EXPORTS" '
++        SHARED_ARGS: '-DBUILD_SHARED_LIBS=On'
+       static:
+         SHARED_ARGS: '-DBUILD_SHARED_LIBS=Off'
+   pool:
+diff --git a/include/RAJA/config.hpp.in b/include/RAJA/config.hpp.in
+index 26b0b0dbde..0347650fc4 100644
+--- a/include/RAJA/config.hpp.in
++++ b/include/RAJA/config.hpp.in
+@@ -364,7 +364,7 @@ const int DATA_ALIGN = @RAJA_DATA_ALIGN@;
+ //
+
+ #if (defined(_WIN32) || defined(_WIN64)) && !defined(RAJA_WIN_STATIC_BUILD)
+-#ifdef RAJASHAREDDLL_EXPORTS
++#ifdef RAJA_EXPORTS
+ #define RAJASHAREDDLL_API __declspec(dllexport)
+ #else
+ #define RAJASHAREDDLL_API __declspec(dllimport)

scripts/docker/ascent_build/2023_12_06_vtkm-mr3160-rocthrust-fix.patch

@@ -0,0 +1,74 @@
+From c9ec6ae6a62b9bd257e727e999987ef31384e3ac Mon Sep 17 00:00:00 2001
+From: Vicente Adolfo Bolea Sanchez <[email protected]>
+Date: Thu, 30 Nov 2023 15:55:32 -0500
+Subject: [PATCH] kokkos: let link vtkm_cont to roc::rocthrust
+
+Also reorder the declarion of the option VTKm_ENABLE_KOKKOS_THRUST
+to be set before calling VTKmDeviceAdapters.
+---
+ CMake/VTKmDeviceAdapters.cmake           |  5 +----
+ CMakeLists.txt                           | 10 +++++-----
+ vtkm/cont/kokkos/internal/CMakeLists.txt |  3 +++
+ 3 files changed, 9 insertions(+), 9 deletions(-)
+
+diff --git a/CMake/VTKmDeviceAdapters.cmake b/CMake/VTKmDeviceAdapters.cmake
+index fb13d0bf85..7b8bf2df9b 100644
+--- a/CMake/VTKmDeviceAdapters.cmake
++++ b/CMake/VTKmDeviceAdapters.cmake
+@@ -360,10 +360,7 @@ if(VTKm_ENABLE_KOKKOS AND NOT TARGET vtkm_kokkos)
+
+     # Make sure rocthrust is available if requested
+     if(VTKm_ENABLE_KOKKOS_THRUST)
+-      find_package(rocthrust)
+-      if(NOT rocthrust_FOUND)
+-        message(FATAL_ERROR "rocthrust not found. Please set VTKm_ENABLE_KOKKOS_THRUST to OFF.")
+-      endif()
++      find_package(rocthrust REQUIRED CONFIG)
+     endif()
+   endif()
+
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 39a9b3bc09..d8204114c7 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -191,6 +191,11 @@ vtkm_option(VTKm_OVERRIDE_CTEST_TIMEOUT "Disable default ctest timeout" OFF)
+ # VTKm_ENABLE_MPI=ON.
+ cmake_dependent_option(VTKm_ENABLE_GPU_MPI "Enable GPU AWARE MPI support" OFF "VTKm_ENABLE_MPI" OFF)
+
++# By default: Set VTKm_ENABLE_KOKKOS_THRUST to ON if VTKm_ENABLE_KOKKOS is ON, otherwise
++# disable it (or if the user explicitly turns this option OFF)
++cmake_dependent_option(VTKm_ENABLE_KOKKOS_THRUST "Enable Kokkos thrust support (only valid with CUDA and HIP)"
++  ON "VTKm_ENABLE_KOKKOS;Kokkos_ENABLE_CUDA OR Kokkos_ENABLE_HIP" OFF)
++
+ mark_as_advanced(
+   VTKm_ENABLE_LOGGING
+   VTKm_NO_ASSERT
+@@ -232,11 +237,6 @@ include(VTKmBuildType)
+ # Include the vtk-m wrappers
+ include(VTKmWrappers)
+
+-# By default: Set VTKm_ENABLE_KOKKOS_THRUST to ON if VTKm_ENABLE_KOKKOS is ON, otherwise
+-# disable it (or if the user explicitly turns this option OFF)
+-cmake_dependent_option(VTKm_ENABLE_KOKKOS_THRUST "Enable Kokkos thrust support (only valid with CUDA and HIP)"
+-  ON "VTKm_ENABLE_KOKKOS;Kokkos_ENABLE_CUDA OR Kokkos_ENABLE_HIP" OFF)
+-
+ # Create vtkm_compiler_flags library. This is an interface library that
+ # holds all the C++ compiler flags that are needed for consumers and
+ # when building VTK-m.
+diff --git a/vtkm/cont/kokkos/internal/CMakeLists.txt b/vtkm/cont/kokkos/internal/CMakeLists.txt
+index 9f924b0f4b..9b731c9fdd 100644
+--- a/vtkm/cont/kokkos/internal/CMakeLists.txt
++++ b/vtkm/cont/kokkos/internal/CMakeLists.txt
+@@ -34,6 +34,9 @@ if (TARGET vtkm_kokkos)
+   elseif(TARGET vtkm_kokkos_hip)
+     set_source_files_properties(${sources} TARGET_DIRECTORY vtkm_cont PROPERTIES LANGUAGE HIP)
+     kokkos_compilation(SOURCE ${sources})
++    if (VTKm_ENABLE_KOKKOS_THRUST)
++      target_link_libraries(vtkm_cont INTERFACE roc::rocthrust)
++    endif()
+   endif()
+
+ else()
+-- 
+2.35.3
+

scripts/docker/ascent_build/2024_05_03_vtkm-mr3215-ext-geom-fix.patch

@@ -0,0 +1,145 @@
+From 49518e5054c607942f644c82a5289e12b0f50476 Mon Sep 17 00:00:00 2001
+From: Kenneth Moreland <[email protected]>
+Date: Fri, 3 May 2024 09:22:56 -0400
+Subject: [PATCH] Fix bug with ExtractGeometry filter
+
+The `ExtractGeometry` filter was outputing datasets containing
+`CellSetPermutation` as the representation for the cells. Although this is
+technically correct and a very fast implementation, it is essentially
+useless. The problem is that any downstream processing will have to know
+that the data has a `CellSetPermutation`. None do (because the permutation
+can be on any other cell set type, which creates an explosion of possible
+cell types).
+
+Like was done with `Threshold` a while ago, this problem is fixed by deep
+copying the result into a `CellSetExplicit`. This behavior is consistent
+with VTK.
+---
+ .../changelog/extract-geometry-permutation.md | 13 +++++++
+ .../testing/UnitTestExtractGeometryFilter.cxx | 13 ++++++-
+ .../worklet/ExtractGeometry.h                 | 34 +++++++------------
+ 3 files changed, 37 insertions(+), 23 deletions(-)
+ create mode 100644 docs/changelog/extract-geometry-permutation.md
+
+diff --git a/docs/changelog/extract-geometry-permutation.md b/docs/changelog/extract-geometry-permutation.md
+new file mode 100644
+index 0000000000..8a90495f76
+--- /dev/null
++++ b/docs/changelog/extract-geometry-permutation.md
+@@ -0,0 +1,13 @@
++# Fix bug with ExtractGeometry filter
++
++The `ExtractGeometry` filter was outputing datasets containing
++`CellSetPermutation` as the representation for the cells. Although this is
++technically correct and a very fast implementation, it is essentially
++useless. The problem is that any downstream processing will have to know
++that the data has a `CellSetPermutation`. None do (because the permutation
++can be on any other cell set type, which creates an explosion of possible
++cell types).
++
++Like was done with `Threshold` a while ago, this problem is fixed by deep
++copying the result into a `CellSetExplicit`. This behavior is consistent
++with VTK.
+diff --git a/vtkm/filter/entity_extraction/testing/UnitTestExtractGeometryFilter.cxx b/vtkm/filter/entity_extraction/testing/UnitTestExtractGeometryFilter.cxx
+index 675df8f77c..14de333666 100644
+--- a/vtkm/filter/entity_extraction/testing/UnitTestExtractGeometryFilter.cxx
++++ b/vtkm/filter/entity_extraction/testing/UnitTestExtractGeometryFilter.cxx
+@@ -11,6 +11,7 @@
+ #include <vtkm/cont/testing/MakeTestDataSet.h>
+ #include <vtkm/cont/testing/Testing.h>
+
++#include <vtkm/filter/clean_grid/CleanGrid.h>
+ #include <vtkm/filter/entity_extraction/ExtractGeometry.h>
+
+ using vtkm::cont::testing::MakeTestDataSet;
+@@ -41,11 +42,21 @@ public:
+     vtkm::cont::DataSet output = extractGeometry.Execute(dataset);
+     VTKM_TEST_ASSERT(test_equal(output.GetNumberOfCells(), 8), "Wrong result for ExtractGeometry");
+
++    vtkm::filter::clean_grid::CleanGrid cleanGrid;
++    cleanGrid.SetCompactPointFields(true);
++    cleanGrid.SetMergePoints(false);
++    vtkm::cont::DataSet cleanOutput = cleanGrid.Execute(output);
++
+     vtkm::cont::ArrayHandle<vtkm::Float32> outCellData;
+-    output.GetField("cellvar").GetData().AsArrayHandle(outCellData);
++    cleanOutput.GetField("cellvar").GetData().AsArrayHandle(outCellData);
+
+     VTKM_TEST_ASSERT(outCellData.ReadPortal().Get(0) == 21.f, "Wrong cell field data");
+     VTKM_TEST_ASSERT(outCellData.ReadPortal().Get(7) == 42.f, "Wrong cell field data");
++
++    vtkm::cont::ArrayHandle<vtkm::Float32> outPointData;
++    cleanOutput.GetField("pointvar").GetData().AsArrayHandle(outPointData);
++    VTKM_TEST_ASSERT(outPointData.ReadPortal().Get(0) == 99);
++    VTKM_TEST_ASSERT(outPointData.ReadPortal().Get(7) == 90);
+   }
+
+   static void TestUniformByBox1()
+diff --git a/vtkm/filter/entity_extraction/worklet/ExtractGeometry.h b/vtkm/filter/entity_extraction/worklet/ExtractGeometry.h
+index 97521335f2..449d7eae60 100644
+--- a/vtkm/filter/entity_extraction/worklet/ExtractGeometry.h
++++ b/vtkm/filter/entity_extraction/worklet/ExtractGeometry.h
+@@ -10,11 +10,13 @@
+ #ifndef vtkm_m_worklet_ExtractGeometry_h
+ #define vtkm_m_worklet_ExtractGeometry_h
+
++#include <vtkm/worklet/CellDeepCopy.h>
+ #include <vtkm/worklet/WorkletMapTopology.h>
+
+ #include <vtkm/cont/Algorithm.h>
+ #include <vtkm/cont/ArrayCopy.h>
+ #include <vtkm/cont/ArrayHandle.h>
++#include <vtkm/cont/CellSetExplicit.h>
+ #include <vtkm/cont/CellSetPermutation.h>
+ #include <vtkm/cont/CoordinateSystem.h>
+ #include <vtkm/cont/Invoker.h>
+@@ -114,28 +116,13 @@ public:
+     }
+   };
+
+-  ////////////////////////////////////////////////////////////////////////////////////
+-  // Extract cells by ids permutes input data
+-  template <typename CellSetType>
+-  vtkm::cont::CellSetPermutation<CellSetType> Run(const CellSetType& cellSet,
+-                                                  const vtkm::cont::ArrayHandle<vtkm::Id>& cellIds)
+-  {
+-    using OutputType = vtkm::cont::CellSetPermutation<CellSetType>;
+-
+-    vtkm::cont::ArrayCopy(cellIds, this->ValidCellIds);
+-
+-    return OutputType(this->ValidCellIds, cellSet);
+-  }
+-
+-  ////////////////////////////////////////////////////////////////////////////////////
+-  // Extract cells by implicit function permutes input data
+   template <typename CellSetType, typename ImplicitFunction>
+-  vtkm::cont::CellSetPermutation<CellSetType> Run(const CellSetType& cellSet,
+-                                                  const vtkm::cont::CoordinateSystem& coordinates,
+-                                                  const ImplicitFunction& implicitFunction,
+-                                                  bool extractInside,
+-                                                  bool extractBoundaryCells,
+-                                                  bool extractOnlyBoundaryCells)
++  vtkm::cont::CellSetExplicit<> Run(const CellSetType& cellSet,
++                                    const vtkm::cont::CoordinateSystem& coordinates,
++                                    const ImplicitFunction& implicitFunction,
++                                    bool extractInside,
++                                    bool extractBoundaryCells,
++                                    bool extractOnlyBoundaryCells)
+   {
+     // Worklet output will be a boolean passFlag array
+     vtkm::cont::ArrayHandle<bool> passFlags;
+@@ -149,7 +136,10 @@ public:
+     vtkm::cont::Algorithm::CopyIf(indices, passFlags, this->ValidCellIds);
+
+     // generate the cellset
+-    return vtkm::cont::CellSetPermutation<CellSetType>(this->ValidCellIds, cellSet);
++    vtkm::cont::CellSetPermutation<CellSetType> permutedCellSet(this->ValidCellIds, cellSet);
++
++    vtkm::cont::CellSetExplicit<> outputCells;
++    return vtkm::worklet::CellDeepCopy::Run(permutedCellSet);
+   }
+
+   vtkm::cont::ArrayHandle<vtkm::Id> GetValidCellIds() const { return this->ValidCellIds; }
+-- 
+GitLab
+

scripts/docker/ascent_build/2024_07_02_vtkm-mr3246-raysubset_bugfix.patch

@@ -0,0 +1,43 @@
+From 763f13306b719bf6a213d00ead13fc93433e942e Mon Sep 17 00:00:00 2001
+From: Cyrus Harrison <[email protected]>
+Date: Tue, 2 Jul 2024 10:28:43 -0700
+Subject: [PATCH] fix bug with ray subsetting using wrong near and far planes
+
+---
+ vtkm/rendering/raytracing/Camera.cxx | 12 ++++--------
+ 1 file changed, 4 insertions(+), 8 deletions(-)
+
+diff --git a/vtkm/rendering/raytracing/Camera.cxx b/vtkm/rendering/raytracing/Camera.cxx
+index f2a39bef9..10febf39f 100644
+--- a/vtkm/rendering/raytracing/Camera.cxx
++++ b/vtkm/rendering/raytracing/Camera.cxx
+@@ -830,6 +830,7 @@ void Camera::FindSubset(const vtkm::Bounds& bounds)
+         transformed[2] = (transformed[2] * 0.5f + 0.5f);
+         zmin = vtkm::Min(zmin, transformed[2]);
+         zmax = vtkm::Max(zmax, transformed[2]);
++        // skip if outside near and far clipping
+         if (transformed[2] < 0 || transformed[2] > 1)
+         {
+           continue;
+@@ -894,15 +895,10 @@ VTKM_CONT void Camera::UpdateDimensions(Ray<Precision>& rays,
+     this->CameraView.SetLookAt(this->GetLookAt());
+     this->CameraView.SetPosition(this->GetPosition());
+     this->CameraView.SetViewUp(this->GetUp());
+-    //
+-    // Just create come clipping range, we ignore the zmax value in subsetting
+-    //
+-    vtkm::Float64 maxDim = vtkm::Max(
+-      boundingBox.X.Max - boundingBox.X.Min,
+-      vtkm::Max(boundingBox.Y.Max - boundingBox.Y.Min, boundingBox.Z.Max - boundingBox.Z.Min));
+
+-    maxDim *= 100;
+-    this->CameraView.SetClippingRange(.0001, maxDim);
++    // Note:
++    // Use clipping range provided, the subsetting does take into consideration
++    // the near and far clipping planes.
+
+     //Update our ViewProjection matrix
+     this->ViewProjectionMat =
+-- 
+2.39.3 (Apple Git-145)
+