Added subfiling option

.github/workflows/CI.yml

@@ -11,10 +11,30 @@ on:
 
 jobs:
   CI:
+    defaults:
+      run:
+        shell: bash
     strategy:
       matrix:
         distro: ['ubuntu:latest']
         backend: ["SERIAL", "OPENMP"]
+        output: ['HDF5']
+        include:
+          - distro: 'ubuntu:latest'
+            cxx: 'g++'
+            backend: 'SERIAL'
+            cmake_build_type: 'Debug'
+            output: 'SILO'
+          - distro: 'ubuntu:latest'
+            cxx: 'g++'
+            backend: 'SERIAL'
+            cmake_build_type: 'Debug'
+            output: 'NONE'
+          - distro: 'ubuntu:latest'
+            cxx: 'g++'
+            backend: 'SERIAL'
+            cmake_build_type: 'Debug'
+            output: 'BOTH'
     runs-on: ubuntu-20.04
     container:
       image: ghcr.io/ecp-copa/ci-containers/${{ matrix.distro }}
@@ -25,7 +45,7 @@ jobs:
         uses: actions/[email protected]
         with:
           repository: kokkos/kokkos
-          ref: 3.6.01
+          ref: 3.7.02
           path: kokkos
       - name: Build kokkos
         working-directory: kokkos
@@ -37,12 +57,26 @@ jobs:
         uses: actions/[email protected]
         with:
           repository: ECP-copa/Cabana
-          ref: master
+          ref: 0.6.1
           path: Cabana
       - name: Build Cabana
         working-directory: Cabana
         run: |
-          cmake -B build -DCMAKE_INSTALL_PREFIX=$HOME/Cabana -DCMAKE_PREFIX_PATH="$HOME/kokkos" -DCabana_REQUIRE_${{ matrix.backend }}=ON
+          if [[ ${{ matrix.output }} == 'HDF5' ]]; then
+            cabana_cmake_opts+=( -DCabana_REQUIRE_HDF5=ON -DCMAKE_DISABLE_FIND_PACKAGE_SILO=ON )
+          elif [[ ${{ matrix.output }} == 'SILO' ]]; then
+            cabana_cmake_opts+=( -DCabana_REQUIRE_SILO=ON -DCMAKE_DISABLE_FIND_PACKAGE_HDF5=ON )
+          elif [[ ${{ matrix.output }} == 'BOTH' ]]; then
+            cabana_cmake_opts+=( -DCabana_REQUIRE_SILO=ON -DCabana_REQUIRE_HDF5=ON )
+          else
+            cabana_cmake_opts+=( -DCMAKE_DISABLE_FIND_PACKAGE_SILO=ON -DCMAKE_DISABLE_FIND_PACKAGE_HDF5=ON )
+          fi
+          cmake -B build \
+            -DCMAKE_INSTALL_PREFIX=$HOME/Cabana \
+            -DCMAKE_PREFIX_PATH="$HOME/kokkos" \
+            -DCMAKE_CXX_FLAGS="-Wall -Wextra -pedantic -Werror" \
+            -DCabana_REQUIRE_${{ matrix.backend }}=ON \
+            ${cabana_cmake_opts[@]}
           cmake --build build --parallel 2
           cmake --install build
       - name: Checkout code

CMakeLists.txt

@@ -8,13 +8,10 @@ include(GNUInstallDirs)
 
 # find dependencies
 set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH})
-find_package(Cabana REQUIRED COMPONENTS Cabana::Cajita Cabana::cabanacore)
+find_package(Cabana 0.6.1 REQUIRED COMPONENTS Cabana::Grid Cabana::Core)
 if( NOT Cabana_ENABLE_MPI )
   message( FATAL_ERROR "Cabana must be compiled with MPI" )
 endif()
-if( NOT Cabana_ENABLE_CAJITA )
-  message( FATAL_ERROR "Cabana must be compiled with Cajita" )
-endif()
 
 # find Clang Format
 find_package( CLANG_FORMAT 14 )

examples/dam_break.cpp

@@ -3,7 +3,7 @@
 
 #include <Cabana_Core.hpp>
 
-#include <Cajita.hpp>
+#include <Cabana_Grid.hpp>
 
 #include <Kokkos_Core.hpp>
 
@@ -87,7 +87,7 @@ void damBreak( const double cell_size, const int ppc, const int halo_size,
     int comm_size;
     MPI_Comm_size( MPI_COMM_WORLD, &comm_size );
     std::array<int, 3> ranks_per_dim = { 1, comm_size, 1 };
-    Cajita::ManualBlockPartitioner<3> partitioner( ranks_per_dim );
+    Cabana::Grid::ManualBlockPartitioner<3> partitioner( ranks_per_dim );
 
     // Material properties.
     double bulk_modulus = 1.0e5;
@@ -118,7 +118,26 @@ void damBreak( const double cell_size, const int ppc, const int halo_size,
 //---------------------------------------------------------------------------//
 int main( int argc, char* argv[] )
 {
-    MPI_Init( &argc, &argv );
+
+    // enable the use of subfiling by setting enviroment H5FD_SUBFILING
+    const char* env_val = std::getenv( "H5FD_SUBFILING" );
+    if ( env_val != NULL )
+    {
+        int mpi_thread_required = MPI_THREAD_MULTIPLE;
+        int mpi_thread_provided = 0;
+
+        // HDF5 Subfiling VFD requires MPI_Init_thread with MPI_THREAD_MULTIPLE
+
+        MPI_Init_thread( &argc, &argv, mpi_thread_required,
+                         &mpi_thread_provided );
+        if ( mpi_thread_provided < mpi_thread_required )
+        {
+            printf( "MPI_THREAD_MULTIPLE not supported\n" );
+            MPI_Abort( MPI_COMM_WORLD, -1 );
+        }
+    }
+    else
+        MPI_Init( &argc, &argv );
 
     Kokkos::initialize( argc, argv );
 

examples/free_fall.cpp

@@ -3,7 +3,7 @@
 
 #include <Cabana_Core.hpp>
 
-#include <Cajita.hpp>
+#include <Cabana_Grid.hpp>
 
 #include <Kokkos_Core.hpp>
 
@@ -84,7 +84,7 @@ void freeFall( const double cell_size, const int ppc, const int halo_size,
     int comm_size;
     MPI_Comm_size( MPI_COMM_WORLD, &comm_size );
     std::array<int, 3> ranks_per_dim = { 1, comm_size, 1 };
-    Cajita::ManualBlockPartitioner<3> partitioner( ranks_per_dim );
+    Cabana::Grid::ManualBlockPartitioner<3> partitioner( ranks_per_dim );
 
     // Material properties.
     double bulk_modulus = 5.0e5;

src/CMakeLists.txt

@@ -20,8 +20,8 @@ install(FILES ${HEADERS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 add_library(exampm ${SOURCES})
 
 target_link_libraries(exampm
-  Cabana::cabanacore
-  Cabana::Cajita
+  Cabana::Core
+  Cabana::Grid
 )
 
 target_include_directories(exampm

src/ExaMPM_Mesh.hpp

@@ -12,7 +12,7 @@
 #ifndef EXAMPM_MESH_HPP
 #define EXAMPM_MESH_HPP
 
-#include <Cajita.hpp>
+#include <Cabana_Grid.hpp>
 
 #include <Kokkos_Core.hpp>
 
@@ -39,7 +39,7 @@ class Mesh
     Mesh( const Kokkos::Array<double, 6>& global_bounding_box,
           const std::array<int, 3>& global_num_cell,
           const std::array<bool, 3>& periodic,
-          const Cajita::BlockPartitioner<3>& partitioner,
+          const Cabana::Grid::BlockPartitioner<3>& partitioner,
           const int halo_cell_width, const int minimum_halo_cell_width,
           MPI_Comm comm )
     {
@@ -91,20 +91,21 @@ class Mesh
         }
 
         // Create the global mesh.
-        auto global_mesh = Cajita::createUniformGlobalMesh(
+        auto global_mesh = Cabana::Grid::createUniformGlobalMesh(
             global_low_corner, global_high_corner, num_cell );
 
         // Build the global grid.
-        auto global_grid = Cajita::createGlobalGrid( comm, global_mesh,
-                                                     periodic, partitioner );
+        auto global_grid = Cabana::Grid::createGlobalGrid(
+            comm, global_mesh, periodic, partitioner );
 
         // Build the local grid.
         int halo_width = std::max( minimum_halo_cell_width, halo_cell_width );
-        _local_grid = Cajita::createLocalGrid( global_grid, halo_width );
+        _local_grid = Cabana::Grid::createLocalGrid( global_grid, halo_width );
     }
 
     // Get the local grid.
-    const std::shared_ptr<Cajita::LocalGrid<Cajita::UniformMesh<double>>>&
+    const std::shared_ptr<
+        Cabana::Grid::LocalGrid<Cabana::Grid::UniformMesh<double>>>&
     localGrid() const
     {
         return _local_grid;
@@ -129,7 +130,8 @@ class Mesh
     }
 
   public:
-    std::shared_ptr<Cajita::LocalGrid<Cajita::UniformMesh<double>>> _local_grid;
+    std::shared_ptr<Cabana::Grid::LocalGrid<Cabana::Grid::UniformMesh<double>>>
+        _local_grid;
 
     Kokkos::Array<int, 3> _min_domain_global_node_index;
     Kokkos::Array<int, 3> _max_domain_global_node_index;

src/ExaMPM_ParticleInit.hpp

@@ -15,7 +15,7 @@
 #include <ExaMPM_Types.hpp>
 
 #include <Cabana_Core.hpp>
-#include <Cajita.hpp>
+#include <Cabana_Grid.hpp>
 
 #include <Kokkos_Core.hpp>
 #include <Kokkos_Random.hpp>
@@ -114,11 +114,11 @@ void initializeParticles( const ExecSpace& exec_space,
     using particle_type = typename ParticleList::tuple_type;
 
     // Create a local mesh.
-    auto local_mesh = Cajita::createLocalMesh<device_type>( local_grid );
+    auto local_mesh = Cabana::Grid::createLocalMesh<device_type>( local_grid );
 
     // Get the local set of owned cell indices.
-    auto owned_cells =
-        local_grid.indexSpace( Cajita::Own(), Cajita::Cell(), Cajita::Local() );
+    auto owned_cells = local_grid.indexSpace(
+        Cabana::Grid::Own(), Cabana::Grid::Cell(), Cabana::Grid::Local() );
 
     // Allocate enough space for the case the particles consume the entire
     // local grid.
@@ -136,7 +136,7 @@ void initializeParticles( const ExecSpace& exec_space,
     int local_num_create = 0;
     Kokkos::parallel_reduce(
         "init_particles_uniform",
-        Cajita::createExecutionPolicy( owned_cells, exec_space ),
+        Cabana::Grid::createExecutionPolicy( owned_cells, exec_space ),
         KOKKOS_LAMBDA( const int i, const int j, const int k,
                        int& create_count ) {
             // Compute the owned local cell id.
@@ -150,12 +150,14 @@ void initializeParticles( const ExecSpace& exec_space,
             // Get the coordinates of the low cell node.
             int low_node[3] = { i, j, k };
             double low_coords[3];
-            local_mesh.coordinates( Cajita::Node(), low_node, low_coords );
+            local_mesh.coordinates( Cabana::Grid::Node(), low_node,
+                                    low_coords );
 
             // Get the coordinates of the high cell node.
             int high_node[3] = { i + 1, j + 1, k + 1 };
             double high_coords[3];
-            local_mesh.coordinates( Cajita::Node(), high_node, high_coords );
+            local_mesh.coordinates( Cabana::Grid::Node(), high_node,
+                                    high_coords );
 
             // Compute the particle spacing in each dimension.
             double spacing[3] = { ( high_coords[Dim::I] - low_coords[Dim::I] ) /

src/ExaMPM_ProblemManager.hpp

@@ -17,7 +17,7 @@
 
 #include <Cabana_Core.hpp>
 
-#include <Cajita.hpp>
+#include <Cabana_Grid.hpp>
 
 #include <memory>
 
@@ -91,13 +91,15 @@ class ProblemManager
     using particle_list = Cabana::AoSoA<particle_members, MemorySpace>;
     using particle_type = typename particle_list::tuple_type;
 
-    using node_array = Cajita::Array<double, Cajita::Node,
-                                     Cajita::UniformMesh<double>, MemorySpace>;
+    using node_array =
+        Cabana::Grid::Array<double, Cabana::Grid::Node,
+                            Cabana::Grid::UniformMesh<double>, MemorySpace>;
 
-    using cell_array = Cajita::Array<double, Cajita::Cell,
-                                     Cajita::UniformMesh<double>, MemorySpace>;
+    using cell_array =
+        Cabana::Grid::Array<double, Cabana::Grid::Cell,
+                            Cabana::Grid::UniformMesh<double>, MemorySpace>;
 
-    using halo = Cajita::Halo<MemorySpace>;
+    using halo = Cabana::Grid::Halo<MemorySpace>;
 
     using mesh_type = Mesh<MemorySpace>;
 
@@ -117,37 +119,38 @@ class ProblemManager
         initializeParticles( exec_space, *( _mesh->localGrid() ),
                              particles_per_cell, create_functor, _particles );
 
-        auto node_vector_layout =
-            Cajita::createArrayLayout( _mesh->localGrid(), 3, Cajita::Node() );
-        auto node_scalar_layout =
-            Cajita::createArrayLayout( _mesh->localGrid(), 1, Cajita::Node() );
-        auto cell_scalar_layout =
-            Cajita::createArrayLayout( _mesh->localGrid(), 1, Cajita::Cell() );
+        auto node_vector_layout = Cabana::Grid::createArrayLayout(
+            _mesh->localGrid(), 3, Cabana::Grid::Node() );
+        auto node_scalar_layout = Cabana::Grid::createArrayLayout(
+            _mesh->localGrid(), 1, Cabana::Grid::Node() );
+        auto cell_scalar_layout = Cabana::Grid::createArrayLayout(
+            _mesh->localGrid(), 1, Cabana::Grid::Cell() );
 
-        _momentum = Cajita::createArray<double, MemorySpace>(
+        _momentum = Cabana::Grid::createArray<double, MemorySpace>(
             "momentum", node_vector_layout );
-        _mass = Cajita::createArray<double, MemorySpace>( "mass",
-                                                          node_scalar_layout );
-        _force = Cajita::createArray<double, MemorySpace>( "force",
-                                                           node_vector_layout );
-        _velocity = Cajita::createArray<double, MemorySpace>(
+        _mass = Cabana::Grid::createArray<double, MemorySpace>(
+            "mass", node_scalar_layout );
+        _force = Cabana::Grid::createArray<double, MemorySpace>(
+            "force", node_vector_layout );
+        _velocity = Cabana::Grid::createArray<double, MemorySpace>(
             "velocity", node_vector_layout );
-        _position_correction = Cajita::createArray<double, MemorySpace>(
+        _position_correction = Cabana::Grid::createArray<double, MemorySpace>(
             "position_correction", node_vector_layout );
-        _density = Cajita::createArray<double, MemorySpace>(
+        _density = Cabana::Grid::createArray<double, MemorySpace>(
             "density", cell_scalar_layout );
 
-        _mark = Cajita::createArray<double, MemorySpace>( "mark",
-                                                          cell_scalar_layout );
-
-        _node_scatter_halo = Cajita::createHalo(
-            Cajita::NodeHaloPattern<3>(), -1, *_momentum, *_mass, *_force );
-        _node_gather_halo =
-            Cajita::createHalo( Cajita::NodeHaloPattern<3>(), -1, *_velocity );
-        _node_correction_halo = Cajita::createHalo(
-            Cajita::NodeHaloPattern<3>(), -1, *_position_correction );
-        _cell_halo = Cajita::createHalo( Cajita::NodeHaloPattern<3>(), -1,
-                                         *_density, *_mark );
+        _mark = Cabana::Grid::createArray<double, MemorySpace>(
+            "mark", cell_scalar_layout );
+
+        _node_scatter_halo =
+            Cabana::Grid::createHalo( Cabana::Grid::NodeHaloPattern<3>(), -1,
+                                      *_momentum, *_mass, *_force );
+        _node_gather_halo = Cabana::Grid::createHalo(
+            Cabana::Grid::NodeHaloPattern<3>(), -1, *_velocity );
+        _node_correction_halo = Cabana::Grid::createHalo(
+            Cabana::Grid::NodeHaloPattern<3>(), -1, *_position_correction );
+        _cell_halo = Cabana::Grid::createHalo(
+            Cabana::Grid::NodeHaloPattern<3>(), -1, *_density, *_mark );
     }
 
     std::size_t numParticle() const { return _particles.size(); }
@@ -237,21 +240,22 @@ class ProblemManager
     void scatter( Location::Node ) const
     {
         _node_scatter_halo->scatter( execution_space(),
-                                     Cajita::ScatterReduce::Sum(), *_momentum,
-                                     *_mass, *_force );
+                                     Cabana::Grid::ScatterReduce::Sum(),
+                                     *_momentum, *_mass, *_force );
     }
 
     void scatter( Location::Node, Field::PositionCorrection ) const
     {
         _node_correction_halo->scatter( execution_space(),
-                                        Cajita::ScatterReduce::Sum(),
+                                        Cabana::Grid::ScatterReduce::Sum(),
                                         *_position_correction );
     }
 
     void scatter( Location::Cell ) const
     {
-        _cell_halo->scatter( execution_space(), Cajita::ScatterReduce::Sum(),
-                             *_density, *_mark );
+        _cell_halo->scatter( execution_space(),
+                             Cabana::Grid::ScatterReduce::Sum(), *_density,
+                             *_mark );
     }
 
     void gather( Location::Node ) const
@@ -268,8 +272,8 @@ class ProblemManager
     void communicateParticles( const int minimum_halo_width )
     {
         auto positions = get( Location::Particle(), Field::Position() );
-        Cajita::particleGridMigrate( *( _mesh->localGrid() ), positions,
-                                     _particles, minimum_halo_width );
+        Cabana::Grid::particleGridMigrate( *( _mesh->localGrid() ), positions,
+                                           _particles, minimum_halo_width );
     }
 
   private: