Merge branch 'develop' into lroberts36/refactor-mesh-constructors

.github/workflows/ci-macos.yml

@@ -24,23 +24,27 @@ jobs:
     - uses: actions/setup-python@v5
       with:
         python-version: '3.11'
-        cache: 'pip' # caching pip dependencies
+        cache: 'pip'
+        cache-dependency-path: '**/requirements.txt'
+    - run: pip install -r requirements.txt
+
+    - name: Install dependencies
+      run: |
+        brew install openmpi hdf5-mpi adios2 || true
 
     - name: Configure
-        run: |
-          cmake -B build \
-            -DCMAKE_BUILD_TYPE=Release \
+      run: cmake -B build -DCMAKE_BUILD_TYPE=Release
 
     - name: Build
-        run: cmake --build build
+      run: cmake --build build
 
     - name: Test
-        run: |
-          cd build
-          ctest -LE 'regression'
-
-     - uses: actions/upload-artifact@v3
-        with:
-          name: configure-log-unit-macos
-          path: build/CMakeFiles/CMakeOutput.log
-          retention-days: 3
+      run: |
+        cd build
+        ctest -LE mpi
+
+    - uses: actions/upload-artifact@v3
+      with:
+        name: configure-log-unit-macos
+        path: build/CMakeFiles/CMakeOutput.log
+        retention-days: 3

CHANGELOG.md

@@ -3,7 +3,9 @@
 ## Current develop
 
 ### Added (new features/APIs/variables/...)
-- [[PR 1084]](https://github.com/parthenon-hpc-lab/parthenon/pull/1084) Properly free swarm boundary MPI requests 
+- [[PR 1090]](https://github.com/parthenon-hpc-lab/parthenon/pull/1090) SMR with swarms
+- [[PR 1079]](https://github.com/parthenon-hpc-lab/parthenon/pull/1079) Address XDMF/Visit Issues
+- [[PR 1084]](https://github.com/parthenon-hpc-lab/parthenon/pull/1084) Properly free swarm boundary MPI requests
 - [[PR 1020]](https://github.com/parthenon-hpc-lab/parthenon/pull/1020) Add bi- and trilinear interpolation routines
 - [[PR 1081]](https://github.com/parthenon-hpc-lab/parthenon/pull/1081) Add GetSize and GetIndex to sparse pack
 - [[PR 1026]](https://github.com/parthenon-hpc-lab/parthenon/pull/1026) Particle BCs without relocatable device code
@@ -21,6 +23,11 @@
 - [[PR 1004]](https://github.com/parthenon-hpc-lab/parthenon/pull/1004) Allow parameter modification from an input file for restarts
 
 ### Fixed (not changing behavior/API/variables/...)
+- [[PR 1089]](https://github.com/parthenon-hpc-lab/parthenon/pull/1089) Fix loading restart files without derefinement counter
+- [[PR 1079]](https://github.com/parthenon-hpc-lab/parthenon/pull/1079) Address XDMF/Visit Issues
+- [[PR 1088]](https://github.com/parthenon-hpc-lab/parthenon/pull/1088) Correctly fill fluxes for non-cell variables in SparsePacks
+- [[PR 1083]](https://github.com/parthenon-hpc-lab/parthenon/pull/1083) Correctly fill VariableFluxPack for edge fluxes in 2D
+- [[PR 1087]](https://github.com/parthenon-hpc-lab/parthenon/pull/1087) Make sure InnerLoopPatternTVR is resolved on device properly when it is the default loop pattern
 - [[PR 1071]](https://github.com/parthenon-hpc-lab/parthenon/pull/1070) Fix bug in static mesh refinement related to redefinition of Mesh::root_level
 - [[PR 1073]](https://github.com/parthenon-hpc-lab/parthenon/pull/1073) Fix bug in AMR and sparse restarts
 - [[PR 1070]](https://github.com/parthenon-hpc-lab/parthenon/pull/1070) Correctly exclude flux vars from searches by default

example/particle_leapfrog/parthinput.particle_leapfrog

@@ -3,7 +3,7 @@
 #  Copyright(C) 2021 The Parthenon collaboration
 #  Licensed under the 3-clause BSD License, see LICENSE file for details
 # ========================================================================================
-#  (C) (or copyright) 2021. Triad National Security, LLC. All rights reserved.
+#  (C) (or copyright) 2021-2024. Triad National Security, LLC. All rights reserved.
 #
 #  This program was produced under U.S. Government contract 89233218CNA000001 for Los
 #  Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
@@ -19,7 +19,7 @@
 problem_id = particles
 
 <parthenon/mesh>
-refinement = none
+refinement = static
 
 nx1 = 16
 x1min = -0.5
@@ -44,6 +44,15 @@ nx1 = 8
 nx2 = 8
 nx3 = 8
 
+<parthenon/static_refinement1>
+level = 1
+x1min = -0.5
+x1max = 0
+x2min = -0.5
+x2max = 0
+x3min = -0.5
+x3max = 0
+
 <parthenon/time>
 tlim = 1.0
 nlim = 100000

requirements.txt

@@ -0,0 +1,3 @@
+h5py
+numpy
+matplotlib

scripts/python/packages/parthenon_tools/parthenon_tools/phdf_diff.py

@@ -285,7 +285,8 @@ def compare_metadata(f0, f1, quiet=False, one=False, check_input=False, tol=1.0e
             for key in f0.fid[var].keys():
                 if var == "Blocks" and key == "loc.level-gid-lid-cnghost-gflag":
                     continue  # depends on number of MPI ranks and distribution of blocks among ranks
-
+                if var == "Blocks" and key == "derefinement_count":
+                    continue  # Gold files have not been updated to include derefinement_count
                 # Compare raw data of these variables
                 val0 = f0.fid[var][key]
                 val1 = f1.fid[var][key]

src/CMakeLists.txt

@@ -78,7 +78,7 @@ else()
 endif()
 
 if (${PAR_LOOP_INNER_LAYOUT} STREQUAL "TVR_INNER_LOOP")
-  set(PAR_LOOP_INNER_LAYOUT_TAG inner_loop_pattern_tvr_tag)
+  set(PAR_LOOP_INNER_LAYOUT_TAG "InnerLoopPatternTVR()")
 elseif (${PAR_LOOP_INNER_LAYOUT} STREQUAL "SIMDFOR_INNER_LOOP")
   set(PAR_LOOP_INNER_LAYOUT_TAG inner_loop_pattern_simdfor_tag)
 else()

src/interface/sparse_pack.hpp

@@ -337,22 +337,22 @@ class SparsePack : public SparsePackBase {
   auto &flux(const int b, const int dir, const int idx) const {
     PARTHENON_DEBUG_REQUIRE(!flat_, "Accessor cannot be used for flat packs");
     PARTHENON_DEBUG_REQUIRE(dir > 0 && dir < 4 && with_fluxes_, "Bad input to flux call");
-    return pack_(dir, b, idx);
+    return pack_(dir - 1 + flx_idx_, b, idx);
   }
 
   KOKKOS_INLINE_FUNCTION
   Real &flux(const int b, const int dir, const int idx, const int k, const int j,
              const int i) const {
     PARTHENON_DEBUG_REQUIRE(!flat_, "Accessor cannot be used for flat packs");
     PARTHENON_DEBUG_REQUIRE(dir > 0 && dir < 4 && with_fluxes_, "Bad input to flux call");
-    return pack_(dir, b, idx)(k, j, i);
+    return pack_(dir - 1 + flx_idx_, b, idx)(k, j, i);
   }
 
   KOKKOS_INLINE_FUNCTION
   Real &flux(const int dir, const int idx, const int k, const int j, const int i) const {
     PARTHENON_DEBUG_REQUIRE(flat_, "Accessor must only be used for flat packs");
     PARTHENON_DEBUG_REQUIRE(dir > 0 && dir < 4 && with_fluxes_, "Bad input to flux call");
-    return pack_(dir, 0, idx)(k, j, i);
+    return pack_(dir - 1 + flx_idx_, 0, idx)(k, j, i);
   }
 
   KOKKOS_INLINE_FUNCTION
@@ -362,7 +362,7 @@ class SparsePack : public SparsePackBase {
     PARTHENON_DEBUG_REQUIRE(!flat_, "Accessor cannot be used for flat packs");
     PARTHENON_DEBUG_REQUIRE(dir > 0 && dir < 4 && with_fluxes_, "Bad input to flux call");
     const int n = bounds_(0, b, idx.VariableIdx()) + idx.Offset();
-    return pack_(dir, b, n)(k, j, i);
+    return pack_(dir - 1 + flx_idx_, b, n)(k, j, i);
   }
 
   template <class TIn, REQUIRES(IncludesType<TIn, Ts...>::value)>
@@ -371,7 +371,7 @@ class SparsePack : public SparsePackBase {
     PARTHENON_DEBUG_REQUIRE(!flat_, "Accessor cannot be used for flat packs");
     PARTHENON_DEBUG_REQUIRE(dir > 0 && dir < 4 && with_fluxes_, "Bad input to flux call");
     const int vidx = GetLowerBound(b, t) + t.idx;
-    return pack_(dir, b, vidx)(k, j, i);
+    return pack_(dir - 1 + flx_idx_, b, vidx)(k, j, i);
   }
 
   template <class... VTs>

src/interface/sparse_pack_base.cpp

@@ -110,6 +110,7 @@ SparsePackBase SparsePackBase::Build(T *pmd, const PackDescriptor &desc,
   int max_size = 0;
   int nblocks = 0;
   bool contains_face_or_edge = false;
+  bool contains_face_with_fluxes = false;
   int size = 0; // local var used to compute size/block
   ForEachBlock(pmd, include_block, [&](int b, mbd_t *pmbd) {
     if (!desc.flat) {
@@ -122,8 +123,13 @@ SparsePackBase SparsePackBase::Build(T *pmd, const PackDescriptor &desc,
         if (uid_map.count(uid) > 0) {
           const auto pv = uid_map.at(uid);
           if (pv->IsAllocated()) {
-            if (pv->IsSet(Metadata::Face) || pv->IsSet(Metadata::Edge))
+            if (pv->IsSet(Metadata::Edge)) contains_face_or_edge = true;
+            if (pv->IsSet(Metadata::Face)) {
+              if (pv->IsSet(Metadata::WithFluxes) && desc.with_fluxes) {
+                contains_face_with_fluxes = true;
+              }
               contains_face_or_edge = true;
+            }
             int prod = pv->GetDim(6) * pv->GetDim(5) * pv->GetDim(4);
             size += prod;       // max size/block (or total size for flat)
             pack.size_ += prod; // total ragged size
@@ -138,7 +144,11 @@ SparsePackBase SparsePackBase::Build(T *pmd, const PackDescriptor &desc,
 
   // Allocate the views
   int leading_dim = 1;
-  if (desc.with_fluxes) {
+  pack.flx_idx_ = 1;
+  if (contains_face_with_fluxes) {
+    leading_dim += 5;
+    pack.flx_idx_ = 3;
+  } else if (desc.with_fluxes) {
     leading_dim += 3;
   } else if (contains_face_or_edge) {
     leading_dim += 2;
@@ -205,11 +215,11 @@ SparsePackBase SparsePackBase::Build(T *pmd, const PackDescriptor &desc,
 
                     if (pv->IsSet(Metadata::Face)) {
                       pack.pack_h_(0, b, idx).topological_element =
-                          TopologicalElement::E1;
+                          TopologicalElement::F1;
                       pack.pack_h_(1, b, idx).topological_element =
-                          TopologicalElement::E2;
+                          TopologicalElement::F2;
                       pack.pack_h_(2, b, idx).topological_element =
-                          TopologicalElement::E3;
+                          TopologicalElement::F3;
                     }
 
                   } else { // This is a cell, node, or a variable that doesn't have
@@ -221,13 +231,16 @@ SparsePackBase SparsePackBase::Build(T *pmd, const PackDescriptor &desc,
                     }
                     if (pv->IsSet(Metadata::Vector))
                       pack.pack_h_(0, b, idx).vector_component = v + 1;
+                  }
 
-                    if (desc.with_fluxes && pv->IsSet(Metadata::WithFluxes)) {
-                      pack.pack_h_(1, b, idx) = pvf->data.Get(0, t, u, v);
-                      pack.pack_h_(2, b, idx) = pvf->data.Get(1, t, u, v);
-                      pack.pack_h_(3, b, idx) = pvf->data.Get(2, t, u, v);
+                  if (desc.with_fluxes && pv->IsSet(Metadata::WithFluxes)) {
+                    pack.pack_h_(0 + pack.flx_idx_, b, idx) = pvf->data.Get(0, t, u, v);
+                    if (!pv->IsSet(Metadata::Edge)) {
+                      pack.pack_h_(1 + pack.flx_idx_, b, idx) = pvf->data.Get(1, t, u, v);
+                      pack.pack_h_(2 + pack.flx_idx_, b, idx) = pvf->data.Get(2, t, u, v);
                     }
                   }
+
                   for (auto el :
                        GetTopologicalElements(pack.pack_h_(0, b, idx).topological_type)) {
                     pack.pack_h_(static_cast<int>(el) % 3, b, idx).topological_element =

src/interface/sparse_pack_base.hpp

@@ -92,6 +92,7 @@ class SparsePackBase {
   bounds_h_t bounds_h_;
   coords_t coords_;
 
+  int flx_idx_;
   bool with_fluxes_;
   bool coarse_;
   bool flat_;

src/interface/swarm.hpp

@@ -249,9 +249,7 @@ class Swarm {
   template <class T>
   vpack_types::SwarmVarList<T> MakeVarList_(const std::vector<std::string> &names);
 
-  void SetNeighborIndices1D_();
-  void SetNeighborIndices2D_();
-  void SetNeighborIndices3D_();
+  void SetNeighborIndices_();
 
   void CountReceivedParticles_();
   void UpdateNeighborBufferReceiveIndices_(ParArray1D<int> &neighbor_index,