Add pragma omp parallel

seahorce-scidac · Jan 17, 2025 · b4fc263 · b4fc263
1 parent 4b03704
commit b4fc263
Show file tree

Hide file tree

Showing 18 changed files with 144 additions and 0 deletions.
diff --git a/Source/BoundaryConditions/REMORA_FillPatch.cpp b/Source/BoundaryConditions/REMORA_FillPatch.cpp
@@ -138,6 +138,9 @@ REMORA::FillPatch (int lev, Real time, MultiFab& mf_to_fill, Vector<MultiFab*> c
              (mf_box.ixType() == IndexType(IntVect(0,1,0))) )
         {
             int khi = geom[lev].Domain().bigEnd(2);
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
             for (MFIter mfi(mf_to_fill); mfi.isValid(); ++mfi)
             {
                 Box gbx  = mfi.growntilebox(); // Note this is face-centered since vel is
@@ -415,6 +418,9 @@ REMORA::FillBdyCCVels (int lev, MultiFab& mf_cc_vel)
     // Impose periodicity first
     mf_cc_vel.FillBoundary(geom[lev].periodicity());
 
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for (MFIter mfi(mf_cc_vel, TilingIfNotGPU()); mfi.isValid(); ++mfi)
     {
         // Note that we don't fill corners here -- only the cells that share a face

diff --git a/Source/BoundaryConditions/REMORA_FillPatcher.cpp b/Source/BoundaryConditions/REMORA_FillPatcher.cpp
@@ -162,6 +162,9 @@ void REMORAFillPatcher::BuildMask (BoxArray const& fba,
     com_ba.complementIn(com_bl, fba_bnd);
 
     // Fill mask based upon the com_bl BoxList
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for (MFIter mfi(*m_cf_mask); mfi.isValid(); ++mfi) {
         const Box& vbx = mfi.validbox();
         const Array4<int>& mask_arr = m_cf_mask->array(mfi);
@@ -232,6 +235,9 @@ void REMORAFillPatcher::InterpFace (MultiFab& fine,
         }
     }
 
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for (MFIter mfi(fine); mfi.isValid(); ++mfi)
     {
         Box const& fbx = mfi.validbox();
@@ -335,6 +341,9 @@ void REMORAFillPatcher::InterpCell (MultiFab& fine,
     IndexType m_ixt = fine.boxArray().ixType();
     Box const& cdomain = amrex::convert(m_cgeom.Domain(), m_ixt);
 
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for (MFIter mfi(fine); mfi.isValid(); ++mfi) {
         Box const& fbx = mfi.validbox();
 

diff --git a/Source/IO/REMORA_NCPlotFile.cpp b/Source/IO/REMORA_NCPlotFile.cpp
@@ -503,6 +503,9 @@ void REMORA::WriteNCPlotFile_which(int lev, int which_subdomain, bool write_head
 
     mask_arrays_for_write(lev, (Real) fill_value);
 
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for (MFIter mfi(*cons_new[lev], false); mfi.isValid(); ++mfi) {
         auto bx = mfi.validbox();
         if (subdomain.contains(bx)) {
@@ -693,6 +696,9 @@ void REMORA::WriteNCPlotFile_which(int lev, int which_subdomain, bool write_head
     //requests.resize(0);
     //irq = 0;
     // Writing u (we loop over cons to get cell-centered box)
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for (MFIter mfi(*cons_new[lev], false); mfi.isValid(); ++mfi) {
         Box bx = mfi.validbox();
 
@@ -777,6 +783,9 @@ void REMORA::WriteNCPlotFile_which(int lev, int which_subdomain, bool write_head
     } // mfi
 
     // Writing v (we loop over cons to get cell-centered box)
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for (MFIter mfi(*cons_new[lev], false); mfi.isValid(); ++mfi) {
         Box bx = mfi.validbox();
 
@@ -864,6 +873,9 @@ void REMORA::WriteNCPlotFile_which(int lev, int which_subdomain, bool write_head
         } // in subdomain
     } // mfi
 
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for (MFIter mfi(*cons_new[lev], false); mfi.isValid(); ++mfi) {
         Box bx = mfi.validbox();
 

diff --git a/Source/IO/REMORA_Plotfile.cpp b/Source/IO/REMORA_Plotfile.cpp
@@ -274,6 +274,9 @@ REMORA::WritePlotFile ()
             containerHasElement(plot_var_names, "z_cc"))
         {
             MultiFab dmf(mf[lev], make_alias, mf_comp, AMREX_SPACEDIM);
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
             for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                 const Box& bx = mfi.tilebox();
                 const Array4<Real> loc_arr = dmf.array(mfi);
@@ -321,6 +324,9 @@ REMORA::WritePlotFile ()
         Real dz = Geom()[lev].CellSizeArray()[2];
         int N = Geom()[lev].Domain().size()[2];
 
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
         for (MFIter mfi(mf_nd[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
         {
             const Box& bx = mfi.tilebox();
@@ -607,6 +613,9 @@ REMORA::WriteGenericPlotfileHeaderWithBathymetry (std::ostream &HeaderFile,
 
 void
 REMORA::mask_arrays_for_write(int lev, Real fill_value) {
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for (MFIter mfi(*cons_new[lev],false); mfi.isValid(); ++mfi) {
         Box bx = mfi.tilebox();
         Box gbx1 = mfi.growntilebox(IntVect(NGROW-1,NGROW-1,0));

diff --git a/Source/Initialization/REMORA_init.cpp b/Source/Initialization/REMORA_init.cpp
@@ -78,6 +78,9 @@ REMORA::set_zeta_average (int lev)
 {
     std::unique_ptr<MultiFab>& mf_zeta = vec_zeta[lev];
     std::unique_ptr<MultiFab>& mf_Zt_avg1  = vec_Zt_avg1[lev];
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for ( MFIter mfi(*cons_new[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
     {
         int nstp = 0;
@@ -109,6 +112,9 @@ REMORA::set_2darrays (int lev)
     std::unique_ptr<MultiFab>& mf_Hz  = vec_Hz[lev];
     int nstp = 0;
 
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for ( MFIter mfi(*cons_new[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
     {
         Array4<Real> const& ubar = (mf_ubar)->array(mfi);
@@ -166,6 +172,9 @@ REMORA::init_gls_vmix (int lev, SolverChoice solver_choice)
 
     auto N = Geom(lev).Domain().size()[2]-1; // Number of vertical "levs" aka, NZ
 
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for (MFIter mfi(*vec_Akk[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
         Array4<Real> const& Akk = vec_Akk[lev]->array(mfi);
         Array4<Real> const& Akp = vec_Akp[lev]->array(mfi);

diff --git a/Source/Initialization/REMORA_init_from_netcdf.cpp b/Source/Initialization/REMORA_init_from_netcdf.cpp
@@ -100,6 +100,9 @@ REMORA::init_data_from_netcdf (int lev)
 #endif
     {
     // Don't tile this since we are operating on full FABs in this routine
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for ( MFIter mfi(*cons_new[lev], false); mfi.isValid(); ++mfi )
     {
         // Define fabs for holding the initial data
@@ -141,6 +144,9 @@ REMORA::init_zeta_from_netcdf (int lev)
 #endif
         {
         // Don't tile this since we are operating on full FABs in this routine
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
         for ( MFIter mfi(*cons_new[lev], false); mfi.isValid(); ++mfi )
         {
             FArrayBox &zeta_fab  = (*vec_zeta[lev])[mfi];
@@ -201,6 +207,9 @@ REMORA::init_bathymetry_from_netcdf (int lev)
 #endif
         {
         // Don't tile this since we are operating on full FABs in this routine
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
         for ( MFIter mfi(*cons_new[lev], false); mfi.isValid(); ++mfi )
         {
             FArrayBox &h_fab     = (*vec_hOfTheConfusingName[lev])[mfi];
@@ -273,6 +282,9 @@ REMORA::init_bathymetry_from_netcdf (int lev)
     vec_xp[lev]->FillBoundary(geom[lev].periodicity());
     vec_yp[lev]->FillBoundary(geom[lev].periodicity());
 
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for ( MFIter mfi(*vec_pm[lev]); mfi.isValid(); ++mfi )
     {
         Box bx   = mfi.tilebox();
@@ -344,6 +356,9 @@ REMORA::init_coriolis_from_netcdf (int lev)
 #endif
         {
         // Don't tile this since we are operating on full FABs in this routine
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
         for ( MFIter mfi(*cons_new[lev], false); mfi.isValid(); ++mfi )
         {
             FArrayBox &fcor_fab  = (*vec_fcor[lev])[mfi];
@@ -385,6 +400,9 @@ REMORA::init_masks_from_netcdf (int lev)
 #endif
         {
         // Don't tile this since we are operating on full FABs in this routine
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
         for ( MFIter mfi(*cons_new[lev], false); mfi.isValid(); ++mfi )
         {
             FArrayBox &mskr_fab  = (*vec_mskr[lev])[mfi];

diff --git a/Source/Initialization/REMORA_make_new_level.cpp b/Source/Initialization/REMORA_make_new_level.cpp
@@ -552,6 +552,9 @@ REMORA::set_pm_pn (int lev)
     vec_pm[lev]->setVal(dxi[0]); vec_pm[lev]->FillBoundary(geom[lev].periodicity());
     vec_pn[lev]->setVal(dxi[1]); vec_pn[lev]->FillBoundary(geom[lev].periodicity());
 
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for ( MFIter mfi(*vec_xr[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
     {
         Array4<Real> const& xr = vec_xr[lev]->array(mfi);
@@ -598,6 +601,9 @@ REMORA::set_zeta_to_Ztavg (int lev)
 {
     std::unique_ptr<MultiFab>& mf_zeta = vec_zeta[lev];
     std::unique_ptr<MultiFab>& mf_Zt_avg1  = vec_Zt_avg1[lev];
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for ( MFIter mfi(*vec_zeta[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
     {
         Array4<const Real> const& Zt_avg1 = (mf_Zt_avg1)->const_array(mfi);
@@ -616,6 +622,9 @@ REMORA::set_zeta_to_Ztavg (int lev)
 void
 REMORA::update_mskp (int lev)
 {
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for ( MFIter mfi(*vec_mskr[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
     {
         Array4<const Real> const& mskr = vec_mskr[lev]->const_array(mfi);

diff --git a/Source/Particles/REMORA_PC_Init.cpp b/Source/Particles/REMORA_PC_Init.cpp
@@ -81,6 +81,9 @@ void REMORAPC::initializeParticlesUniformDistributionInBox (const std::unique_pt
                              ParticleDistributionMap(lev),
                              1, 0 );
     num_particles.setVal(0);
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for(MFIter mfi = MakeMFIter(lev); mfi.isValid(); ++mfi) {
         const Box& tile_box  = mfi.tilebox();
         auto num_particles_arr = num_particles[mfi].array();
@@ -117,6 +120,9 @@ void REMORAPC::initializeParticlesUniformDistributionInBox (const std::unique_pt
                        1, 0 );
     offsets.setVal(0);
 
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for(MFIter mfi = MakeMFIter(lev); mfi.isValid(); ++mfi) {
         const Box& tile_box  = mfi.tilebox();
 

diff --git a/Source/REMORA_SumIQ.cpp b/Source/REMORA_SumIQ.cpp
@@ -27,6 +27,9 @@ REMORA::sum_integrated_quantities(Real time)
         MultiFab ones_mf(grids[lev], dmap[lev], 1, 0);
         ones_mf.setVal(1.0_rt);
 
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
         for (MFIter mfi(*cons_new[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
             const Box& bx = mfi.tilebox();
             const Array4<      Real> kineng_arr = kineng_mf.array(mfi);
@@ -120,6 +123,9 @@ REMORA::volWgtSumMF(int lev, const MultiFab& mf, int comp, bool local, bool fine
     }
 
     MultiFab volume(grids[lev], dmap[lev], 1, 0);
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for (MFIter mfi(*cons_new[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
         const Box& bx = mfi.tilebox();
         const Array4<      Real> vol_arr = volume.array(mfi);

diff --git a/Source/REMORA_Tagging.cpp b/Source/REMORA_Tagging.cpp
@@ -50,6 +50,9 @@ REMORA::ErrorEst (int levc, TagBoxArray& tags, Real time, int /*ngrow*/)
             // Impose bc's at domain boundaries at all levels
             FillBdyCCVels(levc, mf_cc_vel);
 
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
             for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();

diff --git a/Source/TimeIntegration/REMORA_advance_2d.cpp b/Source/TimeIntegration/REMORA_advance_2d.cpp
@@ -105,6 +105,9 @@ REMORA::advance_2d (int lev,
     MultiFab mf_DUon(convert(ba,IntVect(1,0,0)),dm,1,IntVect(NGROW,NGROW,0));
     MultiFab mf_DVom(convert(ba,IntVect(0,1,0)),dm,1,IntVect(NGROW,NGROW,0));
 
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for ( MFIter mfi(*mf_rhoS, TilingIfNotGPU()); mfi.isValid(); ++mfi )
     {
         Array4<Real      > const& ubar = mf_ubar->array(mfi);
@@ -170,6 +173,9 @@ REMORA::advance_2d (int lev,
     mf_DUon.FillBoundary(geom[lev].periodicity());
     mf_DVom.FillBoundary(geom[lev].periodicity());
 
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for ( MFIter mfi(*mf_rhoS, TilingIfNotGPU()); mfi.isValid(); ++mfi )
     {
         Array4<Real const> const& rhoS = mf_rhoS->const_array(mfi);

diff --git a/Source/TimeIntegration/REMORA_advance_3d.cpp b/Source/TimeIntegration/REMORA_advance_3d.cpp
@@ -50,6 +50,9 @@ REMORA::advance_3d (int lev, MultiFab& mf_cons,
     MultiFab mf_DC (ba,dm,1,IntVect(NGROW,NGROW,NGROW-1)); //2d missing j coordinate
     MultiFab mf_Hzk(ba,dm,1,IntVect(NGROW,NGROW,NGROW-1)); //2d missing j coordinate
 
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for ( MFIter mfi(mf_cons, TilingIfNotGPU()); mfi.isValid(); ++mfi )
     {
         Array4<Real      > const& u = mf_u.array(mfi);
@@ -163,6 +166,9 @@ REMORA::advance_3d (int lev, MultiFab& mf_cons,
         }
 #endif
 
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for ( MFIter mfi(mf_cons, TilingIfNotGPU()); mfi.isValid(); ++mfi )
     {
         Array4<Real      > const& u = mf_u.array(mfi);
@@ -229,6 +235,9 @@ REMORA::advance_3d (int lev, MultiFab& mf_cons,
 
     MultiFab mf_W(convert(ba,IntVect(0,0,1)),dm,1,IntVect(NGROW+1,NGROW+1,0));
     mf_W.setVal(0.0_rt);
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for ( MFIter mfi(mf_cons, TilingIfNotGPU()); mfi.isValid(); ++mfi )
     {
         Array4<Real> const& Huon = mf_Huon->array(mfi);
@@ -303,6 +312,9 @@ REMORA::advance_3d (int lev, MultiFab& mf_cons,
     }
     nnew = 0;
 
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for ( MFIter mfi(mf_cons, TilingIfNotGPU()); mfi.isValid(); ++mfi )
     {
         Array4<Real> const& Hz  = mf_Hz->array(mfi);
@@ -349,6 +361,9 @@ REMORA::advance_3d (int lev, MultiFab& mf_cons,
 
     FillPatch(lev, t_old[lev], mf_cons, cons_new, BCVars::cons_bc, BdyVars::t,0,true,false,0,0,dt_lev,*cons_old[lev]);
 
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for ( MFIter mfi(mf_cons, TilingIfNotGPU()); mfi.isValid(); ++mfi )
     {
         Array4<Real> const& AK = mf_AK.array(mfi);

diff --git a/Source/TimeIntegration/REMORA_advance_3d_ml.cpp b/Source/TimeIntegration/REMORA_advance_3d_ml.cpp
@@ -48,6 +48,9 @@ void REMORA::advance_3d_ml (int lev, Real dt_lev)
     FillPatch(lev, t_old[lev], *zvel_new[lev], zvel_new, BCVars::zvel_bc, BdyVars::null);
 
     // Apply land/sea mask to tracers
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for ( MFIter mfi(*cons_new[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
     {
         Array4<Real> const& cons = cons_new[lev]->array(mfi);

diff --git a/Source/TimeIntegration/REMORA_gls.cpp b/Source/TimeIntegration/REMORA_gls.cpp
@@ -9,6 +9,9 @@ REMORA::gls_prestep (int lev, MultiFab* mf_gls, MultiFab* mf_tke,
                      const int iic, const int ntfirst, const int N, const Real dt_lev)
 {
     // temps: grad, gradL, XF, FX, FXL, EF, FE, FEL
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for ( MFIter mfi(*mf_gls, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
         Array4<Real> const& gls = mf_gls->array(mfi);
         Array4<Real> const& tke = mf_tke->array(mfi);
@@ -384,6 +387,9 @@ REMORA::gls_corrector (int lev, MultiFab* mf_gls, MultiFab* mf_tke,
     bool is_periodic_in_y = geomdata.isPeriodic(1);
 
 
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for ( MFIter mfi(*mf_gls, TilingIfNotGPU()); mfi.isValid(); ++mfi )
     {
         Box   bx = mfi.tilebox();
@@ -434,6 +440,9 @@ REMORA::gls_corrector (int lev, MultiFab* mf_gls, MultiFab* mf_tke,
     (*physbcs[lev])(mf_shear2_cached,*mf_mskr,0,1,mf_shear2_cached.nGrowVect(),t_new[lev],BCVars::foextrap_bc);
     mf_CF.setVal(0.0_rt);
 
+#ifdef _OPENMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
     for ( MFIter mfi(*mf_gls, TilingIfNotGPU()); mfi.isValid(); ++mfi )
     {
         Box  bx = mfi.tilebox();