Skip to content

Commit

Permalink
Merge branch 'development' into fix_ubar_nodal
Browse files Browse the repository at this point in the history
  • Loading branch information
@hklion
hklion committed Dec 9, 2023
2 parents 2d6e390 + 9a6287e commit 41e1c60
Show file tree
Hide file tree
Showing 24 changed files with 370 additions and 460 deletions.
329 changes: 125 additions & 204 deletions Source/BoundaryConditions/ROMSX_FillPatch.cpp
View file

Large diffs are not rendered by default.

7 changes: 3 additions & 4 deletions Source/BoundaryConditions/ROMSX_PhysBCFunct.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public:
ROMSXPhysBCFunct (const int lev,
const amrex::Geometry& geom, const amrex::Vector<amrex::BCRec>& domain_bcs_type,
const amrex::Gpu::DeviceVector<amrex::BCRec>& domain_bcs_type_d,
const int& var_idx, TimeInterpolatedData& data,
TimeInterpolatedData& data,
amrex::Array<amrex::Array<amrex::Real,AMREX_SPACEDIM*2>,AMREX_SPACEDIM+NVAR> bc_extdir_vals
#ifdef ROMSX_USE_NETCDF
,const std::string& init_type,
Expand All @@ -49,7 +49,7 @@ public:
)
: m_lev(lev), m_geom(geom), m_domain_bcs_type(domain_bcs_type),
m_domain_bcs_type_d(domain_bcs_type_d),
m_var_idx(var_idx), m_data(data),
m_data(data),
m_bc_extdir_vals(bc_extdir_vals)
#ifdef ROMSX_USE_NETCDF
,m_init_type(init_type),
Expand All @@ -67,7 +67,7 @@ public:
// mf is the multifab to be filled
// icomp is the index into the MultiFab -- if cell-centered this can be any value
// from 0 to NVAR-1, if face-centered this must be 0
// ncomp is the number of components -- if cell-centered (var_idx = 0) this can be any value
// ncomp is the number of components -- if cell-centered this can be any value
// from 1 to NVAR as long as icomp+ncomp <= NVAR-1. If face-centered this
// must be 1
// nghost is how many ghost cells to be filled
Expand Down Expand Up @@ -117,7 +117,6 @@ private:
amrex::Geometry m_geom;
amrex::Vector<amrex::BCRec> m_domain_bcs_type;
amrex::Gpu::DeviceVector<amrex::BCRec> m_domain_bcs_type_d;
int m_var_idx;
TimeInterpolatedData& m_data;
amrex::Array<amrex::Array<amrex::Real, AMREX_SPACEDIM*2>,AMREX_SPACEDIM+NVAR> m_bc_extdir_vals;
#ifdef ROMSX_USE_NETCDF
Expand Down
40 changes: 13 additions & 27 deletions Source/BoundaryConditions/ROMSX_PhysBCFunct.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ using namespace amrex;
// mf is the multifab to be filled
// icomp is the index into the MultiFab -- if cell-centered this can be any value
// from 0 to NVAR-1, if face-centered this must be 0
// ncomp is the number of components -- if cell-centered (var_idx = 0) this can be any value
// ncomp is the number of components -- if cell-centered this can be any value
// from 1 to NVAR as long as icomp+ncomp <= NVAR-1. If face-centered this
// must be 1
// nghost is how many ghost cells to be filled
Expand All @@ -17,7 +17,7 @@ using namespace amrex;
// so this follows the BCVars enum
//
void ROMSXPhysBCFunct::operator() (MultiFab& mf, int icomp, int ncomp, IntVect const& nghost,
Real time, int bccomp)
Real time, int bccomp)
{
if (m_geom.isAllPeriodic()) return;

Expand Down Expand Up @@ -53,30 +53,34 @@ void ROMSXPhysBCFunct::operator() (MultiFab& mf, int icomp, int ncomp, IntVect c
//! if there are cells not in the valid + periodic grown box
//! we need to fill them here
//!
if (!gdomain.contains(bx) || (m_var_idx == Vars::zvel))
if (!gdomain.contains(bx) || (mf[0].box().ixType() == IndexType(IntVect(0,0,1))) )
{
if (m_var_idx == Vars::xvel) {
if (mf[0].box().ixType() == IndexType(IntVect(1,0,0)))
{
AMREX_ALWAYS_ASSERT(ncomp == 1 && icomp == 0);
impose_xvel_bcs(dest_arr,bx,domain,
dxInv,time,bccomp);

} else if (m_var_idx == Vars::yvel) {
} else if (mf[0].box().ixType() == IndexType(IntVect(0,1,0)))
{
AMREX_ALWAYS_ASSERT(ncomp == 1 && icomp == 0);
impose_yvel_bcs(dest_arr,bx,domain,
dxInv,time,bccomp);

} else if (m_var_idx == Vars::zvel) {
} else if (mf[0].box().ixType() == IndexType(IntVect(0,0,1)))
{
AMREX_ALWAYS_ASSERT(ncomp == 1 && icomp == 0);
impose_zvel_bcs(dest_arr,bx,domain,
velx_arr,vely_arr,dx,dxInv,
time,bccomp);

} else if (m_var_idx == Vars::cons) {
} else if (mf[0].box().ixType() == IndexType(IntVect(0,0,0)))
{
AMREX_ALWAYS_ASSERT(icomp == 0 && icomp+ncomp <= NVAR);
impose_cons_bcs(dest_arr,bx,domain,
dxInv,icomp,ncomp,time,bccomp);
} else {
amrex::Abort("Dont know this var_idx in ROMSX_PhysBC");
amrex::Abort("Dont know this box type in ROMSX_PhysBC");
}

// ****************************************************************************
Expand All @@ -102,26 +106,8 @@ void ROMSXPhysBCFunct::operator() (MultiFab& mf, int icomp, int ncomp, IntVect c
(bcrs_d.data(), bcrs.data(), sizeof(BCRec)*ncomp);
#endif

#ifdef ROMSX_USE_NETCDF
const amrex::BCRec* bc_ptr = bcrs_d.data();
if (m_init_type == "real") {
int icomp_for_wrfbdy, ncomp_for_wrfbdy, bccomp_for_wrfbdy;
if (m_var_idx == Vars::cons) {
icomp_for_wrfbdy = Temp_comp;
bccomp_for_wrfbdy = BCVars::Temp_bc_comp;
ncomp_for_wrfbdy = 1; // (Because we are currently only filling U, V, W, T)
} else {
icomp_for_wrfbdy = icomp;
bccomp_for_wrfbdy = bccomp;
ncomp_for_wrfbdy = 1; // (Because we are currently only filling U, V, W, T)
}
fill_from_wrfbdy(m_lev, bx, dest_arr, icomp_for_wrfbdy, bccomp_for_wrfbdy, ncomp_for_wrfbdy,
domain, bc_ptr,
time, m_bdy_time_interval);
}
#endif
Gpu::streamSynchronize(); // because of bcrs_d
} // !gdomain.contains(bx)
} // MFIter
} // OpenMP
} // operator()
} // operator()
16 changes: 8 additions & 8 deletions Source/IO/Checkpoint.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,19 +117,19 @@ ROMSX::WriteCheckpointFile () const
BoxArray ba2d(std::move(bl2d));

MultiFab cons(grids[lev],dmap[lev],Cons::NumVars,0);
MultiFab::Copy(cons,vars_new[lev][Vars::cons],0,0,NVAR,0);
MultiFab::Copy(cons,*cons_new[lev],0,0,NVAR,0);
VisMF::Write(cons, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Cell"));

MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
MultiFab::Copy(xvel,vars_new[lev][Vars::xvel],0,0,1,0);
MultiFab::Copy(xvel,*xvel_new[lev],0,0,1,0);
VisMF::Write(xvel, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "XFace"));

MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
MultiFab::Copy(yvel,vars_new[lev][Vars::yvel],0,0,1,0);
MultiFab::Copy(yvel,*yvel_new[lev],0,0,1,0);
VisMF::Write(yvel, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "YFace"));

MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
MultiFab::Copy(zvel,vars_new[lev][Vars::zvel],0,0,1,0);
MultiFab::Copy(zvel,*zvel_new[lev],0,0,1,0);
VisMF::Write(zvel, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "ZFace"));

MultiFab mf_ru(grids[lev],dmap[lev],2,NGROW);
Expand Down Expand Up @@ -292,19 +292,19 @@ ROMSX::ReadCheckpointFile ()

MultiFab cons(grids[lev],dmap[lev],Cons::NumVars,0);
VisMF::Read(cons, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
MultiFab::Copy(vars_new[lev][Vars::cons],cons,0,0,NVAR,0);
MultiFab::Copy(*cons_new[lev],cons,0,0,NVAR,0);

MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
VisMF::Read(xvel, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "XFace"));
MultiFab::Copy(vars_new[lev][Vars::xvel],xvel,0,0,1,0);
MultiFab::Copy(*xvel_new[lev],xvel,0,0,1,0);

MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
VisMF::Read(yvel, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "YFace"));
MultiFab::Copy(vars_new[lev][Vars::yvel],yvel,0,0,1,0);
MultiFab::Copy(*yvel_new[lev],yvel,0,0,1,0);

MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
VisMF::Read(zvel, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "ZFace"));
MultiFab::Copy(vars_new[lev][Vars::zvel],zvel,0,0,1,0);
MultiFab::Copy(*zvel_new[lev],zvel,0,0,1,0);

MultiFab mf_ru(grids[lev],dmap[lev],2,NGROW);
VisMF::Read(mf_ru, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "XRHS"));
Expand Down
44 changes: 20 additions & 24 deletions Source/IO/NCCheckpoint.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,19 +98,19 @@ ROMSX::WriteNCCheckpointFile () const
for (int lev = 0; lev <= finest_level; ++lev) {

MultiFab cons(grids[lev],dmap[lev],Cons::NumVars,0);
MultiFab::Copy(cons,vars_new[lev][Vars::cons],0,0,NVAR,0);
MultiFab::Copy(cons,cons_new[lev],0,0,NVAR,0);
WriteNCMultiFab(cons, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Cell"));

MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
MultiFab::Copy(xvel,vars_new[lev][Vars::xvel],0,0,1,0);
MultiFab::Copy(xvel,xvel_new[lev],0,0,1,0);
WriteNCMultiFab(xvel, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "XFace"));

MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
MultiFab::Copy(yvel,vars_new[lev][Vars::yvel],0,0,1,0);
MultiFab::Copy(yvel,yvel_new[lev],0,0,1,0);
WriteNCMultiFab(yvel, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "YFace"));

MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
MultiFab::Copy(zvel,vars_new[lev][Vars::zvel],0,0,1,0);
MultiFab::Copy(zvel,zvel_new[lev],0,0,1,0);
WriteNCMultiFab(zvel, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "ZFace"));
}
}
Expand Down Expand Up @@ -190,21 +190,17 @@ ROMSX::ReadNCCheckpointFile ()
// build MultiFab data
int ncomp = Cons::NumVars;

auto& lev_old = vars_old[lev];
auto& lev_new = vars_new[lev];
cons_new[lev].define(grids[lev], dmap[lev], ncomp, ngrow_state);
cons_old[lev].define(grids[lev], dmap[lev], ncomp, ngrow_state);

lev_new[Vars::cons].define(grids[lev], dmap[lev], ncomp, ngrow_state);
lev_old[Vars::cons].define(grids[lev], dmap[lev], ncomp, ngrow_state);
xvel_new[levl].define(convert(grids[lev], IntVect(1,0,0)), dmap[lev], 1, ngrow_vels);
xvel_old[levl].define(convert(grids[lev], IntVect(1,0,0)), dmap[lev], 1, ngrow_vels);

//!don: get the ghost cells right here
lev_new[Vars::xvel].define(convert(grids[lev], IntVect(1,0,0)), dmap[lev], 1, ngrow_vels);
lev_old[Vars::xvel].define(convert(grids[lev], IntVect(1,0,0)), dmap[lev], 1, ngrow_vels);
yvel_new[levl].define(convert(grids[lev], IntVect(0,1,0)), dmap[lev], 1, ngrow_vels);
yvel_old[levl].define(convert(grids[lev], IntVect(0,1,0)), dmap[lev], 1, ngrow_vels);

lev_new[Vars::yvel].define(convert(grids[lev], IntVect(0,1,0)), dmap[lev], 1, ngrow_vels);
lev_old[Vars::yvel].define(convert(grids[lev], IntVect(0,1,0)), dmap[lev], 1, ngrow_vels);

lev_new[Vars::zvel].define(convert(grids[lev], IntVect(0,0,1)), dmap[lev], 1, ngrow_vels);
lev_old[Vars::zvel].define(convert(grids[lev], IntVect(0,0,1)), dmap[lev], 1, ngrow_vels);
zvel_new[levl].define(convert(grids[lev], IntVect(0,0,1)), dmap[lev], 1, ngrow_vels);
zvel_old[levl].define(convert(grids[lev], IntVect(0,0,1)), dmap[lev], 1, ngrow_vels);
}

// read in the MultiFab data
Expand All @@ -213,24 +209,24 @@ ROMSX::ReadNCCheckpointFile ()

MultiFab cons(grids[lev],dmap[lev],Cons::NumVars,0);
WriteNCMultiFab(cons, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
MultiFab::Copy(vars_new[lev][Vars::cons],cons,0,0,Cons::NumVars,0);
MultiFab::Copy(cons_new[lev],cons,0,0,Cons::NumVars,0);

MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
WriteNCMultiFab(xvel, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
MultiFab::Copy(vars_new[lev][Vars::xvel],xvel,0,0,1,0);
MultiFab::Copy(xvel_new[lev],xvel,0,0,1,0);

MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
WriteNCMultiFab(yvel, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
MultiFab::Copy(vars_new[lev][Vars::yvel],yvel,0,0,1,0);
MultiFab::Copy(yvel_new[lev],yvel,0,0,1,0);

MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
WriteNCMultiFab(zvel, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
MultiFab::Copy(vars_new[lev][Vars::zvel],zvel,0,0,1,0);
MultiFab::Copy(zvel_new[lev],zvel,0,0,1,0);

// Copy from new into old just in case
MultiFab::Copy(vars_old[lev][Vars::cons],vars_new[lev][Vars::cons],0,0,NVAR,0);
MultiFab::Copy(vars_old[lev][Vars::xvel],vars_new[lev][Vars::xvel],0,0,1,0);
MultiFab::Copy(vars_old[lev][Vars::yvel],vars_new[lev][Vars::yvel],0,0,1,0);
MultiFab::Copy(vars_old[lev][Vars::zvel],vars_new[lev][Vars::zvel],0,0,1,0);
MultiFab::Copy(cons_old[lev],cons_new[lev],0,0,NVAR,0);
MultiFab::Copy(xvel_old[lev],xvel_new[lev],0,0,1,0);
MultiFab::Copy(yvel_old[lev],yvel_new[lev],0,0,1,0);
MultiFab::Copy(zvel_old[lev],zvel_new[lev],0,0,1,0);
}
}
13 changes: 8 additions & 5 deletions Source/IO/NCColumnFile.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,11 +103,14 @@ ROMSX::writeToNCColumnFile(const int lev,
IntVect{iloc+1, jloc+1, kend});

// Need data in one grow cell for interpolation
FillPatch(lev, t_new[lev], vars_new[lev]);

MultiFab& S_new = vars_new[lev][Vars::cons];
MultiFab& U_new = vars_new[lev][Vars::xvel];
MultiFab& V_new = vars_new[lev][Vars::yvel];
FillPatch(lev, t_new[lev], cons_new[lev]);
FillPatch(lev, t_new[lev], xvel_new[lev]);
FillPatch(lev, t_new[lev], yvel_new[lev]);
FillPatch(lev, t_new[lev], zvel_new[lev]);

MultiFab& S_new = cons_new[lev];
MultiFab& U_new = xvel_new[lev];
MultiFab& V_new = yvel_new[lev];

// No tiling - we're just interested in one location
for ( MFIter mfi(S_new); mfi.isValid(); ++mfi){
Expand Down
17 changes: 10 additions & 7 deletions Source/IO/Plotfile.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,10 @@ ROMSX::WritePlotFile (int which, Vector<std::string> plot_var_names)
// We fillpatch here because some of the derived quantities require derivatives
// which require ghost cells to be filled
for (int lev = 0; lev <= finest_level; ++lev) {
FillPatch(lev, t_new[lev], vars_new[lev]);
FillPatch(lev, t_new[lev], cons_new[lev], cons_new);
FillPatch(lev, t_new[lev], xvel_new[lev], xvel_new);
FillPatch(lev, t_new[lev], yvel_new[lev], yvel_new);
FillPatch(lev, t_new[lev], zvel_new[lev], zvel_new);
}

if (ncomp_mf == 0)
Expand All @@ -115,7 +118,7 @@ ROMSX::WritePlotFile (int which, Vector<std::string> plot_var_names)
AMREX_ALWAYS_ASSERT(cons_names.size() == Cons::NumVars);
for (int i = 0; i < Cons::NumVars; ++i) {
if (containerHasElement(plot_var_names, cons_names[i])) {
MultiFab::Copy(mf[lev],vars_new[lev][Vars::cons],i,mf_comp,1,ngrow_vars);
MultiFab::Copy(mf[lev],*cons_new[lev],i,mf_comp,1,ngrow_vars);
mf_comp++;
}
}
Expand All @@ -126,7 +129,7 @@ ROMSX::WritePlotFile (int which, Vector<std::string> plot_var_names)
containerHasElement(plot_var_names, "z_velocity")) {
amrex::Print()<<"For now, print faces as if they are at cell centers"<<std::endl;
// average_face_to_cellcenter(mf[lev],mf_comp,
// Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
// Array<const MultiFab*,3>{&xvel_new[lev],&yvel_new[lev],&zvel_new[lev]});
//
// Convert the map-factor-scaled-velocities back to velocities
//
Expand All @@ -135,9 +138,9 @@ ROMSX::WritePlotFile (int which, Vector<std::string> plot_var_names)
{
const Box& bx = mfi.growntilebox();
const Array4<Real> vel_arr = dmf.array(mfi);
const Array4<const Real> velx_arr = vars_new[lev][Vars::xvel].array(mfi);
const Array4<const Real> vely_arr = vars_new[lev][Vars::yvel].array(mfi);
const Array4<const Real> velz_arr = vars_new[lev][Vars::zvel].array(mfi);
const Array4<const Real> velx_arr = xvel_new[lev]->const_array(mfi);
const Array4<const Real> vely_arr = yvel_new[lev]->const_array(mfi);
const Array4<const Real> velz_arr = zvel_new[lev]->const_array(mfi);
ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
{
vel_arr(i,j,k,0) = velx_arr(i,j,k);
Expand All @@ -160,7 +163,7 @@ ROMSX::WritePlotFile (int which, Vector<std::string> plot_var_names)
{
const Box& bx = mfi.tilebox();
auto& dfab = dmf[mfi];
auto& sfab = vars_new[lev][Vars::cons][mfi];
auto& sfab = (*cons_new[lev])[mfi];
der_function(bx, dfab, 0, 1, sfab, Geom(lev), t_new[0], nullptr, lev);
}

Expand Down
13 changes: 0 additions & 13 deletions Source/IndexDefines.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,19 +66,6 @@ namespace RHSVar {
};
}

namespace Vars {
enum {
cons = 0,
xvel,
yvel,
zvel,
NumTypes,
xmom,
ymom,
zmom
};
}

namespace Cons {
enum {
Rho = 0,
Expand Down
Loading

0 comments on commit 41e1c60

Please sign in to comment.