Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use FFT for MAC projection cases #152

Merged
merged 2 commits into from
Oct 22, 2024
Merged

Use FFT for MAC projection cases #152

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9fd7cc8
Use FFT for MAC projection cases
WeiqunZhang Oct 21, 2024
0441d28
Fix warning
WeiqunZhang Oct 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ option( HYDRO_PROJECTIONS "Enable projections" YES)
option( HYDRO_EB "Enable Embedded-Boundary support" YES)
option( HYDRO_OMP "Enable OpenMP" NO )
option( HYDRO_MPI "Enable MPI" YES )
option( HYDRO_FFT "Enable FFT" NO )


set(HYDRO_GPU_BACKEND_VALUES NONE SYCL CUDA HIP)
Expand Down Expand Up @@ -71,6 +72,9 @@ if (NOT TARGET AMReX::amrex)
if (HYDRO_EB)
list(APPEND AMREX_REQUIRED_COMPONENTS EB)
endif ()
if (HYDRO_FFT)
list(APPEND AMREX_REQUIRED_COMPONENTS FFT)
endif ()
find_package(AMReX CONFIG REQUIRED ${AMREX_REQUIRED_COMPONENTS} )
endif ()

Expand Down
10 changes: 10 additions & 0 deletions Projections/hydro_MacProjector.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@
#include <AMReX_MLEBABecLap.H>
#endif

#ifdef AMREX_USE_FFT
#include <AMReX_FFT_Poisson.H>
#endif

#ifdef AMREX_USE_HYPRE
#include <AMReX_HypreMLABecLap.H>
#endif
Expand Down Expand Up @@ -212,6 +216,12 @@ private:
#ifdef AMREX_USE_HYPRE
std::unique_ptr<amrex::HypreMLABecLap> m_hypremlabeclap;
#endif

bool m_use_fft = false;
#ifdef AMREX_USE_FFT
std::unique_ptr<amrex::FFT::Poisson<amrex::MultiFab>> m_fft_poisson;
std::unique_ptr<amrex::FFT::PoissonHybrid<amrex::MultiFab>> m_fft_poisson_hybrid;
#endif
};

}
Expand Down
54 changes: 52 additions & 2 deletions Projections/hydro_MacProjector.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ MacProjector::MacProjector(
m_phi_loc(a_phi_loc),
m_divu_loc(a_divu_loc)
{
amrex::ignore_unused(m_divu_loc, m_beta_loc, m_phi_loc, m_umac_loc);
amrex::ignore_unused(m_divu_loc, m_beta_loc, m_phi_loc, m_umac_loc, m_use_fft);
}

MacProjector::MacProjector (const Vector<Array<MultiFab*,AMREX_SPACEDIM> >& a_umac,
Expand Down Expand Up @@ -259,6 +259,13 @@ MacProjector::setDomainBC (const Array<LinOpBCType,AMREX_SPACEDIM>& lobc,
#ifdef AMREX_USE_HYPRE
m_hypremlabeclap.reset();
#endif
#ifdef AMREX_USE_FFT
if (m_fft_poisson_hybrid) {
AMREX_ALWAYS_ASSERT_WITH_MESSAGE(lobc.back() == LinOpBCType::Neumann &&
hibc.back() == LinOpBCType::Neumann,
"FFT::PoissonHybrid supports Neumann BC in z-direction only");
}
#endif
}


Expand Down Expand Up @@ -387,6 +394,18 @@ MacProjector::project (Real reltol, Real atol)
m_mlmg->prepareForFluxes(amrex::GetVecOfConstPtrs(m_phi));
}
} else
#endif
#ifdef AMREX_USE_FFT
if (m_fft_poisson || m_fft_poisson_hybrid) {
m_mlmg->makeSolvable(0,0,m_rhs[0]);
if (m_fft_poisson) {
m_fft_poisson->solve(m_phi[0], m_rhs[0]);
} else {
m_fft_poisson_hybrid->solve(m_phi[0], m_rhs[0]);
}
// The solve below should not do any iterations.
m_mlmg->solve(amrex::GetVecOfPtrs(m_phi), amrex::GetVecOfConstPtrs(m_rhs), reltol, atol);
} else
#endif
{
m_mlmg->solve(amrex::GetVecOfPtrs(m_phi), amrex::GetVecOfConstPtrs(m_rhs), reltol, atol);
Expand Down Expand Up @@ -565,6 +584,27 @@ void MacProjector::initProjector (Vector<BoxArray> const& a_grids,
}
auto const& dm = a_dmap;

#ifdef AMREX_USE_FFT
{
ParmParse pp("mac_proj");
pp.query("use_fft", m_use_fft);
}
if (m_use_fft) {
bool fft_supported = (nlevs == 1) && (a_overset_mask.empty()) &&
m_geom[0].Domain().numPts() == ba[0].numPts();
#if (AMREX_SPACEDIM == 3)
fft_supported = fft_supported && (m_geom[0].isPeriodic(0) &&
m_geom[0].isPeriodic(1));
#else
fft_supported = fft_supported && m_geom[0].isAllPeriodic();
#endif
if (!fft_supported) {
m_use_fft = false;
amrex::Warning("MacProjector: FFT not support");
}
}
#endif

m_rhs.resize(nlevs);
m_phi.resize(nlevs);
m_fluxes.resize(nlevs);
Expand All @@ -582,7 +622,7 @@ void MacProjector::initProjector (Vector<BoxArray> const& a_grids,
}
}

if (m_use_mlhypre) { a_lpinfo.setMaxCoarseningLevel(0); }
if (m_use_mlhypre || m_use_fft) { a_lpinfo.setMaxCoarseningLevel(0); }

if (a_overset_mask.empty()) {
m_poisson = std::make_unique<MLPoisson>(m_geom, ba, dm, a_lpinfo);
Expand All @@ -608,6 +648,16 @@ void MacProjector::initProjector (Vector<BoxArray> const& a_grids,
}
#endif

#ifdef AMREX_USE_FFT
if (m_use_fft) {
if (m_geom[0].isAllPeriodic()) {
m_fft_poisson = std::make_unique<FFT::Poisson<MultiFab>>(m_geom[0]);
} else {
m_fft_poisson_hybrid = std::make_unique<FFT::PoissonHybrid<MultiFab>>(m_geom[0]);
}
}
#endif

setOptions();

m_needs_init = false;
Expand Down