diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H index fa00f6288f9..64080b16625 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H @@ -27,12 +27,12 @@ struct SelectParticles * \brief Constructor of SelectParticles functor. * * @param[in] a_pti WarpX particle iterator - * @param[in] tmp_particle_data temporary particle data * @param[in] current_z_boost current z-position of the slice in boosted frame * @param[in] old_z_boost previous z-position of the slice in boosted frame * @param[in] a_offset index offset for particles to be selected */ - SelectParticles( const WarpXParIter& a_pti, TmpParticles& tmp_particle_data, + SelectParticles (const WarpXParticleContainer& pc, + WarpXParIter& a_pti, amrex::Real current_z_boost, amrex::Real old_z_boost, int a_offset = 0); @@ -83,15 +83,15 @@ struct LorentzTransformParticles * \brief Constructor of the LorentzTransformParticles functor. * * @param[in] a_pti WarpX particle iterator - * @param[in] tmp_particle_data temporary particle data * @param[in] t_boost time in boosted frame * @param[in] dt timestep in boosted-frame * @param[in] t_lab time in lab-frame * @param[in] a_offset index offset for particles to be transformed */ - LorentzTransformParticles ( const WarpXParIter& a_pti, TmpParticles& tmp_particle_data, - amrex::Real t_boost, amrex::Real dt, - amrex::Real t_lab, int a_offset = 0); + LorentzTransformParticles (const WarpXParticleContainer& pc, + WarpXParIter& a_pti, + amrex::Real t_boost, amrex::Real dt, + amrex::Real t_lab, int a_offset = 0); /** * \brief Functor call. This method computes the Lorentz-transform for particle diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp index 9c2d10ad819..1e5d92411f7 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp @@ -15,29 +15,37 @@ #include #include -SelectParticles::SelectParticles (const WarpXParIter& a_pti, TmpParticles& tmp_particle_data, - amrex::Real current_z_boost, amrex::Real old_z_boost, - int a_offset) +SelectParticles::SelectParticles ( + const WarpXParticleContainer& pc, + WarpXParIter& a_pti, + amrex::Real current_z_boost, + amrex::Real old_z_boost, + int a_offset +) : m_current_z_boost(current_z_boost), m_old_z_boost(old_z_boost) { m_get_position = GetParticlePosition(a_pti, a_offset); - const auto lev = a_pti.GetLevel(); - const auto index = a_pti.GetPairIndex(); - - zpold = tmp_particle_data[lev][index][TmpIdx::zold].dataPtr(); + auto particle_comps = pc.getParticleComps(); + zpold = a_pti.GetAttribs(particle_comps["z_n_btd"]).dataPtr(); } -LorentzTransformParticles::LorentzTransformParticles ( const WarpXParIter& a_pti, - TmpParticles& tmp_particle_data, - amrex::Real t_boost, amrex::Real dt, - amrex::Real t_lab, int a_offset) +LorentzTransformParticles::LorentzTransformParticles ( + const WarpXParticleContainer& pc, + WarpXParIter& a_pti, + amrex::Real t_boost, + amrex::Real dt, + amrex::Real t_lab, + int a_offset +) : m_t_boost(t_boost), m_dt(dt), m_t_lab(t_lab) { using namespace amrex::literals; - if (tmp_particle_data.empty()) { return; } + // FIXME + //if (tmp_particle_data.empty()) { return; } + m_get_position = GetParticlePosition(a_pti, a_offset); const auto& attribs = a_pti.GetAttribs(); @@ -46,15 +54,13 @@ LorentzTransformParticles::LorentzTransformParticles ( const WarpXParIter& a_pti m_uypnew = attribs[PIdx::uy].dataPtr(); m_uzpnew = attribs[PIdx::uz].dataPtr(); - const auto lev = a_pti.GetLevel(); - const auto index = a_pti.GetPairIndex(); - - m_xpold = tmp_particle_data[lev][index][TmpIdx::xold].dataPtr(); - m_ypold = tmp_particle_data[lev][index][TmpIdx::yold].dataPtr(); - m_zpold = tmp_particle_data[lev][index][TmpIdx::zold].dataPtr(); - m_uxpold = tmp_particle_data[lev][index][TmpIdx::uxold].dataPtr(); - m_uypold = tmp_particle_data[lev][index][TmpIdx::uyold].dataPtr(); - m_uzpold = tmp_particle_data[lev][index][TmpIdx::uzold].dataPtr(); + auto particle_comps = pc.getParticleComps(); + m_xpold = a_pti.GetAttribs(particle_comps["x_n_btd"]).dataPtr(); + m_ypold = a_pti.GetAttribs(particle_comps["y_n_btd"]).dataPtr(); + m_zpold = a_pti.GetAttribs(particle_comps["z_n_btd"]).dataPtr(); + m_uxpold = a_pti.GetAttribs(particle_comps["ux_n_btd"]).dataPtr(); + m_uypold = a_pti.GetAttribs(particle_comps["uy_n_btd"]).dataPtr(); + m_uzpold = a_pti.GetAttribs(particle_comps["uz_n_btd"]).dataPtr(); m_betaboost = WarpX::beta_boost; m_gammaboost = WarpX::gamma_boost; @@ -84,7 +90,6 @@ BackTransformParticleFunctor::operator () (PinnedMemoryParticleContainer& pc_dst auto &warpx = WarpX::GetInstance(); // get particle slice const int nlevs = std::max(0, m_pc_src->finestLevel()+1); - auto tmp_particle_data = m_pc_src->getTmpParticleData(); for (int lev = 0; lev < nlevs; ++lev) { const amrex::Real t_boost = warpx.gett_new(0); const amrex::Real dt = warpx.getdt(0); @@ -107,13 +112,19 @@ BackTransformParticleFunctor::operator () (PinnedMemoryParticleContainer& pc_dst auto index = std::make_pair(pti.index(), pti.LocalTileIndex()); - const auto GetParticleFilter = SelectParticles(pti, tmp_particle_data, - m_current_z_boost[i_buffer], - m_old_z_boost[i_buffer]); + const auto GetParticleFilter = SelectParticles( + *m_pc_src, + pti, + m_current_z_boost[i_buffer], + m_old_z_boost[i_buffer] + ); const auto GetParticleLorentzTransform = LorentzTransformParticles( - pti, tmp_particle_data, - t_boost, dt, - m_t_lab[i_buffer]); + *m_pc_src, + pti, + t_boost, + dt, + m_t_lab[i_buffer] + ); long const np = pti.numParticles(); diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 5aca5ad89da..3c96de99fcb 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -539,6 +539,7 @@ WarpX::InitData () WarpX::PrintDtDxDyDz(); InitFromScratch(); InitDiagnostics(); + InitRuntimeComps(); } else { @@ -701,6 +702,31 @@ WarpX::InitFromScratch () } +void WarpX::InitRuntimeComps () +{ + if (mypc->getDoBackTransformedParticles()) + { + // Set comm to false so that the attributes are not communicated + // nor written to the checkpoint files + int const comm = 0; + + // Add space to save the positions and velocities at the start of the time steps + for (auto const& pc : *mypc) + { +#if (AMREX_SPACEDIM >= 2) + pc->NewRealComp("x_n_btd", comm); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + pc->NewRealComp("y_n_btd", comm); +#endif + pc->NewRealComp("z_n_btd", comm); + pc->NewRealComp("ux_n_btd", comm); + pc->NewRealComp("uy_n_btd", comm); + pc->NewRealComp("uz_n_btd", comm); + } + } +} + void WarpX::InitPML () { diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 0e33b6bac3c..64560dfd6e1 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -257,6 +257,13 @@ public: */ void SetDoBackTransformedParticles (const std::string& species_name, bool do_back_transformed_particles); + /** Get value of private member m_do_back_transformed_particles + */ + bool getDoBackTransformedParticles () const + { + return m_do_back_transformed_particles; + } + [[nodiscard]] int nSpeciesDepositOnMainGrid () const { bool const onMainGrid = true; diff --git a/Source/Particles/PhotonParticleContainer.cpp b/Source/Particles/PhotonParticleContainer.cpp index 47c426cd6ff..af5ca4278f0 100644 --- a/Source/Particles/PhotonParticleContainer.cpp +++ b/Source/Particles/PhotonParticleContainer.cpp @@ -126,7 +126,7 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, } #endif - auto copyAttribs = CopyParticleAttribs(pti, tmp_particle_data, offset); + auto copyAttribs = CopyParticleAttribs(*this, pti, offset); const int do_copy = (m_do_back_transformed_particles && (a_dt_type!=DtType::SecondHalf) ); const auto GetPosition = GetParticlePosition(pti, offset); diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index baac138dd38..be6e40b03e4 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1838,20 +1838,6 @@ PhysicalParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, amrex::MultiFab & By = *fields.get(FieldType::Bfield_aux, Direction{1}, lev); amrex::MultiFab & Bz = *fields.get(FieldType::Bfield_aux, Direction{2}, lev); - if (m_do_back_transformed_particles) - { - for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) - { - const auto np = pti.numParticles(); - const auto t_lev = pti.GetLevel(); - const auto index = pti.GetPairIndex(); - tmp_particle_data.resize(finestLevel()+1); - for (int i = 0; i < TmpIdx::nattribs; ++i) { - tmp_particle_data[t_lev][index][i].resize(np); - } - } - } - #ifdef AMREX_USE_OMP #pragma omp parallel #endif @@ -2623,7 +2609,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, const int do_copy = (m_do_back_transformed_particles && (a_dt_type!=DtType::SecondHalf) ); CopyParticleAttribs copyAttribs; if (do_copy) { - copyAttribs = CopyParticleAttribs(pti, tmp_particle_data, offset); + copyAttribs = CopyParticleAttribs(*this, pti, offset); } int* AMREX_RESTRICT ion_lev = nullptr; @@ -2880,7 +2866,7 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, const int do_copy = (m_do_back_transformed_particles && (a_dt_type!=DtType::SecondHalf) ); CopyParticleAttribs copyAttribs; if (do_copy) { - copyAttribs = CopyParticleAttribs(pti, tmp_particle_data, offset); + copyAttribs = CopyParticleAttribs(*this, pti, offset); } int* AMREX_RESTRICT ion_lev = nullptr; diff --git a/Source/Particles/Pusher/CopyParticleAttribs.H b/Source/Particles/Pusher/CopyParticleAttribs.H index 4f961fe94aa..ca625cb15d6 100644 --- a/Source/Particles/Pusher/CopyParticleAttribs.H +++ b/Source/Particles/Pusher/CopyParticleAttribs.H @@ -20,8 +20,6 @@ */ struct CopyParticleAttribs { - using TmpParticles = WarpXParticleContainer::TmpParticles; - GetParticlePosition m_get_position; const amrex::ParticleReal* AMREX_RESTRICT uxp = nullptr; @@ -41,15 +39,17 @@ struct CopyParticleAttribs /** \brief Construct a new functor * * \param a_pti iterator to the tile containing the macroparticles - * \param tmp_particle_data holder for the temporary particle data * \param a_offset offset to apply when reading / writing particle data * This is needed because when we use field gather buffers we don't * always start at the particle with index 0. */ - CopyParticleAttribs (const WarpXParIter& a_pti, TmpParticles& tmp_particle_data, - long a_offset = 0) noexcept + CopyParticleAttribs ( + const WarpXParticleContainer& pc, + WarpXParIter& a_pti, + long a_offset = 0) noexcept { - if (tmp_particle_data.empty()) { return; } + // FIXME + //if (tmp_particle_data.empty()) { return; } const auto& attribs = a_pti.GetAttribs(); @@ -57,14 +57,13 @@ struct CopyParticleAttribs uyp = attribs[PIdx::uy].dataPtr() + a_offset; uzp = attribs[PIdx::uz].dataPtr() + a_offset; - const auto lev = a_pti.GetLevel(); - const auto index = a_pti.GetPairIndex(); - xpold = tmp_particle_data[lev].at(index)[TmpIdx::xold ].dataPtr() + a_offset; - ypold = tmp_particle_data[lev].at(index)[TmpIdx::yold ].dataPtr() + a_offset; - zpold = tmp_particle_data[lev].at(index)[TmpIdx::zold ].dataPtr() + a_offset; - uxpold = tmp_particle_data[lev].at(index)[TmpIdx::uxold].dataPtr() + a_offset; - uypold = tmp_particle_data[lev].at(index)[TmpIdx::uyold].dataPtr() + a_offset; - uzpold = tmp_particle_data[lev].at(index)[TmpIdx::uzold].dataPtr() + a_offset; + auto particle_comps = pc.getParticleComps(); + xpold = a_pti.GetAttribs(particle_comps["x_n_btd"]).dataPtr(); + ypold = a_pti.GetAttribs(particle_comps["y_n_btd"]).dataPtr(); + zpold = a_pti.GetAttribs(particle_comps["z_n_btd"]).dataPtr(); + uxpold = a_pti.GetAttribs(particle_comps["ux_n_btd"]).dataPtr(); + uypold = a_pti.GetAttribs(particle_comps["uy_n_btd"]).dataPtr(); + uzpold = a_pti.GetAttribs(particle_comps["uz_n_btd"]).dataPtr(); m_get_position = GetParticlePosition(a_pti, a_offset); } diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 9c316b110ee..636a0707f47 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -496,13 +496,8 @@ public: TmpIdx::nattribs>; using TmpParticles = amrex::Vector >; - TmpParticles getTmpParticleData () const noexcept {return tmp_particle_data;} - int getIonizationInitialLevel () const noexcept {return ionization_initial_level;} -protected: - TmpParticles tmp_particle_data; - private: void particlePostLocate(ParticleType& p, const amrex::ParticleLocData& pld, int lev) override; diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 36793c8619b..17ce8eb05f4 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -1533,18 +1533,6 @@ void WarpXParticleContainer::defineAllParticleTiles () noexcept { // Call the parent class's method NamedComponentParticleContainer::defineAllParticleTiles(); - - // Resize the tmp_particle_data (no present in parent class) - tmp_particle_data.resize(finestLevel()+1); - for (int lev = 0; lev <= finestLevel(); ++lev) - { - for (auto mfi = MakeMFIter(lev); mfi.isValid(); ++mfi) - { - const int grid_id = mfi.index(); - const int tile_id = mfi.LocalTileIndex(); - tmp_particle_data[lev][std::make_pair(grid_id,tile_id)]; - } - } } // This function is called in Redistribute, just after locate diff --git a/Source/WarpX.H b/Source/WarpX.H index 2c949cb514a..0272e707d67 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1269,6 +1269,8 @@ private: void InitFromScratch (); + void InitRuntimeComps (); + void AllocLevelData (int lev, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm);