From 244dff83b10bb4e26238ee74dacc4e90ba9956b9 Mon Sep 17 00:00:00 2001 From: Andrea Iob Date: Tue, 10 Jan 2023 10:24:44 +0100 Subject: [PATCH 01/15] containers/piercedVector: properly release memory of holes' storage --- src/containers/piercedKernel.tpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/containers/piercedKernel.tpp b/src/containers/piercedKernel.tpp index 1556dba43f..388dfb244b 100644 --- a/src/containers/piercedKernel.tpp +++ b/src/containers/piercedKernel.tpp @@ -1911,12 +1911,11 @@ void PiercedKernel::holesClearPending(bool release) template void PiercedKernel::holesResize(std::size_t offset, std::size_t nRegulars, std::size_t nPendings, bool release) { + m_holes.resize(offset + nRegulars + MAX_PENDING_HOLES); if (release) { m_holes.shrink_to_fit(); } - m_holes.resize(offset + nRegulars + MAX_PENDING_HOLES); - m_holes_regular_begin = offset; m_holes_regular_end = m_holes_regular_begin + nRegulars; m_holes_pending_begin = m_holes_regular_end; From bef7efb8ebf30802b47e67edf40cf087927578eb Mon Sep 17 00:00:00 2001 From: Andrea Iob Date: Fri, 3 Feb 2023 09:11:32 +0100 Subject: [PATCH 02/15] PABLO: mark octants as "new for refinement" only if they are generated by a refinement --- src/PABLO/LocalTree.cpp | 5 +++++ src/PABLO/Octant.cpp | 2 -- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/PABLO/LocalTree.cpp b/src/PABLO/LocalTree.cpp index 760a2c6891..0d783dfeb8 100644 --- a/src/PABLO/LocalTree.cpp +++ b/src/PABLO/LocalTree.cpp @@ -524,6 +524,11 @@ namespace bitpit { uint32_t firstChildIdx = futureIdx - (nChildren - 1); fatherOctant.buildChildren(m_octants.data() + firstChildIdx); + // Set children information + for (int i = 0; i < nChildren; ++i) { + m_octants[firstChildIdx + i].m_info[Octant::INFO_NEW4REFINEMENT] = true; + } + // Update the mapping if(!mapidx.empty()){ for (uint8_t i=0; i Date: Sat, 7 Jan 2023 21:11:53 +0100 Subject: [PATCH 03/15] patchkernel: fix comments --- src/patchkernel/patch_kernel.cpp | 5 ++--- src/patchkernel/patch_kernel_parallel.cpp | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/patchkernel/patch_kernel.cpp b/src/patchkernel/patch_kernel.cpp index 29b2364fff..72eeea147e 100644 --- a/src/patchkernel/patch_kernel.cpp +++ b/src/patchkernel/patch_kernel.cpp @@ -1054,7 +1054,7 @@ void PatchKernel::resetInterfaces() \param release if it's true the memory hold by the interfaces will be released, otherwise the interfaces will be reset but their memory will - not be relased + not be released */ void PatchKernel::_resetInterfaces(bool release) { @@ -6440,7 +6440,7 @@ void PatchKernel::destroyInterfaces() // Destroy the interfaces _resetInterfaces(true); - // Clear list of cells with dirty adjacencies + // Clear list of cells with dirty interfaces unsetCellAlterationFlags(FLAG_INTERFACES_DIRTY); // Clear list of altered interfaces @@ -6515,7 +6515,6 @@ void PatchKernel::pruneStaleInterfaces() */ void PatchKernel::_updateInterfaces() { - // // Update interfaces // // Adjacencies and interfaces of a face are paired: the i-th face adjacency diff --git a/src/patchkernel/patch_kernel_parallel.cpp b/src/patchkernel/patch_kernel_parallel.cpp index 88d07bf9af..c442ca297b 100644 --- a/src/patchkernel/patch_kernel_parallel.cpp +++ b/src/patchkernel/patch_kernel_parallel.cpp @@ -2922,7 +2922,7 @@ std::vector PatchKernel::_partitioningAlter_sendCells(const std: // Update adaption info // if (trackPartitioning) { - // Update partition + // Track cells that have been sent // // The ids of the cells send will be stored accordingly to the send // order, this is the same order that will be used on the process From e4a11f74924f4c7bc568e2fba7a504327f2019d9 Mon Sep 17 00:00:00 2001 From: Andrea Iob Date: Sat, 7 Jan 2023 21:17:16 +0100 Subject: [PATCH 04/15] patchkernel: rename some variables --- src/patchkernel/patch_kernel.cpp | 38 ++++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/patchkernel/patch_kernel.cpp b/src/patchkernel/patch_kernel.cpp index 72eeea147e..fcf5bc6bcc 100644 --- a/src/patchkernel/patch_kernel.cpp +++ b/src/patchkernel/patch_kernel.cpp @@ -624,14 +624,14 @@ PatchKernel::~PatchKernel() */ std::vector PatchKernel::update(bool trackAdaption, bool squeezeStorage) { - std::vector updateInfo; + std::vector adaptionData; // Early return if the patch is not dirty // // If we need to squeeze the storage we need to perform the update also // if the patch is not dirty. if (!squeezeStorage && !isDirty(true)) { - return updateInfo; + return adaptionData; } // Finalize alterations @@ -640,10 +640,10 @@ std::vector PatchKernel::update(bool trackAdaption, bool squeeze // Adaption bool adaptionDirty = (getAdaptionStatus(true) == ADAPTION_DIRTY); if (adaptionDirty) { - mergeAdaptionInfo(adaption(trackAdaption, squeezeStorage), updateInfo); + mergeAdaptionInfo(adaption(trackAdaption, squeezeStorage), adaptionData); } - return updateInfo; + return adaptionData; } /*! @@ -678,17 +678,17 @@ void PatchKernel::simulateCellUpdate(const long id, adaption::Marker marker, std */ std::vector PatchKernel::adaption(bool trackAdaption, bool squeezeStorage) { - std::vector adaptionInfo; + std::vector adaptionData; // Early return if adaption cannot be performed AdaptionMode adaptionMode = getAdaptionMode(); if (adaptionMode == ADAPTION_DISABLED) { - return adaptionInfo; + return adaptionData; } AdaptionStatus adaptionStatus = getAdaptionStatus(true); if (adaptionStatus == ADAPTION_CLEAN) { - return adaptionInfo; + return adaptionData; } else if (adaptionStatus != ADAPTION_DIRTY) { throw std::runtime_error ("An adaption is already in progress."); } @@ -696,11 +696,11 @@ std::vector PatchKernel::adaption(bool trackAdaption, bool squee // Run adaption adaptionPrepare(false); - adaptionInfo = adaptionAlter(trackAdaption, squeezeStorage); + adaptionData = adaptionAlter(trackAdaption, squeezeStorage); adaptionCleanup(); - return adaptionInfo; + return adaptionData; } /*! @@ -718,28 +718,28 @@ std::vector PatchKernel::adaption(bool trackAdaption, bool squee */ std::vector PatchKernel::adaptionPrepare(bool trackAdaption) { - std::vector adaptionInfo; + std::vector adaptionData; // Early return if adaption cannot be performed AdaptionMode adaptionMode = getAdaptionMode(); if (adaptionMode == ADAPTION_DISABLED) { - return adaptionInfo; + return adaptionData; } AdaptionStatus adaptionStatus = getAdaptionStatus(true); if (adaptionStatus == ADAPTION_CLEAN) { - return adaptionInfo; + return adaptionData; } else if (adaptionStatus != ADAPTION_DIRTY) { throw std::runtime_error ("An adaption is already in progress."); } // Execute the adaption preparation - adaptionInfo = _adaptionPrepare(trackAdaption); + adaptionData = _adaptionPrepare(trackAdaption); // Update the status setAdaptionStatus(ADAPTION_PREPARED); - return adaptionInfo; + return adaptionData; } /*! @@ -759,23 +759,23 @@ std::vector PatchKernel::adaptionPrepare(bool trackAdaption) */ std::vector PatchKernel::adaptionAlter(bool trackAdaption, bool squeezeStorage) { - std::vector adaptionInfo; + std::vector adaptionData; // Early return if adaption cannot be performed AdaptionMode adaptionMode = getAdaptionMode(); if (adaptionMode == ADAPTION_DISABLED) { - return adaptionInfo; + return adaptionData; } AdaptionStatus adaptionStatus = getAdaptionStatus(); if (adaptionStatus == ADAPTION_CLEAN) { - return adaptionInfo; + return adaptionData; } else if (adaptionStatus != ADAPTION_PREPARED) { throw std::runtime_error ("The prepare function has not been called."); } // Adapt the patch - adaptionInfo = _adaptionAlter(trackAdaption); + adaptionData = _adaptionAlter(trackAdaption); // Finalize patch alterations finalizeAlterations(squeezeStorage); @@ -783,7 +783,7 @@ std::vector PatchKernel::adaptionAlter(bool trackAdaption, bool // Update the status setAdaptionStatus(ADAPTION_ALTERED); - return adaptionInfo; + return adaptionData; } /*! From 0755fa12c035c63d496659b9a8c2eb3095216141 Mon Sep 17 00:00:00 2001 From: Andrea Iob Date: Tue, 1 Aug 2023 12:45:51 +0200 Subject: [PATCH 05/15] patchkernel: update Doxygen documentation --- src/patchkernel/patch_kernel_parallel.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/patchkernel/patch_kernel_parallel.cpp b/src/patchkernel/patch_kernel_parallel.cpp index c442ca297b..5e80c4ece1 100644 --- a/src/patchkernel/patch_kernel_parallel.cpp +++ b/src/patchkernel/patch_kernel_parallel.cpp @@ -2191,7 +2191,7 @@ std::vector PatchKernel::_partitioningAlter(bool trackPartitioni } /*! - Get the ghost that will change ownership after partitioning. + Get the ghost cells that will change ownership after partitioning. Some of the cells that are send during a partitioning may be ghosts on other processes. We need to find out the final ghost owners after the From 6110c70a23a37e71ce31310821e85cad9f011240 Mon Sep 17 00:00:00 2001 From: Andrea Iob Date: Sat, 7 Jan 2023 21:09:06 +0100 Subject: [PATCH 06/15] patchkernel: no need to prune the interfaces before resetting them --- src/patchkernel/patch_kernel.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/patchkernel/patch_kernel.cpp b/src/patchkernel/patch_kernel.cpp index fcf5bc6bcc..f0b46ee07f 100644 --- a/src/patchkernel/patch_kernel.cpp +++ b/src/patchkernel/patch_kernel.cpp @@ -1034,9 +1034,6 @@ void PatchKernel::resetInterfaces() return; } - // Prune stale interfaces - pruneStaleInterfaces(); - // Reset the interfaces _resetInterfaces(false); From d7a61bf7ba9d3fa3388f866e6065bf3abaa066e2 Mon Sep 17 00:00:00 2001 From: Andrea Iob Date: Sat, 7 Jan 2023 21:11:42 +0100 Subject: [PATCH 07/15] patchkernel: clear list of altered interfaces after resetting them --- src/patchkernel/patch_kernel.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/patchkernel/patch_kernel.cpp b/src/patchkernel/patch_kernel.cpp index f0b46ee07f..4ea31309fe 100644 --- a/src/patchkernel/patch_kernel.cpp +++ b/src/patchkernel/patch_kernel.cpp @@ -1037,11 +1037,11 @@ void PatchKernel::resetInterfaces() // Reset the interfaces _resetInterfaces(false); - // All remaining interfaces will be deleted - setInterfaceAlterationFlags(FLAG_DELETED); - // Mark cell interfaces as dirty setCellAlterationFlags(FLAG_INTERFACES_DIRTY); + + // Clear list of altered interfaces + m_alteredInterfaces.clear(); } /*! From e2ef8728101ddca0519c51ad1cd6a6a64eefd2b8 Mon Sep 17 00:00:00 2001 From: Andrea Iob Date: Mon, 30 Jan 2023 14:54:45 +0100 Subject: [PATCH 08/15] patchkernel: add functions to get vertices/interfaces that are on some specified cells --- src/patchkernel/patch_kernel.cpp | 80 ++++++++++++++++++++++++++++++++ src/patchkernel/patch_kernel.hpp | 3 ++ 2 files changed, 83 insertions(+) diff --git a/src/patchkernel/patch_kernel.cpp b/src/patchkernel/patch_kernel.cpp index 4ea31309fe..ea44b462fd 100644 --- a/src/patchkernel/patch_kernel.cpp +++ b/src/patchkernel/patch_kernel.cpp @@ -5774,6 +5774,86 @@ bool PatchKernel::isSameFace(const Cell &cell_A, int face_A, const Cell &cell_B, return (faceVertexIds_A == faceVertexIds_B); } +/*! + * Get the vertices that are on the specified cells. + * + * The result will contain a list of unique interfaces that are on the given + * cells. The list will be populated iterating the cells and addding the + * interfaces as they are encountered. + * + * \param cellIds are the ids of the cell whose vertices will be gathered + * \param interior controls if interior vertices will be gathered + * \param ghost controls if ghost vertices will be gathered + * \result The vertices that are on the specified cells. + */ +std::vector PatchKernel::getOrderedCellsVertices(const std::vector &cellIds, + bool interior, bool ghost) const +{ + std::vector orderedVertices; +#if BITPIT_ENABLE_MPI==1 + if (!interior && (!ghost || !isPartitioned())) { + return orderedVertices; + } +#else + if (!interior) { + return orderedVertices; + } +#endif + + std::unordered_set verticesSet; + for (long cellId : cellIds) { + const Cell &cell = getCell(cellId); + for (long vertexId : cell.getVertexIds()) { + if (verticesSet.count(vertexId) == 0) { + continue; + } + +#if BITPIT_ENABLE_MPI==1 + if (!interior || !ghost) { + auto vertexGhostInfoItr = m_ghostVertexInfo.find(vertexId); + if (!interior && vertexGhostInfoItr == m_ghostVertexInfo.cend()) { + continue; + } else if (!ghost && vertexGhostInfoItr != m_ghostVertexInfo.cend()) { + continue; + } + } +#endif + + verticesSet.insert(vertexId); + orderedVertices.push_back(vertexId); + } + } + + return orderedVertices; +} + +/*! + * Get the interfaces that are on the specified cells. + * + * The result will contain a list of unique interfaces that are on the given + * cells. The list will be populated iterating the cells and addding the + * interfaces as they are encountered. + */ +std::vector PatchKernel::getOrderedCellsInterfaces(const std::vector &cellIds) const +{ + std::vector orderedInterfaces; + std::unordered_set interfacesSet; + for (long cellId : cellIds) { + const Cell &cell = getCell(cellId); + const long *interfaces = cell.getInterfaces(); + int nCellInterfaces = cell.getInterfaceCount(); + for (int i = 0; i < nCellInterfaces; ++i) { + long interfaceId = interfaces[i]; + if (interfacesSet.count(interfaceId) == 0) { + interfacesSet.insert(interfaceId); + orderedInterfaces.push_back(interfaceId); + } + } + } + + return orderedInterfaces; +} + /*! Returns the current adjacencies build strategy. diff --git a/src/patchkernel/patch_kernel.hpp b/src/patchkernel/patch_kernel.hpp index dab4c909b1..29340765c8 100644 --- a/src/patchkernel/patch_kernel.hpp +++ b/src/patchkernel/patch_kernel.hpp @@ -963,6 +963,9 @@ friend class PatchManager; virtual int findAdjoinNeighFace(const Cell &cell, int cellFace, const Cell &neigh) const; virtual bool isSameFace(const Cell &cell_A, int face_A, const Cell &cell_B, int face_B) const; + std::vector getOrderedCellsVertices(const std::vector &cellIds, bool interior, bool ghost) const; + std::vector getOrderedCellsInterfaces(const std::vector &cellIds) const; + private: struct GhostVertexInfo { int owner; From d2a46ef63eb6feca3208709ec8a0acf8b5316891 Mon Sep 17 00:00:00 2001 From: Andrea Iob Date: Tue, 31 Jan 2023 15:47:40 +0100 Subject: [PATCH 09/15] patchkernel: rework collection for storing adaption information --- src/patchkernel/adaption.cpp | 294 ++++++++++++++++++++++++++++++++--- src/patchkernel/adaption.hpp | 26 +++- src/voloctree/voloctree.cpp | 14 +- 3 files changed, 306 insertions(+), 28 deletions(-) diff --git a/src/patchkernel/adaption.cpp b/src/patchkernel/adaption.cpp index 8d54b06b36..2d6b500281 100644 --- a/src/patchkernel/adaption.cpp +++ b/src/patchkernel/adaption.cpp @@ -61,7 +61,7 @@ namespace adaption /*! \struct Info - \brief The Info struct defines the information associated to an + \brief The Info struct defines the infomation associated to an adaption. */ @@ -77,12 +77,84 @@ namespace adaption */ InfoCollection::InfoCollection() { + m_cachedTypes.insert(adaption::TYPE_RENUMBERING); m_cachedTypes.insert(adaption::TYPE_DELETION); m_cachedTypes.insert(adaption::TYPE_CREATION); m_cachedTypes.insert(adaption::TYPE_PARTITION_RECV); m_cachedTypes.insert(adaption::TYPE_PARTITION_SEND); } + +/*! + * Get the size of the collection + * + * \return The number of entries in the collection. + */ +std::size_t InfoCollection::size() const +{ + return m_collection.size(); +} + +/*! + * Get a read/write iterator to the first entry of the collection. + * + * \return The iterator pointing to the first entry of the collection. + */ +InfoCollection::iterator InfoCollection::begin() noexcept +{ + return iterator(m_collection.begin()); +} + +/*! + * Get a read/write iterator that points one past the last enty of the collection. + * + * \return The iterator pointing to one past the last entry of the collection. + */ +InfoCollection::iterator InfoCollection::end() noexcept +{ + return iterator(m_collection.end()); +} + +/*! + * Get a read-only iterator to the first enty of the collection. + * + * \return the const iterator pointing to the first entry of the collection. + */ +InfoCollection::const_iterator InfoCollection::begin() const noexcept +{ + return const_iterator(cbegin()); +} + +/*! + * Get a read-only iterator that points one past the last enty of the collection. + * + * \return the const iterator pointing to one past the last entry of the collection. + */ +InfoCollection::const_iterator InfoCollection::end() const noexcept +{ + return const_iterator(cend()); +} + +/*! + * Get a read-only iterator to the first enty of the collection. + * + * \return the const iterator pointing to the first entry of the collection. + */ +InfoCollection::const_iterator InfoCollection::cbegin() const noexcept +{ + return const_iterator(m_collection.cbegin()); +} + +/*! + * Get a read-only iterator that points one past the last enty of the collection. + * + * \return the const iterator pointing to one past the last entry of the collection. + */ +InfoCollection::const_iterator InfoCollection::cend() const noexcept +{ + return const_iterator(m_collection.cend()); +} + /*! Returns a constant reference to the vector used internally by the collection to store its owned adaption info. @@ -108,11 +180,11 @@ std::vector & InfoCollection::data() noexcept } /*! - Creates an empty adaption info. + Insert an empty adaption info. - \result An empty adaption info. + \result The index of the newly added adaption info. */ -std::size_t InfoCollection::create() +std::size_t InfoCollection::insert() { m_collection.emplace_back(); @@ -120,19 +192,21 @@ std::size_t InfoCollection::create() } /*! - Creates an adaption info with the requested data. + Insert an adaption info with the requested data. - If an adaption info with the requested data already exists, the existing - value will be returned, otherwise a new adaption info will be created. + If the requested type is among the types that are cached and an adaption + info with the requested data already exists, the existing value will be + returned, otherwise a new adaption info will be created. If the requested + type is not among the types that are cached a new adaption info will be + created even if the collection already contains adaption info with the + requested data. \param type is the type of adaption info \param entity is the entity associated to the adaption info \param rank is the rank associated to the adaption info - \result The requested adaption info. If an adaption info with the - requested data already exists, the existing value will be returned, - otherwise a new adaption info will be created. + \result The position inside the collection of the adaption info. */ -std::size_t InfoCollection::create(Type type, Entity entity, int rank) +std::size_t InfoCollection::insert(Type type, Entity entity, int rank) { infoData_t infoData = infoData_t(type, entity, rank); bool useCache = (m_cachedTypes.count(type) > 0); @@ -152,36 +226,143 @@ std::size_t InfoCollection::create(Type type, Entity entity, int rank) return id; } +/*! + Copy the specified adaption in the collection. + + If the requested type is among the types that are cached and an adaption + info with the requested data already exists, the existing value will be + returned, otherwise a new adaption info will be created. If the requested + type is not among the types that are cached a new adaption info will be + created even if the collection already contains adaption info with the + same data. + + \param info is the adaption info that will be moved in the collection + \result The index of the newly added (or already existing) adaption + info +*/ +std::size_t InfoCollection::insert(const Info &other) +{ + return insert(Info(other)); +} + +/*! + Move the specified adaption in the collection. + + If the requested type is among the types that are cached and an adaption + info with the requested data already exists, the existing value will be + returned, otherwise a new adaption info will be created. If the requested + type is not among the types that are cached a new adaption info will be + created even if the collection already contains adaption info with the + same data. + + \param info is the adaption info that will be moved in the collection + \result The index of the newly added (or already existing) adaption + info +*/ +std::size_t InfoCollection::insert(Info &&other) +{ + std::size_t id = insert(other.type, other.entity, other.rank); + Info &info = m_collection[id]; + + if (info.previous.empty()) { + info.previous = std::move(other.previous); + } else { + appendIds(std::move(other.previous), true, &(info.previous)); + } + + if (info.current.empty()) { + info.current = std::move(other.current); + } else { + appendIds(std::move(other.current), true, &(info.current)); + } + + return id; +} + +/*! + Erase the adaption info at the specified position in the collection. + + If the provided position is not valid, no actions will be performed. + + \param id is the index of the adaption info + \result Returns true if the adaption info was deleted, otherwise it + return false. +*/ +bool InfoCollection::erase(std::size_t id) +{ + if (id >= size()) { + return false; + } + + const Info &info = m_collection[id]; + if (m_cachedTypes.count(info.type)) { + infoData_t infoData = infoData_t(info.type, info.entity, info.rank); + auto cacheItr = m_cache.find(infoData); + if (cacheItr != m_cache.end()) { + m_cache.erase(cacheItr); + } + } + + m_collection.erase(m_collection.begin() + id); + + return true; +} + +/*! + Erase all the adaption info of the specified type. + + \param type is the type of adaption info that wil be deleted + \result Returns the number of adaption info that have been deleted +*/ +std::size_t InfoCollection::erase(Type type) +{ + std::size_t nErasedInfo = 0; + + std::size_t id = 0; + while (id < m_collection.size()) { + const Info &info = at(id); + if (info.type != type) { + ++id; + continue; + } + + erase(id); + ++nErasedInfo; + } + + return nErasedInfo; +} + /*! Returns a reference to the adaption info at the specified position in the collection. - \param n is the position of an adaption info in the collection. + \param id is the index of the adaption info \result Returns a reference to requested adaption info. */ -Info & InfoCollection::at(std::size_t n) +Info & InfoCollection::at(std::size_t id) { - if (n >= m_collection.size()) { + if (id >= m_collection.size()) { throw std::out_of_range("Requested adaption info is not in the collection"); } - return (*this)[n]; + return (*this)[id]; } /*! Returns a constant reference to the adaption info at the specified position in the collection. - \param n is the position of an adaption info in the collection. + \param id is the index of the adaption info \result Returns a constant reference to requested adaption info. */ -const Info & InfoCollection::at(std::size_t n) const +const Info & InfoCollection::at(std::size_t id) const { - if (n >= m_collection.size()) { + if (id >= m_collection.size()) { throw std::out_of_range("Requested adaption info is not in the collection"); } - return (*this)[n]; + return (*this)[id]; } /*! @@ -224,6 +405,81 @@ std::vector InfoCollection::dump() return exportedCollection; } +/*! + Append the ids contained in the source list into the destination list. + + Optionally, it is possible to append only the ids of the source list that + are not already contained in the destination list. + + \param src is the list that contains the ids to be appended + \param unique if set to true, only the ids of the source list that are not + already contained in the destination list will be appended + \param dst is the list where the ids will be copied to +*/ +void InfoCollection::appendIds(std::vector src, bool unique, std::vector *dst) +{ + // Early return if the source is empty + if (src.empty()) { + return; + } + + // Early return if the destination is empty + if (dst->empty()) { + *dst = std::move(src); + return; + } + + // Append the ids + if (unique) { + std::unordered_set dstSet(dst->begin(), dst->end()); + for (long id : src) { + if (dstSet.count(id) == 0) { + dst->push_back(id); + } + } + } else { + dst->insert(dst->end(), src.begin(), src.end()); + } +} + +/*! + Remove the ids contained in the source list from the destination list. + + \param src is the list that contains the ids to be removed + \param dst is the list where the ids will be copied to + \result The number of ids that have been deleted. +*/ +std::size_t InfoCollection::removeIds(std::unordered_set src, std::vector *dst) +{ + // Early return if the source is empty + if (src.empty()) { + return 0; + } + + // Early return if the destination is empty + if (dst->empty()) { + return 0; + } + + // Remove the ids + std::size_t nDeletedIds = 0; + for (auto itr = dst->begin(); itr != dst->end(); ++itr) { + long vertexId = *itr; + if (src.count(vertexId) > 0) { + ++nDeletedIds; + } else if (nDeletedIds > 0) { + *(itr - nDeletedIds) = *itr; + } + } + + if (nDeletedIds > 0) { + dst->resize(dst->size() - nDeletedIds); + dst->shrink_to_fit(); + } + + return nDeletedIds; +} + } /*! diff --git a/src/patchkernel/adaption.hpp b/src/patchkernel/adaption.hpp index 96b62aad4b..c785c225ce 100644 --- a/src/patchkernel/adaption.hpp +++ b/src/patchkernel/adaption.hpp @@ -82,10 +82,32 @@ namespace adaption { public: + typedef std::vector::iterator iterator; + typedef std::vector::const_iterator const_iterator; + + static void appendIds(std::vector src, bool unique, std::vector *dst); + static std::size_t removeIds(std::unordered_set src, std::vector *dst); + InfoCollection(); - std::size_t create(); - std::size_t create(Type type, Entity entity, int rank = -1); + std::size_t size() const; + + iterator begin() noexcept; + iterator end() noexcept; + + const_iterator begin() const noexcept; + const_iterator end() const noexcept; + + const_iterator cbegin() const noexcept; + const_iterator cend() const noexcept; + + std::size_t insert(); + std::size_t insert(Type type, Entity entity, int rank = -1); + std::size_t insert(const Info &other); + std::size_t insert(Info &&other); + + bool erase(std::size_t id); + std::size_t erase(Type type); Info & at(std::size_t id); const Info & at(std::size_t id) const; diff --git a/src/voloctree/voloctree.cpp b/src/voloctree/voloctree.cpp index 49c30c47a4..73764dc5e3 100644 --- a/src/voloctree/voloctree.cpp +++ b/src/voloctree/voloctree.cpp @@ -983,7 +983,7 @@ std::vector VolOctree::_adaptionPrepare(bool trackAdaption) } if (createAdaption) { - std::size_t adaptionInfoId = adaptionData.create(adaptionType, adaption::ENTITY_CELL, currentRank); + std::size_t adaptionInfoId = adaptionData.insert(adaptionType, adaption::ENTITY_CELL, currentRank); adaptionInfo = &(adaptionData[adaptionInfoId]); } @@ -997,7 +997,7 @@ std::vector VolOctree::_adaptionPrepare(bool trackAdaption) CellConstIterator beginItr = ghostCellConstBegin(); CellConstIterator endItr = ghostCellConstEnd(); - std::size_t adaptionInfoId = adaptionData.create(adaption::TYPE_DELETION, adaption::ENTITY_CELL, currentRank); + std::size_t adaptionInfoId = adaptionData.insert(adaption::TYPE_DELETION, adaption::ENTITY_CELL, currentRank); adaption::Info &adaptionInfo = adaptionData[adaptionInfoId]; adaptionInfo.previous.reserve(getGhostCellCount()); for (CellConstIterator itr = beginItr; itr != endItr; ++itr) { @@ -1256,7 +1256,7 @@ std::vector VolOctree::sync(bool trackChanges) #endif // Get the adaption info - std::size_t infoId = adaptionData.create(adaptionType, adaption::ENTITY_CELL, rank); + std::size_t infoId = adaptionData.insert(adaptionType, adaption::ENTITY_CELL, rank); adaption::Info &adaptionInfo = adaptionData[infoId]; // Current status @@ -1409,7 +1409,7 @@ std::vector VolOctree::sync(bool trackChanges) if (adaptionIsDeletion || adaptionIsSend) { int rank = deleteInfo.rank; - std::size_t adaptionInfoId = adaptionData.create(adaptionType, adaption::ENTITY_CELL, rank); + std::size_t adaptionInfoId = adaptionData.insert(adaptionType, adaption::ENTITY_CELL, rank); adaption::Info &adaptionInfo = adaptionData[adaptionInfoId]; adaptionInfo.previous.emplace_back(); long &deletedId = adaptionInfo.previous.back(); @@ -1447,7 +1447,7 @@ std::vector VolOctree::sync(bool trackChanges) #endif // Adaption info for the deleted interfaces - std::size_t adaptionInfoId = adaptionData.create(adaption::TYPE_DELETION, adaption::ENTITY_INTERFACE, currentRank); + std::size_t adaptionInfoId = adaptionData.insert(adaption::TYPE_DELETION, adaption::ENTITY_INTERFACE, currentRank); adaption::Info &adaptionInfo = adaptionData[adaptionInfoId]; adaptionInfo.previous.reserve(removedInterfaces.size()); for (long interfaceId : removedInterfaces) { @@ -1513,7 +1513,7 @@ std::vector VolOctree::sync(bool trackChanges) #if BITPIT_ENABLE_MPI==1 // Track created ghosts cells if (nGhostsOctants > 0) { - std::size_t adaptionInfoId = adaptionData.create(adaption::TYPE_CREATION, adaption::ENTITY_CELL, currentRank); + std::size_t adaptionInfoId = adaptionData.insert(adaption::TYPE_CREATION, adaption::ENTITY_CELL, currentRank); adaption::Info &adaptionInfo = adaptionData[adaptionInfoId]; adaptionInfo.current.reserve(nGhostsOctants); @@ -1545,7 +1545,7 @@ std::vector VolOctree::sync(bool trackChanges) } // Adaption info - std::size_t infoId = adaptionData.create(adaption::TYPE_CREATION, adaption::ENTITY_INTERFACE, currentRank); + std::size_t infoId = adaptionData.insert(adaption::TYPE_CREATION, adaption::ENTITY_INTERFACE, currentRank); adaption::Info &adaptionInfo = adaptionData[infoId]; adaptionInfo.current.reserve(createdInterfaces.size()); for (long interfaceId : createdInterfaces) { From 82fbd37e181bbfca23b879948d5543b9abc96d59 Mon Sep 17 00:00:00 2001 From: Andrea Iob Date: Sat, 7 Jan 2023 21:13:38 +0100 Subject: [PATCH 10/15] volcartesian: remove unneeded overload of resetInterfaces function --- src/volcartesian/volcartesian.cpp | 9 --------- src/volcartesian/volcartesian.hpp | 2 -- 2 files changed, 11 deletions(-) diff --git a/src/volcartesian/volcartesian.cpp b/src/volcartesian/volcartesian.cpp index 60c3541552..62fe7a1602 100644 --- a/src/volcartesian/volcartesian.cpp +++ b/src/volcartesian/volcartesian.cpp @@ -251,15 +251,6 @@ void VolCartesian::reset() m_nInterfaces = 0; } -/*! - Resest the interfaces of the patch. -*/ -void VolCartesian::resetInterfaces() -{ - PatchKernel::resetInterfaces(); -} - - /*! Internal function to update the adjacencies of the patch. diff --git a/src/volcartesian/volcartesian.hpp b/src/volcartesian/volcartesian.hpp index 0dec239eef..aa449abf00 100644 --- a/src/volcartesian/volcartesian.hpp +++ b/src/volcartesian/volcartesian.hpp @@ -66,8 +66,6 @@ class VolCartesian : public VolumeKernel { void reset() override; - void resetInterfaces() override; - long getVertexCount() const override; int getVertexCount(int direction) const; From ea4eef7be792b3e8f747fe8a75bbf0e36a1c3d3a Mon Sep 17 00:00:00 2001 From: Andrea Iob Date: Sat, 7 Jan 2023 21:17:09 +0100 Subject: [PATCH 11/15] voloctree: rename some variables --- src/voloctree/voloctree_mapper.cpp | 48 ++++++++++++++-------------- src/voloctree/voloctree_mapper.hpp | 10 +++--- src/voloctree/voloctree_parallel.cpp | 14 ++++---- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/src/voloctree/voloctree_mapper.cpp b/src/voloctree/voloctree_mapper.cpp index 0fb484693a..f8f79de881 100644 --- a/src/voloctree/voloctree_mapper.cpp +++ b/src/voloctree/voloctree_mapper.cpp @@ -98,12 +98,12 @@ void VolOctreeMapper::clearPartitionMappingLists() * The adaptation of only one mesh at time is allowed. Calling this method * is mandatory to perform an update of the mapper after an adaptation. * - * \param[in] adaptionInfo are the adaptation info that describe the changes + * \param[in] adaptionData are the adaptation info that describe the changes * of the patch * \param[in] reference if set to true the reference mesh will be adapted, * is set to false the mapped one will be adapted */ -void VolOctreeMapper::adaptionPrepare(const std::vector &adaptionInfo, bool reference) +void VolOctreeMapper::adaptionPrepare(const std::vector &adaptionData, bool reference) { m_previousMapping.clear(); PiercedStorage *pmapper; @@ -118,7 +118,7 @@ void VolOctreeMapper::adaptionPrepare(const std::vector &adaptio #if BITPIT_ENABLE_MPI m_partitionIR.map_rank_previousMapping.clear(); #endif - for (const adaption::Info &info : adaptionInfo) { + for (const adaption::Info &info : adaptionData) { if (info.type == adaption::Type::TYPE_PARTITION_SEND || info.type == adaption::Type::TYPE_PARTITION_RECV || info.type == adaption::Type::TYPE_PARTITION_NOTICE) { @@ -139,7 +139,7 @@ void VolOctreeMapper::adaptionPrepare(const std::vector &adaptio * The adaptationPrepare method of the mapper has to called before the * adaptation of the mesh. * - * \param[in] adaptionInfo are the adaptation info that describe the changes of + * \param[in] adaptionData are the adaptation info that describe the changes of * the patch * \param[in] reference if set to true the reference mesh will be adapted, * is set to false the mapped one will be adapted @@ -148,12 +148,12 @@ void VolOctreeMapper::adaptionPrepare(const std::vector &adaptio * the inverse mapper has to be necessarily filled during the first computation * in initialization) */ -void VolOctreeMapper::adaptionAlter(const std::vector &adaptionInfo, bool reference, bool inverseFilled) +void VolOctreeMapper::adaptionAlter(const std::vector &adaptionData, bool reference, bool inverseFilled) { if (reference) { - _mappingAdaptionReferenceUpdate(adaptionInfo, inverseFilled); + _mappingAdaptionReferenceUpdate(adaptionData, inverseFilled); } else { - _mappingAdaptionMappedUpdate(adaptionInfo); + _mappingAdaptionMappedUpdate(adaptionData); } } @@ -168,14 +168,14 @@ void VolOctreeMapper::adaptionCleanup() /** * Update the mapper after an adaption of the reference mesh. * - * \param[in] adaptionInfo are the adaptation info that describe the changes of + * \param[in] adaptionData are the adaptation info that describe the changes of * the patch * \param[in] inverseFilled if set to true the inverse mapped was filled * during the mesh mapper computing. If the adapted mesh is the mapped one, * the inverse mapper has to be necessarily filled during the first computation * in initialization) */ -void VolOctreeMapper::_mappingAdaptionReferenceUpdate(const std::vector &adaptionInfo, bool inverseFilled) +void VolOctreeMapper::_mappingAdaptionReferenceUpdate(const std::vector &adaptionData, bool inverseFilled) { const VolOctree *adaptedPatch = static_cast(m_referencePatch); const VolOctree *mappedPatch = static_cast(m_mappedPatch); @@ -193,7 +193,7 @@ void VolOctreeMapper::_mappingAdaptionReferenceUpdate(const std::vector &adaptionInfo) +void VolOctreeMapper::_mappingAdaptionMappedUpdate(const std::vector &adaptionData) { const VolOctree *adaptedPatch = static_cast(m_mappedPatch); const VolOctree *referencePatch = static_cast(m_referencePatch); @@ -481,18 +481,18 @@ void VolOctreeMapper::_mappingAdaptionMappedUpdate(const std::vector adaptionInfoRef; + std::vector adaptionDataRef; #if BITPIT_ENABLE_MPI if (!checkPart) { - _communicateMappedAdaptionInfo(adaptionInfo, &adaptionInfoRef); + _communicateMappedAdaptionInfo(adaptionData, &adaptionDataRef); } else { - adaptionInfoRef = adaptionInfo; + adaptionDataRef = adaptionData; } #else - adaptionInfoRef = adaptionInfo; + adaptionDataRef = adaptionData; #endif - for (const adaption::Info &info : adaptionInfoRef) { + for (const adaption::Info &info : adaptionDataRef) { if (info.type == adaption::Type::TYPE_PARTITION_SEND || info.type == adaption::Type::TYPE_PARTITION_RECV || info.type == adaption::Type::TYPE_PARTITION_NOTICE) { @@ -1596,12 +1596,12 @@ void VolOctreeMapper::_communicateInverseMapperBack() * Communicate adaption info of overlapped partitions of the mapped mesh to * the processes of reference partitions. * - * \param[in] adaptionInfoMap are the adaptation info that describe the changes + * \param[in] adaptionDataMap are the adaptation info that describe the changes * of the local mapped partitions - * \param[out] adaptionInfoRef are the adaptation info that describe the + * \param[out] adaptionDataRef are the adaptation info that describe the * changes of the local reference partitions */ -void VolOctreeMapper::_communicateMappedAdaptionInfo(const std::vector &adaptionInfoMap, std::vector *adaptionInfoRef) +void VolOctreeMapper::_communicateMappedAdaptionInfo(const std::vector &adaptionDataMap, std::vector *adaptionDataRef) { // Recover mapping elements to send (to partitions of mapped mesh) std::set toRanks; @@ -1609,7 +1609,7 @@ void VolOctreeMapper::_communicateMappedAdaptionInfo(const std::vector> toRankId; std::size_t n = 0; - for (auto &info : adaptionInfoMap) { + for (auto &info : adaptionDataMap) { for (long id : info.previous) { auto itPreviousMapping = m_previousMapping.find(id); if (itPreviousMapping == m_previousMapping.end()){ @@ -1639,7 +1639,7 @@ void VolOctreeMapper::_communicateMappedAdaptionInfo(const std::vector> arank; info.rank = arank; - adaptionInfoRef->push_back(info); + adaptionDataRef->push_back(info); } ++nCompletedRecvs; diff --git a/src/voloctree/voloctree_mapper.hpp b/src/voloctree/voloctree_mapper.hpp index d4aa693f36..6356242bac 100644 --- a/src/voloctree/voloctree_mapper.hpp +++ b/src/voloctree/voloctree_mapper.hpp @@ -47,8 +47,8 @@ class VolOctreeMapper: public VolumeMapper { VolOctreeMapper(const VolOctree *referencePatch, const VolOctree *mappedPatch); #endif - void adaptionPrepare(const std::vector &adaptionInfo, bool reference = true) override; - void adaptionAlter(const std::vector &adaptionInfo, bool reference = true, bool inverseFilled = false) override; + void adaptionPrepare(const std::vector &adaptionData, bool reference = true) override; + void adaptionAlter(const std::vector &adaptionData, bool reference = true, bool inverseFilled = false) override; void adaptionCleanup() override; #if BITPIT_ENABLE_MPI @@ -161,8 +161,8 @@ class VolOctreeMapper: public VolumeMapper { void _mapMeshesSamePartition(const std::vector * octantsReference, const std::vector * octantsMapped, bool fillInverse, long *indRef); - void _mappingAdaptionReferenceUpdate(const std::vector &adaptionInfo, bool fillInverse = false); - void _mappingAdaptionMappedUpdate(const std::vector &adaptionInfo); + void _mappingAdaptionReferenceUpdate(const std::vector &adaptionData, bool fillInverse = false); + void _mappingAdaptionMappedUpdate(const std::vector &adaptionData); #if BITPIT_ENABLE_MPI void clearPartitionMappingLists(); @@ -177,7 +177,7 @@ class VolOctreeMapper: public VolumeMapper { void _communicateInverseMapperBack(); - void _communicateMappedAdaptionInfo(const std::vector &adaptionInfoMap, std::vector *adaptionInfoRef); + void _communicateMappedAdaptionInfo(const std::vector &adaptionDataMap, std::vector *adaptionDataRef); #endif }; diff --git a/src/voloctree/voloctree_parallel.cpp b/src/voloctree/voloctree_parallel.cpp index 96ffc1b72a..6c7b7a14ed 100644 --- a/src/voloctree/voloctree_parallel.cpp +++ b/src/voloctree/voloctree_parallel.cpp @@ -147,19 +147,19 @@ std::vector VolOctree::_partitioningPrepare(const std::unordered } partitioningData.emplace_back(); - adaption::Info &partitioningInfo = partitioningData.back(); - partitioningInfo.entity = adaption::ENTITY_CELL; - partitioningInfo.type = adaptionType; - partitioningInfo.rank = receiver; + adaption::Info &partitioningCellInfo = partitioningData.back(); + partitioningCellInfo.entity = adaption::ENTITY_CELL; + partitioningCellInfo.type = adaptionType; + partitioningCellInfo.rank = receiver; const std::array &sendRange = entry.second; uint32_t beginTreeId = sendRange[0]; uint32_t endTreeId = sendRange[1]; - partitioningInfo.previous.reserve(endTreeId - beginTreeId); + partitioningCellInfo.previous.reserve(endTreeId - beginTreeId); for (uint32_t treeId = beginTreeId; treeId < endTreeId; ++treeId) { OctantInfo octantInfo(treeId, true); - partitioningInfo.previous.emplace_back(); - long &cellId = partitioningInfo.previous.back(); + partitioningCellInfo.previous.emplace_back(); + long &cellId = partitioningCellInfo.previous.back(); cellId = getOctantId(octantInfo); } } From 61d20b0a6996c4404be66cced27cdf254450db3c Mon Sep 17 00:00:00 2001 From: Andrea Iob Date: Fri, 3 Feb 2023 09:41:58 +0100 Subject: [PATCH 12/15] voloctree: rework patch synchronization --- src/voloctree/voloctree.cpp | 1078 +++++++++++++++-------------------- src/voloctree/voloctree.hpp | 8 +- 2 files changed, 475 insertions(+), 611 deletions(-) diff --git a/src/voloctree/voloctree.cpp b/src/voloctree/voloctree.cpp index 73764dc5e3..5e68f9d254 100644 --- a/src/voloctree/voloctree.cpp +++ b/src/voloctree/voloctree.cpp @@ -24,6 +24,7 @@ #include #include +#include #include "bitpit_common.hpp" @@ -1070,7 +1071,7 @@ void VolOctree::settleAdaptionMarkers() */ std::vector VolOctree::sync(bool trackChanges) { - log::cout() << ">> Syncing patch..." << std::endl; + log::cout() << "Syncing patch..." << std::endl; // Detect if we are in import-from-scratch mode // @@ -1086,17 +1087,17 @@ std::vector VolOctree::sync(bool trackChanges) } // Info on the tree - uint32_t nOctants = static_cast(m_tree->getNumOctants()); - uint32_t nPreviousOctants = static_cast(m_octantToCell.size()); - - log::cout() << ">> Number of octants : " << nOctants << std::endl; - - // Info on the tree - uint32_t nGhostsOctants = static_cast(m_tree->getNumGhosts()); - uint32_t nPreviousGhosts = static_cast(m_ghostToCell.size()); - - // Initialize tracking data - adaption::InfoCollection adaptionData; + std::size_t nOctants = m_tree->getNumOctants(); + std::size_t nGhostsOctants = m_tree->getNumGhosts(); + std::size_t nPreviousOctants = m_octantToCell.size(); + + log::cout() << " Previous number of octants: " << nPreviousOctants << std::endl; + log::cout() << " Current number of octants: " << nOctants << std::endl; + if (nOctants > nPreviousOctants) { + log::cout() << " Number of octants that will be created: " << (nOctants - nPreviousOctants) << std::endl; + } else if (nOctants < nPreviousOctants) { + log::cout() << " Number of octants that will be deleted: " << (nPreviousOctants - nOctants) << std::endl; + } // Current rank int currentRank = -1; @@ -1108,513 +1109,566 @@ std::vector VolOctree::sync(bool trackChanges) // // If there are no cells in the mesh we need to import all // octants. - log::cout() << ">> Extract information for transforming the patch..."; - - std::vector unmappedOctants(nPreviousOctants, true); - std::vector addedOctants; - std::vector renumberedOctants; - std::vector deletedOctants; - - addedOctants.reserve(nOctants + nGhostsOctants); - if (!importFromScratch) { - renumberedOctants.reserve(nPreviousOctants + nPreviousGhosts); - deletedOctants.reserve(nPreviousOctants + nPreviousGhosts); - } - - uint32_t treeId = 0; - std::vector mapper_octantMap; - std::vector mapper_ghostFlag; - std::vector mapper_octantRank; - while (treeId < (uint32_t) nOctants) { - // Octant mapping - mapper_octantMap.clear(); - mapper_ghostFlag.clear(); - mapper_octantRank.clear(); - if (!importFromScratch) { - m_tree->getMapping(treeId, mapper_octantMap, mapper_ghostFlag, mapper_octantRank); - } + log::cout() << " Extract information for transforming the patch..."; + + adaption::InfoCollection synchronizationData; + if (importFromScratch) { + std::size_t adaptionId = synchronizationData.insert(adaption::TYPE_CREATION, adaption::ENTITY_CELL); + adaption::Info &adaptionInfo = synchronizationData[adaptionId]; + adaptionInfo.current.resize(nOctants + nGhostsOctants); + std::iota(adaptionInfo.current.begin(), adaptionInfo.current.end(), 0); + } else { + // Identify changes using the tree mapper + std::vector mappedInternalOctants(nPreviousOctants, false); + + std::vector mapper_octantMap; + std::vector mapper_ghostFlag; + std::vector mapper_octantRank; + + uint32_t treeId = 0; + while (treeId < (uint32_t) nOctants) { + // Octant mapping + mapper_octantMap.clear(); + mapper_ghostFlag.clear(); + mapper_octantRank.clear(); + if (!importFromScratch) { + m_tree->getMapping(treeId, mapper_octantMap, mapper_ghostFlag, mapper_octantRank); + } - // Adaption type - adaption::Type adaptionType = adaption::TYPE_NONE; - if (importFromScratch) { - adaptionType = adaption::TYPE_CREATION; - } else if (lastTreeOperation == ParaTree::OP_ADAPT_MAPPED) { - bool isNewR = m_tree->getIsNewR(treeId); - if (isNewR) { - adaptionType = adaption::TYPE_REFINEMENT; - } else { - bool isNewC = m_tree->getIsNewC(treeId); - if (isNewC) { - adaptionType = adaption::TYPE_COARSENING; + // Identify adaption type + adaption::Type adaptionType = adaption::TYPE_NONE; + if (lastTreeOperation == ParaTree::OP_ADAPT_MAPPED) { + bool isNewR = m_tree->getIsNewR(treeId); + if (isNewR) { + adaptionType = adaption::TYPE_REFINEMENT; + } else { + bool isNewC = m_tree->getIsNewC(treeId); + if (isNewC) { + adaptionType = adaption::TYPE_COARSENING; + } else if (treeId != mapper_octantMap.front()) { + adaptionType = adaption::TYPE_RENUMBERING; + } + } +#if BITPIT_ENABLE_MPI==1 + } else if (lastTreeOperation == ParaTree::OP_LOADBALANCE || lastTreeOperation == ParaTree::OP_LOADBALANCE_FIRST) { + if (currentRank != mapper_octantRank.front()) { + adaptionType = adaption::TYPE_PARTITION_RECV; } else if (treeId != mapper_octantMap.front()) { adaptionType = adaption::TYPE_RENUMBERING; } +#endif + } + + // Skip to the next octant if no changes have been detected + if (adaptionType == adaption::TYPE_NONE) { + mappedInternalOctants[treeId] = true; + ++treeId; + continue; } + + // Rank assocated with the adaption + int adaptionRank = currentRank; #if BITPIT_ENABLE_MPI==1 - } else if (lastTreeOperation == ParaTree::OP_LOADBALANCE || lastTreeOperation == ParaTree::OP_LOADBALANCE_FIRST) { - if (currentRank != mapper_octantRank.front()) { - adaptionType = adaption::TYPE_PARTITION_RECV; - } else if (treeId != mapper_octantMap.front()) { - adaptionType = adaption::TYPE_RENUMBERING; + if (adaptionType == adaption::TYPE_PARTITION_RECV) { + adaptionRank = mapper_octantRank[0]; } #endif - } - - // If the octant cell has not been modified we can skip to the next - // octant. - if (adaptionType == adaption::TYPE_NONE) { - unmappedOctants[treeId] = false; - ++treeId; - continue; - } - // Re-numbered cells just need to be added to the proper list. - // - // Renumbered cells are not tracked, because the re-numbering - // only happens inside VolOctree. - if (adaptionType == adaption::TYPE_RENUMBERING) { - uint32_t previousTreeId = mapper_octantMap.front(); - OctantInfo previousOctantInfo(previousTreeId, !mapper_ghostFlag.front()); - long cellId = getOctantId(previousOctantInfo); - renumberedOctants.emplace_back(cellId, treeId); - unmappedOctants[previousTreeId] = false; - - // No more work needed, skip to the next octant - ++treeId; - continue; - } + // Get adaption information + std::size_t adaptionInfoId = synchronizationData.insert(adaptionType, adaption::ENTITY_CELL, adaptionRank); + adaption::Info &adaptionInfo = synchronizationData[adaptionInfoId]; - // Handle other kind of adaption - // - // New octants need to be imported into the patch, - // whereas cells associated to previous octants - // need to be removed. - // - // If the user want to track adaption, adaption - // data needs to be filled. - - // Current tree ids that will be imported - long nCurrentTreeIds; - if (importFromScratch) { - nCurrentTreeIds = nOctants - treeId; - } else if (adaptionType == adaption::TYPE_REFINEMENT) { - nCurrentTreeIds = uipow(2, getDimension()); - } else { - nCurrentTreeIds = 1; - } + // Fill adaption information about current status + // + // Current status of the adaption information is filled with dummy ids set equal + // to the tree ids. During cell creation, these dummy will define the octant to + // cell association. + long nCurrentTreeIds; + if (adaptionType == adaption::TYPE_REFINEMENT) { + nCurrentTreeIds = m_tree->getNchildren(); + } else if (adaptionType == adaption::TYPE_DELETION) { + nCurrentTreeIds = 0; + } else if (adaptionType == adaption::TYPE_PARTITION_SEND) { + nCurrentTreeIds = 0; + } else { + nCurrentTreeIds = 1; + } - const long lastCurrentTreeId = treeId + nCurrentTreeIds; - for (int currentTreeId = treeId; currentTreeId < lastCurrentTreeId; ++currentTreeId) { - addedOctants.emplace_back(currentTreeId, true); - } + for (int k = 0; k < nCurrentTreeIds; ++k) { + adaptionInfo.current.push_back(treeId + k); + } - // Cells that will be removed - // - // Mark the cells associated to previous local octants for deletion. - if (!importFromScratch) { + // Fill adaption information about previous status int nPreviousTreeIds = mapper_octantMap.size(); for (int k = 0; k < nPreviousTreeIds; ++k) { #if BITPIT_ENABLE_MPI==1 - // Only local cells can be deleted + // Previous information should be filled only for local octants if (mapper_octantRank[k] != currentRank) { continue; } #endif - // Ghost octants will be processed later - if (mapper_ghostFlag[k]) { - continue; - } - - // Mark previous octant for deletion + // Idenfity cell id uint32_t previousTreeId = mapper_octantMap[k]; OctantInfo previousOctantInfo(previousTreeId, !mapper_ghostFlag[k]); - long cellId = getOctantId(previousOctantInfo); - deletedOctants.emplace_back(cellId, adaptionType); + long previousCellId = getOctantId(previousOctantInfo); - unmappedOctants[previousTreeId] = false; - } - } + // Update information on previous status + adaptionInfo.previous.push_back(previousCellId); - // Adaption tracking - // - // The adaption info associated to the octants that has been received - // from external partitions will contain the current octants sorted by - // their tree id (we are looping over the octants in that order), this - // is the same order that will be used on the process that has sent - // the octants. Since the order is the same, the two processes are able - // to exchange cell data without any additional extra communication - // (they already know the list of cells for which data is needed and - // the order in which these data will be sent). - if (trackChanges) { - // Rank assocated to the adaption info - int rank = currentRank; -#if BITPIT_ENABLE_MPI==1 - if (adaptionType == adaption::TYPE_PARTITION_RECV) { - rank = mapper_octantRank[0]; - } -#endif - - // Get the adaption info - std::size_t infoId = adaptionData.insert(adaptionType, adaption::ENTITY_CELL, rank); - adaption::Info &adaptionInfo = adaptionData[infoId]; - - // Current status - // - // We don't know the id of the current status, because those - // cells are not yet in the mesh. Store the trre id, and - // make the translation later. - // - // WARNING: tree id are uint32_t wherase adaptionInfo stores - // id as long. - adaptionInfo.current.reserve(nCurrentTreeIds); - auto addedOctantsIter = addedOctants.cend() - nCurrentTreeIds; - while (addedOctantsIter != addedOctants.cend()) { - adaptionInfo.current.emplace_back(); - long &adaptionId = adaptionInfo.current.back(); - adaptionId = (*addedOctantsIter).id; - - addedOctantsIter++; - } - - // Previous cells - // - // A coarsening can merge togheter cells of different processes. - // However, since the coarsening is limited to one level, the - // previous cells will always be internal or among the ghost of - // the current process. - int nPreviousCellIds = mapper_octantMap.size(); - adaptionInfo.previous.reserve(nPreviousCellIds); - for (int k = 0; k < nPreviousCellIds; ++k) { - long previousCellId; -#if BITPIT_ENABLE_MPI==1 - if (mapper_octantRank[k] != currentRank) { - previousCellId = Cell::NULL_ID; - } else -#endif - { - OctantInfo previousOctantInfo(mapper_octantMap[k], !mapper_ghostFlag[k]); - previousCellId = getOctantId(previousOctantInfo); + // Previous octant has been mapped + if (previousOctantInfo.internal) { + mappedInternalOctants[previousTreeId] = true; } - - adaptionInfo.previous.emplace_back(); - long &adaptionId = adaptionInfo.previous.back(); - adaptionId = previousCellId; } - } - - // Incremente tree id - treeId += nCurrentTreeIds; - } - - log::cout() << " Done" << std::endl; -#if BITPIT_ENABLE_MPI==1 - // New ghost octants need to be added - for (uint32_t treeId = 0; treeId < (uint32_t) nGhostsOctants; ++treeId) { - addedOctants.emplace_back(treeId, false); - } -#endif + // Incremente tree id + treeId += std::max(nCurrentTreeIds, 1L); + } - // Remove octants that are no more in the tree - if (!importFromScratch) { #if BITPIT_ENABLE_MPI==1 - // Cells that have been send to other processes need to be removed + // Identify cells sent to other partitions + // + // The adaption info associated to the octants that has been sent + // to external partitions will contain the current octants sorted by + // their tree id (they were added to the deleted octants list in that + // order), this is the same order that will be used on the process + // that has received the octants. Since the order is the same, the two + // processes are able to exchange cell data without any additional + // extra communication (they already know the list of cells for which + // data is needed and the order in which these data will be sent). PabloUniform::LoadBalanceRanges loadBalanceRanges = m_tree->getLoadBalanceRanges(); for (const auto &rankEntry : loadBalanceRanges.sendRanges) { - int rank = rankEntry.first; - if (rank == currentRank) { + // Consider only cells sent to other partitions + // + // During first tree load balance, cells are deleted and not sent + // to other processes (because all processes contain the whole + // tree). + if (loadBalanceRanges.sendAction != PabloUniform::LoadBalanceRanges::ACTION_SEND) { continue; } - adaption::Type deletionType; - if (loadBalanceRanges.sendAction == PabloUniform::LoadBalanceRanges::ACTION_DELETE) { - deletionType = adaption::TYPE_DELETION; - } else { - deletionType = adaption::TYPE_PARTITION_SEND; + int receiverRank = rankEntry.first; + if (receiverRank == currentRank) { + continue; } - uint32_t beginTreeId = rankEntry.second[0]; - uint32_t endTreeId = rankEntry.second[1]; - for (uint32_t treeId = beginTreeId; treeId < endTreeId; ++treeId) { - OctantInfo octantInfo(treeId, true); - long cellId = getOctantId(octantInfo); - deletedOctants.emplace_back(cellId, deletionType, rank); - unmappedOctants[treeId] = false; + // Get adaption information + std::size_t adaptionId = synchronizationData.insert(adaption::TYPE_PARTITION_SEND, adaption::ENTITY_CELL, receiverRank); + adaption::Info &adaptionInfo = synchronizationData[adaptionId]; + + // Fill adaption information about previous status + uint32_t beginPreviousTreeId = rankEntry.second[0]; + uint32_t endPreviousTreeId = rankEntry.second[1]; + for (uint32_t previousTreeId = beginPreviousTreeId; previousTreeId < endPreviousTreeId; ++previousTreeId) { + // Idenfity cell id + OctantInfo octantInfo(previousTreeId, true); + long previousCellId = getOctantId(octantInfo); + + // Update information on previous status + adaptionInfo.previous.push_back(previousCellId); + + // Previous octant has been mapped + mappedInternalOctants[previousTreeId] = true; } } - // Previous ghosts cells need to be removed - if (nPreviousGhosts > 0) { - for (uint32_t ghostTreeId = 0; ghostTreeId < nPreviousGhosts; ++ghostTreeId) { - OctantInfo ghostOctantInfo(ghostTreeId, false); - long ghostCellId = getOctantId(ghostOctantInfo); - deletedOctants.emplace_back(ghostCellId, adaption::TYPE_DELETION); - } + // Cells previously associated with ghost octants should be deleted + std::size_t ghostDeletionAdaptionId = synchronizationData.insert(adaption::TYPE_CREATION, adaption::ENTITY_CELL); + adaption::Info &ghostDeletionAdaptionInfo = synchronizationData[ghostDeletionAdaptionId]; + for (auto ghostItr = ghostCellConstBegin(); ghostItr != ghostCellConstEnd(); ++ghostItr) { + ghostDeletionAdaptionInfo.previous.emplace_back(ghostItr.getId()); + } + + // Cells associated with current ghost octants should be created + // + // Current status of the adaption information is filled with dummy ids set equal + // to the tree ids incremented by the number of octants. During cell creation, + // these dummy will define the octant to cell association. + std::size_t ghostCreationInfoId = synchronizationData.insert(adaption::TYPE_CREATION, adaption::ENTITY_CELL); + adaption::Info &ghostCreationInfo = synchronizationData[ghostCreationInfoId]; + for (uint32_t treeId = 0; treeId < (uint32_t) nGhostsOctants; ++treeId) { + ghostCreationInfo.current.emplace_back(nOctants + treeId); } #endif - // Remove unmapped octants + // Cells associated with unmapped octants should be removed // // A coarsening that merges cells from different processes, can leave, on // the processes which own the ghost octants involved in the coarsening, // some octants that are not mapped. + // + // During first tree load balance, cells are deleted and not sent to + // other processes (because all processes contain the whole tree). + // These cells were not mapped and need to be deleted. + std::size_t unmappedDeletionInfoId = synchronizationData.insert(adaption::TYPE_DELETION, adaption::ENTITY_CELL); + adaption::Info &unmappedDeletionInfo = synchronizationData[unmappedDeletionInfoId]; for (uint32_t previousTreeId = 0; previousTreeId < nPreviousOctants; ++previousTreeId) { - if (unmappedOctants[previousTreeId]) { - OctantInfo octantInfo = OctantInfo(previousTreeId, true); - long cellId = getOctantId(octantInfo); - deletedOctants.emplace_back(cellId, adaption::TYPE_DELETION); + if (mappedInternalOctants[previousTreeId]) { + continue; } + + // Identify cell id + OctantInfo octantInfo = OctantInfo(previousTreeId, true); + long cellId = getOctantId(octantInfo); + + // Update information on previous status + unmappedDeletionInfo.previous.push_back(cellId); } } + log::cout() << " Done" << std::endl; + // Enable manual adaption AdaptionMode previousAdaptionMode = getAdaptionMode(); setAdaptionMode(ADAPTION_MANUAL); // Renumber cells - renumberCells(renumberedOctants); + renumberCells(synchronizationData); // Remove deleted cells StitchInfo stitchInfo; - if (deletedOctants.size() > 0) { - log::cout() << ">> Removing non-existing cells..."; + if (!importFromScratch) { + log::cout() << " Deleting stale elements..." << std::endl; - // Track changes - // - // The adaption info associated to the octants that has been sent - // to external partitions will contain the current octants sorted by - // their tree id (they were added to the deleted octants list in that - // order), this is the same order that will be used on the process - // that has received the octants. Since the order is the same, the two - // processes are able to exchange cell data without any additional - // extra communication (they already know the list of cells for which - // data is needed and the order in which these data will be sent). - if (trackChanges) { - // Track deleted cells - std::unordered_set sendAdaptionInfo; - std::unordered_set removedInterfaces; - for (const DeleteInfo &deleteInfo : deletedOctants) { - // Cell id - long cellId = deleteInfo.cellId; - - // Adaption tracking - // - // Only cells deleted from a real deletion or a partition send - // needs to be tracked here, the other cells will be tracked - // where the adaption that deleted the cell is tracked. - adaption::Type adaptionType = deleteInfo.trigger; - bool adaptionIsDeletion = (adaptionType == adaption::TYPE_DELETION); - bool adaptionIsSend = (adaptionType == adaption::TYPE_PARTITION_SEND); - - if (adaptionIsDeletion || adaptionIsSend) { - int rank = deleteInfo.rank; - std::size_t adaptionInfoId = adaptionData.insert(adaptionType, adaption::ENTITY_CELL, rank); - adaption::Info &adaptionInfo = adaptionData[adaptionInfoId]; - adaptionInfo.previous.emplace_back(); - long &deletedId = adaptionInfo.previous.back(); - deletedId = cellId; - - // Keep track of adaption info for the send cells - if (adaptionIsSend) { - sendAdaptionInfo.insert(adaptionInfoId); - } - } + stitchInfo = deleteCells(synchronizationData); - // List of deleted interfaces - const Cell &cell = m_cells.at(cellId); - long nCellInterfaces = cell.getInterfaceCount(); - const long *interfaces = cell.getInterfaces(); - for (int k = 0; k < nCellInterfaces; ++k) { - long interfaceId = interfaces[k]; - if (interfaceId >= 0) { - removedInterfaces.insert(interfaceId); - } - } - } + log::cout() << " Stale element successfully deleted." << std::endl; + } -#if BITPIT_ENABLE_MPI==1 - // Sort sent cells - // - // We cannot use native functions to evaluate the position of the - // cells because the octants associated to the cells no longer - // exist on the octree. The cells are still there, therefore we - // can evaluate the cell positions using generic patch functions. - for (int adaptionInfoId : sendAdaptionInfo) { - adaption::Info &adaptionInfo = adaptionData[adaptionInfoId]; - std::sort(adaptionInfo.previous.begin(), adaptionInfo.previous.end(), CellPositionLess(*this, false)); + // Import added cells + log::cout() << " Creating new elements..." << std::endl; + + createCells(stitchInfo, nullptr, &synchronizationData); + + log::cout() << " New elements successfully created." << std::endl; + + StitchInfo().swap(stitchInfo); + + // Restore previous adaption mode + setAdaptionMode(previousAdaptionMode); + + // Remove renumberings from adaption data + // + // Renumbering is only needed for synchronizing the tree with the patch. + synchronizationData.erase(adaption::TYPE_RENUMBERING); + + // Done + log::cout() << " Synchronization completed successfully." << std::endl; + + if (trackChanges) { + return synchronizationData.dump(); + } else { + return std::vector(); + } +} + +/*! + Renumber the specified cells. + + \param cellAdaptionData are the information that describe the changes + that need to be performed to the cells +*/ +void VolOctree::renumberCells(const adaption::InfoCollection &cellAdaptionData) +{ + // Remove old patch-to-tree and tree-to-patch associations + for (const adaption::Info &adaptionInfo : cellAdaptionData) { + if (adaptionInfo.type != adaption::TYPE_RENUMBERING) { + continue; + } + + for (long cellId : adaptionInfo.previous) { + const OctantInfo &previousOctantInfo = getCellOctant(cellId); + if (previousOctantInfo.internal) { + m_octantToCell.erase(previousOctantInfo.id); + } else { + m_ghostToCell.erase(previousOctantInfo.id); } -#endif + } + } - // Adaption info for the deleted interfaces - std::size_t adaptionInfoId = adaptionData.insert(adaption::TYPE_DELETION, adaption::ENTITY_INTERFACE, currentRank); - adaption::Info &adaptionInfo = adaptionData[adaptionInfoId]; - adaptionInfo.previous.reserve(removedInterfaces.size()); - for (long interfaceId : removedInterfaces) { - adaptionInfo.previous.emplace_back(); - long &deletedInterfaceId = adaptionInfo.previous.back(); - deletedInterfaceId = interfaceId; + // Create new patch-to-tree and tree-to-patch associations + std::size_t nOctants = m_tree->getNumOctants(); + for (const adaption::Info &adaptionInfo : cellAdaptionData) { + if (adaptionInfo.type != adaption::TYPE_RENUMBERING) { + continue; + } + + std::size_t nRenumberedCells = adaptionInfo.previous.size(); + for (std::size_t i = 0; i < nRenumberedCells; ++i) { + long cellId = adaptionInfo.previous[i]; + + auto treeIdItr = m_cellToOctant.find(cellId); + if (treeIdItr != m_cellToOctant.end()) { + uint32_t currentTreeId = adaptionInfo.current[i]; + treeIdItr->second = currentTreeId; + m_octantToCell[currentTreeId] = cellId; + } else { + uint32_t currentTreeId = adaptionInfo.current[i] - nOctants; + treeIdItr = m_cellToGhost.find(cellId); + assert(treeIdItr != m_cellToGhost.end()); + treeIdItr->second = currentTreeId; + m_ghostToCell[currentTreeId] = cellId; } } + } +} + +/*! + Imports a list of octants into the patch. + + \param stitchInfo is the list of vertices that will be used to stitch the + the new octants + \param restoreStream if a is an optional stream from which the information about + the restore will be read. If no restore stream is given the the cells will + be created from scratch + \param[in,out] cellAdaptionData are the information that describe the changes + that need to be performed to the cells, on output the dummy ids contained in + the current statuses will be replaced with the actual ids of the cells that + have been created +*/ +void VolOctree::createCells(StitchInfo &stitchInfo, std::istream *restoreStream, + adaption::InfoCollection *cellAdaptionData) +{ + // Tree information + long nOctants = m_tree->getNumOctants(); + long nGhostsOctants = m_tree->getNumGhosts(); + + int nOctantVertices = m_tree->getNnodes(); + + // Create the new vertices + std::vector trackedCreatedVertices; + for (const adaption::Info &adaptionInfo : *cellAdaptionData) { + if (adaptionInfo.type == adaption::TYPE_RENUMBERING) { + continue; + } + + for (long currentId : adaptionInfo.current) { + // Octant + bool isInternal = (currentId < nOctants); + + std::size_t treeId = currentId; + if (!isInternal) { + treeId -= nOctants; + } + + const OctantInfo octantInfo(treeId, isInternal); + Octant *octant = getOctantPointer(octantInfo); + + // Create vertices + for (int k = 0; k < nOctantVertices; ++k) { + uint64_t vertexTreeKey = m_tree->computeNodePersistentKey(octant, k); + if (stitchInfo.count(vertexTreeKey) == 0) { + // Vertex coordinates + std::array nodeCoords = m_tree->getNode(octant, k); - // Delete cells - stitchInfo = deleteCells(deletedOctants); + // Create the vertex + long vertexId; + if (!restoreStream) { +#if BITPIT_ENABLE_MPI==1 + VertexIterator vertexIterator = addVertex(nodeCoords); +#else + VertexIterator vertexIterator = addVertex(nodeCoords); +#endif + vertexId = vertexIterator.getId(); + } else { + utils::binary::read(*restoreStream, vertexId); + +#if BITPIT_ENABLE_MPI==1 + int rank; + utils::binary::read(*restoreStream, rank); - log::cout() << " Done" << std::endl; - log::cout() << ">> Cells removed: " << deletedOctants.size() << std::endl; + restoreVertex(nodeCoords, rank, vertexId); +#else + int dummtRank; + utils::binary::read(*restoreStream, dummtRank); + + restoreVertex(nodeCoords, vertexId); +#endif + } + + // Add the vertex to the stitching info + stitchInfo[vertexTreeKey] = vertexId; + } + } + } } - std::vector().swap(deletedOctants); + // Reserve space for the maps + m_cellToOctant.reserve(nOctants); + m_octantToCell.reserve(nOctants); - // Import added cells - std::vector createdCells; - if (addedOctants.size() > 0) { - log::cout() << ">> Importing new octants..."; + m_cellToGhost.reserve(nGhostsOctants); + m_ghostToCell.reserve(nGhostsOctants); - createdCells = importCells(addedOctants, stitchInfo); + // Add the cells + std::size_t updatedCellCount = m_cells.size(); + for (const adaption::Info &adaptionInfo : *cellAdaptionData) { + if (adaptionInfo.type == adaption::TYPE_RENUMBERING) { + continue; + } - log::cout() << " Done" << std::endl; - log::cout() << ">> Octants imported: " << addedOctants.size() << std::endl; + updatedCellCount += adaptionInfo.current.size(); } + m_cells.reserve(updatedCellCount); - StitchInfo().swap(stitchInfo); + for (adaption::Info &adaptionInfo : *cellAdaptionData) { + if (adaptionInfo.type == adaption::TYPE_RENUMBERING) { + continue; + } - // Restore previous adaption mode - setAdaptionMode(previousAdaptionMode); + for (long ¤tId : adaptionInfo.current) { + // Octant + bool isInternal = (currentId < nOctants); - // Track mesh adaption - if (trackChanges) { - // Complete mesh adaption info for the cells - for (auto &adaptionInfo : adaptionData.data()) { - if (adaptionInfo.entity != adaption::ENTITY_CELL) { - continue; + std::size_t treeId = currentId; + if (!isInternal) { + treeId -= nOctants; } - // Map ids of the added cells - int nCurrentIds = adaptionInfo.current.size(); - for (int k = 0; k < nCurrentIds; ++k) { - long cellId = m_octantToCell.at(adaptionInfo.current[k]); - adaptionInfo.current[k] = cellId; + const OctantInfo octantInfo(treeId, isInternal); + Octant *octant = getOctantPointer(octantInfo); + + // Cell connectivity + std::unique_ptr cellConnect = std::unique_ptr(new long[nOctantVertices]); + for (int k = 0; k < nOctantVertices; ++k) { + uint64_t vertexTreeKey = m_tree->computeNodePersistentKey(octant, k); + cellConnect[k] = stitchInfo.at(vertexTreeKey); } #if BITPIT_ENABLE_MPI==1 - // Sort received cells - // - // To match the sorting done on the procesor that sent the cells, - // we don't use the native functions to evaluate the position of - // the cells. - adaption::Type adaptionType = adaptionInfo.type; - if (adaptionType == adaption::TYPE_PARTITION_RECV) { - std::sort(adaptionInfo.current.begin(), adaptionInfo.current.end(), CellPositionLess(*this, false)); + // Cell owner + int owner; + if (octantInfo.internal) { + owner = getRank(); + } else { + uint64_t globalTreeId = m_tree->getGhostGlobalIdx(octantInfo.id); + owner = m_tree->getOwnerRank(globalTreeId); } + + // Cell halo layer + int haloLayer = m_tree->getGhostLayer(octant); #endif - } + // Add cell + long cellId; + if (!restoreStream) { #if BITPIT_ENABLE_MPI==1 - // Track created ghosts cells - if (nGhostsOctants > 0) { - std::size_t adaptionInfoId = adaptionData.insert(adaption::TYPE_CREATION, adaption::ENTITY_CELL, currentRank); - adaption::Info &adaptionInfo = adaptionData[adaptionInfoId]; + CellIterator cellIterator = addCell(m_cellTypeInfo->type, std::move(cellConnect), owner, haloLayer); +#else + CellIterator cellIterator = addCell(m_cellTypeInfo->type, std::move(cellConnect)); +#endif + cellId = cellIterator.getId(); + } else { + utils::binary::read(*restoreStream, cellId); - adaptionInfo.current.reserve(nGhostsOctants); - auto cellIterator = m_cellToGhost.cbegin(); - while (cellIterator != m_cellToGhost.cend()) { - adaptionInfo.current.emplace_back(); - long &adaptionId = adaptionInfo.current.back(); - adaptionId = cellIterator->first; +#if BITPIT_ENABLE_MPI==1 + restoreCell(m_cellTypeInfo->type, std::move(cellConnect), owner, haloLayer, cellId); +#else + restoreCell(m_cellTypeInfo->type, std::move(cellConnect), cellId); +#endif + } - cellIterator++; + // Create patch-tree associations + if (octantInfo.internal) { + m_cellToOctant.insert({{cellId, octantInfo.id}}); + m_octantToCell.insert({{octantInfo.id, cellId}}); + } else { + m_cellToGhost.insert({{cellId, octantInfo.id}}); + m_ghostToCell.insert({{octantInfo.id, cellId}}); } + + // Replace current id with the actual cell id + currentId = cellId; } -#endif + } - // Track created interfaces - if (createdCells.size() > 0) { - // List of unique interfaces that have been created - std::unordered_set createdInterfaces; - for (const auto &cellId : createdCells) { - const Cell &cell = m_cells.at(cellId); - long nCellInterfaces = cell.getInterfaceCount(); - const long *interfaces = cell.getInterfaces(); - for (int k = 0; k < nCellInterfaces; ++k) { - long interfaceId = interfaces[k]; - if (interfaceId >= 0) { - createdInterfaces.insert(interfaceId); - } - } - } + // Update first ghost and last internal information + if (restoreStream) { + updateLastInternalCellId(); +#if BITPIT_ENABLE_MPI==1 + updateFirstGhostCellId(); +#endif + } - // Adaption info - std::size_t infoId = adaptionData.insert(adaption::TYPE_CREATION, adaption::ENTITY_INTERFACE, currentRank); - adaption::Info &adaptionInfo = adaptionData[infoId]; - adaptionInfo.current.reserve(createdInterfaces.size()); - for (long interfaceId : createdInterfaces) { - adaptionInfo.current.emplace_back(); - long &createdInterfaceId = adaptionInfo.current.back(); - createdInterfaceId = interfaceId; - } + // Update cell PIDs + if (restoreStream) { + for (Cell &cell : getCells()) { + int PID; + utils::binary::read(*restoreStream, PID); + cell.setPID(PID); } } - // Done - return adaptionData.dump(); -} + // Update adjacencies + updateAdjacencies(); + // Update interfaces + if (!restoreStream) { + updateInterfaces(false); + } +} /*! Delete the specified cells. - \param deletedOctants contains a list with the information about the - deleted octants - \result Returns the stitch information that can the faces created - after deleting the octants. + \param cellAdaptionData are the information that describe the changes + that need to be performed to the cells + \param stitchInfo if a valid pointer is provided, on output will contain + the stitch information that can used to stich the faces created after + deleting the octants */ -VolOctree::StitchInfo VolOctree::deleteCells(const std::vector &deletedOctants) +VolOctree::StitchInfo VolOctree::deleteCells(const adaption::InfoCollection &cellAdaptionData) { // Info of the cells int nCellVertices = m_cellTypeInfo->nVertices; int nCellFaces = m_cellTypeInfo->nFaces; - // Delete the cells + // Delete cells + std::vector deadCells; std::unordered_set deadVertices; + for (const adaption::Info &adaptionInfo : cellAdaptionData) { + if (adaptionInfo.type == adaption::TYPE_RENUMBERING) { + continue; + } - std::vector deadCells; - deadCells.reserve(deletedOctants.size()); - for (const DeleteInfo &deleteInfo : deletedOctants) { - long cellId = deleteInfo.cellId; - const Cell &cell = m_cells[cellId]; + // Add previous cells to the list of dead cells + deadCells.insert(deadCells.begin(), adaptionInfo.previous.begin(), adaptionInfo.previous.end()); - // List vertices to remove - // - // For now, all cell vertices will be listed. Later, the vertex of - // the dangling faces will be removed from the list. - ConstProxyVector cellVertexIds = cell.getVertexIds(); - deadVertices.insert(cellVertexIds.cbegin(), cellVertexIds.cend()); + // Prepare cell deletion + for (long cellId : adaptionInfo.previous) { + const Cell &cell = m_cells[cellId]; - // Remove patch-tree associations - const OctantInfo &octantInfo = getCellOctant(cellId); - uint32_t treeId = octantInfo.id; + const OctantInfo &octantInfo = getCellOctant(cellId); + uint32_t treeId = octantInfo.id; - if (cell.isInterior()) { - m_cellToOctant.erase(cellId); + // List vertices to remove + // + // For now, all cell vertices will be listed. Later, the vertex of + // the dangling faces will be removed from the list. + ConstProxyVector cellVertexIds = cell.getVertexIds(); + deadVertices.insert(cellVertexIds.cbegin(), cellVertexIds.cend()); - auto octantToCellItr = m_octantToCell.find(treeId); - if (octantToCellItr->second == cellId) { - m_octantToCell.erase(octantToCellItr); - } - } else { - m_cellToGhost.erase(cellId); + // Remove patch-tree associations + if (cell.isInterior()) { + m_cellToOctant.erase(cellId); + + auto octantToCellItr = m_octantToCell.find(treeId); + if (octantToCellItr->second == cellId) { + m_octantToCell.erase(octantToCellItr); + } + } else { + m_cellToGhost.erase(cellId); - auto ghostToCellItr = m_ghostToCell.find(treeId); - if (ghostToCellItr->second == cellId) { - m_ghostToCell.erase(ghostToCellItr); + auto ghostToCellItr = m_ghostToCell.find(treeId); + if (ghostToCellItr->second == cellId) { + m_ghostToCell.erase(ghostToCellItr); + } } } - - // Cell needs to be removed - deadCells.push_back(cellId); } + log::cout() << " " << deadCells.size() << " cell will be removed." << std::endl; PatchKernel::deleteCells(deadCells); // Prune cell adjacencies and interfaces @@ -1625,7 +1679,9 @@ VolOctree::StitchInfo VolOctree::deleteCells(const std::vector &dele pruneStaleInterfaces(); - // All the vertices belonging to the dangling cells has to be kept + // Delete vertices + // + // All the vertices belonging to the dangling cells has to be kept. // // We need to keep all the vertices that lie on a dangling face and belong // to cells that were not deleted. These are the vertices of the dangling @@ -1648,7 +1704,7 @@ VolOctree::StitchInfo VolOctree::deleteCells(const std::vector &dele // octree numbering of the vertices of the dangling cells. This map will // be used when imprting the octants to stitch the imported octants to // the existing cells. - StitchInfo stitchVertices; + StitchInfo stitchInfo; for (const auto &entry: m_alteredCells) { // Consider only dangling faces if (!testAlterationFlags(entry.second, FLAG_DANGLING)) { @@ -1666,7 +1722,7 @@ VolOctree::StitchInfo VolOctree::deleteCells(const std::vector &dele for (int k = 0; k < nCellVertices; ++k) { long vertexId = cellVertexIds[k]; uint64_t vertexTreeKey = m_tree->computeNodePersistentKey(octant, k); - stitchVertices.insert({vertexTreeKey, vertexId}); + stitchInfo.insert({vertexTreeKey, vertexId}); deadVertices.erase(vertexId); } @@ -1696,209 +1752,18 @@ VolOctree::StitchInfo VolOctree::deleteCells(const std::vector &dele for (std::size_t k = 0; k < nFaceVertexIds; ++k) { long vertexId = faceVertexIds[k]; uint64_t vertexTreeKey = m_tree->computeNodePersistentKey(neighOctant, localNeighFaceConnect[k]); - stitchVertices.insert({vertexTreeKey, vertexId}); + stitchInfo.insert({vertexTreeKey, vertexId}); deadVertices.erase(vertexId); } } } } - // Delete the vertices + log::cout() << " " << deadVertices.size() << " vertices will be removed." << std::endl; PatchKernel::deleteVertices(deadVertices); // Done - return stitchVertices; -} - -/*! - Renumber the specified cells. - - \param renumberedOctants contains the information about the renumbered - octants -*/ -void VolOctree::renumberCells(const std::vector &renumberedOctants) -{ - // Remove old patch-to-tree and tree-to-patch associations - for (const RenumberInfo &renumberInfo : renumberedOctants) { - long cellId = renumberInfo.cellId; - if (!m_cells[cellId].isInterior()) { - continue; - } - - const OctantInfo &previousOctantInfo = getCellOctant(cellId); - uint32_t previousTreeId = previousOctantInfo.id; - - m_octantToCell.erase(previousTreeId); - } - - // Create new patch-to-tree and tree-to-patch associations - for (const RenumberInfo &renumberInfo : renumberedOctants) { - long cellId = renumberInfo.cellId; - if (!m_cells[cellId].isInterior()) { - continue; - } - - uint32_t treeId = renumberInfo.newTreeId; - - m_cellToOctant[cellId] = treeId; - m_octantToCell[treeId] = cellId; - } -} - -/*! - Imports a list of octants into the patch. - - \param octantInfoList is the list of octant to import - \param stitchInfo is the list of vertices that will be used to stitch the - the new octants - \param restoreStream is an optional stream from which the information about - the restore will be read. If no restore stream is given the the cells will - be created from scratch -*/ -std::vector VolOctree::importCells(const std::vector &octantInfoList, - StitchInfo &stitchInfo, std::istream *restoreStream) -{ - // Create the new vertices - int nCellVertices = m_cellTypeInfo->nVertices; - for (const OctantInfo &octantInfo : octantInfoList) { - Octant *octant = getOctantPointer(octantInfo); - for (int k = 0; k < nCellVertices; ++k) { - uint64_t vertexTreeKey = m_tree->computeNodePersistentKey(octant, k); - if (stitchInfo.count(vertexTreeKey) == 0) { - // Vertex coordinates - std::array nodeCoords = m_tree->getNode(octant, k); - - // Create the vertex - long vertexId; - if (!restoreStream) { -#if BITPIT_ENABLE_MPI==1 - VertexIterator vertexIterator = addVertex(std::move(nodeCoords)); -#else - VertexIterator vertexIterator = addVertex(std::move(nodeCoords)); -#endif - vertexId = vertexIterator.getId(); - } else { - utils::binary::read(*restoreStream, vertexId); - -#if BITPIT_ENABLE_MPI==1 - int rank; - utils::binary::read(*restoreStream, rank); - - restoreVertex(nodeCoords, rank, vertexId); -#else - int dummyRank; - utils::binary::read(*restoreStream, dummyRank); - - restoreVertex(nodeCoords, vertexId); -#endif - } - - // Add the vertex to the stitching info - stitchInfo[vertexTreeKey] = vertexId; - } - } - } - - // Reserve space for the maps - long nOctants = m_tree->getNumOctants(); - m_cellToOctant.reserve(nOctants); - m_octantToCell.reserve(nOctants); - - long nGhostsOctants = m_tree->getNumGhosts(); - m_cellToGhost.reserve(nGhostsOctants); - m_ghostToCell.reserve(nGhostsOctants); - - // Add the cells - size_t octantInfoListSize = octantInfoList.size(); - m_cells.reserve(m_cells.size() + octantInfoListSize); - - std::vector createdCells(octantInfoListSize); - for (size_t i = 0; i < octantInfoListSize; ++i) { - const OctantInfo &octantInfo = octantInfoList[i]; - - // Octant connectivity - Octant *octant = getOctantPointer(octantInfo); - - // Cell connectivity - std::unique_ptr cellConnect = std::unique_ptr(new long[nCellVertices]); - for (int k = 0; k < nCellVertices; ++k) { - uint64_t vertexTreeKey = m_tree->computeNodePersistentKey(octant, k); - cellConnect[k] = stitchInfo.at(vertexTreeKey); - } - -#if BITPIT_ENABLE_MPI==1 - // Cell owner - int owner; - if (octantInfo.internal) { - owner = getRank(); - } else { - uint64_t globalTreeId = m_tree->getGhostGlobalIdx(octantInfo.id); - owner = m_tree->getOwnerRank(globalTreeId); - } - - // Cell halo layer - int haloLayer = m_tree->getGhostLayer(octant); -#endif - - // Add cell - long cellId; - if (!restoreStream) { -#if BITPIT_ENABLE_MPI==1 - CellIterator cellIterator = addCell(m_cellTypeInfo->type, std::move(cellConnect), owner, haloLayer); -#else - CellIterator cellIterator = addCell(m_cellTypeInfo->type, std::move(cellConnect)); -#endif - cellId = cellIterator.getId(); - } else { - utils::binary::read(*restoreStream, cellId); - -#if BITPIT_ENABLE_MPI==1 - restoreCell(m_cellTypeInfo->type, std::move(cellConnect), owner, haloLayer, cellId); -#else - restoreCell(m_cellTypeInfo->type, std::move(cellConnect), cellId); -#endif - } - - // Create patch-tree associations - if (octantInfo.internal) { - m_cellToOctant.insert({{cellId, octantInfo.id}}); - m_octantToCell.insert({{octantInfo.id, cellId}}); - } else { - m_cellToGhost.insert({{cellId, octantInfo.id}}); - m_ghostToCell.insert({{octantInfo.id, cellId}}); - } - - // Add the cell to the list of created cells - createdCells[i] = cellId; - } - - // Update first ghost and last internal information - if (restoreStream) { - updateLastInternalCellId(); -#if BITPIT_ENABLE_MPI==1 - updateFirstGhostCellId(); -#endif - } - - // Update cell PIDs - if (restoreStream) { - for (Cell &cell : getCells()) { - int PID; - utils::binary::read(*restoreStream, PID); - cell.setPID(PID); - } - } - - // Update adjacencies - updateAdjacencies(); - - // Update interfaces - if (!restoreStream) { - updateInterfaces(); - } - - // Done - return createdCells; + return stitchInfo; } /*! @@ -2331,16 +2196,13 @@ void VolOctree::_restore(std::istream &stream) StitchInfo stitchInfo; - std::vector octantInfoList; - octantInfoList.reserve(nOctants + nGhostsOctants); - for (std::size_t n = 0; n < nOctants; ++n) { - octantInfoList.emplace_back(n, true); - } - for (std::size_t n = 0; n < nGhostsOctants; ++n) { - octantInfoList.emplace_back(n, false); - } + adaption::InfoCollection adaptionData; + std::size_t adaptionId = adaptionData.insert(adaption::TYPE_CREATION, adaption::ENTITY_CELL); + adaption::Info &adaptionInfo = adaptionData[adaptionId]; + adaptionInfo.current.resize(nOctants + nGhostsOctants); + std::iota(adaptionInfo.current.begin(), adaptionInfo.current.end(), 0); - importCells(octantInfoList, stitchInfo, &stream); + createCells(stitchInfo, &stream, &adaptionData); // Restore interfaces restoreInterfaces(stream); diff --git a/src/voloctree/voloctree.hpp b/src/voloctree/voloctree.hpp index 0ce5910532..80455b39a6 100644 --- a/src/voloctree/voloctree.hpp +++ b/src/voloctree/voloctree.hpp @@ -260,9 +260,11 @@ class VolOctree : public VolumeKernel { OctantHash evaluateOctantHash(const OctantInfo &octantInfo); - StitchInfo deleteCells(const std::vector &deletedOctants); - void renumberCells(const std::vector &renumberedOctants); - std::vector importCells(const std::vector &octantTreeIds, StitchInfo &stitchInfo, std::istream *stream = nullptr); + void renumberCells(const adaption::InfoCollection &treeAdaptionData); + + void createCells(StitchInfo &stitchInfo, std::istream *stream, adaption::InfoCollection *cellAdaptionData); + + StitchInfo deleteCells(const adaption::InfoCollection &cellAdaptionData); std::vector sync(bool trackChanges); From 3510f6888e33795fb25829a6a6cc164b331f5d85 Mon Sep 17 00:00:00 2001 From: Andrea Iob Date: Mon, 23 Jan 2023 11:49:47 +0100 Subject: [PATCH 13/15] patchkernel: add tracking of ghost cells during adaptation/partitioning --- src/patchkernel/patch_kernel.cpp | 25 +- src/patchkernel/patch_kernel.hpp | 2 +- src/patchkernel/patch_kernel_parallel.cpp | 326 +++++++++++++++++----- src/voloctree/voloctree.cpp | 11 +- src/voloctree/voloctree_parallel.cpp | 4 + 5 files changed, 291 insertions(+), 77 deletions(-) diff --git a/src/patchkernel/patch_kernel.cpp b/src/patchkernel/patch_kernel.cpp index ea44b462fd..09782facd0 100644 --- a/src/patchkernel/patch_kernel.cpp +++ b/src/patchkernel/patch_kernel.cpp @@ -710,6 +710,13 @@ std::vector PatchKernel::adaption(bool trackAdaption, bool squee track the changes that will be performed to the patch. During this phase no changes will be performed to the patch. + Tracking will only list changes that involve data collection on the internal + data structures (e.g., adaption preparation will not track adaption of ghost + cells). + + Information available for tracking purposes are the following: + - internal cells that will be coarsened/refined. + \param trackAdaption if set to true the function will return the changes that will be performed in the alter step \result If the adaption is tracked, returns a vector of adaption::Info that @@ -747,7 +754,17 @@ std::vector PatchKernel::adaptionPrepare(bool trackAdaption) The actual modification of the patch takes place during this phase. After this phase the adaption is completed and the patch is in its final state. - Optionally the patch can track the changes performed to the patch. + + Optionally, this funciton can track the changes performed to the patch. + + Tracking information will not contain changes that involve data collection + on ghost data structures (e.g., ghost cells adaption is not tracked, only + deletion and creation of ghost cells will be tracked). + + Information available for tracking purposes are the following: + - internal cells that have been coarsened/refined; + - new ghost cells that have been created; + - ghost cells that have been deleted. \param trackAdaption if set to true the function will return the changes done to the patch during the adaption @@ -5183,6 +5200,9 @@ void PatchKernel::restoreInterfaces(std::istream &stream) Default implementation is a no-op function. + See PatchKernel::adaptionPrepare(bool trackPartitioning) for the + documentation about the tracking information returned by this function. + \param trackAdaption if set to true the function will return the changes that will be performed in the alter step \result If the adaption is tracked, returns a vector of adaption::Info that @@ -5201,6 +5221,9 @@ std::vector PatchKernel::_adaptionPrepare(bool trackAdaption) Default implementation is a no-op function. + See PatchKernel::adaptionAlter(bool trackPartitioning) for the + documentation about the tracking information returned by this function. + \param trackAdaption if set to true the function will return the changes done to the patch during the adaption \result If the adaption is tracked, returns a vector of adaption::Info diff --git a/src/patchkernel/patch_kernel.hpp b/src/patchkernel/patch_kernel.hpp index 29340765c8..507302da7f 100644 --- a/src/patchkernel/patch_kernel.hpp +++ b/src/patchkernel/patch_kernel.hpp @@ -1061,7 +1061,7 @@ friend class PatchManager; void computeCellHaloLayer(int id); - void _partitioningAlter_deleteGhosts(); + std::vector _partitioningAlter_deleteGhosts(bool trackPartitioning); std::unordered_map _partitioningAlter_evalGhostCellOwnershipChanges(); void _partitioningAlter_applyGhostCellOwnershipChanges(int sendRank, std::unordered_map *ghostCellOwnershipChanges); diff --git a/src/patchkernel/patch_kernel_parallel.cpp b/src/patchkernel/patch_kernel_parallel.cpp index 5e80c4ece1..903782cd50 100644 --- a/src/patchkernel/patch_kernel_parallel.cpp +++ b/src/patchkernel/patch_kernel_parallel.cpp @@ -1303,6 +1303,11 @@ void PatchKernel::updateFirstGhostCellId() Partitions the patch among the processes. Each cell will be assigned to a specific process according to the specified input. + Optionally, this funciton can track the changes performed to the patch. See + PatchKernel::partitioningAlter(bool trackPartitioning, bool squeezeStorage) + for the documentation about the tracking information returned by this + function. + \param communicator is the communicator that will be used \param cellRanks are the ranks of the cells after the partitioning \param trackPartitioning if set to true, the changes to the patch will be @@ -1327,6 +1332,11 @@ std::vector PatchKernel::partition(MPI_Comm communicator, const Partitions the patch among the processes. Each cell will be assigned to a specific process according to the specified input. + Optionally, this funciton can track the changes performed to the patch. See + PatchKernel::partitioningAlter(bool trackPartitioning, bool squeezeStorage) + for the documentation about the tracking information returned by this + function. + \param cellRanks are the ranks of the cells after the partitioning \param trackPartitioning if set to true, the changes to the patch will be tracked @@ -1350,6 +1360,11 @@ std::vector PatchKernel::partition(const std::unordered_map PatchKernel::partition(MPI_Comm communicator, bool t Partitions the patch among the processes. The partitioning is done using a criteria that tries to balance the load among the processes. + Optionally, this funciton can track the changes performed to the patch. See + PatchKernel::partitioningAlter(bool trackPartitioning, bool squeezeStorage) + for the documentation about the tracking information returned by this + function. + \param trackPartitioning if set to true, the changes to the patch will be tracked \param squeezeStorage if set to true the vector that store patch information @@ -1399,6 +1419,11 @@ std::vector PatchKernel::partition(bool trackPartitioning, bool Partitions the patch among the processes. Each cell will be assigned to a specific process according to the specified input. + Optionally, this funciton can track the changes performed to the patch. See + PatchKernel::partitioningAlter(bool trackPartitioning, bool squeezeStorage) + for the documentation about the tracking information returned by this + function. + \param communicator is the communicator that will be used \param cellWeights are the weights of the cells, the weight represents the relative computational cost associated with a specified cell. If no weight @@ -1425,6 +1450,11 @@ std::vector PatchKernel::partition(MPI_Comm communicator, const Partitions the patch among the processes. Each cell will be assigned to a specific process according to the specified input. + Optionally, this funciton can track the changes performed to the patch. See + PatchKernel::partitioningAlter(bool trackPartitioning, bool squeezeStorage) + for the documentation about the tracking information returned by this + function. + \param cellWeights are the weights of the cells, the weight represents the relative computational cost associated with a specified cell. If no weight is specified for a cell, a weight equal to one is used @@ -1450,6 +1480,9 @@ std::vector PatchKernel::partition(const std::unordered_map PatchKernel::partitioningPrepare(MPI_Comm communicat Partitions the patch among the processes. Each cell will be assigned to a specific process according to the specified input. + See PatchKernel::partitioningPrepare(bool trackPartitioning) for the + documentation about the tracking information returned by this function. + \param cellRanks are the ranks of the cells after the partitioning \param trackPartitioning if set to true, the changes to the patch will be tracked @@ -1508,6 +1544,9 @@ std::vector PatchKernel::partitioningPrepare(const std::unordere Partitions the patch among the processes. The partitioning is done using a criteria that tries to balance the load among the processes. + See PatchKernel::partitioningPrepare(bool trackPartitioning) for the + documentation about the tracking information returned by this function. + \param communicator is the communicator that will be used \param trackPartitioning if set to true, the changes to the patch will be tracked @@ -1531,6 +1570,16 @@ std::vector PatchKernel::partitioningPrepare(MPI_Comm communicat Partitions the patch among the processes. The partitioning is done using a criteria that tries to balance the load among the processes. + Tracking will only list changes that involve data exchange among internal + data structured (e.g., partitioning preparation will not track ghost cells + nor cell deletion). + + Information available on the sender side for tracking purposes are the + following: + - internal cells that will be send. + + No information about tracking are provided on the receiver side. + \param trackPartitioning if set to true, the changes to the patch will be tracked \result Returns a vector of adaption::Info that can be used to track @@ -1547,6 +1596,9 @@ std::vector PatchKernel::partitioningPrepare(bool trackPartition Partitions the patch among the processes. The partitioning is done using a criteria that tries to balance the load among the processes. + See PatchKernel::partitioningPrepare(bool trackPartitioning) for the + documentation about the tracking information returned by this function. + \param communicator is the communicator that will be used \param cellWeights are the weights of the cells, the weight represents the relative computational cost associated with a specified cell. If no weight @@ -1571,6 +1623,9 @@ std::vector PatchKernel::partitioningPrepare(MPI_Comm communicat Partitions the patch among the processes. The partitioning is done using a criteria that tries to balance the load among the processes. + See PatchKernel::partitioningPrepare(bool trackPartitioning) for the + documentation about the tracking information returned by this function. + \param cellWeights are the weights of the cells, the weight represents the relative computational cost associated with a specified cell. If no weight is specified for a cell, a weight equal to one is used @@ -1606,7 +1661,30 @@ std::vector PatchKernel::partitioningPrepare(const std::unordere The actual modification of the patch takes place during this phase. After this phase the adaption is completed and the patch is in its final state. - Optionally the patch can track the changes performed to the patch. + + Optionally, this funciton can track the changes performed to the patch. + + Tracking information will not contain changes that involve data exchange + among ghost data structured (e.g., ghost cells exchanges are not tracked, + only deletion and creation of ghost cells will be tracked). + + Multiple adaption information can be associated to a single element, for + example a cell can be sent to another processor and then "recreated" to + become a ghost cell. + + Information available on the sender side for tracking purposes are the + following: + - internal cells that have been sent; + - new ghost cells that have been created (some of the internal cells that + have been sent may have become ghosts cells); + - ghost cells that have been deleted. + + Information available on the receiver side for tracking purposes are the + following: + - internal cells that have been received; + - new ghost cells that have been created; + - ghost cells that have been deleted (some ghost cells may have been + replaced by internal cells that have just been received). \param trackPartitioning if set to true the function will return the changes done to the patch during the partitioning @@ -1833,6 +1911,10 @@ double PatchKernel::evalPartitioningUnbalance(const std::unordered_map PatchKernel::_partitioningPrepare(const std::unorder Partitions the patch among the processes. Each cell will be assigned to a specific process according to the specified input. + See PatchKernel::partitioningPrepare(bool trackPartitioningge) for the + documentation about the tracking information returned by this function. + \param cellRanks are the ranks of the cells after the partitioning \param trackPartitioning if set to true, the changes to the patch will be tracked @@ -2055,20 +2140,21 @@ std::vector PatchKernel::_partitioningPrepare(const std::unorder MPI_Allreduce(MPI_IN_PLACE, &m_partitioningSerialization, 1, MPI_C_BOOL, MPI_LAND, m_communicator); } - // Build the information on the cells that will be sent - // - // Only internal cells are tracked. + // Track changes that involve data exchange std::vector partitioningData; if (trackPartitioning) { + std::vector cellsToSend; for (const std::pair &entry : m_partitioningGlobalExchanges) { + // Get send/receive ranks int sendRank = entry.first; if (sendRank != patchRank) { continue; } + // Get list of cells to be sent int recvRank = entry.second; - std::vector previous; + cellsToSend.clear(); for (const auto &entry : m_partitioningOutgoings) { int cellRank = entry.second; if (cellRank != recvRank) { @@ -2076,17 +2162,20 @@ std::vector PatchKernel::_partitioningPrepare(const std::unorder } long cellId = entry.first; - previous.push_back(cellId); + cellsToSend.push_back(cellId); } - if (!previous.empty()) { - partitioningData.emplace_back(); - adaption::Info &partitioningInfo = partitioningData.back(); - partitioningInfo.entity = adaption::ENTITY_CELL; - partitioningInfo.type = adaption::TYPE_PARTITION_SEND; - partitioningInfo.rank = recvRank; - partitioningInfo.previous = std::move(previous); + if (cellsToSend.empty()) { + continue; } + + // Fill tracking data structures + partitioningData.emplace_back(); + adaption::Info &partitioningCellInfo = partitioningData.back(); + partitioningCellInfo.entity = adaption::ENTITY_CELL; + partitioningCellInfo.type = adaption::TYPE_PARTITION_SEND; + partitioningCellInfo.rank = recvRank; + partitioningCellInfo.previous = std::move(cellsToSend); } } @@ -2096,6 +2185,9 @@ std::vector PatchKernel::_partitioningPrepare(const std::unorder /*! Alter the patch performing the partitioning. + See PatchKernel::partitioningAlter(bool trackPartitioning, bool squeezeStorage) + for the documentation about the tracking information returned by this function. + \param trackPartitioning if set to true the function will return the changes done to the patch during the partitioning \result If the partitioning is tracked, returns a vector of adaption::Info @@ -2104,6 +2196,8 @@ std::vector PatchKernel::_partitioningPrepare(const std::unorder */ std::vector PatchKernel::_partitioningAlter(bool trackPartitioning) { + std::vector partitioningData; + // Adjacencies need to be up-to-date updateAdjacencies(); @@ -2120,7 +2214,7 @@ std::vector PatchKernel::_partitioningAlter(bool trackPartitioni ghostCellOwnershipChanges = _partitioningAlter_evalGhostCellOwnershipChanges(); } } else { - _partitioningAlter_deleteGhosts(); + mergeAdaptionInfo(_partitioningAlter_deleteGhosts(trackPartitioning), partitioningData); } // Communicate patch data structures @@ -2132,7 +2226,6 @@ std::vector PatchKernel::_partitioningAlter(bool trackPartitioni std::unordered_set batchSendRanks; std::unordered_set batchRecvRanks; - std::vector partitioningData; std::vector rankPartitioningData; while (!m_partitioningGlobalExchanges.empty()) { // Add exchanges to the batch @@ -2306,9 +2399,35 @@ void PatchKernel::_partitioningAlter_applyGhostCellOwnershipChanges(int sendRank /*! Delete ghosts. + + See PatchKernel::partitioningAlter(bool trackPartitioning, bool squeezeStorage) + for the documentation about the tracking information returned by this function. + + \param trackPartitioning if set to true the function will return the changes + done to the patch during the partitioning + \result If the partitioning is tracked, returns a vector of adaption::Info + with all the changes done to the patch during the partitioning, otherwise + an empty vector will be returned. */ -void PatchKernel::_partitioningAlter_deleteGhosts() +std::vector PatchKernel::_partitioningAlter_deleteGhosts(bool trackPartitioning) { + std::vector partitioningData; + if (getGhostCellCount() == 0) { + return partitioningData; + } + + // Track changes + if (trackPartitioning) { + partitioningData.emplace_back(); + adaption::Info &partitioningCellInfo = partitioningData.back(); + partitioningCellInfo.entity = adaption::ENTITY_CELL; + partitioningCellInfo.type = adaption::TYPE_DELETION; + partitioningCellInfo.current.reserve(getGhostCellCount()); + for (CellConstIterator itr = ghostCellConstBegin(); itr != ghostCellConstEnd(); ++itr) { + partitioningCellInfo.current.push_back(itr.getId()); + } + } + // Delete ghost cells std::vector cellsDeleteList; cellsDeleteList.reserve(m_ghostCellInfo.size()); @@ -2334,11 +2453,16 @@ void PatchKernel::_partitioningAlter_deleteGhosts() while (getGhostVertexCount() > 0) { ghostVertex2InternalVertex(m_firstGhostVertexId); } + + return partitioningData; } /*! Sends the given list of cells to the process with the specified rank. + See PatchKernel::partitioningAlter(bool trackPartitioning, bool squeezeStorage) + for the documentation about the tracking information returned by this function. + \param recvRanks are the receiver ranks \param trackPartitioning if set to true the function will return the changes done to the patch during the partitioning @@ -2734,6 +2858,24 @@ std::vector PatchKernel::_partitioningAlter_sendCells(const std: ghostHaloCellsOverall.insert(cellId); } + // Track changes that involve data exchange + if (trackPartitioning && !cellSendList.empty()) { + // Track cells that have been sent to other processes + // + // The ids of the cells send will be stored accordingly to the send + // order, this is the same order that will be used on the process + // that has received the cell. Since the order is the same, the + // two processes are able to exchange cell data without additional + // communications (they already know the list of cells for which + // data is needed and the order in which these data will be sent). + partitioningData.emplace_back(); + adaption::Info &partitioningCellInfo = partitioningData.back(); + partitioningCellInfo.entity = adaption::ENTITY_CELL; + partitioningCellInfo.type = adaption::TYPE_PARTITION_SEND; + partitioningCellInfo.rank = recvRank; + partitioningCellInfo.previous = std::vector(cellSendList.begin(), cellSendList.begin() + nOutgoingCells); + } + // // Communicate cell buffer size // @@ -2918,31 +3060,6 @@ std::vector PatchKernel::_partitioningAlter_sendCells(const std: MPI_Isend(cellsBuffer.data(), cellsBuffer.getSize(), MPI_CHAR, recvRank, m_partitioningCellsTag, m_communicator, &cellsRequest); - // - // Update adaption info - // - if (trackPartitioning) { - // Track cells that have been sent - // - // The ids of the cells send will be stored accordingly to the send - // order, this is the same order that will be used on the process - // that has received the cell. Since the order is the same, the - // two processes are able to exchange cell data without additional - // communications (they already know the list of cells for which - // data is needed and the order in which these data will be sent). - partitioningData.emplace_back(); - adaption::Info &partitioningInfo = partitioningData.back(); - partitioningInfo.entity = adaption::ENTITY_CELL; - partitioningInfo.type = adaption::TYPE_PARTITION_SEND; - partitioningInfo.rank = recvRank; - - partitioningInfo.previous.resize(nOutgoingCells); - for (std::size_t i = 0; i < nOutgoingCells; ++i) { - long cellId = cellSendList[i]; - partitioningInfo.previous[i] = cellId; - } - } - // Delete outgoing cells not in the frame // // Frame cells cannot be deleted just now, because they may be ghost cells @@ -2988,13 +3105,12 @@ std::vector PatchKernel::_partitioningAlter_sendCells(const std: // If the process is sending all its cells we can just clear the patch. if (!sendingAllCells) { - std::vector deleteList; - // Convert inner cells into ghosts // // All cells on the inner frontier should become ghost cells. These // cells belongs to the first halo layer and defines the seeds for // growing the subsequent halo layers. + std::vector trackedCreatedGhostCells; for (long cellId : innerFrontierCellsOverall) { const Cell &cell = getCell(cellId); std::size_t cellHaloLayer = 0; @@ -3004,26 +3120,40 @@ std::vector PatchKernel::_partitioningAlter_sendCells(const std: int cellOwner = m_partitioningOutgoings.at(cellId); internalCell2GhostCell(cellId, cellOwner, cellHaloLayer); + if (trackPartitioning) { + trackedCreatedGhostCells.push_back(cellId); + } } auto ghostFrameSelector = [this](long cellId) { return (m_partitioningOutgoings.count(cellId) != 0); }; - auto ghostFrameBuilder = [this](long cellId, int layer) { + auto ghostFrameBuilder = [this, trackPartitioning, &trackedCreatedGhostCells](long cellId, int layer) { int cellOwner = m_partitioningOutgoings.at(cellId); internalCell2GhostCell(cellId, cellOwner, layer + 1); + if (trackPartitioning) { + trackedCreatedGhostCells.push_back(cellId); + } return false; }; processCellsNeighbours(innerFrontierCellsOverall, m_haloSize - 1, ghostFrameSelector, ghostFrameBuilder); + if (!trackedCreatedGhostCells.empty()) { + partitioningData.emplace_back(); + adaption::Info &cellCreationInfo = partitioningData.back(); + cellCreationInfo.entity = adaption::ENTITY_CELL; + cellCreationInfo.type = adaption::TYPE_CREATION; + cellCreationInfo.rank = getRank(); + cellCreationInfo.current = getOrderedCellsVertices(trackedCreatedGhostCells, false, true); + } // Delete frame cells that are not ghosts // // Now that the new ghosts have been created, we can delete all the // frontier cells that are not ghosts. - deleteList.clear(); + std::vector deleteList; for (long cellId : frameCellsOverall) { const Cell &cell = getCell(cellId); if (!cell.isInterior()) { @@ -3055,8 +3185,10 @@ std::vector PatchKernel::_partitioningAlter_sendCells(const std: std::vector neighIds; std::vector updateList; + std::vector haloDeleteList; + std::vector trackedDeletedGhostCells; for (int haloLayer = 0; haloLayer < static_cast(m_haloSize); ++haloLayer) { - deleteList.clear(); + haloDeleteList.clear(); for (long cellId : ghostHaloCellsOverall) { // Consider only cells in the current halo layer const GhostCellInfo &ghostInfo = m_ghostCellInfo.at(cellId); @@ -3070,21 +3202,37 @@ std::vector PatchKernel::_partitioningAlter_sendCells(const std: if (confirmCellHaloLayer(cell, haloLayer, m_partitioningOutgoings)) { updateList.push_back(cellId); } else { - deleteList.push_back(cellId); + haloDeleteList.push_back(cellId); } } // Delete stale ghost in this layer - for (long cellId : deleteList) { + for (long cellId : haloDeleteList) { ghostCellOwnershipChanges->erase(cellId); ghostHaloCellsOverall.erase(cellId); } - deleteCells(deleteList); + deleteCells(haloDeleteList); + + // Track deleted cells + if (trackPartitioning) { + trackedDeletedGhostCells.insert(trackedDeletedGhostCells.end(), haloDeleteList.begin(), haloDeleteList.end()); + } // Prune stale adjacencies pruneStaleAdjacencies(); } + // Track ghost cell deletion + if (trackPartitioning && !trackedDeletedGhostCells.empty()) { + partitioningData.emplace_back(); + adaption::Info &cellDeletionInfo = partitioningData.back(); + cellDeletionInfo.entity = adaption::ENTITY_CELL; + cellDeletionInfo.type = adaption::TYPE_DELETION; + cellDeletionInfo.rank = getRank(); + cellDeletionInfo.current = std::move(trackedDeletedGhostCells); + trackedDeletedGhostCells.clear(); + } + // Compute layer associated with ghosts in the halo of the outgoing cells // // Some cells have been deleted because they are now on a different processor, @@ -3100,6 +3248,18 @@ std::vector PatchKernel::_partitioningAlter_sendCells(const std: // Delete orphan vertices deleteOrphanVertices(); } else { + // All ghost cells will be deleted + if (trackPartitioning) { + partitioningData.emplace_back(); + adaption::Info &partitioningCellInfo = partitioningData.back(); + partitioningCellInfo.entity = adaption::ENTITY_CELL; + partitioningCellInfo.type = adaption::TYPE_DELETION; + partitioningCellInfo.current.reserve(getGhostCellCount()); + for (CellConstIterator itr = ghostCellConstBegin(); itr != ghostCellConstEnd(); ++itr) { + partitioningCellInfo.current.push_back(itr.getId()); + } + } + // The process has sent all its cells, the patch is now empty reset(); ghostCellOwnershipChanges->clear(); @@ -3128,6 +3288,9 @@ std::vector PatchKernel::_partitioningAlter_sendCells(const std: /*! Receives a list of cells from the specified process. + See PatchKernel::partitioningAlter(bool trackPartitioning, bool squeezeStorage) + for the documentation about the tracking information returned by this function. + \param sendRanks are the rank of the processes sending the cells \param trackPartitioning if set to true the function will return the changes done to the patch during the partitioning @@ -3179,19 +3342,8 @@ std::vector PatchKernel::_partitioningAlter_receiveCells(const s std::vector> cellsBuffers(nSendRanks); std::vector> verticesBuffers(nSendRanks); - // Fill partitioning info + // Initialize partitioning info std::vector partitioningData; - if (trackPartitioning) { - partitioningData.resize(nSendRanks); - for (int sendRank : sendRanks) { - int sendRankIndex = sendRankIndexes.at(sendRank); - - adaption::Info &partitioningInfo = partitioningData[sendRankIndex]; - partitioningInfo.entity = adaption::ENTITY_CELL; - partitioningInfo.type = adaption::TYPE_PARTITION_RECV; - partitioningInfo.rank = sendRank; - } - } // Mark border interfaces of ghost cells as dangling // @@ -3485,9 +3637,10 @@ std::vector PatchKernel::_partitioningAlter_receiveCells(const s reserveCells(getCellCount() + nReceivedCells); + std::vector trackedReceivedInteriorCells; + std::vector trackedCreatedGhostCells; if (trackPartitioning) { - // Only internal cells are tracked. - partitioningData[sendRankIndex].current.reserve(nReceivedInternalCells); + trackedReceivedInteriorCells.reserve(nReceivedInternalCells); } validReceivedAdjacencies.clear(); @@ -3549,12 +3702,14 @@ std::vector PatchKernel::_partitioningAlter_receiveCells(const s } } + bool cellAlreadyExists = (cellId >= 0); + // If the cell is not a duplicate add it in the cell data structure, // otherwise merge the connectivity of the duplicate cell to the // existing cell. This ensure that the received cell will be // properly connected to the received cells - bool isTracked = false; - if (cellId < 0) { + CellIterator cellIterator; + if (!cellAlreadyExists) { // Reset the interfaces of the cell if (getInterfacesBuildStrategy() != INTERFACES_NONE) { cell.resetInterfaces(); @@ -3574,7 +3729,7 @@ std::vector PatchKernel::_partitioningAlter_receiveCells(const s throw std::runtime_error("A cell with the same id of the received cell already exists."); } - CellIterator cellIterator = addCell(std::move(cell), cellOwner, cellHaloLayer); + cellIterator = addCell(std::move(cell), cellOwner, cellHaloLayer); cellId = cellIterator.getId(); addedCells.push_back(cellId); @@ -3584,16 +3739,12 @@ std::vector PatchKernel::_partitioningAlter_receiveCells(const s ghostCellOwnershipChanges->insert({cellId, ghostCellOwnershipChange}); } } - - // Interior cells are tacked - isTracked = (trackPartitioning && isInterior); } else { // Check if the existing cells needs to become an internal cell - const Cell &localCell = m_cells[cellId]; - if (isInterior && !localCell.isInterior()) { + cellIterator = getCellIterator(cellId); + if (isInterior && !cellIterator->isInterior()) { ghostCell2InternalCell(cellId); ghostCellOwnershipChanges->erase(cellId); - isTracked = trackPartitioning; } // Update the halo layer associated with ghost cells @@ -3643,8 +3794,12 @@ std::vector PatchKernel::_partitioningAlter_receiveCells(const s // additional communications (they already know the list of cells // for which data is needed and the order in which these data will // be sent). - if (isTracked) { - partitioningData[sendRankIndex].current.emplace_back(cellId); + if (trackPartitioning) { + if (cellIterator->isInterior()) { + trackedReceivedInteriorCells.emplace_back(cellId); + } else if (!cellAlreadyExists) { + trackedCreatedGhostCells.emplace_back(cellId); + } } } @@ -3747,6 +3902,29 @@ std::vector PatchKernel::_partitioningAlter_receiveCells(const s } } + // Track changes + if (trackPartitioning) { + if (!trackedReceivedInteriorCells.empty()) { + partitioningData.emplace_back(); + adaption::Info &cellRecvInfo = partitioningData.back(); + cellRecvInfo.entity = adaption::ENTITY_CELL; + cellRecvInfo.type = adaption::TYPE_PARTITION_RECV; + cellRecvInfo.rank = sendRank; + cellRecvInfo.current = std::move(trackedReceivedInteriorCells); + trackedReceivedInteriorCells.clear(); + } + + if (!trackedCreatedGhostCells.empty()) { + partitioningData.emplace_back(); + adaption::Info &cellCreationInfo = partitioningData.back(); + cellCreationInfo.entity = adaption::ENTITY_CELL; + cellCreationInfo.type = adaption::TYPE_CREATION; + cellCreationInfo.rank = patchRank; + cellCreationInfo.current = std::move(trackedCreatedGhostCells); + trackedCreatedGhostCells.clear(); + } + } + // Receive is now complete awaitingSendRanks.erase(sendRank); } diff --git a/src/voloctree/voloctree.cpp b/src/voloctree/voloctree.cpp index 5e68f9d254..a793605772 100644 --- a/src/voloctree/voloctree.cpp +++ b/src/voloctree/voloctree.cpp @@ -929,7 +929,8 @@ int VolOctree::getCellFamilySplitLocalVertex(long id) const /*! Prepares the patch for performing the adaption. - NOTE: only cells are tracked. + See PatchKernel::adaptionPrepare(bool trackPartitioning) for the + documentation about the tracking information returned by this function. \param trackAdaption if set to true the function will return the changes that will be performed in the alter step @@ -1016,6 +1017,9 @@ std::vector VolOctree::_adaptionPrepare(bool trackAdaption) /*! Alter the patch performing the adpation. + See PatchKernel::adaptionAlter(bool trackPartitioning) for the + documentation about the tracking information returned by this function. + \param trackAdaption if set to true the function will return the changes done to the patch during the adaption \result If the adaption is tracked, returns a vector of adaption::Info @@ -1065,6 +1069,11 @@ void VolOctree::settleAdaptionMarkers() /*! Syncronizes the patch with the underlying octree. + Optionally, this function can track the changes performed to the patch. See + PatchKernel::partitioningAlter(bool trackPartitioning, bool squeezeStorage) + for the documentation about the tracking information returned by this + function. + \param trackChanges if set to true the changes to the patch will be tracked \result Returns all the changes applied to the patch. diff --git a/src/voloctree/voloctree_parallel.cpp b/src/voloctree/voloctree_parallel.cpp index 6c7b7a14ed..3867510c81 100644 --- a/src/voloctree/voloctree_parallel.cpp +++ b/src/voloctree/voloctree_parallel.cpp @@ -112,6 +112,10 @@ void VolOctree::_setHaloSize(std::size_t haloSize) /*! Prepares the patch for performing the partitioning. + See PatchKernel::partitioningPrepare(bool trackPartitioning) for the + documentation about the tracking information that should be returned + when re-implmeneting by this function. + \param cellWeights are the weights of the cells, the weight represents the relative computational cost associated to a specified cell. If no weight is specified for a cell, the default weight will be used From feecfb20c879b05f43581f4ca0ba82af3b31d2a7 Mon Sep 17 00:00:00 2001 From: Andrea Iob Date: Mon, 30 Jan 2023 14:51:24 +0100 Subject: [PATCH 14/15] patchkernel: add tracking of vertices during adaptation/partitioning --- src/patchkernel/adaption.hpp | 3 +- src/patchkernel/patch_kernel.cpp | 6 +- src/patchkernel/patch_kernel_parallel.cpp | 97 ++++++++++++++++++++++- src/voloctree/voloctree.cpp | 82 ++++++++++++++++++- src/voloctree/voloctree.hpp | 6 +- src/voloctree/voloctree_parallel.cpp | 7 ++ 6 files changed, 189 insertions(+), 12 deletions(-) diff --git a/src/patchkernel/adaption.hpp b/src/patchkernel/adaption.hpp index c785c225ce..4dbca87517 100644 --- a/src/patchkernel/adaption.hpp +++ b/src/patchkernel/adaption.hpp @@ -56,7 +56,8 @@ namespace adaption enum Entity { ENTITY_UNKNOWN = -1, ENTITY_CELL, - ENTITY_INTERFACE + ENTITY_INTERFACE, + ENTITY_VERTEX }; struct Info diff --git a/src/patchkernel/patch_kernel.cpp b/src/patchkernel/patch_kernel.cpp index 09782facd0..bf42f1fc99 100644 --- a/src/patchkernel/patch_kernel.cpp +++ b/src/patchkernel/patch_kernel.cpp @@ -763,8 +763,12 @@ std::vector PatchKernel::adaptionPrepare(bool trackAdaption) Information available for tracking purposes are the following: - internal cells that have been coarsened/refined; + - new internal vertices that have been created; + - internal vertices that have been deleted; - new ghost cells that have been created; - - ghost cells that have been deleted. + - new ghost vertices that have been created; + - ghost cells that have been deleted; + - ghost vertices that have been deleted. \param trackAdaption if set to true the function will return the changes done to the patch during the adaption diff --git a/src/patchkernel/patch_kernel_parallel.cpp b/src/patchkernel/patch_kernel_parallel.cpp index 903782cd50..8736f8da98 100644 --- a/src/patchkernel/patch_kernel_parallel.cpp +++ b/src/patchkernel/patch_kernel_parallel.cpp @@ -1576,7 +1576,8 @@ std::vector PatchKernel::partitioningPrepare(MPI_Comm communicat Information available on the sender side for tracking purposes are the following: - - internal cells that will be send. + - internal cells that will be send; + - internal vertices that will be send. No information about tracking are provided on the receiver side. @@ -1675,16 +1676,24 @@ std::vector PatchKernel::partitioningPrepare(const std::unordere Information available on the sender side for tracking purposes are the following: - internal cells that have been sent; + - internal vertices that have been sent; - new ghost cells that have been created (some of the internal cells that have been sent may have become ghosts cells); - - ghost cells that have been deleted. + - new ghost vertices that have been created (some of the internal vertices + that have been sent may have become ghosts vertices); + - ghost cells that have been deleted; + - ghost vertices that have been deleted. Information available on the receiver side for tracking purposes are the following: - internal cells that have been received; + - internal vertices that have been received; - new ghost cells that have been created; + - new ghost vertices that have been created; - ghost cells that have been deleted (some ghost cells may have been - replaced by internal cells that have just been received). + replaced by internal cells that have just been received); + - ghost vertices that have been deleted (some ghost vertices may have been + replaced by internal vertices that have just been received). \param trackPartitioning if set to true the function will return the changes done to the patch during the partitioning @@ -2170,6 +2179,13 @@ std::vector PatchKernel::_partitioningPrepare(const std::unorder } // Fill tracking data structures + partitioningData.emplace_back(); + adaption::Info &partitioningVertexInfo = partitioningData.back(); + partitioningVertexInfo.entity = adaption::ENTITY_VERTEX; + partitioningVertexInfo.type = adaption::TYPE_PARTITION_SEND; + partitioningVertexInfo.rank = recvRank; + partitioningVertexInfo.previous = getOrderedCellsVertices(cellsToSend, true, false); + partitioningData.emplace_back(); adaption::Info &partitioningCellInfo = partitioningData.back(); partitioningCellInfo.entity = adaption::ENTITY_CELL; @@ -2418,6 +2434,15 @@ std::vector PatchKernel::_partitioningAlter_deleteGhosts(bool tr // Track changes if (trackPartitioning) { + partitioningData.emplace_back(); + adaption::Info &partitioningVertexInfo = partitioningData.back(); + partitioningVertexInfo.entity= adaption::ENTITY_VERTEX; + partitioningVertexInfo.type = adaption::TYPE_DELETION; + partitioningVertexInfo.current.reserve(getGhostVertexCount()); + for (VertexConstIterator itr = ghostVertexConstBegin(); itr != ghostVertexConstEnd(); ++itr) { + partitioningVertexInfo.current.push_back(itr.getId()); + } + partitioningData.emplace_back(); adaption::Info &partitioningCellInfo = partitioningData.back(); partitioningCellInfo.entity = adaption::ENTITY_CELL; @@ -2868,6 +2893,13 @@ std::vector PatchKernel::_partitioningAlter_sendCells(const std: // two processes are able to exchange cell data without additional // communications (they already know the list of cells for which // data is needed and the order in which these data will be sent). + partitioningData.emplace_back(); + adaption::Info &partitioningVertexInfo = partitioningData.back(); + partitioningVertexInfo.entity = adaption::ENTITY_VERTEX; + partitioningVertexInfo.type = adaption::TYPE_PARTITION_SEND; + partitioningVertexInfo.rank = recvRank; + partitioningVertexInfo.current = getOrderedCellsVertices(cellSendList, true, false); + partitioningData.emplace_back(); adaption::Info &partitioningCellInfo = partitioningData.back(); partitioningCellInfo.entity = adaption::ENTITY_CELL; @@ -3147,6 +3179,16 @@ std::vector PatchKernel::_partitioningAlter_sendCells(const std: cellCreationInfo.type = adaption::TYPE_CREATION; cellCreationInfo.rank = getRank(); cellCreationInfo.current = getOrderedCellsVertices(trackedCreatedGhostCells, false, true); + + std::vector deletedGhostVertices = getOrderedCellsVertices(trackedCreatedGhostCells, false, true); + if (!deletedGhostVertices.empty()) { + partitioningData.emplace_back(); + adaption::Info &vertexCreationInfo = partitioningData.back(); + vertexCreationInfo.entity = adaption::ENTITY_VERTEX; + vertexCreationInfo.type = adaption::TYPE_CREATION; + vertexCreationInfo.rank = getRank(); + vertexCreationInfo.current = std::move(deletedGhostVertices); + } } // Delete frame cells that are not ghosts @@ -3245,11 +3287,43 @@ std::vector PatchKernel::_partitioningAlter_sendCells(const std: // Prune stale interfaces pruneStaleInterfaces(); + // Identify orphan vertices + std::vector orphanVertices = findOrphanVertices(); + + // Track ghost vertices deletion + // + // Only ghost vertices need to be tracked, all orphan internal vertex + // have already been tracked among the vertices that have been send. + if (trackPartitioning && !orphanVertices.empty()) { + partitioningData.emplace_back(); + adaption::Info &vertexDeletionInfo = partitioningData.back(); + vertexDeletionInfo.entity = adaption::ENTITY_VERTEX; + vertexDeletionInfo.type = adaption::TYPE_DELETION; + vertexDeletionInfo.rank = getRank(); + for (long vertexId : orphanVertices) { + const Vertex &vertex = getVertex(vertexId); + if (vertex.isInterior()) { + continue; + } + + vertexDeletionInfo.current.push_back(vertexId); + } + } + // Delete orphan vertices - deleteOrphanVertices(); + deleteVertices(orphanVertices); } else { // All ghost cells will be deleted if (trackPartitioning) { + partitioningData.emplace_back(); + adaption::Info &partitioningVertexInfo = partitioningData.back(); + partitioningVertexInfo.entity= adaption::ENTITY_VERTEX; + partitioningVertexInfo.type = adaption::TYPE_DELETION; + partitioningVertexInfo.current.reserve(getGhostVertexCount()); + for (VertexConstIterator itr = ghostVertexConstBegin(); itr != ghostVertexConstEnd(); ++itr) { + partitioningVertexInfo.current.push_back(itr.getId()); + } + partitioningData.emplace_back(); adaption::Info &partitioningCellInfo = partitioningData.back(); partitioningCellInfo.entity = adaption::ENTITY_CELL; @@ -3905,6 +3979,13 @@ std::vector PatchKernel::_partitioningAlter_receiveCells(const s // Track changes if (trackPartitioning) { if (!trackedReceivedInteriorCells.empty()) { + partitioningData.emplace_back(); + adaption::Info &vertexRecvInfo = partitioningData.back(); + vertexRecvInfo.entity = adaption::ENTITY_VERTEX; + vertexRecvInfo.type = adaption::TYPE_PARTITION_RECV; + vertexRecvInfo.rank = sendRank; + vertexRecvInfo.current = getOrderedCellsVertices(trackedReceivedInteriorCells, true, false); + partitioningData.emplace_back(); adaption::Info &cellRecvInfo = partitioningData.back(); cellRecvInfo.entity = adaption::ENTITY_CELL; @@ -3915,6 +3996,14 @@ std::vector PatchKernel::_partitioningAlter_receiveCells(const s } if (!trackedCreatedGhostCells.empty()) { + partitioningData.emplace_back(); + adaption::Info &vertexCreationInfo = partitioningData.back(); + vertexCreationInfo.entity = adaption::ENTITY_VERTEX; + vertexCreationInfo.type = adaption::TYPE_CREATION; + vertexCreationInfo.rank = patchRank; + vertexCreationInfo.current = getOrderedCellsVertices(trackedCreatedGhostCells, false, true); + + partitioningData.emplace_back(); adaption::Info &cellCreationInfo = partitioningData.back(); cellCreationInfo.entity = adaption::ENTITY_CELL; diff --git a/src/voloctree/voloctree.cpp b/src/voloctree/voloctree.cpp index a793605772..b7433bf6f6 100644 --- a/src/voloctree/voloctree.cpp +++ b/src/voloctree/voloctree.cpp @@ -1340,7 +1340,11 @@ std::vector VolOctree::sync(bool trackChanges) if (!importFromScratch) { log::cout() << " Deleting stale elements..." << std::endl; - stitchInfo = deleteCells(synchronizationData); + if (trackChanges) { + stitchInfo = deleteCells(synchronizationData, &synchronizationData); + } else { + stitchInfo = deleteCells(synchronizationData, nullptr); + } log::cout() << " Stale element successfully deleted." << std::endl; } @@ -1348,7 +1352,11 @@ std::vector VolOctree::sync(bool trackChanges) // Import added cells log::cout() << " Creating new elements..." << std::endl; - createCells(stitchInfo, nullptr, &synchronizationData); + if (trackChanges) { + createCells(stitchInfo, nullptr, &synchronizationData, &synchronizationData); + } else { + createCells(stitchInfo, nullptr, &synchronizationData, nullptr); + } log::cout() << " New elements successfully created." << std::endl; @@ -1435,9 +1443,12 @@ void VolOctree::renumberCells(const adaption::InfoCollection &cellAdaptionData) that need to be performed to the cells, on output the dummy ids contained in the current statuses will be replaced with the actual ids of the cells that have been created + \param vertexAdaptionData if a valid pointer is provided, on output will + contain the changes applied to the vertices */ void VolOctree::createCells(StitchInfo &stitchInfo, std::istream *restoreStream, - adaption::InfoCollection *cellAdaptionData) + adaption::InfoCollection *cellAdaptionData, + adaption::InfoCollection *vertexAdaptionData) { // Tree information long nOctants = m_tree->getNumOctants(); @@ -1496,6 +1507,11 @@ void VolOctree::createCells(StitchInfo &stitchInfo, std::istream *restoreStream, #endif } + // Track vertex creation + if (vertexAdaptionData) { + trackedCreatedVertices.push_back(vertexId); + } + // Add the vertex to the stitching info stitchInfo[vertexTreeKey] = vertexId; } @@ -1616,6 +1632,34 @@ void VolOctree::createCells(StitchInfo &stitchInfo, std::istream *restoreStream, if (!restoreStream) { updateInterfaces(false); } + +#if BITPIT_ENABLE_MPI==1 + // Track internal vertices received from other partitions + std::unordered_set recvVertices; + if (vertexAdaptionData) { + for (const adaption::Info &adaptionInfo : *cellAdaptionData) { + if (adaptionInfo.type != adaption::TYPE_PARTITION_RECV) { + continue; + } + + std::size_t vertexRecvInfoId = vertexAdaptionData->insert(adaption::TYPE_PARTITION_RECV, adaption::ENTITY_VERTEX, adaptionInfo.rank); + adaption::Info &vertexRecvInfo = vertexAdaptionData->at(vertexRecvInfoId); + vertexRecvInfo.current = getOrderedCellsVertices(adaptionInfo.current, true, false); + recvVertices.insert(vertexRecvInfo.current.begin(), vertexRecvInfo.current.end()); + } + } +#endif + + // Track vertex creation + if (!trackedCreatedVertices.empty()) { + std::size_t vertexCreationInfoId = vertexAdaptionData->insert(adaption::TYPE_CREATION, adaption::ENTITY_VERTEX); + adaption::Info &vertexCreationInfo = vertexAdaptionData->at(vertexCreationInfoId); + vertexCreationInfo.current = std::move(trackedCreatedVertices); +#if BITPIT_ENABLE_MPI==1 + adaption::InfoCollection::removeIds(recvVertices, &(vertexCreationInfo.current)); +#endif + trackedCreatedVertices.clear(); + } } /*! @@ -1623,16 +1667,36 @@ void VolOctree::createCells(StitchInfo &stitchInfo, std::istream *restoreStream, \param cellAdaptionData are the information that describe the changes that need to be performed to the cells + \param vertexAdaptionData if a valid pointer is provided, on output will + contain the changes applied to the vertices \param stitchInfo if a valid pointer is provided, on output will contain the stitch information that can used to stich the faces created after deleting the octants */ -VolOctree::StitchInfo VolOctree::deleteCells(const adaption::InfoCollection &cellAdaptionData) +VolOctree::StitchInfo VolOctree::deleteCells(const adaption::InfoCollection &cellAdaptionData, + adaption::InfoCollection *vertexAdaptionData) { // Info of the cells int nCellVertices = m_cellTypeInfo->nVertices; int nCellFaces = m_cellTypeInfo->nFaces; +#if BITPIT_ENABLE_MPI==1 + // Track internal vertices sent to other partitions + std::unordered_set trackedSentVertices; + if (vertexAdaptionData) { + for (const adaption::Info &adaptionInfo : cellAdaptionData) { + if (adaptionInfo.type != adaption::TYPE_PARTITION_SEND) { + continue; + } + + std::size_t vertexSendInfoId = vertexAdaptionData->insert(adaption::TYPE_PARTITION_SEND, adaption::ENTITY_VERTEX, adaptionInfo.rank); + adaption::Info &vertexSendInfo = vertexAdaptionData->at(vertexSendInfoId); + vertexSendInfo.previous = getOrderedCellsVertices(adaptionInfo.previous, true, false); + trackedSentVertices.insert(vertexSendInfo.previous.begin(), vertexSendInfo.previous.end()); + } + } +#endif + // Delete cells std::vector deadCells; std::unordered_set deadVertices; @@ -1771,6 +1835,16 @@ VolOctree::StitchInfo VolOctree::deleteCells(const adaption::InfoCollection &cel log::cout() << " " << deadVertices.size() << " vertices will be removed." << std::endl; PatchKernel::deleteVertices(deadVertices); + // Track vertex deletion + if (vertexAdaptionData && !deadVertices.empty()) { + std::size_t deletionInfoId = vertexAdaptionData->insert(adaption::TYPE_DELETION, adaption::ENTITY_VERTEX); + adaption::Info &deletionInfo = vertexAdaptionData->at(deletionInfoId); + deletionInfo.previous = std::vector(deadVertices.begin(), deadVertices.end()); +#if BITPIT_ENABLE_MPI==1 + adaption::InfoCollection::removeIds(trackedSentVertices, &(deletionInfo.previous)); +#endif + } + // Done return stitchInfo; } diff --git a/src/voloctree/voloctree.hpp b/src/voloctree/voloctree.hpp index 80455b39a6..d16c353711 100644 --- a/src/voloctree/voloctree.hpp +++ b/src/voloctree/voloctree.hpp @@ -262,9 +262,11 @@ class VolOctree : public VolumeKernel { void renumberCells(const adaption::InfoCollection &treeAdaptionData); - void createCells(StitchInfo &stitchInfo, std::istream *stream, adaption::InfoCollection *cellAdaptionData); + void createCells(StitchInfo &stitchInfo, std::istream *stream, adaption::InfoCollection *cellAdaptionData, + adaption::InfoCollection *vertexAdaptionData = nullptr); - StitchInfo deleteCells(const adaption::InfoCollection &cellAdaptionData); + StitchInfo deleteCells(const adaption::InfoCollection &cellAdaptionData, + adaption::InfoCollection *vertexAdaptionData = nullptr); std::vector sync(bool trackChanges); diff --git a/src/voloctree/voloctree_parallel.cpp b/src/voloctree/voloctree_parallel.cpp index 3867510c81..db18228252 100644 --- a/src/voloctree/voloctree_parallel.cpp +++ b/src/voloctree/voloctree_parallel.cpp @@ -166,6 +166,13 @@ std::vector VolOctree::_partitioningPrepare(const std::unordered long &cellId = partitioningCellInfo.previous.back(); cellId = getOctantId(octantInfo); } + + partitioningData.emplace_back(); + adaption::Info &partitioningVertexInfo = partitioningData.back(); + partitioningVertexInfo.entity = adaption::ENTITY_VERTEX; + partitioningVertexInfo.type = adaption::TYPE_PARTITION_SEND; + partitioningVertexInfo.rank = receiver; + partitioningVertexInfo.previous = getOrderedCellsVertices(partitioningCellInfo.previous, true, false); } } From d63a6c762d750c41f5dbbde5a7dcd3ffadced295 Mon Sep 17 00:00:00 2001 From: Andrea Iob Date: Mon, 30 Jan 2023 14:49:45 +0100 Subject: [PATCH 15/15] patchkernel: add tracking of interfaces during adaptation/partitioning --- src/patchkernel/patch_kernel.cpp | 167 +++++++++++++++++----- src/patchkernel/patch_kernel.hpp | 16 +-- src/patchkernel/patch_kernel_parallel.cpp | 20 +-- src/volcartesian/volcartesian.cpp | 22 ++- src/volcartesian/volcartesian.hpp | 2 +- src/voloctree/voloctree.cpp | 32 +++-- src/voloctree/voloctree.hpp | 9 +- 7 files changed, 207 insertions(+), 61 deletions(-) diff --git a/src/patchkernel/patch_kernel.cpp b/src/patchkernel/patch_kernel.cpp index bf42f1fc99..bafd23ccc1 100644 --- a/src/patchkernel/patch_kernel.cpp +++ b/src/patchkernel/patch_kernel.cpp @@ -635,7 +635,7 @@ std::vector PatchKernel::update(bool trackAdaption, bool squeeze } // Finalize alterations - finalizeAlterations(squeezeStorage); + mergeAdaptionInfo(finalizeAlterations(trackAdaption, squeezeStorage), adaptionData); // Adaption bool adaptionDirty = (getAdaptionStatus(true) == ADAPTION_DIRTY); @@ -768,7 +768,9 @@ std::vector PatchKernel::adaptionPrepare(bool trackAdaption) - new ghost cells that have been created; - new ghost vertices that have been created; - ghost cells that have been deleted; - - ghost vertices that have been deleted. + - ghost vertices that have been deleted; + - new interfaces that have been created; + - interfaces that have been deleted. \param trackAdaption if set to true the function will return the changes done to the patch during the adaption @@ -796,10 +798,10 @@ std::vector PatchKernel::adaptionAlter(bool trackAdaption, bool } // Adapt the patch - adaptionData = _adaptionAlter(trackAdaption); + mergeAdaptionInfo(_adaptionAlter(trackAdaption), adaptionData); // Finalize patch alterations - finalizeAlterations(squeezeStorage); + mergeAdaptionInfo(finalizeAlterations(trackAdaption, squeezeStorage), adaptionData); // Update the status setAdaptionStatus(ADAPTION_ALTERED); @@ -848,11 +850,15 @@ void PatchKernel::settleAdaptionMarkers() /*! Finalize patch alterations. + \param trackAdaption if set to true the function will return the changes + that will be performed in the alter step \param squeezeStorage if set to true patch data structures will be squeezed */ -void PatchKernel::finalizeAlterations(bool squeezeStorage) +std::vector PatchKernel::finalizeAlterations(bool trackAdaption, bool squeezeStorage) { + std::vector adaptionData; + // Flush vertex data structures m_vertices.flush(); @@ -874,7 +880,7 @@ void PatchKernel::finalizeAlterations(bool squeezeStorage) // Update interfaces bool interfacesDirty = areInterfacesDirty(); if (interfacesDirty) { - updateInterfaces(); + mergeAdaptionInfo(updateInterfaces(false, trackAdaption), adaptionData); } // Flush interfaces data structures @@ -901,6 +907,8 @@ void PatchKernel::finalizeAlterations(bool squeezeStorage) m_cells.sync(); m_interfaces.sync(); m_vertices.sync(); + + return adaptionData; } /*! @@ -1047,22 +1055,29 @@ void PatchKernel::resetCells() This function doesn't change the build strategy, it only resets the existing interface. + + \param trackAdaption if set to true the changes to the patch will be + tracked */ -void PatchKernel::resetInterfaces() +std::vector PatchKernel::resetInterfaces(bool trackAdaption) { + std::vector adaptionData; + // Early return if no interfaces have been built if (getInterfacesBuildStrategy() == INTERFACES_NONE) { - return; + return adaptionData; } // Reset the interfaces - _resetInterfaces(false); + adaptionData = _resetInterfaces(trackAdaption, false); // Mark cell interfaces as dirty setCellAlterationFlags(FLAG_INTERFACES_DIRTY); // Clear list of altered interfaces m_alteredInterfaces.clear(); + + return adaptionData; } /*! @@ -1070,20 +1085,49 @@ void PatchKernel::resetInterfaces() This function doesn't change the alteration flags. + \param trackAdaption if set to true the changes to the patch will be + tracked \param release if it's true the memory hold by the interfaces will be released, otherwise the interfaces will be reset but their memory will not be released */ -void PatchKernel::_resetInterfaces(bool release) +std::vector PatchKernel::_resetInterfaces(bool trackAdaption, bool release) { + // Reset cell interfaces for (auto &cell : m_cells) { cell.resetInterfaces(!release); } + // Track deleted interfaces + adaption::InfoCollection adaptionData; + if (trackAdaption) { + // Identify interior interfaces + std::unordered_set internalInterfaces; + for (CellConstIterator cellItr = internalCellBegin(); cellItr != internalCellEnd(); ++cellItr) { + const Cell &cell = *cellItr; + const int nCellInterfaces = cell.getInterfaceCount(); + const long *cellInterfaces = cell.getInterfaces(); + for (int k = 0; k < nCellInterfaces; ++k) { + long interfaceId = cellInterfaces[k]; + internalInterfaces.insert(interfaceId); + } + } + + // Track interfaces that will be deleted + // + // Only interfaces on interior cells will be tracked. + std::size_t adaptionInfoId = adaptionData.insert(adaption::TYPE_DELETION, adaption::ENTITY_INTERFACE); + adaption::Info &adaptionInfo = adaptionData[adaptionInfoId]; + adaptionInfo.previous = std::vector(internalInterfaces.begin(), internalInterfaces.end()); + } + + // Delete interfaces m_interfaces.clear(release); if (m_interfaceIdGenerator) { m_interfaceIdGenerator->reset(); } + + return adaptionData.dump(); } /*! @@ -6449,9 +6493,12 @@ void PatchKernel::buildInterfaces() adjacencies are not yet initialized an exception is thrown. \param strategy is the build strategy that will be used + \param trackAdaption if set to true the changes to the patch will be tracked */ -void PatchKernel::initializeInterfaces(InterfacesBuildStrategy strategy) +std::vector PatchKernel::initializeInterfaces(InterfacesBuildStrategy strategy, bool trackAdaption) { + std::vector adaptionData; + // Interfaces need adjacencies if (getAdjacenciesBuildStrategy() == ADJACENCIES_NONE) { throw std::runtime_error ("Adjacencies are mandatory for building the interfaces."); @@ -6463,10 +6510,10 @@ void PatchKernel::initializeInterfaces(InterfacesBuildStrategy strategy) // Early return if we don't need interfaces if (strategy == INTERFACES_NONE) { if (currentStrategy != INTERFACES_NONE) { - destroyInterfaces(); + mergeAdaptionInfo(destroyInterfaces(trackAdaption), adaptionData); } - return; + return adaptionData; } // Update the build strategy @@ -6475,10 +6522,12 @@ void PatchKernel::initializeInterfaces(InterfacesBuildStrategy strategy) } // Reset interfaces - resetInterfaces(); + mergeAdaptionInfo(resetInterfaces(trackAdaption), adaptionData); // Update the interfaces - updateInterfaces(); + mergeAdaptionInfo(updateInterfaces(false, trackAdaption), adaptionData); + + return adaptionData; } /*! @@ -6486,19 +6535,22 @@ void PatchKernel::initializeInterfaces(InterfacesBuildStrategy strategy) \param forcedUpdated if set to true, bounding box information will be updated also if they are not marked as dirty + \param trackAdaption if set to true the changes to the patch will be tracked */ -void PatchKernel::updateInterfaces(bool forcedUpdated) +std::vector PatchKernel::updateInterfaces(bool forcedUpdated, bool trackAdaption) { + std::vector adaptionData; + // Early return if interfaces are not built InterfacesBuildStrategy currentStrategy = getInterfacesBuildStrategy(); if (currentStrategy == INTERFACES_NONE) { - return; + return adaptionData; } // Check if the interfaces are dirty bool interfacesDirty = areInterfacesDirty(); if (!interfacesDirty && !forcedUpdated) { - return; + return adaptionData; } // Interfaces need up-to-date adjacencies @@ -6512,10 +6564,10 @@ void PatchKernel::updateInterfaces(bool forcedUpdated) setAdaptionMode(ADAPTION_MANUAL); // Prune stale interfaces - pruneStaleInterfaces(); + mergeAdaptionInfo(pruneStaleInterfaces(trackAdaption), adaptionData); // Update interfaces - _updateInterfaces(); + mergeAdaptionInfo(_updateInterfaces(trackAdaption), adaptionData); // Interfaces are now updated unsetCellAlterationFlags(FLAG_INTERFACES_DIRTY); @@ -6524,8 +6576,10 @@ void PatchKernel::updateInterfaces(bool forcedUpdated) // Restore previous adaption mode setAdaptionMode(previousAdaptionMode); } else { - initializeInterfaces(currentStrategy); + mergeAdaptionInfo(initializeInterfaces(currentStrategy, trackAdaption), adaptionData); } + + return adaptionData; } /*! @@ -6533,16 +6587,21 @@ void PatchKernel::updateInterfaces(bool forcedUpdated) After deleting the interfaces, this function changes the build strategy to "None". + + \param trackAdaption if set to true the changes to the patch will be + tracked */ -void PatchKernel::destroyInterfaces() +std::vector PatchKernel::destroyInterfaces(bool trackAdaption) { + std::vector adaptionData; + // Early return if no interfaces have been built if (getInterfacesBuildStrategy() == INTERFACES_NONE) { - return; + return adaptionData; } - // Destroy the interfaces - _resetInterfaces(true); + // Reset the interfaces + adaptionData = _resetInterfaces(trackAdaption, true); // Clear list of cells with dirty interfaces unsetCellAlterationFlags(FLAG_INTERFACES_DIRTY); @@ -6552,6 +6611,8 @@ void PatchKernel::destroyInterfaces() // Set interface build strategy setInterfacesBuildStrategy(INTERFACES_NONE); + + return adaptionData; } /*! @@ -6559,12 +6620,17 @@ void PatchKernel::destroyInterfaces() The list of cells to process and the list of stale interfaces are filled during cell deletion. + + \param trackAdaption if set to true the changes to the patch will be tracked + \result If the adaption is tracked, returns a vector of adaption::Info + with all the changes done to the patch during the adaption, otherwise an + empty vector will be returned. */ -void PatchKernel::pruneStaleInterfaces() +std::vector PatchKernel::pruneStaleInterfaces(bool trackAdaption) { // Early return if no interfaces have been built if (getInterfacesBuildStrategy() == INTERFACES_NONE) { - return; + return std::vector(); } // Remove dangling interfaces from cells @@ -6609,6 +6675,16 @@ void PatchKernel::pruneStaleInterfaces() danglingInterfaces.push_back(interfaceId); } deleteInterfaces(danglingInterfaces); + + // Track changes + adaption::InfoCollection adaptionData; + if (trackAdaption) { + std::size_t adaptionInfoId = adaptionData.insert(adaption::TYPE_DELETION, adaption::ENTITY_INTERFACE); + adaption::Info &adaptionInfo = adaptionData[adaptionInfoId]; + adaptionInfo.previous = std::move(danglingInterfaces); + } + + return adaptionData.dump(); } /*! @@ -6616,8 +6692,10 @@ void PatchKernel::pruneStaleInterfaces() The function will process the cells whose interfaces have been marked as dirty. + + \param trackAdaption if set to true the changes to the patch will be tracked */ -void PatchKernel::_updateInterfaces() +std::vector PatchKernel::_updateInterfaces(bool trackAdaption) { // Update interfaces // @@ -6630,6 +6708,7 @@ void PatchKernel::_updateInterfaces() // // On border faces of internal cells we need to build an interface, also // if there are no adjacencies. + std::vector createdInterfaces; for (const auto &entry : m_alteredCells) { AlterationFlags cellAlterationFlags = entry.second; if (!testAlterationFlags(cellAlterationFlags, FLAG_INTERFACES_DIRTY)) { @@ -6659,14 +6738,35 @@ void PatchKernel::_updateInterfaces() int neighFace = findAdjoinNeighFace(cell, face, *neigh); - buildCellInterface(&cell, face, neigh, neighFace); + // Build the interface + InterfaceIterator interfaceIterator = buildCellInterface(&cell, face, neigh, neighFace); + + // Track changes + if (trackAdaption) { + createdInterfaces.push_back(interfaceIterator.getId()); + } } } else if (nFaceInterfaces == 0) { // Internal borderes need an interface - buildCellInterface(&cell, face, nullptr, -1); + InterfaceIterator interfaceIterator = buildCellInterface(&cell, face, nullptr, -1); + + // Track changes + if (trackAdaption) { + createdInterfaces.push_back(interfaceIterator.getId()); + } } } } + + // Track changes + adaption::InfoCollection adaptionData; + if (trackAdaption) { + std::size_t adaptionInfoId = adaptionData.insert(adaption::TYPE_CREATION, adaption::ENTITY_INTERFACE); + adaption::Info &adaptionInfo = adaptionData[adaptionInfoId]; + adaptionInfo.current = std::move(createdInterfaces); + } + + return adaptionData.dump(); } /*! @@ -8526,12 +8626,15 @@ void PatchKernel::mergeAdaptionInfo(std::vector &&source, std::v { if (source.empty()) { return; - } else if (destination.empty()) { + } + + if (destination.empty()) { destination.swap(source); return; } - throw std::runtime_error ("Unable to merge the adaption info."); + destination.insert(destination.end(), std::make_move_iterator(source.begin()), std::make_move_iterator(source.end())); + source.clear(); } } diff --git a/src/patchkernel/patch_kernel.hpp b/src/patchkernel/patch_kernel.hpp index 507302da7f..8fc1fbf4d0 100644 --- a/src/patchkernel/patch_kernel.hpp +++ b/src/patchkernel/patch_kernel.hpp @@ -384,7 +384,7 @@ friend class PatchManager; virtual void reset(); virtual void resetVertices(); virtual void resetCells(); - virtual void resetInterfaces(); + virtual std::vector resetInterfaces(bool trackAdaption = false); bool reserveVertices(size_t nVertices); bool reserveCells(size_t nCells); @@ -655,9 +655,9 @@ friend class PatchManager; InterfacesBuildStrategy getInterfacesBuildStrategy() const; bool areInterfacesDirty(bool global = false) const; BITPIT_DEPRECATED(void buildInterfaces()); - void initializeInterfaces(InterfacesBuildStrategy strategy = INTERFACES_AUTOMATIC); - void updateInterfaces(bool forcedUpdated = false); - void destroyInterfaces(); + std::vector initializeInterfaces(InterfacesBuildStrategy strategy = INTERFACES_AUTOMATIC, bool trackAdaption = false); + std::vector updateInterfaces(bool forcedUpdated = false, bool trackAdaption = false); + std::vector destroyInterfaces(bool trackAdaption = false); void getBoundingBox(std::array &minPoint, std::array &maxPoint) const; void getBoundingBox(bool global, std::array &minPoint, std::array &maxPoint) const; @@ -883,9 +883,9 @@ friend class PatchManager; virtual void _updateAdjacencies(); void setInterfacesBuildStrategy(InterfacesBuildStrategy status); - void pruneStaleInterfaces(); - virtual void _resetInterfaces(bool release); - virtual void _updateInterfaces(); + std::vector pruneStaleInterfaces(bool trackAdaption); + virtual std::vector _resetInterfaces(bool trackAdaption, bool release); + virtual std::vector _updateInterfaces(bool trackAdaption); bool testCellAlterationFlags(long id, AlterationFlags flags) const; AlterationFlags getCellAlterationFlags(long id) const; @@ -1098,7 +1098,7 @@ friend class PatchManager; void initializeSerialCommunicator(); #endif - void finalizeAlterations(bool squeezeStorage = false); + std::vector finalizeAlterations(bool trackAdaption, bool squeezeStorage = false); InterfaceIterator buildCellInterface(Cell *cell_1, int face_1, Cell *cell_2, int face_2, long interfaceId = Element::NULL_ID); diff --git a/src/patchkernel/patch_kernel_parallel.cpp b/src/patchkernel/patch_kernel_parallel.cpp index 8736f8da98..8a817a5d8d 100644 --- a/src/patchkernel/patch_kernel_parallel.cpp +++ b/src/patchkernel/patch_kernel_parallel.cpp @@ -1682,7 +1682,9 @@ std::vector PatchKernel::partitioningPrepare(const std::unordere - new ghost vertices that have been created (some of the internal vertices that have been sent may have become ghosts vertices); - ghost cells that have been deleted; - - ghost vertices that have been deleted. + - ghost vertices that have been deleted; + - new interfaces that have been created; + - interfaces that have been deleted. Information available on the receiver side for tracking purposes are the following: @@ -1693,7 +1695,9 @@ std::vector PatchKernel::partitioningPrepare(const std::unordere - ghost cells that have been deleted (some ghost cells may have been replaced by internal cells that have just been received); - ghost vertices that have been deleted (some ghost vertices may have been - replaced by internal vertices that have just been received). + replaced by internal vertices that have just been received); + - new interfaces that have been created; + - interfaces that have been deleted. \param trackPartitioning if set to true the function will return the changes done to the patch during the partitioning @@ -1719,10 +1723,10 @@ std::vector PatchKernel::partitioningAlter(bool trackPartitionin } // Partition the patch - partitioningData = _partitioningAlter(trackPartitioning); + mergeAdaptionInfo(_partitioningAlter(trackPartitioning), partitioningData); // Finalize patch alterations - finalizeAlterations(squeezeStorage); + mergeAdaptionInfo(finalizeAlterations(trackPartitioning, squeezeStorage), partitioningData); // Update the status setPartitioningStatus(PARTITIONING_ALTERED); @@ -2469,7 +2473,7 @@ std::vector PatchKernel::_partitioningAlter_deleteGhosts(bool tr // Prune stale interfaces // // Interfaces will be re-created after the partitioning. - pruneStaleInterfaces(); + mergeAdaptionInfo(pruneStaleInterfaces(trackPartitioning), partitioningData); // Delete vertices no longer used deleteOrphanVertices(); @@ -3114,7 +3118,7 @@ std::vector PatchKernel::_partitioningAlter_sendCells(const std: // we need to remove stale adjacencies and interfaces. pruneStaleAdjacencies(); - pruneStaleInterfaces(); + mergeAdaptionInfo(pruneStaleInterfaces(trackPartitioning), partitioningData); // If we are sending many cells try to reduced the used memory bool keepMemoryLimited = (nOutgoingCells > ACTIVATE_MEMORY_LIMIT_THRESHOLD * getInternalCellCount()); @@ -3285,7 +3289,7 @@ std::vector PatchKernel::_partitioningAlter_sendCells(const std: } // Prune stale interfaces - pruneStaleInterfaces(); + mergeAdaptionInfo(pruneStaleInterfaces(trackPartitioning), partitioningData); // Identify orphan vertices std::vector orphanVertices = findOrphanVertices(); @@ -4025,7 +4029,7 @@ std::vector PatchKernel::_partitioningAlter_receiveCells(const s // Update interfaces if (getInterfacesBuildStrategy() == INTERFACES_AUTOMATIC) { - updateInterfaces(); + mergeAdaptionInfo(updateInterfaces(false, trackPartitioning), partitioningData); } // Return adaption data diff --git a/src/volcartesian/volcartesian.cpp b/src/volcartesian/volcartesian.cpp index 62fe7a1602..d24f04972a 100644 --- a/src/volcartesian/volcartesian.cpp +++ b/src/volcartesian/volcartesian.cpp @@ -304,8 +304,13 @@ void VolCartesian::_updateAdjacencies() It is not possible to partially update the interfaces of this patch. The function will always update all the interfaces. + + \param trackAdaption if set to true the changes to the patch will be tracked + \result If the adaption is tracked, returns a vector of adaption::Info + with all the changes done to the patch during the adaption, otherwise an + empty vector will be returned. */ -void VolCartesian::_updateInterfaces() +std::vector VolCartesian::_updateInterfaces(bool trackAdaption) { // Partial updates are not supported CellConstIterator beginItr = cellConstBegin(); @@ -326,6 +331,7 @@ void VolCartesian::_updateInterfaces() } // Build interfaces + adaption::InfoCollection adaptionData; if (getMemoryMode() == MemoryMode::MEMORY_NORMAL) { int nCellFaces = 2 * getDimension(); @@ -345,6 +351,8 @@ void VolCartesian::_updateInterfaces() } // Build interfaces + std::vector createdInterfaces; + createdInterfaces.reserve(m_nInterfaces); for (Cell &cell : getCells()) { long cellId = cell.getId(); for (int face = 0; face < nCellFaces; ++face) { @@ -393,12 +401,24 @@ void VolCartesian::_updateInterfaces() } else { interface.unsetNeigh(); } + + // Track changes + createdInterfaces.push_back(interfaceId); } } // Restore previous adaption mode setAdaptionMode(previousAdaptionMode); + + // Track changes + if (trackAdaption) { + std::size_t adaptionInfoId = adaptionData.insert(adaption::TYPE_CREATION, adaption::ENTITY_INTERFACE); + adaption::Info &adaptionInfo = adaptionData[adaptionInfoId]; + adaptionInfo.previous = std::move(createdInterfaces); + } } + + return adaptionData.dump(); } /*! diff --git a/src/volcartesian/volcartesian.hpp b/src/volcartesian/volcartesian.hpp index aa449abf00..0e60053914 100644 --- a/src/volcartesian/volcartesian.hpp +++ b/src/volcartesian/volcartesian.hpp @@ -149,7 +149,7 @@ class VolCartesian : public VolumeKernel { protected: void _updateAdjacencies() override; - void _updateInterfaces() override; + std::vector _updateInterfaces(bool trackAdaption) override; int _getDumpVersion() const override; void _dump(std::ostream &stream) const override; diff --git a/src/voloctree/voloctree.cpp b/src/voloctree/voloctree.cpp index b7433bf6f6..f5a5f34983 100644 --- a/src/voloctree/voloctree.cpp +++ b/src/voloctree/voloctree.cpp @@ -1341,9 +1341,9 @@ std::vector VolOctree::sync(bool trackChanges) log::cout() << " Deleting stale elements..." << std::endl; if (trackChanges) { - stitchInfo = deleteCells(synchronizationData, &synchronizationData); + stitchInfo = deleteCells(synchronizationData, &synchronizationData, &synchronizationData); } else { - stitchInfo = deleteCells(synchronizationData, nullptr); + stitchInfo = deleteCells(synchronizationData, nullptr, nullptr); } log::cout() << " Stale element successfully deleted." << std::endl; @@ -1353,9 +1353,9 @@ std::vector VolOctree::sync(bool trackChanges) log::cout() << " Creating new elements..." << std::endl; if (trackChanges) { - createCells(stitchInfo, nullptr, &synchronizationData, &synchronizationData); + createCells(stitchInfo, nullptr, &synchronizationData, &synchronizationData, &synchronizationData); } else { - createCells(stitchInfo, nullptr, &synchronizationData, nullptr); + createCells(stitchInfo, nullptr, &synchronizationData, nullptr, nullptr); } log::cout() << " New elements successfully created." << std::endl; @@ -1445,10 +1445,13 @@ void VolOctree::renumberCells(const adaption::InfoCollection &cellAdaptionData) have been created \param vertexAdaptionData if a valid pointer is provided, on output will contain the changes applied to the vertices + \param interfaceAdaptionData if a valid pointer is provided, on output will + contain the changes applied to the interfaces */ void VolOctree::createCells(StitchInfo &stitchInfo, std::istream *restoreStream, adaption::InfoCollection *cellAdaptionData, - adaption::InfoCollection *vertexAdaptionData) + adaption::InfoCollection *vertexAdaptionData, + adaption::InfoCollection *interfaceAdaptionData) { // Tree information long nOctants = m_tree->getNumOctants(); @@ -1630,7 +1633,12 @@ void VolOctree::createCells(StitchInfo &stitchInfo, std::istream *restoreStream, // Update interfaces if (!restoreStream) { - updateInterfaces(false); + std::vector interfaceUpdateData = updateInterfaces(false, interfaceAdaptionData); + if (interfaceAdaptionData) { + for (adaption::Info &info : interfaceUpdateData) { + interfaceAdaptionData->insert(std::move(info)); + } + } } #if BITPIT_ENABLE_MPI==1 @@ -1669,12 +1677,15 @@ void VolOctree::createCells(StitchInfo &stitchInfo, std::istream *restoreStream, that need to be performed to the cells \param vertexAdaptionData if a valid pointer is provided, on output will contain the changes applied to the vertices + \param interfaceAdaptionData if a valid pointer is provided, on output will + contain the changes applied to the interfaces \param stitchInfo if a valid pointer is provided, on output will contain the stitch information that can used to stich the faces created after deleting the octants */ VolOctree::StitchInfo VolOctree::deleteCells(const adaption::InfoCollection &cellAdaptionData, - adaption::InfoCollection *vertexAdaptionData) + adaption::InfoCollection *vertexAdaptionData, + adaption::InfoCollection *interfaceAdaptionData) { // Info of the cells int nCellVertices = m_cellTypeInfo->nVertices; @@ -1750,7 +1761,12 @@ VolOctree::StitchInfo VolOctree::deleteCells(const adaption::InfoCollection &cel // we need to remove stale adjacencies and interfaces. pruneStaleAdjacencies(); - pruneStaleInterfaces(); + std::vector interfacePruneData = pruneStaleInterfaces(interfaceAdaptionData); + if (interfaceAdaptionData) { + for (adaption::Info &info : interfacePruneData) { + interfaceAdaptionData->insert(std::move(info)); + } + } // Delete vertices // diff --git a/src/voloctree/voloctree.hpp b/src/voloctree/voloctree.hpp index d16c353711..5d0f2c0ab2 100644 --- a/src/voloctree/voloctree.hpp +++ b/src/voloctree/voloctree.hpp @@ -262,11 +262,14 @@ class VolOctree : public VolumeKernel { void renumberCells(const adaption::InfoCollection &treeAdaptionData); - void createCells(StitchInfo &stitchInfo, std::istream *stream, adaption::InfoCollection *cellAdaptionData, - adaption::InfoCollection *vertexAdaptionData = nullptr); + void createCells(StitchInfo &stitchInfo, std::istream *stream, + adaption::InfoCollection *cellAdaptionData, + adaption::InfoCollection *vertexAdaptionData = nullptr, + adaption::InfoCollection *interfaceAdaptionData = nullptr); StitchInfo deleteCells(const adaption::InfoCollection &cellAdaptionData, - adaption::InfoCollection *vertexAdaptionData = nullptr); + adaption::InfoCollection *vertexAdaptionData = nullptr, + adaption::InfoCollection *interfaceAdaptionData = nullptr); std::vector sync(bool trackChanges);