From c421a2665831d273c476d9f87f66974ffbcb7081 Mon Sep 17 00:00:00 2001 From: Ziyan Date: Wed, 27 Apr 2022 17:20:24 +0900 Subject: [PATCH 1/5] Port implementation of doubly linked memory pool allocator. --- include/rapidjson/allocators.h | 186 ++++++++++++++++++++++++++------- 1 file changed, 147 insertions(+), 39 deletions(-) diff --git a/include/rapidjson/allocators.h b/include/rapidjson/allocators.h index 12bc5bafc..efcb80492 100644 --- a/include/rapidjson/allocators.h +++ b/include/rapidjson/allocators.h @@ -82,7 +82,7 @@ concept Allocator { class CrtAllocator { public: static const bool kNeedFree = true; - void* Malloc(size_t size) { + void* Malloc(size_t size) { if (size) // behavior of malloc(0) is implementation defined. return RAPIDJSON_MALLOC(size); else @@ -128,16 +128,19 @@ class CrtAllocator { template class MemoryPoolAllocator { //! Chunk header for perpending to each chunk. - /*! Chunks are stored as a singly linked list. + /*! Chunks are stored as a doubly linked list. */ struct ChunkHeader { size_t capacity; //!< Capacity of the chunk in bytes (excluding the header itself). size_t size; //!< Current size of allocated memory in bytes. ChunkHeader *next; //!< Next chunk in the linked list. + ChunkHeader *prev; //!< Prev chunk in the linked list. }; struct SharedData { - ChunkHeader *chunkHead; //!< Head of the chunk linked-list. Only the head chunk serves allocation. + ChunkHeader *chunkHead; //!< Head of the chunk linked-list. + ChunkHeader *chunkNext; //!< Next available chunk in the linked-list. Only the next chunk serves allocation. + ChunkHeader *chunkTail; //!< Tail of the chunk linked-list. BaseAllocator* ownBaseAllocator; //!< base allocator created by this object. size_t refcount; bool ownBuffer; @@ -152,7 +155,7 @@ class MemoryPoolAllocator { } static inline uint8_t *GetChunkBuffer(SharedData *shared) { - return reinterpret_cast(shared->chunkHead) + SIZEOF_CHUNK_HEADER; + return reinterpret_cast(shared->chunkNext) + SIZEOF_CHUNK_HEADER; } static const size_t kDefaultChunkCapacity = RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY; //!< Default chunk capacity. @@ -183,6 +186,9 @@ class MemoryPoolAllocator { shared_->chunkHead->capacity = 0; shared_->chunkHead->size = 0; shared_->chunkHead->next = 0; + shared_->chunkHead->prev = 0; + shared_->chunkNext = shared_->chunkHead; + shared_->chunkTail = shared_->chunkHead; shared_->ownBuffer = true; shared_->refcount = 1; } @@ -207,6 +213,9 @@ class MemoryPoolAllocator { shared_->chunkHead->capacity = size - SIZEOF_SHARED_DATA - SIZEOF_CHUNK_HEADER; shared_->chunkHead->size = 0; shared_->chunkHead->next = 0; + shared_->chunkHead->prev = 0; + shared_->chunkNext = shared_->chunkHead; + shared_->chunkTail = shared_->chunkHead; shared_->ownBaseAllocator = 0; shared_->ownBuffer = false; shared_->refcount = 1; @@ -264,7 +273,7 @@ class MemoryPoolAllocator { --shared_->refcount; return; } - Clear(); + Release(); BaseAllocator *a = shared_->ownBaseAllocator; if (shared_->ownBuffer) { baseAllocator_->Free(shared_); @@ -273,27 +282,48 @@ class MemoryPoolAllocator { } //! Deallocates all memory chunks, excluding the first/user one. - void Clear() RAPIDJSON_NOEXCEPT { + void Release() RAPIDJSON_NOEXCEPT { RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); - for (;;) { - ChunkHeader* c = shared_->chunkHead; - if (!c->next) { - break; - } - shared_->chunkHead = c->next; - baseAllocator_->Free(c); + + ChunkHeader *chunkFirst = GetChunkHead(shared_); + while (shared_->chunkTail != chunkFirst) { + ChunkHeader* prev = shared_->chunkTail->prev; + baseAllocator_->Free(shared_->chunkTail); + shared_->chunkTail = prev; } + shared_->chunkNext = shared_->chunkHead = shared_->chunkTail; + + RAPIDJSON_NOEXCEPT_ASSERT(shared_->chunkHead->prev == 0); + shared_->chunkHead->next = 0; shared_->chunkHead->size = 0; } + //! Release all memory chunks, allowing them to be reused + void Clear() RAPIDJSON_NOEXCEPT { + RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); + for (ChunkHeader* c = shared_->chunkHead; c != 0; c = c->next) { + c->size = 0; + } + shared_->chunkNext = shared_->chunkHead; + } + + //! Reserve enough memory + void Reserve(size_t size) RAPIDJSON_NOEXCEPT { + size_t capacity = Capacity(); + if (size > capacity) { + AllocateChunk(RAPIDJSON_ALIGN(size - capacity)); + } + } + //! Computes the total capacity of allocated memory chunks. /*! \return total capacity in bytes. */ size_t Capacity() const RAPIDJSON_NOEXCEPT { RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); size_t capacity = 0; - for (ChunkHeader* c = shared_->chunkHead; c != 0; c = c->next) + for (ChunkHeader* c = shared_->chunkHead; c != 0; c = c->next) { capacity += c->capacity; + } return capacity; } @@ -303,8 +333,12 @@ class MemoryPoolAllocator { size_t Size() const RAPIDJSON_NOEXCEPT { RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); size_t size = 0; - for (ChunkHeader* c = shared_->chunkHead; c != 0; c = c->next) + for (ChunkHeader* c = shared_->chunkHead; c != 0; c = c->next) { size += c->size; + if (c == shared_->chunkNext) { + break; // the rest will be zeros + } + } return size; } @@ -319,52 +353,58 @@ class MemoryPoolAllocator { //! Allocates a memory block. (concept Allocator) void* Malloc(size_t size) { RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); - if (!size) + if (!size) { return NULL; + } size = RAPIDJSON_ALIGN(size); - if (RAPIDJSON_UNLIKELY(shared_->chunkHead->size + size > shared_->chunkHead->capacity)) - if (!AddChunk(chunk_capacity_ > size ? chunk_capacity_ : size)) - return NULL; + if (!EnsureChunk(size)) { + return NULL; + } - void *buffer = GetChunkBuffer(shared_) + shared_->chunkHead->size; - shared_->chunkHead->size += size; + void *buffer = GetChunkBuffer(shared_) + shared_->chunkNext->size; + shared_->chunkNext->size += size; return buffer; } //! Resizes a memory block (concept Allocator) void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) { - if (originalPtr == 0) + if (originalPtr == 0) { return Malloc(newSize); + } RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); - if (newSize == 0) + if (newSize == 0) { return NULL; + } originalSize = RAPIDJSON_ALIGN(originalSize); newSize = RAPIDJSON_ALIGN(newSize); // Do not shrink if new size is smaller than original - if (originalSize >= newSize) + if (originalSize >= newSize) { return originalPtr; + } // Simply expand it if it is the last allocation and there is sufficient space - if (originalPtr == GetChunkBuffer(shared_) + shared_->chunkHead->size - originalSize) { + if (originalPtr == GetChunkBuffer(shared_) + shared_->chunkNext->size - originalSize) { size_t increment = static_cast(newSize - originalSize); - if (shared_->chunkHead->size + increment <= shared_->chunkHead->capacity) { - shared_->chunkHead->size += increment; + if (shared_->chunkNext->size + increment <= shared_->chunkNext->capacity) { + shared_->chunkNext->size += increment; return originalPtr; } } // Realloc process: allocate and copy memory, do not free original buffer. if (void* newBuffer = Malloc(newSize)) { - if (originalSize) + if (originalSize) { std::memcpy(newBuffer, originalPtr, originalSize); + } return newBuffer; } - else + else { return NULL; + } } //! Frees a memory block (concept Allocator) @@ -386,18 +426,87 @@ class MemoryPoolAllocator { /*! \param capacity Capacity of the chunk in bytes. \return true if success. */ - bool AddChunk(size_t capacity) { - if (!baseAllocator_) - shared_->ownBaseAllocator = baseAllocator_ = RAPIDJSON_NEW(BaseAllocator)(); - if (ChunkHeader* chunk = static_cast(baseAllocator_->Malloc(SIZEOF_CHUNK_HEADER + capacity))) { - chunk->capacity = capacity; - chunk->size = 0; - chunk->next = shared_->chunkHead; - shared_->chunkHead = chunk; + bool EnsureChunk(size_t size) { + RAPIDJSON_ASSERT(shared_->chunkNext != 0); // there is always at least one chunk + // check if chunk is big enough + if (shared_->chunkNext->size + size <= shared_->chunkNext->capacity) { return true; } - else + // search for for a big enough chunk that we already had before + for (ChunkHeader* c = shared_->chunkNext->next; c != 0; c = c->next) { + if (c->size + size > c->capacity) { + // not big enough, continue + continue; + } + if (c == shared_->chunkNext->next) { + // found the chunk as the next in list + shared_->chunkNext = c; + return true; + } + // found a usable chunk, but somewhere later in the list + // re-arrange the link list so that it becomes the next chunk + // remove chunk from link list + RAPIDJSON_ASSERT(c != shared_->chunkHead); + RAPIDJSON_ASSERT(c->prev != 0); + c->prev->next = c->next; + if (c->next) { + RAPIDJSON_ASSERT(c != shared_->chunkTail); + c->next->prev = c->prev; + } else { + shared_->chunkTail = c->prev; + RAPIDJSON_ASSERT(c == shared_->chunkTail); + } + c->prev = c->next = 0; + // add chunk to link list after shared_->chunkNext + RAPIDJSON_ASSERT(shared_->chunkNext->next != 0); + c->next = shared_->chunkNext->next; + shared_->chunkNext->next->prev = c; + c->prev = shared_->chunkNext; + shared_->chunkNext->next = c; + shared_->chunkNext = c; + return true; + } + // if no existing chunk can satisfy, need to allocate a new chunk + ChunkHeader* chunk = AllocateChunk(size); + if (!chunk) { return false; + } + shared_->chunkNext = chunk; + return true; + } + + //! Allocate new chunk, but do not change shared_->chunkNext + ChunkHeader* AllocateChunk(size_t size) { + if (!baseAllocator_) { + shared_->ownBaseAllocator = baseAllocator_ = RAPIDJSON_NEW(BaseAllocator)(); + } + size_t capacity = chunk_capacity_; + if (size > capacity) { + capacity = size; + } + ChunkHeader* chunk = static_cast(baseAllocator_->Malloc(SIZEOF_CHUNK_HEADER + capacity)); + if (!chunk) { + return 0; + } + chunk->capacity = capacity; + chunk->size = 0; + chunk->next = 0; + chunk->prev = 0; + RAPIDJSON_ASSERT(shared_->chunkNext != 0); // there is always at least one chunk + if (shared_->chunkNext->next == 0) { + // last chunk in the list + RAPIDJSON_ASSERT(shared_->chunkNext == shared_->chunkTail); + chunk->prev = shared_->chunkTail; + shared_->chunkTail->next = chunk; + shared_->chunkTail = chunk; + return chunk; + } + // insert chunk to link list after shared_->chunkNext + chunk->next = shared_->chunkNext->next; + shared_->chunkNext->next->prev = chunk; + chunk->prev = shared_->chunkNext; + shared_->chunkNext->next = chunk; + return chunk; } static inline void* AlignBuffer(void* buf, size_t &size) @@ -418,7 +527,6 @@ class MemoryPoolAllocator { BaseAllocator* baseAllocator_; //!< base allocator for allocating memory chunks. SharedData *shared_; //!< The shared data of the allocator }; - namespace internal { template struct IsRefCounted : From ed33b705a8934dbfd38eceb8dca7e692485e72a2 Mon Sep 17 00:00:00 2001 From: Ziyan Date: Wed, 27 Apr 2022 17:43:19 +0900 Subject: [PATCH 2/5] Switch default stack allocator. --- include/rapidjson/document.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/rapidjson/document.h b/include/rapidjson/document.h index 74089cb9b..5e60744c0 100644 --- a/include/rapidjson/document.h +++ b/include/rapidjson/document.h @@ -85,7 +85,7 @@ class GenericDocument; User can define this to use CrtAllocator or MemoryPoolAllocator. */ #ifndef RAPIDJSON_DEFAULT_STACK_ALLOCATOR -#define RAPIDJSON_DEFAULT_STACK_ALLOCATOR ::RAPIDJSON_NAMESPACE::CrtAllocator +#define RAPIDJSON_DEFAULT_STACK_ALLOCATOR RAPIDJSON_DEFAULT_ALLOCATOR #endif /*! \def RAPIDJSON_VALUE_DEFAULT_OBJECT_CAPACITY From 980a5d7ea5fef0d86122fc1657a43bd21dc041e2 Mon Sep 17 00:00:00 2001 From: Ziyan Date: Thu, 28 Apr 2022 13:39:32 +0900 Subject: [PATCH 3/5] Preallocate first chunk. --- include/rapidjson/allocators.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/rapidjson/allocators.h b/include/rapidjson/allocators.h index efcb80492..84b8d5779 100644 --- a/include/rapidjson/allocators.h +++ b/include/rapidjson/allocators.h @@ -172,7 +172,7 @@ class MemoryPoolAllocator { MemoryPoolAllocator(size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : chunk_capacity_(chunkSize), baseAllocator_(baseAllocator ? baseAllocator : RAPIDJSON_NEW(BaseAllocator)()), - shared_(static_cast(baseAllocator_ ? baseAllocator_->Malloc(SIZEOF_SHARED_DATA + SIZEOF_CHUNK_HEADER) : 0)) + shared_(static_cast(baseAllocator_ ? baseAllocator_->Malloc(SIZEOF_SHARED_DATA + SIZEOF_CHUNK_HEADER + chunkSize) : 0)) { RAPIDJSON_ASSERT(baseAllocator_ != 0); RAPIDJSON_ASSERT(shared_ != 0); @@ -183,7 +183,7 @@ class MemoryPoolAllocator { shared_->ownBaseAllocator = baseAllocator_; } shared_->chunkHead = GetChunkHead(shared_); - shared_->chunkHead->capacity = 0; + shared_->chunkHead->capacity = chunkSize; shared_->chunkHead->size = 0; shared_->chunkHead->next = 0; shared_->chunkHead->prev = 0; From cec4916e3af0cc49853f40773cdc2a728fdd9edf Mon Sep 17 00:00:00 2001 From: Ziyan Date: Thu, 28 Apr 2022 17:51:42 +0900 Subject: [PATCH 4/5] Reuse stack allocator, but clear it when the stack becomes empty. --- include/rapidjson/document.h | 4 ++++ include/rapidjson/internal/stack.h | 3 +++ 2 files changed, 7 insertions(+) diff --git a/include/rapidjson/document.h b/include/rapidjson/document.h index 5e60744c0..d7390af77 100644 --- a/include/rapidjson/document.h +++ b/include/rapidjson/document.h @@ -2785,6 +2785,10 @@ class GenericDocument : public GenericValue { //! Get the capacity of stack in bytes. size_t GetStackCapacity() const { return stack_.GetCapacity(); } + StackAllocatorType& GetStackAllocator() { + return stack_.GetAllocator(); + } + private: // clear stack on any exit from ParseStream, e.g. due to exception struct ClearStackOnExit { diff --git a/include/rapidjson/internal/stack.h b/include/rapidjson/internal/stack.h index 73abd706e..f1e1f05c8 100644 --- a/include/rapidjson/internal/stack.h +++ b/include/rapidjson/internal/stack.h @@ -39,6 +39,8 @@ class Stack { // Optimization note: Do not allocate memory for stack_ in constructor. // Do it lazily when first Push() -> Expand() -> Resize(). Stack(Allocator* allocator, size_t stackCapacity) : allocator_(allocator), ownAllocator_(0), stack_(0), stackTop_(0), stackEnd_(0), initialCapacity_(stackCapacity) { + if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS @@ -105,6 +107,7 @@ class Stack { stack_ = 0; stackTop_ = 0; stackEnd_ = 0; + allocator_->Clear(); } else Resize(GetSize()); From 2518b13a2dfe46c5b197dacefa0f9708134ea49c Mon Sep 17 00:00:00 2001 From: Ziyan Date: Fri, 29 Apr 2022 14:32:56 +0900 Subject: [PATCH 5/5] Provide a Clear for CrtAllocator as well since some stack still uses it. --- include/rapidjson/allocators.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/rapidjson/allocators.h b/include/rapidjson/allocators.h index 84b8d5779..380b3cc33 100644 --- a/include/rapidjson/allocators.h +++ b/include/rapidjson/allocators.h @@ -104,6 +104,8 @@ class CrtAllocator { bool operator!=(const CrtAllocator&) const RAPIDJSON_NOEXCEPT { return false; } + + void Clear() RAPIDJSON_NOEXCEPT {} }; ///////////////////////////////////////////////////////////////////////////////