From b4842bdf693e11c07a41b37d71234ed436fd89b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 7 Aug 2024 17:48:03 +0200 Subject: [PATCH 1/4] Make shape an optional attribute for constant components --- include/openPMD/backend/Attributable.hpp | 29 ++++++++ src/Iteration.cpp | 3 +- src/ParticleSpecies.cpp | 3 +- src/RecordComponent.cpp | 93 +++++++++++++++++------- 4 files changed, 96 insertions(+), 32 deletions(-) diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index a77d8fe524..6c33897b1c 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -20,6 +20,7 @@ */ #pragma once +#include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/ThrowError.hpp" #include "openPMD/auxiliary/OutOfRangeMsg.hpp" @@ -30,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -106,6 +108,7 @@ namespace internal friend class openPMD::Attributable; using SharedData_t = std::shared_ptr; + using A_MAP = SharedData_t::element_type::A_MAP; public: AttributableData(); @@ -152,6 +155,32 @@ namespace internal std::shared_ptr(self, [](auto const *) {})); return res; } + + inline auto attributes() -> A_MAP & + { + return operator*().m_attributes; + } + [[nodiscard]] inline auto attributes() const -> A_MAP const & + { + return operator*().m_attributes; + } + [[nodiscard]] inline auto + readAttribute(std::string const &name) const -> Attribute const & + { + auto const & attr = attributes(); + if (auto it = attr.find(name); it != attr.end()) + { + return it->second; + } + else + { + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::NotFound, + std::nullopt, + "Not found: '" + name + "'."); + } + } }; template diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 931a1f9e3d..82dc946f89 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -626,8 +626,7 @@ void Iteration::readMeshes(std::string const &meshesPath) auto att_begin = aList.attributes->begin(); auto att_end = aList.attributes->end(); auto value = std::find(att_begin, att_end, "value"); - auto shape = std::find(att_begin, att_end, "shape"); - if (value != att_end && shape != att_end) + if (value != att_end) { MeshRecordComponent &mrc = m; IOHandler()->enqueue(IOTask(&mrc, pOpen)); diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index 4006cc82ba..81e7cdcd8c 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -76,8 +76,7 @@ void ParticleSpecies::read() auto att_begin = aList.attributes->begin(); auto att_end = aList.attributes->end(); auto value = std::find(att_begin, att_end, "value"); - auto shape = std::find(att_begin, att_end, "shape"); - if (value != att_end && shape != att_end) + if (value != att_end) { RecordComponent &rc = r; IOHandler()->enqueue(IOTask(&rc, pOpen)); diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index fc17909fc6..d44a65ee4b 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -108,7 +108,14 @@ RecordComponent &RecordComponent::resetDataset(Dataset d) throw std::runtime_error("Dataset extent must be at least 1D."); if (d.empty()) { - if (d.dtype != Datatype::UNDEFINED) + if (d.extent.empty()) + { + throw error::Internal( + "A zero-dimensional dataset is not to be considered empty, but " + "undefined. This is an internal safeguard against future " + "changes that might not consider this."); + } + else if (d.dtype != Datatype::UNDEFINED) { return makeEmpty(std::move(d)); } @@ -280,6 +287,13 @@ void RecordComponent::flush( { setUnitSI(1); } + auto constant_component_write_shape = [&]() { + auto extent = getExtent(); + return !extent.empty() && + std::none_of(extent.begin(), extent.end(), [](auto val) { + return val == Dataset::JOINED_DIMENSION; + }); + }; if (!written()) { if (constant()) @@ -299,16 +313,20 @@ void RecordComponent::flush( Operation::WRITE_ATT>::ChangesOverSteps::IfPossible; } IOHandler()->enqueue(IOTask(this, aWrite)); - aWrite.name = "shape"; - Attribute a(getExtent()); - aWrite.dtype = a.dtype; - aWrite.resource = a.getResource(); - if (isVBased) + if (constant_component_write_shape()) { - aWrite.changesOverSteps = Parameter< - Operation::WRITE_ATT>::ChangesOverSteps::IfPossible; + + aWrite.name = "shape"; + Attribute a(getExtent()); + aWrite.dtype = a.dtype; + aWrite.resource = a.getResource(); + if (isVBased) + { + aWrite.changesOverSteps = Parameter< + Operation::WRITE_ATT>::ChangesOverSteps::IfPossible; + } + IOHandler()->enqueue(IOTask(this, aWrite)); } - IOHandler()->enqueue(IOTask(this, aWrite)); } else { @@ -326,6 +344,13 @@ void RecordComponent::flush( { if (constant()) { + if (!constant_component_write_shape()) + { + throw error::WrongAPIUsage( + "Extended constant component from a previous shape to " + "one that cannot be written (empty or with joined " + "dimension)."); + } bool isVBased = retrieveSeries().iterationEncoding() == IterationEncoding::variableBased; Parameter aWrite; @@ -390,28 +415,35 @@ namespace }; } // namespace +inline void breakpoint() +{} + void RecordComponent::readBase(bool require_unit_si) { using DT = Datatype; - // auto & rc = get(); - Parameter aRead; + auto &rc = get(); - if (constant() && !empty()) - { - aRead.name = "value"; - IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(internal::defaultFlushParams); + readAttributes(ReadMode::FullyReread); - Attribute a(*aRead.resource); - DT dtype = *aRead.dtype; + auto read_constant = + [&]() // comment for forcing clang-format into putting a newline here + { + Attribute a = rc.readAttribute("value"); + DT dtype = a.dtype; setWritten(false, Attributable::EnqueueAsynchronously::No); switchNonVectorType(dtype, *this, a); setWritten(true, Attributable::EnqueueAsynchronously::No); - aRead.name = "shape"; - IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(internal::defaultFlushParams); - a = Attribute(*aRead.resource); + if (!containsAttribute("shape")) + { + setWritten(false, Attributable::EnqueueAsynchronously::No); + resetDataset(Dataset(dtype, {})); + setWritten(true, Attributable::EnqueueAsynchronously::No); + + return; + } + + a = rc.attributes().at("shape"); Extent e; // uint64_t check @@ -421,7 +453,7 @@ void RecordComponent::readBase(bool require_unit_si) else { std::ostringstream oss; - oss << "Unexpected datatype (" << *aRead.dtype + oss << "Unexpected datatype (" << a.dtype << ") for attribute 'shape' (" << determineDatatype() << " aka uint64_t)"; throw error::ReadError( @@ -434,9 +466,13 @@ void RecordComponent::readBase(bool require_unit_si) setWritten(false, Attributable::EnqueueAsynchronously::No); resetDataset(Dataset(dtype, e)); setWritten(true, Attributable::EnqueueAsynchronously::No); - } + }; - readAttributes(ReadMode::FullyReread); + if (constant() && !empty()) + { + breakpoint(); + read_constant(); + } if (require_unit_si) { @@ -450,7 +486,8 @@ void RecordComponent::readBase(bool require_unit_si) "'" + myPath().openPMDPath() + "'."); } - if (!getAttribute("unitSI").getOptional().has_value()) + if (auto attr = getAttribute("unitSI"); + !attr.getOptional().has_value()) { throw error::ReadError( error::AffectedObject::Attribute, @@ -458,8 +495,8 @@ void RecordComponent::readBase(bool require_unit_si) {}, "Unexpected Attribute datatype for 'unitSI' (expected double, " "found " + - datatypeToString(Attribute(*aRead.resource).dtype) + - ") in '" + myPath().openPMDPath() + "'."); + datatypeToString(attr.dtype) + ") in '" + + myPath().openPMDPath() + "'."); } } } From 2592b8dd32f228b472f7ac0bacc09437c04f4739 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 11 Oct 2024 13:34:05 +0200 Subject: [PATCH 2/4] Adhere to the standard changes more closely --- include/openPMD/RecordComponent.hpp | 3 ++- .../openPMD/backend/MeshRecordComponent.hpp | 3 ++- src/Iteration.cpp | 6 +++++- src/Mesh.cpp | 17 +++++++++++------ src/ParticleSpecies.cpp | 2 ++ src/Record.cpp | 17 +++++++++++------ src/RecordComponent.cpp | 19 ++++++++++++++++++- src/backend/MeshRecordComponent.cpp | 6 ++++-- src/backend/PatchRecord.cpp | 6 +++--- 9 files changed, 58 insertions(+), 21 deletions(-) diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index ee29a6d7fa..2a5ea6adca 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -487,7 +487,8 @@ class RecordComponent : public BaseRecordComponent static constexpr char const *const SCALAR = "\vScalar"; protected: - void flush(std::string const &, internal::FlushParams const &); + void + flush(std::string const &, internal::FlushParams const &, bool is_scalar); void read(bool require_unit_si); private: diff --git a/include/openPMD/backend/MeshRecordComponent.hpp b/include/openPMD/backend/MeshRecordComponent.hpp index d05163d754..f8229635ba 100644 --- a/include/openPMD/backend/MeshRecordComponent.hpp +++ b/include/openPMD/backend/MeshRecordComponent.hpp @@ -47,7 +47,8 @@ class MeshRecordComponent : public RecordComponent MeshRecordComponent(); MeshRecordComponent(NoInit); void read(); - void flush(std::string const &, internal::FlushParams const &); + void + flush(std::string const &, internal::FlushParams const &, bool is_scalar); public: ~MeshRecordComponent() override = default; diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 82dc946f89..021bee3f6b 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -623,10 +623,14 @@ void Iteration::readMeshes(std::string const &meshesPath) IOHandler()->enqueue(IOTask(&m, aList)); IOHandler()->flush(internal::defaultFlushParams); + // Find constant scalar meshes. shape generally required for meshes, + // shape also required for scalars. + // https://github.com/openPMD/openPMD-standard/pull/289 auto att_begin = aList.attributes->begin(); auto att_end = aList.attributes->end(); auto value = std::find(att_begin, att_end, "value"); - if (value != att_end) + auto shape = std::find(att_begin, att_end, "shape"); + if (value != att_end && shape != att_end) { MeshRecordComponent &mrc = m; IOHandler()->enqueue(IOTask(&mrc, pOpen)); diff --git a/src/Mesh.cpp b/src/Mesh.cpp index c5cdefe483..7daf26194c 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -371,12 +371,14 @@ void Mesh::flush_impl( auto &m = get(); if (m.m_datasetDefined) { - T_RecordComponent::flush(SCALAR, flushParams); + T_RecordComponent::flush( + SCALAR, flushParams, /* is_scalar = */ true); } else { for (auto &comp : *this) - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* is_scalar = */ false); } } else @@ -386,7 +388,7 @@ void Mesh::flush_impl( if (scalar()) { MeshRecordComponent &mrc = *this; - mrc.flush(name, flushParams); + mrc.flush(name, flushParams, /* is_scalar = */ true); } else { @@ -396,7 +398,8 @@ void Mesh::flush_impl( for (auto &comp : *this) { comp.second.parent() = &this->writable(); - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* is_scalar = */ false); } } } @@ -404,12 +407,14 @@ void Mesh::flush_impl( { if (scalar()) { - T_RecordComponent::flush(name, flushParams); + T_RecordComponent::flush( + name, flushParams, /* is_scalar = */ true); } else { for (auto &comp : *this) - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* is_scalar = */ false); } } if (!containsAttribute("gridUnitSI")) diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index 81e7cdcd8c..1c3da720ce 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -76,6 +76,8 @@ void ParticleSpecies::read() auto att_begin = aList.attributes->begin(); auto att_end = aList.attributes->end(); auto value = std::find(att_begin, att_end, "value"); + // @todo see this comment: + // https://github.com/openPMD/openPMD-standard/pull/289#issuecomment-2407263974 if (value != att_end) { RecordComponent &rc = r; diff --git a/src/Record.cpp b/src/Record.cpp index 7d41fce5c2..63e9583db4 100644 --- a/src/Record.cpp +++ b/src/Record.cpp @@ -56,12 +56,14 @@ void Record::flush_impl( { if (scalar()) { - T_RecordComponent::flush(SCALAR, flushParams); + T_RecordComponent::flush( + SCALAR, flushParams, /* is_scalar = */ true); } else { for (auto &comp : *this) - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* is_scalar = */ false); } } else @@ -71,7 +73,7 @@ void Record::flush_impl( if (scalar()) { RecordComponent &rc = *this; - rc.flush(name, flushParams); + rc.flush(name, flushParams, /* is_scalar = */ true); } else { @@ -81,7 +83,8 @@ void Record::flush_impl( for (auto &comp : *this) { comp.second.parent() = getWritable(this); - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* is_scalar = */ false); } } } @@ -90,12 +93,14 @@ void Record::flush_impl( if (scalar()) { - T_RecordComponent::flush(name, flushParams); + T_RecordComponent::flush( + name, flushParams, /* is_scalar = */ true); } else { for (auto &comp : *this) - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* is_scalar = */ false); } } diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index d44a65ee4b..aaebcf88a7 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -244,7 +244,9 @@ bool RecordComponent::empty() const } void RecordComponent::flush( - std::string const &name, internal::FlushParams const &flushParams) + std::string const &name, + internal::FlushParams const &flushParams, + bool is_scalar) { auto &rc = get(); if (flushParams.flushLevel == FlushLevel::SkeletonOnly) @@ -288,6 +290,21 @@ void RecordComponent::flush( setUnitSI(1); } auto constant_component_write_shape = [&]() { + if (is_scalar) + { + // Must write shape in any case: + // 1. Non-scalar constant components can be distinguished from + // normal components by checking if the backend reports a + // group or a dataset. This does not work for scalar constant + // components, so the parser needs to check if the attributes + // value and shape are there. If they're not, the group is + // not considered as a constant component. + // 2. Scalar constant components are required to write the shape + // by standard anyway since the standard requires that at + // least one component in a record have a shape. For scalars, + // there is only one component, so it must have a shape. + return true; + } auto extent = getExtent(); return !extent.empty() && std::none_of(extent.begin(), extent.end(), [](auto val) { diff --git a/src/backend/MeshRecordComponent.cpp b/src/backend/MeshRecordComponent.cpp index ed50080757..af2e683eb5 100644 --- a/src/backend/MeshRecordComponent.cpp +++ b/src/backend/MeshRecordComponent.cpp @@ -68,14 +68,16 @@ void MeshRecordComponent::read() } void MeshRecordComponent::flush( - std::string const &name, internal::FlushParams const ¶ms) + std::string const &name, + internal::FlushParams const ¶ms, + bool is_scalar) { if (access::write(IOHandler()->m_frontendAccess) && !containsAttribute("position")) { setPosition(std::vector{0}); } - RecordComponent::flush(name, params); + RecordComponent::flush(name, params, is_scalar); } template diff --git a/src/backend/PatchRecord.cpp b/src/backend/PatchRecord.cpp index 5d2b38d50f..2b0baf6db1 100644 --- a/src/backend/PatchRecord.cpp +++ b/src/backend/PatchRecord.cpp @@ -41,17 +41,17 @@ PatchRecord::setUnitDimension(std::map const &udim) void PatchRecord::flush_impl( std::string const &path, internal::FlushParams const &flushParams) { - if (!this->datasetDefined()) + if (!this->scalar()) { if (IOHandler()->m_frontendAccess != Access::READ_ONLY) Container::flush( path, flushParams); // warning (clang-tidy-10): // bugprone-parent-virtual-call for (auto &comp : *this) - comp.second.flush(comp.first, flushParams); + comp.second.flush(comp.first, flushParams, /* is_scalar = */ false); } else - T_RecordComponent::flush(path, flushParams); + T_RecordComponent::flush(path, flushParams, /* is_scalar = */ true); if (flushParams.flushLevel != FlushLevel::SkeletonOnly) { setDirty(false); From 7b4a6d4eafc4add6a8e696038ea81d474930403e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 11 Oct 2024 15:08:21 +0000 Subject: [PATCH 3/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- include/openPMD/backend/Attributable.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 6c33897b1c..00d20aa99a 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -164,8 +164,8 @@ namespace internal { return operator*().m_attributes; } - [[nodiscard]] inline auto - readAttribute(std::string const &name) const -> Attribute const & + [[nodiscard]] inline auto readAttribute(std::string const &name) const + -> Attribute const & { auto const & attr = attributes(); if (auto it = attr.find(name); it != attr.end()) From 1b835b2625338b4c6feaf3b44e734d16c258b82d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 11 Dec 2024 09:14:45 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- include/openPMD/backend/Attributable.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 00d20aa99a..7713d729a1 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -167,7 +167,7 @@ namespace internal [[nodiscard]] inline auto readAttribute(std::string const &name) const -> Attribute const & { - auto const & attr = attributes(); + auto const &attr = attributes(); if (auto it = attr.find(name); it != attr.end()) { return it->second;