Skip to content

Commit

Permalink
No constexpr rank via constexpr min-max rank.
Browse files Browse the repository at this point in the history 
The commit removes the requirement for a constexpr rank. Instead
containers need a minimum and maximum constexpr rank.

The difficulties are:

  * Empty arrays can't figure out their runtime rank. This is an issue
    when deducing the dimension when writing to disk. The "solution" is
    to deduce with lowest rank possible.

  * Broadcasting happens before the array has been resized. This is an
    issue when reading, because determining the dimensions for resizing
    the array requires the rank of the array. Which in general can't be
    determined until after resizing. For variable rank arrays, as little
    broadcasting as possible happens.
  • Loading branch information
@1uc
1uc committed Feb 3, 2024
1 parent d1402a4 commit 77c0b28
Show file tree
Hide file tree
Showing 9 changed files with 161 additions and 40 deletions.
18 changes: 10 additions & 8 deletions include/highfive/bits/H5Attribute_misc.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

#include "../H5DataSpace.hpp"
#include "H5Converter_misc.hpp"
#include "H5Inspector_misc.hpp"
#include "H5ReadWrite_misc.hpp"
#include "H5Utils.hpp"
#include "h5a_wrapper.hpp"
Expand Down Expand Up @@ -66,17 +67,17 @@ inline void Attribute::read(T& array) const {
[this]() -> std::string { return this->getName(); },
details::BufferInfo<T>::Operation::read);

if (!details::checkDimensions(mem_space, buffer_info.n_dimensions)) {
if (!details::checkDimensions(mem_space, buffer_info.getMinRank(), buffer_info.getMaxRank())) {
std::ostringstream ss;
ss << "Impossible to read Attribute of dimensions " << mem_space.getNumberDimensions()
<< " into arrays of dimensions " << buffer_info.n_dimensions;
ss << "Impossible to read attribute of dimensions " << mem_space.getNumberDimensions()
<< " into arrays of dimensions: " << buffer_info.getMinRank() << "(min) to "
<< buffer_info.getMaxRank() << "(max)";
throw DataSpaceException(ss.str());
}
auto dims = mem_space.getDimensions();

if (mem_space.getElementCount() == 0) {
auto effective_dims = details::squeezeDimensions(dims,
details::inspector<T>::recursive_ndim);
auto effective_dims = details::squeezeDimensions(dims, details::inspector<T>::max_ndim);

details::inspector<T>::prepare(array, effective_dims);
return;
Expand Down Expand Up @@ -132,10 +133,11 @@ inline void Attribute::write(const T& buffer) {
[this]() -> std::string { return this->getName(); },
details::BufferInfo<T>::Operation::write);

if (!details::checkDimensions(mem_space, buffer_info.n_dimensions)) {
if (!details::checkDimensions(mem_space, buffer_info.getMinRank(), buffer_info.getMaxRank())) {
std::ostringstream ss;
ss << "Impossible to write buffer of dimensions " << buffer_info.n_dimensions
<< " into dataset of dimensions " << mem_space.getNumberDimensions();
ss << "Impossible to write attribute of dimensions " << mem_space.getNumberDimensions()
<< " into arrays of dimensions: " << buffer_info.getMinRank() << "(min) to "
<< buffer_info.getMaxRank() << "(max)";
throw DataSpaceException(ss.str());
}
auto w = details::data_converter::serialize<T>(buffer, file_datatype);
Expand Down
2 changes: 1 addition & 1 deletion include/highfive/bits/H5Converter_misc.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -411,7 +411,7 @@ struct data_converter {
T& val,
const DataType& file_datatype) {
// TODO Use bufferinfo for recursive_ndim
auto effective_dims = details::squeezeDimensions(dims, inspector<T>::recursive_ndim);
auto effective_dims = details::squeezeDimensions(dims, details::inspector<T>::max_ndim);
inspector<T>::prepare(val, effective_dims);
return Reader<T>(effective_dims, val, file_datatype);
}
Expand Down
6 changes: 6 additions & 0 deletions include/highfive/bits/H5Dataspace_misc.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,5 +136,11 @@ inline bool checkDimensions(const DataSpace& mem_space, size_t n_dim_requested)
return checkDimensions(mem_space.getDimensions(), n_dim_requested);
}

inline bool checkDimensions(const DataSpace& mem_space,
size_t min_dim_requested,
size_t max_dim_requested) {
return checkDimensions(mem_space.getDimensions(), min_dim_requested, max_dim_requested);
}

} // namespace details
} // namespace HighFive
89 changes: 75 additions & 14 deletions include/highfive/bits/H5Inspector_misc.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,18 @@
namespace HighFive {
namespace details {

inline bool checkDimensions(const std::vector<size_t>& dims, size_t n_dim_requested) {
inline bool checkDimensions(const std::vector<size_t>& dims,
size_t min_dim_requested,
size_t max_dim_requested) {
size_t n_dim_actual = dims.size();

// Empty, non-scalar is always good.
if (compute_total_size(dims) == 0ul) {
return (n_dim_actual == 0ul) == (max_dim_requested == 0ul);
}

// We should allow reading scalar from shapes like `(1, 1, 1)`.
if (n_dim_requested == 0) {
if (max_dim_requested == 0) {
if (n_dim_actual == 0ul) {
return true;
}
Expand All @@ -41,20 +48,20 @@ inline bool checkDimensions(const std::vector<size_t>& dims, size_t n_dim_reques

// For non-scalar datasets, we can squeeze away singleton dimension, but
// we never add any.
if (n_dim_actual < n_dim_requested) {
if (n_dim_actual < min_dim_requested) {
return false;
}

// Special case for 1-dimensional arrays, which can squeeze `1`s from either
// side simultaneously if needed.
if (n_dim_requested == 1ul) {
if (min_dim_requested <= 1ul && 1ul <= max_dim_requested) {
return n_dim_actual >= 1ul &&
size_t(std::count(dims.begin(), dims.end(), 1ul)) >= n_dim_actual - 1ul;
}

// All other cases strip front only. This avoid unstable behaviour when
// squeezing singleton dimensions.
size_t n_dim_excess = n_dim_actual - n_dim_requested;
size_t n_dim_excess = n_dim_actual - std::min(max_dim_requested, n_dim_actual);

bool squeeze_back = true;
for (size_t i = 1; i <= n_dim_excess; ++i) {
Expand All @@ -67,6 +74,10 @@ inline bool checkDimensions(const std::vector<size_t>& dims, size_t n_dim_reques
return squeeze_back;
}

inline bool checkDimensions(const std::vector<size_t>& dims, size_t n_dim_requested) {
return checkDimensions(dims, n_dim_requested, n_dim_requested);
}


inline std::vector<size_t> squeezeDimensions(const std::vector<size_t>& dims,
size_t n_dim_requested) {
Expand Down Expand Up @@ -163,9 +174,15 @@ struct type_helper {
using hdf5_type = base_type;

static constexpr size_t ndim = 0;
static constexpr size_t recursive_ndim = ndim;
static constexpr size_t min_ndim = ndim;
static constexpr size_t max_ndim = ndim;

static constexpr bool is_trivially_copyable = std::is_trivially_copyable<type>::value;

static size_t getRank(const type& /* val */) {
return ndim;
}

static std::vector<size_t> getDimensions(const type& /* val */) {
return {};
}
Expand Down Expand Up @@ -296,9 +313,15 @@ struct inspector<deprecated::FixedLenStringArray<N>> {
using hdf5_type = char;

static constexpr size_t ndim = 1;
static constexpr size_t recursive_ndim = ndim;
static constexpr size_t min_ndim = ndim;
static constexpr size_t max_ndim = ndim;

static constexpr bool is_trivially_copyable = false;

static size_t getRank(const type& /* val */) {
return ndim;
}

static std::vector<size_t> getDimensions(const type& val) {
return std::vector<size_t>{val.size()};
}
Expand Down Expand Up @@ -351,16 +374,25 @@ struct inspector<std::vector<T>> {
using hdf5_type = typename inspector<value_type>::hdf5_type;

static constexpr size_t ndim = 1;
static constexpr size_t recursive_ndim = ndim + inspector<value_type>::recursive_ndim;
static constexpr size_t min_ndim = ndim + inspector<value_type>::min_ndim;
static constexpr size_t max_ndim = ndim + inspector<value_type>::max_ndim;

static constexpr bool is_trivially_copyable = std::is_trivially_copyable<value_type>::value &&
inspector<value_type>::is_trivially_copyable;

static size_t getRank(const type& val) {
if (!val.empty()) {
return ndim + inspector<value_type>::getRank(val[0]);
} else {
return ndim;
}
}

static std::vector<size_t> getDimensions(const type& val) {
std::vector<size_t> sizes(recursive_ndim, 1ul);
std::vector<size_t> sizes(min_ndim, 1ul);
sizes[0] = val.size();
if (!val.empty()) {
auto s = inspector<value_type>::getDimensions(val[0]);
assert(s.size() + ndim == sizes.size());
for (size_t i = 0; i < s.size(); ++i) {
sizes[i + ndim] = s[i];
}
Expand Down Expand Up @@ -421,9 +453,15 @@ struct inspector<std::vector<bool>> {
using hdf5_type = uint8_t;

static constexpr size_t ndim = 1;
static constexpr size_t recursive_ndim = ndim;
static constexpr size_t min_ndim = ndim;
static constexpr size_t max_ndim = ndim;

static constexpr bool is_trivially_copyable = false;

static size_t getRank(const type& /* val */) {
return ndim;
}

static std::vector<size_t> getDimensions(const type& val) {
std::vector<size_t> sizes{val.size()};
return sizes;
Expand Down Expand Up @@ -478,11 +516,17 @@ struct inspector<std::array<T, N>> {
using hdf5_type = typename inspector<value_type>::hdf5_type;

static constexpr size_t ndim = 1;
static constexpr size_t recursive_ndim = ndim + inspector<value_type>::recursive_ndim;
static constexpr size_t min_ndim = ndim + inspector<value_type>::min_ndim;
static constexpr size_t max_ndim = ndim + inspector<value_type>::max_ndim;

static constexpr bool is_trivially_copyable = std::is_trivially_copyable<value_type>::value &&
sizeof(type) == N * sizeof(T) &&
inspector<value_type>::is_trivially_copyable;

static size_t getRank(const type& val) {
return ndim + inspector<value_type>::getRank(val[0]);
}

static std::vector<size_t> getDimensions(const type& val) {
std::vector<size_t> sizes{N};
if (!val.empty()) {
Expand Down Expand Up @@ -555,10 +599,20 @@ struct inspector<T*> {
using hdf5_type = typename inspector<value_type>::hdf5_type;

static constexpr size_t ndim = 1;
static constexpr size_t recursive_ndim = ndim + inspector<value_type>::recursive_ndim;
static constexpr size_t min_ndim = ndim + inspector<value_type>::min_ndim;
static constexpr size_t max_ndim = ndim + inspector<value_type>::max_ndim;

static constexpr bool is_trivially_copyable = std::is_trivially_copyable<value_type>::value &&
inspector<value_type>::is_trivially_copyable;

static size_t getRank(const type& val) {
if (val != nullptr) {
return ndim + inspector<value_type>::getRank(val[0]);
} else {
return ndim;
}
}

static size_t getSizeVal(const type& /* val */) {
throw DataSpaceException("Not possible to have size of a T*");
}
Expand Down Expand Up @@ -587,7 +641,9 @@ struct inspector<T[N]> {
using hdf5_type = typename inspector<value_type>::hdf5_type;

static constexpr size_t ndim = 1;
static constexpr size_t recursive_ndim = ndim + inspector<value_type>::recursive_ndim;
static constexpr size_t min_ndim = ndim + inspector<value_type>::min_ndim;
static constexpr size_t max_ndim = ndim + inspector<value_type>::max_ndim;

static constexpr bool is_trivially_copyable = std::is_trivially_copyable<value_type>::value &&
inspector<value_type>::is_trivially_copyable;

Expand All @@ -606,6 +662,10 @@ struct inspector<T[N]> {
}
}

static size_t getRank(const type& val) {
return ndim + inspector<value_type>::getRank(val[0]);
}

static size_t getSizeVal(const type& val) {
return compute_total_size(getDimensions(val));
}
Expand Down Expand Up @@ -637,6 +697,7 @@ struct inspector<T[N]> {
}
};


} // namespace details
} // namespace HighFive

Expand Down
27 changes: 23 additions & 4 deletions include/highfive/bits/H5ReadWrite_misc.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#pragma once

#include <H5Tpublic.h>
#include "H5Inspector_misc.hpp"
#include "H5Utils.hpp"

namespace HighFive {
Expand Down Expand Up @@ -57,10 +58,14 @@ struct BufferInfo {
template <class F>
BufferInfo(const DataType& dtype, F getName, Operation _op);

size_t getRank(const T& array) const;
size_t getMinRank() const;
size_t getMaxRank() const;

// member data for info depending on the destination dataset type
const bool is_fixed_len_string;
const size_t n_dimensions;
const DataType data_type;
const size_t rank_correction;
};

// details implementation
Expand Down Expand Up @@ -135,10 +140,9 @@ BufferInfo<T>::BufferInfo(const DataType& file_data_type, F getName, Operation _
: op(_op)
, is_fixed_len_string(file_data_type.isFixedLenStr())
// In case we are using Fixed-len strings we need to subtract one dimension
, n_dimensions(details::inspector<type_no_const>::recursive_ndim -
((is_fixed_len_string && is_char_array) ? 1 : 0))
, data_type(string_type_checker<char_array_t>::getDataType(create_datatype<elem_type>(),
file_data_type)) {
file_data_type))
, rank_correction((is_fixed_len_string && is_char_array) ? 1 : 0) {
// We warn. In case they are really not convertible an exception will rise on read/write
if (file_data_type.getClass() != data_type.getClass()) {
HIGHFIVE_LOG_WARN(getName() + "\": data and hdf5 dataset have different types: " +
Expand All @@ -157,6 +161,21 @@ BufferInfo<T>::BufferInfo(const DataType& file_data_type, F getName, Operation _
}
}

template <typename T>
size_t BufferInfo<T>::getRank(const T& array) const {
return details::inspector<type_no_const>::getRank(array) - rank_correction;
}

template <typename T>
size_t BufferInfo<T>::getMinRank() const {
return details::inspector<T>::min_ndim - rank_correction;
}

template <typename T>
size_t BufferInfo<T>::getMaxRank() const {
return details::inspector<T>::max_ndim - rank_correction;
}

} // namespace details

} // namespace HighFive
9 changes: 5 additions & 4 deletions include/highfive/bits/H5Slice_traits_misc.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,10 +177,11 @@ inline void SliceTraits<Derivate>::read(T& array, const DataTransferProps& xfer_
[&slice]() -> std::string { return details::get_dataset(slice).getPath(); },
details::BufferInfo<T>::Operation::read);

if (!details::checkDimensions(mem_space, buffer_info.n_dimensions)) {
if (!details::checkDimensions(mem_space, buffer_info.getMinRank(), buffer_info.getMaxRank())) {
std::ostringstream ss;
ss << "Impossible to read DataSet of dimensions " << mem_space.getNumberDimensions()
<< " into arrays of dimensions " << buffer_info.n_dimensions;
<< " into arrays of dimensions: " << buffer_info.getMinRank() << "(min) to "
<< buffer_info.getMaxRank() << "(max)";
throw DataSpaceException(ss.str());
}
auto dims = mem_space.getDimensions();
Expand Down Expand Up @@ -249,11 +250,11 @@ inline void SliceTraits<Derivate>::write(const T& buffer, const DataTransferProp
[&slice]() -> std::string { return details::get_dataset(slice).getPath(); },
details::BufferInfo<T>::Operation::write);

if (!details::checkDimensions(mem_space, buffer_info.n_dimensions)) {
if (!details::checkDimensions(mem_space, buffer_info.getRank(buffer))) {
std::ostringstream ss;
ss << "Impossible to write buffer of dimensions "
<< details::format_vector(mem_space.getDimensions())
<< " into dataset with n = " << buffer_info.n_dimensions << " dimensions.";
<< " into dataset with n = " << buffer_info.getRank(buffer) << " dimensions.";
throw DataSpaceException(ss.str());
}
auto w = details::data_converter::serialize<T>(buffer, file_datatype);
Expand Down
Loading

0 comments on commit 77c0b28

Please sign in to comment.