Merge pull request #779 from kordejong/gh733

Refactor I/O library and make it easy to try out and compare alternative implementations
computationalgeography · Jan 17, 2025 · efc535a · efc535a
2 parents 8fd4522 + 5472c33
commit efc535a
Show file tree

Hide file tree

Showing 39 changed files with 1,817 additions and 2,621 deletions.
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
@@ -18,8 +18,7 @@ jobs:
                     # To be recognized by Conan, name must be one of:
                     #     'Visual Studio', 'apple-clang', 'clang', 'gcc', 'intel', 'intel-cc', 'mcst-lcc',
                     #     'msvc', 'qcc', 'sun-cc'
-                    - { name: gcc  , version: 10, c: gcc-10  , cxx: g++-10    , package: g++-10   }
-                    - { name: gcc  , version: 12, c: gcc-12  , cxx: g++-12    , package: g++-12   }
+                    - { name: gcc  , version: 13, c: gcc-13  , cxx: g++-13    , package: g++-13   }
                     - { name: clang, version: 18, c: clang-18, cxx: clang++-18, package: clang-18 }
             fail-fast: false
 

diff --git a/environment/cmake/Lue.cmake b/environment/cmake/Lue.cmake
@@ -95,6 +95,7 @@ message(STATUS "+ python api                  : ${LUE_DATA_MODEL_WITH_PYTHON_API
 message(STATUS "+ utilities                   : ${LUE_DATA_MODEL_WITH_UTILITIES}")
 message(STATUS "Build framework               : ${LUE_BUILD_FRAMEWORK}")
 message(STATUS "+ python api                  : ${LUE_FRAMEWORK_WITH_PYTHON_API}")
+message(STATUS "+ parallel i/o                : ${LUE_FRAMEWORK_WITH_PARALLEL_IO}")
 message(STATUS "Build view                    : ${LUE_BUILD_VIEW}")
 message(STATUS "+ value inspection            : ${LUE_BUILD_FRAMEWORK}")
 message(STATUS "Build documentation           : ${LUE_BUILD_DOCUMENTATION}")

diff --git a/environment/cmake/LueConfiguration.cmake b/environment/cmake/LueConfiguration.cmake
@@ -1,4 +1,5 @@
 include(CheckCXXSymbolExists)
+include(CMakeDependentOption)
 
 
 # Options to be set by the user ------------------------------------------------
@@ -442,6 +443,7 @@ if(LUE_BUILD_DOCUMENTATION)
     set(LUE_GRAPHVIZ_REQUIRED TRUE)
     set(LUE_JUPYTER_BOOK_REQUIRED TRUE)
     set(LUE_SPHINX_REQUIRED TRUE)
+    set(LUE_DOXYGEN_AWESOME_CSS_REQUIRED TRUE)
 
     find_program(EDIT_DOT_GRAPH
         edit_dot_graph.py
@@ -730,6 +732,9 @@ if(LUE_HPX_REQUIRED)
     #             endif()
     #         endif()
     endif()
+
+    message(STATUS "HPX_WITH_NETWORKING           : ${HPX_WITH_NETWORKING}")
+    message(STATUS "HPX_WITH_PARCELPORT_MPI       : ${HPX_WITH_PARCELPORT_MPI}")
 endif()
 
 
@@ -765,6 +770,25 @@ if(LUE_DOXYGEN_REQUIRED)
 endif()
 
 
+if(LUE_DOXYGEN_AWESOME_CSS_REQUIRED)
+    FetchContent_Declare(doxygen-awesome-css
+        GIT_REPOSITORY https://github.com/jothepro/doxygen-awesome-css.git
+        GIT_TAG 568f56cde6ac78b6dfcc14acd380b2e745c301ea  # v2.3.4
+        # URL https://github.com/jothepro/doxygen-awesome-css/archive/refs/heads/main.zip
+    )
+    FetchContent_MakeAvailable(doxygen-awesome-css)
+
+    # Save the location the files were cloned into
+    # This allows us to get the path to doxygen-awesome.css
+    FetchContent_GetProperties(doxygen-awesome-css SOURCE_DIR AWESOME_CSS_DIR)
+
+    # # Generate the Doxyfile
+    # set(DOXYFILE_IN ${CMAKE_CURRENT_SOURCE_DIR}/doc/Doxyfile.in)
+    # set(DOXYFILE_OUT ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile)
+    # configure_file(${DOXYFILE_IN} ${DOXYFILE_OUT} @ONLY)
+endif()
+
+
 if(LUE_GDAL_REQUIRED)
     # https://gdal.org/en/stable/development/cmake.html
     find_package(GDAL REQUIRED CONFIG)
@@ -799,7 +823,11 @@ if(LUE_HDF5_REQUIRED)
     # Note that Conan prefers Config Mode (it sets CMAKE_FIND_PACKAGE_PREFER_CONFIG in the
     # toolchain).
     find_package(HDF5 MODULE REQUIRED COMPONENTS C)
+    set(CMAKE_REQUIRED_INCLUDES "${HDF5_C_INCLUDE_DIRS}")
+    check_symbol_exists(H5_HAVE_THREADSAFE "hdf5.h" HDF5_IS_THREADSAFE)
+    unset(CMAKE_REQUIRED_INCLUDED)
     message(STATUS "HDF5_IS_PARALLEL              : ${HDF5_IS_PARALLEL}")
+    message(STATUS "HDF5_IS_THREADSAFE            : ${HDF5_IS_THREADSAFE}")
 endif()
 
 if(LUE_JUPYTER_BOOK_REQUIRED)
@@ -825,3 +853,12 @@ if(LUE_SPHINX_REQUIRED)
         message(FATAL_ERROR "sphinx not found")
     endif()
 endif()
+
+# Only allow the user to configure the use of parallel I/O if this is something that is supported by the
+# platform. If so, the default is to support parallel I/O.
+cmake_dependent_option(LUE_FRAMEWORK_WITH_PARALLEL_IO
+    "Use parallel I/O for formats that support it"
+    TRUE
+    "LUE_BUILD_DATA_MODEL;LUE_HDF5_REQUIRED;HDF5_IS_PARALLEL;LUE_BUILD_FRAMEWORK;LUE_HPX_REQUIRED;HPX_WITH_NETWORKING;HPX_WITH_PARCELPORT_MPI"
+    FALSE
+)
diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt
@@ -58,6 +58,14 @@ if(LUE_BUILD_DOCUMENTATION)
     ### )
 
     # set(DOXYGEN_ENABLE_PREPROCESSING FALSE)
+    set(DOXYGEN_GENERATE_TREEVIEW TRUE)  # optional. Also works without treeview
+    set(DOXYGEN_DISABLE_INDEX FALSE)
+    set(DOXYGEN_FULL_SIDEBAR FALSE)
+    set(DOXYGEN_HTML_EXTRA_STYLESHEET "${AWESOME_CSS_DIR}/doxygen-awesome.css")
+    set(DOXYGEN_EXCLUDE_SYMBOLS "*::detail::*")
+    set(DOXYGEN_HTML_COLORSTYLE "LIGHT") # required with Doxygen >= 1.9.5
+    set(DOXYGEN_HIDE_UNDOC_MEMBERS TRUE)
+
     doxygen_add_docs(cpp_doc
         doc
         shared

diff --git a/source/data_model/hdf5/include/configure.hpp.in b/source/data_model/hdf5/include/configure.hpp.in
@@ -2,10 +2,10 @@
 
 
 #cmakedefine HDF5_IS_PARALLEL
+#cmakedefine HDF5_IS_THREADSAFE
 
 
-namespace lue {
-    namespace hdf5 {
+namespace lue::hdf5 {
 
     class BuildOptions
     {
@@ -20,7 +20,14 @@ namespace lue {
 #endif
                 };
 
+            static constexpr bool hdf5_is_threadsafe{
+#ifdef HDF5_IS_THREADSAFE
+                    true
+#else
+                    false
+#endif
+                };
+
         };
 
-    }  // namespace hdf5
-}  // namespace lue
+} // namespace lue::hdf5
diff --git a/source/framework/api/cxx/source/io.cpp b/source/framework/api/cxx/source/io.cpp
@@ -3,7 +3,7 @@
 #include "fmt/format.h"
 #include "overload.hpp"
 #include "lue/framework/core/domain_decomposition.hpp"
-#include "lue/framework/io/raster.hpp"
+#include "lue/framework/io/gdal.hpp"
 #include "lue/framework.hpp"
 #include "lue/gdal.hpp"
 // #include <any>
@@ -43,7 +43,7 @@ namespace lue {
 
                 if constexpr (lue::arithmetic_element_supported<Element>)
                 {
-                    result = read<Element>(name, partition_shape);
+                    result = lue::from_gdal<Element>(name, partition_shape);
                 }
 
                 return result;

diff --git a/source/framework/core/include/lue/framework/configure.hpp.in b/source/framework/core/include/lue/framework/configure.hpp.in
@@ -6,9 +6,9 @@
 #include <hpx/config.hpp>
 
 
-#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_MPI)) || defined(HPX_HAVE_MODULE_MPI_BASE)
-#define LUE_HPX_WITH_MPI
-#endif
+// #if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_MPI)) || defined(HPX_HAVE_MODULE_MPI_BASE)
+// #define LUE_HPX_WITH_MPI
+// #endif
 
 
 namespace lue {

diff --git a/source/framework/core/include/lue/framework/core/type_traits.hpp b/source/framework/core/include/lue/framework/core/type_traits.hpp
@@ -1,5 +1,6 @@
 #pragma once
 #include "lue/framework/core/define.hpp"
+#include "lue/concept.hpp"
 #include <string>
 #include <type_traits>
 
@@ -76,12 +77,30 @@ namespace lue {
     using ElementsT = typename detail::ArrayTraits<detail::remove_cvref_t<Array>>::Elements;
 
 
+    namespace detail {
+
+        template<typename Shaped>
+        struct ShapedTraits
+        {
+                static constexpr Rank rank = detail::ArrayTraits<detail::remove_cvref_t<Shaped>>::rank;
+        };
+
+
+        template<TupleLike T>
+        struct ShapedTraits<T>
+        {
+                static constexpr Rank rank = std::tuple_size_v<T>;
+        };
+
+    }  // namespace detail
+
+
     /*!
-        @brief      Rank of an array-like type
-        @tparam     Array-like type
+        @brief      Rank of a type
+        @tparam     Array-like type or a tuple containing extents
     */
-    template<typename Array>
-    static constexpr Rank rank = detail::ArrayTraits<detail::remove_cvref_t<Array>>::rank;
+    template<typename Shaped>
+    static constexpr Rank rank = detail::ShapedTraits<Shaped>::rank;
 
 
     template<typename Array>

diff --git a/source/framework/core/source/debug.cpp b/source/framework/core/source/debug.cpp
@@ -1,9 +1,9 @@
 #include "lue/framework/core/debug.hpp"
 #include "lue/framework/configure.hpp"
 #include <hpx/include/compute.hpp>
-#ifdef LUE_HPX_WITH_MPI
-#include <hpx/mpi_base/mpi_environment.hpp>
-#endif
+// #ifdef LUE_HPX_WITH_MPI
+// #include <hpx/mpi_base/mpi_environment.hpp>
+// #endif
 #include "hpx/runtime_distributed/find_localities.hpp"
 #include "hpx/runtime_distributed/get_locality_name.hpp"
 #include <hpx/include/lcos.hpp>
@@ -78,28 +78,28 @@ namespace lue {
         }
 
 
-#ifdef LUE_HPX_WITH_MPI
-        std::string locality_rank_mapping()
-        {
-            std::ostringstream stream;
-            stream << hpx::get_locality_id();
-
-            return fmt::format(
-                "locality: {} -- rank(HPX): {}",  // -- rank(env): {}",
-                stream.str(),
-                hpx::util::mpi_environment::rank()
-                // , std::getenv("OMPI_COMM_WORLD_RANK")
-            );
-        }
-#endif
+        // #ifdef LUE_HPX_WITH_MPI
+        //         std::string locality_rank_mapping()
+        //         {
+        //             std::ostringstream stream;
+        //             stream << hpx::get_locality_id();
+        //
+        //             return fmt::format(
+        //                 "locality: {} -- rank(HPX): {}",  // -- rank(env): {}",
+        //                 stream.str(),
+        //                 hpx::util::mpi_environment::rank()
+        //                 // , std::getenv("OMPI_COMM_WORLD_RANK")
+        //             );
+        //         }
+        // #endif
 
     }  // Anonymous namespace
 }  // namespace lue
 
 
-#ifdef LUE_HPX_WITH_MPI
-HPX_PLAIN_ACTION(lue::locality_rank_mapping, LocalityRankMapping)
-#endif
+// #ifdef LUE_HPX_WITH_MPI
+// HPX_PLAIN_ACTION(lue::locality_rank_mapping, LocalityRankMapping)
+// #endif
 
 
 namespace lue {
@@ -136,20 +136,20 @@ namespace lue {
         auto all_locality_names = join(lue::all_locality_names(), ", ");
 
 
-#ifdef LUE_HPX_WITH_MPI
-        auto const locality_ids = hpx::find_all_localities();
-        std::vector<hpx::future<std::string>> locality_rank_mappings(locality_ids.size());
-        LocalityRankMapping action{};
-
-        hpx::transform(
-            hpx::execution::par,
-            locality_ids.begin(),
-            locality_ids.end(),
-            locality_rank_mappings.begin(),
-            [action](auto const locality_id) { return hpx::async(action, locality_id); });
-
-        auto all_locality_rank_mappings = join(std::move(locality_rank_mappings), ", ");
-#endif
+        // #ifdef LUE_HPX_WITH_MPI
+        //         auto const locality_ids = hpx::find_all_localities();
+        //         std::vector<hpx::future<std::string>> locality_rank_mappings(locality_ids.size());
+        //         LocalityRankMapping action{};
+        //
+        //         hpx::transform(
+        //             hpx::execution::par,
+        //             locality_ids.begin(),
+        //             locality_ids.end(),
+        //             locality_rank_mappings.begin(),
+        //             [action](auto const locality_id) { return hpx::async(action, locality_id); });
+        //
+        //         auto all_locality_rank_mappings = join(std::move(locality_rank_mappings), ", ");
+        // #endif
 
 
         auto format_message = [this_locality_nr,
@@ -197,11 +197,11 @@ namespace lue {
             root_locality_name,
             current_nr_localities,
             all_locality_names,
-#ifdef LUE_HPX_WITH_MPI
-            all_locality_rank_mappings
-#else
+            // #ifdef LUE_HPX_WITH_MPI
+            //             all_locality_rank_mappings
+            // #else
             "-"
-#endif
+            // #endif
         );
 
         return description;

diff --git a/source/framework/io/CMakeLists.txt b/source/framework/io/CMakeLists.txt
@@ -8,37 +8,17 @@ block()
 
     generate_template_instantiation(
         INPUT_PATHNAME
-            "${CMAKE_CURRENT_SOURCE_DIR}/source/raster.cpp.in"
+            "${CMAKE_CURRENT_SOURCE_DIR}/source/gdal.cpp.in"
         OUTPUT_PATHNAME
-            "${CMAKE_CURRENT_BINARY_DIR}/source/raster.cpp"
-        DICTIONARY
-            '{"Elements":[${Elements}],"ranks":[${LUE_FRAMEWORK_RANKS}]}'
-    )
-
-    generate_template_instantiation(
-        INPUT_PATHNAME
-            "${CMAKE_CURRENT_SOURCE_DIR}/source/read.cpp.in"
-        OUTPUT_PATHNAME
-            "${CMAKE_CURRENT_BINARY_DIR}/source/read.cpp"
-        DICTIONARY
-            '{"Elements":[${Elements}],"ranks":[${LUE_FRAMEWORK_RANKS}]}'
-    )
-
-    generate_template_instantiation(
-        INPUT_PATHNAME
-            "${CMAKE_CURRENT_SOURCE_DIR}/source/write.cpp.in"
-        OUTPUT_PATHNAME
-            "${CMAKE_CURRENT_BINARY_DIR}/source/write.cpp"
+            "${CMAKE_CURRENT_BINARY_DIR}/source/gdal.cpp"
         DICTIONARY
             '{"Elements":[${Elements}],"ranks":[${LUE_FRAMEWORK_RANKS}]}'
     )
 endblock()
 
 add_library(lue_framework_io
-    source/util.cpp
-    ${CMAKE_CURRENT_BINARY_DIR}/source/raster.cpp
-    ${CMAKE_CURRENT_BINARY_DIR}/source/read.cpp
-    ${CMAKE_CURRENT_BINARY_DIR}/source/write.cpp
+    source/dataset.cpp
+    ${CMAKE_CURRENT_BINARY_DIR}/source/gdal.cpp
 )
 
 add_library(lue::framework_io ALIAS lue_framework_io)