Merge pull request #396 from agri-gaia/feat/defer-fb-point-index-crea…

…tion

Defer index creation when storing points using the flatbuffers service

seerep_srv/seerep_core_fb/include/seerep_core_fb/core_fb_point.h

@@ -71,15 +71,31 @@ class CoreFbPoint
       grpc::ServerWriter<
           flatbuffers::grpc::Message<seerep::fb::PointStamped>>* const writer);
   /**
-   * @brief Add an point to the indices and write the data to hdf5
-   * @param img the flatbuffer message containing the point
+   * @brief write the point data to hdf5
+   * @param point the flatbuffer message containing the point
    * @return the uuid of the stored point
    *
-   * The point is stored in the hdf5 file via hdf5-io-fb. The data needed for
-   * the indices is extracted and added to the core. If the uuid of point is not
-   * defined yet, a uuid is generated and returned.
+   * The point is stored in the hdf5 file via hdf5-io-fb. If the uuid of point
+   * is not defined yet, a uuid is generated and returned.
    */
-  boost::uuids::uuid addData(const seerep::fb::PointStamped* point);
+  boost::uuids::uuid addDataToHdf5(const seerep::fb::PointStamped* point);
+
+  /**
+   * @brief Extract point data from hdf5 and build the indices.
+   *
+   * This method complements @ref addDataToHdf5 and should be called after
+   * the data has been added to hdf5. The data for the indices is retrieved from
+   * hdf5 and added to the indices to the core.
+   *
+   * @param projectPointUuids a vector of pairs of first project uuids and
+   * second point uuids, where the point belongs to the specified project
+   *
+   * @see seerep_core_fb::CoreFbPoint::addDataToHdf5
+   */
+  void
+  buildIndices(const std::vector<std::pair<std::string, boost::uuids::uuid>>&
+                   projectPointUuids);
+
   /**
    * @brief Adds attributes to an existing point
    * @param uuidAttributePair a pair of the uuid of the targeted point and the attributes

seerep_srv/seerep_core_fb/src/core_fb_point.cpp

@@ -30,23 +30,24 @@ void CoreFbPoint::getData(
     BOOST_LOG_SEV(m_logger, boost::log::trivial::severity_level::info)
         << "sending images from project"
         << boost::lexical_cast<std::string>(project.projectUuid);
-    for (auto uuidImg : project.dataOrInstanceUuids)
+    for (auto uuidPoint : project.dataOrInstanceUuids)
     {
       auto point =
           CoreFbGeneral::getHdf5(project.projectUuid, m_seerepCore, m_hdf5IoMap)
-              ->readPoint(boost::lexical_cast<std::string>(uuidImg));
+              ->readPoint(boost::lexical_cast<std::string>(uuidPoint));
 
       if (point)
       {
         BOOST_LOG_SEV(m_logger, boost::log::trivial::severity_level::info)
-            << "sending point " << boost::lexical_cast<std::string>(uuidImg);
+            << "sending point " << boost::lexical_cast<std::string>(uuidPoint);
         writer->Write(point.value());
       }
     }
   }
 }
 
-boost::uuids::uuid CoreFbPoint::addData(const seerep::fb::PointStamped* point)
+boost::uuids::uuid
+CoreFbPoint::addDataToHdf5(const seerep::fb::PointStamped* point)
 {
   seerep_core_msgs::DatasetIndexable dataForIndices =
       CoreFbConversion::fromFb(point);
@@ -56,11 +57,29 @@ boost::uuids::uuid CoreFbPoint::addData(const seerep::fb::PointStamped* point)
   hdf5io->writePoint(
       boost::lexical_cast<std::string>(dataForIndices.header.uuidData), point);
 
-  m_seerepCore->addDataset(dataForIndices);
-
   return dataForIndices.header.uuidData;
 }
 
+void CoreFbPoint::buildIndices(
+    const std::vector<std::pair<std::string, boost::uuids::uuid>>&
+        projectPointUuids)
+{
+  for (auto [projectUuid, pointUuid] : projectPointUuids)
+  {
+    auto point = CoreFbGeneral::getHdf5(projectUuid, m_seerepCore, m_hdf5IoMap)
+                     ->readPoint(boost::lexical_cast<std::string>(pointUuid));
+
+    if (!point.has_value())
+    {
+      throw std::runtime_error("points couldn't be retrieved correctly!");
+    }
+    seerep_core_msgs::DatasetIndexable dataForIndices =
+        CoreFbConversion::fromFb(point.value().GetRoot());
+
+    m_seerepCore->addDataset(dataForIndices);
+  }
+}
+
 void CoreFbPoint::addAttributes(
     const seerep::fb::AttributesStamped& attributesStamped)
 {

seerep_srv/seerep_server/src/fb_point_service.cpp

@@ -110,6 +110,9 @@ grpc::Status FbPointService::TransferPoint(
   std::string answer = "everything stored!";
 
   flatbuffers::grpc::Message<seerep::fb::PointStamped> pointMsg;
+  std::vector<std::pair<std::string, boost::uuids::uuid>> projectPointUuids;
+
+  // write the point data to hdf5
   while (reader->Read(&pointMsg))
   {
     BOOST_LOG_SEV(m_logger, boost::log::trivial::severity_level::info)
@@ -121,7 +124,8 @@ grpc::Status FbPointService::TransferPoint(
     {
       try
       {
-        pointFb->addData(point);
+        projectPointUuids.push_back(
+            { uuidProject, pointFb->addDataToHdf5(point) });
       }
       catch (std::runtime_error const& e)
       {
@@ -166,6 +170,46 @@ grpc::Status FbPointService::TransferPoint(
     }
   }
 
+  // build the indices from the written data
+  try
+  {
+    pointFb->buildIndices(projectPointUuids);
+  }
+  catch (std::runtime_error const& e)
+  {
+    // mainly catching "invalid uuid string" when transforming uuid_project
+    // from string to uuid also catching core doesn't have project with uuid
+    // error
+    BOOST_LOG_SEV(m_logger, boost::log::trivial::severity_level::error)
+        << e.what();
+
+    seerep_server_util::createResponseFb(std::string(e.what()),
+                                         seerep::fb::TRANSMISSION_STATE_FAILURE,
+                                         response);
+
+    return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, e.what());
+  }
+  catch (const std::exception& e)
+  {
+    // specific handling for all exceptions extending std::exception, except
+    // std::runtime_error which is handled explicitly
+    BOOST_LOG_SEV(m_logger, boost::log::trivial::severity_level::error)
+        << e.what();
+    seerep_server_util::createResponseFb(std::string(e.what()),
+                                         seerep::fb::TRANSMISSION_STATE_FAILURE,
+                                         response);
+    return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, e.what());
+  }
+  catch (...)
+  {
+    // catch any other errors (that we have no information about)
+    std::string msg = "Unknown failure occurred. Possible memory corruption";
+    BOOST_LOG_SEV(m_logger, boost::log::trivial::severity_level::error) << msg;
+    seerep_server_util::createResponseFb(
+        msg, seerep::fb::TRANSMISSION_STATE_FAILURE, response);
+    return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, msg);
+  }
+
   seerep_server_util::createResponseFb(
       answer, seerep::fb::TRANSMISSION_STATE_SUCCESS, response);