Extend CMSSW to a distributed application over MPI

DataFormats/Provenance/interface/BranchPattern.h

@@ -0,0 +1,124 @@
+#ifndef DataFormats_Provenance_interface_BranchPattern_h
+#define DataFormats_Provenance_interface_BranchPattern_h
+
+#include <algorithm>
+#include <string>
+#include <string_view>
+#include <regex>
+#include <vector>
+
+#include <boost/algorithm/string/replace.hpp>
+
+#include "DataFormats/Provenance/interface/BranchDescription.h"
+#include "FWCore/Utilities/interface/EDMException.h"
+
+namespace edm {
+
+  /* BranchPattern
+   *
+   * A BranchPattern is constructed from a string representing either a module label (e.g. "<module label>") or a
+   * a branch name (e.g. "<product type>_<module label>_<instance name>_<process name>").
+   *
+   * A BranchPattern object can be compared with a BranchDescription object using the match() method:
+   *
+   *     branchPattern.match(branch)
+   *
+   * .
+   * Glob expressions ("?" and "*") are supported in module labels and within the individual fields of branch names,
+   * similar to an OutputModule's "keep" statements.
+   * Use "*" to match all products of a given category.
+   *
+   * If a module label is used, it must not contain any underscores ("_"); the resulting BranchPattern will match all
+   * the branches prodced by a module with the given label, including those with a non-empty instance names, and those
+   * produced by the Transformer functionality (such as the implicitly copied-to-host products in case of Alpaka-based
+   * modules).
+   * If a branch name is used, all four fields must be present, separated by underscores; the resulting BranchPattern
+   * will match the branches matching all four fields.
+   *
+   * For example, in the case of products from an Alpaka-based producer running on a device
+   *
+   *     BranchPattern("module")
+   *
+   * would match all branches produced by "module", including the automatic host copy of its device products.
+   * While
+   *
+   *     BranchPattern( "*DeviceProduct_module_*_*" )
+   *
+   * would match only the branches corresponding to the device products.
+   */
+
+  class BranchPattern {
+  public:
+    explicit BranchPattern(std::string const& label) {
+      static const char kSeparator = '_';
+      static const std::string_view kWildcard{"*"};
+      static const std::regex kAny{".*"};
+
+      // wildcard
+      if (label == kWildcard) {
+        type_ = kAny;
+        moduleLabel_ = kAny;
+        productInstanceName_ = kAny;
+        processName_ = kAny;
+        return;
+      }
+
+      int fields = std::count(label.begin(), label.end(), kSeparator) + 1;
+      if (fields == 1) {
+        // convert the module label into a regular expression
+        type_ = kAny;
+        moduleLabel_ = glob_to_regex(label);
+        productInstanceName_ = kAny;
+        processName_ = kAny;
+      } else if (fields == 4) {
+        // split the branch name into <product type>_<module label>_<instance name>_<process name>
+        // and convert the glob expressions into regular expressions
+        size_t first = 0, last = 0;
+        last = label.find(kSeparator, first);
+        type_ = glob_to_regex(label.substr(first, last - first));
+        first = last + 1;
+        last = label.find(kSeparator, first);
+        moduleLabel_ = glob_to_regex(label.substr(first, last - first));
+        first = last + 1;
+        last = label.find(kSeparator, first);
+        productInstanceName_ = glob_to_regex(label.substr(first, last - first));
+        first = last + 1;
+        last = label.find(kSeparator, first);
+        processName_ = glob_to_regex(label.substr(first, last - first));
+      } else {
+        // invalid input
+        throw edm::Exception(edm::errors::Configuration) << "Invalid module label or branch name: \"" << label << "\"";
+      }
+    }
+
+    bool match(edm::BranchDescription const& branch) const {
+      return (std::regex_match(branch.friendlyClassName(), type_) and
+              std::regex_match(branch.moduleLabel(), moduleLabel_) and
+              std::regex_match(branch.productInstanceName(), productInstanceName_) and
+              std::regex_match(branch.processName(), processName_));
+    }
+
+  private:
+    static std::regex glob_to_regex(std::string pattern) {
+      boost::replace_all(pattern, "*", ".*");
+      boost::replace_all(pattern, "?", ".");
+      return std::regex(pattern);
+    }
+
+    std::regex type_;
+    std::regex moduleLabel_;
+    std::regex productInstanceName_;
+    std::regex processName_;
+  };
+
+  inline std::vector<BranchPattern> branchPatterns(std::vector<std::string> const& labels) {
+    std::vector<BranchPattern> patterns;
+    patterns.reserve(labels.size());
+    for (auto const& label : labels)
+      patterns.emplace_back(label);
+    return patterns;
+  }
+
+}  // namespace edm
+
+#endif  // DataFormats_Provenance_interface_BranchPattern_h

FWCore/Framework/interface/GenericProduct.h

@@ -0,0 +1,72 @@
+#include <cstring>
+#include <memory>
+#include <typeinfo>
+#include <utility>
+
+#include "DataFormats/Common/interface/OrphanHandle.h"
+#include "FWCore/Framework/interface/Event.h"
+#include "FWCore/Reflection/interface/ObjectWithDict.h"
+#include "FWCore/Reflection/interface/TypeWithDict.h"
+
+namespace edm {
+
+  class GenericProduct {
+  public:
+    // TODO make private and add accessors and a constructor
+    TypeWithDict wrappedType_;
+    ObjectWithDict object_;
+  };
+
+  template <>
+  class OrphanHandle<GenericProduct> : public OrphanHandleBase {
+  public:
+    OrphanHandle(ObjectWithDict prod, ProductID const& id) : OrphanHandleBase(prod.address(), id) {}
+  };
+
+  // specialise Event::putImpl for GenericProduct
+  template <>
+  OrphanHandle<GenericProduct> Event::putImpl(EDPutToken::value_type index, std::unique_ptr<GenericProduct> product) {
+    /* TODO implement for ObjectWithDict
+    // The following will call post_insert if T has such a function,
+    // and do nothing if T has no such function.
+    if constexpr (not std::derived_from<T, DoNotSortUponInsertion> and requires(T& p) { p.post_insert(); }) {
+        iProduct.post_insert();
+    }
+     */
+
+    assert(index < putProducts().size());
+    ObjectWithDict wrapper = product->wrappedType_.construct();
+
+    // memcpy the object representation of product->object_ into the wrapper
+    std::memcpy(wrapper.get("obj").address(), product->object_.address(), product->object_.typeOf().size());
+    ObjectWithDict prod(product->object_.typeOf(), wrapper.get("obj").address());
+
+    // mark the object as present
+    *reinterpret_cast<bool*>(wrapper.get("present").address()) = true;
+
+    std::unique_ptr<WrapperBase> wp(reinterpret_cast<WrapperBase*>(wrapper.address()));
+    putProducts()[index] = std::move(wp);
+    auto const& prodID = provRecorder_.getProductID(index);
+    return OrphanHandle<GenericProduct>(prod, prodID);
+  }
+
+  // specialise Event::put for GenericProduct
+  template <>
+  OrphanHandle<GenericProduct> Event::put(EDPutToken token, std::unique_ptr<GenericProduct> product) {
+    if (UNLIKELY(product.get() == nullptr)) {  // null pointer is illegal
+      TypeID typeID(typeid(GenericProduct));
+      principal_get_adapter_detail::throwOnPutOfNullProduct("Event", typeID, provRecorder_.productInstanceLabel(token));
+    }
+    std::type_info const& type = product->object_.typeOf().typeInfo();
+    if (UNLIKELY(token.isUninitialized())) {
+      principal_get_adapter_detail::throwOnPutOfUninitializedToken("Event", type);
+    }
+    TypeID const& expected = provRecorder_.getTypeIDForPutTokenIndex(token.index());
+    if (UNLIKELY(expected != TypeID{type})) {
+      principal_get_adapter_detail::throwOnPutOfWrongType(type, expected);
+    }
+
+    return putImpl(token.index(), std::move(product));
+  }
+
+}  // namespace edm

FWCore/Modules/src/GenericConsumer.cc

@@ -34,87 +34,14 @@
 #include <boost/algorithm/string/replace.hpp>
 
 #include "DataFormats/Provenance/interface/BranchDescription.h"
+#include "DataFormats/Provenance/interface/BranchPattern.h"
 #include "FWCore/Framework/interface/global/EDAnalyzer.h"
 #include "FWCore/MessageLogger/interface/MessageLogger.h"
 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
 #include "FWCore/ParameterSet/interface/ParameterDescriptionNode.h"
 #include "FWCore/ParameterSet/interface/ParameterSet.h"
 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
 
-namespace {
-  struct ProductBranch {
-  public:
-    ProductBranch(std::string const& label) {
-      static const char kSeparator = '_';
-      static const char kWildcard = '*';
-      static const std::regex kAny{".*"};
-
-      // wildcard
-      if (label == kWildcard) {
-        type_ = kAny;
-        moduleLabel_ = kAny;
-        productInstanceName_ = kAny;
-        processName_ = kAny;
-        return;
-      }
-
-      int fields = std::count(label.begin(), label.end(), kSeparator) + 1;
-      if (fields == 1) {
-        // convert the module label into a regular expression
-        type_ = kAny;
-        moduleLabel_ = glob_to_regex(label);
-        productInstanceName_ = kAny;
-        processName_ = kAny;
-      } else if (fields == 4) {
-        // split the branch name into <product type>_<module label>_<instance name>_<process name>
-        // and convert the glob expressions into regular expressions
-        size_t first = 0, last = 0;
-        last = label.find(kSeparator, first);
-        type_ = glob_to_regex(label.substr(first, last - first));
-        first = last + 1;
-        last = label.find(kSeparator, first);
-        moduleLabel_ = glob_to_regex(label.substr(first, last - first));
-        first = last + 1;
-        last = label.find(kSeparator, first);
-        productInstanceName_ = glob_to_regex(label.substr(first, last - first));
-        first = last + 1;
-        last = label.find(kSeparator, first);
-        processName_ = glob_to_regex(label.substr(first, last - first));
-      } else {
-        // invalid input
-        throw edm::Exception(edm::errors::Configuration) << "Invalid module label or branch name: \"" << label << "\"";
-      }
-    }
-
-    bool match(edm::BranchDescription const& branch) const {
-      return (std::regex_match(branch.friendlyClassName(), type_) and
-              std::regex_match(branch.moduleLabel(), moduleLabel_) and
-              std::regex_match(branch.productInstanceName(), productInstanceName_) and
-              std::regex_match(branch.processName(), processName_));
-    }
-
-  private:
-    static std::regex glob_to_regex(std::string pattern) {
-      boost::replace_all(pattern, "*", ".*");
-      boost::replace_all(pattern, "?", ".");
-      return std::regex(pattern);
-    }
-
-    std::regex type_;
-    std::regex moduleLabel_;
-    std::regex productInstanceName_;
-    std::regex processName_;
-  };
-
-  std::vector<ProductBranch> make_patterns(std::vector<std::string> const& labels) {
-    std::vector<ProductBranch> patterns;
-    patterns.reserve(labels.size());
-    for (auto const& label : labels)
-      patterns.emplace_back(label);
-    return patterns;
-  }
-}  // namespace
-
 namespace edm {
   class GenericConsumer : public edm::global::EDAnalyzer<> {
   public:
@@ -126,19 +53,20 @@ namespace edm {
     static void fillDescriptions(ConfigurationDescriptions& descriptions);
 
   private:
-    std::vector<ProductBranch> eventProducts_;
-    std::vector<ProductBranch> lumiProducts_;
-    std::vector<ProductBranch> runProducts_;
-    std::vector<ProductBranch> processProducts_;
+    std::vector<edm::BranchPattern> eventProducts_;
+    std::vector<edm::BranchPattern> lumiProducts_;
+    std::vector<edm::BranchPattern> runProducts_;
+    std::vector<edm::BranchPattern> processProducts_;
     std::string label_;
     bool verbose_;
   };
 
   GenericConsumer::GenericConsumer(ParameterSet const& config)
-      : eventProducts_(make_patterns(config.getUntrackedParameter<std::vector<std::string>>("eventProducts"))),
-        lumiProducts_(make_patterns(config.getUntrackedParameter<std::vector<std::string>>("lumiProducts"))),
-        runProducts_(make_patterns(config.getUntrackedParameter<std::vector<std::string>>("runProducts"))),
-        processProducts_(make_patterns(config.getUntrackedParameter<std::vector<std::string>>("processProducts"))),
+      : eventProducts_(edm::branchPatterns(config.getUntrackedParameter<std::vector<std::string>>("eventProducts"))),
+        lumiProducts_(edm::branchPatterns(config.getUntrackedParameter<std::vector<std::string>>("lumiProducts"))),
+        runProducts_(edm::branchPatterns(config.getUntrackedParameter<std::vector<std::string>>("runProducts"))),
+        processProducts_(
+            edm::branchPatterns(config.getUntrackedParameter<std::vector<std::string>>("processProducts"))),
         label_(config.getParameter<std::string>("@module_label")),
         verbose_(config.getUntrackedParameter<bool>("verbose")) {
     callWhenNewProductsRegistered([this](edm::BranchDescription const& branch) {

FWCore/TestModules/README.md

@@ -24,3 +24,14 @@ product read from the `Event`.
 Together `edmtest::EventIDProducer` and `edmtest::EventIDValidator` can be used
 to validate that an object produced in a given event is being read back in the
 same event.
+
+
+## `edmtest::GenericCloner`
+
+This module will clone all the event products declared by its configuration,
+using their ROOT dictionaries.
+The products can be specified either as module labels (_e.g._ `<module label>`)
+or as branch names (_e.g._ `<product type>_<module label>_<instance name>_<process name>`).
+Glob expressions (`?` and `*`) are supported in module labels and within the
+individual fields of branch names, similar to an `OutputModule`'s `keep`
+statements.