From 58094c1052a288d88c079fee6fa8c8c361785f1a Mon Sep 17 00:00:00 2001 From: Andrea Bocci Date: Sat, 30 Nov 2024 10:43:27 +0100 Subject: [PATCH] Fix PFClusterSoAProducer to read a device collection Make the PFRecHitSoAProducer produce an additional host-only collection with the number of PF rechits. Make the the PFClusterSoAProducer consume the device collection of PF rechits, and the host collection with the number of PF rechits. --- .../plugins/alpaka/PFClusterECLCC.h | 18 +++++----- .../plugins/alpaka/PFClusterSoAProducer.cc | 28 +++++++-------- .../alpaka/PFClusterSoAProducerKernel.dev.cc | 30 ++++++++-------- .../alpaka/PFClusterSoAProducerKernel.h | 15 ++++---- .../plugins/alpaka/PFRecHitSoAProducer.cc | 34 ++++++++++++++----- 5 files changed, 69 insertions(+), 56 deletions(-) diff --git a/RecoParticleFlow/PFClusterProducer/plugins/alpaka/PFClusterECLCC.h b/RecoParticleFlow/PFClusterProducer/plugins/alpaka/PFClusterECLCC.h index abf63c01e9531..2b6fba2c69d5c 100644 --- a/RecoParticleFlow/PFClusterProducer/plugins/alpaka/PFClusterECLCC.h +++ b/RecoParticleFlow/PFClusterProducer/plugins/alpaka/PFClusterECLCC.h @@ -1,10 +1,11 @@ #ifndef RecoParticleFlow_PFClusterProducer_plugins_alpaka_PFClusterECLCC_h #define RecoParticleFlow_PFClusterProducer_plugins_alpaka_PFClusterECLCC_h +#include "DataFormats/ParticleFlowReco/interface/alpaka/PFRecHitDeviceCollection.h" #include "HeterogeneousCore/AlpakaInterface/interface/config.h" #include "HeterogeneousCore/AlpakaInterface/interface/workdivision.h" -#include "RecoParticleFlow/PFClusterProducer/interface/alpaka/PFClusteringVarsDeviceCollection.h" #include "RecoParticleFlow/PFClusterProducer/interface/alpaka/PFClusteringEdgeVarsDeviceCollection.h" +#include "RecoParticleFlow/PFClusterProducer/interface/alpaka/PFClusteringVarsDeviceCollection.h" // The following comment block is required in using the ECL-CC algorithm for topological clustering @@ -79,9 +80,8 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { // Initial step of ECL-CC. Uses ID of first neighbour in edgeList with a smaller ID class ECLCCInit { public: - template >> - ALPAKA_FN_ACC void operator()(const TAcc& acc, - reco::PFRecHitHostCollection::ConstView pfRecHits, + ALPAKA_FN_ACC void operator()(Acc1D const& acc, + reco::PFRecHitDeviceCollection::ConstView pfRecHits, reco::PFClusteringVarsDeviceCollection::View pfClusteringVars, reco::PFClusteringEdgeVarsDeviceCollection::View pfClusteringEdgeVars) const { const int nRH = pfRecHits.size(); @@ -103,9 +103,8 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { // Processes vertices class ECLCCCompute1 { public: - template >> - ALPAKA_FN_ACC void operator()(const TAcc& acc, - reco::PFRecHitHostCollection::ConstView pfRecHits, + ALPAKA_FN_ACC void operator()(Acc1D const& acc, + reco::PFRecHitDeviceCollection::ConstView pfRecHits, reco::PFClusteringVarsDeviceCollection::View pfClusteringVars, reco::PFClusteringEdgeVarsDeviceCollection::View pfClusteringEdgeVars) const { const int nRH = pfRecHits.size(); @@ -148,9 +147,8 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { /* link all vertices to sink */ class ECLCCFlatten { public: - template >> - ALPAKA_FN_ACC void operator()(const TAcc& acc, - reco::PFRecHitHostCollection::ConstView pfRecHits, + ALPAKA_FN_ACC void operator()(Acc1D const& acc, + reco::PFRecHitDeviceCollection::ConstView pfRecHits, reco::PFClusteringVarsDeviceCollection::View pfClusteringVars, reco::PFClusteringEdgeVarsDeviceCollection::View pfClusteringEdgeVars) const { const int nRH = pfRecHits.size(); diff --git a/RecoParticleFlow/PFClusterProducer/plugins/alpaka/PFClusterSoAProducer.cc b/RecoParticleFlow/PFClusterProducer/plugins/alpaka/PFClusterSoAProducer.cc index 65c0e4f5c33f3..ac5cbe5ab6ba2 100644 --- a/RecoParticleFlow/PFClusterProducer/plugins/alpaka/PFClusterSoAProducer.cc +++ b/RecoParticleFlow/PFClusterProducer/plugins/alpaka/PFClusterSoAProducer.cc @@ -1,7 +1,7 @@ #include #include -#include "DataFormats/ParticleFlowReco/interface/PFRecHitHostCollection.h" +#include "DataFormats/ParticleFlowReco/interface/alpaka/PFRecHitDeviceCollection.h" #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" #include "FWCore/ParameterSet/interface/ParameterSetDescription.h" @@ -16,12 +16,14 @@ #include "RecoParticleFlow/PFRecHitProducer/interface/PFRecHitTopologyRecord.h" namespace ALPAKA_ACCELERATOR_NAMESPACE { + class PFClusterSoAProducer : public stream::SynchronizingEDProducer<> { public: PFClusterSoAProducer(edm::ParameterSet const& config) : pfClusParamsToken(esConsumes(config.getParameter("pfClusterParams"))), topologyToken_(esConsumes(config.getParameter("topology"))), inputPFRecHitSoA_Token_{consumes(config.getParameter("pfRecHits"))}, + inputPFRecHitNum_Token_{consumes(config.getParameter("pfRecHits"))}, outputPFClusterSoA_Token_{produces()}, outputPFRHFractionSoA_Token_{produces()}, numRHF_{cms::alpakatools::make_host_buffer()}, @@ -30,10 +32,8 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { void acquire(device::Event const& event, device::EventSetup const& setup) override { const reco::PFClusterParamsDeviceCollection& params = setup.getData(pfClusParamsToken); const reco::PFRecHitHCALTopologyDeviceCollection& topology = setup.getData(topologyToken_); - const reco::PFRecHitHostCollection& pfRecHits = event.get(inputPFRecHitSoA_Token_); - int nRH = 0; - if (pfRecHits->metadata().size() != 0) - nRH = pfRecHits->size(); + const reco::PFRecHitDeviceCollection& pfRecHits = event.get(inputPFRecHitSoA_Token_); + int nRH = event.get(inputPFRecHitNum_Token_); pfClusteringVars_.emplace(nRH, event.queue()); pfClusteringEdgeVars_.emplace(nRH * 8, event.queue()); @@ -42,13 +42,14 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { *numRHF_ = 0; if (nRH != 0) { - PFClusterProducerKernel kernel(event.queue(), pfRecHits); + PFClusterProducerKernel kernel(event.queue()); kernel.seedTopoAndContract(event.queue(), params, topology, *pfClusteringVars_, *pfClusteringEdgeVars_, pfRecHits, + nRH, *pfClusters_, numRHF_.data()); } @@ -57,23 +58,21 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { void produce(device::Event& event, device::EventSetup const& setup) override { const reco::PFClusterParamsDeviceCollection& params = setup.getData(pfClusParamsToken); const reco::PFRecHitHCALTopologyDeviceCollection& topology = setup.getData(topologyToken_); - const reco::PFRecHitHostCollection& pfRecHits = event.get(inputPFRecHitSoA_Token_); - int nRH = 0; + const reco::PFRecHitDeviceCollection& pfRecHits = event.get(inputPFRecHitSoA_Token_); std::optional pfrhFractions; - if (pfRecHits->metadata().size() != 0) - nRH = pfRecHits->size(); - + int nRH = event.get(inputPFRecHitNum_Token_); if (nRH != 0) { - pfrhFractions.emplace(*numRHF_.data(), event.queue()); - PFClusterProducerKernel kernel(event.queue(), pfRecHits); + pfrhFractions.emplace(*numRHF_, event.queue()); + PFClusterProducerKernel kernel(event.queue()); kernel.cluster(event.queue(), params, topology, *pfClusteringVars_, *pfClusteringEdgeVars_, pfRecHits, + nRH, *pfClusters_, *pfrhFractions); } else { @@ -99,7 +98,8 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { private: const device::ESGetToken pfClusParamsToken; const device::ESGetToken topologyToken_; - const edm::EDGetTokenT inputPFRecHitSoA_Token_; + const device::EDGetToken inputPFRecHitSoA_Token_; + const edm::EDGetTokenT inputPFRecHitNum_Token_; const device::EDPutToken outputPFClusterSoA_Token_; const device::EDPutToken outputPFRHFractionSoA_Token_; cms::alpakatools::host_buffer numRHF_; diff --git a/RecoParticleFlow/PFClusterProducer/plugins/alpaka/PFClusterSoAProducerKernel.dev.cc b/RecoParticleFlow/PFClusterProducer/plugins/alpaka/PFClusterSoAProducerKernel.dev.cc index e0c5e8d5a24fa..b9ec4e557009c 100644 --- a/RecoParticleFlow/PFClusterProducer/plugins/alpaka/PFClusterSoAProducerKernel.dev.cc +++ b/RecoParticleFlow/PFClusterProducer/plugins/alpaka/PFClusterSoAProducerKernel.dev.cc @@ -1,13 +1,12 @@ #include +#include "DataFormats/ParticleFlowReco/interface/PFLayer.h" #include "FWCore/Utilities/interface/bit_cast.h" +#include "HeterogeneousCore/AlpakaInterface/interface/atomicMaxF.h" #include "HeterogeneousCore/AlpakaInterface/interface/config.h" #include "HeterogeneousCore/AlpakaInterface/interface/workdivision.h" -#include "HeterogeneousCore/AlpakaInterface/interface/atomicMaxF.h" - -#include "DataFormats/ParticleFlowReco/interface/PFLayer.h" -#include "RecoParticleFlow/PFClusterProducer/plugins/alpaka/PFClusterSoAProducerKernel.h" #include "RecoParticleFlow/PFClusterProducer/plugins/alpaka/PFClusterECLCC.h" +#include "RecoParticleFlow/PFClusterProducer/plugins/alpaka/PFClusterSoAProducerKernel.h" namespace ALPAKA_ACCELERATOR_NAMESPACE { @@ -1088,7 +1087,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { reco::PFClusteringVarsDeviceCollection::View pfClusteringVars, const reco::PFClusterParamsDeviceCollection::ConstView pfClusParams, const reco::PFRecHitHCALTopologyDeviceCollection::ConstView topology, - const reco::PFRecHitHostCollection::ConstView pfRecHits, + const reco::PFRecHitDeviceCollection::ConstView pfRecHits, reco::PFClusterDeviceCollection::View clusterView, uint32_t* __restrict__ nSeeds) const { const int nRH = pfRecHits.size(); @@ -1165,7 +1164,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { public: template >> ALPAKA_FN_ACC void operator()(const TAcc& acc, - const reco::PFRecHitHostCollection::ConstView pfRecHits, + const reco::PFRecHitDeviceCollection::ConstView pfRecHits, reco::PFClusteringVarsDeviceCollection::View pfClusteringVars, reco::PFClusteringEdgeVarsDeviceCollection::View pfClusteringEdgeVars, uint32_t* __restrict__ nSeeds) const { @@ -1195,7 +1194,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { public: template >> ALPAKA_FN_ACC void operator()(const TAcc& acc, - const reco::PFRecHitHostCollection::ConstView pfRecHits, + const reco::PFRecHitDeviceCollection::ConstView pfRecHits, reco::PFClusteringVarsDeviceCollection::View pfClusteringVars, reco::PFClusterDeviceCollection::View clusterView, uint32_t* __restrict__ nSeeds, @@ -1319,7 +1318,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { public: template >> ALPAKA_FN_ACC void operator()(const TAcc& acc, - const reco::PFRecHitHostCollection::ConstView pfRecHits, + const reco::PFRecHitDeviceCollection::ConstView pfRecHits, reco::PFClusteringVarsDeviceCollection::View pfClusteringVars, reco::PFRecHitFractionDeviceCollection::View fracView) const { const int nRH = pfRecHits.size(); @@ -1350,7 +1349,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { public: template >> ALPAKA_FN_ACC void operator()(const TAcc& acc, - const reco::PFRecHitHostCollection::ConstView pfRecHits, + const reco::PFRecHitDeviceCollection::ConstView pfRecHits, const reco::PFClusterParamsDeviceCollection::ConstView pfClusParams, const reco::PFRecHitHCALTopologyDeviceCollection::ConstView topology, reco::PFClusteringVarsDeviceCollection::View pfClusteringVars, @@ -1412,7 +1411,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { public: template >> ALPAKA_FN_ACC void operator()(const TAcc& acc, - const reco::PFRecHitHostCollection::ConstView pfRecHits, + const reco::PFRecHitDeviceCollection::ConstView pfRecHits, const reco::PFClusterParamsDeviceCollection::ConstView pfClusParams, const reco::PFRecHitHCALTopologyDeviceCollection::ConstView topology, reco::PFClusteringVarsDeviceCollection::View pfClusteringVars, @@ -1454,7 +1453,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { } }; - PFClusterProducerKernel::PFClusterProducerKernel(Queue& queue, const reco::PFRecHitHostCollection& pfRecHits) + PFClusterProducerKernel::PFClusterProducerKernel(Queue& queue) : nSeeds(cms::alpakatools::make_device_buffer(queue)), globalClusterPos( cms::alpakatools::make_device_buffer(queue, blocksForExoticClusters * maxTopoInput)), @@ -1473,10 +1472,10 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { const reco::PFRecHitHCALTopologyDeviceCollection& topology, reco::PFClusteringVarsDeviceCollection& pfClusteringVars, reco::PFClusteringEdgeVarsDeviceCollection& pfClusteringEdgeVars, - const reco::PFRecHitHostCollection& pfRecHits, + const reco::PFRecHitDeviceCollection& pfRecHits, + int nRH, reco::PFClusterDeviceCollection& pfClusters, uint32_t* __restrict__ nRHF) { - const int nRH = pfRecHits->size(); const int threadsPerBlock = 256; const int blocks = divide_up_by(nRH, threadsPerBlock); @@ -1533,11 +1532,10 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { const reco::PFRecHitHCALTopologyDeviceCollection& topology, reco::PFClusteringVarsDeviceCollection& pfClusteringVars, reco::PFClusteringEdgeVarsDeviceCollection& pfClusteringEdgeVars, - const reco::PFRecHitHostCollection& pfRecHits, + const reco::PFRecHitDeviceCollection& pfRecHits, + int nRH, reco::PFClusterDeviceCollection& pfClusters, reco::PFRecHitFractionDeviceCollection& pfrhFractions) { - const int nRH = pfRecHits->size(); - // fillRhfIndex alpaka::exec(queue, make_workdiv({divide_up_by(nRH, 32), divide_up_by(nRH, 32)}, {32, 32}), diff --git a/RecoParticleFlow/PFClusterProducer/plugins/alpaka/PFClusterSoAProducerKernel.h b/RecoParticleFlow/PFClusterProducer/plugins/alpaka/PFClusterSoAProducerKernel.h index 32c68a64f4b24..cfc79b8d7ad69 100644 --- a/RecoParticleFlow/PFClusterProducer/plugins/alpaka/PFClusterSoAProducerKernel.h +++ b/RecoParticleFlow/PFClusterProducer/plugins/alpaka/PFClusterSoAProducerKernel.h @@ -1,15 +1,14 @@ #ifndef RecoParticleFlow_PFClusterProducer_PFClusterProducerAlpakaKernel_h #define RecoParticleFlow_PFClusterProducer_PFClusterProducerAlpakaKernel_h -#include "DataFormats/ParticleFlowReco/interface/alpaka/PFRecHitDeviceCollection.h" -#include "DataFormats/ParticleFlowReco/interface/PFRecHitHostCollection.h" #include "DataFormats/ParticleFlowReco/interface/alpaka/PFClusterDeviceCollection.h" +#include "DataFormats/ParticleFlowReco/interface/alpaka/PFRecHitDeviceCollection.h" #include "DataFormats/ParticleFlowReco/interface/alpaka/PFRecHitFractionDeviceCollection.h" +#include "HeterogeneousCore/AlpakaInterface/interface/config.h" #include "RecoParticleFlow/PFClusterProducer/interface/alpaka/PFClusterParamsDeviceCollection.h" -#include "RecoParticleFlow/PFClusterProducer/interface/alpaka/PFClusteringVarsDeviceCollection.h" #include "RecoParticleFlow/PFClusterProducer/interface/alpaka/PFClusteringEdgeVarsDeviceCollection.h" +#include "RecoParticleFlow/PFClusterProducer/interface/alpaka/PFClusteringVarsDeviceCollection.h" #include "RecoParticleFlow/PFRecHitProducer/interface/alpaka/PFRecHitTopologyDeviceCollection.h" -#include "HeterogeneousCore/AlpakaInterface/interface/config.h" namespace ALPAKA_ACCELERATOR_NAMESPACE { @@ -37,14 +36,15 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { class PFClusterProducerKernel { public: - PFClusterProducerKernel(Queue& queue, const reco::PFRecHitHostCollection& pfRecHits); + explicit PFClusterProducerKernel(Queue& queue); void seedTopoAndContract(Queue& queue, const reco::PFClusterParamsDeviceCollection& params, const reco::PFRecHitHCALTopologyDeviceCollection& topology, reco::PFClusteringVarsDeviceCollection& pfClusteringVars, reco::PFClusteringEdgeVarsDeviceCollection& pfClusteringEdgeVars, - const reco::PFRecHitHostCollection& pfRecHits, + const reco::PFRecHitDeviceCollection& pfRecHits, + int nRH, reco::PFClusterDeviceCollection& pfClusters, uint32_t* __restrict__ nRHF); @@ -53,7 +53,8 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { const reco::PFRecHitHCALTopologyDeviceCollection& topology, reco::PFClusteringVarsDeviceCollection& pfClusteringVars, reco::PFClusteringEdgeVarsDeviceCollection& pfClusteringEdgeVars, - const reco::PFRecHitHostCollection& pfRecHits, + const reco::PFRecHitDeviceCollection& pfRecHits, + int nRH, reco::PFClusterDeviceCollection& pfClusters, reco::PFRecHitFractionDeviceCollection& pfrhFractions); diff --git a/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitSoAProducer.cc b/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitSoAProducer.cc index 18c160f3cc0c1..ebaf910afd299 100644 --- a/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitSoAProducer.cc +++ b/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitSoAProducer.cc @@ -3,11 +3,12 @@ #include +#include "DataFormats/ParticleFlowReco/interface/alpaka/PFRecHitDeviceCollection.h" #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" #include "FWCore/ParameterSet/interface/ParameterSetDescription.h" #include "FWCore/Utilities/interface/InputTag.h" -#include "HeterogeneousCore/AlpakaCore/interface/alpaka/global/EDProducer.h" +#include "HeterogeneousCore/AlpakaCore/interface/alpaka/stream/SynchronizingEDProducer.h" #include "RecoParticleFlow/PFRecHitProducer/interface/PFRecHitParamsRecord.h" #include "RecoParticleFlow/PFRecHitProducer/interface/PFRecHitTopologyRecord.h" @@ -18,12 +19,14 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { using namespace particleFlowRecHitProducer; template - class PFRecHitSoAProducer : public global::EDProducer<> { + class PFRecHitSoAProducer : public stream::SynchronizingEDProducer<> { public: PFRecHitSoAProducer(edm::ParameterSet const& config) : topologyToken_(esConsumes(config.getParameter("topology"))), pfRecHitsToken_(produces()), - synchronise_(config.getUntrackedParameter("synchronise")) { + sizeToken_(produces()), + synchronise_(config.getUntrackedParameter("synchronise")), + size_{cms::alpakatools::make_host_buffer()} { const std::vector producers = config.getParameter>("producers"); recHitsToken_.reserve(producers.size()); for (const edm::ParameterSet& producer : producers) { @@ -32,27 +35,34 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { } } - void produce(edm::StreamID, device::Event& event, const device::EventSetup& setup) const override { + void acquire(device::Event const& event, const device::EventSetup& setup) override { const typename CAL::TopologyTypeDevice& topology = setup.getData(topologyToken_); uint32_t num_recHits = 0; for (const auto& token : recHitsToken_) num_recHits += event.get(token.first)->metadata().size(); - reco::PFRecHitDeviceCollection pfRecHits{(int)num_recHits, event.queue()}; + pfRecHits_.emplace((int)num_recHits, event.queue()); + *size_ = 0; if (num_recHits != 0) { PFRecHitProducerKernel kernel{event.queue(), num_recHits}; for (const auto& token : recHitsToken_) kernel.processRecHits( - event.queue(), event.get(token.first), setup.getData(token.second), topology, pfRecHits); - kernel.associateTopologyInfo(event.queue(), topology, pfRecHits); + event.queue(), event.get(token.first), setup.getData(token.second), topology, *pfRecHits_); + kernel.associateTopologyInfo(event.queue(), topology, *pfRecHits_); + auto size_d = cms::alpakatools::make_device_view(event.queue(), pfRecHits_->view().size()); + alpaka::memcpy(event.queue(), size_, size_d); } + } + + void produce(device::Event& event, const device::EventSetup& setup) override { + event.emplace(pfRecHitsToken_, std::move(*pfRecHits_)); + event.emplace(sizeToken_, *size_); + pfRecHits_.reset(); if (synchronise_) alpaka::wait(event.queue()); - - event.emplace(pfRecHitsToken_, std::move(pfRecHits)); } static void fillDescriptions(edm::ConfigurationDescriptions& descriptions) { @@ -71,12 +81,18 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { } private: + // module configuration const device::ESGetToken topologyToken_; std::vector, device::ESGetToken>> recHitsToken_; const device::EDPutToken pfRecHitsToken_; + const edm::EDPutTokenT sizeToken_; const bool synchronise_; + + // data members used to communicate between acquire() and produce() + cms::alpakatools::host_buffer size_; + std::optional pfRecHits_; }; using PFRecHitSoAProducerHCAL = PFRecHitSoAProducer;