Fixes for ECAL rechits on GPU

EventFilter/EcalRawToDigi/python/ecalDigis_cff.py

@@ -18,11 +18,11 @@
 
 # copy the digi from the GPU to the CPU and convert to legacy format
 from EventFilter.EcalRawToDigi.ecalCPUDigisProducer_cfi import ecalCPUDigisProducer as _ecalCPUDigisProducer
-_gpu_ecalDigis = _ecalCPUDigisProducer.clone(
-  digisInLabelEB = 'ecalDigisGPU:ebDigisGPU',
-  digisInLabelEE = 'ecalDigisGPU:eeDigisGPU',
-  produceDummyIntegrityCollections = True,
+_ecalDigis_gpu = _ecalCPUDigisProducer.clone(
+  digisInLabelEB = cms.InputTag('ecalDigisGPU', 'ebDigisGPU'),
+  digisInLabelEE = cms.InputTag('ecalDigisGPU', 'eeDigisGPU'),
+  produceDummyIntegrityCollections = True
 )
-gpu.toReplaceWith(ecalDigis, _gpu_ecalDigis)
+gpu.toReplaceWith(ecalDigis, _ecalDigis_gpu)
 
 gpu.toReplaceWith(ecalDigisTask, cms.Task(ecalElectronicsMappingGPUESProducer, ecalDigisGPU, ecalDigis))

RecoLocalCalo/Configuration/python/ecalLocalRecoSequence_cff.py

@@ -1,4 +1,5 @@
 import FWCore.ParameterSet.Config as cms
+from Configuration.ProcessModifiers.gpu_cff import gpu
 
 # TPG condition needed by ecalRecHit producer if TT recovery is ON
 from RecoLocalCalo.EcalRecProducers.ecalRecHitTPGConditions_cff import *
@@ -43,6 +44,57 @@
 
 ecalOnlyLocalRecoSequence = cms.Sequence(ecalOnlyLocalRecoTask)
 
+# ECAL rechit calibrations on GPU
+from RecoLocalCalo.EcalRecProducers.ecalRechitADCToGeVConstantGPUESProducer_cfi import ecalRechitADCToGeVConstantGPUESProducer
+from RecoLocalCalo.EcalRecProducers.ecalRechitChannelStatusGPUESProducer_cfi import ecalRechitChannelStatusGPUESProducer
+from RecoLocalCalo.EcalRecProducers.ecalIntercalibConstantsGPUESProducer_cfi import ecalIntercalibConstantsGPUESProducer
+from RecoLocalCalo.EcalRecProducers.ecalLaserAPDPNRatiosGPUESProducer_cfi import ecalLaserAPDPNRatiosGPUESProducer
+from RecoLocalCalo.EcalRecProducers.ecalLaserAPDPNRatiosRefGPUESProducer_cfi import ecalLaserAPDPNRatiosRefGPUESProducer
+from RecoLocalCalo.EcalRecProducers.ecalLaserAlphasGPUESProducer_cfi import ecalLaserAlphasGPUESProducer
+from RecoLocalCalo.EcalRecProducers.ecalLinearCorrectionsGPUESProducer_cfi import ecalLinearCorrectionsGPUESProducer
+
+# ECAL rechits running on GPU
+from RecoLocalCalo.EcalRecProducers.ecalRecHitGPU_cfi import ecalRecHitGPU as _ecalRecHitGPU
+ecalRecHitGPU = _ecalRecHitGPU.clone(
+    uncalibrecHitsInLabelEB = cms.InputTag('ecalMultiFitUncalibRecHitGPU', 'EcalUncalibRecHitsEB'),
+    uncalibrecHitsInLabelEE = cms.InputTag('ecalMultiFitUncalibRecHitGPU', 'EcalUncalibRecHitsEE')
+)
+
+# copy the rechits from GPU to CPU
+from RecoLocalCalo.EcalRecProducers.ecalCPURecHitProducer_cfi import ecalCPURecHitProducer as _ecalCPURecHitProducer
+ecalRecHitSoA = _ecalCPURecHitProducer.clone(
+    recHitsInLabelEB = cms.InputTag('ecalRecHitGPU', 'EcalRecHitsEB'),
+    recHitsInLabelEE = cms.InputTag('ecalRecHitGPU', 'EcalRecHitsEE')
+)
+
+# convert the rechits from SoA to legacy format
+from RecoLocalCalo.EcalRecProducers.ecalRecHitConvertGPU2CPUFormat_cfi import ecalRecHitConvertGPU2CPUFormat as _ecalRecHitConvertGPU2CPUFormat
+_ecalRecHit_gpu = _ecalRecHitConvertGPU2CPUFormat.clone(
+    recHitsLabelGPUEB = cms.InputTag('ecalRecHitSoA', 'EcalRecHitsEB'),
+    recHitsLabelGPUEE = cms.InputTag('ecalRecHitSoA', 'EcalRecHitsEE')
+)
+gpu.toReplaceWith(ecalRecHit, _ecalRecHit_gpu)
+
+# ECAL reconstruction on GPU
+gpu.toReplaceWith(ecalRecHitNoTPTask, cms.Task(
+  # ECAL rechit calibrations on GPU
+  ecalRechitADCToGeVConstantGPUESProducer,
+  ecalRechitChannelStatusGPUESProducer,
+  ecalIntercalibConstantsGPUESProducer,
+  ecalLaserAPDPNRatiosGPUESProducer,
+  ecalLaserAPDPNRatiosRefGPUESProducer,
+  ecalLaserAlphasGPUESProducer,
+  ecalLinearCorrectionsGPUESProducer,
+  # ECAL rechits running on GPU
+  ecalRecHitGPU,
+  # copy the rechits from GPU to CPU
+  ecalRecHitSoA,
+  # convert the rechits from SoA to legacy format
+  ecalRecHit,
+  # ECAL preshower rechit legacy module
+  ecalPreshowerRecHit
+))
+
 # Phase 2 modifications
 from RecoLocalCalo.EcalRecProducers.ecalDetailedTimeRecHit_cfi import *
 _phase2_timing_ecalRecHitTask = cms.Task( ecalRecHitTask.copy() , ecalDetailedTimeRecHit )

RecoLocalCalo/EcalRecAlgos/src/EcalLinearCorrectionsGPU.cc

@@ -38,6 +38,10 @@ EcalLinearCorrectionsGPU::Product::~Product() {
   // deallocation
   cudaCheck(cudaFree(p1));
   cudaCheck(cudaFree(p2));
+  cudaCheck(cudaFree(p3));
+  cudaCheck(cudaFree(t1));
+  cudaCheck(cudaFree(t2));
+  cudaCheck(cudaFree(t3));
 }
 
 EcalLinearCorrectionsGPU::Product const& EcalLinearCorrectionsGPU::getProduct(cudaStream_t cudaStream) const {

RecoLocalCalo/EcalRecAlgos/src/EcalRecHitBuilderKernels.cu

@@ -157,7 +157,7 @@ namespace ecal {
 
         ::ecal::reco::StorageScalarType const* amplitude = isEndcap ? amplitude_ee : amplitude_eb;
 
-        ::ecal::reco::StorageScalarType const* time_in = isEndcap ? time_ee : time_eb;
+        //::ecal::reco::StorageScalarType const* time_in = isEndcap ? time_ee : time_eb;
 
         ::ecal::reco::StorageScalarType const* chi2_in = isEndcap ? chi2_ee : chi2_eb;
 
@@ -287,16 +287,31 @@ namespace ecal {
         //
         energy[ch] = -1;  //---- AM: default, un-physical, ok
 
-        //
-        static const int chStatusMask = 0x1F;
+        // truncate the chi2
+        if (chi2_in[inputCh] > 64)
+          chi2[ch] = 64;
+        else
+          chi2[ch] = chi2_in[inputCh];
+
+        // default values for the flags
+        flagBits[ch] = 0;
+        extra[ch] = 0;
+
+        static const int chStatusMask = 0x1f;
         // ChannelStatusToBeExcluded is a "int" then I put "dbstatus" to be the same
         int dbstatus = EcalChannelStatusCode_Code((status[hashedId]) & chStatusMask);
         if (ChannelStatusToBeExcludedSize != 0) {
+          bool skip_this_channel = false;
           for (int ich_to_check = 0; ich_to_check < ChannelStatusToBeExcludedSize; ich_to_check++) {
             if (ChannelStatusToBeExcluded[ich_to_check] == dbstatus) {
-              return;
+              skip_this_channel = true;
+              break;
             }
           }
+          if (skip_this_channel) {
+            // skip this channel
+            continue;
+          }
         }
 
         // Take our association map of dbstatuses-> recHit flagbits and return the apporpriate flagbit word
@@ -335,13 +350,13 @@ namespace ecal {
           flagbit_counter += 1;
         }
 
+        flagBits[ch] = temporary_flagBits;
+
         if ((flagmask & temporary_flagBits) && killDeadChannels) {
-          return;
+          // skip this channel
+          continue;
         }
 
-        //
-        flagBits[ch] = temporary_flagBits;
-
         //
         // multiply the adc counts with factors to get GeV
         //
@@ -352,13 +367,7 @@ namespace ecal {
         // Time is not saved so far, FIXME
         //         time[ch] = time_in[inputCh];
 
-        if (chi2_in[inputCh] > 64)
-          chi2[ch] = 64;
-        else
-          chi2[ch] = chi2_in[inputCh];
-
         // NB: calculate the "flagBits extra"  --> not really "flags", but actually an encoded version of energy uncertainty, time unc., ...
-        extra[ch] = 0;
 
         //
         // extra packing ...

RecoLocalCalo/EcalRecProducers/plugins/EcalRecHitProducerGPU.cc

@@ -7,10 +7,6 @@
 #include "FWCore/Framework/interface/EventSetup.h"
 #include "FWCore/Framework/interface/MakerMacros.h"
 
-//
-//
-//
-
 // format
 #include "CUDADataFormats/EcalRecHitSoA/interface/EcalUncalibratedRecHit_soa.h"
 #include "CUDADataFormats/EcalRecHitSoA/interface/EcalRecHit_soa.h"
@@ -302,7 +298,7 @@ void EcalRecHitProducerGPU::acquire(edm::Event const& event,
 
   neb_ = ebUncalibRecHits.size;
   nee_ = eeUncalibRecHits.size;
-  //   std::cout << " [EcalRecHitProducerGPU::acquire]  neb_:nee_ = " << neb_ << " : " << nee_ << std::endl;
+  // std::cout << " [EcalRecHitProducerGPU::acquire]  neb_:nee_ = " << neb_ << " : " << nee_ << std::endl;
 
   int nchannelsEB = ebUncalibRecHits.size;  // --> offsetForInput, first EB and then EE
 
@@ -321,8 +317,6 @@ void EcalRecHitProducerGPU::acquire(edm::Event const& event,
   setup.get<EcalLaserAlphasRcd>().get(LaserAlphasHandle_);
   setup.get<EcalLinearCorrectionsRcd>().get(LinearCorrectionsHandle_);
 
-  //
-
   auto const& ADCToGeVConstantProduct = ADCToGeVConstantHandle_->getProduct(ctx.stream());
   auto const& IntercalibConstantsProduct = IntercalibConstantsHandle_->getProduct(ctx.stream());
   auto const& ChannelStatusProduct = ChannelStatusHandle_->getProduct(ctx.stream());

RecoLocalCalo/EcalRecProducers/python/ecalMultiFitUncalibRecHit_cff.py

@@ -29,12 +29,27 @@
   recHitsInLabelEE = cms.InputTag('ecalMultiFitUncalibRecHitGPU', 'EcalUncalibRecHitsEE'),
 )
 
-# convert the uncalibrated rechits legacy format
+# convert the uncalibrated rechits from SoA to legacy format
 from RecoLocalCalo.EcalRecProducers.ecalUncalibRecHitConvertGPU2CPUFormat_cfi import ecalUncalibRecHitConvertGPU2CPUFormat as _ecalUncalibRecHitConvertGPU2CPUFormat
-_gpu_ecalMultiFitUncalibRecHit = _ecalUncalibRecHitConvertGPU2CPUFormat.clone(
+_ecalMultiFitUncalibRecHit_gpu = _ecalUncalibRecHitConvertGPU2CPUFormat.clone(
   recHitsLabelGPUEB = cms.InputTag('ecalMultiFitUncalibRecHitSoA', 'EcalUncalibRecHitsEB'),
   recHitsLabelGPUEE = cms.InputTag('ecalMultiFitUncalibRecHitSoA', 'EcalUncalibRecHitsEE'),
 )
-gpu.toReplaceWith(ecalMultiFitUncalibRecHit, _gpu_ecalMultiFitUncalibRecHit)
+gpu.toReplaceWith(ecalMultiFitUncalibRecHit, _ecalMultiFitUncalibRecHit_gpu)
 
-gpu.toReplaceWith(ecalMultiFitUncalibRecHitTask, cms.Task(ecalMultiFitUncalibRecHitGPU, ecalMultiFitUncalibRecHitSoA, ecalMultiFitUncalibRecHit))
+gpu.toReplaceWith(ecalMultiFitUncalibRecHitTask, cms.Task(
+  # ECAL conditions used by the multifit running on GPU
+  ecalPedestalsGPUESProducer,
+  ecalGainRatiosGPUESProducer,
+  ecalPulseShapesGPUESProducer,
+  ecalPulseCovariancesGPUESProducer,
+  ecalSamplesCorrelationGPUESProducer,
+  ecalTimeBiasCorrectionsGPUESProducer,
+  ecalTimeCalibConstantsGPUESProducer,
+  # ECAL multifit running on GP
+  ecalMultiFitUncalibRecHitGPU,
+  # copy the uncalibrated rechits from GPU to CPU
+  ecalMultiFitUncalibRecHitSoA,
+  # convert the uncalibrated rechits legacy format
+  ecalMultiFitUncalibRecHit,
+))