Skip to content

Commit

Permalink
Use up to 10 time samples for the HBHE digis in Run 3 MC (#531)
Browse files Browse the repository at this point in the history
  • Loading branch information
@vkhristenko @fwyzard
vkhristenko authored and fwyzard committed Aug 8, 2020
1 parent 96c3294 commit 146b6fd
Show file tree
Hide file tree
Showing 4 changed files with 99 additions and 77 deletions.
2 changes: 1 addition & 1 deletion CUDADataFormats/HcalDigi/interface/DigiCollection.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ namespace hcal {

typename StoragePolicy::template StorageSelector<uint32_t>::type ids;
typename StoragePolicy::template StorageSelector<uint16_t>::type data;
uint32_t stride;
uint32_t stride{0};
};

template <typename Flavor, typename StoragePolicy>
Expand Down
6 changes: 3 additions & 3 deletions EventFilter/HcalRawToDigi/bin/makeHcalRaw2DigiGpuValidationPlots.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,15 +130,15 @@ int main(int argc, char* argv[]) {
rt->SetBranchAddress("QIE11DataFrameHcalDataFrameContainer_hcalDigis__RECO.", &wcpuf01he);
rt->SetBranchAddress("HBHEDataFramesSorted_hcalDigis__RECO.", &wcpuf5hb);
rt->SetBranchAddress(
"hcalFlavor5hcalCUDAHostAllocatorAliashcalcommonVecStoragePolicyhcalDigiCollection_hcalCPUDigisProducer_"
"hcalFlavor5calocommonCUDAHostAllocatorAliascalocommonVecStoragePolicyhcalDigiCollection_hcalCPUDigisProducer_"
"f5HBDigis_RECO.",
&wgpuf5hb);
rt->SetBranchAddress(
"hcalFlavor01hcalCUDAHostAllocatorAliashcalcommonVecStoragePolicyhcalDigiCollection_hcalCPUDigisProducer_"
"hcalFlavor01calocommonCUDAHostAllocatorAliascalocommonVecStoragePolicyhcalDigiCollection_hcalCPUDigisProducer_"
"f01HEDigis_RECO.",
&wgpuf01he);
rt->SetBranchAddress(
"hcalFlavor3hcalCUDAHostAllocatorAliashcalcommonVecStoragePolicyhcalDigiCollection_hcalCPUDigisProducer_"
"hcalFlavor3calocommonCUDAHostAllocatorAliascalocommonVecStoragePolicyhcalDigiCollection_hcalCPUDigisProducer_"
"f3HBDigis_RECO.",
&wgpuf3hb);

Expand Down
92 changes: 52 additions & 40 deletions EventFilter/HcalRawToDigi/plugins/HcalDigisProducerGPU.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
class HcalDigisProducerGPU : public edm::stream::EDProducer<edm::ExternalWork> {
public:
explicit HcalDigisProducerGPU(edm::ParameterSet const& ps);
~HcalDigisProducerGPU() override;
~HcalDigisProducerGPU() override = default;
static void fillDescriptions(edm::ConfigurationDescriptions&);

private:
Expand Down Expand Up @@ -49,11 +49,11 @@ class HcalDigisProducerGPU : public edm::stream::EDProducer<edm::ExternalWork> {
cms::cuda::ContextState cudaState_;

struct ConfigParameters {
uint32_t maxChannelsF01HE, maxChannelsF5HB, maxChannelsF3HB, nsamplesF01HE, nsamplesF5HB, nsamplesF3HB;
uint32_t maxChannelsF01HE, maxChannelsF5HB, maxChannelsF3HB;
};
ConfigParameters config_;

// tmp on the host
// per event host buffers
HostCollectionf01 hf01_;
HostCollectionf5 hf5_;
HostCollectionf3 hf3_;
Expand All @@ -76,9 +76,6 @@ void HcalDigisProducerGPU::fillDescriptions(edm::ConfigurationDescriptions& conf
desc.add<uint32_t>("maxChannelsF01HE", 10000u);
desc.add<uint32_t>("maxChannelsF5HB", 10000u);
desc.add<uint32_t>("maxChannelsF3HB", 10000u);
desc.add<uint32_t>("nsamplesF01HE", 8);
desc.add<uint32_t>("nsamplesF5HB", 8);
desc.add<uint32_t>("nsamplesF3HB", 8);

confDesc.addWithDefaultLabel(desc);
}
Expand All @@ -92,27 +89,24 @@ HcalDigisProducerGPU::HcalDigisProducerGPU(const edm::ParameterSet& ps)
config_.maxChannelsF01HE = ps.getParameter<uint32_t>("maxChannelsF01HE");
config_.maxChannelsF5HB = ps.getParameter<uint32_t>("maxChannelsF5HB");
config_.maxChannelsF3HB = ps.getParameter<uint32_t>("maxChannelsF3HB");
config_.nsamplesF01HE = ps.getParameter<uint32_t>("nsamplesF01HE");
config_.nsamplesF5HB = ps.getParameter<uint32_t>("nsamplesF5HB");
config_.nsamplesF3HB = ps.getParameter<uint32_t>("nsamplesF3HB");

// preallocate on the host
hf01_.stride = hcal::compute_stride<hcal::Flavor01>(config_.nsamplesF01HE);
hf5_.stride = hcal::compute_stride<hcal::Flavor5>(config_.nsamplesF5HB);
hf3_.stride = hcal::compute_stride<hcal::Flavor3>(config_.nsamplesF3HB);

// this is a preallocation for the max statically known number of time samples
// actual stride/nsamples will be inferred from data
hf5_.stride = hcal::compute_stride<hcal::Flavor5>(HBHEDataFrame::MAXSAMPLES);
hf01_.stride = hcal::compute_stride<hcal::Flavor01>(QIE11DigiCollection::MAXSAMPLES);
hf3_.stride = hcal::compute_stride<hcal::Flavor3>(QIE11DigiCollection::MAXSAMPLES);
hf01_.reserve(config_.maxChannelsF01HE);
hf5_.reserve(config_.maxChannelsF5HB);
hf3_.reserve(config_.maxChannelsF3HB);
}

HcalDigisProducerGPU::~HcalDigisProducerGPU() {}

void HcalDigisProducerGPU::acquire(edm::Event const& event,
edm::EventSetup const& setup,
edm::WaitingTaskWithArenaHolder holder) {
// raii
cms::cuda::ScopedContextAcquire ctx{event.streamID(), std::move(holder), cudaState_};

// clear host buffers
hf01_.clear();
hf5_.clear();
hf3_.clear();
Expand All @@ -123,35 +117,54 @@ void HcalDigisProducerGPU::acquire(edm::Event const& event,
event.getByToken(hbheDigiToken_, hbheDigis);
event.getByToken(qie11DigiToken_, qie11Digis);

// flavor 0/1 get devie blobs
df01_.data = cms::cuda::make_device_unique<uint16_t[]>(
config_.maxChannelsF01HE * hcal::compute_stride<hcal::Flavor01>(config_.nsamplesF01HE), ctx.stream());
df01_.ids = cms::cuda::make_device_unique<uint32_t[]>(config_.maxChannelsF01HE, ctx.stream());

// flavor3 get device blobs
df3_.data = cms::cuda::make_device_unique<uint16_t[]>(
config_.maxChannelsF3HB * hcal::compute_stride<hcal::Flavor3>(config_.nsamplesF3HB), ctx.stream());
df3_.ids = cms::cuda::make_device_unique<uint32_t[]>(config_.maxChannelsF3HB, ctx.stream());
// init f5 collection
if (hbheDigis->size() > 0) {
auto const nsamples = (*hbheDigis)[0].size();
auto const stride = hcal::compute_stride<hcal::Flavor5>(nsamples);
hf5_.stride = stride;

// flavor5 get device blobs
df5_.stride = stride;
df5_.data = cms::cuda::make_device_unique<uint16_t[]>(
config_.maxChannelsF5HB * stride, ctx.stream());
df5_.ids = cms::cuda::make_device_unique<uint32_t[]>(config_.maxChannelsF5HB, ctx.stream());
df5_.npresamples = cms::cuda::make_device_unique<uint8_t[]>(config_.maxChannelsF5HB, ctx.stream());
}

// flavor5 get device blobs
df5_.data = cms::cuda::make_device_unique<uint16_t[]>(
config_.maxChannelsF5HB * hcal::compute_stride<hcal::Flavor5>(config_.nsamplesF5HB), ctx.stream());
df5_.ids = cms::cuda::make_device_unique<uint32_t[]>(config_.maxChannelsF5HB, ctx.stream());
df5_.npresamples = cms::cuda::make_device_unique<uint8_t[]>(config_.maxChannelsF5HB, ctx.stream());
if (qie11Digis->size() > 0) {
auto const nsamples = qie11Digis->samples();
auto const stride01 = hcal::compute_stride<hcal::Flavor01>(nsamples);
auto const stride3 = hcal::compute_stride<hcal::Flavor3>(nsamples);

hf01_.stride = stride01;
hf3_.stride = stride3;

// flavor 0/1 get devie blobs
df01_.stride = stride01;
df01_.data = cms::cuda::make_device_unique<uint16_t[]>(
config_.maxChannelsF01HE * stride01, ctx.stream());
df01_.ids = cms::cuda::make_device_unique<uint32_t[]>(config_.maxChannelsF01HE, ctx.stream());

// flavor3 get device blobs
df3_.stride = stride3;
df3_.data = cms::cuda::make_device_unique<uint16_t[]>(
config_.maxChannelsF3HB * stride3, ctx.stream());
df3_.ids = cms::cuda::make_device_unique<uint32_t[]>(config_.maxChannelsF3HB, ctx.stream());
}

for (auto const& hbhe : *hbheDigis) {
auto const id = hbhe.id().rawId();
auto const presamples = hbhe.presamples();
hf5_.ids.push_back(id);
hf5_.npresamples.push_back(presamples);
int stride = hcal::compute_stride<hcal::Flavor5>(config_.nsamplesF5HB);
auto const stride = hcal::compute_stride<hcal::Flavor5>(hbhe.size());
assert(stride == hf5_.stride && "strides must be the same for every single digi of the collection");
// simple for now...
static_assert(hcal::Flavor5::HEADER_WORDS == 1);
uint16_t header_word = (1 << 15) | (0x5 << 12) | (0 << 10) | ((hbhe.sample(0).capid() & 0x3) << 8);
hf5_.data.push_back(header_word);
//for (int i=0; i<hcal::Flavor5::HEADER_WORDS; i++)
// hf5_.data.push_back(0);
for (int i = 0; i < stride - hcal::Flavor5::HEADER_WORDS; i++) {
for (unsigned int i = 0; i < stride - hcal::Flavor5::HEADER_WORDS; i++) {
uint16_t s0 = (0 << 7) | (static_cast<uint8_t>(hbhe.sample(2 * i).adc()) & 0x7f);
uint16_t s1 = (0 << 7) | (static_cast<uint8_t>(hbhe.sample(2 * i + 1).adc()) & 0x7f);
uint16_t sample = (s1 << 8) | s0;
Expand All @@ -161,6 +174,7 @@ void HcalDigisProducerGPU::acquire(edm::Event const& event,

for (unsigned int i = 0; i < qie11Digis->size(); i++) {
auto const& digi = QIE11DataFrame{(*qie11Digis)[i]};
assert(digi.samples() == qie11Digis->samples() && "collection nsamples must equal per digi samples");
if (digi.flavor() == 0 or digi.flavor() == 1) {
if (digi.detid().subdetId() != HcalEndcap)
continue;
Expand All @@ -185,6 +199,7 @@ void HcalDigisProducerGPU::acquire(edm::Event const& event,
}

auto lambdaToTransfer = [&ctx](auto* dest, auto const& src) {
if (src.size() == 0) return;
using vector_type = typename std::remove_reference<decltype(src)>::type;
using type = typename vector_type::value_type;
using dest_data_type = typename std::remove_pointer<decltype(dest)>::type;
Expand All @@ -201,17 +216,14 @@ void HcalDigisProducerGPU::acquire(edm::Event const& event,

lambdaToTransfer(df3_.data.get(), hf3_.data);
lambdaToTransfer(df3_.ids.get(), hf3_.ids);
}

void HcalDigisProducerGPU::produce(edm::Event& event, edm::EventSetup const& setup) {
cms::cuda::ScopedContextProduce ctx{cudaState_};

df01_.stride = hcal::compute_stride<hcal::Flavor01>(config_.nsamplesF01HE);
df01_.size = hf01_.ids.size();
df5_.stride = hcal::compute_stride<hcal::Flavor5>(config_.nsamplesF5HB);
df5_.size = hf5_.ids.size();
df3_.stride = hcal::compute_stride<hcal::Flavor3>(config_.nsamplesF3HB);
df3_.size = hf3_.ids.size();
}

void HcalDigisProducerGPU::produce(edm::Event& event, edm::EventSetup const& setup) {
cms::cuda::ScopedContextProduce ctx{cudaState_};

ctx.emplace(event, digisF01HEToken_, std::move(df01_));
ctx.emplace(event, digisF5HBToken_, std::move(df5_));
Expand Down
Loading

0 comments on commit 146b6fd

Please sign in to comment.