Project
Loading...
Searching...
No Matches
GPUChainTrackingClusterizer.cxx
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
14
15#include "GPUChainTracking.h"
18#include "GPULogging.h"
19#include "GPUO2DataTypes.h"
20#include "GPUTPCExtraADC.h"
23#include "GPUNewCalibValues.h"
24#include "GPUConstantMem.h"
25#include "CfChargePos.h"
26#include "CfArray2D.h"
27#include "GPUGeneralKernels.h"
28#include "GPUDefParametersRuntime.h"
31#include "GPUTPCCFDecodeZS.h"
33#include "GPUTPCCFPeakFinder.h"
36#include "GPUTPCCFClusterizer.h"
37#include "GPUTPCCFGather.h"
39#include "GPUTriggerOutputs.h"
40#include "GPUHostDataTypes.h"
46#include "TPCBase/RDHUtils.h"
47
48#ifdef GPUCA_HAS_ONNX
51#include "ORTRootSerializer.h"
52#endif
53
54#ifndef GPUCA_STANDALONE
56#endif
57
58#include "utils/VcShim.h"
59#include "utils/strtag.h"
60#include "utils/vecpod.h"
61#include <numeric>
62#include <random>
63#include <vector>
64
65// #define INSERT_SATURATED_SIGNALS
66
67using namespace o2::gpu;
68using namespace o2::tpc;
69using namespace o2::tpc::constants;
70using namespace o2::dataformats;
71
72#ifndef GPUCA_RUN2
73std::pair<uint32_t, uint32_t> GPUChainTracking::TPCClusterizerDecodeZSCountUpdate(uint32_t iSector, const CfFragment& frag)
74{
78 uint32_t digits = 0;
79 uint32_t pages = 0;
80 for (uint16_t j = 0; j < GPUTrackingInOutZS::NENDPOINTS; j++) {
81 clusterer.mMinMaxCN[j] = mCFContext->fragmentData[frag.index].minMaxCN[iSector][j];
82 if (doGPU) {
83 uint16_t posInEndpoint = 0;
84 uint16_t pagesEndpoint = 0;
85 for (uint32_t k = clusterer.mMinMaxCN[j].zsPtrFirst; k < clusterer.mMinMaxCN[j].zsPtrLast; k++) {
86 const uint32_t pageFirst = (k == clusterer.mMinMaxCN[j].zsPtrFirst) ? clusterer.mMinMaxCN[j].zsPageFirst : 0;
87 const uint32_t pageLast = (k + 1 == clusterer.mMinMaxCN[j].zsPtrLast) ? clusterer.mMinMaxCN[j].zsPageLast : mIOPtrs.tpcZS->sector[iSector].nZSPtr[j][k];
88 for (uint32_t l = pageFirst; l < pageLast; l++) {
89 uint16_t pageDigits = mCFContext->fragmentData[frag.index].pageDigits[iSector][j][posInEndpoint++];
90 if (pageDigits) {
91 *(o++) = GPUTPCClusterFinder::ZSOffset{digits, j, pagesEndpoint};
92 digits += pageDigits;
93 }
94 pagesEndpoint++;
95 }
96 }
97 if (pagesEndpoint != mCFContext->fragmentData[frag.index].pageDigits[iSector][j].size()) {
98 if (GetProcessingSettings().ignoreNonFatalGPUErrors) {
99 GPUError("TPC raw page count mismatch in TPCClusterizerDecodeZSCountUpdate: expected %d / buffered %lu", pagesEndpoint, mCFContext->fragmentData[frag.index].pageDigits[iSector][j].size());
100 return {0, 0};
101 } else {
102 GPUFatal("TPC raw page count mismatch in TPCClusterizerDecodeZSCountUpdate: expected %d / buffered %lu", pagesEndpoint, mCFContext->fragmentData[frag.index].pageDigits[iSector][j].size());
103 }
104 }
105 } else {
107 digits += mCFContext->fragmentData[frag.index].nDigits[iSector][j];
108 pages += mCFContext->fragmentData[frag.index].nPages[iSector][j];
109 }
110 }
111 if (doGPU) {
113 }
114 if (GetProcessingSettings().clusterizerZSSanityCheck && mCFContext->zsVersion >= ZSVersion::ZSVersionDenseLinkBased) {
115 TPCClusterizerEnsureZSOffsets(iSector, frag);
116 }
117 return {digits, pages};
118}
119
120void GPUChainTracking::TPCClusterizerEnsureZSOffsets(uint32_t iSector, const CfFragment& frag)
121{
123 uint32_t nAdcs = 0;
124 for (uint16_t endpoint = 0; endpoint < GPUTrackingInOutZS::NENDPOINTS; endpoint++) {
125 const auto& data = mCFContext->fragmentData[frag.index];
126 uint32_t pagesEndpoint = 0;
127 const uint32_t nAdcsExpected = data.nDigits[iSector][endpoint];
128 const uint32_t nPagesExpected = data.nPages[iSector][endpoint];
129
130 uint32_t nAdcDecoded = 0;
131 const auto& zs = mIOPtrs.tpcZS->sector[iSector];
132 for (uint32_t i = data.minMaxCN[iSector][endpoint].zsPtrFirst; i < data.minMaxCN[iSector][endpoint].zsPtrLast; i++) {
133 const uint32_t pageFirst = (i == data.minMaxCN[iSector][endpoint].zsPtrFirst) ? data.minMaxCN[iSector][endpoint].zsPageFirst : 0;
134 const uint32_t pageLast = (i + 1 == data.minMaxCN[iSector][endpoint].zsPtrLast) ? data.minMaxCN[iSector][endpoint].zsPageLast : zs.nZSPtr[endpoint][i];
135 for (uint32_t j = pageFirst; j < pageLast; j++) {
136 const uint8_t* page = static_cast<const uint8_t*>(zs.zsPtr[endpoint][i]) + j * TPCZSHDR::TPC_ZS_PAGE_SIZE;
137 const header::RAWDataHeader* rawDataHeader = reinterpret_cast<const header::RAWDataHeader*>(page);
138 const TPCZSHDRV2* decHdr = reinterpret_cast<const TPCZSHDRV2*>(page + raw::RDHUtils::getMemorySize(*rawDataHeader) - sizeof(TPCZSHDRV2));
139 const uint16_t nSamplesInPage = decHdr->nADCsamples;
140
141 nAdcDecoded += nSamplesInPage;
142 pagesEndpoint++;
143 }
144 }
145
146 if (pagesEndpoint != nPagesExpected) {
147 GPUFatal("Sector %d, Endpoint %d, Fragment %d: TPC raw page count mismatch: expected %d / buffered %u", iSector, endpoint, frag.index, pagesEndpoint, nPagesExpected);
148 }
149
150 if (nAdcDecoded != nAdcsExpected) {
151 GPUFatal("Sector %d, Endpoint %d, Fragment %d: TPC ADC count mismatch: expected %u, buffered %u", iSector, endpoint, frag.index, nAdcsExpected, nAdcDecoded);
152 }
153
154 if (nAdcs != clusterer.mPzsOffsets[endpoint].offset) {
155 GPUFatal("Sector %d, Endpoint %d, Fragment %d: TPC ADC offset mismatch: expected %u, buffered %u", iSector, endpoint, frag.index, nAdcs, clusterer.mPzsOffsets[endpoint].offset);
156 }
157
158 nAdcs += nAdcsExpected;
159 }
160}
161
162void GPUChainTracking::TPCClusterizerTransferExtraADC(GPUTPCClusterFinder& clusterer, GPUTPCClusterFinder& clustererShadow, int lane, const GPUTPCExtraADC& extraADCs)
163{
164 const int32_t iSector = clusterer.mISector;
165 const auto& frag = clusterer.mPmemory->frag;
166 const auto& digits = extraADCs.digitsBySector[iSector];
167
168 if (frag.index != 0) {
169 return;
170 }
171
172 if (digits.empty()) {
173 return;
174 }
175
176 const size_t chargeMapSize = TPCMapMemoryLayout<PackedCharge>::items(GetProcessingSettings().overrideClusterizerFragmentLen);
177 const size_t chargeMapSizeBytes = chargeMapSize * sizeof(PackedCharge);
178
179 vecpod<uint16_t> chargeMapHostData;
180 chargeMapHostData.resize(chargeMapSize);
181
182 CfArray2D<PackedCharge> chargeMapHost(reinterpret_cast<PackedCharge*>(chargeMapHostData.data()));
183
184 vecpod<CfChargePos> extraPositions;
185 extraPositions.reserve(digits.size());
186
187 GPUMemCpy(RecoStep::TPCClusterFinding, chargeMapHostData.data(), clustererShadow.mPchargeMap, chargeMapSizeBytes, lane, false);
188 SynchronizeStream(lane);
189
190 for (const auto& d : digits) {
191 if (!frag.contains(d.getTimeStamp())) {
192 continue;
193 }
194
195 CfChargePos pos{(tpccf::Row)d.getRow(), (tpccf::Pad)d.getPad(), (tpccf::TPCFragmentTime)(d.getTimeStamp() - frag.start)};
196 chargeMapHost[pos] = PackedCharge(d.getChargeFloat());
197
198 extraPositions.push_back(pos);
199 }
200
201 GPUMemCpy(RecoStep::TPCClusterFinding, clustererShadow.mPchargeMap, chargeMapHostData.data(), chargeMapSizeBytes, lane, true);
202
203 const size_t nPositions = clusterer.mPmemory->counters.nPositions;
204 const size_t extraPositionsOffset = nPositions - extraPositions.size();
205 GPUMemCpy(RecoStep::TPCClusterFinding, clustererShadow.mPpositions + extraPositionsOffset, extraPositions.data(), extraPositions.size() * sizeof(CfChargePos), lane, true);
206}
207
208void GPUChainTracking::TPCClusterizerCheckExtraADCZeros(GPUTPCClusterFinder& clusterer, GPUTPCClusterFinder& clustererShadow, int lane, const GPUTPCExtraADC& extraADCs)
209{
210 const int32_t iSector = clusterer.mISector;
211 const auto& frag = clusterer.mPmemory->frag;
212 const auto& digits = extraADCs.digitsBySector[iSector];
213
214 if (frag.index != 0) {
215 return;
216 }
217
218 if (digits.empty()) {
219 return;
220 }
221
222 const size_t chargeMapSize = TPCMapMemoryLayout<PackedCharge>::items(GetProcessingSettings().overrideClusterizerFragmentLen);
223 const size_t chargeMapSizeBytes = chargeMapSize * sizeof(PackedCharge);
224
225 vecpod<uint16_t> chargeMapHostData;
226 chargeMapHostData.resize(chargeMapSize);
227
228 CfArray2D<PackedCharge> chargeMapHost(reinterpret_cast<PackedCharge*>(chargeMapHostData.data()));
229
230 GPUMemCpy(RecoStep::TPCClusterFinding, chargeMapHostData.data(), clustererShadow.mPchargeMap, chargeMapSizeBytes, lane, false);
231 SynchronizeStream(lane);
232
233 size_t nNonZeroADCs = 0;
234
235 for (const auto& d : digits) {
236 if (!frag.contains(d.getTimeStamp())) {
237 continue;
238 }
239
240 CfChargePos pos{(tpccf::Row)d.getRow(), (tpccf::Pad)d.getPad(), (tpccf::TPCFragmentTime)(d.getTimeStamp() - frag.start)};
241
242 auto adc = chargeMapHost[pos].unpack();
243
244 if (adc != 0) {
245 nNonZeroADCs++;
246 }
247 }
248
249 if (nNonZeroADCs > 0) {
250 GPUInfo("Non Zero ADCs: %zu", nNonZeroADCs);
251 } else {
252 GPUInfo("Cleared all extra ADC values!");
253 }
254}
255
256namespace
257{
258struct TPCCFDecodeScanTmp {
259 int32_t zsPtrFirst, zsPageFirst, zsPtrLast, zsPageLast, hasData, pageCounter;
260};
261
262// Additional ADC values must be generated at start of clusterizer
263// This is required, so enough memory is allocated for the charge points
264// And ADCs can be injected by "simply"
265// -> copying chargeMap + chargePositions to host
266// -> writing additional adcs to chargeMap + positions
267// -> copying values to device
268GPUTPCExtraADC GenerateSaturatedSignals(size_t seed = 42)
269{
270 constexpr int32_t MinTailLength = 50;
271 constexpr int32_t MaxTailLength = 200;
272 constexpr int32_t TailWidth = 3; // Assume tails are 3 pads wide at the moment
273
274 constexpr GPUTPCGeometry geo;
275
276 GPUTPCExtraADC adcs;
277
278 const int32_t nHIPs = 50;
279 const int32_t firstTB = 0; // Place all HIPs in first fragment for now
280 const int32_t lastTB = 4000 - MaxTailLength; // Don't allow cut off tails at fragment borders
281 const int32_t tailADC = 250; // charge should decrease over time, but for now just hardcode ADC above the threshold
282
283 std::mt19937 gen{(uint32_t)seed};
284 std::uniform_int_distribution<> randomRow(0, GPUTPCGeometry::NROWS - 1);
285 std::uniform_int_distribution<> randomTB(firstTB, lastTB);
286 std::uniform_int_distribution<> randomTailLength(MinTailLength, MaxTailLength);
287 // std::normal_distribution<> tailLengthNoise(8, 2.0);
288
289 for (int32_t iHIP = 0; iHIP < nHIPs; iHIP++) {
290
291 const int32_t row = randomRow(gen);
292 const int32_t nPads = geo.NPads(row);
293 std::uniform_int_distribution<> randomPad(0, nPads - 1);
294
295 const int32_t basePad = randomPad(gen);
296 const int32_t baseTb = randomTB(gen);
297
298 auto& digits = adcs.digitsBySector[0];
299
300 const int32_t tailLength = randomTailLength(gen);
301
302 for (int32_t dPad = -TailWidth; dPad <= TailWidth; dPad++) {
303 const int32_t iPad = basePad + dPad;
304 if (iPad < 0 || iPad >= nPads) {
305 continue;
306 }
307
308 for (int32_t dTime = 0; dTime < tailLength; dTime++) {
309 const int32_t iTime = baseTb + dTime;
310
311 if (iTime >= 4000) {
312 break;
313 }
314
315 const auto adc = dTime == 0 && dPad == 0 ? 1023 : tailADC;
316
317 digits.emplace_back(0, adc, row, iPad, iTime);
318 }
319 }
320 }
321
322 GPUInfo("Generated %zu ADCs!", adcs.digitsBySector[0].size());
323
324 return adcs;
325}
326
327} // namespace
328
329std::pair<uint32_t, uint32_t> GPUChainTracking::TPCClusterizerDecodeZSCount(uint32_t iSector, const CfFragment& frag)
330{
331 mRec->getGeneralStepTimer(GeneralStep::Prepare).Start();
332 uint32_t nDigits = 0;
333 uint32_t nPages = 0;
334 uint32_t endpointAdcSamples[GPUTrackingInOutZS::NENDPOINTS];
335 memset(endpointAdcSamples, 0, sizeof(endpointAdcSamples));
337 int32_t firstHBF = (mIOPtrs.settingsTF && mIOPtrs.settingsTF->hasTfStartOrbit) ? mIOPtrs.settingsTF->tfStartOrbit : ((mIOPtrs.tpcZS->sector[iSector].count[0] && mIOPtrs.tpcZS->sector[iSector].nZSPtr[0][0]) ? o2::raw::RDHUtils::getHeartBeatOrbit(*(const o2::header::RAWDataHeader*)mIOPtrs.tpcZS->sector[iSector].zsPtr[0][0]) : 0);
338
339 for (uint16_t j = 0; j < GPUTrackingInOutZS::NENDPOINTS; j++) {
340
341 if (GetProcessingSettings().prefetchTPCpageScan >= 3 && j < GPUTrackingInOutZS::NENDPOINTS - 1) {
342 for (uint32_t k = 0; k < mIOPtrs.tpcZS->sector[iSector].count[j + 1]; k++) {
343 for (uint32_t l = 0; l < mIOPtrs.tpcZS->sector[iSector].nZSPtr[j + 1][k]; l++) {
344 Vc::Common::prefetchMid(((const uint8_t*)mIOPtrs.tpcZS->sector[iSector].zsPtr[j + 1][k]) + l * TPCZSHDR::TPC_ZS_PAGE_SIZE);
345 Vc::Common::prefetchMid(((const uint8_t*)mIOPtrs.tpcZS->sector[iSector].zsPtr[j + 1][k]) + l * TPCZSHDR::TPC_ZS_PAGE_SIZE + sizeof(o2::header::RAWDataHeader));
346 }
347 }
348 }
349
350 std::vector<std::pair<CfFragment, TPCCFDecodeScanTmp>> fragments;
351 fragments.reserve(mCFContext->nFragments);
352 fragments.emplace_back(std::pair<CfFragment, TPCCFDecodeScanTmp>{frag, {0, 0, 0, 0, 0, -1}});
353 for (uint32_t i = 1; i < mCFContext->nFragments; i++) {
354 fragments.emplace_back(std::pair<CfFragment, TPCCFDecodeScanTmp>{fragments.back().first.next(), {0, 0, 0, 0, 0, -1}});
355 }
356 std::vector<bool> fragmentExtends(mCFContext->nFragments, false);
357
358 uint32_t firstPossibleFragment = 0;
360 uint32_t emptyPages = 0;
361 for (uint32_t k = 0; k < mIOPtrs.tpcZS->sector[iSector].count[j]; k++) {
362 if (GetProcessingSettings().tpcSingleSector != -1 && GetProcessingSettings().tpcSingleSector != (int32_t)iSector) {
363 break;
364 }
365 nPages += mIOPtrs.tpcZS->sector[iSector].nZSPtr[j][k];
366 for (uint32_t l = 0; l < mIOPtrs.tpcZS->sector[iSector].nZSPtr[j][k]; l++) {
367
368 if (GetProcessingSettings().prefetchTPCpageScan >= 2 && l + 1 < mIOPtrs.tpcZS->sector[iSector].nZSPtr[j][k]) {
369 Vc::Common::prefetchForOneRead(((const uint8_t*)mIOPtrs.tpcZS->sector[iSector].zsPtr[j][k]) + (l + 1) * TPCZSHDR::TPC_ZS_PAGE_SIZE);
370 Vc::Common::prefetchForOneRead(((const uint8_t*)mIOPtrs.tpcZS->sector[iSector].zsPtr[j][k]) + (l + 1) * TPCZSHDR::TPC_ZS_PAGE_SIZE + sizeof(o2::header::RAWDataHeader));
371 }
372
373 const uint8_t* const page = ((const uint8_t*)mIOPtrs.tpcZS->sector[iSector].zsPtr[j][k]) + l * TPCZSHDR::TPC_ZS_PAGE_SIZE;
375 if (o2::raw::RDHUtils::getMemorySize(*rdh) == sizeof(o2::header::RAWDataHeader)) {
376 emptyPages++;
377 continue;
378 }
379 pageCounter++;
380 const TPCZSHDR* const hdr = (const TPCZSHDR*)(rdh_utils::getLink(o2::raw::RDHUtils::getFEEID(*rdh)) == rdh_utils::DLBZSLinkID ? (page + o2::raw::RDHUtils::getMemorySize(*rdh) - sizeof(TPCZSHDRV2)) : (page + sizeof(o2::header::RAWDataHeader)));
381 if (mCFContext->zsVersion == -1) {
382 mCFContext->zsVersion = hdr->version;
383 if (GetProcessingSettings().param.tpcTriggerHandling && mCFContext->zsVersion < ZSVersion::ZSVersionDenseLinkBased) { // TODO: Move tpcTriggerHandling to recoSteps bitmask
384 static bool errorShown = false;
385 if (errorShown == false) {
386 GPUAlarm("Trigger handling only possible with TPC Dense Link Based data, received version %d, disabling", mCFContext->zsVersion);
387 }
388 errorShown = true;
389 }
390 } else if (mCFContext->zsVersion != (int32_t)hdr->version) {
391 GPUError("Received TPC ZS 8kb page of mixed versions, expected %d, received %d (linkid %d, feeCRU %d, feeEndpoint %d, feelinkid %d)", mCFContext->zsVersion, (int32_t)hdr->version, (int32_t)o2::raw::RDHUtils::getLinkID(*rdh), (int32_t)rdh_utils::getCRU(*rdh), (int32_t)rdh_utils::getEndPoint(*rdh), (int32_t)rdh_utils::getLink(*rdh));
392 constexpr size_t bufferSize = 3 * std::max(sizeof(*rdh), sizeof(*hdr)) + 1;
393 char dumpBuffer[bufferSize];
394 for (size_t i = 0; i < sizeof(*rdh); i++) {
395 // "%02X " guaranteed to be 3 chars + ending 0.
396 snprintf(dumpBuffer + 3 * i, 4, "%02X ", (int32_t)((uint8_t*)rdh)[i]);
397 }
398 GPUAlarm("RDH of page: %s", dumpBuffer);
399 for (size_t i = 0; i < sizeof(*hdr); i++) {
400 // "%02X " guaranteed to be 3 chars + ending 0.
401 snprintf(dumpBuffer + 3 * i, 4, "%02X ", (int32_t)((uint8_t*)hdr)[i]);
402 }
403 GPUAlarm("Metainfo of page: %s", dumpBuffer);
404 if (GetProcessingSettings().ignoreNonFatalGPUErrors) {
405 mCFContext->abandonTimeframe = true;
406 return {0, 0};
407 } else {
408 GPUFatal("Cannot process with invalid TPC ZS data, exiting");
409 }
410 }
411 if (GetProcessingSettings().param.tpcTriggerHandling) {
412 const TPCZSHDRV2* const hdr2 = (const TPCZSHDRV2*)hdr;
413 if (hdr2->flags & TPCZSHDRV2::ZSFlags::TriggerWordPresent) {
414 const char* triggerWord = (const char*)hdr - TPCZSHDRV2::TRIGGER_WORD_SIZE;
416 memcpy((void*)&tmp.triggerWord, triggerWord, TPCZSHDRV2::TRIGGER_WORD_SIZE);
417 tmp.orbit = o2::raw::RDHUtils::getHeartBeatOrbit(*rdh);
418 if (tmp.triggerWord.isValid(0)) {
419 mTriggerBuffer->triggers.emplace(tmp);
420 }
421 }
422 }
423 nDigits += hdr->nADCsamples;
424 endpointAdcSamples[j] += hdr->nADCsamples;
425 uint32_t timeBin = (hdr->timeOffset + (o2::raw::RDHUtils::getHeartBeatOrbit(*rdh) - firstHBF) * o2::constants::lhc::LHCMaxBunches) / LHCBCPERTIMEBIN;
426 uint32_t maxTimeBin = timeBin + hdr->nTimeBinSpan;
427 if (mCFContext->zsVersion >= ZSVersion::ZSVersionDenseLinkBased) {
428 const TPCZSHDRV2* const hdr2 = (const TPCZSHDRV2*)hdr;
429 if (hdr2->flags & TPCZSHDRV2::ZSFlags::nTimeBinSpanBit8) {
430 maxTimeBin += 256;
431 }
432 }
433 if (maxTimeBin > mCFContext->tpcMaxTimeBin) {
434 mCFContext->tpcMaxTimeBin = maxTimeBin;
435 }
436 bool extendsInNextPage = false;
437 if (mCFContext->zsVersion >= ZSVersion::ZSVersionDenseLinkBased) {
438 if (l + 1 < mIOPtrs.tpcZS->sector[iSector].nZSPtr[j][k] && o2::raw::RDHUtils::getMemorySize(*rdh) == TPCZSHDR::TPC_ZS_PAGE_SIZE) {
439 const o2::header::RAWDataHeader* nextrdh = (const o2::header::RAWDataHeader*)(page + TPCZSHDR::TPC_ZS_PAGE_SIZE);
440 extendsInNextPage = o2::raw::RDHUtils::getHeartBeatOrbit(*nextrdh) == o2::raw::RDHUtils::getHeartBeatOrbit(*rdh) && o2::raw::RDHUtils::getMemorySize(*nextrdh) > sizeof(o2::header::RAWDataHeader);
441 }
442 }
443 while (firstPossibleFragment && (uint32_t)fragments[firstPossibleFragment - 1].first.last() > timeBin) {
444 firstPossibleFragment--;
445 }
446 auto handleExtends = [&](uint32_t ff) {
447 if (fragmentExtends[ff]) {
448 if (doGPU) {
449 // Only add extended page on GPU. On CPU the pages are in consecutive memory anyway.
450 // Not adding the page prevents an issue where a page is decoded twice on CPU, when only the extend should be decoded.
451 fragments[ff].second.zsPageLast++;
452 mCFContext->fragmentData[ff].nPages[iSector][j]++;
453 mCFContext->fragmentData[ff].pageDigits[iSector][j].emplace_back(0);
454 }
455 fragmentExtends[ff] = false;
456 }
457 };
458 if (mCFContext->zsVersion >= ZSVersion::ZSVersionDenseLinkBased) {
459 for (uint32_t ff = 0; ff < firstPossibleFragment; ff++) {
460 handleExtends(ff);
461 }
462 }
463 for (uint32_t f = firstPossibleFragment; f < mCFContext->nFragments; f++) {
464 if (timeBin < (uint32_t)fragments[f].first.last() && (uint32_t)fragments[f].first.first() <= maxTimeBin) {
465 if (!fragments[f].second.hasData) {
466 fragments[f].second.hasData = 1;
467 fragments[f].second.zsPtrFirst = k;
468 fragments[f].second.zsPageFirst = l;
469 } else {
470 if (pageCounter > (uint32_t)fragments[f].second.pageCounter + 1) {
471 mCFContext->fragmentData[f].nPages[iSector][j] += emptyPages + pageCounter - fragments[f].second.pageCounter - 1;
472 for (uint32_t k2 = fragments[f].second.zsPtrLast - 1; k2 <= k; k2++) {
473 for (uint32_t l2 = ((int32_t)k2 == fragments[f].second.zsPtrLast - 1) ? fragments[f].second.zsPageLast : 0; l2 < (k2 < k ? mIOPtrs.tpcZS->sector[iSector].nZSPtr[j][k2] : l); l2++) {
474 if (doGPU) {
475 mCFContext->fragmentData[f].pageDigits[iSector][j].emplace_back(0);
476 } else {
477 // CPU cannot skip unneeded pages, so we must keep space to store the invalid dummy clusters
478 const uint8_t* const pageTmp = ((const uint8_t*)mIOPtrs.tpcZS->sector[iSector].zsPtr[j][k2]) + l2 * TPCZSHDR::TPC_ZS_PAGE_SIZE;
479 const o2::header::RAWDataHeader* rdhTmp = (const o2::header::RAWDataHeader*)pageTmp;
480 if (o2::raw::RDHUtils::getMemorySize(*rdhTmp) != sizeof(o2::header::RAWDataHeader)) {
481 const TPCZSHDR* const hdrTmp = (const TPCZSHDR*)(rdh_utils::getLink(o2::raw::RDHUtils::getFEEID(*rdhTmp)) == rdh_utils::DLBZSLinkID ? (pageTmp + o2::raw::RDHUtils::getMemorySize(*rdhTmp) - sizeof(TPCZSHDRV2)) : (pageTmp + sizeof(o2::header::RAWDataHeader)));
482 mCFContext->fragmentData[f].nDigits[iSector][j] += hdrTmp->nADCsamples;
483 }
484 }
485 }
486 }
487 } else if (emptyPages) {
488 mCFContext->fragmentData[f].nPages[iSector][j] += emptyPages;
489 if (doGPU) {
490 for (uint32_t m = 0; m < emptyPages; m++) {
491 mCFContext->fragmentData[f].pageDigits[iSector][j].emplace_back(0);
492 }
493 }
494 }
495 }
496 fragments[f].second.zsPtrLast = k + 1;
497 fragments[f].second.zsPageLast = l + 1;
498 fragments[f].second.pageCounter = pageCounter;
499 mCFContext->fragmentData[f].nPages[iSector][j]++;
500 mCFContext->fragmentData[f].nDigits[iSector][j] += hdr->nADCsamples;
501 if (doGPU) {
502 mCFContext->fragmentData[f].pageDigits[iSector][j].emplace_back(hdr->nADCsamples);
503 }
504 fragmentExtends[f] = extendsInNextPage;
505 } else {
506 handleExtends(f);
507 if (timeBin < (uint32_t)fragments[f].first.last()) {
508 if (mCFContext->zsVersion >= ZSVersion::ZSVersionDenseLinkBased) {
509 for (uint32_t ff = f + 1; ff < mCFContext->nFragments; ff++) {
510 handleExtends(ff);
511 }
512 }
513 break;
514 } else {
515 firstPossibleFragment = f + 1;
516 }
517 }
518 }
519 emptyPages = 0;
520 }
521 }
522 for (uint32_t f = 0; f < mCFContext->nFragments; f++) {
523 mCFContext->fragmentData[f].minMaxCN[iSector][j].zsPtrLast = fragments[f].second.zsPtrLast;
524 mCFContext->fragmentData[f].minMaxCN[iSector][j].zsPtrFirst = fragments[f].second.zsPtrFirst;
525 mCFContext->fragmentData[f].minMaxCN[iSector][j].zsPageLast = fragments[f].second.zsPageLast;
526 mCFContext->fragmentData[f].minMaxCN[iSector][j].zsPageFirst = fragments[f].second.zsPageFirst;
527 }
528 }
529 mCFContext->nPagesTotal += nPages;
530 mCFContext->nPagesSector[iSector] = nPages;
531
532 mCFContext->nDigitsEndpointMax[iSector] = 0;
533 for (uint32_t i = 0; i < GPUTrackingInOutZS::NENDPOINTS; i++) {
534 if (endpointAdcSamples[i] > mCFContext->nDigitsEndpointMax[iSector]) {
535 mCFContext->nDigitsEndpointMax[iSector] = endpointAdcSamples[i];
536 }
537 }
538 uint32_t nDigitsFragmentMax = 0;
539 for (uint32_t i = 0; i < mCFContext->nFragments; i++) {
540 uint32_t pagesInFragment = 0;
541 uint32_t digitsInFragment = 0;
542 for (uint16_t j = 0; j < GPUTrackingInOutZS::NENDPOINTS; j++) {
543 pagesInFragment += mCFContext->fragmentData[i].nPages[iSector][j];
544 digitsInFragment += mCFContext->fragmentData[i].nDigits[iSector][j];
545 }
546 mCFContext->nPagesFragmentMax = std::max(mCFContext->nPagesFragmentMax, pagesInFragment);
547 nDigitsFragmentMax = std::max(nDigitsFragmentMax, digitsInFragment);
548 }
549 mRec->getGeneralStepTimer(GeneralStep::Prepare).Stop();
550 return {nDigits, nDigitsFragmentMax};
551}
552
553void GPUChainTracking::RunTPCClusterizer_compactPeaks(GPUTPCClusterFinder& clusterer, GPUTPCClusterFinder& clustererShadow, int32_t stage, bool doGPU, int32_t lane)
554{
555 auto& in = stage ? clustererShadow.mPpeakPositions : clustererShadow.mPpositions;
556 auto& out = stage ? clustererShadow.mPfilteredPeakPositions : clustererShadow.mPpeakPositions;
557 if (doGPU && mRec->getGPUParameters(doGPU).par_CF_SCAN_WORKGROUP_SIZE > 1) { // In case we have only 1 device thread, run the CPU version
558 const uint32_t iSector = clusterer.mISector;
559 auto& count = stage ? clusterer.mPmemory->counters.nPeaks : clusterer.mPmemory->counters.nPositions;
560
561 std::vector<size_t> counts;
562
563 uint32_t nSteps = clusterer.getNSteps(count);
564 if (nSteps > clusterer.mNBufs) {
565 GPUError("Clusterer buffers exceeded (%u > %u)", nSteps, (int32_t)clusterer.mNBufs);
566 exit(1);
567 }
568
569 int32_t scanWorkgroupSize = mRec->getGPUParameters(doGPU).par_CF_SCAN_WORKGROUP_SIZE;
570 size_t tmpCount = count;
571 if (nSteps > 1) {
572 for (uint32_t i = 1; i < nSteps; i++) {
573 counts.push_back(tmpCount);
574 if (i == 1) {
575 runKernel<GPUTPCCFStreamCompaction, GPUTPCCFStreamCompaction::scanStart>({GetGrid(tmpCount, scanWorkgroupSize, lane), {iSector}}, i, stage);
576 } else {
577 runKernel<GPUTPCCFStreamCompaction, GPUTPCCFStreamCompaction::scanUp>({GetGrid(tmpCount, scanWorkgroupSize, lane), {iSector}}, i, tmpCount);
578 }
579 tmpCount = (tmpCount + scanWorkgroupSize - 1) / scanWorkgroupSize;
580 }
581
582 runKernel<GPUTPCCFStreamCompaction, GPUTPCCFStreamCompaction::scanTop>({GetGrid(tmpCount, scanWorkgroupSize, lane), {iSector}}, nSteps, tmpCount);
583
584 for (uint32_t i = nSteps - 1; i > 1; i--) {
585 tmpCount = counts[i - 1];
586 runKernel<GPUTPCCFStreamCompaction, GPUTPCCFStreamCompaction::scanDown>({GetGrid(tmpCount - scanWorkgroupSize, scanWorkgroupSize, lane), {iSector}}, i, scanWorkgroupSize, tmpCount);
587 }
588 }
589
590 runKernel<GPUTPCCFStreamCompaction, GPUTPCCFStreamCompaction::compactDigits>({GetGrid(count, scanWorkgroupSize, lane), {iSector}}, 1, stage, in, out);
591 } else {
592 if (doGPU) {
593 SynchronizeStream(lane); // doGPU with CF_SCAN_WORKGROUP_SIZE <= 1
594 }
595 auto& nOut = stage ? clustererShadow.mPmemory->counters.nClusters : clustererShadow.mPmemory->counters.nPeaks;
596 auto& nIn = stage ? clustererShadow.mPmemory->counters.nPeaks : clustererShadow.mPmemory->counters.nPositions;
597 size_t count = 0;
598 for (size_t i = 0; i < nIn; i++) {
599 if (clustererShadow.mPisPeak[i]) {
600 out[count++] = in[i];
601 }
602 }
603 nOut = count;
604 }
605}
606
607std::pair<uint32_t, uint32_t> GPUChainTracking::RunTPCClusterizer_transferZS(int32_t iSector, const CfFragment& frag, int32_t lane, const GPUTPCExtraADC& extraADCs)
608{
609 bool doGPU = GetRecoStepsGPU() & RecoStep::TPCClusterFinding;
610 if (mCFContext->abandonTimeframe) {
611 return {0, 0};
612 }
613 auto retVal = TPCClusterizerDecodeZSCountUpdate(iSector, frag);
614 if (frag.index == 0) {
615 retVal.first += extraADCs.digitsBySector[iSector].size();
616 }
617 if (doGPU) {
619 GPUTPCClusterFinder& clustererShadow = doGPU ? processorsShadow()->tpcClusterer[iSector] : clusterer;
620 uint32_t nPagesSector = 0;
621 for (uint32_t j = 0; j < GPUTrackingInOutZS::NENDPOINTS; j++) {
622 uint32_t nPages = 0;
623 mInputsHost->mPzsMeta->sector[iSector].zsPtr[j] = &mInputsShadow->mPzsPtrs[iSector * GPUTrackingInOutZS::NENDPOINTS + j];
624 mInputsHost->mPzsPtrs[iSector * GPUTrackingInOutZS::NENDPOINTS + j] = clustererShadow.mPzs + (nPagesSector + nPages) * TPCZSHDR::TPC_ZS_PAGE_SIZE;
625 for (uint32_t k = clusterer.mMinMaxCN[j].zsPtrFirst; k < clusterer.mMinMaxCN[j].zsPtrLast; k++) {
626 const uint32_t min = (k == clusterer.mMinMaxCN[j].zsPtrFirst) ? clusterer.mMinMaxCN[j].zsPageFirst : 0;
627 const uint32_t max = (k + 1 == clusterer.mMinMaxCN[j].zsPtrLast) ? clusterer.mMinMaxCN[j].zsPageLast : mIOPtrs.tpcZS->sector[iSector].nZSPtr[j][k];
628 if (max > min) {
629 char* src = (char*)mIOPtrs.tpcZS->sector[iSector].zsPtr[j][k] + min * TPCZSHDR::TPC_ZS_PAGE_SIZE;
630 char* ptrLast = (char*)mIOPtrs.tpcZS->sector[iSector].zsPtr[j][k] + (max - 1) * TPCZSHDR::TPC_ZS_PAGE_SIZE;
631 size_t size = (ptrLast - src) + o2::raw::RDHUtils::getMemorySize(*(const o2::header::RAWDataHeader*)ptrLast);
632 GPUMemCpy(RecoStep::TPCClusterFinding, clustererShadow.mPzs + (nPagesSector + nPages) * TPCZSHDR::TPC_ZS_PAGE_SIZE, src, size, lane, true);
633 }
634 nPages += max - min;
635 }
636 mInputsHost->mPzsMeta->sector[iSector].nZSPtr[j] = &mInputsShadow->mPzsSizes[iSector * GPUTrackingInOutZS::NENDPOINTS + j];
637 mInputsHost->mPzsSizes[iSector * GPUTrackingInOutZS::NENDPOINTS + j] = nPages;
638 mInputsHost->mPzsMeta->sector[iSector].count[j] = 1;
639 nPagesSector += nPages;
640 }
641 GPUMemCpy(RecoStep::TPCClusterFinding, clustererShadow.mPzsOffsets, clusterer.mPzsOffsets, clusterer.mNMaxPages * sizeof(*clusterer.mPzsOffsets), lane, true);
642 }
643 return retVal;
644}
645
646int32_t GPUChainTracking::RunTPCClusterizer_prepare(bool restorePointers, const GPUTPCExtraADC& extraADCs)
647{
649 if (restorePointers) {
650 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
651 processors()->tpcClusterer[iSector].mPzsOffsets = mCFContext->ptrSave[iSector].zsOffsetHost;
652 processorsShadow()->tpcClusterer[iSector].mPzsOffsets = mCFContext->ptrSave[iSector].zsOffsetDevice;
654 }
655 processorsShadow()->ioPtrs.clustersNative = mCFContext->ptrClusterNativeSave;
656 return 0;
657 }
658 const auto& threadContext = GetThreadContext();
660 if (mCFContext == nullptr) {
662 }
663 const int16_t maxFragmentLen = GetProcessingSettings().overrideClusterizerFragmentLen;
664 const uint32_t maxAllowedTimebin = param().par.continuousTracking ? std::max<int32_t>(param().continuousMaxTimeBin, maxFragmentLen) : constants::TPC_MAX_TIME_BIN_TRIGGERED;
665 mCFContext->tpcMaxTimeBin = maxAllowedTimebin;
666 const CfFragment fragmentMax{(tpccf::TPCTime)mCFContext->tpcMaxTimeBin + 1, maxFragmentLen};
667 mCFContext->prepare(mIOPtrs.tpcZS, fragmentMax);
668 if (GetProcessingSettings().param.tpcTriggerHandling) {
669 mTriggerBuffer->triggers.clear();
670 }
671 if (mIOPtrs.tpcZS) {
672 uint32_t nDigitsFragmentMax[NSECTORS];
673 mCFContext->zsVersion = -1;
674 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
676 const void* rdh = mIOPtrs.tpcZS->sector[iSector].zsPtr[0][0];
677 if (rdh && o2::raw::RDHUtils::getVersion<o2::header::RAWDataHeaderV6>() > o2::raw::RDHUtils::getVersion(rdh)) {
678 GPUError("Data has invalid RDH version %d, %d required (sector %d)\n", o2::raw::RDHUtils::getVersion(rdh), o2::raw::RDHUtils::getVersion<o2::header::RAWDataHeader>(), iSector);
679 return 1;
680 }
681 }
682
683 if (GetProcessingSettings().prefetchTPCpageScan >= 1 && iSector < NSECTORS - 1) {
684 for (uint32_t j = 0; j < GPUTrackingInOutZS::NENDPOINTS; j++) {
685 for (uint32_t k = 0; k < mIOPtrs.tpcZS->sector[iSector].count[j]; k++) {
686 for (uint32_t l = 0; l < mIOPtrs.tpcZS->sector[iSector].nZSPtr[j][k]; l++) {
687 Vc::Common::prefetchFar(((const uint8_t*)mIOPtrs.tpcZS->sector[iSector + 1].zsPtr[j][k]) + l * TPCZSHDR::TPC_ZS_PAGE_SIZE);
688 Vc::Common::prefetchFar(((const uint8_t*)mIOPtrs.tpcZS->sector[iSector + 1].zsPtr[j][k]) + l * TPCZSHDR::TPC_ZS_PAGE_SIZE + sizeof(o2::header::RAWDataHeader));
689 }
690 }
691 }
692 }
693
694 const auto& x = TPCClusterizerDecodeZSCount(iSector, fragmentMax);
695 nDigitsFragmentMax[iSector] = x.first;
697 mRec->MemoryScalers()->nTPCdigits += x.first;
698 }
699 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
700 uint32_t nDigitsBase = nDigitsFragmentMax[iSector];
701 uint32_t threshold = 40000000;
702 uint32_t nDigitsScaled = nDigitsBase > threshold ? nDigitsBase : std::min((threshold + nDigitsBase) / 2, 2 * nDigitsBase);
703 processors()->tpcClusterer[iSector].SetNMaxDigits(processors()->tpcClusterer[iSector].mPmemory->counters.nDigits, mCFContext->nPagesFragmentMax, nDigitsScaled, mCFContext->nDigitsEndpointMax[iSector]);
704 if (doGPU) {
705 processorsShadow()->tpcClusterer[iSector].SetNMaxDigits(processors()->tpcClusterer[iSector].mPmemory->counters.nDigits, mCFContext->nPagesFragmentMax, nDigitsScaled, mCFContext->nDigitsEndpointMax[iSector]);
706 }
707 if (mPipelineNotifyCtx && GetProcessingSettings().doublePipelineClusterizer) {
708 mPipelineNotifyCtx->rec->AllocateRegisteredForeignMemory(processors()->tpcClusterer[iSector].mZSOffsetId, mRec);
709 mPipelineNotifyCtx->rec->AllocateRegisteredForeignMemory(processors()->tpcClusterer[iSector].mZSId, mRec);
710 } else {
711 AllocateRegisteredMemory(processors()->tpcClusterer[iSector].mZSOffsetId);
712 AllocateRegisteredMemory(processors()->tpcClusterer[iSector].mZSId);
713 }
714 }
715 } else {
716 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
718 mRec->MemoryScalers()->nTPCdigits += nDigits;
719 processors()->tpcClusterer[iSector].SetNMaxDigits(nDigits, mCFContext->nPagesFragmentMax, nDigits, 0);
720 }
721 }
722
723 if (mIOPtrs.tpcZS) {
724 GPUInfo("Event has %u 8kb TPC ZS pages (version %d), %ld digits", mCFContext->nPagesTotal, mCFContext->zsVersion, (int64_t)mRec->MemoryScalers()->nTPCdigits);
725 } else {
726 GPUInfo("Event has %ld TPC Digits", (int64_t)mRec->MemoryScalers()->nTPCdigits);
727 }
728
729 if (mCFContext->tpcMaxTimeBin > maxAllowedTimebin) {
730 GPUError("Input data has invalid time bin %u > %d", mCFContext->tpcMaxTimeBin, maxAllowedTimebin);
731 if (GetProcessingSettings().ignoreNonFatalGPUErrors) {
732 mCFContext->abandonTimeframe = true;
733 mCFContext->tpcMaxTimeBin = maxAllowedTimebin;
734 } else {
735 return 1;
736 }
737 }
738
739 mCFContext->fragmentFirst = CfFragment{std::max<int32_t>(mCFContext->tpcMaxTimeBin + 1, maxFragmentLen), maxFragmentLen};
740 for (int32_t iSector = 0; iSector < GetProcessingSettings().nTPCClustererLanes && iSector < NSECTORS; iSector++) {
741 if (mIOPtrs.tpcZS && mCFContext->nPagesSector[iSector] && mCFContext->zsVersion != -1) {
742 mCFContext->nextPos[iSector] = RunTPCClusterizer_transferZS(iSector, mCFContext->fragmentFirst, GetProcessingSettings().nTPCClustererLanes + iSector, extraADCs);
743 }
744 }
745
746 if (mPipelineNotifyCtx && GetProcessingSettings().doublePipelineClusterizer) {
747 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
748 mCFContext->ptrSave[iSector].zsOffsetHost = processors()->tpcClusterer[iSector].mPzsOffsets;
749 mCFContext->ptrSave[iSector].zsOffsetDevice = processorsShadow()->tpcClusterer[iSector].mPzsOffsets;
751 }
752 }
753 return 0;
754}
755#endif
756
758{
759 if (param().rec.fwdTPCDigitsAsClusters) {
760 return ForwardTPCDigits();
761 }
762#ifndef GPUCA_RUN2
763 int32_t tpcTimeBinCut = (mUpdateNewCalibObjects && mNewCalibValues->newTPCTimeBinCut) ? mNewCalibValues->tpcTimeBinCut : param().tpcCutTimeBin;
764
766 const auto& threadContext = GetThreadContext();
767 const bool doGPU = GetRecoStepsGPU() & RecoStep::TPCClusterFinding;
768
769 GPUTPCExtraADC extraADCs;
770#ifdef INSERT_SATURATED_SIGNALS
771 extraADCs = GenerateSaturatedSignals();
772#endif
773
774 if (RunTPCClusterizer_prepare(mPipelineNotifyCtx && GetProcessingSettings().doublePipelineClusterizer, extraADCs)) {
776 }
777 if (GetProcessingSettings().autoAdjustHostThreads && !doGPU) {
779 }
780
782 float tpcHitLowOccupancyScalingFactor = 1.f;
784 uint32_t nHitsBase = mRec->MemoryScalers()->nTPCHits;
785 uint32_t threshold = 30000000 / 256 * mIOPtrs.settingsTF->nHBFPerTF;
786 if (mIOPtrs.settingsTF->nHBFPerTF < 64) {
787 threshold *= 2;
788 }
789 mRec->MemoryScalers()->nTPCHits = std::max<uint32_t>(nHitsBase, std::min<uint32_t>(threshold, nHitsBase * 3.5f)); // Increase the buffer size for low occupancy data to compensate for noisy pads creating exceiive clusters
790 if (nHitsBase < threshold) {
791 float maxFactor = mRec->MemoryScalers()->nTPCHits < threshold * 2 / 3 ? 3 : (mRec->MemoryScalers()->nTPCHits < threshold ? 2.25f : 1.75f);
792 mRec->MemoryScalers()->temporaryFactor *= std::min(maxFactor, (float)threshold / nHitsBase);
793 tpcHitLowOccupancyScalingFactor = std::min(3.5f, (float)threshold / nHitsBase);
794 }
795 }
796 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
797 processors()->tpcClusterer[iSector].SetMaxData(mIOPtrs); // First iteration to set data sizes
798 }
799 mRec->ComputeReuseMax(nullptr); // Resolve maximums for shared buffers
800 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
801 SetupGPUProcessor(&processors()->tpcClusterer[iSector], true); // Now we allocate
802 }
803 if (mPipelineNotifyCtx && GetProcessingSettings().doublePipelineClusterizer) {
804 RunTPCClusterizer_prepare(true, extraADCs); // Restore some pointers, allocated by the other pipeline, and set to 0 by SetupGPUProcessor (since not allocated in this pipeline)
805 }
806
807 if (doGPU && mIOPtrs.tpcZS) {
809 WriteToConstantMemory(RecoStep::TPCClusterFinding, (char*)&processors()->ioPtrs - (char*)processors(), &processorsShadow()->ioPtrs, sizeof(processorsShadow()->ioPtrs), mRec->NStreams() - 1);
810 }
811 if (doGPU) {
812 WriteToConstantMemory(RecoStep::TPCClusterFinding, (char*)processors()->tpcClusterer - (char*)processors(), processorsShadow()->tpcClusterer, sizeof(GPUTPCClusterFinder) * NSECTORS, mRec->NStreams() - 1, &mEvents->init);
813 }
814
815#ifdef GPUCA_HAS_ONNX
816 const GPUSettingsProcessingNNclusterizer& nn_settings = GetProcessingSettings().nn;
817 GPUTPCNNClusterizerHost nnApplications[GetProcessingSettings().nTPCClustererLanes];
818
819 // Maximum of 4 lanes supported
820 HighResTimer* nnTimers[12];
821
822 if (nn_settings.applyNNclusterizer) {
823 int32_t deviceId = -1;
824 int32_t numLanes = GetProcessingSettings().nTPCClustererLanes;
825 int32_t maxThreads = mRec->getNKernelHostThreads(true);
826 // bool recreateMemoryAllocator = false;
827
828 if (GetProcessingSettings().debugLevel >= 1) {
829 nnTimers[0] = &getTimer<GPUTPCNNClusterizer, 0>("GPUTPCNNClusterizer_ONNXClassification_0_", 0);
830 nnTimers[1] = &getTimer<GPUTPCNNClusterizer, 1>("GPUTPCNNClusterizer_ONNXRegression_1_", 1);
831 nnTimers[2] = &getTimer<GPUTPCNNClusterizer, 2>("GPUTPCNNClusterizer_ONNXRegression2_2_", 2);
832 nnTimers[3] = &getTimer<GPUTPCNNClusterizer, 3>("GPUTPCNNClusterizer_ONNXClassification_0_", 3);
833 nnTimers[4] = &getTimer<GPUTPCNNClusterizer, 4>("GPUTPCNNClusterizer_ONNXRegression_1_", 4);
834 nnTimers[5] = &getTimer<GPUTPCNNClusterizer, 5>("GPUTPCNNClusterizer_ONNXRegression2_2_", 5);
835 nnTimers[6] = &getTimer<GPUTPCNNClusterizer, 6>("GPUTPCNNClusterizer_ONNXClassification_0_", 6);
836 nnTimers[7] = &getTimer<GPUTPCNNClusterizer, 7>("GPUTPCNNClusterizer_ONNXRegression_1_", 7);
837 nnTimers[8] = &getTimer<GPUTPCNNClusterizer, 8>("GPUTPCNNClusterizer_ONNXRegression2_2_", 8);
838 nnTimers[9] = &getTimer<GPUTPCNNClusterizer, 9>("GPUTPCNNClusterizer_ONNXClassification_0_", 9);
839 nnTimers[10] = &getTimer<GPUTPCNNClusterizer, 10>("GPUTPCNNClusterizer_ONNXRegression_1_", 10);
840 nnTimers[11] = &getTimer<GPUTPCNNClusterizer, 11>("GPUTPCNNClusterizer_ONNXRegression2_2_", 11);
841 }
842
843 mRec->runParallelOuterLoop(doGPU, numLanes, [&](uint32_t lane) {
844 nnApplications[lane].init(nn_settings, GetProcessingSettings().deterministicGPUReconstruction);
845 if (nnApplications[lane].mModelsUsed[0]) {
846 SetONNXGPUStream(*(nnApplications[lane].mModelClass).getSessionOptions(), lane, &deviceId);
847 (nnApplications[lane].mModelClass).setDeviceId(deviceId);
848 if (nnApplications[lane].mModelClass.getIntraOpNumThreads() > maxThreads) {
849 nnApplications[lane].mModelClass.setIntraOpNumThreads(maxThreads);
850 }
851 (nnApplications[lane].mModelClass).initEnvironment();
852 // Registering this once seems to be enough, even with different environmnents / models. ONNX apparently uses this per device and stores the OrtAllocator internally. All models will then use the volatile allocation.
853 // But environment must be valid, so we init the model environment first and use it here afterwards.
854 // Either this is done in one environment with lane == 0 or by recreating the allocator using recreateMemoryAllocator.
855 // TODO: Volatile allocation works for reserving, but not yet for allocations when binding the input tensor
856 // if (lane == 0) {
857 // nnApplications[lane].directOrtAllocator((nnApplications[lane].mModelClass).getEnv(), (nnApplications[lane].mModelClass).getMemoryInfo(), mRec, recreateMemoryAllocator);
858 // }
859 // recreateMemoryAllocator = true;
860 if (!nn_settings.nnLoadFromCCDB) {
861 (nnApplications[lane].mModelClass).initSession(); // loads from file
862 } else {
863 (nnApplications[lane].mModelClass).initSessionFromBuffer((processors()->calibObjects.nnClusterizerNetworks[0])->getONNXModel(), (processors()->calibObjects.nnClusterizerNetworks[0])->getONNXModelSize()); // loads from CCDB
864 }
865 }
866 if (nnApplications[lane].mModelsUsed[1]) {
867 SetONNXGPUStream(*(nnApplications[lane].mModelReg1).getSessionOptions(), lane, &deviceId);
868 (nnApplications[lane].mModelReg1).setDeviceId(deviceId);
869 if (nnApplications[lane].mModelReg1.getIntraOpNumThreads() > maxThreads) {
870 nnApplications[lane].mModelReg1.setIntraOpNumThreads(maxThreads);
871 }
872 // (nnApplications[lane].mModelReg1).setEnv((nnApplications[lane].mModelClass).getEnv());
873 (nnApplications[lane].mModelReg1).initEnvironment();
874 // nnApplications[lane].directOrtAllocator((nnApplications[lane].mModelReg1).getEnv(), (nnApplications[lane].mModelReg1).getMemoryInfo(), mRec, recreateMemoryAllocator);
875 if (!nn_settings.nnLoadFromCCDB) {
876 (nnApplications[lane].mModelReg1).initSession(); // loads from file
877 } else {
878 (nnApplications[lane].mModelReg1).initSessionFromBuffer((processors()->calibObjects.nnClusterizerNetworks[1])->getONNXModel(), (processors()->calibObjects.nnClusterizerNetworks[1])->getONNXModelSize()); // loads from CCDB
879 }
880 }
881 if (nnApplications[lane].mModelsUsed[2]) {
882 SetONNXGPUStream(*(nnApplications[lane].mModelReg2).getSessionOptions(), lane, &deviceId);
883 (nnApplications[lane].mModelReg2).setDeviceId(deviceId);
884 if (nnApplications[lane].mModelReg2.getIntraOpNumThreads() > maxThreads) {
885 nnApplications[lane].mModelReg2.setIntraOpNumThreads(maxThreads);
886 }
887 // (nnApplications[lane].mModelReg2).setEnv((nnApplications[lane].mModelClass).getEnv());
888 (nnApplications[lane].mModelReg2).initEnvironment();
889 // nnApplications[lane].directOrtAllocator((nnApplications[lane].mModelClass).getEnv(), (nnApplications[lane].mModelClass).getMemoryInfo(), mRec, recreateMemoryAllocator);
890 if (!nn_settings.nnLoadFromCCDB) {
891 (nnApplications[lane].mModelReg2).initSession(); // loads from file
892 } else {
893 (nnApplications[lane].mModelReg2).initSessionFromBuffer((processors()->calibObjects.nnClusterizerNetworks[2])->getONNXModel(), (processors()->calibObjects.nnClusterizerNetworks[2])->getONNXModelSize()); // loads from CCDB
894 }
895 }
896 if (nn_settings.nnClusterizerVerbosity > 0) {
897 LOG(info) << "(ORT) Allocated ONNX stream for lane " << lane << " and device " << deviceId;
898 }
899 });
900 const int16_t maxFragmentLen = GetProcessingSettings().overrideClusterizerFragmentLen;
901 const uint32_t maxAllowedTimebin = param().par.continuousTracking ? std::max<int32_t>(param().continuousMaxTimeBin, maxFragmentLen) : constants::TPC_MAX_TIME_BIN_TRIGGERED;
902 for (int32_t sector = 0; sector < NSECTORS; sector++) {
903 GPUTPCNNClusterizer& clustererNN = processors()->tpcNNClusterer[sector];
904 GPUTPCNNClusterizer& clustererNNShadow = doGPU ? processorsShadow()->tpcNNClusterer[sector] : clustererNN;
905 int32_t lane = sector % numLanes;
906 clustererNN.mDeviceId = deviceId;
907 clustererNN.mISector = sector;
909 nnApplications[lane].initClusterizer(nn_settings, clustererNN, maxFragmentLen, maxAllowedTimebin);
910 if (doGPU) {
911 clustererNNShadow.mDeviceId = deviceId;
912 clustererNNShadow.mISector = sector;
914 nnApplications[lane].initClusterizer(nn_settings, clustererNNShadow, maxFragmentLen, maxAllowedTimebin);
915 }
916 if (nn_settings.nnClusterizerVerbosity > 2) {
917 LOG(info) << "(NNCLUS, GPUChainTrackingClusterizer, this=" << this << ") Processor initialized. Sector " << sector << ", lane " << lane << ", max clusters " << clustererNN.mNnClusterizerTotalClusters << " (clustererNN=" << &clustererNN << ", clustererNNShadow=" << &clustererNNShadow << ")";
918 }
920 if (nn_settings.nnClusterizerVerbosity > 2) {
921 LOG(info) << "(NNCLUS, GPUChainTrackingClusterizer, this=" << this << ") Memory registered for memoryId " << clustererNN.mMemoryId << " (clustererNN=" << &clustererNN << ", clustererNNShadow=" << &clustererNNShadow << ")";
922 }
923 // nnApplications[lane].createBoundary(clustererNNShadow);
924 // nnApplications[lane].createIndexLookup(clustererNNShadow);
925 }
926 if (doGPU) {
927 if (nn_settings.nnClusterizerVerbosity > 2) {
928 LOG(info) << "(NNCLUS, GPUChainTrackingClusterizer, this=" << this << ") Writing to constant memory...";
929 }
930 WriteToConstantMemory(RecoStep::TPCClusterFinding, (char*)&processors()->tpcNNClusterer - (char*)processors(), &processorsShadow()->tpcNNClusterer, sizeof(GPUTPCNNClusterizer) * NSECTORS, mRec->NStreams() - 1, &mEvents->init);
931 if (nn_settings.nnClusterizerVerbosity > 2) {
932 LOG(info) << "(NNCLUS, GPUChainTrackingClusterizer, this=" << this << ") Writing to constant memory done";
933 }
934 }
935 }
936#endif
937
938 size_t nClsTotal = 0;
939 ClusterNativeAccess* tmpNativeAccess = mClusterNativeAccess.get();
940 ClusterNative* tmpNativeClusters = nullptr;
941 std::unique_ptr<ClusterNative[]> tmpNativeClusterBuffer;
942
943 const bool buildNativeGPU = doGPU && NeedTPCClustersOnGPU();
944 const bool buildNativeHost = (mRec->GetRecoStepsOutputs() & gpudatatypes::InOutType::TPCClusters) || GetProcessingSettings().deterministicGPUReconstruction; // TODO: Should do this also when clusters are needed for later steps on the host but not requested as output
945 const bool propagateMCLabels = buildNativeHost && GetProcessingSettings().runMC && processors()->ioPtrs.tpcPackedDigits && processors()->ioPtrs.tpcPackedDigits->tpcDigitsMC;
946 const bool sortClusters = buildNativeHost && (GetProcessingSettings().deterministicGPUReconstruction || GetProcessingSettings().debugLevel >= 4);
947
949 GPUWarning("Requested to process MC labels, but no labels present");
950 }
951
952 auto* digitsMC = propagateMCLabels ? processors()->ioPtrs.tpcPackedDigits->tpcDigitsMC : nullptr;
953
954 mInputsHost->mNClusterNative = mInputsShadow->mNClusterNative = mRec->MemoryScalers()->nTPCHits * tpcHitLowOccupancyScalingFactor;
955 if (buildNativeGPU) {
956 AllocateRegisteredMemory(mInputsHost->mResourceClusterNativeBuffer);
957 }
958 if (mWaitForFinalInputs && GetProcessingSettings().nTPCClustererLanes > 6) {
959 GPUFatal("ERROR, mWaitForFinalInputs cannot be called with nTPCClustererLanes > 6");
960 }
961 if (buildNativeHost && !(buildNativeGPU && GetProcessingSettings().delayedOutput)) {
962 if (mWaitForFinalInputs) {
963 GPUFatal("Cannot use waitForFinalInput callback without delayed output");
964 }
965 if (!GetProcessingSettings().tpcApplyClusterFilterOnCPU) {
966 AllocateRegisteredMemory(mInputsHost->mResourceClusterNativeOutput, GetProcessingSettings().tpcWriteClustersAfterRejection ? nullptr : mSubOutputControls[GPUTrackingOutputs::getIndex(&GPUTrackingOutputs::clustersNative)]);
967 tmpNativeClusters = mInputsHost->mPclusterNativeOutput;
968 } else {
969 tmpNativeClusterBuffer = std::make_unique<ClusterNative[]>(mInputsHost->mNClusterNative);
970 tmpNativeClusters = tmpNativeClusterBuffer.get();
971 }
972 }
973
974 GPUTPCLinearLabels mcLinearLabels;
975 if (propagateMCLabels) {
976 // No need to overallocate here, nTPCHits is anyway an upper bound used for the GPU cluster buffer, and we can always enlarge the buffer anyway
977 mcLinearLabels.header.reserve(mRec->MemoryScalers()->nTPCHits / 2);
978 mcLinearLabels.data.reserve(mRec->MemoryScalers()->nTPCHits);
979 }
980
981 int8_t transferRunning[NSECTORS] = {0};
982 uint32_t outputQueueStart = mOutputQueue.size();
983
984 auto notifyForeignChainFinished = [this]() {
985 if (mPipelineNotifyCtx) {
986 SynchronizeStream(OutputStream()); // Must finish before updating ioPtrs in (global) constant memory
987 {
988 std::lock_guard<std::mutex> lock(mPipelineNotifyCtx->mutex);
989 mPipelineNotifyCtx->ready = true;
990 }
991 mPipelineNotifyCtx->cond.notify_one();
992 }
993 };
994 bool synchronizeCalibUpdate = false;
995
996 for (uint32_t iSectorBase = 0; iSectorBase < NSECTORS; iSectorBase += GetProcessingSettings().nTPCClustererLanes) {
997 std::vector<bool> laneHasData(GetProcessingSettings().nTPCClustererLanes, false);
998 static_assert(NSECTORS <= constants::GPU_MAX_STREAMS, "Stream events must be able to hold all sectors");
999 const int32_t maxLane = std::min<int32_t>(GetProcessingSettings().nTPCClustererLanes, NSECTORS - iSectorBase);
1000 for (CfFragment frag = mCFContext->fragmentFirst; !frag.isEnd(); frag = frag.next()) {
1001 if (GetProcessingSettings().debugLevel >= 3) {
1002 GPUInfo("Processing time bins [%d, %d) for sectors %d to %d", frag.start, frag.last(), iSectorBase, iSectorBase + GetProcessingSettings().nTPCClustererLanes - 1);
1003 }
1004 mRec->runParallelOuterLoop(doGPU, maxLane, [&](uint32_t lane) {
1005 if (doGPU && frag.index != 0) {
1006 SynchronizeStream(lane); // Don't overwrite charge map from previous iteration until cluster computation is finished
1007 }
1008
1009 uint32_t iSector = iSectorBase + lane;
1011 GPUTPCClusterFinder& clustererShadow = doGPU ? processorsShadow()->tpcClusterer[iSector] : clusterer;
1012 clusterer.mPmemory->counters.nPeaks = clusterer.mPmemory->counters.nClusters = 0;
1013 clusterer.mPmemory->frag = frag;
1014
1016 bool setDigitsOnGPU = doGPU && not mIOPtrs.tpcZS;
1017 bool setDigitsOnHost = (not doGPU && not mIOPtrs.tpcZS) || propagateMCLabels;
1018 auto* inDigits = mIOPtrs.tpcPackedDigits;
1019 size_t numDigits = inDigits->nTPCDigits[iSector];
1020 if (setDigitsOnGPU) {
1021 GPUMemCpy(RecoStep::TPCClusterFinding, clustererShadow.mPdigits, inDigits->tpcDigits[iSector], sizeof(clustererShadow.mPdigits[0]) * numDigits, lane, true);
1022 }
1023 if (setDigitsOnHost) {
1024 clusterer.mPdigits = const_cast<o2::tpc::Digit*>(inDigits->tpcDigits[iSector]); // TODO: Needs fixing (invalid const cast)
1025 }
1026 clusterer.mPmemory->counters.nDigits = numDigits;
1027 }
1028
1029 if (mIOPtrs.tpcZS) {
1030 if (mCFContext->nPagesSector[iSector] && mCFContext->zsVersion != -1) {
1031 clusterer.mPmemory->counters.nPositions = mCFContext->nextPos[iSector].first;
1032 clusterer.mPmemory->counters.nPagesSubsector = mCFContext->nextPos[iSector].second;
1033 } else {
1034 clusterer.mPmemory->counters.nPositions = clusterer.mPmemory->counters.nPagesSubsector = 0;
1035 }
1036 }
1037 TransferMemoryResourceLinkToGPU(RecoStep::TPCClusterFinding, clusterer.mMemoryId, lane);
1038
1039 using ChargeMapType = decltype(*clustererShadow.mPchargeMap);
1040 using PeakMapType = decltype(*clustererShadow.mPpeakMap);
1041 runKernel<GPUMemClean16>({GetGridAutoStep(lane, RecoStep::TPCClusterFinding)}, clustererShadow.mPchargeMap, TPCMapMemoryLayout<ChargeMapType>::items(GetProcessingSettings().overrideClusterizerFragmentLen) * sizeof(ChargeMapType));
1042 runKernel<GPUMemClean16>({GetGridAutoStep(lane, RecoStep::TPCClusterFinding)}, clustererShadow.mPpeakMap, TPCMapMemoryLayout<PeakMapType>::items(GetProcessingSettings().overrideClusterizerFragmentLen) * sizeof(PeakMapType));
1043 if (frag.index == 0) {
1044 runKernel<GPUMemClean16>({GetGridAutoStep(lane, RecoStep::TPCClusterFinding)}, clustererShadow.mPpadIsNoisy, TPC_CLUSTERER_STRIDED_PAD_COUNT * sizeof(*clustererShadow.mPpadIsNoisy));
1045 }
1047
1048 if (doGPU) {
1049 if (mIOPtrs.tpcZS && mCFContext->nPagesSector[iSector] && mCFContext->zsVersion != -1) {
1050 TransferMemoryResourceLinkToGPU(RecoStep::TPCClusterFinding, mInputsHost->mResourceZS, lane);
1051 SynchronizeStream(GetProcessingSettings().nTPCClustererLanes + lane);
1052 }
1053 SynchronizeStream(mRec->NStreams() - 1); // Wait for copying to constant memory
1054 }
1055
1056 if (mIOPtrs.tpcZS && (mCFContext->abandonTimeframe || !mCFContext->nPagesSector[iSector] || mCFContext->zsVersion == -1)) {
1057 clusterer.mPmemory->counters.nPositions = 0;
1058 return;
1059 }
1061 clusterer.mPmemory->counters.nPositions = 0;
1062 return;
1063 }
1064
1065 if (propagateMCLabels) {
1066 if (frag.index == 0) {
1067 // Must be only called on the first fragment as some buffers are used across the whole timeframe
1068 clusterer.AllocMCBuffers();
1069 }
1070 clusterer.InitMCBuffersForFragment();
1071 clusterer.mPinputLabels = digitsMC->v[iSector];
1072 if (clusterer.mPinputLabels == nullptr) {
1073 GPUFatal("MC label container missing, sector %d", iSector);
1074 }
1076 GPUFatal("MC label container has incorrect number of entries: %d expected, has %d\n", (int32_t)mIOPtrs.tpcPackedDigits->nTPCDigits[iSector], (int32_t)clusterer.mPinputLabels->getIndexedSize());
1077 }
1078 }
1079
1080 if (GetProcessingSettings().tpcSingleSector == -1 || GetProcessingSettings().tpcSingleSector == (int32_t)iSector) {
1081 if (not mIOPtrs.tpcZS) {
1082 runKernel<GPUTPCCFChargeMapFiller, GPUTPCCFChargeMapFiller::findFragmentStart>({GetGrid(1, lane), {iSector}}, mIOPtrs.tpcZS == nullptr);
1083 TransferMemoryResourceLinkToHost(RecoStep::TPCClusterFinding, clusterer.mMemoryId, lane);
1084 } else if (propagateMCLabels) {
1085 runKernel<GPUTPCCFChargeMapFiller, GPUTPCCFChargeMapFiller::findFragmentStart>({GetGrid(1, lane, GPUReconstruction::krnlDeviceType::CPU), {iSector}}, mIOPtrs.tpcZS == nullptr);
1086 TransferMemoryResourceLinkToGPU(RecoStep::TPCClusterFinding, clusterer.mMemoryId, lane);
1087 }
1088 }
1089
1090 if (mIOPtrs.tpcZS) {
1091 int32_t firstHBF = (mIOPtrs.settingsTF && mIOPtrs.settingsTF->hasTfStartOrbit) ? mIOPtrs.settingsTF->tfStartOrbit : ((mIOPtrs.tpcZS->sector[iSector].count[0] && mIOPtrs.tpcZS->sector[iSector].nZSPtr[0][0]) ? o2::raw::RDHUtils::getHeartBeatOrbit(*(const o2::header::RAWDataHeader*)mIOPtrs.tpcZS->sector[iSector].zsPtr[0][0]) : 0);
1092 uint32_t nBlocks = doGPU ? clusterer.mPmemory->counters.nPagesSubsector : GPUTrackingInOutZS::NENDPOINTS;
1093
1094 switch (mCFContext->zsVersion) {
1095 default:
1096 GPUFatal("Data with invalid TPC ZS mode (%d) received", mCFContext->zsVersion);
1097 break;
1100 runKernel<GPUTPCCFDecodeZS>({GetGridBlk(nBlocks, lane), {iSector}}, firstHBF, tpcTimeBinCut);
1101 break;
1103 runKernel<GPUTPCCFDecodeZSLink>({GetGridBlk(nBlocks, lane), {iSector}}, firstHBF, tpcTimeBinCut);
1104 break;
1106 runKernel<GPUTPCCFDecodeZSDenseLink>({GetGridBlk(nBlocks, lane), {iSector}}, firstHBF, tpcTimeBinCut);
1107 break;
1108 }
1109 TransferMemoryResourceLinkToHost(RecoStep::TPCClusterFinding, clusterer.mMemoryId, lane);
1110 } // clang-format off
1111 });
1112 mRec->runParallelOuterLoop(doGPU, maxLane, [&](uint32_t lane) {
1113 uint32_t iSector = iSectorBase + lane;
1114 if (doGPU) {
1115 SynchronizeStream(lane);
1116 }
1117 if (mIOPtrs.tpcZS) {
1118 CfFragment f = frag.next();
1119 int32_t nextSector = iSector;
1120 if (f.isEnd()) {
1121 nextSector += GetProcessingSettings().nTPCClustererLanes;
1122 f = mCFContext->fragmentFirst;
1123 }
1124 if (nextSector < NSECTORS && mIOPtrs.tpcZS && mCFContext->nPagesSector[nextSector] && mCFContext->zsVersion != -1 && !mCFContext->abandonTimeframe) {
1125 mCFContext->nextPos[nextSector] = RunTPCClusterizer_transferZS(nextSector, f, GetProcessingSettings().nTPCClustererLanes + lane, extraADCs);
1126 }
1127 }
1129 GPUTPCClusterFinder& clustererShadow = doGPU ? processorsShadow()->tpcClusterer[iSector] : clusterer;
1130 if (clusterer.mPmemory->counters.nPositions == 0) {
1131 return;
1132 }
1133 if (!mIOPtrs.tpcZS) {
1134 runKernel<GPUTPCCFChargeMapFiller, GPUTPCCFChargeMapFiller::fillFromDigits>({GetGrid(clusterer.mPmemory->counters.nPositions, lane), {iSector}});
1135 }
1136
1137 TPCClusterizerTransferExtraADC(clusterer, clustererShadow, lane, extraADCs);
1138
1139 if (propagateMCLabels) {
1140 runKernel<GPUTPCCFChargeMapFiller, GPUTPCCFChargeMapFiller::fillIndexMap>({GetGrid(clusterer.mPmemory->counters.nDigitsInFragment, lane, GPUReconstruction::krnlDeviceType::CPU), {iSector}});
1141 }
1142
1143 bool checkForNoisyPads = (rec()->GetParam().rec.tpc.maxTimeBinAboveThresholdIn1000Bin > 0) || (rec()->GetParam().rec.tpc.maxConsecTimeBinAboveThreshold > 0);
1144 checkForNoisyPads &= (rec()->GetParam().rec.tpc.noisyPadsQuickCheck ? frag.index == 0 : true);
1145 checkForNoisyPads &= !GetProcessingSettings().disableTPCNoisyPadFilter;
1146 // TODO Move hipTailFilter flag to ProcessingSettings?
1147 // TODO Add some warning when re enabling pad filter with this flag, so it's not just silently enabled when disabling was requested
1148 checkForNoisyPads |= rec()->GetParam().rec.tpc.hipTailFilter;
1149
1150 if (checkForNoisyPads) {
1151 if (rec()->GetParam().rec.tpc.hipTailFilter) {
1152 runKernel<GPUMemClean16>({GetGridAutoStep(lane, RecoStep::TPCClusterFinding)}, clustererShadow.mPhipTailsByRow, GPUTPCGeometry::NROWS * sizeof(*clustererShadow.mPhipTailsByRow) * GPUTPCCFHIPClusterizer::MaxHIPTailsPerRow);
1153 runKernel<GPUMemClean16>({GetGridAutoStep(lane, RecoStep::TPCClusterFinding)}, clustererShadow.mPnHIPTails, GPUTPCGeometry::NROWS * sizeof(*clustererShadow.mPnHIPTails));
1154 }
1155 const int32_t nBlocks = GPUTPCGeometry::NROWS;
1156
1157 runKernel<GPUTPCCFCheckPadBaseline>({GetGridBlk(nBlocks, lane), {iSector}});
1158 getKernelTimer<GPUTPCCFCheckPadBaseline>(RecoStep::TPCClusterFinding, iSector, TPC_REAL_PADS_IN_SECTOR * frag.lengthWithoutOverlap() * sizeof(PackedCharge), false);
1159 }
1160
1162 // Avoid additional sync when also dumping digits
1163 const bool debugSyncChargeMap = !(GetProcessingSettings().debugMask & GPUChainTrackingDebugFlags::TPCClustererDigits);
1164 // DumpChargeMap should run after noisy pad filter to avoid yet another dump of intermediate data. When chargemap without pad filter is required, disable pad filter instead.
1165 DoDebugAndDump(RecoStep::TPCClusterFinding, GPUChainTrackingDebugFlags::TPCClustererChargeMap, debugSyncChargeMap, clusterer, &GPUTPCClusterFinder::DumpChargeMap, *mDebugFile, "Charges");
1166
1167 TPCClusterizerCheckExtraADCZeros(clusterer, clustererShadow, lane, extraADCs);
1168
1169 runKernel<GPUTPCCFPeakFinder>({GetGrid(clusterer.mPmemory->counters.nPositions, lane), {iSector}});
1171 clusterer.DumpPeakMap(*mDebugFile, "Peaks");
1172 }
1173
1174 RunTPCClusterizer_compactPeaks(clusterer, clustererShadow, 0, doGPU, lane);
1175 TransferMemoryResourceLinkToHost(RecoStep::TPCClusterFinding, clusterer.mMemoryId, lane);
1176 DoDebugAndDump(RecoStep::TPCClusterFinding, GPUChainTrackingDebugFlags::TPCClustererPeaks, clusterer, &GPUTPCClusterFinder::DumpPeaksCompacted, *mDebugFile); // clang-format off
1177 });
1178 mRec->runParallelOuterLoop(doGPU, maxLane, [&](uint32_t lane) {
1179 uint32_t iSector = iSectorBase + lane;
1181 GPUTPCClusterFinder& clustererShadow = doGPU ? processorsShadow()->tpcClusterer[iSector] : clusterer;
1182 if (doGPU) {
1183 SynchronizeStream(lane);
1184 }
1185 if (clusterer.mPmemory->counters.nPeaks == 0) {
1186 return;
1187 }
1188 runKernel<GPUTPCCFNoiseSuppression, GPUTPCCFNoiseSuppression::noiseSuppression>({GetGrid(clusterer.mPmemory->counters.nPeaks, lane), {iSector}});
1189 runKernel<GPUTPCCFNoiseSuppression, GPUTPCCFNoiseSuppression::updatePeaks>({GetGrid(clusterer.mPmemory->counters.nPeaks, lane), {iSector}});
1191 clusterer.DumpPeakMap(*mDebugFile, "Suppressed Peaks");
1192 }
1193
1194 RunTPCClusterizer_compactPeaks(clusterer, clustererShadow, 1, doGPU, lane);
1195 TransferMemoryResourceLinkToHost(RecoStep::TPCClusterFinding, clusterer.mMemoryId, lane);
1197 });
1198 mRec->runParallelOuterLoop(doGPU, maxLane, [&](uint32_t lane) {
1199 uint32_t iSector = iSectorBase + lane;
1201 GPUTPCClusterFinder& clustererShadow = doGPU ? processorsShadow()->tpcClusterer[iSector] : clusterer;
1202
1203 const bool resetClusterCounters = frag.index == 0;
1204 // The reset must also run for an empty first fragment since later fragments can contain data.
1205 if (clusterer.mPmemory->counters.nPositions == 0 && !resetClusterCounters) {
1206 return;
1207 }
1208
1209 if (doGPU) {
1210 SynchronizeStream(lane);
1211 }
1212
1213 if (resetClusterCounters) {
1214 deviceEvent* waitEvent = nullptr;
1215 if (transferRunning[lane] == 1) {
1216 waitEvent = &mEvents->stream[lane];
1217 transferRunning[lane] = 2;
1218 }
1219 runKernel<GPUMemClean16>({GetGridAutoStep(lane, RecoStep::TPCClusterFinding), krnlRunRangeNone, {nullptr, waitEvent}}, clustererShadow.mPclusterInRow, GPUTPCGeometry::NROWS * sizeof(*clustererShadow.mPclusterInRow));
1220 }
1221
1222 if (clusterer.mPmemory->counters.nPositions == 0) {
1223 return;
1224 }
1225
1226 const auto nRegularClusters = clusterer.mPmemory->counters.nClusters;
1227 if (nRegularClusters != 0) {
1228 if (GetProcessingSettings().nn.applyNNclusterizer) {
1229#ifdef GPUCA_HAS_ONNX
1230 GPUTPCNNClusterizer& clustererNN = processors()->tpcNNClusterer[lane];
1231 GPUTPCNNClusterizer& clustererNNShadow = doGPU ? processorsShadow()->tpcNNClusterer[lane] : clustererNN;
1232 GPUTPCNNClusterizerHost& nnApplication = nnApplications[lane];
1233
1234 // int withMC = (doGPU && propagateMCLabels);
1235
1236 if (nn_settings.nnClusterizerApplyCfDeconvolution) {
1237 runKernel<GPUTPCCFDeconvolution>({GetGrid(clusterer.mPmemory->counters.nPositions, lane), {iSector}}, true);
1238 } else if (clustererNNShadow.mNnClusterizerSetDeconvolutionFlags) {
1239 runKernel<GPUTPCCFDeconvolution>({GetGrid(clusterer.mPmemory->counters.nPositions, lane), {iSector}}, false);
1240 }
1241
1242 // float time_clusterizer = 0, time_fill = 0, time_networks = 0;
1243 if (nn_settings.nnClusterizerVerbosity > 2) {
1244 LOG(info) << "(NNCLUS, GPUChainTrackingClusterizer, this=" << this << ") Starting loop over batched data. clustererNNShadow.mNnClusterizerBatchedMode=" << clustererNNShadow.mNnClusterizerBatchedMode << ", numLoops=" << std::ceil((float)clusterer.mPmemory->counters.nClusters / clustererNNShadow.mNnClusterizerBatchedMode) << ", numClusters=" << clusterer.mPmemory->counters.nClusters << ". (clustererNN=" << &clustererNN << ", clustererNNShadow=" << &clustererNNShadow << ")";
1245 }
1246 for (int batch = 0; batch < std::ceil((float)clusterer.mPmemory->counters.nClusters / clustererNNShadow.mNnClusterizerBatchedMode); batch++) {
1247 if (nn_settings.nnClusterizerVerbosity > 3) {
1248 LOG(info) << "(NNCLUS, GPUChainTrackingClusterizer, this=" << this << ") Start. Loop=" << batch << ". (clustererNN=" << &clustererNN << ", clustererNNShadow=" << &clustererNNShadow << ")";
1249 }
1250 uint batchStart = batch * clustererNNShadow.mNnClusterizerBatchedMode;
1251 size_t iSize = CAMath::Min((uint)clustererNNShadow.mNnClusterizerBatchedMode, (uint)(clusterer.mPmemory->counters.nClusters - batchStart));
1252
1253 // Filling the data
1254 if (mRec->IsGPU() || GetProcessingSettings().nn.nnClusterizerForceGpuInputFill) {
1255 // Fills element by element of each input matrix -> better parallelizability, but worse on CPU due to unnecessary computations
1256 runKernel<GPUTPCNNClusterizerKernels, GPUTPCNNClusterizerKernels::fillInputNNGPU>({GetGrid(iSize * clustererNNShadow.mNnClusterizerRowTimeSizeThreads , lane), krnlRunRangeNone}, iSector, clustererNNShadow.mNnInferenceInputDType, propagateMCLabels, batchStart);
1257 } else {
1258 // Fills the whole input matrix at once -> better performance on CPU, but worse parallelizability
1259 runKernel<GPUTPCNNClusterizerKernels, GPUTPCNNClusterizerKernels::fillInputNNCPU>({GetGrid(iSize, lane), krnlRunRangeNone}, iSector, clustererNNShadow.mNnInferenceInputDType, propagateMCLabels, batchStart);
1260 }
1261 if (nn_settings.nnClusterizerVerbosity > 3) {
1262 LOG(info) << "(NNCLUS, GPUChainTrackingClusterizer, this=" << this << ") Done filling data. Loop=" << batch << ". (clustererNN=" << &clustererNN << ", clustererNNShadow=" << &clustererNNShadow << ")";
1263 }
1264
1265 if (clustererNNShadow.mNnClusterizerSetDeconvolutionFlags) {
1266 runKernel<GPUTPCNNClusterizerKernels, GPUTPCNNClusterizerKernels::publishDeconvolutionFlags>({GetGrid(iSize, lane), krnlRunRangeNone}, iSector, clustererNNShadow.mNnInferenceInputDType, propagateMCLabels, batchStart); // Publishing the deconvolution flags
1267 if (nn_settings.nnClusterizerVerbosity > 3) {
1268 LOG(info) << "(NNCLUS, GPUChainTrackingClusterizer, this=" << this << ") Done setting deconvolution flags. Loop=" << batch << ". (clustererNN=" << &clustererNN << ", clustererNNShadow=" << &clustererNNShadow << ")";
1269 }
1270 }
1271
1272 // NN evaluations
1273 if(clustererNNShadow.mNnClusterizerUseClassification) {
1274 if(GetProcessingSettings().debugLevel >= 1 && (doGPU || lane < 4)) { nnTimers[3*lane]->Start(); }
1275 if (clustererNNShadow.mNnInferenceInputDType == 0) {
1276 if (clustererNNShadow.mNnInferenceOutputDType == 0) {
1277 (nnApplication.mModelClass).inference(clustererNNShadow.mInputData_32, iSize, clustererNNShadow.mModelProbabilities_32);
1278 } else {
1279 (nnApplication.mModelClass).inference(clustererNNShadow.mInputData_32, iSize, clustererNNShadow.mModelProbabilities_16);
1280 }
1281 } else if (clustererNNShadow.mNnInferenceInputDType == 1) {
1282 if (clustererNNShadow.mNnInferenceOutputDType == 0) {
1283 (nnApplication.mModelClass).inference(clustererNNShadow.mInputData_16, iSize, clustererNNShadow.mModelProbabilities_32);
1284 } else {
1285 (nnApplication.mModelClass).inference(clustererNNShadow.mInputData_16, iSize, clustererNNShadow.mModelProbabilities_16);
1286 }
1287 }
1288 if(GetProcessingSettings().debugLevel >= 1 && (doGPU || lane < 4)) { nnTimers[3*lane]->Stop(); } // doGPU || lane<4 -> only for GPU or first 4 CPU lanes (to limit number of concurrent timers). At least gives some statistics for CPU time...
1289 if (nn_settings.nnClusterizerVerbosity > 3) {
1290 LOG(info) << "(NNCLUS, GPUChainTrackingClusterizer, this=" << this << ") Done with NN classification inference. Loop=" << batch << ". (clustererNN=" << &clustererNN << ", clustererNNShadow=" << &clustererNNShadow << ")";
1291 }
1292 }
1293 if (!clustererNNShadow.mNnClusterizerUseCfRegression) {
1294 if(GetProcessingSettings().debugLevel >= 1 && (doGPU || lane < 4)) { nnTimers[3*lane + 1]->Start(); }
1295 if (clustererNNShadow.mNnInferenceInputDType == 0) {
1296 if (clustererNNShadow.mNnInferenceOutputDType == 0) {
1297 (nnApplication.mModelReg1).inference(clustererNNShadow.mInputData_32, iSize, clustererNNShadow.mOutputDataReg1_32);
1298 } else {
1299 (nnApplication.mModelReg1).inference(clustererNNShadow.mInputData_32, iSize, clustererNNShadow.mOutputDataReg1_16);
1300 }
1301 } else {
1302 if (clustererNNShadow.mNnInferenceOutputDType == 0) {
1303 (nnApplication.mModelReg1).inference(clustererNNShadow.mInputData_16, iSize, clustererNNShadow.mOutputDataReg1_32);
1304 } else {
1305 (nnApplication.mModelReg1).inference(clustererNNShadow.mInputData_16, iSize, clustererNNShadow.mOutputDataReg1_16);
1306 }
1307 }
1308 if(GetProcessingSettings().debugLevel >= 1 && (doGPU || lane < 4)) { nnTimers[3*lane + 1]->Stop(); }
1309 if (nnApplication.mModelClass.getNumOutputNodes()[0][1] > 1 && nnApplication.mModelReg2.isInitialized()) {
1310 if(GetProcessingSettings().debugLevel >= 1 && (doGPU || lane < 4)) { nnTimers[3*lane + 2]->Start(); }
1311 if (clustererNNShadow.mNnInferenceInputDType == 0) {
1312 if (clustererNNShadow.mNnInferenceOutputDType == 0) {
1313 (nnApplication.mModelReg2).inference(clustererNNShadow.mInputData_32, iSize, clustererNNShadow.mOutputDataReg2_32);
1314 } else {
1315 (nnApplication.mModelReg2).inference(clustererNNShadow.mInputData_32, iSize, clustererNNShadow.mOutputDataReg2_16);
1316 }
1317 } else if (clustererNNShadow.mNnInferenceInputDType == 1) {
1318 if (clustererNNShadow.mNnInferenceOutputDType == 0) {
1319 (nnApplication.mModelReg2).inference(clustererNNShadow.mInputData_16, iSize, clustererNNShadow.mOutputDataReg2_32);
1320 } else {
1321 (nnApplication.mModelReg2).inference(clustererNNShadow.mInputData_16, iSize, clustererNNShadow.mOutputDataReg2_16);
1322 }
1323 }
1324 if(GetProcessingSettings().debugLevel >= 1 && (doGPU || lane < 4)) { nnTimers[3*lane + 2]->Stop(); }
1325 }
1326 if (nn_settings.nnClusterizerVerbosity > 3) {
1327 LOG(info) << "(NNCLUS, GPUChainTrackingClusterizer, this=" << this << ") Done with NN regression inference. Loop=" << batch << ". (clustererNN=" << &clustererNN << ", clustererNNShadow=" << &clustererNNShadow << ")";
1328 }
1329 }
1330
1331 // Publishing kernels for class labels and regression results
1332 // In case classification should not be used, this kernel should still be executed to fill the mOutputDataClass array with default values
1333 if (nnApplication.mModelClass.getNumOutputNodes()[0][1] == 1) {
1334 runKernel<GPUTPCNNClusterizerKernels, GPUTPCNNClusterizerKernels::determineClass1Labels>({GetGrid(iSize, lane), krnlRunRangeNone}, iSector, clustererNNShadow.mNnInferenceOutputDType, propagateMCLabels, batchStart); // Assigning class labels
1335 } else {
1336 runKernel<GPUTPCNNClusterizerKernels, GPUTPCNNClusterizerKernels::determineClass2Labels>({GetGrid(iSize, lane), krnlRunRangeNone}, iSector, clustererNNShadow.mNnInferenceOutputDType, propagateMCLabels, batchStart); // Assigning class labels
1337 }
1338 if (!clustererNNShadow.mNnClusterizerUseCfRegression) {
1339 runKernel<GPUTPCNNClusterizerKernels, GPUTPCNNClusterizerKernels::publishClass1Regression>({GetGrid(iSize, lane), krnlRunRangeNone}, iSector, clustererNNShadow.mNnInferenceOutputDType, propagateMCLabels, batchStart); // Publishing class 1 regression results
1340 if (nnApplication.mModelClass.getNumOutputNodes()[0][1] > 1 && nnApplication.mModelReg2.isInitialized()) {
1341 runKernel<GPUTPCNNClusterizerKernels, GPUTPCNNClusterizerKernels::publishClass2Regression>({GetGrid(iSize, lane), krnlRunRangeNone}, iSector, clustererNNShadow.mNnInferenceOutputDType, propagateMCLabels, batchStart); // Publishing class 2 regression results
1342 }
1343 }
1344 if (nn_settings.nnClusterizerVerbosity > 3) {
1345 LOG(info) << "(NNCLUS, GPUChainTrackingClusterizer, this=" << this << ") Done publishing. Loop=" << batch << ". (clustererNN=" << &clustererNN << ", clustererNNShadow=" << &clustererNNShadow << ")";
1346 }
1347 }
1348
1349 if (clustererNNShadow.mNnClusterizerUseCfRegression) {
1350 if(!nn_settings.nnClusterizerApplyCfDeconvolution) { // If it is already applied don't do it twice, otherwise apply now
1351 runKernel<GPUTPCCFDeconvolution>({GetGrid(clusterer.mPmemory->counters.nPositions, lane), {iSector}}, true);
1352 }
1354 runKernel<GPUTPCNNClusterizerKernels, GPUTPCNNClusterizerKernels::runCfClusterizer>({GetGrid(clusterer.mPmemory->counters.nClusters, lane), krnlRunRangeNone}, iSector, clustererNNShadow.mNnInferenceInputDType, propagateMCLabels, 0); // Running the CF regression kernel - no batching needed: batchStart = 0
1355 if (nn_settings.nnClusterizerVerbosity > 3) {
1356 LOG(info) << "(NNCLUS, GPUChainTrackingClusterizer, this=" << this << ") Done with CF regression. (clustererNN=" << &clustererNN << ", clustererNNShadow=" << &clustererNNShadow << ")";
1357 }
1358 }
1359#else
1360 GPUFatal("Project not compiled with neural network clusterization. Aborting.");
1361#endif
1362 } else {
1363 runKernel<GPUTPCCFDeconvolution>({GetGrid(clusterer.mPmemory->counters.nPositions, lane), {iSector}}, true);
1365 runKernel<GPUTPCCFClusterizer>({GetGrid(clusterer.mPmemory->counters.nClusters, lane), {iSector}}, 0);
1366 } // if (GetProcessingSettings().nn.applyNNclusterizer)
1367
1368 if (doGPU && propagateMCLabels) {
1369 TransferMemoryResourceLinkToHost(RecoStep::TPCClusterFinding, clusterer.mScratchId, lane);
1370 TransferMemoryResourceLinkToHost(RecoStep::TPCClusterFinding, clusterer.mScratchClustersId, lane);
1371 SynchronizeStream(lane);
1372 runKernel<GPUTPCCFClusterizer>({GetGrid(clusterer.mPmemory->counters.nClusters, lane, GPUReconstruction::krnlDeviceType::CPU), {iSector}}, 1); // Computes MC labels
1373 }
1374 } // if (nRegularClusters != 0) {
1375
1376
1377 // TODO: Move this right after CheckPadBaseline once tail zeroing is moved into this kernel.
1378 if (rec()->GetParam().rec.tpc.hipTailFilter) {
1379 runKernel<GPUTPCCFHIPTailConnector>({GetGridBlk(GPUTPCGeometry::NROWS, lane), {iSector}});
1380 runKernel<GPUTPCCFHIPClusterizer>({GetGridBlk(GPUTPCGeometry::NROWS, lane), {iSector}}, 0);
1381 if (doGPU && (nRegularClusters == 0 || GetProcessingSettings().debugLevel >= 3)) {
1382 TransferMemoryResourceLinkToHost(RecoStep::TPCClusterFinding, clusterer.mMemoryId, lane);
1383 SynchronizeStream(lane);
1384 }
1385 if (doGPU && propagateMCLabels) {
1386 TransferMemoryResourceLinkToHost(RecoStep::TPCClusterFinding, clusterer.mScratchClustersId, lane);
1387 SynchronizeStream(lane);
1388 runKernel<GPUTPCCFHIPClusterizer>({GetGrid(GPUTPCGeometry::NROWS, lane, GPUReconstruction::krnlDeviceType::CPU), {iSector}}, 1); // Computes MC labels
1389 }
1390 }
1391
1392 bool hasClusters = nRegularClusters != 0;
1393
1394 // Paranoid edge case: If no regular clusters were found, need to still check that no HIP clusters were created
1395 // HIPClusterizer kernel doesn't update counters.nClusters, because:
1396 // - 64bit atomic support in OpenCL is flaky
1397 // - nClusters is only used internally by clusterizer to track #peaks that will probably become clusters,
1398 // so storing the number of HIP clusters there is only asking for trouble anyway
1399 if (rec()->GetParam().rec.tpc.hipTailFilter && nRegularClusters == 0) {
1400 for (uint32_t row = 0; row < GPUTPCGeometry::NROWS; row++) {
1401 hasClusters |= clusterer.mPclusterInRow[row] != 0;
1402 }
1403 }
1404
1405 if (!hasClusters) {
1406 return;
1407 }
1408
1409 if (GetProcessingSettings().debugLevel >= 3) {
1410 GPUInfo("Sector %02d Fragment %02d Lane %d: Found clusters: digits %u peaks %u clusters %u", iSector, frag.index, lane, (int32_t)clusterer.mPmemory->counters.nPositions, (int32_t)clusterer.mPmemory->counters.nPeaks, (int32_t)clusterer.mPmemory->counters.nClusters);
1411 }
1412
1413 TransferMemoryResourcesToHost(RecoStep::TPCClusterFinding, &clusterer, lane);
1414 laneHasData[lane] = true;
1415 });
1417 }
1418
1419 size_t nClsFirst = nClsTotal;
1420 bool anyLaneHasData = false;
1421 for (int32_t lane = 0; lane < maxLane; lane++) {
1422 uint32_t iSector = iSectorBase + lane;
1423 std::fill(&tmpNativeAccess->nClusters[iSector][0], &tmpNativeAccess->nClusters[iSector][0] + MAXGLOBALPADROW, 0);
1424 if (doGPU) {
1425 SynchronizeStream(lane);
1426 }
1428 GPUTPCClusterFinder& clustererShadow = doGPU ? processorsShadow()->tpcClusterer[iSector] : clusterer;
1429
1430 if (laneHasData[lane]) {
1431 anyLaneHasData = true;
1432 // Include clusters in default debug mask, exclude other debug output by default.
1433 // The cluster buffers are accumulated per sector, so dump them once after all fragments.
1434 DoDebugAndDump(RecoStep::TPCClusterFinding, GPUChainTrackingDebugFlags::TPCClustererClusters, clusterer, &GPUTPCClusterFinder::DumpClusters, *mDebugFile); // clang-format off
1435 if (buildNativeGPU && GetProcessingSettings().tpccfGatherKernel) {
1436 runKernel<GPUTPCCFGather>({GetGridBlk(GPUTPCGeometry::NROWS, mRec->NStreams() - 1), {iSector}}, &mInputsShadow->mPclusterNativeBuffer[nClsTotal]);
1437 }
1438 for (uint32_t j = 0; j < GPUTPCGeometry::NROWS; j++) {
1439 if (nClsTotal + clusterer.mPclusterInRow[j] > mInputsHost->mNClusterNative) {
1440 clusterer.raiseError(GPUErrors::ERROR_CF_GLOBAL_CLUSTER_OVERFLOW, iSector * 1000 + j, nClsTotal + clusterer.mPclusterInRow[j], mInputsHost->mNClusterNative);
1441 continue;
1442 }
1443 if (buildNativeGPU) {
1444 if (!GetProcessingSettings().tpccfGatherKernel) {
1445 GPUMemCpyAlways(RecoStep::TPCClusterFinding, (void*)&mInputsShadow->mPclusterNativeBuffer[nClsTotal], (const void*)&clustererShadow.mPclusterByRow[j * clusterer.mNMaxClusterPerRow], sizeof(mIOPtrs.clustersNative->clustersLinear[0]) * clusterer.mPclusterInRow[j], mRec->NStreams() - 1, -2);
1446 }
1447 } else if (buildNativeHost) {
1448 GPUMemCpyAlways(RecoStep::TPCClusterFinding, (void*)&tmpNativeClusters[nClsTotal], (const void*)&clustererShadow.mPclusterByRow[j * clusterer.mNMaxClusterPerRow], sizeof(mIOPtrs.clustersNative->clustersLinear[0]) * clusterer.mPclusterInRow[j], mRec->NStreams() - 1, false);
1449 }
1450 tmpNativeAccess->nClusters[iSector][j] += clusterer.mPclusterInRow[j];
1451 nClsTotal += clusterer.mPclusterInRow[j];
1452 }
1453 if (transferRunning[lane]) {
1454 ReleaseEvent(mEvents->stream[lane], doGPU);
1455 }
1456 RecordMarker(&mEvents->stream[lane], mRec->NStreams() - 1);
1457 transferRunning[lane] = 1;
1458 }
1459
1460 if (not propagateMCLabels || not laneHasData[lane]) {
1461 continue;
1462 }
1463
1464 runKernel<GPUTPCCFMCLabelFlattener, GPUTPCCFMCLabelFlattener::setRowOffsets>({GetGrid(GPUTPCGeometry::NROWS, lane, GPUReconstruction::krnlDeviceType::CPU), {iSector}});
1466 runKernel<GPUTPCCFMCLabelFlattener, GPUTPCCFMCLabelFlattener::flatten>({GetGrid(GPUTPCGeometry::NROWS, lane, GPUReconstruction::krnlDeviceType::CPU), {iSector}}, &mcLinearLabels);
1467 assert(propagateMCLabels ? mcLinearLabels.header.size() == nClsTotal : true);
1468 }
1469 for (int32_t lane = 0; lane < maxLane; lane++) {
1470 processors()->tpcClusterer[iSectorBase + lane].FreeMCBuffers();
1471 }
1472 if (buildNativeHost && buildNativeGPU && anyLaneHasData) {
1473 if (GetProcessingSettings().delayedOutput) {
1474 mOutputQueue.emplace_back(outputQueueEntry{(void*)((char*)&tmpNativeClusters[nClsFirst] - (char*)&tmpNativeClusters[0]), &mInputsShadow->mPclusterNativeBuffer[nClsFirst], (nClsTotal - nClsFirst) * sizeof(tmpNativeClusters[0]), RecoStep::TPCClusterFinding});
1475 } else {
1476 GPUMemCpy(RecoStep::TPCClusterFinding, (void*)&tmpNativeClusters[nClsFirst], (const void*)&mInputsShadow->mPclusterNativeBuffer[nClsFirst], (nClsTotal - nClsFirst) * sizeof(tmpNativeClusters[0]), mRec->NStreams() - 1, false);
1477 }
1478 }
1479
1480 if (mWaitForFinalInputs && iSectorBase >= 21 && (int32_t)iSectorBase < 21 + GetProcessingSettings().nTPCClustererLanes) {
1481 notifyForeignChainFinished();
1482 }
1483 if (mWaitForFinalInputs && iSectorBase >= 30 && (int32_t)iSectorBase < 30 + GetProcessingSettings().nTPCClustererLanes) {
1484 if (mWaitForFinalInputs()) {
1486 }
1487 synchronizeCalibUpdate = DoQueuedUpdates(0, false);
1488 }
1489 }
1490 for (int32_t i = 0; i < GetProcessingSettings().nTPCClustererLanes; i++) {
1491#ifdef GPUCA_HAS_ONNX
1492 if (GetProcessingSettings().nn.applyNNclusterizer) {
1493 if (GetProcessingSettings().nn.nnClusterizerVerbosity > 0) {
1494 LOG(info) << "(ORT) Environment releasing...";
1495 }
1496 GPUTPCNNClusterizerHost& nnApplication = nnApplications[i];
1497 nnApplication.mModelClass.release(true);
1498 nnApplication.mModelReg1.release(true);
1499 nnApplication.mModelReg2.release(true);
1500 }
1501#endif
1502 if (transferRunning[i]) {
1503 ReleaseEvent(mEvents->stream[i], doGPU);
1504 }
1505 }
1506
1507 if (GetProcessingSettings().param.tpcTriggerHandling) {
1509 if (triggerOutput && triggerOutput->allocator) {
1510 // GPUInfo("Storing %lu trigger words", mTriggerBuffer->triggers.size());
1511 auto* outputBuffer = (decltype(mTriggerBuffer->triggers)::value_type*)triggerOutput->allocator(mTriggerBuffer->triggers.size() * sizeof(decltype(mTriggerBuffer->triggers)::value_type));
1512 std::copy(mTriggerBuffer->triggers.begin(), mTriggerBuffer->triggers.end(), outputBuffer);
1513 }
1514 mTriggerBuffer->triggers.clear();
1515 }
1516
1517 // Number of clusters is logged by tracking. This ensures clusters are still printed if it's not running
1519 GPUInfo("Event has %zu TPC Clusters", nClsTotal);
1520 }
1521
1522 ClusterNativeAccess::ConstMCLabelContainerView* mcLabelsConstView = nullptr;
1523 if (propagateMCLabels) { // TODO: write to buffer directly
1525 std::pair<ConstMCLabelContainer*, ConstMCLabelContainerView*> buffer;
1527 if (!GetProcessingSettings().tpcWriteClustersAfterRejection && !sortClusters && labelOutputControl && labelOutputControl->useExternal()) {
1528 if (!labelOutputControl->allocator) {
1529 throw std::runtime_error("Cluster MC Label buffer missing");
1530 }
1532 buffer = {&container->first, &container->second};
1533 } else {
1534 mIOMem.clusterNativeMCView = std::make_unique<ConstMCLabelContainerView>();
1535 mIOMem.clusterNativeMCBuffer = std::make_unique<ConstMCLabelContainer>();
1537 }
1538
1539 assert(propagateMCLabels ? mcLinearLabels.header.size() == nClsTotal : true);
1540 assert(propagateMCLabels ? mcLinearLabels.data.size() >= nClsTotal : true);
1541
1542 mcLabels.setFrom(mcLinearLabels.header, mcLinearLabels.data);
1543 mcLabels.flatten_to(*buffer.first);
1544 *buffer.second = *buffer.first;
1545 mcLabelsConstView = buffer.second;
1546 }
1547
1548 if (buildNativeHost && buildNativeGPU && GetProcessingSettings().delayedOutput) {
1549 mInputsHost->mNClusterNative = mInputsShadow->mNClusterNative = nClsTotal;
1550 AllocateRegisteredMemory(mInputsHost->mResourceClusterNativeOutput, GetProcessingSettings().tpcWriteClustersAfterRejection ? nullptr : mSubOutputControls[GPUTrackingOutputs::getIndex(&GPUTrackingOutputs::clustersNative)]);
1551 tmpNativeClusters = mInputsHost->mPclusterNativeOutput;
1552 for (uint32_t i = outputQueueStart; i < mOutputQueue.size(); i++) {
1553 mOutputQueue[i].dst = (char*)tmpNativeClusters + (size_t)mOutputQueue[i].dst;
1554 }
1555 }
1556
1557 if (buildNativeHost) {
1558 tmpNativeAccess->clustersLinear = tmpNativeClusters;
1559 tmpNativeAccess->clustersMCTruth = mcLabelsConstView;
1560 tmpNativeAccess->setOffsetPtrs();
1561 mIOPtrs.clustersNative = tmpNativeAccess;
1562 if (GetProcessingSettings().tpcApplyClusterFilterOnCPU) {
1563 auto allocator = [this, &tmpNativeClusters](size_t size) {
1564 this->mInputsHost->mNClusterNative = size;
1565 this->AllocateRegisteredMemory(this->mInputsHost->mResourceClusterNativeOutput, this->GetProcessingSettings().tpcWriteClustersAfterRejection ? nullptr : this->mSubOutputControls[GPUTrackingOutputs::getIndex(&GPUTrackingOutputs::clustersNative)]);
1566 return (tmpNativeClusters = this->mInputsHost->mPclusterNativeOutput);
1567 };
1568 RunTPCClusterFilter(tmpNativeAccess, allocator, false);
1569 nClsTotal = tmpNativeAccess->nClustersTotal;
1570 }
1571 }
1572
1573 if (!mWaitForFinalInputs) {
1574 notifyForeignChainFinished();
1575 }
1576
1577 if (buildNativeGPU) {
1578 processorsShadow()->ioPtrs.clustersNative = mInputsShadow->mPclusterNativeAccess;
1579 WriteToConstantMemory(RecoStep::TPCClusterFinding, (char*)&processors()->ioPtrs - (char*)processors(), &processorsShadow()->ioPtrs, sizeof(processorsShadow()->ioPtrs), 0);
1580 *mInputsHost->mPclusterNativeAccess = *mIOPtrs.clustersNative;
1581 mInputsHost->mPclusterNativeAccess->clustersLinear = mInputsShadow->mPclusterNativeBuffer;
1582 mInputsHost->mPclusterNativeAccess->setOffsetPtrs();
1583 TransferMemoryResourceLinkToGPU(RecoStep::TPCClusterFinding, mInputsHost->mResourceClusterNativeAccess, 0);
1584 }
1585 if (doGPU && synchronizeOutput) {
1587 }
1588 if (doGPU && synchronizeCalibUpdate) {
1590 }
1591 if (sortClusters) {
1592 SortClusters(buildNativeGPU, propagateMCLabels, tmpNativeAccess, tmpNativeClusters);
1593 }
1594 mRec->MemoryScalers()->nTPCHits = nClsTotal;
1595 mRec->PopNonPersistentMemory(RecoStep::TPCClusterFinding, qStr2Tag("TPCCLUST"));
1596 if (mPipelineNotifyCtx) {
1598 mPipelineNotifyCtx = nullptr;
1599 }
1600
1601 if (GetProcessingSettings().autoAdjustHostThreads && !doGPU) {
1603 }
1604
1605#endif
1606 return 0;
1607}
1608
1609void GPUChainTracking::SortClusters(bool buildNativeGPU, bool propagateMCLabels, ClusterNativeAccess* clusterAccess, ClusterNative* clusters)
1610{
1611 if (propagateMCLabels) {
1612 std::vector<uint32_t> clsOrder(clusterAccess->nClustersTotal);
1613 std::iota(clsOrder.begin(), clsOrder.end(), 0);
1614 std::vector<ClusterNative> tmpClusters;
1615 for (uint32_t i = 0; i < NSECTORS; i++) {
1616 for (uint32_t j = 0; j < GPUTPCGeometry::NROWS; j++) {
1617 const uint32_t offset = clusterAccess->clusterOffset[i][j];
1618 std::sort(&clsOrder[offset], &clsOrder[offset + clusterAccess->nClusters[i][j]], [&clusters](const uint32_t a, const uint32_t b) {
1619 return clusters[a] < clusters[b];
1620 });
1621 tmpClusters.resize(clusterAccess->nClusters[i][j]);
1622 memcpy(tmpClusters.data(), &clusters[offset], clusterAccess->nClusters[i][j] * sizeof(tmpClusters[0]));
1623 for (uint32_t k = 0; k < tmpClusters.size(); k++) {
1624 clusters[offset + k] = tmpClusters[clsOrder[offset + k] - offset];
1625 }
1626 }
1627 }
1628 tmpClusters.clear();
1629
1630 std::pair<o2::dataformats::ConstMCLabelContainer*, o2::dataformats::ConstMCLabelContainerView*> labelBuffer;
1632 std::unique_ptr<ConstMCLabelContainerView> tmpUniqueContainerView;
1633 std::unique_ptr<ConstMCLabelContainer> tmpUniqueContainerBuffer;
1634 if (labelOutput && labelOutput->allocator) {
1636 labelBuffer = {&labelContainer->first, &labelContainer->second};
1637 } else {
1638 tmpUniqueContainerView = std::move(mIOMem.clusterNativeMCView);
1639 tmpUniqueContainerBuffer = std::move(mIOMem.clusterNativeMCBuffer);
1640 mIOMem.clusterNativeMCView = std::make_unique<ConstMCLabelContainerView>();
1641 mIOMem.clusterNativeMCBuffer = std::make_unique<ConstMCLabelContainer>();
1642 labelBuffer = {mIOMem.clusterNativeMCBuffer.get(), mIOMem.clusterNativeMCView.get()};
1643 }
1644
1646 for (uint32_t i = 0; i < clusterAccess->nClustersTotal; i++) {
1647 for (const auto& element : clusterAccess->clustersMCTruth->getLabels(clsOrder[i])) {
1648 tmpContainer.addElement(i, element);
1649 }
1650 }
1651 tmpContainer.flatten_to(*labelBuffer.first);
1652 *labelBuffer.second = *labelBuffer.first;
1653 clusterAccess->clustersMCTruth = labelBuffer.second;
1654 } else {
1655 for (uint32_t i = 0; i < NSECTORS; i++) {
1656 for (uint32_t j = 0; j < GPUTPCGeometry::NROWS; j++) {
1657 std::sort(&clusters[clusterAccess->clusterOffset[i][j]], &clusters[clusterAccess->clusterOffset[i][j] + clusterAccess->nClusters[i][j]]);
1658 }
1659 }
1660 }
1661 if (buildNativeGPU) {
1662 GPUMemCpy(RecoStep::TPCClusterFinding, (void*)mInputsShadow->mPclusterNativeBuffer, (const void*)clusters, clusterAccess->nClustersTotal * sizeof(clusters[0]), -1, true);
1663 }
1664}
Definition of the TPC Digit.
default_random_engine gen(dev())
atype::type element
unsigned short int uint16_t
unsigned int uint32_t
std::vector< std::string > header
int32_t i
TPCZSHDR * hdr
zsPage * page
uint32_t iSector
uint32_t pageCounter
o2::raw::RawFileWriter * raw
int32_t retVal
configStandalone Light OpenGL display display_light Apply smoothing to points Enable Z buffer Highlight links Highlight seeds Highlight sector tracks Highlight final tracks Show individual sector
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error First event to Number of events to Number of iterations to Number of iterations to Number of initial iterations excluded from const char Directory with events to Events directory is and not inside events d
configStandalone Reconstruction rec_tpc multiplier for the combined cluster track error during track following area in cm for the search of for z only used if decrease removal by when above this local occupancy rowx Shift of TPC revert cluster flag edge bit distance to edge exceeds this ZOffset for secondary multiple of radius max P of electron K BB bands crossing max P of electron p BB bands crossing max P of electron d BB bands crossing max P of electron t BB bands crossing min accepted dEdxMaxTPC of the track Additive extra cluster error for Y2 if edge flag set Additive extra cluster error for Y2 if single set Additive extra cluster error for Y2 if or single set Additive extra cluster error for Z2 if or single set Extra errors to add to track when crossing depending on addErrorsCECrossing Inner portion of y range in sector that is not used in searching for extrapolated track candidates factor2YS for track merging factor2K for track merging Margin in cm of Y position when rejecting edge clusters based on uncorrected track Y Max fraction of shared clusters for tracklet Except pad from cluster finding if total number of charges in a fragment is above this Except pad from cluster finding if number of consecutive charges in a fragment is above this Threshold where a timebin is considered disabling the noisy pad check for that pad Treat signal above this minimum as saturated Smoothing factor for the exponential Highly Ionising Particle tail filter Number of timebins to use for the averaging Add additional custom track errors when crossing relative as fraction of Do N fit passes in final fit of< 0:reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking(prolong tracks to adjacent sectors to find short segments)") AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit(1:attachment, 2:propagation, 4:loop following, 8:mirroring)") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression(0=no, 1=strategy A, 2=strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits(1=truncate charge/width LSB, 2=differences, 4=track-model)") AddOptionRTC(compressionSortOrder, uint8_t, o2::gpu::GPUSettings::SortTime, "", 0, "Sort order of TPC compression(0=time, 1=pad, 2=Z-time-pad, 3=Z-pad-time, 4=no sorting(use incoming order))") AddOptionRTC(sigBitsCharge, uint8_t, 4, "", 0, "Number of significant bits for TPC cluster charge in compression mode 1") AddOptionRTC(sigBitsWidth, uint8_t, 3, "", 0, "Number of significant bits for TPC cluster width in compression mode 1") AddOptionRTC(dropLoopers, uint8_t, 0, "", 0, "Drop looping tracks starting from second loop") AddOptionRTC(mergerInterpolateErrors, uint8_t, 1, "", 0, "Use interpolation instead of extrapolation for chi2 based cluster rejection") AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF Merge tracks accross the central electrode Enable PID response Add sub threshold clusters in TPC dEdx computation OR mask of TPC flags that will reject the cluster in alternative dEdx Reject edge clusters based on uncorrected track Y during track fit Increase decrease removal by when on this number of pads from the edge h configStandalone Reconstruction rec_trd Max chi2 for TRD tracklets to be matched to a track Chi2 cut for strict matching mode Number of sigmas for ITS TPC track time error estimate Addition to search road around track prolongation along Y in cm Optimal tracklet rphi resolution in maximum chi2 per attached tracklet for TRD tracks take into account the full likelihood of the slope including exponential tails Tracks with less attached tracklets are discarded after the tracking Post trigger Pile up integration time in BCs fits all iterations but the last one in projections Use material lookup table for TPC refit Apply workarounds to avoid cyclic merge should not be needed tpc dyn configStandalone Processing proc_rtc Replace constant variables static by constexpr expressions Compile RTC in deterministic with NO_FAST_MATH flags and GPUCA_DETERMINISTIC_MODE define Use RTC to optimize GPU code h configStandalone Processing proc_rtctech Use a file lock to serialize access to the cache folder int32_t
configStandalone Reconstruction rec_tpc multiplier for the combined cluster track error during track following area in cm for the search of for z only used if decrease removal by when above this local occupancy rowx Shift of TPC revert cluster flag edge bit distance to edge exceeds this ZOffset for secondary multiple of radius max P of electron K BB bands crossing max P of electron p BB bands crossing max P of electron d BB bands crossing max P of electron t BB bands crossing min accepted dEdxMaxTPC of the track Additive extra cluster error for Y2 if edge flag set Additive extra cluster error for Y2 if single set Additive extra cluster error for Y2 if or single set Additive extra cluster error for Z2 if or single set Extra errors to add to track when crossing depending on addErrorsCECrossing Inner portion of y range in sector that is not used in searching for extrapolated track candidates factor2YS for track merging factor2K for track merging Margin in cm of Y position when rejecting edge clusters based on uncorrected track Y Max fraction of shared clusters for tracklet Except pad from cluster finding if total number of charges in a fragment is above this Except pad from cluster finding if number of consecutive charges in a fragment is above this Threshold where a timebin is considered disabling the noisy pad check for that pad Treat signal above this minimum as saturated Smoothing factor for the exponential Highly Ionising Particle tail filter Number of timebins to use for the averaging Add additional custom track errors when crossing relative as fraction of Do N fit passes in final fit of< 0:reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking(prolong tracks to adjacent sectors to find short segments)") AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit(1:attachment, 2:propagation, 4:loop following, 8:mirroring)") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression(0=no, 1=strategy A, 2=strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits(1=truncate charge/width LSB, 2=differences, 4=track-model)") AddOptionRTC(compressionSortOrder, uint8_t, o2::gpu::GPUSettings::SortTime, "", 0, "Sort order of TPC compression(0=time, 1=pad, 2=Z-time-pad, 3=Z-pad-time, 4=no sorting(use incoming order))") AddOptionRTC(sigBitsCharge, uint8_t, 4, "", 0, "Number of significant bits for TPC cluster charge in compression mode 1") AddOptionRTC(sigBitsWidth, uint8_t, 3, "", 0, "Number of significant bits for TPC cluster width in compression mode 1") AddOptionRTC(dropLoopers, uint8_t, 0, "", 0, "Drop looping tracks starting from second loop") AddOptionRTC(mergerInterpolateErrors, uint8_t, 1, "", 0, "Use interpolation instead of extrapolation for chi2 based cluster rejection") AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF Merge tracks accross the central electrode Enable PID response Add sub threshold clusters in TPC dEdx computation OR mask of TPC flags that will reject the cluster in alternative dEdx Reject edge clusters based on uncorrected track Y during track fit Increase decrease removal by when on this number of pads from the edge h configStandalone Reconstruction rec_trd Max chi2 for TRD tracklets to be matched to a track Chi2 cut for strict matching mode Number of sigmas for ITS TPC track time error estimate Addition to search road around track prolongation along Y in cm Optimal tracklet rphi resolution in maximum chi2 per attached tracklet for TRD tracks take into account the full likelihood of the slope including exponential tails Tracks with less attached tracklets are discarded after the tracking Post trigger Pile up integration time in BCs fits all iterations but the last one in projections Use material lookup table for TPC refit Apply workarounds to avoid cyclic merge should not be needed tpc dyn configStandalone Processing proc_rtc Replace constant variables static by constexpr expressions Compile RTC in deterministic with NO_FAST_MATH flags and GPUCA_DETERMINISTIC_MODE define Use RTC to optimize GPU code h configStandalone Processing proc_rtctech Use a file lock to serialize access to the cache folder Print launch bounds used for RTC code as debugging for exit after Allow RTC with slave GPUReconstruction instances with optConstexpr and optSpecialcode Maximum number of RTC cache files per backend in the cache the oldest are Prepend RTC compilation commands by this string Load a parameter object containing the launch bounds from a file h configStandalone Processing proc_param TPC Cluster Z Error Parameterization h nn
Class to serialize ONNX objects for ROOT snapshots of CCDB objects at runtime.
std::enable_if_t< std::is_signed< T >::value, bool > hasData(const CalArray< T > &cal)
Definition Painter.cxx:600
uint16_t pos
Definition RawData.h:3
uint32_t j
Definition RawData.h:0
uint8_t endpoint
Definition RawData.h:0
Provides a basic fallback implementation for Vc.
Definitions of TPC Zero Suppression Data Headers.
void Start()
Definition timer.cxx:64
void Stop()
Definition timer.cxx:76
A container to hold and manage MC truth information/labels.
void addElement(uint32_t dataindex, TruthElement const &element, bool noElement=false)
void setFrom(std::vector< MCTruthHeaderElement > &header, std::vector< TruthElement > &truthArray)
size_t flatten_to(ContainerType &container) const
std::unique_ptr< o2::tpc::ClusterNativeAccess > mClusterNativeAccess
int32_t RunTPCClusterizer(bool synchronizeOutput=true)
std::unique_ptr< GPUTrackingInputProvider > mInputsHost
std::unique_ptr< std::ostream > mDebugFile
std::array< GPUOutputControl *, GPUTrackingOutputs::count()> mSubOutputControls
std::unique_ptr< GPUTriggerOutputs > mTriggerBuffer
std::vector< outputQueueEntry > mOutputQueue
std::unique_ptr< GPUTPCCFChainContext > mCFContext
int32_t DoQueuedUpdates(int32_t stream, bool updateSlave=true)
std::unique_ptr< GPUNewCalibValues > mNewCalibValues
GPUTrackingInOutPointers & mIOPtrs
struct o2::gpu::GPUChainTracking::InOutMemory mIOMem
std::unique_ptr< GPUTrackingInputProvider > mInputsShadow
void RecordMarker(deviceEvent *ev, int32_t stream)
Definition GPUChain.h:108
void TransferMemoryResourceLinkToGPU(RecoStep step, int16_t res, int32_t stream=-1, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
Definition GPUChain.h:124
void GPUMemCpyAlways(RecoStep step, void *dst, const void *src, size_t size, int32_t stream, int32_t toGPU, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
Definition GPUChain.h:130
void GPUMemCpy(RecoStep step, void *dst, const void *src, size_t size, int32_t stream, int32_t toGPU, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
Definition GPUChain.h:129
bool DoDebugAndDump(RecoStep step, uint32_t mask, T &processor, S T::*func, Args &&... args)
Definition GPUChain.h:239
GPUReconstruction::RecoStepField GetRecoStepsGPU() const
Definition GPUChain.h:72
GPUReconstruction::RecoStepField GetRecoSteps() const
Definition GPUChain.h:71
void WriteToConstantMemory(RecoStep step, size_t offset, const void *src, size_t size, int32_t stream=-1, deviceEvent *ev=nullptr)
Definition GPUChain.h:128
void ReleaseEvent(deviceEvent ev, bool doGPU=true)
Definition GPUChain.h:111
krnlExec GetGrid(uint32_t totalItems, uint32_t nThreads, int32_t stream, GPUReconstruction::krnlDeviceType d=GPUReconstruction::krnlDeviceType::Auto, gpudatatypes::RecoStep st=gpudatatypes::RecoStep::NoRecoStep)
Definition GPUChain.cxx:21
size_t AllocateRegisteredMemory(GPUProcessor *proc)
Definition GPUChain.h:228
virtual std::unique_ptr< GPUReconstructionProcessing::threadContext > GetThreadContext()
Definition GPUChain.h:109
GPUConstantMem * processors()
Definition GPUChain.h:84
static constexpr krnlRunRange krnlRunRangeNone
Definition GPUChain.h:41
void SetONNXGPUStream(Ort::SessionOptions &opt, int32_t stream, int32_t *deviceId)
Definition GPUChain.h:90
krnlExec GetGridAutoStep(int32_t stream, gpudatatypes::RecoStep st=gpudatatypes::RecoStep::NoRecoStep)
Definition GPUChain.cxx:47
GPUParam & param()
Definition GPUChain.h:87
void SetupGPUProcessor(T *proc, bool allocate)
Definition GPUChain.h:231
const GPUSettingsProcessing & GetProcessingSettings() const
Definition GPUChain.h:76
void SynchronizeStream(int32_t stream)
Definition GPUChain.h:89
GPUReconstructionCPU * mRec
Definition GPUChain.h:79
GPUConstantMem * processorsShadow()
Definition GPUChain.h:85
krnlExec GetGridBlk(uint32_t nBlocks, int32_t stream, GPUReconstruction::krnlDeviceType d=GPUReconstruction::krnlDeviceType::Auto, gpudatatypes::RecoStep st=gpudatatypes::RecoStep::NoRecoStep)
Definition GPUChain.cxx:32
static constexpr int32_t NSECTORS
Definition GPUChain.h:58
const GPUParam & GetParam() const
Definition GPUChain.h:63
void TransferMemoryResourceLinkToHost(RecoStep step, int16_t res, int32_t stream=-1, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
Definition GPUChain.h:125
void TransferMemoryResourcesToHost(RecoStep step, GPUProcessor *proc, int32_t stream=-1, bool all=false)
Definition GPUChain.h:123
GPUReconstruction * rec()
Definition GPUChain.h:66
HighResTimer & getGeneralStepTimer(GeneralStep step)
void runParallelOuterLoop(bool doGPU, uint32_t nThreads, std::function< void(uint32_t)> lambda)
const GPUDefParameters & getGPUParameters(bool doGPU) const override
void AllocateRegisteredForeignMemory(int16_t res, GPUReconstruction *rec, GPUOutputControl *control=nullptr)
void ComputeReuseMax(GPUProcessor *proc)
RecoStepField GetRecoStepsGPU() const
void PopNonPersistentMemory(RecoStep step, uint64_t tag, const GPUProcessor *proc=nullptr)
const GPUParam & GetParam() const
void PushNonPersistentMemory(uint64_t tag)
InOutTypeField GetRecoStepsOutputs() const
GPUMemorySizeScalers * MemoryScalers()
static void setGlobalOffsetsAndAllocate(GPUTPCClusterFinder &, GPUTPCLinearLabels &)
void SetMaxData(const GPUTrackingInOutPointers &io)
void SetNMaxDigits(size_t nDigits, size_t nPages, size_t nDigitsFragment, size_t nDigitsEndpointMax)
void DumpSuppressedPeaks(std::ostream &out)
void DumpPeakMap(std::ostream &out, std::string_view)
o2::dataformats::ConstMCTruthContainerView< o2::MCCompLabel > const * mPinputLabels
void DumpChargeMap(std::ostream &out, std::string_view)
uint32_t getNSteps(size_t items) const
void DumpSuppressedPeaksCompacted(std::ostream &out)
void DumpPeaksCompacted(std::ostream &out)
tpc::ClusterNative * mPclusterByRow
void init(const GPUSettingsProcessingNNclusterizer &, bool=false)
void initClusterizer(const GPUSettingsProcessingNNclusterizer &, GPUTPCNNClusterizer &, int32_t=-1, int32_t=-1)
OrtDataType::Float16_t * mInputData_16
OrtDataType::Float16_t * mOutputDataReg2_16
OrtDataType::Float16_t * mModelProbabilities_16
OrtDataType::Float16_t * mOutputDataReg1_16
void release(bool=false)
void setIntraOpNumThreads(int threads)
std::vector< std::vector< int64_t > > getNumOutputNodes() const
#define TPC_REAL_PADS_IN_SECTOR
#define TPC_CLUSTERER_STRIDED_PAD_COUNT
GLint GLenum GLint x
Definition glcorearb.h:403
const GLfloat * m
Definition glcorearb.h:4066
GLenum src
Definition glcorearb.h:1767
GLint GLsizei count
Definition glcorearb.h:399
GLuint buffer
Definition glcorearb.h:655
GLsizeiptr size
Definition glcorearb.h:659
GLint first
Definition glcorearb.h:399
GLdouble f
Definition glcorearb.h:310
GLboolean GLboolean GLboolean b
Definition glcorearb.h:1233
GLintptr offset
Definition glcorearb.h:660
GLenum GLfloat param
Definition glcorearb.h:271
GLboolean GLboolean GLboolean GLboolean a
Definition glcorearb.h:1233
Definition of a container to keep/associate and arbitrary number of labels associated to an index wit...
RAWDataHeaderV7 RAWDataHeader
const float k2
Definition MathUtils.h:75
void dumpBuffer(gsl::span< const std::byte > buffer, std::ostream &out=std::cout, size_t maxbytes=std::numeric_limits< size_t >::max())
Definition DumpBuffer.h:139
std::unique_ptr< const o2::dataformats::MCTruthContainer< MCLabel > > getLabels(framework::ProcessingContext &pc, std::string_view dataBind, EventType eventType=EventType::Standard)
Global TPC definitions and constants.
@ ZSVersionDenseLinkBased
@ ZSVersionLinkBasedWithMeta
@ ZSVersionRowBased10BitADC
@ ZSVersionRowBased12BitADC
a couple of static helper functions to create timestamp values for CCDB queries or override obsolete ...
constexpr T qStr2Tag(const char(&str)[N])
Definition strtag.h:24
tpccf::TPCTime start
Definition CfFragment.h:31
S< o2::tpc::ORTRootSerializer >::type * nnClusterizerNetworks[3]
std::unique_ptr< o2::dataformats::ConstMCTruthContainerView< o2::MCCompLabel > > clusterNativeMCView
std::unique_ptr< o2::dataformats::ConstMCTruthContainer< o2::MCCompLabel > > clusterNativeMCBuffer
deviceEvent stream[constants::GPU_MAX_STREAMS]
GPUCalibObjectsConst calibObjects
GPUTPCClusterFinder tpcClusterer[GPUTPCGeometry::NSECTORS]
GPUTrackingInOutPointers ioPtrs
size_t NTPCClusters(size_t tpcDigits, bool perSector=false)
std::function< void *(size_t)> allocator
struct o2::gpu::GPUTPCClusterFinder::Memory::counters_t counters
std::array< std::vector< tpc::Digit >, tpc::constants::MAXSECTOR > digitsBySector
std::vector< o2::MCCompLabel > data
std::vector< o2::dataformats::MCTruthHeaderElement > header
const GPUTPCDigitsMCInput * tpcDigitsMC
const o2::tpc::ClusterNativeAccess * clustersNative
const GPUTrackingInOutZS * tpcZS
const GPUTrackingInOutDigits * tpcPackedDigits
GPUTrackingInOutZSSector sector[NSECTORS]
size_t getIndex(const GPUOutputControl &v)
static constexpr int getVersion()
get numeric version of the RDH
Definition RDHUtils.h:60
unsigned int nClusters[constants::MAXSECTOR][constants::MAXGLOBALPADROW]
const o2::dataformats::ConstMCTruthContainerView< o2::MCCompLabel > * clustersMCTruth
std::pair< ConstMCLabelContainer, ConstMCLabelContainerView > ConstMCLabelContainerViewWithBuffer
unsigned int clusterOffset[constants::MAXSECTOR][constants::MAXGLOBALPADROW]
const ClusterNative * clustersLinear
unsigned char nTimeBinSpan
unsigned char version
unsigned short timeOffset
unsigned short nADCsamples
Trigger info including the orbit.
uint32_t orbit
orbit of the trigger word
TriggerWordDLBZS triggerWord
trigger Word information
bool isValid(int entry=0) const
constexpr size_t min
constexpr size_t max
LOG(info)<< "Compressed in "<< sw.CpuTime()<< " s"
std::vector< Cluster > clusters
std::vector< Digit > digits
std::vector< int > row
ArrayADC adc
typename std::vector< T, vecpod_allocator< T > > vecpod
Definition vecpod.h:31