Project
Loading...
Searching...
No Matches
GPUChainTrackingClusterizer.cxx
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
14
15#include "GPUChainTracking.h"
18#include "GPULogging.h"
19#include "GPUO2DataTypes.h"
22#include "GPUNewCalibValues.h"
23#include "GPUConstantMem.h"
24#include "CfChargePos.h"
25#include "CfArray2D.h"
26#include "GPUGeneralKernels.h"
27#include "GPUDefParametersRuntime.h"
30#include "GPUTPCCFDecodeZS.h"
32#include "GPUTPCCFPeakFinder.h"
35#include "GPUTPCCFClusterizer.h"
36#include "GPUTPCCFGather.h"
38#include "GPUTriggerOutputs.h"
39#include "GPUHostDataTypes.h"
45#include "TPCBase/RDHUtils.h"
46
47#ifdef GPUCA_HAS_ONNX
50#endif
51
52#ifdef GPUCA_O2_LIB
54#endif
55
56#include "utils/strtag.h"
57#include <fstream>
58
59#ifndef GPUCA_NO_VC
60#include <Vc/Vc>
61#endif
62
63using namespace o2::gpu;
64using namespace o2::tpc;
65using namespace o2::tpc::constants;
66using namespace o2::dataformats;
67
68#ifdef GPUCA_TPC_GEOMETRY_O2
69std::pair<uint32_t, uint32_t> GPUChainTracking::TPCClusterizerDecodeZSCountUpdate(uint32_t iSector, const CfFragment& fragment)
70{
72 GPUTPCClusterFinder& clusterer = processors()->tpcClusterer[iSector];
74 uint32_t digits = 0;
75 uint32_t pages = 0;
76 for (uint16_t j = 0; j < GPUTrackingInOutZS::NENDPOINTS; j++) {
77 clusterer.mMinMaxCN[j] = mCFContext->fragmentData[fragment.index].minMaxCN[iSector][j];
78 if (doGPU) {
79 uint16_t posInEndpoint = 0;
80 uint16_t pagesEndpoint = 0;
81 for (uint32_t k = clusterer.mMinMaxCN[j].zsPtrFirst; k < clusterer.mMinMaxCN[j].zsPtrLast; k++) {
82 const uint32_t pageFirst = (k == clusterer.mMinMaxCN[j].zsPtrFirst) ? clusterer.mMinMaxCN[j].zsPageFirst : 0;
83 const uint32_t pageLast = (k + 1 == clusterer.mMinMaxCN[j].zsPtrLast) ? clusterer.mMinMaxCN[j].zsPageLast : mIOPtrs.tpcZS->sector[iSector].nZSPtr[j][k];
84 for (uint32_t l = pageFirst; l < pageLast; l++) {
85 uint16_t pageDigits = mCFContext->fragmentData[fragment.index].pageDigits[iSector][j][posInEndpoint++];
86 if (pageDigits) {
87 *(o++) = GPUTPCClusterFinder::ZSOffset{digits, j, pagesEndpoint};
88 digits += pageDigits;
89 }
90 pagesEndpoint++;
91 }
92 }
93 if (pagesEndpoint != mCFContext->fragmentData[fragment.index].pageDigits[iSector][j].size()) {
94 if (GetProcessingSettings().ignoreNonFatalGPUErrors) {
95 GPUError("TPC raw page count mismatch in TPCClusterizerDecodeZSCountUpdate: expected %d / buffered %lu", pagesEndpoint, mCFContext->fragmentData[fragment.index].pageDigits[iSector][j].size());
96 return {0, 0};
97 } else {
98 GPUFatal("TPC raw page count mismatch in TPCClusterizerDecodeZSCountUpdate: expected %d / buffered %lu", pagesEndpoint, mCFContext->fragmentData[fragment.index].pageDigits[iSector][j].size());
99 }
100 }
101 } else {
103 digits += mCFContext->fragmentData[fragment.index].nDigits[iSector][j];
104 pages += mCFContext->fragmentData[fragment.index].nPages[iSector][j];
105 }
106 }
107 if (doGPU) {
108 pages = o - processors()->tpcClusterer[iSector].mPzsOffsets;
109 }
110 if (GetProcessingSettings().clusterizerZSSanityCheck && mCFContext->zsVersion >= ZSVersion::ZSVersionDenseLinkBased) {
111 TPCClusterizerEnsureZSOffsets(iSector, fragment);
112 }
113 return {digits, pages};
114}
115
116void GPUChainTracking::TPCClusterizerEnsureZSOffsets(uint32_t iSector, const CfFragment& fragment)
117{
118 GPUTPCClusterFinder& clusterer = processors()->tpcClusterer[iSector];
119 uint32_t nAdcs = 0;
121 const auto& data = mCFContext->fragmentData[fragment.index];
122 uint32_t pagesEndpoint = 0;
123 const uint32_t nAdcsExpected = data.nDigits[iSector][endpoint];
124 const uint32_t nPagesExpected = data.nPages[iSector][endpoint];
125
126 uint32_t nAdcDecoded = 0;
127 const auto& zs = mIOPtrs.tpcZS->sector[iSector];
128 for (uint32_t i = data.minMaxCN[iSector][endpoint].zsPtrFirst; i < data.minMaxCN[iSector][endpoint].zsPtrLast; i++) {
129 const uint32_t pageFirst = (i == data.minMaxCN[iSector][endpoint].zsPtrFirst) ? data.minMaxCN[iSector][endpoint].zsPageFirst : 0;
130 const uint32_t pageLast = (i + 1 == data.minMaxCN[iSector][endpoint].zsPtrLast) ? data.minMaxCN[iSector][endpoint].zsPageLast : zs.nZSPtr[endpoint][i];
131 for (uint32_t j = pageFirst; j < pageLast; j++) {
132 const uint8_t* page = static_cast<const uint8_t*>(zs.zsPtr[endpoint][i]) + j * TPCZSHDR::TPC_ZS_PAGE_SIZE;
133 const header::RAWDataHeader* rawDataHeader = reinterpret_cast<const header::RAWDataHeader*>(page);
134 const TPCZSHDRV2* decHdr = reinterpret_cast<const TPCZSHDRV2*>(page + raw::RDHUtils::getMemorySize(*rawDataHeader) - sizeof(TPCZSHDRV2));
135 const uint16_t nSamplesInPage = decHdr->nADCsamples;
136
137 nAdcDecoded += nSamplesInPage;
138 pagesEndpoint++;
139 }
140 }
141
142 if (pagesEndpoint != nPagesExpected) {
143 GPUFatal("Sector %d, Endpoint %d, Fragment %d: TPC raw page count mismatch: expected %d / buffered %u", iSector, endpoint, fragment.index, pagesEndpoint, nPagesExpected);
144 }
145
146 if (nAdcDecoded != nAdcsExpected) {
147 GPUFatal("Sector %d, Endpoint %d, Fragment %d: TPC ADC count mismatch: expected %u, buffered %u", iSector, endpoint, fragment.index, nAdcsExpected, nAdcDecoded);
148 }
149
150 if (nAdcs != clusterer.mPzsOffsets[endpoint].offset) {
151 GPUFatal("Sector %d, Endpoint %d, Fragment %d: TPC ADC offset mismatch: expected %u, buffered %u", iSector, endpoint, fragment.index, nAdcs, clusterer.mPzsOffsets[endpoint].offset);
152 }
153
154 nAdcs += nAdcsExpected;
155 }
156}
157
158namespace
159{
160struct TPCCFDecodeScanTmp {
161 int32_t zsPtrFirst, zsPageFirst, zsPtrLast, zsPageLast, hasData, pageCounter;
162};
163} // namespace
164
165std::pair<uint32_t, uint32_t> GPUChainTracking::TPCClusterizerDecodeZSCount(uint32_t iSector, const CfFragment& fragment)
166{
167 mRec->getGeneralStepTimer(GeneralStep::Prepare).Start();
168 uint32_t nDigits = 0;
169 uint32_t nPages = 0;
170 uint32_t endpointAdcSamples[GPUTrackingInOutZS::NENDPOINTS];
171 memset(endpointAdcSamples, 0, sizeof(endpointAdcSamples));
173 int32_t firstHBF = (mIOPtrs.settingsTF && mIOPtrs.settingsTF->hasTfStartOrbit) ? mIOPtrs.settingsTF->tfStartOrbit : ((mIOPtrs.tpcZS->sector[iSector].count[0] && mIOPtrs.tpcZS->sector[iSector].nZSPtr[0][0]) ? o2::raw::RDHUtils::getHeartBeatOrbit(*(const o2::header::RAWDataHeader*)mIOPtrs.tpcZS->sector[iSector].zsPtr[0][0]) : 0);
174
175 for (uint16_t j = 0; j < GPUTrackingInOutZS::NENDPOINTS; j++) {
176#ifndef GPUCA_NO_VC
177 if (GetProcessingSettings().prefetchTPCpageScan >= 3 && j < GPUTrackingInOutZS::NENDPOINTS - 1) {
178 for (uint32_t k = 0; k < mIOPtrs.tpcZS->sector[iSector].count[j + 1]; k++) {
179 for (uint32_t l = 0; l < mIOPtrs.tpcZS->sector[iSector].nZSPtr[j + 1][k]; l++) {
180 Vc::Common::prefetchMid(((const uint8_t*)mIOPtrs.tpcZS->sector[iSector].zsPtr[j + 1][k]) + l * TPCZSHDR::TPC_ZS_PAGE_SIZE);
181 Vc::Common::prefetchMid(((const uint8_t*)mIOPtrs.tpcZS->sector[iSector].zsPtr[j + 1][k]) + l * TPCZSHDR::TPC_ZS_PAGE_SIZE + sizeof(o2::header::RAWDataHeader));
182 }
183 }
184 }
185#endif
186
187 std::vector<std::pair<CfFragment, TPCCFDecodeScanTmp>> fragments;
188 fragments.reserve(mCFContext->nFragments);
189 fragments.emplace_back(std::pair<CfFragment, TPCCFDecodeScanTmp>{fragment, {0, 0, 0, 0, 0, -1}});
190 for (uint32_t i = 1; i < mCFContext->nFragments; i++) {
191 fragments.emplace_back(std::pair<CfFragment, TPCCFDecodeScanTmp>{fragments.back().first.next(), {0, 0, 0, 0, 0, -1}});
192 }
193 std::vector<bool> fragmentExtends(mCFContext->nFragments, false);
194
195 uint32_t firstPossibleFragment = 0;
196 uint32_t pageCounter = 0;
197 uint32_t emptyPages = 0;
198 for (uint32_t k = 0; k < mIOPtrs.tpcZS->sector[iSector].count[j]; k++) {
199 if (GetProcessingSettings().tpcSingleSector != -1 && GetProcessingSettings().tpcSingleSector != (int32_t)iSector) {
200 break;
201 }
202 nPages += mIOPtrs.tpcZS->sector[iSector].nZSPtr[j][k];
203 for (uint32_t l = 0; l < mIOPtrs.tpcZS->sector[iSector].nZSPtr[j][k]; l++) {
204#ifndef GPUCA_NO_VC
205 if (GetProcessingSettings().prefetchTPCpageScan >= 2 && l + 1 < mIOPtrs.tpcZS->sector[iSector].nZSPtr[j][k]) {
206 Vc::Common::prefetchForOneRead(((const uint8_t*)mIOPtrs.tpcZS->sector[iSector].zsPtr[j][k]) + (l + 1) * TPCZSHDR::TPC_ZS_PAGE_SIZE);
207 Vc::Common::prefetchForOneRead(((const uint8_t*)mIOPtrs.tpcZS->sector[iSector].zsPtr[j][k]) + (l + 1) * TPCZSHDR::TPC_ZS_PAGE_SIZE + sizeof(o2::header::RAWDataHeader));
208 }
209#endif
210 const uint8_t* const page = ((const uint8_t*)mIOPtrs.tpcZS->sector[iSector].zsPtr[j][k]) + l * TPCZSHDR::TPC_ZS_PAGE_SIZE;
212 if (o2::raw::RDHUtils::getMemorySize(*rdh) == sizeof(o2::header::RAWDataHeader)) {
213 emptyPages++;
214 continue;
215 }
216 pageCounter++;
217 const TPCZSHDR* const hdr = (const TPCZSHDR*)(rdh_utils::getLink(o2::raw::RDHUtils::getFEEID(*rdh)) == rdh_utils::DLBZSLinkID ? (page + o2::raw::RDHUtils::getMemorySize(*rdh) - sizeof(TPCZSHDRV2)) : (page + sizeof(o2::header::RAWDataHeader)));
218 if (mCFContext->zsVersion == -1) {
219 mCFContext->zsVersion = hdr->version;
220 if (GetProcessingSettings().param.tpcTriggerHandling && mCFContext->zsVersion < ZSVersion::ZSVersionDenseLinkBased) { // TODO: Move tpcTriggerHandling to recoSteps bitmask
221 static bool errorShown = false;
222 if (errorShown == false) {
223 GPUAlarm("Trigger handling only possible with TPC Dense Link Based data, received version %d, disabling", mCFContext->zsVersion);
224 }
225 errorShown = true;
226 }
227 } else if (mCFContext->zsVersion != (int32_t)hdr->version) {
228 GPUError("Received TPC ZS 8kb page of mixed versions, expected %d, received %d (linkid %d, feeCRU %d, feeEndpoint %d, feelinkid %d)", mCFContext->zsVersion, (int32_t)hdr->version, (int32_t)o2::raw::RDHUtils::getLinkID(*rdh), (int32_t)rdh_utils::getCRU(*rdh), (int32_t)rdh_utils::getEndPoint(*rdh), (int32_t)rdh_utils::getLink(*rdh));
229 constexpr size_t bufferSize = 3 * std::max(sizeof(*rdh), sizeof(*hdr)) + 1;
230 char dumpBuffer[bufferSize];
231 for (size_t i = 0; i < sizeof(*rdh); i++) {
232 // "%02X " guaranteed to be 3 chars + ending 0.
233 snprintf(dumpBuffer + 3 * i, 4, "%02X ", (int32_t)((uint8_t*)rdh)[i]);
234 }
235 GPUAlarm("RDH of page: %s", dumpBuffer);
236 for (size_t i = 0; i < sizeof(*hdr); i++) {
237 // "%02X " guaranteed to be 3 chars + ending 0.
238 snprintf(dumpBuffer + 3 * i, 4, "%02X ", (int32_t)((uint8_t*)hdr)[i]);
239 }
240 GPUAlarm("Metainfo of page: %s", dumpBuffer);
241 if (GetProcessingSettings().ignoreNonFatalGPUErrors) {
242 mCFContext->abandonTimeframe = true;
243 return {0, 0};
244 } else {
245 GPUFatal("Cannot process with invalid TPC ZS data, exiting");
246 }
247 }
248 if (GetProcessingSettings().param.tpcTriggerHandling) {
249 const TPCZSHDRV2* const hdr2 = (const TPCZSHDRV2*)hdr;
250 if (hdr2->flags & TPCZSHDRV2::ZSFlags::TriggerWordPresent) {
251 const char* triggerWord = (const char*)hdr - TPCZSHDRV2::TRIGGER_WORD_SIZE;
253 memcpy((void*)&tmp.triggerWord, triggerWord, TPCZSHDRV2::TRIGGER_WORD_SIZE);
254 tmp.orbit = o2::raw::RDHUtils::getHeartBeatOrbit(*rdh);
255 if (tmp.triggerWord.isValid(0)) {
256 mTriggerBuffer->triggers.emplace(tmp);
257 }
258 }
259 }
260 nDigits += hdr->nADCsamples;
261 endpointAdcSamples[j] += hdr->nADCsamples;
262 uint32_t timeBin = (hdr->timeOffset + (o2::raw::RDHUtils::getHeartBeatOrbit(*rdh) - firstHBF) * o2::constants::lhc::LHCMaxBunches) / LHCBCPERTIMEBIN;
263 uint32_t maxTimeBin = timeBin + hdr->nTimeBinSpan;
264 if (mCFContext->zsVersion >= ZSVersion::ZSVersionDenseLinkBased) {
265 const TPCZSHDRV2* const hdr2 = (const TPCZSHDRV2*)hdr;
266 if (hdr2->flags & TPCZSHDRV2::ZSFlags::nTimeBinSpanBit8) {
267 maxTimeBin += 256;
268 }
269 }
270 if (maxTimeBin > mCFContext->tpcMaxTimeBin) {
271 mCFContext->tpcMaxTimeBin = maxTimeBin;
272 }
273 bool extendsInNextPage = false;
274 if (mCFContext->zsVersion >= ZSVersion::ZSVersionDenseLinkBased) {
275 if (l + 1 < mIOPtrs.tpcZS->sector[iSector].nZSPtr[j][k] && o2::raw::RDHUtils::getMemorySize(*rdh) == TPCZSHDR::TPC_ZS_PAGE_SIZE) {
277 extendsInNextPage = o2::raw::RDHUtils::getHeartBeatOrbit(*nextrdh) == o2::raw::RDHUtils::getHeartBeatOrbit(*rdh) && o2::raw::RDHUtils::getMemorySize(*nextrdh) > sizeof(o2::header::RAWDataHeader);
278 }
279 }
280 while (firstPossibleFragment && (uint32_t)fragments[firstPossibleFragment - 1].first.last() > timeBin) {
281 firstPossibleFragment--;
282 }
283 auto handleExtends = [&](uint32_t ff) {
284 if (fragmentExtends[ff]) {
285 if (doGPU) {
286 // Only add extended page on GPU. On CPU the pages are in consecutive memory anyway.
287 // Not adding the page prevents an issue where a page is decoded twice on CPU, when only the extend should be decoded.
288 fragments[ff].second.zsPageLast++;
289 mCFContext->fragmentData[ff].nPages[iSector][j]++;
290 mCFContext->fragmentData[ff].pageDigits[iSector][j].emplace_back(0);
291 }
292 fragmentExtends[ff] = false;
293 }
294 };
295 if (mCFContext->zsVersion >= ZSVersion::ZSVersionDenseLinkBased) {
296 for (uint32_t ff = 0; ff < firstPossibleFragment; ff++) {
297 handleExtends(ff);
298 }
299 }
300 for (uint32_t f = firstPossibleFragment; f < mCFContext->nFragments; f++) {
301 if (timeBin < (uint32_t)fragments[f].first.last() && (uint32_t)fragments[f].first.first() <= maxTimeBin) {
302 if (!fragments[f].second.hasData) {
303 fragments[f].second.hasData = 1;
304 fragments[f].second.zsPtrFirst = k;
305 fragments[f].second.zsPageFirst = l;
306 } else {
307 if (pageCounter > (uint32_t)fragments[f].second.pageCounter + 1) {
308 mCFContext->fragmentData[f].nPages[iSector][j] += emptyPages + pageCounter - fragments[f].second.pageCounter - 1;
309 for (uint32_t k2 = fragments[f].second.zsPtrLast - 1; k2 <= k; k2++) {
310 for (uint32_t l2 = ((int32_t)k2 == fragments[f].second.zsPtrLast - 1) ? fragments[f].second.zsPageLast : 0; l2 < (k2 < k ? mIOPtrs.tpcZS->sector[iSector].nZSPtr[j][k2] : l); l2++) {
311 if (doGPU) {
312 mCFContext->fragmentData[f].pageDigits[iSector][j].emplace_back(0);
313 } else {
314 // CPU cannot skip unneeded pages, so we must keep space to store the invalid dummy clusters
315 const uint8_t* const pageTmp = ((const uint8_t*)mIOPtrs.tpcZS->sector[iSector].zsPtr[j][k2]) + l2 * TPCZSHDR::TPC_ZS_PAGE_SIZE;
316 const o2::header::RAWDataHeader* rdhTmp = (const o2::header::RAWDataHeader*)pageTmp;
317 if (o2::raw::RDHUtils::getMemorySize(*rdhTmp) != sizeof(o2::header::RAWDataHeader)) {
318 const TPCZSHDR* const hdrTmp = (const TPCZSHDR*)(rdh_utils::getLink(o2::raw::RDHUtils::getFEEID(*rdhTmp)) == rdh_utils::DLBZSLinkID ? (pageTmp + o2::raw::RDHUtils::getMemorySize(*rdhTmp) - sizeof(TPCZSHDRV2)) : (pageTmp + sizeof(o2::header::RAWDataHeader)));
319 mCFContext->fragmentData[f].nDigits[iSector][j] += hdrTmp->nADCsamples;
320 }
321 }
322 }
323 }
324 } else if (emptyPages) {
325 mCFContext->fragmentData[f].nPages[iSector][j] += emptyPages;
326 if (doGPU) {
327 for (uint32_t m = 0; m < emptyPages; m++) {
328 mCFContext->fragmentData[f].pageDigits[iSector][j].emplace_back(0);
329 }
330 }
331 }
332 }
333 fragments[f].second.zsPtrLast = k + 1;
334 fragments[f].second.zsPageLast = l + 1;
335 fragments[f].second.pageCounter = pageCounter;
336 mCFContext->fragmentData[f].nPages[iSector][j]++;
337 mCFContext->fragmentData[f].nDigits[iSector][j] += hdr->nADCsamples;
338 if (doGPU) {
339 mCFContext->fragmentData[f].pageDigits[iSector][j].emplace_back(hdr->nADCsamples);
340 }
341 fragmentExtends[f] = extendsInNextPage;
342 } else {
343 handleExtends(f);
344 if (timeBin < (uint32_t)fragments[f].first.last()) {
345 if (mCFContext->zsVersion >= ZSVersion::ZSVersionDenseLinkBased) {
346 for (uint32_t ff = f + 1; ff < mCFContext->nFragments; ff++) {
347 handleExtends(ff);
348 }
349 }
350 break;
351 } else {
352 firstPossibleFragment = f + 1;
353 }
354 }
355 }
356 emptyPages = 0;
357 }
358 }
359 for (uint32_t f = 0; f < mCFContext->nFragments; f++) {
360 mCFContext->fragmentData[f].minMaxCN[iSector][j].zsPtrLast = fragments[f].second.zsPtrLast;
361 mCFContext->fragmentData[f].minMaxCN[iSector][j].zsPtrFirst = fragments[f].second.zsPtrFirst;
362 mCFContext->fragmentData[f].minMaxCN[iSector][j].zsPageLast = fragments[f].second.zsPageLast;
363 mCFContext->fragmentData[f].minMaxCN[iSector][j].zsPageFirst = fragments[f].second.zsPageFirst;
364 }
365 }
366 mCFContext->nPagesTotal += nPages;
367 mCFContext->nPagesSector[iSector] = nPages;
368
369 mCFContext->nDigitsEndpointMax[iSector] = 0;
370 for (uint32_t i = 0; i < GPUTrackingInOutZS::NENDPOINTS; i++) {
371 if (endpointAdcSamples[i] > mCFContext->nDigitsEndpointMax[iSector]) {
372 mCFContext->nDigitsEndpointMax[iSector] = endpointAdcSamples[i];
373 }
374 }
375 uint32_t nDigitsFragmentMax = 0;
376 for (uint32_t i = 0; i < mCFContext->nFragments; i++) {
377 uint32_t pagesInFragment = 0;
378 uint32_t digitsInFragment = 0;
379 for (uint16_t j = 0; j < GPUTrackingInOutZS::NENDPOINTS; j++) {
380 pagesInFragment += mCFContext->fragmentData[i].nPages[iSector][j];
381 digitsInFragment += mCFContext->fragmentData[i].nDigits[iSector][j];
382 }
383 mCFContext->nPagesFragmentMax = std::max(mCFContext->nPagesFragmentMax, pagesInFragment);
384 nDigitsFragmentMax = std::max(nDigitsFragmentMax, digitsInFragment);
385 }
386 mRec->getGeneralStepTimer(GeneralStep::Prepare).Stop();
387 return {nDigits, nDigitsFragmentMax};
388}
389
390void GPUChainTracking::RunTPCClusterizer_compactPeaks(GPUTPCClusterFinder& clusterer, GPUTPCClusterFinder& clustererShadow, int32_t stage, bool doGPU, int32_t lane)
391{
392 auto& in = stage ? clustererShadow.mPpeakPositions : clustererShadow.mPpositions;
393 auto& out = stage ? clustererShadow.mPfilteredPeakPositions : clustererShadow.mPpeakPositions;
394 if (doGPU) {
395 const uint32_t iSector = clusterer.mISector;
396 auto& count = stage ? clusterer.mPmemory->counters.nPeaks : clusterer.mPmemory->counters.nPositions;
397
398 std::vector<size_t> counts;
399
400 uint32_t nSteps = clusterer.getNSteps(count);
401 if (nSteps > clusterer.mNBufs) {
402 GPUError("Clusterer buffers exceeded (%u > %u)", nSteps, (int32_t)clusterer.mNBufs);
403 exit(1);
404 }
405
406 int32_t scanWorkgroupSize = mRec->getGPUParameters(doGPU).par_CF_SCAN_WORKGROUP_SIZE;
407 size_t tmpCount = count;
408 if (nSteps > 1) {
409 for (uint32_t i = 1; i < nSteps; i++) {
410 counts.push_back(tmpCount);
411 if (i == 1) {
412 runKernel<GPUTPCCFStreamCompaction, GPUTPCCFStreamCompaction::scanStart>({GetGrid(tmpCount, scanWorkgroupSize, lane), {iSector}}, i, stage);
413 } else {
414 runKernel<GPUTPCCFStreamCompaction, GPUTPCCFStreamCompaction::scanUp>({GetGrid(tmpCount, scanWorkgroupSize, lane), {iSector}}, i, tmpCount);
415 }
416 tmpCount = (tmpCount + scanWorkgroupSize - 1) / scanWorkgroupSize;
417 }
418
419 runKernel<GPUTPCCFStreamCompaction, GPUTPCCFStreamCompaction::scanTop>({GetGrid(tmpCount, scanWorkgroupSize, lane), {iSector}}, nSteps, tmpCount);
420
421 for (uint32_t i = nSteps - 1; i > 1; i--) {
422 tmpCount = counts[i - 1];
423 runKernel<GPUTPCCFStreamCompaction, GPUTPCCFStreamCompaction::scanDown>({GetGrid(tmpCount - scanWorkgroupSize, scanWorkgroupSize, lane), {iSector}}, i, scanWorkgroupSize, tmpCount);
424 }
425 }
426
427 runKernel<GPUTPCCFStreamCompaction, GPUTPCCFStreamCompaction::compactDigits>({GetGrid(count, scanWorkgroupSize, lane), {iSector}}, 1, stage, in, out);
428 } else {
429 auto& nOut = stage ? clusterer.mPmemory->counters.nClusters : clusterer.mPmemory->counters.nPeaks;
430 auto& nIn = stage ? clusterer.mPmemory->counters.nPeaks : clusterer.mPmemory->counters.nPositions;
431 size_t count = 0;
432 for (size_t i = 0; i < nIn; i++) {
433 if (clusterer.mPisPeak[i]) {
434 out[count++] = in[i];
435 }
436 }
437 nOut = count;
438 }
439}
440
441std::pair<uint32_t, uint32_t> GPUChainTracking::RunTPCClusterizer_transferZS(int32_t iSector, const CfFragment& fragment, int32_t lane)
442{
443 bool doGPU = GetRecoStepsGPU() & RecoStep::TPCClusterFinding;
444 if (mCFContext->abandonTimeframe) {
445 return {0, 0};
446 }
447 const auto& retVal = TPCClusterizerDecodeZSCountUpdate(iSector, fragment);
448 if (doGPU) {
449 GPUTPCClusterFinder& clusterer = processors()->tpcClusterer[iSector];
450 GPUTPCClusterFinder& clustererShadow = doGPU ? processorsShadow()->tpcClusterer[iSector] : clusterer;
451 uint32_t nPagesSector = 0;
452 for (uint32_t j = 0; j < GPUTrackingInOutZS::NENDPOINTS; j++) {
453 uint32_t nPages = 0;
454 mInputsHost->mPzsMeta->sector[iSector].zsPtr[j] = &mInputsShadow->mPzsPtrs[iSector * GPUTrackingInOutZS::NENDPOINTS + j];
455 mInputsHost->mPzsPtrs[iSector * GPUTrackingInOutZS::NENDPOINTS + j] = clustererShadow.mPzs + (nPagesSector + nPages) * TPCZSHDR::TPC_ZS_PAGE_SIZE;
456 for (uint32_t k = clusterer.mMinMaxCN[j].zsPtrFirst; k < clusterer.mMinMaxCN[j].zsPtrLast; k++) {
457 const uint32_t min = (k == clusterer.mMinMaxCN[j].zsPtrFirst) ? clusterer.mMinMaxCN[j].zsPageFirst : 0;
458 const uint32_t max = (k + 1 == clusterer.mMinMaxCN[j].zsPtrLast) ? clusterer.mMinMaxCN[j].zsPageLast : mIOPtrs.tpcZS->sector[iSector].nZSPtr[j][k];
459 if (max > min) {
460 char* src = (char*)mIOPtrs.tpcZS->sector[iSector].zsPtr[j][k] + min * TPCZSHDR::TPC_ZS_PAGE_SIZE;
461 char* ptrLast = (char*)mIOPtrs.tpcZS->sector[iSector].zsPtr[j][k] + (max - 1) * TPCZSHDR::TPC_ZS_PAGE_SIZE;
462 size_t size = (ptrLast - src) + o2::raw::RDHUtils::getMemorySize(*(const o2::header::RAWDataHeader*)ptrLast);
463 GPUMemCpy(RecoStep::TPCClusterFinding, clustererShadow.mPzs + (nPagesSector + nPages) * TPCZSHDR::TPC_ZS_PAGE_SIZE, src, size, lane, true);
464 }
465 nPages += max - min;
466 }
467 mInputsHost->mPzsMeta->sector[iSector].nZSPtr[j] = &mInputsShadow->mPzsSizes[iSector * GPUTrackingInOutZS::NENDPOINTS + j];
468 mInputsHost->mPzsSizes[iSector * GPUTrackingInOutZS::NENDPOINTS + j] = nPages;
469 mInputsHost->mPzsMeta->sector[iSector].count[j] = 1;
470 nPagesSector += nPages;
471 }
472 GPUMemCpy(RecoStep::TPCClusterFinding, clustererShadow.mPzsOffsets, clusterer.mPzsOffsets, clusterer.mNMaxPages * sizeof(*clusterer.mPzsOffsets), lane, true);
473 }
474 return retVal;
475}
476
477int32_t GPUChainTracking::RunTPCClusterizer_prepare(bool restorePointers)
478{
480 if (restorePointers) {
481 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
482 processors()->tpcClusterer[iSector].mPzsOffsets = mCFContext->ptrSave[iSector].zsOffsetHost;
483 processorsShadow()->tpcClusterer[iSector].mPzsOffsets = mCFContext->ptrSave[iSector].zsOffsetDevice;
484 processorsShadow()->tpcClusterer[iSector].mPzs = mCFContext->ptrSave[iSector].zsDevice;
485 }
486 processorsShadow()->ioPtrs.clustersNative = mCFContext->ptrClusterNativeSave;
487 return 0;
488 }
489 const auto& threadContext = GetThreadContext();
491 if (mCFContext == nullptr) {
493 }
494 const int16_t maxFragmentLen = GetProcessingSettings().overrideClusterizerFragmentLen;
495 const uint32_t maxAllowedTimebin = param().par.continuousTracking ? std::max<int32_t>(param().continuousMaxTimeBin, maxFragmentLen) : TPC_MAX_TIME_BIN_TRIGGERED;
496 mCFContext->tpcMaxTimeBin = maxAllowedTimebin;
497 const CfFragment fragmentMax{(tpccf::TPCTime)mCFContext->tpcMaxTimeBin + 1, maxFragmentLen};
498 mCFContext->prepare(mIOPtrs.tpcZS, fragmentMax);
499 if (GetProcessingSettings().param.tpcTriggerHandling) {
500 mTriggerBuffer->triggers.clear();
501 }
502 if (mIOPtrs.tpcZS) {
503 uint32_t nDigitsFragmentMax[NSECTORS];
504 mCFContext->zsVersion = -1;
505 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
506 if (mIOPtrs.tpcZS->sector[iSector].count[0]) {
507 const void* rdh = mIOPtrs.tpcZS->sector[iSector].zsPtr[0][0];
508 if (rdh && o2::raw::RDHUtils::getVersion<o2::header::RAWDataHeaderV6>() > o2::raw::RDHUtils::getVersion(rdh)) {
509 GPUError("Data has invalid RDH version %d, %d required\n", o2::raw::RDHUtils::getVersion(rdh), o2::raw::RDHUtils::getVersion<o2::header::RAWDataHeader>());
510 return 1;
511 }
512 }
513#ifndef GPUCA_NO_VC
514 if (GetProcessingSettings().prefetchTPCpageScan >= 1 && iSector < NSECTORS - 1) {
515 for (uint32_t j = 0; j < GPUTrackingInOutZS::NENDPOINTS; j++) {
516 for (uint32_t k = 0; k < mIOPtrs.tpcZS->sector[iSector].count[j]; k++) {
517 for (uint32_t l = 0; l < mIOPtrs.tpcZS->sector[iSector].nZSPtr[j][k]; l++) {
518 Vc::Common::prefetchFar(((const uint8_t*)mIOPtrs.tpcZS->sector[iSector + 1].zsPtr[j][k]) + l * TPCZSHDR::TPC_ZS_PAGE_SIZE);
519 Vc::Common::prefetchFar(((const uint8_t*)mIOPtrs.tpcZS->sector[iSector + 1].zsPtr[j][k]) + l * TPCZSHDR::TPC_ZS_PAGE_SIZE + sizeof(o2::header::RAWDataHeader));
520 }
521 }
522 }
523 }
524#endif
525 const auto& x = TPCClusterizerDecodeZSCount(iSector, fragmentMax);
526 nDigitsFragmentMax[iSector] = x.first;
527 processors()->tpcClusterer[iSector].mPmemory->counters.nDigits = x.first;
528 mRec->MemoryScalers()->nTPCdigits += x.first;
529 }
530 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
531 uint32_t nDigitsBase = nDigitsFragmentMax[iSector];
532 uint32_t threshold = 40000000;
533 uint32_t nDigitsScaled = nDigitsBase > threshold ? nDigitsBase : std::min((threshold + nDigitsBase) / 2, 2 * nDigitsBase);
534 processors()->tpcClusterer[iSector].SetNMaxDigits(processors()->tpcClusterer[iSector].mPmemory->counters.nDigits, mCFContext->nPagesFragmentMax, nDigitsScaled, mCFContext->nDigitsEndpointMax[iSector]);
535 if (doGPU) {
536 processorsShadow()->tpcClusterer[iSector].SetNMaxDigits(processors()->tpcClusterer[iSector].mPmemory->counters.nDigits, mCFContext->nPagesFragmentMax, nDigitsScaled, mCFContext->nDigitsEndpointMax[iSector]);
537 }
538 if (mPipelineNotifyCtx && GetProcessingSettings().doublePipelineClusterizer) {
539 mPipelineNotifyCtx->rec->AllocateRegisteredForeignMemory(processors()->tpcClusterer[iSector].mZSOffsetId, mRec);
540 mPipelineNotifyCtx->rec->AllocateRegisteredForeignMemory(processors()->tpcClusterer[iSector].mZSId, mRec);
541 } else {
542 AllocateRegisteredMemory(processors()->tpcClusterer[iSector].mZSOffsetId);
543 AllocateRegisteredMemory(processors()->tpcClusterer[iSector].mZSId);
544 }
545 }
546 } else {
547 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
548 uint32_t nDigits = mIOPtrs.tpcPackedDigits->nTPCDigits[iSector];
549 mRec->MemoryScalers()->nTPCdigits += nDigits;
550 processors()->tpcClusterer[iSector].SetNMaxDigits(nDigits, mCFContext->nPagesFragmentMax, nDigits, 0);
551 }
552 }
553
554 if (mIOPtrs.tpcZS) {
555 GPUInfo("Event has %u 8kb TPC ZS pages (version %d), %ld digits", mCFContext->nPagesTotal, mCFContext->zsVersion, (int64_t)mRec->MemoryScalers()->nTPCdigits);
556 } else {
557 GPUInfo("Event has %ld TPC Digits", (int64_t)mRec->MemoryScalers()->nTPCdigits);
558 }
559
560 if (mCFContext->tpcMaxTimeBin > maxAllowedTimebin) {
561 GPUError("Input data has invalid time bin %u > %d", mCFContext->tpcMaxTimeBin, maxAllowedTimebin);
562 if (GetProcessingSettings().ignoreNonFatalGPUErrors) {
563 mCFContext->abandonTimeframe = true;
564 mCFContext->tpcMaxTimeBin = maxAllowedTimebin;
565 } else {
566 return 1;
567 }
568 }
569
570 mCFContext->fragmentFirst = CfFragment{std::max<int32_t>(mCFContext->tpcMaxTimeBin + 1, maxFragmentLen), maxFragmentLen};
571 for (int32_t iSector = 0; iSector < GetProcessingSettings().nTPCClustererLanes && iSector < NSECTORS; iSector++) {
572 if (mIOPtrs.tpcZS && mCFContext->nPagesSector[iSector] && mCFContext->zsVersion != -1) {
573 mCFContext->nextPos[iSector] = RunTPCClusterizer_transferZS(iSector, mCFContext->fragmentFirst, GetProcessingSettings().nTPCClustererLanes + iSector);
574 }
575 }
576
577 if (mPipelineNotifyCtx && GetProcessingSettings().doublePipelineClusterizer) {
578 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
579 mCFContext->ptrSave[iSector].zsOffsetHost = processors()->tpcClusterer[iSector].mPzsOffsets;
580 mCFContext->ptrSave[iSector].zsOffsetDevice = processorsShadow()->tpcClusterer[iSector].mPzsOffsets;
581 mCFContext->ptrSave[iSector].zsDevice = processorsShadow()->tpcClusterer[iSector].mPzs;
582 }
583 }
584 return 0;
585}
586#endif
587
588int32_t GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput)
589{
590 if (param().rec.fwdTPCDigitsAsClusters) {
591 return ForwardTPCDigits();
592 }
593#ifdef GPUCA_TPC_GEOMETRY_O2
594 [[maybe_unused]] int32_t tpcTimeBinCut = mUpdateNewCalibObjects && mNewCalibValues->newTPCTimeBinCut ? mNewCalibValues->tpcTimeBinCut : param().tpcCutTimeBin;
596 const auto& threadContext = GetThreadContext();
597 const bool doGPU = GetRecoStepsGPU() & RecoStep::TPCClusterFinding;
598 if (RunTPCClusterizer_prepare(mPipelineNotifyCtx && GetProcessingSettings().doublePipelineClusterizer)) {
599 return 1;
600 }
601 if (GetProcessingSettings().autoAdjustHostThreads && !doGPU) {
603 }
604
606 float tpcHitLowOccupancyScalingFactor = 1.f;
608 uint32_t nHitsBase = mRec->MemoryScalers()->nTPCHits;
609 uint32_t threshold = 30000000 / 256 * mIOPtrs.settingsTF->nHBFPerTF;
610 if (mIOPtrs.settingsTF->nHBFPerTF < 64) {
611 threshold *= 2;
612 }
613 mRec->MemoryScalers()->nTPCHits = std::max<uint32_t>(nHitsBase, std::min<uint32_t>(threshold, nHitsBase * 3.5f)); // Increase the buffer size for low occupancy data to compensate for noisy pads creating exceiive clusters
614 if (nHitsBase < threshold) {
615 float maxFactor = mRec->MemoryScalers()->nTPCHits < threshold * 2 / 3 ? 3 : (mRec->MemoryScalers()->nTPCHits < threshold ? 2.25f : 1.75f);
616 mRec->MemoryScalers()->temporaryFactor *= std::min(maxFactor, (float)threshold / nHitsBase);
617 tpcHitLowOccupancyScalingFactor = std::min(3.5f, (float)threshold / nHitsBase);
618 }
619 }
620 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
621 processors()->tpcClusterer[iSector].SetMaxData(mIOPtrs); // First iteration to set data sizes
622 }
623 mRec->ComputeReuseMax(nullptr); // Resolve maximums for shared buffers
624 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
625 SetupGPUProcessor(&processors()->tpcClusterer[iSector], true); // Now we allocate
626 }
627 if (mPipelineNotifyCtx && GetProcessingSettings().doublePipelineClusterizer) {
628 RunTPCClusterizer_prepare(true); // Restore some pointers, allocated by the other pipeline, and set to 0 by SetupGPUProcessor (since not allocated in this pipeline)
629 }
630
631 if (doGPU && mIOPtrs.tpcZS) {
633 WriteToConstantMemory(RecoStep::TPCClusterFinding, (char*)&processors()->ioPtrs - (char*)processors(), &processorsShadow()->ioPtrs, sizeof(processorsShadow()->ioPtrs), mRec->NStreams() - 1);
634 }
635 if (doGPU) {
636 WriteToConstantMemory(RecoStep::TPCClusterFinding, (char*)processors()->tpcClusterer - (char*)processors(), processorsShadow()->tpcClusterer, sizeof(GPUTPCClusterFinder) * NSECTORS, mRec->NStreams() - 1, &mEvents->init);
637 }
638
639#ifdef GPUCA_HAS_ONNX
640 const GPUSettingsProcessingNNclusterizer& nn_settings = GetProcessingSettings().nn;
641 GPUTPCNNClusterizerHost nnApplications[GetProcessingSettings().nTPCClustererLanes];
642
643 // Maximum of 4 lanes supported
644 HighResTimer* nnTimers[12];
645
646 if (GetProcessingSettings().nn.applyNNclusterizer) {
647 int32_t deviceId = -1;
648 int32_t numLanes = GetProcessingSettings().nTPCClustererLanes;
649 int32_t maxThreads = mRec->getNKernelHostThreads(true);
650 // bool recreateMemoryAllocator = false;
651
652 if (GetProcessingSettings().debugLevel >= 1) {
653 nnTimers[0] = &getTimer<GPUTPCNNClusterizer, 0>("GPUTPCNNClusterizer_ONNXClassification_0_", 0);
654 nnTimers[1] = &getTimer<GPUTPCNNClusterizer, 1>("GPUTPCNNClusterizer_ONNXRegression_1_", 1);
655 nnTimers[2] = &getTimer<GPUTPCNNClusterizer, 2>("GPUTPCNNClusterizer_ONNXRegression2_2_", 2);
656 nnTimers[3] = &getTimer<GPUTPCNNClusterizer, 3>("GPUTPCNNClusterizer_ONNXClassification_0_", 3);
657 nnTimers[4] = &getTimer<GPUTPCNNClusterizer, 4>("GPUTPCNNClusterizer_ONNXRegression_1_", 4);
658 nnTimers[5] = &getTimer<GPUTPCNNClusterizer, 5>("GPUTPCNNClusterizer_ONNXRegression2_2_", 5);
659 nnTimers[6] = &getTimer<GPUTPCNNClusterizer, 6>("GPUTPCNNClusterizer_ONNXClassification_0_", 6);
660 nnTimers[7] = &getTimer<GPUTPCNNClusterizer, 7>("GPUTPCNNClusterizer_ONNXRegression_1_", 7);
661 nnTimers[8] = &getTimer<GPUTPCNNClusterizer, 8>("GPUTPCNNClusterizer_ONNXRegression2_2_", 8);
662 nnTimers[9] = &getTimer<GPUTPCNNClusterizer, 9>("GPUTPCNNClusterizer_ONNXClassification_0_", 9);
663 nnTimers[10] = &getTimer<GPUTPCNNClusterizer, 10>("GPUTPCNNClusterizer_ONNXRegression_1_", 10);
664 nnTimers[11] = &getTimer<GPUTPCNNClusterizer, 11>("GPUTPCNNClusterizer_ONNXRegression2_2_", 11);
665 }
666
667 mRec->runParallelOuterLoop(doGPU, numLanes, [&](uint32_t lane) {
668 nnApplications[lane].init(nn_settings, GetProcessingSettings().deterministicGPUReconstruction);
669 if (nnApplications[lane].mModelsUsed[0]) {
670 SetONNXGPUStream(*(nnApplications[lane].mModelClass).getSessionOptions(), lane, &deviceId);
671 (nnApplications[lane].mModelClass).setDeviceId(deviceId);
672 if (nnApplications[lane].mModelClass.getIntraOpNumThreads() > maxThreads) {
673 nnApplications[lane].mModelClass.setIntraOpNumThreads(maxThreads);
674 }
675 (nnApplications[lane].mModelClass).initEnvironment();
676 // Registering this once seems to be enough, even with different environmnents / models. ONNX apparently uses this per device and stores the OrtAllocator internally. All models will then use the volatile allocation.
677 // But environment must be valid, so we init the model environment first and use it here afterwards.
678 // Either this is done in one environment with lane == 0 or by recreating the allocator using recreateMemoryAllocator.
679 // TODO: Volatile allocation works for reserving, but not yet for allocations when binding the input tensor
680 // if (lane == 0) {
681 // nnApplications[lane].directOrtAllocator((nnApplications[lane].mModelClass).getEnv(), (nnApplications[lane].mModelClass).getMemoryInfo(), mRec, recreateMemoryAllocator);
682 // }
683 // recreateMemoryAllocator = true;
684 (nnApplications[lane].mModelClass).initSession();
685 }
686 if (nnApplications[lane].mModelsUsed[1]) {
687 SetONNXGPUStream(*(nnApplications[lane].mModelReg1).getSessionOptions(), lane, &deviceId);
688 (nnApplications[lane].mModelReg1).setDeviceId(deviceId);
689 if (nnApplications[lane].mModelReg1.getIntraOpNumThreads() > maxThreads) {
690 nnApplications[lane].mModelReg1.setIntraOpNumThreads(maxThreads);
691 }
692 // (nnApplications[lane].mModelReg1).setEnv((nnApplications[lane].mModelClass).getEnv());
693 (nnApplications[lane].mModelReg1).initEnvironment();
694 // nnApplications[lane].directOrtAllocator((nnApplications[lane].mModelReg1).getEnv(), (nnApplications[lane].mModelReg1).getMemoryInfo(), mRec, recreateMemoryAllocator);
695 (nnApplications[lane].mModelReg1).initSession();
696 }
697 if (nnApplications[lane].mModelsUsed[2]) {
698 SetONNXGPUStream(*(nnApplications[lane].mModelReg2).getSessionOptions(), lane, &deviceId);
699 (nnApplications[lane].mModelReg2).setDeviceId(deviceId);
700 if (nnApplications[lane].mModelReg2.getIntraOpNumThreads() > maxThreads) {
701 nnApplications[lane].mModelReg2.setIntraOpNumThreads(maxThreads);
702 }
703 // (nnApplications[lane].mModelReg2).setEnv((nnApplications[lane].mModelClass).getEnv());
704 (nnApplications[lane].mModelReg2).initEnvironment();
705 // nnApplications[lane].directOrtAllocator((nnApplications[lane].mModelClass).getEnv(), (nnApplications[lane].mModelClass).getMemoryInfo(), mRec, recreateMemoryAllocator);
706 (nnApplications[lane].mModelReg2).initSession();
707 }
708 if (nn_settings.nnClusterizerVerbosity > 0) {
709 LOG(info) << "(ORT) Allocated ONNX stream for lane " << lane << " and device " << deviceId;
710 }
711 });
712 const int16_t maxFragmentLen = GetProcessingSettings().overrideClusterizerFragmentLen;
713 const uint32_t maxAllowedTimebin = param().par.continuousTracking ? std::max<int32_t>(param().continuousMaxTimeBin, maxFragmentLen) : TPC_MAX_TIME_BIN_TRIGGERED;
714 for (int32_t sector = 0; sector < NSECTORS; sector++) {
715 GPUTPCNNClusterizer& clustererNN = processors()->tpcNNClusterer[sector];
716 GPUTPCNNClusterizer& clustererNNShadow = doGPU ? processorsShadow()->tpcNNClusterer[sector] : clustererNN;
717 int32_t lane = sector % numLanes;
718 clustererNN.mDeviceId = deviceId;
719 clustererNN.mISector = sector;
721 nnApplications[lane].initClusterizer(nn_settings, clustererNN, maxFragmentLen, maxAllowedTimebin);
722 if (doGPU) {
723 clustererNNShadow.mDeviceId = deviceId;
724 clustererNNShadow.mISector = sector;
726 nnApplications[lane].initClusterizer(nn_settings, clustererNNShadow, maxFragmentLen, maxAllowedTimebin);
727 }
728 if (nn_settings.nnClusterizerVerbosity > 2) {
729 LOG(info) << "(NNCLUS, GPUChainTrackingClusterizer, this=" << this << ") Processor initialized. Sector " << sector << ", lane " << lane << ", max clusters " << clustererNN.mNnClusterizerTotalClusters << " (clustererNN=" << &clustererNN << ", clustererNNShadow=" << &clustererNNShadow << ")";
730 }
732 if (nn_settings.nnClusterizerVerbosity > 2) {
733 LOG(info) << "(NNCLUS, GPUChainTrackingClusterizer, this=" << this << ") Memory registered for memoryId " << clustererNN.mMemoryId << " (clustererNN=" << &clustererNN << ", clustererNNShadow=" << &clustererNNShadow << ")";
734 }
735 // nnApplications[lane].createBoundary(clustererNNShadow);
736 // nnApplications[lane].createIndexLookup(clustererNNShadow);
737 }
738 if (doGPU) {
739 if (nn_settings.nnClusterizerVerbosity > 2) {
740 LOG(info) << "(NNCLUS, GPUChainTrackingClusterizer, this=" << this << ") Writing to constant memory...";
741 }
742 WriteToConstantMemory(RecoStep::TPCClusterFinding, (char*)&processors()->tpcNNClusterer - (char*)processors(), &processorsShadow()->tpcNNClusterer, sizeof(GPUTPCNNClusterizer) * NSECTORS, mRec->NStreams() - 1, &mEvents->init);
743 if (nn_settings.nnClusterizerVerbosity > 2) {
744 LOG(info) << "(NNCLUS, GPUChainTrackingClusterizer, this=" << this << ") Writing to constant memory done";
745 }
746 }
747 }
748#endif
749
750 size_t nClsTotal = 0;
751 ClusterNativeAccess* tmpNativeAccess = mClusterNativeAccess.get();
752 ClusterNative* tmpNativeClusters = nullptr;
753 std::unique_ptr<ClusterNative[]> tmpNativeClusterBuffer;
754
755 // setup MC Labels
757
758 auto* digitsMC = propagateMCLabels ? processors()->ioPtrs.tpcPackedDigits->tpcDigitsMC : nullptr;
759
760 bool buildNativeGPU = doGPU && NeedTPCClustersOnGPU();
761 bool buildNativeHost = (mRec->GetRecoStepsOutputs() & GPUDataTypes::InOutType::TPCClusters) || GetProcessingSettings().deterministicGPUReconstruction; // TODO: Should do this also when clusters are needed for later steps on the host but not requested as output
762
763 mInputsHost->mNClusterNative = mInputsShadow->mNClusterNative = mRec->MemoryScalers()->nTPCHits * tpcHitLowOccupancyScalingFactor;
764 if (buildNativeGPU) {
765 AllocateRegisteredMemory(mInputsHost->mResourceClusterNativeBuffer);
766 }
767 if (mWaitForFinalInputs && GetProcessingSettings().nTPCClustererLanes > 6) {
768 GPUFatal("ERROR, mWaitForFinalInputs cannot be called with nTPCClustererLanes > 6");
769 }
770 if (buildNativeHost && !(buildNativeGPU && GetProcessingSettings().delayedOutput)) {
771 if (mWaitForFinalInputs) {
772 GPUFatal("Cannot use waitForFinalInput callback without delayed output");
773 }
774 if (!GetProcessingSettings().tpcApplyClusterFilterOnCPU) {
775 AllocateRegisteredMemory(mInputsHost->mResourceClusterNativeOutput, GetProcessingSettings().tpcWriteClustersAfterRejection ? nullptr : mSubOutputControls[GPUTrackingOutputs::getIndex(&GPUTrackingOutputs::clustersNative)]);
776 tmpNativeClusters = mInputsHost->mPclusterNativeOutput;
777 } else {
778 tmpNativeClusterBuffer = std::make_unique<ClusterNative[]>(mInputsHost->mNClusterNative);
779 tmpNativeClusters = tmpNativeClusterBuffer.get();
780 }
781 }
782
783 GPUTPCLinearLabels mcLinearLabels;
784 if (propagateMCLabels) {
785 // No need to overallocate here, nTPCHits is anyway an upper bound used for the GPU cluster buffer, and we can always enlarge the buffer anyway
786 mcLinearLabels.header.reserve(mRec->MemoryScalers()->nTPCHits / 2);
787 mcLinearLabels.data.reserve(mRec->MemoryScalers()->nTPCHits);
788 }
789
790 int8_t transferRunning[NSECTORS] = {0};
791 uint32_t outputQueueStart = mOutputQueue.size();
792
793 auto notifyForeignChainFinished = [this]() {
794 if (mPipelineNotifyCtx) {
795 SynchronizeStream(OutputStream()); // Must finish before updating ioPtrs in (global) constant memory
796 {
797 std::lock_guard<std::mutex> lock(mPipelineNotifyCtx->mutex);
798 mPipelineNotifyCtx->ready = true;
799 }
800 mPipelineNotifyCtx->cond.notify_one();
801 }
802 };
803 bool synchronizeCalibUpdate = false;
804
805 for (uint32_t iSectorBase = 0; iSectorBase < NSECTORS; iSectorBase += GetProcessingSettings().nTPCClustererLanes) {
806 std::vector<bool> laneHasData(GetProcessingSettings().nTPCClustererLanes, false);
807 static_assert(NSECTORS <= GPUCA_MAX_STREAMS, "Stream events must be able to hold all sectors");
808 const int32_t maxLane = std::min<int32_t>(GetProcessingSettings().nTPCClustererLanes, NSECTORS - iSectorBase);
809 for (CfFragment fragment = mCFContext->fragmentFirst; !fragment.isEnd(); fragment = fragment.next()) {
810 if (GetProcessingSettings().debugLevel >= 3) {
811 GPUInfo("Processing time bins [%d, %d) for sectors %d to %d", fragment.start, fragment.last(), iSectorBase, iSectorBase + GetProcessingSettings().nTPCClustererLanes - 1);
812 }
813 mRec->runParallelOuterLoop(doGPU, maxLane, [&](uint32_t lane) {
814 if (doGPU && fragment.index != 0) {
815 SynchronizeStream(lane); // Don't overwrite charge map from previous iteration until cluster computation is finished
816 }
817
818 uint32_t iSector = iSectorBase + lane;
819 GPUTPCClusterFinder& clusterer = processors()->tpcClusterer[iSector];
820 GPUTPCClusterFinder& clustererShadow = doGPU ? processorsShadow()->tpcClusterer[iSector] : clusterer;
821 clusterer.mPmemory->counters.nPeaks = clusterer.mPmemory->counters.nClusters = 0;
822 clusterer.mPmemory->fragment = fragment;
823
825 bool setDigitsOnGPU = doGPU && not mIOPtrs.tpcZS;
826 bool setDigitsOnHost = (not doGPU && not mIOPtrs.tpcZS) || propagateMCLabels;
827 auto* inDigits = mIOPtrs.tpcPackedDigits;
828 size_t numDigits = inDigits->nTPCDigits[iSector];
829 if (setDigitsOnGPU) {
830 GPUMemCpy(RecoStep::TPCClusterFinding, clustererShadow.mPdigits, inDigits->tpcDigits[iSector], sizeof(clustererShadow.mPdigits[0]) * numDigits, lane, true);
831 }
832 if (setDigitsOnHost) {
833 clusterer.mPdigits = const_cast<o2::tpc::Digit*>(inDigits->tpcDigits[iSector]); // TODO: Needs fixing (invalid const cast)
834 }
835 clusterer.mPmemory->counters.nDigits = numDigits;
836 }
837
838 if (mIOPtrs.tpcZS) {
839 if (mCFContext->nPagesSector[iSector] && mCFContext->zsVersion != -1) {
840 clusterer.mPmemory->counters.nPositions = mCFContext->nextPos[iSector].first;
841 clusterer.mPmemory->counters.nPagesSubsector = mCFContext->nextPos[iSector].second;
842 } else {
843 clusterer.mPmemory->counters.nPositions = clusterer.mPmemory->counters.nPagesSubsector = 0;
844 }
845 }
846 TransferMemoryResourceLinkToGPU(RecoStep::TPCClusterFinding, clusterer.mMemoryId, lane);
847
848 using ChargeMapType = decltype(*clustererShadow.mPchargeMap);
849 using PeakMapType = decltype(*clustererShadow.mPpeakMap);
850 runKernel<GPUMemClean16>({GetGridAutoStep(lane, RecoStep::TPCClusterFinding)}, clustererShadow.mPchargeMap, TPCMapMemoryLayout<ChargeMapType>::items(GetProcessingSettings().overrideClusterizerFragmentLen) * sizeof(ChargeMapType));
851 runKernel<GPUMemClean16>({GetGridAutoStep(lane, RecoStep::TPCClusterFinding)}, clustererShadow.mPpeakMap, TPCMapMemoryLayout<PeakMapType>::items(GetProcessingSettings().overrideClusterizerFragmentLen) * sizeof(PeakMapType));
852 if (fragment.index == 0) {
853 runKernel<GPUMemClean16>({GetGridAutoStep(lane, RecoStep::TPCClusterFinding)}, clustererShadow.mPpadIsNoisy, TPC_PADS_IN_SECTOR * sizeof(*clustererShadow.mPpadIsNoisy));
854 }
856
857 if (doGPU) {
858 if (mIOPtrs.tpcZS && mCFContext->nPagesSector[iSector] && mCFContext->zsVersion != -1) {
859 TransferMemoryResourceLinkToGPU(RecoStep::TPCClusterFinding, mInputsHost->mResourceZS, lane);
860 SynchronizeStream(GetProcessingSettings().nTPCClustererLanes + lane);
861 }
862 SynchronizeStream(mRec->NStreams() - 1); // Wait for copying to constant memory
863 }
864
865 if (mIOPtrs.tpcZS && (mCFContext->abandonTimeframe || !mCFContext->nPagesSector[iSector] || mCFContext->zsVersion == -1)) {
866 clusterer.mPmemory->counters.nPositions = 0;
867 return;
868 }
869 if (!mIOPtrs.tpcZS && mIOPtrs.tpcPackedDigits->nTPCDigits[iSector] == 0) {
870 clusterer.mPmemory->counters.nPositions = 0;
871 return;
872 }
873
874 if (propagateMCLabels && fragment.index == 0) {
875 clusterer.PrepareMC();
876 clusterer.mPinputLabels = digitsMC->v[iSector];
877 if (clusterer.mPinputLabels == nullptr) {
878 GPUFatal("MC label container missing, sector %d", iSector);
879 }
880 if (clusterer.mPinputLabels->getIndexedSize() != mIOPtrs.tpcPackedDigits->nTPCDigits[iSector]) {
881 GPUFatal("MC label container has incorrect number of entries: %d expected, has %d\n", (int32_t)mIOPtrs.tpcPackedDigits->nTPCDigits[iSector], (int32_t)clusterer.mPinputLabels->getIndexedSize());
882 }
883 }
884
885 if (GetProcessingSettings().tpcSingleSector == -1 || GetProcessingSettings().tpcSingleSector == (int32_t)iSector) {
886 if (not mIOPtrs.tpcZS) {
887 runKernel<GPUTPCCFChargeMapFiller, GPUTPCCFChargeMapFiller::findFragmentStart>({GetGrid(1, lane), {iSector}}, mIOPtrs.tpcZS == nullptr);
888 TransferMemoryResourceLinkToHost(RecoStep::TPCClusterFinding, clusterer.mMemoryId, lane);
889 } else if (propagateMCLabels) {
890 runKernel<GPUTPCCFChargeMapFiller, GPUTPCCFChargeMapFiller::findFragmentStart>({GetGrid(1, lane, GPUReconstruction::krnlDeviceType::CPU), {iSector}}, mIOPtrs.tpcZS == nullptr);
891 TransferMemoryResourceLinkToGPU(RecoStep::TPCClusterFinding, clusterer.mMemoryId, lane);
892 }
893 }
894
895 if (mIOPtrs.tpcZS) {
896 int32_t firstHBF = (mIOPtrs.settingsTF && mIOPtrs.settingsTF->hasTfStartOrbit) ? mIOPtrs.settingsTF->tfStartOrbit : ((mIOPtrs.tpcZS->sector[iSector].count[0] && mIOPtrs.tpcZS->sector[iSector].nZSPtr[0][0]) ? o2::raw::RDHUtils::getHeartBeatOrbit(*(const o2::header::RAWDataHeader*)mIOPtrs.tpcZS->sector[iSector].zsPtr[0][0]) : 0);
897 uint32_t nBlocks = doGPU ? clusterer.mPmemory->counters.nPagesSubsector : GPUTrackingInOutZS::NENDPOINTS;
898
899 switch (mCFContext->zsVersion) {
900 default:
901 GPUFatal("Data with invalid TPC ZS mode (%d) received", mCFContext->zsVersion);
902 break;
905 runKernel<GPUTPCCFDecodeZS>({GetGridBlk(nBlocks, lane), {iSector}}, firstHBF);
906 break;
908 runKernel<GPUTPCCFDecodeZSLink>({GetGridBlk(nBlocks, lane), {iSector}}, firstHBF);
909 break;
911 runKernel<GPUTPCCFDecodeZSDenseLink>({GetGridBlk(nBlocks, lane), {iSector}}, firstHBF);
912 break;
913 }
914 TransferMemoryResourceLinkToHost(RecoStep::TPCClusterFinding, clusterer.mMemoryId, lane);
915 } // clang-format off
916 });
917 mRec->runParallelOuterLoop(doGPU, maxLane, [&](uint32_t lane) {
918 uint32_t iSector = iSectorBase + lane;
919 if (doGPU) {
920 SynchronizeStream(lane);
921 }
922 if (mIOPtrs.tpcZS) {
923 CfFragment f = fragment.next();
924 int32_t nextSector = iSector;
925 if (f.isEnd()) {
926 nextSector += GetProcessingSettings().nTPCClustererLanes;
927 f = mCFContext->fragmentFirst;
928 }
929 if (nextSector < NSECTORS && mIOPtrs.tpcZS && mCFContext->nPagesSector[nextSector] && mCFContext->zsVersion != -1 && !mCFContext->abandonTimeframe) {
930 mCFContext->nextPos[nextSector] = RunTPCClusterizer_transferZS(nextSector, f, GetProcessingSettings().nTPCClustererLanes + lane);
931 }
932 }
933 GPUTPCClusterFinder& clusterer = processors()->tpcClusterer[iSector];
934 GPUTPCClusterFinder& clustererShadow = doGPU ? processorsShadow()->tpcClusterer[iSector] : clusterer;
935 if (clusterer.mPmemory->counters.nPositions == 0) {
936 return;
937 }
938 if (!mIOPtrs.tpcZS) {
939 runKernel<GPUTPCCFChargeMapFiller, GPUTPCCFChargeMapFiller::fillFromDigits>({GetGrid(clusterer.mPmemory->counters.nPositions, lane), {iSector}});
940 }
942 clusterer.DumpChargeMap(*mDebugFile, "Charges");
943 }
944
945 if (propagateMCLabels) {
946 runKernel<GPUTPCCFChargeMapFiller, GPUTPCCFChargeMapFiller::fillIndexMap>({GetGrid(clusterer.mPmemory->counters.nDigitsInFragment, lane, GPUReconstruction::krnlDeviceType::CPU), {iSector}});
947 }
948
949 bool checkForNoisyPads = (rec()->GetParam().rec.tpc.maxTimeBinAboveThresholdIn1000Bin > 0) || (rec()->GetParam().rec.tpc.maxConsecTimeBinAboveThreshold > 0);
950 checkForNoisyPads &= (rec()->GetParam().rec.tpc.noisyPadsQuickCheck ? fragment.index == 0 : true);
951 checkForNoisyPads &= !GetProcessingSettings().disableTPCNoisyPadFilter;
952
953 if (checkForNoisyPads) {
955
956 runKernel<GPUTPCCFCheckPadBaseline>({GetGridBlk(nBlocks, lane), {iSector}});
957 }
958
959 runKernel<GPUTPCCFPeakFinder>({GetGrid(clusterer.mPmemory->counters.nPositions, lane), {iSector}});
961 clusterer.DumpPeakMap(*mDebugFile, "Peaks");
962 }
963
964 RunTPCClusterizer_compactPeaks(clusterer, clustererShadow, 0, doGPU, lane);
965 TransferMemoryResourceLinkToHost(RecoStep::TPCClusterFinding, clusterer.mMemoryId, lane);
966 DoDebugAndDump(RecoStep::TPCClusterFinding, GPUChainTrackingDebugFlags::TPCClustererPeaks, clusterer, &GPUTPCClusterFinder::DumpPeaksCompacted, *mDebugFile); // clang-format off
967 });
968 mRec->runParallelOuterLoop(doGPU, maxLane, [&](uint32_t lane) {
969 uint32_t iSector = iSectorBase + lane;
970 GPUTPCClusterFinder& clusterer = processors()->tpcClusterer[iSector];
971 GPUTPCClusterFinder& clustererShadow = doGPU ? processorsShadow()->tpcClusterer[iSector] : clusterer;
972 if (doGPU) {
973 SynchronizeStream(lane);
974 }
975 if (clusterer.mPmemory->counters.nPeaks == 0) {
976 return;
977 }
978 runKernel<GPUTPCCFNoiseSuppression, GPUTPCCFNoiseSuppression::noiseSuppression>({GetGrid(clusterer.mPmemory->counters.nPeaks, lane), {iSector}});
979 runKernel<GPUTPCCFNoiseSuppression, GPUTPCCFNoiseSuppression::updatePeaks>({GetGrid(clusterer.mPmemory->counters.nPeaks, lane), {iSector}});
981 clusterer.DumpPeakMap(*mDebugFile, "Suppressed Peaks");
982 }
983
984 RunTPCClusterizer_compactPeaks(clusterer, clustererShadow, 1, doGPU, lane);
985 TransferMemoryResourceLinkToHost(RecoStep::TPCClusterFinding, clusterer.mMemoryId, lane);
987 });
988 mRec->runParallelOuterLoop(doGPU, maxLane, [&](uint32_t lane) {
989 uint32_t iSector = iSectorBase + lane;
990 GPUTPCClusterFinder& clusterer = processors()->tpcClusterer[iSector];
991 GPUTPCClusterFinder& clustererShadow = doGPU ? processorsShadow()->tpcClusterer[iSector] : clusterer;
992
993 if (doGPU) {
994 SynchronizeStream(lane);
995 }
996
997 if (fragment.index == 0) {
998 deviceEvent* waitEvent = nullptr;
999 if (transferRunning[lane] == 1) {
1000 waitEvent = &mEvents->stream[lane];
1001 transferRunning[lane] = 2;
1002 }
1003 runKernel<GPUMemClean16>({GetGridAutoStep(lane, RecoStep::TPCClusterFinding), krnlRunRangeNone, {nullptr, waitEvent}}, clustererShadow.mPclusterInRow, GPUCA_ROW_COUNT * sizeof(*clustererShadow.mPclusterInRow));
1004 }
1005
1006 if (clusterer.mPmemory->counters.nClusters == 0) {
1007 return;
1008 }
1009
1010 if (GetProcessingSettings().nn.applyNNclusterizer) {
1011#ifdef GPUCA_HAS_ONNX
1012 GPUTPCNNClusterizer& clustererNN = processors()->tpcNNClusterer[lane];
1013 GPUTPCNNClusterizer& clustererNNShadow = doGPU ? processorsShadow()->tpcNNClusterer[lane] : clustererNN;
1014 GPUTPCNNClusterizerHost& nnApplication = nnApplications[lane];
1015
1016 // int withMC = (doGPU && propagateMCLabels);
1017
1018 if (nn_settings.nnClusterizerApplyCfDeconvolution) {
1019 runKernel<GPUTPCCFDeconvolution>({GetGrid(clusterer.mPmemory->counters.nPositions, lane), {iSector}}, true);
1020 } else if (clustererNNShadow.mNnClusterizerSetDeconvolutionFlags) {
1021 runKernel<GPUTPCCFDeconvolution>({GetGrid(clusterer.mPmemory->counters.nPositions, lane), {iSector}}, false);
1022 }
1023
1024 // float time_clusterizer = 0, time_fill = 0, time_networks = 0;
1025 if (nn_settings.nnClusterizerVerbosity > 2) {
1026 LOG(info) << "(NNCLUS, GPUChainTrackingClusterizer, this=" << this << ") Starting loop over batched data. clustererNNShadow.mNnClusterizerBatchedMode=" << clustererNNShadow.mNnClusterizerBatchedMode << ", numLoops=" << std::ceil((float)clusterer.mPmemory->counters.nClusters / clustererNNShadow.mNnClusterizerBatchedMode) << ", numClusters=" << clusterer.mPmemory->counters.nClusters << ". (clustererNN=" << &clustererNN << ", clustererNNShadow=" << &clustererNNShadow << ")";
1027 }
1028 for (int batch = 0; batch < std::ceil((float)clusterer.mPmemory->counters.nClusters / clustererNNShadow.mNnClusterizerBatchedMode); batch++) {
1029 if (nn_settings.nnClusterizerVerbosity > 3) {
1030 LOG(info) << "(NNCLUS, GPUChainTrackingClusterizer, this=" << this << ") Start. Loop=" << batch << ". (clustererNN=" << &clustererNN << ", clustererNNShadow=" << &clustererNNShadow << ")";
1031 }
1032 uint batchStart = batch * clustererNNShadow.mNnClusterizerBatchedMode;
1033 size_t iSize = CAMath::Min((uint)clustererNNShadow.mNnClusterizerBatchedMode, (uint)(clusterer.mPmemory->counters.nClusters - batchStart));
1034
1035 // Filling the data
1036 if (mRec->IsGPU() || GetProcessingSettings().nn.nnClusterizerForceGpuInputFill) {
1037 // Fills element by element of each input matrix -> better parallelizability, but worse on CPU due to unnecessary computations
1038 runKernel<GPUTPCNNClusterizerKernels, GPUTPCNNClusterizerKernels::fillInputNNGPU>({GetGrid(iSize * clustererNNShadow.mNnClusterizerRowTimeSizeFull, lane), krnlRunRangeNone}, iSector, clustererNNShadow.mNnInferenceInputDType, propagateMCLabels, batchStart);
1039 } else {
1040 // Fills the whole input matrix at once -> better performance on CPU, but worse parallelizability
1041 runKernel<GPUTPCNNClusterizerKernels, GPUTPCNNClusterizerKernels::fillInputNNCPU>({GetGrid(iSize, lane), krnlRunRangeNone}, iSector, clustererNNShadow.mNnInferenceInputDType, propagateMCLabels, batchStart);
1042 }
1043 if (nn_settings.nnClusterizerVerbosity > 3) {
1044 LOG(info) << "(NNCLUS, GPUChainTrackingClusterizer, this=" << this << ") Done filling data. Loop=" << batch << ". (clustererNN=" << &clustererNN << ", clustererNNShadow=" << &clustererNNShadow << ")";
1045 }
1046
1047 if (clustererNNShadow.mNnClusterizerSetDeconvolutionFlags) {
1048 runKernel<GPUTPCNNClusterizerKernels, GPUTPCNNClusterizerKernels::publishDeconvolutionFlags>({GetGrid(iSize, lane), krnlRunRangeNone}, iSector, clustererNNShadow.mNnInferenceInputDType, propagateMCLabels, batchStart); // Publishing the deconvolution flags
1049 if (nn_settings.nnClusterizerVerbosity > 3) {
1050 LOG(info) << "(NNCLUS, GPUChainTrackingClusterizer, this=" << this << ") Done setting deconvolution flags. Loop=" << batch << ". (clustererNN=" << &clustererNN << ", clustererNNShadow=" << &clustererNNShadow << ")";
1051 }
1052 }
1053
1054 // NN evaluations
1055 if(clustererNNShadow.mNnClusterizerUseClassification) {
1056 if(GetProcessingSettings().debugLevel >= 1 && (doGPU || lane < 4)) { nnTimers[3*lane]->Start(); }
1057 if (clustererNNShadow.mNnInferenceInputDType == 0) {
1058 if (clustererNNShadow.mNnInferenceOutputDType == 0) {
1059 (nnApplication.mModelClass).inference(clustererNNShadow.mInputData_16, iSize, clustererNNShadow.mModelProbabilities_16);
1060 } else if (clustererNNShadow.mNnInferenceOutputDType == 1) {
1061 (nnApplication.mModelClass).inference(clustererNNShadow.mInputData_16, iSize, clustererNNShadow.mModelProbabilities_32);
1062 }
1063 } else if (clustererNNShadow.mNnInferenceInputDType == 1) {
1064 if (clustererNNShadow.mNnInferenceOutputDType == 0) {
1065 (nnApplication.mModelClass).inference(clustererNNShadow.mInputData_32, iSize, clustererNNShadow.mModelProbabilities_16);
1066 } else if (clustererNNShadow.mNnInferenceOutputDType == 1) {
1067 (nnApplication.mModelClass).inference(clustererNNShadow.mInputData_32, iSize, clustererNNShadow.mModelProbabilities_32);
1068 }
1069 }
1070 if(GetProcessingSettings().debugLevel >= 1 && (doGPU || lane < 4)) { nnTimers[3*lane]->Stop(); } // doGPU || lane<4 -> only for GPU or first 4 CPU lanes (to limit number of concurrent timers). At least gives some statistics for CPU time...
1071 if (nn_settings.nnClusterizerVerbosity > 3) {
1072 LOG(info) << "(NNCLUS, GPUChainTrackingClusterizer, this=" << this << ") Done with NN classification inference. Loop=" << batch << ". (clustererNN=" << &clustererNN << ", clustererNNShadow=" << &clustererNNShadow << ")";
1073 }
1074 }
1075 if (!clustererNNShadow.mNnClusterizerUseCfRegression) {
1076 if(GetProcessingSettings().debugLevel >= 1 && (doGPU || lane < 4)) { nnTimers[3*lane + 1]->Start(); }
1077 if (clustererNNShadow.mNnInferenceInputDType == 0) {
1078 if (clustererNNShadow.mNnInferenceOutputDType == 0) {
1079 (nnApplication.mModelReg1).inference(clustererNNShadow.mInputData_16, iSize, clustererNNShadow.mOutputDataReg1_16);
1080 } else if (clustererNNShadow.mNnInferenceOutputDType == 1) {
1081 (nnApplication.mModelReg1).inference(clustererNNShadow.mInputData_16, iSize, clustererNNShadow.mOutputDataReg1_32);
1082 }
1083 } else if (clustererNNShadow.mNnInferenceInputDType == 1) {
1084 if (clustererNNShadow.mNnInferenceOutputDType == 0) {
1085 (nnApplication.mModelReg1).inference(clustererNNShadow.mInputData_32, iSize, clustererNNShadow.mOutputDataReg1_16);
1086 } else if (clustererNNShadow.mNnInferenceOutputDType == 1) {
1087 (nnApplication.mModelReg1).inference(clustererNNShadow.mInputData_32, iSize, clustererNNShadow.mOutputDataReg1_32);
1088 }
1089 }
1090 if(GetProcessingSettings().debugLevel >= 1 && (doGPU || lane < 4)) { nnTimers[3*lane + 1]->Stop(); }
1091 if (nnApplication.mModelClass.getNumOutputNodes()[0][1] > 1 && nnApplication.mModelReg2.isInitialized()) {
1092 if(GetProcessingSettings().debugLevel >= 1 && (doGPU || lane < 4)) { nnTimers[3*lane + 2]->Start(); }
1093 if (clustererNNShadow.mNnInferenceInputDType == 0) {
1094 if (clustererNNShadow.mNnInferenceOutputDType == 0) {
1095 (nnApplication.mModelReg2).inference(clustererNNShadow.mInputData_16, iSize, clustererNNShadow.mOutputDataReg2_16);
1096 } else if (clustererNNShadow.mNnInferenceOutputDType == 1) {
1097 (nnApplication.mModelReg2).inference(clustererNNShadow.mInputData_16, iSize, clustererNNShadow.mOutputDataReg2_32);
1098 }
1099 } else if (clustererNNShadow.mNnInferenceInputDType == 1) {
1100 if (clustererNNShadow.mNnInferenceOutputDType == 0) {
1101 (nnApplication.mModelReg2).inference(clustererNNShadow.mInputData_32, iSize, clustererNNShadow.mOutputDataReg2_16);
1102 } else if (clustererNNShadow.mNnInferenceOutputDType == 1) {
1103 (nnApplication.mModelReg2).inference(clustererNNShadow.mInputData_32, iSize, clustererNNShadow.mOutputDataReg2_32);
1104 }
1105 }
1106 if(GetProcessingSettings().debugLevel >= 1 && (doGPU || lane < 4)) { nnTimers[3*lane + 2]->Stop(); }
1107 }
1108 if (nn_settings.nnClusterizerVerbosity > 3) {
1109 LOG(info) << "(NNCLUS, GPUChainTrackingClusterizer, this=" << this << ") Done with NN regression inference. Loop=" << batch << ". (clustererNN=" << &clustererNN << ", clustererNNShadow=" << &clustererNNShadow << ")";
1110 }
1111 }
1112
1113 // Publishing kernels for class labels and regression results
1114 // In case classification should not be used, this kernel should still be executed to fill the mOutputDataClass array with default values
1115 if (nnApplication.mModelClass.getNumOutputNodes()[0][1] == 1) {
1116 runKernel<GPUTPCNNClusterizerKernels, GPUTPCNNClusterizerKernels::determineClass1Labels>({GetGrid(iSize, lane), krnlRunRangeNone}, iSector, clustererNNShadow.mNnInferenceOutputDType, propagateMCLabels, batchStart); // Assigning class labels
1117 } else {
1118 runKernel<GPUTPCNNClusterizerKernels, GPUTPCNNClusterizerKernels::determineClass2Labels>({GetGrid(iSize, lane), krnlRunRangeNone}, iSector, clustererNNShadow.mNnInferenceOutputDType, propagateMCLabels, batchStart); // Assigning class labels
1119 }
1120 if (!clustererNNShadow.mNnClusterizerUseCfRegression) {
1121 runKernel<GPUTPCNNClusterizerKernels, GPUTPCNNClusterizerKernels::publishClass1Regression>({GetGrid(iSize, lane), krnlRunRangeNone}, iSector, clustererNNShadow.mNnInferenceOutputDType, propagateMCLabels, batchStart); // Publishing class 1 regression results
1122 if (nnApplication.mModelClass.getNumOutputNodes()[0][1] > 1 && nnApplication.mModelReg2.isInitialized()) {
1123 runKernel<GPUTPCNNClusterizerKernels, GPUTPCNNClusterizerKernels::publishClass2Regression>({GetGrid(iSize, lane), krnlRunRangeNone}, iSector, clustererNNShadow.mNnInferenceOutputDType, propagateMCLabels, batchStart); // Publishing class 2 regression results
1124 }
1125 }
1126 if (nn_settings.nnClusterizerVerbosity > 3) {
1127 LOG(info) << "(NNCLUS, GPUChainTrackingClusterizer, this=" << this << ") Done publishing. Loop=" << batch << ". (clustererNN=" << &clustererNN << ", clustererNNShadow=" << &clustererNNShadow << ")";
1128 }
1129 }
1130
1131 if (clustererNNShadow.mNnClusterizerUseCfRegression) {
1132 if(!nn_settings.nnClusterizerApplyCfDeconvolution) { // If it is already applied don't do it twice, otherwise apply now
1133 runKernel<GPUTPCCFDeconvolution>({GetGrid(clusterer.mPmemory->counters.nPositions, lane), {iSector}}, true);
1134 }
1135 DoDebugAndDump(RecoStep::TPCClusterFinding, GPUChainTrackingDebugFlags::TPCClustererChargeMap, clusterer, &GPUTPCClusterFinder::DumpChargeMap, *mDebugFile, "Split Charges");
1136 runKernel<GPUTPCNNClusterizerKernels, GPUTPCNNClusterizerKernels::runCfClusterizer>({GetGrid(clusterer.mPmemory->counters.nClusters, lane), krnlRunRangeNone}, iSector, clustererNNShadow.mNnInferenceInputDType, propagateMCLabels, 0); // Running the CF regression kernel - no batching needed: batchStart = 0
1137 if (nn_settings.nnClusterizerVerbosity > 3) {
1138 LOG(info) << "(NNCLUS, GPUChainTrackingClusterizer, this=" << this << ") Done with CF regression. (clustererNN=" << &clustererNN << ", clustererNNShadow=" << &clustererNNShadow << ")";
1139 }
1140 }
1141#else
1142 GPUFatal("Project not compiled with neural network clusterization. Aborting.");
1143#endif
1144 } else {
1145 runKernel<GPUTPCCFDeconvolution>({GetGrid(clusterer.mPmemory->counters.nPositions, lane), {iSector}}, true);
1146 DoDebugAndDump(RecoStep::TPCClusterFinding, GPUChainTrackingDebugFlags::TPCClustererChargeMap, clusterer, &GPUTPCClusterFinder::DumpChargeMap, *mDebugFile, "Split Charges");
1147 runKernel<GPUTPCCFClusterizer>({GetGrid(clusterer.mPmemory->counters.nClusters, lane), {iSector}}, 0);
1148 }
1149
1150 if (doGPU && propagateMCLabels) {
1151 TransferMemoryResourceLinkToHost(RecoStep::TPCClusterFinding, clusterer.mScratchId, lane);
1152 if (doGPU) {
1153 SynchronizeStream(lane);
1154 }
1155 runKernel<GPUTPCCFClusterizer>({GetGrid(clusterer.mPmemory->counters.nClusters, lane, GPUReconstruction::krnlDeviceType::CPU), {iSector}}, 1); // Computes MC labels
1156 }
1157
1158 if (GetProcessingSettings().debugLevel >= 3) {
1159 GPUInfo("Sector %02d Fragment %02d Lane %d: Found clusters: digits %u peaks %u clusters %u", iSector, fragment.index, lane, (int32_t)clusterer.mPmemory->counters.nPositions, (int32_t)clusterer.mPmemory->counters.nPeaks, (int32_t)clusterer.mPmemory->counters.nClusters);
1160 }
1161
1162 TransferMemoryResourcesToHost(RecoStep::TPCClusterFinding, &clusterer, lane);
1163 laneHasData[lane] = true;
1164 // Include clusters in default debug mask, exclude other debug output by default
1165 DoDebugAndDump(RecoStep::TPCClusterFinding, GPUChainTrackingDebugFlags::TPCClustererClusters, clusterer, &GPUTPCClusterFinder::DumpClusters, *mDebugFile); // clang-format off
1166 });
1168 }
1169
1170 size_t nClsFirst = nClsTotal;
1171 bool anyLaneHasData = false;
1172 for (int32_t lane = 0; lane < maxLane; lane++) {
1173 uint32_t iSector = iSectorBase + lane;
1174 std::fill(&tmpNativeAccess->nClusters[iSector][0], &tmpNativeAccess->nClusters[iSector][0] + MAXGLOBALPADROW, 0);
1175 if (doGPU) {
1176 SynchronizeStream(lane);
1177 }
1178 GPUTPCClusterFinder& clusterer = processors()->tpcClusterer[iSector];
1179 GPUTPCClusterFinder& clustererShadow = doGPU ? processorsShadow()->tpcClusterer[iSector] : clusterer;
1180
1181 if (laneHasData[lane]) {
1182 anyLaneHasData = true;
1183 if (buildNativeGPU && GetProcessingSettings().tpccfGatherKernel) {
1184 runKernel<GPUTPCCFGather>({GetGridBlk(GPUCA_ROW_COUNT, mRec->NStreams() - 1), {iSector}}, &mInputsShadow->mPclusterNativeBuffer[nClsTotal]);
1185 }
1186 for (uint32_t j = 0; j < GPUCA_ROW_COUNT; j++) {
1187 if (nClsTotal + clusterer.mPclusterInRow[j] > mInputsHost->mNClusterNative) {
1188 clusterer.raiseError(GPUErrors::ERROR_CF_GLOBAL_CLUSTER_OVERFLOW, iSector * 1000 + j, nClsTotal + clusterer.mPclusterInRow[j], mInputsHost->mNClusterNative);
1189 continue;
1190 }
1191 if (buildNativeGPU) {
1192 if (!GetProcessingSettings().tpccfGatherKernel) {
1193 GPUMemCpyAlways(RecoStep::TPCClusterFinding, (void*)&mInputsShadow->mPclusterNativeBuffer[nClsTotal], (const void*)&clustererShadow.mPclusterByRow[j * clusterer.mNMaxClusterPerRow], sizeof(mIOPtrs.clustersNative->clustersLinear[0]) * clusterer.mPclusterInRow[j], mRec->NStreams() - 1, -2);
1194 }
1195 } else if (buildNativeHost) {
1196 GPUMemCpyAlways(RecoStep::TPCClusterFinding, (void*)&tmpNativeClusters[nClsTotal], (const void*)&clustererShadow.mPclusterByRow[j * clusterer.mNMaxClusterPerRow], sizeof(mIOPtrs.clustersNative->clustersLinear[0]) * clusterer.mPclusterInRow[j], mRec->NStreams() - 1, false);
1197 }
1198 tmpNativeAccess->nClusters[iSector][j] += clusterer.mPclusterInRow[j];
1199 nClsTotal += clusterer.mPclusterInRow[j];
1200 }
1201 if (transferRunning[lane]) {
1202 ReleaseEvent(mEvents->stream[lane], doGPU);
1203 }
1204 RecordMarker(&mEvents->stream[lane], mRec->NStreams() - 1);
1205 transferRunning[lane] = 1;
1206 }
1207
1208 if (not propagateMCLabels || not laneHasData[lane]) {
1209 assert(propagateMCLabels ? mcLinearLabels.header.size() == nClsTotal : true);
1210 continue;
1211 }
1212
1213 runKernel<GPUTPCCFMCLabelFlattener, GPUTPCCFMCLabelFlattener::setRowOffsets>({GetGrid(GPUCA_ROW_COUNT, lane, GPUReconstruction::krnlDeviceType::CPU), {iSector}});
1215 runKernel<GPUTPCCFMCLabelFlattener, GPUTPCCFMCLabelFlattener::flatten>({GetGrid(GPUCA_ROW_COUNT, lane, GPUReconstruction::krnlDeviceType::CPU), {iSector}}, &mcLinearLabels);
1216 clusterer.clearMCMemory();
1217 assert(propagateMCLabels ? mcLinearLabels.header.size() == nClsTotal : true);
1218 }
1219 if (propagateMCLabels) {
1220 for (int32_t lane = 0; lane < maxLane; lane++) {
1221 processors()->tpcClusterer[iSectorBase + lane].clearMCMemory();
1222 }
1223 }
1224 if (buildNativeHost && buildNativeGPU && anyLaneHasData) {
1225 if (GetProcessingSettings().delayedOutput) {
1226 mOutputQueue.emplace_back(outputQueueEntry{(void*)((char*)&tmpNativeClusters[nClsFirst] - (char*)&tmpNativeClusters[0]), &mInputsShadow->mPclusterNativeBuffer[nClsFirst], (nClsTotal - nClsFirst) * sizeof(tmpNativeClusters[0]), RecoStep::TPCClusterFinding});
1227 } else {
1228 GPUMemCpy(RecoStep::TPCClusterFinding, (void*)&tmpNativeClusters[nClsFirst], (const void*)&mInputsShadow->mPclusterNativeBuffer[nClsFirst], (nClsTotal - nClsFirst) * sizeof(tmpNativeClusters[0]), mRec->NStreams() - 1, false);
1229 }
1230 }
1231
1232 if (mWaitForFinalInputs && iSectorBase >= 21 && (int32_t)iSectorBase < 21 + GetProcessingSettings().nTPCClustererLanes) {
1233 notifyForeignChainFinished();
1234 }
1235 if (mWaitForFinalInputs && iSectorBase >= 30 && (int32_t)iSectorBase < 30 + GetProcessingSettings().nTPCClustererLanes) {
1236 mWaitForFinalInputs();
1237 synchronizeCalibUpdate = DoQueuedUpdates(0, false);
1238 }
1239 }
1240 for (int32_t i = 0; i < GetProcessingSettings().nTPCClustererLanes; i++) {
1241#ifdef GPUCA_HAS_ONNX
1242 if (GetProcessingSettings().nn.applyNNclusterizer) {
1243 if (GetProcessingSettings().nn.nnClusterizerVerbosity > 0) {
1244 LOG(info) << "(ORT) Environment releasing...";
1245 }
1246 GPUTPCNNClusterizerHost& nnApplication = nnApplications[i];
1247 nnApplication.mModelClass.release(true);
1248 nnApplication.mModelReg1.release(true);
1249 nnApplication.mModelReg2.release(true);
1250 }
1251#endif
1252 if (transferRunning[i]) {
1253 ReleaseEvent(mEvents->stream[i], doGPU);
1254 }
1255 }
1256
1257 if (GetProcessingSettings().param.tpcTriggerHandling) {
1259 if (triggerOutput && triggerOutput->allocator) {
1260 // GPUInfo("Storing %lu trigger words", mTriggerBuffer->triggers.size());
1261 auto* outputBuffer = (decltype(mTriggerBuffer->triggers)::value_type*)triggerOutput->allocator(mTriggerBuffer->triggers.size() * sizeof(decltype(mTriggerBuffer->triggers)::value_type));
1262 std::copy(mTriggerBuffer->triggers.begin(), mTriggerBuffer->triggers.end(), outputBuffer);
1263 }
1264 mTriggerBuffer->triggers.clear();
1265 }
1266
1267 ClusterNativeAccess::ConstMCLabelContainerView* mcLabelsConstView = nullptr;
1268 if (propagateMCLabels) {
1269 // TODO: write to buffer directly
1271 std::pair<ConstMCLabelContainer*, ConstMCLabelContainerView*> buffer;
1274 throw std::runtime_error("Cluster MC Label buffer missing");
1275 }
1277 buffer = {&container->first, &container->second};
1278 } else {
1279 mIOMem.clusterNativeMCView = std::make_unique<ConstMCLabelContainerView>();
1280 mIOMem.clusterNativeMCBuffer = std::make_unique<ConstMCLabelContainer>();
1281 buffer.first = mIOMem.clusterNativeMCBuffer.get();
1282 buffer.second = mIOMem.clusterNativeMCView.get();
1283 }
1284
1285 assert(propagateMCLabels ? mcLinearLabels.header.size() == nClsTotal : true);
1286 assert(propagateMCLabels ? mcLinearLabels.data.size() >= nClsTotal : true);
1287
1288 mcLabels.setFrom(mcLinearLabels.header, mcLinearLabels.data);
1289 mcLabels.flatten_to(*buffer.first);
1290 *buffer.second = *buffer.first;
1291 mcLabelsConstView = buffer.second;
1292 }
1293
1294 if (buildNativeHost && buildNativeGPU && GetProcessingSettings().delayedOutput) {
1295 mInputsHost->mNClusterNative = mInputsShadow->mNClusterNative = nClsTotal;
1296 AllocateRegisteredMemory(mInputsHost->mResourceClusterNativeOutput, GetProcessingSettings().tpcWriteClustersAfterRejection ? nullptr : mSubOutputControls[GPUTrackingOutputs::getIndex(&GPUTrackingOutputs::clustersNative)]);
1297 tmpNativeClusters = mInputsHost->mPclusterNativeOutput;
1298 for (uint32_t i = outputQueueStart; i < mOutputQueue.size(); i++) {
1299 mOutputQueue[i].dst = (char*)tmpNativeClusters + (size_t)mOutputQueue[i].dst;
1300 }
1301 }
1302
1303 if (buildNativeHost) {
1304 tmpNativeAccess->clustersLinear = tmpNativeClusters;
1305 tmpNativeAccess->clustersMCTruth = mcLabelsConstView;
1306 tmpNativeAccess->setOffsetPtrs();
1307 mIOPtrs.clustersNative = tmpNativeAccess;
1308 if (GetProcessingSettings().tpcApplyClusterFilterOnCPU) {
1309 auto allocator = [this, &tmpNativeClusters](size_t size) {
1310 this->mInputsHost->mNClusterNative = size;
1311 this->AllocateRegisteredMemory(this->mInputsHost->mResourceClusterNativeOutput, this->GetProcessingSettings().tpcWriteClustersAfterRejection ? nullptr : this->mSubOutputControls[GPUTrackingOutputs::getIndex(&GPUTrackingOutputs::clustersNative)]);
1312 return (tmpNativeClusters = this->mInputsHost->mPclusterNativeOutput);
1313 };
1314 RunTPCClusterFilter(tmpNativeAccess, allocator, false);
1315 nClsTotal = tmpNativeAccess->nClustersTotal;
1316 }
1317 }
1318
1319 if (!mWaitForFinalInputs) {
1320 notifyForeignChainFinished();
1321 }
1322
1323 if (buildNativeGPU) {
1324 processorsShadow()->ioPtrs.clustersNative = mInputsShadow->mPclusterNativeAccess;
1325 WriteToConstantMemory(RecoStep::TPCClusterFinding, (char*)&processors()->ioPtrs - (char*)processors(), &processorsShadow()->ioPtrs, sizeof(processorsShadow()->ioPtrs), 0);
1326 *mInputsHost->mPclusterNativeAccess = *mIOPtrs.clustersNative;
1327 mInputsHost->mPclusterNativeAccess->clustersLinear = mInputsShadow->mPclusterNativeBuffer;
1328 mInputsHost->mPclusterNativeAccess->setOffsetPtrs();
1329 TransferMemoryResourceLinkToGPU(RecoStep::TPCClusterFinding, mInputsHost->mResourceClusterNativeAccess, 0);
1330 }
1331 if (doGPU && synchronizeOutput) {
1333 }
1334 if (doGPU && synchronizeCalibUpdate) {
1336 }
1337 if (buildNativeHost && (GetProcessingSettings().deterministicGPUReconstruction || GetProcessingSettings().debugLevel >= 4)) {
1338 for (uint32_t i = 0; i < NSECTORS; i++) {
1339 for (uint32_t j = 0; j < GPUCA_ROW_COUNT; j++) {
1340 std::sort(&tmpNativeClusters[tmpNativeAccess->clusterOffset[i][j]], &tmpNativeClusters[tmpNativeAccess->clusterOffset[i][j] + tmpNativeAccess->nClusters[i][j]]);
1341 }
1342 }
1343 if (buildNativeGPU) {
1344 GPUMemCpy(RecoStep::TPCClusterFinding, (void*)mInputsShadow->mPclusterNativeBuffer, (const void*)tmpNativeClusters, nClsTotal * sizeof(tmpNativeClusters[0]), -1, true);
1345 }
1346 }
1347 mRec->MemoryScalers()->nTPCHits = nClsTotal;
1348 mRec->PopNonPersistentMemory(RecoStep::TPCClusterFinding, qStr2Tag("TPCCLUST"));
1349 if (mPipelineNotifyCtx) {
1351 mPipelineNotifyCtx = nullptr;
1352 }
1353
1354 if (GetProcessingSettings().autoAdjustHostThreads && !doGPU) {
1356 }
1357
1358#endif
1359 return 0;
1360}
Definition of the TPC Digit.
int32_t i
#define TPC_MAX_TIME_BIN_TRIGGERED
#define GPUCA_MAX_STREAMS
int32_t retVal
#define GPUCA_ROW_COUNT
std::enable_if_t< std::is_signed< T >::value, bool > hasData(const CalArray< T > &cal)
Definition Painter.cxx:599
uint32_t j
Definition RawData.h:0
uint8_t endpoint
Definition RawData.h:0
Definitions of TPC Zero Suppression Data Headers.
void Start()
Definition timer.cxx:64
void Stop()
Definition timer.cxx:76
A container to hold and manage MC truth information/labels.
void setFrom(std::vector< MCTruthHeaderElement > &header, std::vector< TruthElement > &truthArray)
size_t flatten_to(ContainerType &container) const
std::unique_ptr< o2::tpc::ClusterNativeAccess > mClusterNativeAccess
int32_t RunTPCClusterizer(bool synchronizeOutput=true)
std::unique_ptr< GPUTrackingInputProvider > mInputsHost
std::array< GPUOutputControl *, GPUTrackingOutputs::count()> mSubOutputControls
std::unique_ptr< std::ofstream > mDebugFile
std::unique_ptr< GPUTriggerOutputs > mTriggerBuffer
std::vector< outputQueueEntry > mOutputQueue
std::unique_ptr< GPUTPCCFChainContext > mCFContext
int32_t DoQueuedUpdates(int32_t stream, bool updateSlave=true)
std::unique_ptr< GPUNewCalibValues > mNewCalibValues
GPUTrackingInOutPointers & mIOPtrs
struct o2::gpu::GPUChainTracking::InOutMemory mIOMem
std::unique_ptr< GPUTrackingInputProvider > mInputsShadow
void RecordMarker(deviceEvent *ev, int32_t stream)
Definition GPUChain.h:108
void TransferMemoryResourceLinkToGPU(RecoStep step, int16_t res, int32_t stream=-1, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
Definition GPUChain.h:124
void GPUMemCpyAlways(RecoStep step, void *dst, const void *src, size_t size, int32_t stream, int32_t toGPU, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
Definition GPUChain.h:129
void GPUMemCpy(RecoStep step, void *dst, const void *src, size_t size, int32_t stream, int32_t toGPU, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
Definition GPUChain.h:128
bool DoDebugAndDump(RecoStep step, uint32_t mask, T &processor, S T::*func, Args &&... args)
Definition GPUChain.h:230
GPUReconstruction::RecoStepField GetRecoStepsGPU() const
Definition GPUChain.h:72
void WriteToConstantMemory(RecoStep step, size_t offset, const void *src, size_t size, int32_t stream=-1, deviceEvent *ev=nullptr)
Definition GPUChain.h:127
void ReleaseEvent(deviceEvent ev, bool doGPU=true)
Definition GPUChain.h:111
size_t AllocateRegisteredMemory(GPUProcessor *proc)
Definition GPUChain.h:217
virtual std::unique_ptr< GPUReconstructionProcessing::threadContext > GetThreadContext()
Definition GPUChain.h:109
GPUConstantMem * processors()
Definition GPUChain.h:84
static constexpr krnlRunRange krnlRunRangeNone
Definition GPUChain.h:41
void SetONNXGPUStream(Ort::SessionOptions &opt, int32_t stream, int32_t *deviceId)
Definition GPUChain.h:90
GPUParam & param()
Definition GPUChain.h:87
void SetupGPUProcessor(T *proc, bool allocate)
Definition GPUChain.h:220
const GPUSettingsProcessing & GetProcessingSettings() const
Definition GPUChain.h:76
void SynchronizeStream(int32_t stream)
Definition GPUChain.h:89
GPUReconstructionCPU * mRec
Definition GPUChain.h:79
GPUConstantMem * processorsShadow()
Definition GPUChain.h:85
krnlExec GetGridAutoStep(int32_t stream, GPUDataTypes::RecoStep st=GPUDataTypes::RecoStep::NoRecoStep)
Definition GPUChain.cxx:47
static constexpr int32_t NSECTORS
Definition GPUChain.h:58
void TransferMemoryResourceLinkToHost(RecoStep step, int16_t res, int32_t stream=-1, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
Definition GPUChain.h:125
void TransferMemoryResourcesToHost(RecoStep step, GPUProcessor *proc, int32_t stream=-1, bool all=false)
Definition GPUChain.h:123
krnlExec GetGrid(uint32_t totalItems, uint32_t nThreads, int32_t stream, GPUReconstruction::krnlDeviceType d=GPUReconstruction::krnlDeviceType::Auto, GPUDataTypes::RecoStep st=GPUDataTypes::RecoStep::NoRecoStep)
Definition GPUChain.cxx:21
krnlExec GetGridBlk(uint32_t nBlocks, int32_t stream, GPUReconstruction::krnlDeviceType d=GPUReconstruction::krnlDeviceType::Auto, GPUDataTypes::RecoStep st=GPUDataTypes::RecoStep::NoRecoStep)
Definition GPUChain.cxx:32
GPUReconstruction * rec()
Definition GPUChain.h:66
HighResTimer & getGeneralStepTimer(GeneralStep step)
void runParallelOuterLoop(bool doGPU, uint32_t nThreads, std::function< void(uint32_t)> lambda)
const GPUDefParameters & getGPUParameters(bool doGPU) const override
void AllocateRegisteredForeignMemory(int16_t res, GPUReconstruction *rec, GPUOutputControl *control=nullptr)
void PopNonPersistentMemory(RecoStep step, uint64_t tag)
void ComputeReuseMax(GPUProcessor *proc)
RecoStepField GetRecoStepsGPU() const
const GPUParam & GetParam() const
void PushNonPersistentMemory(uint64_t tag)
InOutTypeField GetRecoStepsOutputs() const
GPUMemorySizeScalers * MemoryScalers()
static void setGlobalOffsetsAndAllocate(GPUTPCClusterFinder &, GPUTPCLinearLabels &)
void SetMaxData(const GPUTrackingInOutPointers &io)
void SetNMaxDigits(size_t nDigits, size_t nPages, size_t nDigitsFragment, size_t nDigitsEndpointMax)
void DumpSuppressedPeaks(std::ostream &out)
void DumpPeakMap(std::ostream &out, std::string_view)
o2::dataformats::ConstMCTruthContainerView< o2::MCCompLabel > const * mPinputLabels
void DumpChargeMap(std::ostream &out, std::string_view)
uint32_t getNSteps(size_t items) const
void DumpSuppressedPeaksCompacted(std::ostream &out)
void DumpPeaksCompacted(std::ostream &out)
tpc::ClusterNative * mPclusterByRow
void init(const GPUSettingsProcessingNNclusterizer &, bool=false)
void initClusterizer(const GPUSettingsProcessingNNclusterizer &, GPUTPCNNClusterizer &, int32_t=-1, int32_t=-1)
OrtDataType::Float16_t * mInputData_16
OrtDataType::Float16_t * mOutputDataReg2_16
OrtDataType::Float16_t * mModelProbabilities_16
OrtDataType::Float16_t * mOutputDataReg1_16
void release(bool=false)
void setIntraOpNumThreads(int threads)
std::vector< std::vector< int64_t > > getNumOutputNodes() const
#define TPC_PADS_IN_SECTOR
GLint GLenum GLint x
Definition glcorearb.h:403
const GLfloat * m
Definition glcorearb.h:4066
GLenum src
Definition glcorearb.h:1767
GLint GLsizei count
Definition glcorearb.h:399
GLuint buffer
Definition glcorearb.h:655
GLsizeiptr size
Definition glcorearb.h:659
GLdouble f
Definition glcorearb.h:310
GLboolean * data
Definition glcorearb.h:298
GLenum GLfloat param
Definition glcorearb.h:271
uint8_t itsSharedClusterMap uint8_t
constexpr int LHCMaxBunches
Definition of a container to keep/associate and arbitrary number of labels associated to an index wit...
RAWDataHeaderV7 RAWDataHeader
const float k2
Definition MathUtils.h:72
void dumpBuffer(gsl::span< const std::byte > buffer, std::ostream &out=std::cout, size_t maxbytes=std::numeric_limits< size_t >::max())
Definition DumpBuffer.h:139
constexpr int LHCBCPERTIMEBIN
Definition Constants.h:38
constexpr int MAXGLOBALPADROW
Definition Constants.h:34
Global TPC definitions and constants.
Definition SimTraits.h:167
@ ZSVersionDenseLinkBased
@ ZSVersionLinkBasedWithMeta
@ ZSVersionRowBased10BitADC
@ ZSVersionRowBased12BitADC
a couple of static helper functions to create timestamp values for CCDB queries or override obsolete ...
constexpr T qStr2Tag(const char *str)
Definition strtag.h:22
tpccf::TPCTime start
Definition CfFragment.h:31
std::unique_ptr< o2::dataformats::ConstMCTruthContainerView< o2::MCCompLabel > > clusterNativeMCView
std::unique_ptr< o2::dataformats::ConstMCTruthContainer< o2::MCCompLabel > > clusterNativeMCBuffer
deviceEvent stream[GPUCA_MAX_STREAMS]
GPUTPCClusterFinder tpcClusterer[GPUCA_NSECTORS]
GPUTrackingInOutPointers ioPtrs
size_t NTPCClusters(size_t tpcDigits, bool perSector=false)
std::function< void *(size_t)> allocator
struct o2::gpu::GPUTPCClusterFinder::Memory::counters_t counters
std::vector< o2::MCCompLabel > data
std::vector< o2::dataformats::MCTruthHeaderElement > header
const GPUTPCDigitsMCInput * tpcDigitsMC
const o2::tpc::ClusterNativeAccess * clustersNative
const GPUSettingsTF * settingsTF
const GPUTrackingInOutZS * tpcZS
const GPUTrackingInOutDigits * tpcPackedDigits
GPUTrackingInOutZSSector sector[NSECTORS]
static constexpr uint32_t NENDPOINTS
size_t getIndex(const GPUOutputControl &v)
static constexpr int getVersion()
get numeric version of the RDH
Definition RDHUtils.h:60
unsigned int nClusters[constants::MAXSECTOR][constants::MAXGLOBALPADROW]
const o2::dataformats::ConstMCTruthContainerView< o2::MCCompLabel > * clustersMCTruth
std::pair< ConstMCLabelContainer, ConstMCLabelContainerView > ConstMCLabelContainerViewWithBuffer
unsigned int clusterOffset[constants::MAXSECTOR][constants::MAXGLOBALPADROW]
const ClusterNative * clustersLinear
static constexpr unsigned int TRIGGER_WORD_SIZE
unsigned char version
static constexpr size_t TPC_ZS_PAGE_SIZE
unsigned short nADCsamples
Trigger info including the orbit.
uint32_t orbit
orbit of the trigger word
TriggerWordDLBZS triggerWord
trigger Word information
bool isValid(int entry=0) const
constexpr size_t min
constexpr size_t max
LOG(info)<< "Compressed in "<< sw.CpuTime()<< " s"
std::vector< Digit > digits