52#ifdef GPUCA_TPC_GEOMETRY_O2
53std::pair<uint32_t, uint32_t> GPUChainTracking::TPCClusterizerDecodeZSCountUpdate(uint32_t iSector,
const CfFragment& fragment)
63 uint16_t posInEndpoint = 0;
64 uint16_t pagesEndpoint = 0;
68 for (uint32_t l = pageFirst; l < pageLast; l++) {
69 uint16_t pageDigits =
mCFContext->fragmentData[fragment.
index].pageDigits[iSector][
j][posInEndpoint++];
77 if (pagesEndpoint !=
mCFContext->fragmentData[fragment.
index].pageDigits[iSector][
j].size()) {
79 GPUError(
"TPC raw page count mismatch in TPCClusterizerDecodeZSCountUpdate: expected %d / buffered %lu", pagesEndpoint,
mCFContext->fragmentData[fragment.
index].pageDigits[iSector][
j].size());
82 GPUFatal(
"TPC raw page count mismatch in TPCClusterizerDecodeZSCountUpdate: expected %d / buffered %lu", pagesEndpoint,
mCFContext->fragmentData[fragment.
index].pageDigits[iSector][
j].size());
95 TPCClusterizerEnsureZSOffsets(iSector, fragment);
100void GPUChainTracking::TPCClusterizerEnsureZSOffsets(uint32_t iSector,
const CfFragment& fragment)
106 uint32_t pagesEndpoint = 0;
107 const uint32_t nAdcsExpected =
data.nDigits[iSector][
endpoint];
108 const uint32_t nPagesExpected =
data.nPages[iSector][
endpoint];
110 uint32_t nAdcDecoded = 0;
113 const uint32_t pageFirst = (
i ==
data.minMaxCN[iSector][
endpoint].zsPtrFirst) ?
data.minMaxCN[iSector][
endpoint].zsPageFirst : 0;
115 for (uint32_t
j = pageFirst;
j < pageLast;
j++) {
119 const uint16_t nSamplesInPage = decHdr->
nADCsamples;
121 nAdcDecoded += nSamplesInPage;
126 if (pagesEndpoint != nPagesExpected) {
127 GPUFatal(
"Sector %d, Endpoint %d, Fragment %d: TPC raw page count mismatch: expected %d / buffered %u", iSector,
endpoint, fragment.
index, pagesEndpoint, nPagesExpected);
130 if (nAdcDecoded != nAdcsExpected) {
131 GPUFatal(
"Sector %d, Endpoint %d, Fragment %d: TPC ADC count mismatch: expected %u, buffered %u", iSector,
endpoint, fragment.
index, nAdcsExpected, nAdcDecoded);
138 nAdcs += nAdcsExpected;
144struct TPCCFDecodeScanTmp {
145 int32_t zsPtrFirst, zsPageFirst, zsPtrLast, zsPageLast,
hasData, pageCounter;
149std::pair<uint32_t, uint32_t> GPUChainTracking::TPCClusterizerDecodeZSCount(uint32_t iSector,
const CfFragment& fragment)
152 uint32_t nDigits = 0;
155 memset(endpointAdcSamples, 0,
sizeof(endpointAdcSamples));
171 std::vector<std::pair<CfFragment, TPCCFDecodeScanTmp>> fragments;
173 fragments.emplace_back(std::pair<CfFragment, TPCCFDecodeScanTmp>{fragment, {0, 0, 0, 0, 0, -1}});
175 fragments.emplace_back(std::pair<CfFragment, TPCCFDecodeScanTmp>{fragments.back().
first.next(), {0, 0, 0, 0, 0, -1}});
177 std::vector<bool> fragmentExtends(
mCFContext->nFragments,
false);
179 uint32_t firstPossibleFragment = 0;
180 uint32_t pageCounter = 0;
181 uint32_t emptyPages = 0;
205 static bool errorShown =
false;
206 if (errorShown ==
false) {
207 GPUAlarm(
"Trigger handling only possible with TPC Dense Link Based data, received version %d, disabling",
mCFContext->zsVersion);
212 GPUError(
"Received TPC ZS 8kb page of mixed versions, expected %d, received %d (linkid %d, feeCRU %d, feeEndpoint %d, feelinkid %d)",
mCFContext->zsVersion, (int32_t)hdr->
version, (int32_t)o2::raw::RDHUtils::getLinkID(*rdh), (int32_t)rdh_utils::getCRU(*rdh), (int32_t)rdh_utils::getEndPoint(*rdh), (int32_t)rdh_utils::getLink(*rdh));
213 constexpr size_t bufferSize = 3 * std::max(
sizeof(*rdh),
sizeof(*hdr)) + 1;
215 for (
size_t i = 0;
i <
sizeof(*rdh);
i++) {
217 snprintf(dumpBuffer + 3 *
i, 4,
"%02X ", (int32_t)((uint8_t*)rdh)[
i]);
219 GPUAlarm(
"RDH of page: %s", dumpBuffer);
220 for (
size_t i = 0;
i <
sizeof(*hdr);
i++) {
222 snprintf(dumpBuffer + 3 *
i, 4,
"%02X ", (int32_t)((uint8_t*)hdr)[
i]);
224 GPUAlarm(
"Metainfo of page: %s", dumpBuffer);
229 GPUFatal(
"Cannot process with invalid TPC ZS data, exiting");
234 if (hdr2->
flags & TPCZSHDRV2::ZSFlags::TriggerWordPresent) {
238 tmp.
orbit = o2::raw::RDHUtils::getHeartBeatOrbit(*rdh);
244 nDigits += hdr->nADCsamples;
245 endpointAdcSamples[
j] += hdr->nADCsamples;
247 uint32_t maxTimeBin = timeBin + hdr->nTimeBinSpan;
248 if (
mCFContext->zsVersion >= ZSVersion::ZSVersionDenseLinkBased) {
250 if (hdr2->
flags & TPCZSHDRV2::ZSFlags::nTimeBinSpanBit8) {
257 bool extendsInNextPage =
false;
258 if (
mCFContext->zsVersion >= ZSVersion::ZSVersionDenseLinkBased) {
261 extendsInNextPage = o2::raw::RDHUtils::getHeartBeatOrbit(*nextrdh) == o2::raw::RDHUtils::getHeartBeatOrbit(*rdh) && o2::raw::RDHUtils::getMemorySize(*nextrdh) >
sizeof(
o2::header::RAWDataHeader);
264 while (firstPossibleFragment && (uint32_t)fragments[firstPossibleFragment - 1].first.last() > timeBin) {
265 firstPossibleFragment--;
267 auto handleExtends = [&](uint32_t ff) {
268 if (fragmentExtends[ff]) {
272 fragments[ff].second.zsPageLast++;
274 mCFContext->fragmentData[ff].pageDigits[iSector][
j].emplace_back(0);
276 fragmentExtends[ff] =
false;
279 if (
mCFContext->zsVersion >= ZSVersion::ZSVersionDenseLinkBased) {
280 for (uint32_t ff = 0; ff < firstPossibleFragment; ff++) {
284 for (uint32_t
f = firstPossibleFragment;
f <
mCFContext->nFragments;
f++) {
285 if (timeBin < (uint32_t)fragments[
f].
first.last() && (uint32_t)fragments[
f].first.first() <= maxTimeBin) {
286 if (!fragments[
f].second.hasData) {
287 fragments[
f].second.hasData = 1;
288 fragments[
f].second.zsPtrFirst = k;
289 fragments[
f].second.zsPageFirst = l;
291 if (pageCounter > (uint32_t)fragments[
f].second.pageCounter + 1) {
292 mCFContext->fragmentData[
f].nPages[iSector][
j] += emptyPages + pageCounter - fragments[
f].second.pageCounter - 1;
293 for (uint32_t k2 = fragments[
f].second.zsPtrLast - 1; k2 <= k; k2++) {
294 for (uint32_t l2 = ((int32_t)k2 == fragments[
f].second.zsPtrLast - 1) ? fragments[
f].second.zsPageLast : 0; l2 < (k2 < k ?
mIOPtrs.
tpcZS->
sector[iSector].
nZSPtr[
j][k2] : l); l2++) {
296 mCFContext->fragmentData[
f].pageDigits[iSector][
j].emplace_back(0);
302 const TPCZSHDR*
const hdrTmp = (
const TPCZSHDR*)(rdh_utils::getLink(o2::raw::RDHUtils::getFEEID(*rdhTmp)) == rdh_utils::DLBZSLinkID ? (pageTmp + o2::raw::RDHUtils::getMemorySize(*rdhTmp) -
sizeof(
TPCZSHDRV2)) : (pageTmp +
sizeof(
o2::header::RAWDataHeader)));
308 }
else if (emptyPages) {
309 mCFContext->fragmentData[
f].nPages[iSector][
j] += emptyPages;
311 for (uint32_t
m = 0;
m < emptyPages;
m++) {
312 mCFContext->fragmentData[
f].pageDigits[iSector][
j].emplace_back(0);
317 fragments[
f].second.zsPtrLast = k + 1;
318 fragments[
f].second.zsPageLast = l + 1;
319 fragments[
f].second.pageCounter = pageCounter;
321 mCFContext->fragmentData[
f].nDigits[iSector][
j] += hdr->nADCsamples;
323 mCFContext->fragmentData[
f].pageDigits[iSector][
j].emplace_back(hdr->nADCsamples);
325 fragmentExtends[
f] = extendsInNextPage;
328 if (timeBin < (uint32_t)fragments[
f].
first.last()) {
329 if (
mCFContext->zsVersion >= ZSVersion::ZSVersionDenseLinkBased) {
330 for (uint32_t ff =
f + 1; ff <
mCFContext->nFragments; ff++) {
336 firstPossibleFragment =
f + 1;
344 mCFContext->fragmentData[
f].minMaxCN[iSector][
j].zsPtrLast = fragments[
f].second.zsPtrLast;
345 mCFContext->fragmentData[
f].minMaxCN[iSector][
j].zsPtrFirst = fragments[
f].second.zsPtrFirst;
346 mCFContext->fragmentData[
f].minMaxCN[iSector][
j].zsPageLast = fragments[
f].second.zsPageLast;
347 mCFContext->fragmentData[
f].minMaxCN[iSector][
j].zsPageFirst = fragments[
f].second.zsPageFirst;
355 if (endpointAdcSamples[
i] >
mCFContext->nDigitsEndpointMax[iSector]) {
356 mCFContext->nDigitsEndpointMax[iSector] = endpointAdcSamples[
i];
359 uint32_t nDigitsFragmentMax = 0;
361 uint32_t pagesInFragment = 0;
362 uint32_t digitsInFragment = 0;
364 pagesInFragment +=
mCFContext->fragmentData[
i].nPages[iSector][
j];
365 digitsInFragment +=
mCFContext->fragmentData[
i].nDigits[iSector][
j];
368 nDigitsFragmentMax = std::max(nDigitsFragmentMax, digitsInFragment);
371 return {nDigits, nDigitsFragmentMax};
379 const uint32_t iSector = clusterer.
mISector;
382 std::vector<size_t> counts;
385 if (nSteps > clusterer.
mNBufs) {
386 GPUError(
"Clusterer buffers exceeded (%u > %u)", nSteps, (int32_t)clusterer.
mNBufs);
390 size_t tmpCount =
count;
392 for (uint32_t
i = 1;
i < nSteps;
i++) {
393 counts.push_back(tmpCount);
395 runKernel<GPUTPCCFStreamCompaction, GPUTPCCFStreamCompaction::scanStart>({
GetGrid(tmpCount, clusterer.
mScanWorkGroupSize, lane), {iSector}},
i, stage);
397 runKernel<GPUTPCCFStreamCompaction, GPUTPCCFStreamCompaction::scanUp>({
GetGrid(tmpCount, clusterer.
mScanWorkGroupSize, lane), {iSector}},
i, tmpCount);
402 runKernel<GPUTPCCFStreamCompaction, GPUTPCCFStreamCompaction::scanTop>({
GetGrid(tmpCount, clusterer.
mScanWorkGroupSize, lane), {iSector}}, nSteps, tmpCount);
404 for (uint32_t
i = nSteps - 1;
i > 1;
i--) {
405 tmpCount = counts[
i - 1];
410 runKernel<GPUTPCCFStreamCompaction, GPUTPCCFStreamCompaction::compactDigits>({
GetGrid(
count, clusterer.
mScanWorkGroupSize, lane), {iSector}}, 1, stage, in, out);
415 for (
size_t i = 0;
i < nIn;
i++) {
424std::pair<uint32_t, uint32_t> GPUChainTracking::RunTPCClusterizer_transferZS(int32_t iSector,
const CfFragment& fragment, int32_t lane)
430 const auto&
retVal = TPCClusterizerDecodeZSCountUpdate(iSector, fragment);
434 uint32_t nPagesSector = 0;
453 nPagesSector += nPages;
460int32_t GPUChainTracking::RunTPCClusterizer_prepare(
bool restorePointers)
463 if (restorePointers) {
464 for (uint32_t iSector = 0; iSector <
NSECTORS; iSector++) {
479 mCFContext->tpcMaxTimeBin = maxAllowedTimebin;
486 uint32_t nDigitsFragmentMax[
NSECTORS];
488 for (uint32_t iSector = 0; iSector <
NSECTORS; iSector++) {
492 GPUError(
"Data has invalid RDH version %d, %d required\n",
o2::raw::RDHUtils::getVersion(rdh), o2::raw::RDHUtils::getVersion<o2::header::RAWDataHeader>());
508 const auto&
x = TPCClusterizerDecodeZSCount(iSector, fragmentMax);
509 nDigitsFragmentMax[iSector] =
x.first;
513 for (uint32_t iSector = 0; iSector <
NSECTORS; iSector++) {
514 uint32_t nDigitsBase = nDigitsFragmentMax[iSector];
515 uint32_t threshold = 40000000;
516 uint32_t nDigitsScaled = nDigitsBase > threshold ? nDigitsBase : std::min((threshold + nDigitsBase) / 2, 2 * nDigitsBase);
530 for (uint32_t iSector = 0; iSector <
NSECTORS; iSector++) {
543 if (
mCFContext->tpcMaxTimeBin > maxAllowedTimebin) {
544 GPUError(
"Input data has invalid time bin %u > %d",
mCFContext->tpcMaxTimeBin, maxAllowedTimebin);
547 mCFContext->tpcMaxTimeBin = maxAllowedTimebin;
561 for (uint32_t iSector = 0; iSector <
NSECTORS; iSector++) {
573 if (
param().
rec.fwdTPCDigitsAsClusters) {
576#ifdef GPUCA_TPC_GEOMETRY_O2
581 if (RunTPCClusterizer_prepare(mPipelineNotifyCtx &&
GetProcessingSettings().doublePipelineClusterizer)) {
589 float tpcHitLowOccupancyScalingFactor = 1.f;
597 if (nHitsBase < threshold) {
600 tpcHitLowOccupancyScalingFactor = std::min(3.5f, (
float)threshold / nHitsBase);
603 for (uint32_t iSector = 0; iSector <
NSECTORS; iSector++) {
607 for (uint32_t iSector = 0; iSector <
NSECTORS; iSector++) {
611 RunTPCClusterizer_prepare(
true);
616 uint32_t maxClusters = -1;
617 for (uint32_t iSector = 0; iSector <
NSECTORS; iSector++) {
618 maxClusters = std::max(maxClusters,
processors()->tpcClusterer[iSector].mNMaxClusters);
620 for (uint32_t iSector = 0; iSector <
NSECTORS; iSector++) {
628 clustererNN.
nnClusterizerElementSize = ((2 * nn_settings.nnClusterizerSizeInputRow + 1) * (2 * nn_settings.nnClusterizerSizeInputPad + 1) * (2 * nn_settings.nnClusterizerSizeInputTime + 1)) + (nn_settings.nnClusterizerAddIndexData ? 3 : 0);
637 if (nn_settings.nnClusterizerVerbosity < 0) {
642 clustererNN.
nnClusterizerDtype = nn_settings.nnInferenceDtype.find(
"32") != std::string::npos;
657 size_t nClsTotal = 0;
660 std::unique_ptr<ClusterNative[]> tmpNativeClusterBuffer;
667 bool buildNativeGPU = doGPU && NeedTPCClustersOnGPU();
671 if (buildNativeGPU) {
675 if (mWaitForFinalInputs) {
676 GPUFatal(
"Cannot use waitForFinalInput callback without delayed output");
680 tmpNativeClusters =
mInputsHost->mPclusterNativeOutput;
682 tmpNativeClusterBuffer = std::make_unique<ClusterNative[]>(
mInputsHost->mNClusterNative);
683 tmpNativeClusters = tmpNativeClusterBuffer.get();
688 if (propagateMCLabels) {
694 int8_t transferRunning[
NSECTORS] = {0};
697 auto notifyForeignChainFinished = [
this]() {
698 if (mPipelineNotifyCtx) {
701 std::lock_guard<std::mutex> lock(mPipelineNotifyCtx->
mutex);
702 mPipelineNotifyCtx->
ready =
true;
704 mPipelineNotifyCtx->
cond.notify_one();
707 bool synchronizeCalibUpdate =
false;
713 for (
CfFragment fragment =
mCFContext->fragmentFirst; !fragment.isEnd(); fragment = fragment.next()) {
715 GPUInfo(
"Processing time bins [%d, %d) for sectors %d to %d", fragment.
start, fragment.last(), iSectorBase, iSectorBase +
GetProcessingSettings().nTPCClustererLanes - 1);
718 if (doGPU && fragment.
index != 0) {
719 SynchronizeStream(lane);
722 uint32_t iSector = iSectorBase + lane;
730 bool setDigitsOnHost = (not doGPU && not
mIOPtrs.
tpcZS) || propagateMCLabels;
732 size_t numDigits = inDigits->
nTPCDigits[iSector];
733 if (setDigitsOnGPU) {
734 GPUMemCpy(RecoStep::TPCClusterFinding, clustererShadow.
mPdigits, inDigits->tpcDigits[iSector],
sizeof(clustererShadow.
mPdigits[0]) * numDigits, lane,
true);
736 if (setDigitsOnHost) {
752 using ChargeMapType =
decltype(*clustererShadow.
mPchargeMap);
753 using PeakMapType =
decltype(*clustererShadow.
mPpeakMap);
756 if (fragment.
index == 0) {
778 if (propagateMCLabels && fragment.
index == 0) {
782 GPUFatal(
"MC label container missing, sector %d", iSector);
791 runKernel<GPUTPCCFChargeMapFiller, GPUTPCCFChargeMapFiller::findFragmentStart>({
GetGrid(1, lane), {iSector}},
mIOPtrs.
tpcZS ==
nullptr);
793 }
else if (propagateMCLabels) {
806 GPUFatal(
"Data with invalid TPC ZS mode (%d) received",
mCFContext->zsVersion);
810 runKernel<GPUTPCCFDecodeZS>({
GetGridBlk(nBlocks, lane), {iSector}}, firstHBF);
813 runKernel<GPUTPCCFDecodeZSLink>({
GetGridBlk(nBlocks, lane), {iSector}}, firstHBF);
816 runKernel<GPUTPCCFDecodeZSDenseLink>({
GetGridBlk(nBlocks, lane), {iSector}}, firstHBF);
823 uint32_t iSector = iSectorBase + lane;
829 int32_t nextSector = iSector;
834 if (nextSector < NSECTORS && mIOPtrs.tpcZS && mCFContext->nPagesSector[nextSector] &&
mCFContext->zsVersion != -1 && !
mCFContext->abandonTimeframe) {
850 if (propagateMCLabels) {
854 bool checkForNoisyPads = (
rec()->
GetParam().
rec.tpc.maxTimeBinAboveThresholdIn1000Bin > 0) || (
rec()->
GetParam().
rec.tpc.maxConsecTimeBinAboveThreshold > 0);
855 checkForNoisyPads &= (
rec()->
GetParam().
rec.tpc.noisyPadsQuickCheck ? fragment.
index == 0 :
true);
858 if (checkForNoisyPads) {
861 runKernel<GPUTPCCFCheckPadBaseline>({
GetGridBlk(nBlocks, lane), {iSector}});
869 RunTPCClusterizer_compactPeaks(clusterer, clustererShadow, 0, doGPU, lane);
874 uint32_t iSector = iSectorBase + lane;
883 runKernel<GPUTPCCFNoiseSuppression, GPUTPCCFNoiseSuppression::noiseSuppression>({
GetGrid(clusterer.
mPmemory->
counters.
nPeaks, lane), {iSector}});
889 RunTPCClusterizer_compactPeaks(clusterer, clustererShadow, 1, doGPU, lane);
894 uint32_t iSector = iSectorBase + lane;
902 if (fragment.
index == 0) {
904 if (transferRunning[lane] == 1) {
906 transferRunning[lane] = 2;
926 float time_clusterizer = 0, time_fill = 0;
931 auto start0 = std::chrono::high_resolution_clock::now();
934 auto stop0 = std::chrono::high_resolution_clock::now();
935 auto start1 = std::chrono::high_resolution_clock::now();
951 auto stop1 = std::chrono::high_resolution_clock::now();
953 time_clusterizer += std::chrono::duration_cast<std::chrono::nanoseconds>(stop1 - start1).count() / 1e9;
954 time_fill += std::chrono::duration_cast<std::chrono::nanoseconds>(stop0 - start0).count() / 1e9;
956 auto start1 = std::chrono::high_resolution_clock::now();
960 auto stop1 = std::chrono::high_resolution_clock::now();
961 time_clusterizer += std::chrono::duration_cast<std::chrono::nanoseconds>(stop1 - start1).count() / 1e9;
963 int acceptedClusters = 0;
967 LOG(info) <<
"[NN CF] Apply NN (fragment " << fragment.
index <<
", lane: " << lane <<
", sector: " << iSector <<
"): filling data " << time_fill <<
"s ; clusterizer: " << time_clusterizer <<
"s ; " << clusterer.
mPmemory->
counters.
nClusters <<
" clusters, " << acceptedClusters <<
" accepted. --> " << clusterer.
mPmemory->
counters.
nClusters / (time_fill + time_clusterizer) <<
" clusters/s";
970 GPUFatal(
"Project not compiled with neural network clusterization. Aborting.");
978 if (doGPU && propagateMCLabels) {
991 laneHasData[lane] =
true;
998 size_t nClsFirst = nClsTotal;
999 bool anyLaneHasData =
false;
1000 for (int32_t lane = 0; lane < maxLane; lane++) {
1001 uint32_t iSector = iSectorBase + lane;
1009 if (laneHasData[lane]) {
1010 anyLaneHasData =
true;
1016 clusterer.raiseError(GPUErrors::ERROR_CF_GLOBAL_CLUSTER_OVERFLOW, iSector * 1000 +
j, nClsTotal + clusterer.
mPclusterInRow[
j],
mInputsHost->mNClusterNative);
1019 if (buildNativeGPU) {
1023 }
else if (buildNativeHost) {
1029 if (transferRunning[lane]) {
1033 transferRunning[lane] = 1;
1036 if (not propagateMCLabels || not laneHasData[lane]) {
1037 assert(propagateMCLabels ? mcLinearLabels.
header.size() == nClsTotal :
true);
1045 assert(propagateMCLabels ? mcLinearLabels.
header.size() == nClsTotal :
true);
1047 if (propagateMCLabels) {
1048 for (int32_t lane = 0; lane < maxLane; lane++) {
1052 if (buildNativeHost && buildNativeGPU && anyLaneHasData) {
1054 mOutputQueue.emplace_back(
outputQueueEntry{(
void*)((
char*)&tmpNativeClusters[nClsFirst] - (
char*)&tmpNativeClusters[0]), &
mInputsShadow->mPclusterNativeBuffer[nClsFirst], (nClsTotal - nClsFirst) *
sizeof(tmpNativeClusters[0]), RecoStep::TPCClusterFinding});
1056 GPUMemCpy(RecoStep::TPCClusterFinding, (
void*)&tmpNativeClusters[nClsFirst], (
const void*)&
mInputsShadow->mPclusterNativeBuffer[nClsFirst], (nClsTotal - nClsFirst) *
sizeof(tmpNativeClusters[0]),
mRec->
NStreams() - 1,
false);
1060 if (mWaitForFinalInputs && iSectorBase >= 21 && (int32_t)iSectorBase < 21 +
GetProcessingSettings().nTPCClustererLanes) {
1061 notifyForeignChainFinished();
1063 if (mWaitForFinalInputs && iSectorBase >= 30 && (int32_t)iSectorBase < 30 +
GetProcessingSettings().nTPCClustererLanes) {
1064 mWaitForFinalInputs();
1069 if (transferRunning[
i]) {
1076 if (triggerOutput && triggerOutput->
allocator) {
1085 if (propagateMCLabels) {
1088 std::pair<ConstMCLabelContainer*, ConstMCLabelContainerView*>
buffer;
1091 throw std::runtime_error(
"Cluster MC Label buffer missing");
1094 buffer = {&container->first, &container->second};
1102 assert(propagateMCLabels ? mcLinearLabels.
header.size() == nClsTotal :
true);
1103 assert(propagateMCLabels ? mcLinearLabels.
data.size() >= nClsTotal :
true);
1108 mcLabelsConstView =
buffer.second;
1114 tmpNativeClusters =
mInputsHost->mPclusterNativeOutput;
1120 if (buildNativeHost) {
1126 auto allocator = [
this, &tmpNativeClusters](
size_t size) {
1129 return (tmpNativeClusters = this->
mInputsHost->mPclusterNativeOutput);
1131 RunTPCClusterFilter(tmpNativeAccess, allocator,
false);
1136 if (!mWaitForFinalInputs) {
1137 notifyForeignChainFinished();
1140 if (buildNativeGPU) {
1145 mInputsHost->mPclusterNativeAccess->setOffsetPtrs();
1148 if (doGPU && synchronizeOutput) {
1151 if (doGPU && synchronizeCalibUpdate) {
1160 if (buildNativeGPU) {
1161 GPUMemCpy(RecoStep::TPCClusterFinding, (
void*)
mInputsShadow->mPclusterNativeBuffer, (
const void*)tmpNativeClusters, nClsTotal *
sizeof(tmpNativeClusters[0]), -1,
true);
1166 if (mPipelineNotifyCtx) {
1168 mPipelineNotifyCtx =
nullptr;
#define TPC_MAX_TIME_BIN_TRIGGERED
#define GPUCA_MAX_STREAMS
std::enable_if_t< std::is_signed< T >::value, bool > hasData(const CalArray< T > &cal)
Definitions of TPC Zero Suppression Data Headers.
std::unique_ptr< o2::tpc::ClusterNativeAccess > mClusterNativeAccess
int32_t RunTPCClusterizer(bool synchronizeOutput=true)
std::unique_ptr< GPUTrackingInputProvider > mInputsHost
std::array< GPUOutputControl *, GPUTrackingOutputs::count()> mSubOutputControls
std::unique_ptr< std::ofstream > mDebugFile
std::unique_ptr< GPUTriggerOutputs > mTriggerBuffer
std::vector< outputQueueEntry > mOutputQueue
bool mUpdateNewCalibObjects
std::unique_ptr< GPUTPCCFChainContext > mCFContext
int32_t DoQueuedUpdates(int32_t stream, bool updateSlave=true)
std::unique_ptr< GPUNewCalibValues > mNewCalibValues
GPUTrackingInOutPointers & mIOPtrs
struct o2::gpu::GPUChainTracking::InOutMemory mIOMem
std::unique_ptr< GPUTrackingInputProvider > mInputsShadow
int32_t ForwardTPCDigits()
void RecordMarker(deviceEvent *ev, int32_t stream)
void TransferMemoryResourceLinkToGPU(RecoStep step, int16_t res, int32_t stream=-1, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
void GPUMemCpyAlways(RecoStep step, void *dst, const void *src, size_t size, int32_t stream, int32_t toGPU, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
void GPUMemCpy(RecoStep step, void *dst, const void *src, size_t size, int32_t stream, int32_t toGPU, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
GPUReconstruction::RecoStepField GetRecoStepsGPU() const
virtual std::unique_ptr< gpu_reconstruction_kernels::threadContext > GetThreadContext()
void WriteToConstantMemory(RecoStep step, size_t offset, const void *src, size_t size, int32_t stream=-1, deviceEvent *ev=nullptr)
void ReleaseEvent(deviceEvent ev, bool doGPU=true)
size_t AllocateRegisteredMemory(GPUProcessor *proc)
GPUConstantMem * processors()
static constexpr krnlRunRange krnlRunRangeNone
void SetupGPUProcessor(T *proc, bool allocate)
const GPUSettingsProcessing & GetProcessingSettings() const
void SynchronizeStream(int32_t stream)
GPUReconstructionCPU * mRec
GPUConstantMem * processorsShadow()
krnlExec GetGridAutoStep(int32_t stream, GPUDataTypes::RecoStep st=GPUDataTypes::RecoStep::NoRecoStep)
static constexpr int32_t NSECTORS
void TransferMemoryResourceLinkToHost(RecoStep step, int16_t res, int32_t stream=-1, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
void TransferMemoryResourcesToHost(RecoStep step, GPUProcessor *proc, int32_t stream=-1, bool all=false)
bool DoDebugAndDump(RecoStep step, int32_t mask, T &processor, S T::*func, Args &&... args)
krnlExec GetGrid(uint32_t totalItems, uint32_t nThreads, int32_t stream, GPUReconstruction::krnlDeviceType d=GPUReconstruction::krnlDeviceType::Auto, GPUDataTypes::RecoStep st=GPUDataTypes::RecoStep::NoRecoStep)
krnlExec GetGridBlk(uint32_t nBlocks, int32_t stream, GPUReconstruction::krnlDeviceType d=GPUReconstruction::krnlDeviceType::Auto, GPUDataTypes::RecoStep st=GPUDataTypes::RecoStep::NoRecoStep)
GPUReconstruction * rec()
HighResTimer & getGeneralStepTimer(GeneralStep step)
void runParallelOuterLoop(bool doGPU, uint32_t nThreads, std::function< void(uint32_t)> lambda)
void SetNActiveThreads(int32_t n)
void SetNActiveThreadsOuterLoop(uint32_t f)
void AllocateRegisteredForeignMemory(int16_t res, GPUReconstruction *rec, GPUOutputControl *control=nullptr)
void PopNonPersistentMemory(RecoStep step, uint64_t tag)
void ComputeReuseMax(GPUProcessor *proc)
const GPUParam & GetParam() const
RecoStepField GetRecoStepsGPU() const
void UnblockStackedMemory()
uint32_t NStreams() const
void PushNonPersistentMemory(uint64_t tag)
InOutTypeField GetRecoStepsOutputs() const
GPUMemorySizeScalers * MemoryScalers()
static void setGlobalOffsetsAndAllocate(GPUTPCClusterFinder &, GPUTPCLinearLabels &)
void DumpDigits(std::ostream &out)
static constexpr int32_t mScanWorkGroupSize
void SetMaxData(const GPUTrackingInOutPointers &io)
void DumpClusters(std::ostream &out)
void SetNMaxDigits(size_t nDigits, size_t nPages, size_t nDigitsFragment, size_t nDigitsEndpointMax)
void DumpSuppressedPeaks(std::ostream &out)
uint32_t mNMaxClusterPerRow
void DumpPeakMap(std::ostream &out, std::string_view)
o2::dataformats::ConstMCTruthContainerView< o2::MCCompLabel > const * mPinputLabels
uint32_t * mPclusterInRow
void DumpChargeMap(std::ostream &out, std::string_view)
uint32_t getNSteps(size_t items) const
ChargePos * mPpeakPositions
ChargePos * mPfilteredPeakPositions
void DumpSuppressedPeaksCompacted(std::ostream &out)
void DumpPeaksCompacted(std::ostream &out)
tpc::ClusterNative * mPclusterByRow
void DumpPeaks(std::ostream &out)
std::vector< std::string > reg_model_paths
void networkInference(o2::ml::OrtModel model, GPUTPCNNClusterizer &clusterer, size_t size, float *output, int32_t dtype)
o2::ml::OrtModel model_reg_2
o2::ml::OrtModel model_reg_1
o2::ml::OrtModel model_class
int nnClusterizerTotalClusters
int nnClusterizerElementSize
int nnClusterizerVerbosity
bool nnSigmoidTrafoClassThreshold
int nnClusterizerUseCfRegression
float * modelProbabilities
int nnClusterizerBatchedMode
bool nnClusterizerAddIndexData
int nnClusterizerSizeInputRow
int nnClusterizerSizeInputTime
int nnClusterizerSizeInputPad
int nnClusterizerBoundaryFillValue
std::vector< std::vector< int64_t > > getNumOutputNodes() const
#define TPC_PADS_IN_SECTOR
typedef void(APIENTRYP PFNGLCULLFACEPROC)(GLenum mode)
uint8_t itsSharedClusterMap uint8_t
constexpr int LHCMaxBunches
void dumpBuffer(gsl::span< const std::byte > buffer, std::ostream &out=std::cout, size_t maxbytes=std::numeric_limits< size_t >::max())
constexpr int LHCBCPERTIMEBIN
constexpr int MAXGLOBALPADROW
Global TPC definitions and constants.
@ ZSVersionDenseLinkBased
@ ZSVersionLinkBasedWithMeta
@ ZSVersionRowBased10BitADC
@ ZSVersionRowBased12BitADC
a couple of static helper functions to create timestamp values for CCDB queries or override obsolete ...
constexpr T qStr2Tag(const char *str)
std::condition_variable cond
std::unique_ptr< o2::dataformats::ConstMCTruthContainerView< o2::MCCompLabel > > clusterNativeMCView
std::unique_ptr< o2::dataformats::ConstMCTruthContainer< o2::MCCompLabel > > clusterNativeMCBuffer
deviceEvent stream[GPUCA_MAX_STREAMS]
GPUTPCClusterFinder tpcClusterer[GPUCA_NSECTORS]
GPUTrackingInOutPointers ioPtrs
size_t NTPCClusters(size_t tpcDigits, bool perSector=false)
std::function< void *(size_t)> allocator
tpccf::SizeT nDigitsInFragment
struct o2::gpu::GPUTPCClusterFinder::Memory::counters_t counters
std::vector< o2::MCCompLabel > data
std::vector< o2::dataformats::MCTruthHeaderElement > header
size_t nTPCDigits[NSECTORS]
const GPUTPCDigitsMCInput * tpcDigitsMC
const o2::tpc::ClusterNativeAccess * clustersNative
const GPUSettingsTF * settingsTF
const GPUTrackingInOutZS * tpcZS
const GPUTrackingInOutDigits * tpcPackedDigits
const void *const * zsPtr[NENDPOINTS]
uint32_t count[NENDPOINTS]
const uint32_t * nZSPtr[NENDPOINTS]
GPUTrackingInOutZSSector sector[NSECTORS]
static constexpr uint32_t NENDPOINTS
GPUOutputControl clustersNative
size_t getIndex(const GPUOutputControl &v)
GPUOutputControl clusterLabels
GPUOutputControl tpcTriggerWords
static constexpr int getVersion()
get numeric version of the RDH
unsigned int nClusters[constants::MAXSECTOR][constants::MAXGLOBALPADROW]
const o2::dataformats::ConstMCTruthContainerView< o2::MCCompLabel > * clustersMCTruth
std::pair< ConstMCLabelContainer, ConstMCLabelContainerView > ConstMCLabelContainerViewWithBuffer
unsigned int nClustersTotal
unsigned int clusterOffset[constants::MAXSECTOR][constants::MAXGLOBALPADROW]
const ClusterNative * clustersLinear
static constexpr unsigned int TRIGGER_WORD_SIZE
static constexpr size_t TPC_ZS_PAGE_SIZE
unsigned short nADCsamples
Trigger info including the orbit.
uint32_t orbit
orbit of the trigger word
TriggerWordDLBZS triggerWord
trigger Word information
bool isValid(int entry=0) const
LOG(info)<< "Compressed in "<< sw.CpuTime()<< " s"
std::vector< Digit > digits