Project
Loading...
Searching...
No Matches
GPUChainTrackingCompression.cxx
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
14
15#include "GPUChainTracking.h"
17#include "GPULogging.h"
18#include "GPUO2DataTypes.h"
22#include "GPUDefParametersRuntime.h"
23#include "GPUConstantMem.h" // TODO: Try to get rid of as many GPUConstantMem includes as possible!
26#include "utils/strtag.h"
27
28#include <numeric>
29
30using namespace o2::gpu;
31using namespace o2::tpc;
32
34{
36 RecoStep myStep = RecoStep::TPCCompression;
37 bool doGPU = GetRecoStepsGPU() & RecoStep::TPCCompression;
38 int32_t gatherMode = mRec->GetProcessingSettings().tpcCompressionGatherMode == -1 ? mRec->getGPUParameters(doGPU).par_COMP_GATHER_MODE : mRec->GetProcessingSettings().tpcCompressionGatherMode;
40 GPUTPCCompression& CompressorShadow = doGPU ? processorsShadow()->tpcCompressor : Compressor;
41 const auto& threadContext = GetThreadContext();
42 if (mPipelineFinalizationCtx && GetProcessingSettings().doublePipelineClusterizer) {
44 }
45
46 if (gatherMode == 3) {
48 }
49 SetupGPUProcessor(&Compressor, true);
50 new (Compressor.mMemory) GPUTPCCompression::memory;
51 WriteToConstantMemory(myStep, (char*)&processors()->tpcCompressor - (char*)processors(), &CompressorShadow, sizeof(CompressorShadow), 0);
52 TransferMemoryResourcesToGPU(myStep, &Compressor, 0);
53 runKernel<GPUMemClean16>(GetGridAutoStep(0, RecoStep::TPCCompression), CompressorShadow.mClusterStatus, Compressor.mMaxClusters * sizeof(CompressorShadow.mClusterStatus[0]));
54 runKernel<GPUTPCCompressionKernels, GPUTPCCompressionKernels::step0attached>(GetGridAuto(0));
55 runKernel<GPUTPCCompressionKernels, GPUTPCCompressionKernels::step1unattached>(GetGridAuto(0));
56 TransferMemoryResourcesToHost(myStep, &Compressor, 0);
57#ifdef GPUCA_TPC_GEOMETRY_O2
58 if (mPipelineFinalizationCtx && GetProcessingSettings().doublePipelineClusterizer) {
60 ((GPUChainTracking*)GetNextChainInQueue())->RunTPCClusterizer_prepare(false);
62 }
63#endif
66 memset((void*)O, 0, sizeof(*O));
67 O->nTracks = Compressor.mMemory->nStoredTracks;
72 O->nComppressionModes = param().rec.tpc.compressionTypeMask;
73 O->solenoidBz = param().bzkG;
76 Compressor.mOutputFlat->set(outputSize, *Compressor.mOutput);
77 char* hostFlatPtr = (char*)Compressor.mOutput->qTotU; // First array as allocated in GPUTPCCompression::SetPointersCompressedClusters
78 size_t copySize = 0;
79 if (gatherMode == 3) {
80 CompressorShadow.mOutputA = Compressor.mOutput;
81 copySize = AllocateRegisteredMemory(Compressor.mMemoryResOutputGPU); // We overwrite Compressor.mOutput with the allocated output pointers on the GPU
82 }
83 const o2::tpc::CompressedClustersPtrs* P = nullptr;
84 HighResTimer* gatherTimer = nullptr;
85 int32_t outputStream = 0;
86 if (GetProcessingSettings().doublePipeline) {
87 SynchronizeStream(OutputStream()); // Synchronize output copies running in parallel from memory that might be released, only the following async copy from stacked memory is safe after the chain finishes.
88 outputStream = OutputStream();
89 }
90 if (gatherMode >= 2) {
91 if (gatherMode == 2) {
92 void* devicePtr = mRec->getGPUPointer(Compressor.mOutputFlat);
93 if (devicePtr != Compressor.mOutputFlat) {
94 CompressedClustersPtrs& ptrs = *Compressor.mOutput; // We need to update the ptrs with the gpu-mapped version of the host address space
95 for (uint32_t i = 0; i < sizeof(ptrs) / sizeof(void*); i++) {
96 reinterpret_cast<char**>(&ptrs)[i] = reinterpret_cast<char**>(&ptrs)[i] + (reinterpret_cast<char*>(devicePtr) - reinterpret_cast<char*>(Compressor.mOutputFlat));
97 }
98 }
99 }
100 TransferMemoryResourcesToGPU(myStep, &Compressor, outputStream);
101 constexpr uint32_t nBlocksDefault = 2;
102 constexpr uint32_t nBlocksMulti = 1 + 2 * 200;
103 int32_t gatherModeKernel = mRec->GetProcessingSettings().tpcCompressionGatherModeKernel == -1 ? mRec->getGPUParameters(doGPU).par_COMP_GATHER_KERNEL : mRec->GetProcessingSettings().tpcCompressionGatherMode;
104 switch (gatherModeKernel) {
105 case 0:
106 runKernel<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::unbuffered>(GetGridBlkStep(nBlocksDefault, outputStream, RecoStep::TPCCompression));
107 getKernelTimer<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::unbuffered>(RecoStep::TPCCompression, 0, outputSize, false);
108 break;
109 case 1:
110 runKernel<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::buffered32>(GetGridBlkStep(nBlocksDefault, outputStream, RecoStep::TPCCompression));
111 getKernelTimer<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::buffered32>(RecoStep::TPCCompression, 0, outputSize, false);
112 break;
113 case 2:
114 runKernel<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::buffered64>(GetGridBlkStep(nBlocksDefault, outputStream, RecoStep::TPCCompression));
115 getKernelTimer<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::buffered64>(RecoStep::TPCCompression, 0, outputSize, false);
116 break;
117 case 3:
118 runKernel<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::buffered128>(GetGridBlkStep(nBlocksDefault, outputStream, RecoStep::TPCCompression));
119 getKernelTimer<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::buffered128>(RecoStep::TPCCompression, 0, outputSize, false);
120 break;
121 case 4:
122 static_assert((nBlocksMulti & 1) && nBlocksMulti >= 3);
123 runKernel<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::multiBlock>(GetGridBlkStep(nBlocksMulti, outputStream, RecoStep::TPCCompression));
124 getKernelTimer<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::multiBlock>(RecoStep::TPCCompression, 0, outputSize, false);
125 break;
126 default:
127 GPUError("Invalid compression kernel %d selected.", (int32_t)gatherModeKernel);
128 return 1;
129 }
130 if (gatherMode == 3) {
131 RecordMarker(&mEvents->stream[outputStream], outputStream);
132 char* deviceFlatPts = (char*)Compressor.mOutput->qTotU;
133 if (GetProcessingSettings().doublePipeline) {
134 const size_t blockSize = CAMath::nextMultipleOf<1024>(copySize / 30);
135 const uint32_t n = (copySize + blockSize - 1) / blockSize;
136 for (uint32_t i = 0; i < n; i++) {
137 GPUMemCpy(myStep, hostFlatPtr + i * blockSize, deviceFlatPts + i * blockSize, CAMath::Min(blockSize, copySize - i * blockSize), outputStream, false);
138 }
139 } else {
140 GPUMemCpy(myStep, hostFlatPtr, deviceFlatPts, copySize, outputStream, false);
141 }
142 }
143 } else {
144 int8_t direction = 0;
145 if (gatherMode == 0) {
146 P = &CompressorShadow.mPtrs;
147 } else if (gatherMode == 1) {
148 P = &Compressor.mPtrs;
149 direction = -1;
150 gatherTimer = &getTimer<GPUTPCCompressionKernels>("GPUTPCCompression_GatherOnCPU", 0);
151 gatherTimer->Start();
152 }
153 GPUMemCpyAlways(myStep, O->nSliceRowClusters, P->nSliceRowClusters, NSECTORS * GPUCA_ROW_COUNT * sizeof(O->nSliceRowClusters[0]), outputStream, direction);
154 GPUMemCpyAlways(myStep, O->nTrackClusters, P->nTrackClusters, O->nTracks * sizeof(O->nTrackClusters[0]), outputStream, direction);
155 SynchronizeStream(outputStream);
156 uint32_t offset = 0;
157 for (uint32_t i = 0; i < NSECTORS; i++) {
158 for (uint32_t j = 0; j < GPUCA_ROW_COUNT; j++) {
159 uint32_t srcOffset = mIOPtrs.clustersNative->clusterOffset[i][j] * Compressor.mMaxClusterFactorBase1024 / 1024;
160 GPUMemCpyAlways(myStep, O->qTotU + offset, P->qTotU + srcOffset, O->nSliceRowClusters[i * GPUCA_ROW_COUNT + j] * sizeof(O->qTotU[0]), outputStream, direction);
161 GPUMemCpyAlways(myStep, O->qMaxU + offset, P->qMaxU + srcOffset, O->nSliceRowClusters[i * GPUCA_ROW_COUNT + j] * sizeof(O->qMaxU[0]), outputStream, direction);
162 GPUMemCpyAlways(myStep, O->flagsU + offset, P->flagsU + srcOffset, O->nSliceRowClusters[i * GPUCA_ROW_COUNT + j] * sizeof(O->flagsU[0]), outputStream, direction);
163 GPUMemCpyAlways(myStep, O->padDiffU + offset, P->padDiffU + srcOffset, O->nSliceRowClusters[i * GPUCA_ROW_COUNT + j] * sizeof(O->padDiffU[0]), outputStream, direction);
164 GPUMemCpyAlways(myStep, O->timeDiffU + offset, P->timeDiffU + srcOffset, O->nSliceRowClusters[i * GPUCA_ROW_COUNT + j] * sizeof(O->timeDiffU[0]), outputStream, direction);
165 GPUMemCpyAlways(myStep, O->sigmaPadU + offset, P->sigmaPadU + srcOffset, O->nSliceRowClusters[i * GPUCA_ROW_COUNT + j] * sizeof(O->sigmaPadU[0]), outputStream, direction);
166 GPUMemCpyAlways(myStep, O->sigmaTimeU + offset, P->sigmaTimeU + srcOffset, O->nSliceRowClusters[i * GPUCA_ROW_COUNT + j] * sizeof(O->sigmaTimeU[0]), outputStream, direction);
168 }
169 }
170 offset = 0;
171 for (uint32_t i = 0; i < O->nTracks; i++) {
172 GPUMemCpyAlways(myStep, O->qTotA + offset, P->qTotA + Compressor.mAttachedClusterFirstIndex[i], O->nTrackClusters[i] * sizeof(O->qTotA[0]), outputStream, direction);
173 GPUMemCpyAlways(myStep, O->qMaxA + offset, P->qMaxA + Compressor.mAttachedClusterFirstIndex[i], O->nTrackClusters[i] * sizeof(O->qMaxA[0]), outputStream, direction);
174 GPUMemCpyAlways(myStep, O->flagsA + offset, P->flagsA + Compressor.mAttachedClusterFirstIndex[i], O->nTrackClusters[i] * sizeof(O->flagsA[0]), outputStream, direction);
175 GPUMemCpyAlways(myStep, O->sigmaPadA + offset, P->sigmaPadA + Compressor.mAttachedClusterFirstIndex[i], O->nTrackClusters[i] * sizeof(O->sigmaPadA[0]), outputStream, direction);
176 GPUMemCpyAlways(myStep, O->sigmaTimeA + offset, P->sigmaTimeA + Compressor.mAttachedClusterFirstIndex[i], O->nTrackClusters[i] * sizeof(O->sigmaTimeA[0]), outputStream, direction);
177
178 // First index stored with track
179 GPUMemCpyAlways(myStep, O->rowDiffA + offset - i, P->rowDiffA + Compressor.mAttachedClusterFirstIndex[i] + 1, (O->nTrackClusters[i] - 1) * sizeof(O->rowDiffA[0]), outputStream, direction);
180 GPUMemCpyAlways(myStep, O->sliceLegDiffA + offset - i, P->sliceLegDiffA + Compressor.mAttachedClusterFirstIndex[i] + 1, (O->nTrackClusters[i] - 1) * sizeof(O->sliceLegDiffA[0]), outputStream, direction);
181 GPUMemCpyAlways(myStep, O->padResA + offset - i, P->padResA + Compressor.mAttachedClusterFirstIndex[i] + 1, (O->nTrackClusters[i] - 1) * sizeof(O->padResA[0]), outputStream, direction);
182 GPUMemCpyAlways(myStep, O->timeResA + offset - i, P->timeResA + Compressor.mAttachedClusterFirstIndex[i] + 1, (O->nTrackClusters[i] - 1) * sizeof(O->timeResA[0]), outputStream, direction);
183 offset += O->nTrackClusters[i];
184 }
185 GPUMemCpyAlways(myStep, O->qPtA, P->qPtA, O->nTracks * sizeof(O->qPtA[0]), outputStream, direction);
186 GPUMemCpyAlways(myStep, O->rowA, P->rowA, O->nTracks * sizeof(O->rowA[0]), outputStream, direction);
187 GPUMemCpyAlways(myStep, O->sliceA, P->sliceA, O->nTracks * sizeof(O->sliceA[0]), outputStream, direction);
188 GPUMemCpyAlways(myStep, O->timeA, P->timeA, O->nTracks * sizeof(O->timeA[0]), outputStream, direction);
189 GPUMemCpyAlways(myStep, O->padA, P->padA, O->nTracks * sizeof(O->padA[0]), outputStream, direction);
190 }
191 if (gatherMode == 1) {
192 gatherTimer->Stop();
193 }
195 if (gatherMode == 3) {
198 }
199
200 if (mPipelineFinalizationCtx == nullptr) {
201 SynchronizeStream(outputStream);
202 } else {
204 }
205 mRec->PopNonPersistentMemory(RecoStep::TPCCompression, qStr2Tag("TPCCOMPR"));
206 if (GetProcessingSettings().deterministicGPUReconstruction) {
209 }
211 return 0;
212}
213
215{
216 const bool needFullFiltering = GetProcessingSettings().tpcApplyCFCutsAtDecoding || (GetProcessingSettings().tpcApplyClusterFilterOnCPU > 0);
217 const bool runTimeBinCutFiltering = param().tpcCutTimeBin > 0;
218 if (needFullFiltering && !GetProcessingSettings().tpcUseOldCPUDecoding) {
219 GPUFatal("tpcApplyCFCutsAtDecoding, tpcApplyClusterFilterOnCPU and tpcCutTimeBin currently require tpcUseOldCPUDecoding");
220 }
221
222 if (GetProcessingSettings().tpcUseOldCPUDecoding) {
223 const bool runFiltering = needFullFiltering || runTimeBinCutFiltering;
224 const auto& threadContext = GetThreadContext();
226 auto allocatorFinal = [this](size_t size) {
227 this->mInputsHost->mNClusterNative = this->mInputsShadow->mNClusterNative = size;
228 this->AllocateRegisteredMemory(this->mInputsHost->mResourceClusterNativeOutput, this->mSubOutputControls[GPUTrackingOutputs::getIndex(&GPUTrackingOutputs::clustersNative)]);
229 return this->mInputsHost->mPclusterNativeOutput;
230 };
231 std::unique_ptr<ClusterNative[]> tmpBuffer;
232 auto allocatorTmp = [&tmpBuffer](size_t size) {
233 return ((tmpBuffer = std::make_unique<ClusterNative[]>(size))).get();
234 };
235 auto& decompressTimer = getTimer<TPCClusterDecompressor>("TPCDecompression", 0);
236 auto allocatorUse = runFiltering ? std::function<ClusterNative*(size_t)>{allocatorTmp} : std::function<ClusterNative*(size_t)>{allocatorFinal};
237 decompressTimer.Start();
238 if (decomp.decompress(mIOPtrs.tpcCompressedClusters, *mClusterNativeAccess, allocatorUse, param(), GetProcessingSettings().deterministicGPUReconstruction)) {
239 GPUError("Error decompressing clusters");
240 return 1;
241 }
242 if (runFiltering) {
243 RunTPCClusterFilter(mClusterNativeAccess.get(), allocatorFinal, GetProcessingSettings().tpcApplyCFCutsAtDecoding);
244 }
245 decompressTimer.Stop();
247 if (mRec->IsGPU()) {
248 AllocateRegisteredMemory(mInputsHost->mResourceClusterNativeBuffer);
249 processorsShadow()->ioPtrs.clustersNative = mInputsShadow->mPclusterNativeAccess;
250 WriteToConstantMemory(RecoStep::TPCDecompression, (char*)&processors()->ioPtrs - (char*)processors(), &processorsShadow()->ioPtrs, sizeof(processorsShadow()->ioPtrs), 0);
251 *mInputsHost->mPclusterNativeAccess = *mIOPtrs.clustersNative;
252 mInputsHost->mPclusterNativeAccess->clustersLinear = mInputsShadow->mPclusterNativeBuffer;
253 mInputsHost->mPclusterNativeAccess->setOffsetPtrs();
254 GPUMemCpy(RecoStep::TPCDecompression, mInputsShadow->mPclusterNativeBuffer, mIOPtrs.clustersNative->clustersLinear, sizeof(mIOPtrs.clustersNative->clustersLinear[0]) * mIOPtrs.clustersNative->nClustersTotal, 0, true);
255 TransferMemoryResourceLinkToGPU(RecoStep::TPCDecompression, mInputsHost->mResourceClusterNativeAccess, 0);
257 }
258 } else {
260 RecoStep myStep = RecoStep::TPCDecompression;
261 bool doGPU = GetRecoStepsGPU() & RecoStep::TPCDecompression;
263 GPUTPCDecompression& DecompressorShadow = doGPU ? processorsShadow()->tpcDecompressor : Decompressor;
264 const auto& threadContext = GetThreadContext();
266 CompressedClusters& inputGPU = Decompressor.mInputGPU;
267 CompressedClusters& inputGPUShadow = DecompressorShadow.mInputGPU;
268
269 if (cmprClsHost.nTracks && cmprClsHost.solenoidBz != -1e6f && cmprClsHost.solenoidBz != param().bzkG) {
270 throw std::runtime_error("Configured solenoid Bz does not match value used for track model encoding");
271 }
272 if (cmprClsHost.nTracks && cmprClsHost.maxTimeBin != -1e6 && cmprClsHost.maxTimeBin != param().continuousMaxTimeBin) {
273 throw std::runtime_error("Configured max time bin does not match value used for track model encoding");
274 }
275
276 int32_t inputStream = 0;
277 int32_t unattachedStream = mRec->NStreams() - 1;
278 inputGPU = cmprClsHost;
279 SetupGPUProcessor(&Decompressor, true);
280 WriteToConstantMemory(myStep, (char*)&processors()->tpcDecompressor - (char*)processors(), &DecompressorShadow, sizeof(DecompressorShadow), inputStream);
281 inputGPU = cmprClsHost;
282
283 bool toGPU = true;
284 runKernel<GPUMemClean16>({GetGridAutoStep(inputStream, RecoStep::TPCDecompression), krnlRunRangeNone, &mEvents->init}, DecompressorShadow.mNativeClustersIndex, NSECTORS * GPUCA_ROW_COUNT * sizeof(DecompressorShadow.mNativeClustersIndex[0]));
285 int32_t nStreams = doGPU ? mRec->NStreams() - 1 : 1;
286 if (cmprClsHost.nAttachedClusters != 0) {
287 std::exclusive_scan(cmprClsHost.nTrackClusters, cmprClsHost.nTrackClusters + cmprClsHost.nTracks, Decompressor.mAttachedClustersOffsets, 0u); // computing clusters offsets for first kernel
288 for (int32_t iStream = 0; iStream < nStreams; iStream++) {
289 uint32_t startTrack = cmprClsHost.nTracks / nStreams * iStream;
290 uint32_t endTrack = cmprClsHost.nTracks / nStreams * (iStream + 1) + (iStream < nStreams - 1 ? 0 : cmprClsHost.nTracks % nStreams); // index of last track (excluded from computation)
291 uint32_t numTracks = endTrack - startTrack;
292 uint32_t* offsets = Decompressor.mAttachedClustersOffsets;
293 uint32_t numClusters = (endTrack == cmprClsHost.nTracks ? offsets[endTrack - 1] + cmprClsHost.nTrackClusters[endTrack - 1] : offsets[endTrack]) - offsets[startTrack];
294 uint32_t numClustersRed = numClusters - numTracks;
295 GPUMemCpy(myStep, DecompressorShadow.mAttachedClustersOffsets + startTrack, Decompressor.mAttachedClustersOffsets + startTrack, numTracks * sizeof(Decompressor.mAttachedClustersOffsets[0]), iStream, toGPU);
296 GPUMemCpy(myStep, inputGPUShadow.nTrackClusters + startTrack, cmprClsHost.nTrackClusters + startTrack, numTracks * sizeof(cmprClsHost.nTrackClusters[0]), iStream, toGPU);
297 GPUMemCpy(myStep, inputGPUShadow.qTotA + offsets[startTrack], cmprClsHost.qTotA + offsets[startTrack], numClusters * sizeof(cmprClsHost.qTotA[0]), iStream, toGPU);
298 GPUMemCpy(myStep, inputGPUShadow.qMaxA + offsets[startTrack], cmprClsHost.qMaxA + offsets[startTrack], numClusters * sizeof(cmprClsHost.qMaxA[0]), iStream, toGPU);
299 GPUMemCpy(myStep, inputGPUShadow.flagsA + offsets[startTrack], cmprClsHost.flagsA + offsets[startTrack], numClusters * sizeof(cmprClsHost.flagsA[0]), iStream, toGPU);
300 GPUMemCpy(myStep, inputGPUShadow.rowDiffA + offsets[startTrack] - startTrack, cmprClsHost.rowDiffA + offsets[startTrack] - startTrack, numClustersRed * sizeof(cmprClsHost.rowDiffA[0]), iStream, toGPU);
301 GPUMemCpy(myStep, inputGPUShadow.sliceLegDiffA + offsets[startTrack] - startTrack, cmprClsHost.sliceLegDiffA + offsets[startTrack] - startTrack, numClustersRed * sizeof(cmprClsHost.sliceLegDiffA[0]), iStream, toGPU);
302 GPUMemCpy(myStep, inputGPUShadow.padResA + offsets[startTrack] - startTrack, cmprClsHost.padResA + offsets[startTrack] - startTrack, numClustersRed * sizeof(cmprClsHost.padResA[0]), iStream, toGPU);
303 GPUMemCpy(myStep, inputGPUShadow.timeResA + offsets[startTrack] - startTrack, cmprClsHost.timeResA + offsets[startTrack] - startTrack, numClustersRed * sizeof(cmprClsHost.timeResA[0]), iStream, toGPU);
304 GPUMemCpy(myStep, inputGPUShadow.sigmaPadA + offsets[startTrack], cmprClsHost.sigmaPadA + offsets[startTrack], numClusters * sizeof(cmprClsHost.sigmaPadA[0]), iStream, toGPU);
305 GPUMemCpy(myStep, inputGPUShadow.sigmaTimeA + offsets[startTrack], cmprClsHost.sigmaTimeA + offsets[startTrack], numClusters * sizeof(cmprClsHost.sigmaTimeA[0]), iStream, toGPU);
306 GPUMemCpy(myStep, inputGPUShadow.qPtA + startTrack, cmprClsHost.qPtA + startTrack, numTracks * sizeof(cmprClsHost.qPtA[0]), iStream, toGPU);
307 GPUMemCpy(myStep, inputGPUShadow.rowA + startTrack, cmprClsHost.rowA + startTrack, numTracks * sizeof(cmprClsHost.rowA[0]), iStream, toGPU);
308 GPUMemCpy(myStep, inputGPUShadow.sliceA + startTrack, cmprClsHost.sliceA + startTrack, numTracks * sizeof(cmprClsHost.sliceA[0]), iStream, toGPU);
309 GPUMemCpy(myStep, inputGPUShadow.timeA + startTrack, cmprClsHost.timeA + startTrack, numTracks * sizeof(cmprClsHost.timeA[0]), iStream, toGPU);
310 GPUMemCpy(myStep, inputGPUShadow.padA + startTrack, cmprClsHost.padA + startTrack, numTracks * sizeof(cmprClsHost.padA[0]), iStream, toGPU);
311 runKernel<GPUTPCDecompressionKernels, GPUTPCDecompressionKernels::step0attached>({GetGridAuto(iStream), krnlRunRangeNone, {&mEvents->stream[iStream], &mEvents->init}}, startTrack, endTrack);
312 }
313 }
314 GPUMemCpy(myStep, inputGPUShadow.nSliceRowClusters, cmprClsHost.nSliceRowClusters, NSECTORS * GPUCA_ROW_COUNT * sizeof(cmprClsHost.nSliceRowClusters[0]), unattachedStream, toGPU);
315 GPUMemCpy(myStep, inputGPUShadow.qTotU, cmprClsHost.qTotU, cmprClsHost.nUnattachedClusters * sizeof(cmprClsHost.qTotU[0]), unattachedStream, toGPU);
316 GPUMemCpy(myStep, inputGPUShadow.qMaxU, cmprClsHost.qMaxU, cmprClsHost.nUnattachedClusters * sizeof(cmprClsHost.qMaxU[0]), unattachedStream, toGPU);
317 GPUMemCpy(myStep, inputGPUShadow.flagsU, cmprClsHost.flagsU, cmprClsHost.nUnattachedClusters * sizeof(cmprClsHost.flagsU[0]), unattachedStream, toGPU);
318 GPUMemCpy(myStep, inputGPUShadow.padDiffU, cmprClsHost.padDiffU, cmprClsHost.nUnattachedClusters * sizeof(cmprClsHost.padDiffU[0]), unattachedStream, toGPU);
319 GPUMemCpy(myStep, inputGPUShadow.timeDiffU, cmprClsHost.timeDiffU, cmprClsHost.nUnattachedClusters * sizeof(cmprClsHost.timeDiffU[0]), unattachedStream, toGPU);
320 GPUMemCpy(myStep, inputGPUShadow.sigmaPadU, cmprClsHost.sigmaPadU, cmprClsHost.nUnattachedClusters * sizeof(cmprClsHost.sigmaPadU[0]), unattachedStream, toGPU);
321 GPUMemCpy(myStep, inputGPUShadow.sigmaTimeU, cmprClsHost.sigmaTimeU, cmprClsHost.nUnattachedClusters * sizeof(cmprClsHost.sigmaTimeU[0]), unattachedStream, toGPU);
322
323 TransferMemoryResourceLinkToHost(RecoStep::TPCDecompression, Decompressor.mResourceTmpIndexes, inputStream, nullptr, mEvents->stream, nStreams);
324 SynchronizeStream(inputStream);
325 uint32_t offset = 0;
326 uint32_t decodedAttachedClusters = 0;
327 for (uint32_t i = 0; i < NSECTORS; i++) {
328 for (uint32_t j = 0; j < GPUCA_ROW_COUNT; j++) {
329 uint32_t linearIndex = i * GPUCA_ROW_COUNT + j;
330 uint32_t unattachedOffset = (linearIndex >= cmprClsHost.nSliceRows) ? 0 : cmprClsHost.nSliceRowClusters[linearIndex];
331 (mClusterNativeAccess->nClusters)[i][j] = Decompressor.mNativeClustersIndex[linearIndex] + unattachedOffset;
332 Decompressor.mUnattachedClustersOffsets[linearIndex] = offset;
333 offset += unattachedOffset;
334 decodedAttachedClusters += Decompressor.mNativeClustersIndex[linearIndex];
335 }
336 }
337 TransferMemoryResourceLinkToGPU(RecoStep::TPCDecompression, Decompressor.mResourceTmpClustersOffsets, inputStream);
338 if (decodedAttachedClusters != cmprClsHost.nAttachedClusters) {
339 GPUWarning("%u / %u clusters failed track model decoding (%f %%)", cmprClsHost.nAttachedClusters - decodedAttachedClusters, cmprClsHost.nAttachedClusters, 100.f * (float)(cmprClsHost.nAttachedClusters - decodedAttachedClusters) / (float)cmprClsHost.nAttachedClusters);
340 }
341 if (runTimeBinCutFiltering) { // If filtering, allocate a temporary buffer and cluster native access in decompressor context
342 Decompressor.mNClusterNativeBeforeFiltering = DecompressorShadow.mNClusterNativeBeforeFiltering = decodedAttachedClusters + cmprClsHost.nUnattachedClusters;
345 mClusterNativeAccess->clustersLinear = DecompressorShadow.mNativeClustersBuffer;
346 mClusterNativeAccess->setOffsetPtrs();
348 WriteToConstantMemory(myStep, (char*)&processors()->tpcDecompressor - (char*)processors(), &DecompressorShadow, sizeof(DecompressorShadow), inputStream);
349 TransferMemoryResourceLinkToGPU(RecoStep::TPCDecompression, Decompressor.mResourceClusterNativeAccess, inputStream, &mEvents->single);
350 } else { // If not filtering, directly allocate the final buffers
351 mInputsHost->mNClusterNative = mInputsShadow->mNClusterNative = cmprClsHost.nAttachedClusters + cmprClsHost.nUnattachedClusters;
353 AllocateRegisteredMemory(mInputsHost->mResourceClusterNativeBuffer);
354 DecompressorShadow.mNativeClustersBuffer = mInputsShadow->mPclusterNativeBuffer;
355 Decompressor.mNativeClustersBuffer = mInputsHost->mPclusterNativeOutput;
356 DecompressorShadow.mClusterNativeAccess = mInputsShadow->mPclusterNativeAccess;
357 Decompressor.mClusterNativeAccess = mInputsHost->mPclusterNativeAccess;
358 WriteToConstantMemory(myStep, (char*)&processors()->tpcDecompressor - (char*)processors(), &DecompressorShadow, sizeof(DecompressorShadow), inputStream);
359 if (doGPU) {
360 mClusterNativeAccess->clustersLinear = mInputsShadow->mPclusterNativeBuffer;
361 mClusterNativeAccess->setOffsetPtrs();
362 *mInputsHost->mPclusterNativeAccess = *mClusterNativeAccess;
363 processorsShadow()->ioPtrs.clustersNative = mInputsShadow->mPclusterNativeAccess;
364 WriteToConstantMemory(RecoStep::TPCDecompression, (char*)&processors()->ioPtrs - (char*)processors(), &processorsShadow()->ioPtrs, sizeof(processorsShadow()->ioPtrs), inputStream);
365 TransferMemoryResourceLinkToGPU(RecoStep::TPCDecompression, mInputsHost->mResourceClusterNativeAccess, inputStream, &mEvents->single);
366 }
368 mClusterNativeAccess->clustersLinear = mInputsHost->mPclusterNativeOutput;
369 mClusterNativeAccess->setOffsetPtrs();
370 *mInputsHost->mPclusterNativeAccess = *mClusterNativeAccess;
371 }
372
373 uint32_t batchSize = doGPU ? 6 : NSECTORS;
374 for (uint32_t iSector = 0; iSector < NSECTORS; iSector = iSector + batchSize) {
375 int32_t iStream = (iSector / batchSize) % mRec->NStreams();
376 runKernel<GPUTPCDecompressionKernels, GPUTPCDecompressionKernels::step1unattached>({GetGridAuto(iStream), krnlRunRangeNone, {nullptr, &mEvents->single}}, iSector, batchSize);
377 uint32_t copySize = std::accumulate(mClusterNativeAccess->nClustersSector + iSector, mClusterNativeAccess->nClustersSector + iSector + batchSize, 0u);
378 if (!runTimeBinCutFiltering) {
379 GPUMemCpy(RecoStep::TPCDecompression, mInputsHost->mPclusterNativeOutput + mClusterNativeAccess->clusterOffset[iSector][0], DecompressorShadow.mNativeClustersBuffer + mClusterNativeAccess->clusterOffset[iSector][0], sizeof(Decompressor.mNativeClustersBuffer[0]) * copySize, iStream, false);
380 }
381 }
383
384 if (runTimeBinCutFiltering) { // If filtering is applied, count how many clusters will remain after filtering and allocate final buffers accordingly
386 WriteToConstantMemory(myStep, (char*)&processors()->tpcDecompressor - (char*)processors(), &DecompressorShadow, sizeof(DecompressorShadow), unattachedStream);
387 runKernel<GPUMemClean16>({GetGridAutoStep(unattachedStream, RecoStep::TPCDecompression), krnlRunRangeNone}, DecompressorShadow.mNClusterPerSectorRow, NSECTORS * GPUCA_ROW_COUNT * sizeof(DecompressorShadow.mNClusterPerSectorRow[0]));
388 runKernel<GPUTPCDecompressionUtilKernels, GPUTPCDecompressionUtilKernels::countFilteredClusters>(GetGridAutoStep(unattachedStream, RecoStep::TPCDecompression));
389 TransferMemoryResourceLinkToHost(RecoStep::TPCDecompression, Decompressor.mResourceNClusterPerSectorRow, unattachedStream);
390 SynchronizeStream(unattachedStream);
391 uint32_t nClustersFinal = std::accumulate(Decompressor.mNClusterPerSectorRow, Decompressor.mNClusterPerSectorRow + inputGPU.nSliceRows, 0u);
392 mInputsHost->mNClusterNative = mInputsShadow->mNClusterNative = nClustersFinal;
394 AllocateRegisteredMemory(mInputsHost->mResourceClusterNativeBuffer);
395 DecompressorShadow.mNativeClustersBuffer = mInputsShadow->mPclusterNativeBuffer;
396 Decompressor.mNativeClustersBuffer = mInputsHost->mPclusterNativeOutput;
397 WriteToConstantMemory(myStep, (char*)&processors()->tpcDecompressor - (char*)processors(), &DecompressorShadow, sizeof(DecompressorShadow), unattachedStream);
398 for (uint32_t i = 0; i < NSECTORS; i++) {
399 for (uint32_t j = 0; j < GPUCA_ROW_COUNT; j++) {
400 mClusterNativeAccess->nClusters[i][j] = Decompressor.mNClusterPerSectorRow[i * GPUCA_ROW_COUNT + j];
401 }
402 }
403 if (doGPU) {
404 mClusterNativeAccess->clustersLinear = mInputsShadow->mPclusterNativeBuffer;
405 mClusterNativeAccess->setOffsetPtrs();
406 *mInputsHost->mPclusterNativeAccess = *mClusterNativeAccess;
407 processorsShadow()->ioPtrs.clustersNative = mInputsShadow->mPclusterNativeAccess;
408 WriteToConstantMemory(RecoStep::TPCDecompression, (char*)&processors()->ioPtrs - (char*)processors(), &processorsShadow()->ioPtrs, sizeof(processorsShadow()->ioPtrs), unattachedStream);
409 TransferMemoryResourceLinkToGPU(RecoStep::TPCDecompression, mInputsHost->mResourceClusterNativeAccess, unattachedStream);
410 }
412 mClusterNativeAccess->clustersLinear = mInputsHost->mPclusterNativeOutput;
413 mClusterNativeAccess->setOffsetPtrs();
414 runKernel<GPUTPCDecompressionUtilKernels, GPUTPCDecompressionUtilKernels::storeFilteredClusters>(GetGridAutoStep(unattachedStream, RecoStep::TPCDecompression));
415 GPUMemCpy(RecoStep::TPCDecompression, mInputsHost->mPclusterNativeOutput, DecompressorShadow.mNativeClustersBuffer, sizeof(Decompressor.mNativeClustersBuffer[0]) * nClustersFinal, unattachedStream, false);
416 SynchronizeStream(unattachedStream);
417 }
418 if (GetProcessingSettings().deterministicGPUReconstruction || GetProcessingSettings().debugLevel >= 4) {
419 runKernel<GPUTPCDecompressionUtilKernels, GPUTPCDecompressionUtilKernels::sortPerSectorRow>(GetGridAutoStep(unattachedStream, RecoStep::TPCDecompression));
421 if (doGPU) {
422 for (uint32_t i = 0; i < NSECTORS; i++) {
423 for (uint32_t j = 0; j < GPUCA_ROW_COUNT; j++) {
424 ClusterNative* begin = mInputsHost->mPclusterNativeOutput + decoded->clusterOffset[i][j];
425 ClusterNative* end = begin + decoded->nClusters[i][j];
426 std::sort(begin, end);
427 }
428 }
429 }
430 SynchronizeStream(unattachedStream);
431 }
432 mRec->PopNonPersistentMemory(RecoStep::TPCDecompression, qStr2Tag("TPCDCMPR"));
433 }
435 return 0;
436}
int32_t i
#define GPUCA_ROW_COUNT
uint32_t j
Definition RawData.h:0
void Start()
Definition timer.cxx:57
void Stop()
Definition timer.cxx:69
static void DebugSortCompressedClusters(o2::tpc::CompressedClustersFlat *cls)
std::unique_ptr< o2::tpc::ClusterNativeAccess > mClusterNativeAccess
static void DumpClusters(std::ostream &out, const o2::tpc::ClusterNativeAccess *clusters)
std::unique_ptr< GPUTrackingInputProvider > mInputsHost
std::array< GPUOutputControl *, GPUTrackingOutputs::count()> mSubOutputControls
std::unique_ptr< std::ofstream > mDebugFile
std::unique_ptr< GPUTPCCFChainContext > mCFContext
GPUTrackingInOutPointers & mIOPtrs
std::unique_ptr< GPUTrackingInputProvider > mInputsShadow
void RecordMarker(deviceEvent *ev, int32_t stream)
Definition GPUChain.h:108
void TransferMemoryResourceLinkToGPU(RecoStep step, int16_t res, int32_t stream=-1, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
Definition GPUChain.h:124
void GPUMemCpyAlways(RecoStep step, void *dst, const void *src, size_t size, int32_t stream, int32_t toGPU, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
Definition GPUChain.h:129
void GPUMemCpy(RecoStep step, void *dst, const void *src, size_t size, int32_t stream, int32_t toGPU, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
Definition GPUChain.h:128
bool DoDebugAndDump(RecoStep step, uint32_t mask, T &processor, S T::*func, Args &&... args)
Definition GPUChain.h:230
void SynchronizeGPU()
Definition GPUChain.h:110
GPUReconstruction::RecoStepField GetRecoStepsGPU() const
Definition GPUChain.h:72
void WriteToConstantMemory(RecoStep step, size_t offset, const void *src, size_t size, int32_t stream=-1, deviceEvent *ev=nullptr)
Definition GPUChain.h:127
GPUChain * GetNextChainInQueue()
Definition GPUChain.h:225
krnlExec GetGridBlkStep(uint32_t nBlocks, int32_t stream, GPUDataTypes::RecoStep st=GPUDataTypes::RecoStep::NoRecoStep)
Definition GPUChain.cxx:37
size_t AllocateRegisteredMemory(GPUProcessor *proc)
Definition GPUChain.h:217
virtual std::unique_ptr< GPUReconstructionProcessing::threadContext > GetThreadContext()
Definition GPUChain.h:109
GPUConstantMem * processors()
Definition GPUChain.h:84
static constexpr krnlRunRange krnlRunRangeNone
Definition GPUChain.h:41
bool DoDebugDump(uint32_t mask, std::function< void(Args &...)> func, Args &... args)
GPUParam & param()
Definition GPUChain.h:87
void SetupGPUProcessor(T *proc, bool allocate)
Definition GPUChain.h:220
const GPUSettingsProcessing & GetProcessingSettings() const
Definition GPUChain.h:76
void SynchronizeStream(int32_t stream)
Definition GPUChain.h:89
GPUReconstructionCPU * mRec
Definition GPUChain.h:79
GPUConstantMem * processorsShadow()
Definition GPUChain.h:85
krnlExec GetGridAutoStep(int32_t stream, GPUDataTypes::RecoStep st=GPUDataTypes::RecoStep::NoRecoStep)
Definition GPUChain.cxx:47
static constexpr int32_t NSECTORS
Definition GPUChain.h:58
void TransferMemoryResourceLinkToHost(RecoStep step, int16_t res, int32_t stream=-1, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
Definition GPUChain.h:125
void TransferMemoryResourcesToHost(RecoStep step, GPUProcessor *proc, int32_t stream=-1, bool all=false)
Definition GPUChain.h:123
krnlExec GetGridAuto(int32_t stream, GPUReconstruction::krnlDeviceType d=GPUReconstruction::krnlDeviceType::Auto, GPUDataTypes::RecoStep st=GPUDataTypes::RecoStep::NoRecoStep)
Definition GPUChain.cxx:42
void SynchronizeEventAndRelease(deviceEvent &ev, bool doGPU=true)
Definition GPUChain.h:92
void TransferMemoryResourcesToGPU(RecoStep step, GPUProcessor *proc, int32_t stream=-1, bool all=false)
Definition GPUChain.h:122
const GPUDefParameters & getGPUParameters(bool doGPU) const override
virtual void * getGPUPointer(void *ptr)
void PopNonPersistentMemory(RecoStep step, uint64_t tag)
void PushNonPersistentMemory(uint64_t tag)
void BlockStackedMemory(GPUReconstruction *rec)
const GPUSettingsProcessing & GetProcessingSettings() const
void DumpCompressedClusters(std::ostream &out)
o2::tpc::CompressedClusters * mOutput
o2::tpc::CompressedClusters * mOutputA
o2::tpc::CompressedClustersPtrs mPtrs
o2::tpc::CompressedClustersFlat * mOutputFlat
o2::tpc::CompressedClusters mInputGPU
o2::tpc::ClusterNative * mNativeClustersBuffer
o2::tpc::ClusterNativeAccess * mClusterNativeAccess
static int32_t decompress(const o2::tpc::CompressedClustersFlat *clustersCompressed, o2::tpc::ClusterNativeAccess &clustersNative, std::function< o2::tpc::ClusterNative *(size_t)> allocator, const GPUParam &param, bool deterministicRec)
GLdouble n
Definition glcorearb.h:1982
GLsizeiptr size
Definition glcorearb.h:659
GLuint GLsizei const GLuint const GLintptr * offsets
Definition glcorearb.h:2595
GLuint GLuint end
Definition glcorearb.h:469
GLintptr offset
Definition glcorearb.h:660
GLenum GLfloat param
Definition glcorearb.h:271
Global TPC definitions and constants.
Definition SimTraits.h:167
Enum< T >::Iterator begin(Enum< T >)
Definition Defs.h:173
constexpr T qStr2Tag(const char *str)
Definition strtag.h:22
deviceEvent stream[GPUCA_MAX_STREAMS]
GPUTPCDecompression tpcDecompressor
GPUTrackingInOutPointers ioPtrs
GPUTPCCompression tpcCompressor
const o2::tpc::ClusterNativeAccess * clustersNative
const o2::tpc::CompressedClustersFlat * tpcCompressedClusters
size_t getIndex(const GPUOutputControl &v)
GPUOutputControl compressedClusters
unsigned int nClusters[constants::MAXSECTOR][constants::MAXGLOBALPADROW]
unsigned int clusterOffset[constants::MAXSECTOR][constants::MAXGLOBALPADROW]
const ClusterNative * clustersLinear
void set(size_t bufferSize, const CompressedClusters &v)