36 RecoStep myStep = RecoStep::TPCCompression;
46 if (gatherMode == 3) {
54 runKernel<GPUTPCCompressionKernels, GPUTPCCompressionKernels::step0attached>(
GetGridAuto(0));
55 runKernel<GPUTPCCompressionKernels, GPUTPCCompressionKernels::step1unattached>(
GetGridAuto(0));
57#ifdef GPUCA_TPC_GEOMETRY_O2
66 memset((
void*)O, 0,
sizeof(*O));
79 if (gatherMode == 3) {
85 int32_t outputStream = 0;
90 if (gatherMode >= 2) {
91 if (gatherMode == 2) {
95 for (uint32_t
i = 0;
i <
sizeof(ptrs) /
sizeof(
void*);
i++) {
96 reinterpret_cast<char**
>(&ptrs)[
i] =
reinterpret_cast<char**
>(&ptrs)[
i] + (
reinterpret_cast<char*
>(devicePtr) -
reinterpret_cast<char*
>(Compressor.
mOutputFlat));
101 constexpr uint32_t nBlocksDefault = 2;
102 constexpr uint32_t nBlocksMulti = 1 + 2 * 200;
104 switch (gatherModeKernel) {
106 runKernel<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::unbuffered>(
GetGridBlkStep(nBlocksDefault, outputStream, RecoStep::TPCCompression));
107 getKernelTimer<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::unbuffered>(RecoStep::TPCCompression, 0, outputSize,
false);
110 runKernel<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::buffered32>(
GetGridBlkStep(nBlocksDefault, outputStream, RecoStep::TPCCompression));
111 getKernelTimer<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::buffered32>(RecoStep::TPCCompression, 0, outputSize,
false);
114 runKernel<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::buffered64>(
GetGridBlkStep(nBlocksDefault, outputStream, RecoStep::TPCCompression));
115 getKernelTimer<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::buffered64>(RecoStep::TPCCompression, 0, outputSize,
false);
118 runKernel<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::buffered128>(
GetGridBlkStep(nBlocksDefault, outputStream, RecoStep::TPCCompression));
119 getKernelTimer<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::buffered128>(RecoStep::TPCCompression, 0, outputSize,
false);
122 static_assert((nBlocksMulti & 1) && nBlocksMulti >= 3);
123 runKernel<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::multiBlock>(
GetGridBlkStep(nBlocksMulti, outputStream, RecoStep::TPCCompression));
124 getKernelTimer<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::multiBlock>(RecoStep::TPCCompression, 0, outputSize,
false);
127 GPUError(
"Invalid compression kernel %d selected.", (int32_t)gatherModeKernel);
130 if (gatherMode == 3) {
134 const size_t blockSize = CAMath::nextMultipleOf<1024>(copySize / 30);
135 const uint32_t
n = (copySize + blockSize - 1) / blockSize;
136 for (uint32_t
i = 0;
i <
n;
i++) {
137 GPUMemCpy(myStep, hostFlatPtr +
i * blockSize, deviceFlatPts +
i * blockSize, CAMath::Min(blockSize, copySize -
i * blockSize), outputStream,
false);
140 GPUMemCpy(myStep, hostFlatPtr, deviceFlatPts, copySize, outputStream,
false);
144 int8_t direction = 0;
145 if (gatherMode == 0) {
146 P = &CompressorShadow.
mPtrs;
147 }
else if (gatherMode == 1) {
150 gatherTimer = &getTimer<GPUTPCCompressionKernels>(
"GPUTPCCompression_GatherOnCPU", 0);
151 gatherTimer->
Start();
191 if (gatherMode == 1) {
195 if (gatherMode == 3) {
200 if (mPipelineFinalizationCtx ==
nullptr) {
219 GPUFatal(
"tpcApplyCFCutsAtDecoding, tpcApplyClusterFilterOnCPU and tpcCutTimeBin currently require tpcUseOldCPUDecoding");
223 const bool runFiltering = needFullFiltering || runTimeBinCutFiltering;
226 auto allocatorFinal = [
this](
size_t size) {
231 std::unique_ptr<ClusterNative[]> tmpBuffer;
232 auto allocatorTmp = [&tmpBuffer](
size_t size) {
235 auto& decompressTimer = getTimer<TPCClusterDecompressor>(
"TPCDecompression", 0);
236 auto allocatorUse = runFiltering ? std::function<
ClusterNative*(size_t)>{allocatorTmp} : std::function<
ClusterNative*(size_t)>{allocatorFinal};
237 decompressTimer.Start();
239 GPUError(
"Error decompressing clusters");
245 decompressTimer.Stop();
253 mInputsHost->mPclusterNativeAccess->setOffsetPtrs();
260 RecoStep myStep = RecoStep::TPCDecompression;
270 throw std::runtime_error(
"Configured solenoid Bz does not match value used for track model encoding");
273 throw std::runtime_error(
"Configured max time bin does not match value used for track model encoding");
276 int32_t inputStream = 0;
278 inputGPU = cmprClsHost;
281 inputGPU = cmprClsHost;
288 for (int32_t iStream = 0; iStream < nStreams; iStream++) {
289 uint32_t startTrack = cmprClsHost.
nTracks / nStreams * iStream;
290 uint32_t endTrack = cmprClsHost.
nTracks / nStreams * (iStream + 1) + (iStream < nStreams - 1 ? 0 : cmprClsHost.
nTracks % nStreams);
291 uint32_t numTracks = endTrack - startTrack;
294 uint32_t numClustersRed = numClusters - numTracks;
306 GPUMemCpy(myStep, inputGPUShadow.
qPtA + startTrack, cmprClsHost.
qPtA + startTrack, numTracks *
sizeof(cmprClsHost.
qPtA[0]), iStream, toGPU);
307 GPUMemCpy(myStep, inputGPUShadow.
rowA + startTrack, cmprClsHost.
rowA + startTrack, numTracks *
sizeof(cmprClsHost.
rowA[0]), iStream, toGPU);
308 GPUMemCpy(myStep, inputGPUShadow.
sliceA + startTrack, cmprClsHost.
sliceA + startTrack, numTracks *
sizeof(cmprClsHost.
sliceA[0]), iStream, toGPU);
309 GPUMemCpy(myStep, inputGPUShadow.
timeA + startTrack, cmprClsHost.
timeA + startTrack, numTracks *
sizeof(cmprClsHost.
timeA[0]), iStream, toGPU);
310 GPUMemCpy(myStep, inputGPUShadow.
padA + startTrack, cmprClsHost.
padA + startTrack, numTracks *
sizeof(cmprClsHost.
padA[0]), iStream, toGPU);
326 uint32_t decodedAttachedClusters = 0;
333 offset += unattachedOffset;
341 if (runTimeBinCutFiltering) {
373 uint32_t batchSize = doGPU ? 6 :
NSECTORS;
374 for (uint32_t iSector = 0; iSector <
NSECTORS; iSector = iSector + batchSize) {
375 int32_t iStream = (iSector / batchSize) %
mRec->
NStreams();
378 if (!runTimeBinCutFiltering) {
384 if (runTimeBinCutFiltering) {
388 runKernel<GPUTPCDecompressionUtilKernels, GPUTPCDecompressionUtilKernels::countFilteredClusters>(
GetGridAutoStep(unattachedStream, RecoStep::TPCDecompression));
414 runKernel<GPUTPCDecompressionUtilKernels, GPUTPCDecompressionUtilKernels::storeFilteredClusters>(
GetGridAutoStep(unattachedStream, RecoStep::TPCDecompression));
419 runKernel<GPUTPCDecompressionUtilKernels, GPUTPCDecompressionUtilKernels::sortPerSectorRow>(
GetGridAutoStep(unattachedStream, RecoStep::TPCDecompression));