35 RecoStep myStep = RecoStep::TPCCompression;
45 if (gatherMode == 3) {
53 runKernel<GPUTPCCompressionKernels, GPUTPCCompressionKernels::step0attached>(
GetGridAuto(0));
54 runKernel<GPUTPCCompressionKernels, GPUTPCCompressionKernels::step1unattached>(
GetGridAuto(0));
56#ifdef GPUCA_TPC_GEOMETRY_O2
65 memset((
void*)O, 0,
sizeof(*O));
78 if (gatherMode == 3) {
84 int32_t outputStream = 0;
89 if (gatherMode >= 2) {
90 if (gatherMode == 2) {
94 for (uint32_t
i = 0;
i <
sizeof(ptrs) /
sizeof(
void*);
i++) {
95 reinterpret_cast<char**
>(&ptrs)[
i] =
reinterpret_cast<char**
>(&ptrs)[
i] + (
reinterpret_cast<char*
>(devicePtr) -
reinterpret_cast<char*
>(Compressor.
mOutputFlat));
100 constexpr uint32_t nBlocksDefault = 2;
101 constexpr uint32_t nBlocksMulti = 1 + 2 * 200;
103 switch (gatherModeKernel) {
105 runKernel<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::unbuffered>(
GetGridBlkStep(nBlocksDefault, outputStream, RecoStep::TPCCompression));
106 getKernelTimer<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::unbuffered>(RecoStep::TPCCompression, 0, outputSize,
false);
109 runKernel<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::buffered32>(
GetGridBlkStep(nBlocksDefault, outputStream, RecoStep::TPCCompression));
110 getKernelTimer<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::buffered32>(RecoStep::TPCCompression, 0, outputSize,
false);
113 runKernel<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::buffered64>(
GetGridBlkStep(nBlocksDefault, outputStream, RecoStep::TPCCompression));
114 getKernelTimer<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::buffered64>(RecoStep::TPCCompression, 0, outputSize,
false);
117 runKernel<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::buffered128>(
GetGridBlkStep(nBlocksDefault, outputStream, RecoStep::TPCCompression));
118 getKernelTimer<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::buffered128>(RecoStep::TPCCompression, 0, outputSize,
false);
121 static_assert((nBlocksMulti & 1) && nBlocksMulti >= 3);
122 runKernel<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::multiBlock>(
GetGridBlkStep(nBlocksMulti, outputStream, RecoStep::TPCCompression));
123 getKernelTimer<GPUTPCCompressionGatherKernels, GPUTPCCompressionGatherKernels::multiBlock>(RecoStep::TPCCompression, 0, outputSize,
false);
126 GPUError(
"Invalid compression kernel %d selected.", (int32_t)gatherModeKernel);
129 if (gatherMode == 3) {
133 const size_t blockSize = CAMath::nextMultipleOf<1024>(copySize / 30);
134 const uint32_t
n = (copySize + blockSize - 1) / blockSize;
135 for (uint32_t
i = 0;
i <
n;
i++) {
136 GPUMemCpy(myStep, hostFlatPtr +
i * blockSize, deviceFlatPts +
i * blockSize, CAMath::Min(blockSize, copySize -
i * blockSize), outputStream,
false);
139 GPUMemCpy(myStep, hostFlatPtr, deviceFlatPts, copySize, outputStream,
false);
143 int8_t direction = 0;
144 if (gatherMode == 0) {
145 P = &CompressorShadow.
mPtrs;
146 }
else if (gatherMode == 1) {
149 gatherTimer = &getTimer<GPUTPCCompressionKernels>(
"GPUTPCCompression_GatherOnCPU", 0);
150 gatherTimer->
Start();
190 if (gatherMode == 1) {
194 if (gatherMode == 3) {
199 if (mPipelineFinalizationCtx ==
nullptr) {
213 GPUFatal(
"tpcApplyCFCutsAtDecoding, tpcApplyClusterFilterOnCPU and tpcCutTimeBin currently require tpcUseOldCPUDecoding");
217 const bool runFiltering = needFullFiltering || runTimeBinCutFiltering;
220 auto allocatorFinal = [
this](
size_t size) {
225 std::unique_ptr<ClusterNative[]> tmpBuffer;
226 auto allocatorTmp = [&tmpBuffer](
size_t size) {
229 auto& decompressTimer = getTimer<TPCClusterDecompressor>(
"TPCDecompression", 0);
230 auto allocatorUse = runFiltering ? std::function<
ClusterNative*(size_t)>{allocatorTmp} : std::function<
ClusterNative*(size_t)>{allocatorFinal};
231 decompressTimer.Start();
233 GPUError(
"Error decompressing clusters");
239 decompressTimer.Stop();
247 mInputsHost->mPclusterNativeAccess->setOffsetPtrs();
254 RecoStep myStep = RecoStep::TPCDecompression;
264 throw std::runtime_error(
"Configured solenoid Bz does not match value used for track model encoding");
267 throw std::runtime_error(
"Configured max time bin does not match value used for track model encoding");
270 int32_t inputStream = 0;
272 inputGPU = cmprClsHost;
275 inputGPU = cmprClsHost;
282 for (int32_t iStream = 0; iStream < nStreams; iStream++) {
283 uint32_t startTrack = cmprClsHost.
nTracks / nStreams * iStream;
284 uint32_t endTrack = cmprClsHost.
nTracks / nStreams * (iStream + 1) + (iStream < nStreams - 1 ? 0 : cmprClsHost.
nTracks % nStreams);
285 uint32_t numTracks = endTrack - startTrack;
288 uint32_t numClustersRed = numClusters - numTracks;
300 GPUMemCpy(myStep, inputGPUShadow.
qPtA + startTrack, cmprClsHost.
qPtA + startTrack, numTracks *
sizeof(cmprClsHost.
qPtA[0]), iStream, toGPU);
301 GPUMemCpy(myStep, inputGPUShadow.
rowA + startTrack, cmprClsHost.
rowA + startTrack, numTracks *
sizeof(cmprClsHost.
rowA[0]), iStream, toGPU);
302 GPUMemCpy(myStep, inputGPUShadow.
sliceA + startTrack, cmprClsHost.
sliceA + startTrack, numTracks *
sizeof(cmprClsHost.
sliceA[0]), iStream, toGPU);
303 GPUMemCpy(myStep, inputGPUShadow.
timeA + startTrack, cmprClsHost.
timeA + startTrack, numTracks *
sizeof(cmprClsHost.
timeA[0]), iStream, toGPU);
304 GPUMemCpy(myStep, inputGPUShadow.
padA + startTrack, cmprClsHost.
padA + startTrack, numTracks *
sizeof(cmprClsHost.
padA[0]), iStream, toGPU);
320 uint32_t decodedAttachedClusters = 0;
327 offset += unattachedOffset;
335 if (runTimeBinCutFiltering) {
367 uint32_t batchSize = doGPU ? 6 :
NSECTORS;
368 for (uint32_t iSector = 0; iSector <
NSECTORS; iSector = iSector + batchSize) {
369 int32_t iStream = (iSector / batchSize) %
mRec->
NStreams();
372 if (!runTimeBinCutFiltering) {
378 if (runTimeBinCutFiltering) {
382 runKernel<GPUTPCDecompressionUtilKernels, GPUTPCDecompressionUtilKernels::countFilteredClusters>(
GetGridAutoStep(unattachedStream, RecoStep::TPCDecompression));
408 runKernel<GPUTPCDecompressionUtilKernels, GPUTPCDecompressionUtilKernels::storeFilteredClusters>(
GetGridAutoStep(unattachedStream, RecoStep::TPCDecompression));
413 runKernel<GPUTPCDecompressionUtilKernels, GPUTPCDecompressionUtilKernels::sortPerSectorRow>(
GetGridAutoStep(unattachedStream, RecoStep::TPCDecompression));