Project
Loading...
Searching...
No Matches
GPUChainTrackingSectorTracker.cxx
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
14
15#include "GPUChainTracking.h"
17#include "GPULogging.h"
18#include "GPUO2DataTypes.h"
20#include "GPUTPCClusterData.h"
23#include "GPUDefParametersRuntime.h"
34#include "utils/strtag.h"
35#include <fstream>
36
37using namespace o2::gpu;
38
39uint32_t GPUChainTracking::StreamForSector(uint32_t sector) const
40{
41 return sector % mRec->NStreams();
42}
43
44int32_t GPUChainTracking::ExtrapolationTracking(uint32_t iSector, bool blocking)
45{
46 const uint32_t stream = StreamForSector(iSector);
47 runKernel<GPUTPCExtrapolationTracking>({GetGridBlk(256, stream), {iSector}});
48 TransferMemoryResourceLinkToHost(RecoStep::TPCSectorTracking, processors()->tpcTrackers[iSector].MemoryResCommon(), stream);
49 if (blocking) {
51 }
52 return (0);
53}
54
56{
57 if (mRec->GPUStuck()) {
58 GPUWarning("This GPU is stuck, processing of tracking for this event is skipped!");
59 return (1);
60 }
61
62 const auto& threadContext = GetThreadContext();
63
64 int32_t retVal = RunTPCTrackingSectors_internal();
65 if (retVal) {
67 }
68 return (retVal != 0);
69}
70
71int32_t GPUChainTracking::RunTPCTrackingSectors_internal()
72{
73 if (GetProcessingSettings().debugLevel >= 2) {
74 GPUInfo("Running TPC Sector Tracker");
75 }
76 bool doGPU = GetRecoStepsGPU() & RecoStep::TPCSectorTracking;
77 if (!param().par.earlyTpcTransform) {
78 for (uint32_t i = 0; i < NSECTORS; i++) {
80 if (doGPU) {
81 processorsShadow()->tpcTrackers[i].Data().SetClusterData(nullptr, mIOPtrs.clustersNative->nClustersSector[i], mIOPtrs.clustersNative->clusterOffset[i][0]); // TODO: not needed I think, anyway copied in SetupGPUProcessor
82 }
83 }
85 } else {
86 int32_t offset = 0;
87 for (uint32_t i = 0; i < NSECTORS; i++) {
88 processors()->tpcTrackers[i].Data().SetClusterData(mIOPtrs.clusterData[i], mIOPtrs.nClusterData[i], offset);
89 if (doGPU && GetRecoSteps().isSet(RecoStep::TPCConversion)) {
90 processorsShadow()->tpcTrackers[i].Data().SetClusterData(processorsShadow()->tpcConverter.mClusters + processors()->tpcTrackers[i].Data().ClusterIdOffset(), processors()->tpcTrackers[i].NHitsTotal(), processors()->tpcTrackers[i].Data().ClusterIdOffset());
91 }
93 }
95 }
96 GPUInfo("Event has %u TPC Clusters, %d TRD Tracklets", (uint32_t)mRec->MemoryScalers()->nTPCHits, mIOPtrs.nTRDTracklets);
97
98 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
99 processors()->tpcTrackers[iSector].SetMaxData(mIOPtrs); // First iteration to set data sizes
100 }
101 mRec->ComputeReuseMax(nullptr); // Resolve maximums for shared buffers
102 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
103 SetupGPUProcessor(&processors()->tpcTrackers[iSector], false); // Prepare custom allocation for 1st stack level
104 mRec->AllocateRegisteredMemory(processors()->tpcTrackers[iSector].MemoryResSectorScratch());
105 }
107 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
108 SetupGPUProcessor(&processors()->tpcTrackers[iSector], true); // Now we allocate
109 mRec->ResetRegisteredMemoryPointers(&processors()->tpcTrackers[iSector]); // TODO: The above call breaks the GPU ptrs to already allocated memory. This fixes them. Should actually be cleaned up at the source.
111 }
112
113 bool streamInit[GPUCA_MAX_STREAMS] = {false};
114 int32_t streamInitAndOccMap = mRec->NStreams() - 1;
115
116 if (doGPU) {
117 // Copy Tracker Object to GPU Memory
118 if (GetProcessingSettings().debugLevel >= 3) {
119 GPUInfo("Copying Tracker objects to GPU");
120 }
121 if (PrepareProfile()) {
122 return 2;
123 }
124
125 WriteToConstantMemory(RecoStep::TPCSectorTracking, (char*)processors()->tpcTrackers - (char*)processors(), processorsShadow()->tpcTrackers, sizeof(GPUTPCTracker) * NSECTORS, streamInitAndOccMap, &mEvents->init);
126
127 std::fill(streamInit, streamInit + mRec->NStreams(), false);
128 streamInit[streamInitAndOccMap] = true;
129 }
130
131 if (param().rec.tpc.occupancyMapTimeBins || param().rec.tpc.sysClusErrorC12Norm) {
133 }
134 if (param().rec.tpc.occupancyMapTimeBins) {
135 if (doGPU) {
137 }
138 uint32_t* ptr = doGPU ? mInputsShadow->mTPCClusterOccupancyMap : mInputsHost->mTPCClusterOccupancyMap;
139 auto* ptrTmp = (GPUTPCClusterOccupancyMapBin*)mRec->AllocateVolatileMemory(GPUTPCClusterOccupancyMapBin::getTotalSize(param()), doGPU);
140 runKernel<GPUMemClean16>(GetGridAutoStep(streamInitAndOccMap, RecoStep::TPCSectorTracking), ptrTmp, GPUTPCClusterOccupancyMapBin::getTotalSize(param()));
141 runKernel<GPUTPCCreateOccupancyMap, GPUTPCCreateOccupancyMap::fill>(GetGridBlk(GPUCA_NSECTORS * GPUCA_ROW_COUNT, streamInitAndOccMap), ptrTmp);
142 runKernel<GPUTPCCreateOccupancyMap, GPUTPCCreateOccupancyMap::fold>(GetGridBlk(GPUTPCClusterOccupancyMapBin::getNBins(param()), streamInitAndOccMap), ptrTmp, ptr + 2);
144 mInputsHost->mTPCClusterOccupancyMap[1] = param().rec.tpc.occupancyMapTimeBins * 0x10000 + param().rec.tpc.occupancyMapTimeBinsAverage;
145 if (doGPU) {
146 GPUMemCpy(RecoStep::TPCSectorTracking, mInputsHost->mTPCClusterOccupancyMap + 2, mInputsShadow->mTPCClusterOccupancyMap + 2, sizeof(*ptr) * GPUTPCClusterOccupancyMapBin::getNBins(mRec->GetParam()), streamInitAndOccMap, false, &mEvents->init);
147 } else {
148 TransferMemoryResourceLinkToGPU(RecoStep::TPCSectorTracking, mInputsHost->mResourceOccupancyMap, streamInitAndOccMap, &mEvents->init);
149 }
150 }
151 if (param().rec.tpc.occupancyMapTimeBins || param().rec.tpc.sysClusErrorC12Norm) {
152 uint32_t& occupancyTotal = *mInputsHost->mTPCClusterOccupancyMap;
153 occupancyTotal = CAMath::Float2UIntRn(mRec->MemoryScalers()->nTPCHits / (mIOPtrs.settingsTF && mIOPtrs.settingsTF->hasNHBFPerTF ? mIOPtrs.settingsTF->nHBFPerTF : 128));
154 mRec->UpdateParamOccupancyMap(param().rec.tpc.occupancyMapTimeBins ? mInputsHost->mTPCClusterOccupancyMap + 2 : nullptr, doGPU && param().rec.tpc.occupancyMapTimeBins ? mInputsShadow->mTPCClusterOccupancyMap + 2 : nullptr, occupancyTotal, streamInitAndOccMap);
155 }
156
157 int32_t streamMap[NSECTORS];
158
159 bool error = false;
160 mRec->runParallelOuterLoop(doGPU, NSECTORS, [&](uint32_t iSector) {
161 GPUTPCTracker& trk = processors()->tpcTrackers[iSector];
162 GPUTPCTracker& trkShadow = doGPU ? processorsShadow()->tpcTrackers[iSector] : trk;
163 int32_t useStream = StreamForSector(iSector);
164 if (GetProcessingSettings().amdMI100SerializationWorkaround) {
165 SynchronizeStream(useStream); // TODO: Remove this workaround once fixed on MI100
166 }
167
168 if (GetProcessingSettings().debugLevel >= 3) {
169 GPUInfo("Creating Sector Data (Sector %d)", iSector);
170 }
171 TransferMemoryResourcesToGPU(RecoStep::TPCSectorTracking, &trk, useStream);
172 runKernel<GPUTPCCreateTrackingData>({doGPU ? GetGridBlk(GPUCA_ROW_COUNT, useStream) : GetGridAuto(0), {iSector}, {nullptr, streamInit[useStream] ? nullptr : &mEvents->init}}); // TODO: Check why GetGridAuto(0) is much fast on CPU
173 streamInit[useStream] = true;
174 if (GetProcessingSettings().deterministicGPUReconstruction) {
175 runKernel<GPUTPCSectorDebugSortKernels, GPUTPCSectorDebugSortKernels::hitData>({GetGridBlk(GPUCA_ROW_COUNT, useStream), {iSector}});
176 }
177 if (!doGPU && trk.CheckEmptySector() && GetProcessingSettings().debugLevel == 0) {
178 return;
179 }
180
181 if (GetProcessingSettings().debugLevel >= 6) {
182 if ((GetProcessingSettings().debugMask & 63)) {
183 *mDebugFile << "\n\nReconstruction: Sector " << iSector << "/" << NSECTORS << std::endl;
184 }
186 if (doGPU) {
187 TransferMemoryResourcesToHost(RecoStep::TPCSectorTracking, &trk, -1, true);
188 }
190 }
191 }
192
193 runKernel<GPUMemClean16>(GetGridAutoStep(useStream, RecoStep::TPCSectorTracking), trkShadow.Data().HitWeights(), trkShadow.Data().NumberOfHitsPlusAlign() * sizeof(*trkShadow.Data().HitWeights()));
194 runKernel<GPUTPCNeighboursFinder>({GetGridBlk(GPUCA_ROW_COUNT, useStream), {iSector}, {nullptr, streamInit[useStream] ? nullptr : &mEvents->init}});
195 streamInit[useStream] = true;
196
197 if (GetProcessingSettings().keepDisplayMemory) {
198 TransferMemoryResourcesToHost(RecoStep::TPCSectorTracking, &trk, -1, true);
199 memcpy(trk.LinkTmpMemory(), mRec->Res(trk.MemoryResLinks()).Ptr(), mRec->Res(trk.MemoryResLinks()).Size());
201 trk.DumpLinks(*mDebugFile, 0);
202 }
203 }
204
205 runKernel<GPUTPCNeighboursCleaner>({GetGridBlk(GPUCA_ROW_COUNT - 2, useStream), {iSector}});
207
208 runKernel<GPUTPCStartHitsFinder>({GetGridBlk(GPUCA_ROW_COUNT - 6, useStream), {iSector}});
209 if (mRec->getGPUParameters(doGPU).par_SORT_STARTHITS) {
210 runKernel<GPUTPCStartHitsSorter>({GetGridAuto(useStream), {iSector}});
211 }
212 if (GetProcessingSettings().deterministicGPUReconstruction) {
213 runKernel<GPUTPCSectorDebugSortKernels, GPUTPCSectorDebugSortKernels::startHits>({GetGrid(1, 1, useStream), {iSector}});
214 }
216
217 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_INDIVIDUAL) {
218 trk.UpdateMaxData();
221 }
222
223 runKernel<GPUTPCTrackletConstructor>({GetGridAuto(useStream), {iSector}});
225 if (GetProcessingSettings().debugMask & GPUChainTrackingDebugFlags::TPCHitWeights && GetProcessingSettings().deterministicGPUReconstruction < 2) {
227 }
228
229 runKernel<GPUTPCTrackletSelector>({GetGridAuto(useStream), {iSector}});
230 runKernel<GPUTPCExtrapolationTrackingCopyNumbers>({{1, -ThreadCount(), useStream}, {iSector}}, 1);
231 if (GetProcessingSettings().deterministicGPUReconstruction) {
232 runKernel<GPUTPCSectorDebugSortKernels, GPUTPCSectorDebugSortKernels::sectorTracks>({GetGrid(1, 1, useStream), {iSector}});
233 }
234 TransferMemoryResourceLinkToHost(RecoStep::TPCSectorTracking, trk.MemoryResCommon(), useStream, &mEvents->sector[iSector]);
235 streamMap[iSector] = useStream;
236 if (GetProcessingSettings().debugLevel >= 3) {
237 GPUInfo("Sector %u, Number of tracks: %d", iSector, *trk.NTracks());
238 }
240 });
242 if (error) {
243 return (3);
244 }
245
246 if (doGPU || GetProcessingSettings().debugLevel >= 1) {
247 if (param().rec.tpc.extrapolationTracking) {
248 std::vector<bool> blocking(NSECTORS * mRec->NStreams());
249 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
250 for (uint32_t iStream = 0; iStream < mRec->NStreams(); iStream++) {
251 blocking[iSector * mRec->NStreams() + iStream] = StreamForSector(iSector) == iStream;
252 }
253 }
254 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
255 uint32_t tmpSector = GPUTPCExtrapolationTracking::ExtrapolationTrackingSectorOrder(iSector);
256 uint32_t sectorLeft, sectorRight;
257 GPUTPCExtrapolationTracking::ExtrapolationTrackingSectorLeftRight(tmpSector, sectorLeft, sectorRight);
258 if (doGPU && !blocking[tmpSector * mRec->NStreams() + StreamForSector(sectorLeft)]) {
259 StreamWaitForEvents(StreamForSector(tmpSector), &mEvents->sector[sectorLeft]);
260 blocking[tmpSector * mRec->NStreams() + StreamForSector(sectorLeft)] = true;
261 }
262 if (doGPU && !blocking[tmpSector * mRec->NStreams() + StreamForSector(sectorRight)]) {
263 StreamWaitForEvents(StreamForSector(tmpSector), &mEvents->sector[sectorRight]);
264 blocking[tmpSector * mRec->NStreams() + StreamForSector(sectorRight)] = true;
265 }
266 ExtrapolationTracking(tmpSector, false);
267 }
268 }
269 if (doGPU) {
271 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
272 ReleaseEvent(mEvents->sector[iSector]);
273 }
274 }
275 } else {
276 mRec->runParallelOuterLoop(doGPU, NSECTORS, [&](uint32_t iSector) {
277 if (param().rec.tpc.extrapolationTracking) {
278 ExtrapolationTracking(iSector, true);
279 }
280 });
282 }
283
284 if (param().rec.tpc.extrapolationTracking && GetProcessingSettings().debugLevel >= 3) {
285 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
286 GPUInfo("Sector %d - Tracks: Local %d Extrapolated %d - Hits: Local %d Extrapolated %d", iSector,
287 processors()->tpcTrackers[iSector].CommonMemory()->nLocalTracks, processors()->tpcTrackers[iSector].CommonMemory()->nTracks, processors()->tpcTrackers[iSector].CommonMemory()->nLocalTrackHits, processors()->tpcTrackers[iSector].CommonMemory()->nTrackHits);
288 }
289 }
290
291 if (DoProfile()) {
292 return (1);
293 }
294 for (uint32_t i = 0; i < NSECTORS; i++) {
295 mIOPtrs.nSectorTracks[i] = *processors()->tpcTrackers[i].NTracks();
297 mIOPtrs.nSectorClusters[i] = *processors()->tpcTrackers[i].NTrackHits();
298 mIOPtrs.sectorClusters[i] = processors()->tpcTrackers[i].TrackHits();
299 if (GetProcessingSettings().keepDisplayMemory && !GetProcessingSettings().keepAllMemory) {
300 TransferMemoryResourcesToHost(RecoStep::TPCSectorTracking, &processors()->tpcTrackers[i], -1, true);
301 }
302 }
303 if (GetProcessingSettings().debugLevel >= 2) {
304 GPUInfo("TPC Sector Tracker finished");
305 }
306 mRec->PopNonPersistentMemory(RecoStep::TPCSectorTracking, qStr2Tag("TPCSLTRK"));
307 return 0;
308}
int32_t i
#define GPUCA_MAX_STREAMS
int32_t retVal
#define GPUCA_NSECTORS
#define GPUCA_ROW_COUNT
TBranch * ptr
std::unique_ptr< GPUTrackingInputProvider > mInputsHost
std::array< GPUOutputControl *, GPUTrackingOutputs::count()> mSubOutputControls
std::unique_ptr< std::ofstream > mDebugFile
GPUTrackingInOutPointers & mIOPtrs
std::unique_ptr< GPUTrackingInputProvider > mInputsShadow
int32_t ExtrapolationTracking(uint32_t iSector, bool blocking)
void TransferMemoryResourceLinkToGPU(RecoStep step, int16_t res, int32_t stream=-1, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
Definition GPUChain.h:124
void GPUMemCpy(RecoStep step, void *dst, const void *src, size_t size, int32_t stream, int32_t toGPU, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
Definition GPUChain.h:128
bool DoDebugAndDump(RecoStep step, uint32_t mask, T &processor, S T::*func, Args &&... args)
Definition GPUChain.h:230
void SynchronizeGPU()
Definition GPUChain.h:110
GPUReconstruction::RecoStepField GetRecoStepsGPU() const
Definition GPUChain.h:72
GPUReconstruction::RecoStepField GetRecoSteps() const
Definition GPUChain.h:71
void WriteToConstantMemory(RecoStep step, size_t offset, const void *src, size_t size, int32_t stream=-1, deviceEvent *ev=nullptr)
Definition GPUChain.h:127
void ReleaseEvent(deviceEvent ev, bool doGPU=true)
Definition GPUChain.h:111
uint32_t ThreadCount() const
Definition GPUChain.h:215
size_t AllocateRegisteredMemory(GPUProcessor *proc)
Definition GPUChain.h:217
virtual std::unique_ptr< GPUReconstructionProcessing::threadContext > GetThreadContext()
Definition GPUChain.h:109
GPUConstantMem * processors()
Definition GPUChain.h:84
void StreamWaitForEvents(int32_t stream, deviceEvent *evList, int32_t nEvents=1)
Definition GPUChain.h:117
GPUParam & param()
Definition GPUChain.h:87
void SetupGPUProcessor(T *proc, bool allocate)
Definition GPUChain.h:220
const GPUSettingsProcessing & GetProcessingSettings() const
Definition GPUChain.h:76
void SynchronizeStream(int32_t stream)
Definition GPUChain.h:89
GPUReconstructionCPU * mRec
Definition GPUChain.h:79
GPUConstantMem * processorsShadow()
Definition GPUChain.h:85
krnlExec GetGridAutoStep(int32_t stream, GPUDataTypes::RecoStep st=GPUDataTypes::RecoStep::NoRecoStep)
Definition GPUChain.cxx:47
static constexpr int32_t NSECTORS
Definition GPUChain.h:58
void TransferMemoryResourceLinkToHost(RecoStep step, int16_t res, int32_t stream=-1, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
Definition GPUChain.h:125
void TransferMemoryResourcesToHost(RecoStep step, GPUProcessor *proc, int32_t stream=-1, bool all=false)
Definition GPUChain.h:123
krnlExec GetGrid(uint32_t totalItems, uint32_t nThreads, int32_t stream, GPUReconstruction::krnlDeviceType d=GPUReconstruction::krnlDeviceType::Auto, GPUDataTypes::RecoStep st=GPUDataTypes::RecoStep::NoRecoStep)
Definition GPUChain.cxx:21
krnlExec GetGridAuto(int32_t stream, GPUReconstruction::krnlDeviceType d=GPUReconstruction::krnlDeviceType::Auto, GPUDataTypes::RecoStep st=GPUDataTypes::RecoStep::NoRecoStep)
Definition GPUChain.cxx:42
krnlExec GetGridBlk(uint32_t nBlocks, int32_t stream, GPUReconstruction::krnlDeviceType d=GPUReconstruction::krnlDeviceType::Auto, GPUDataTypes::RecoStep st=GPUDataTypes::RecoStep::NoRecoStep)
Definition GPUChain.cxx:32
GPUReconstruction * rec()
Definition GPUChain.h:66
void TransferMemoryResourcesToGPU(RecoStep step, GPUProcessor *proc, int32_t stream=-1, bool all=false)
Definition GPUChain.h:122
void UpdateParamOccupancyMap(const uint32_t *mapHost, const uint32_t *mapGPU, uint32_t occupancyTotal, int32_t stream=-1)
void runParallelOuterLoop(bool doGPU, uint32_t nThreads, std::function< void(uint32_t)> lambda)
const GPUDefParameters & getGPUParameters(bool doGPU) const override
void * AllocateVolatileMemory(size_t size, bool device)
void PopNonPersistentMemory(RecoStep step, uint64_t tag)
void ComputeReuseMax(GPUProcessor *proc)
void ResetRegisteredMemoryPointers(GPUProcessor *proc)
GPUMemoryResource & Res(int16_t num)
const GPUParam & GetParam() const
void PushNonPersistentMemory(uint64_t tag)
GPUMemorySizeScalers * MemoryScalers()
size_t AllocateRegisteredMemory(GPUProcessor *proc, bool resetCustom=false)
int16_t MemoryResTracklets() const
int16_t MemoryResLinks() const
void SetMaxData(const GPUTrackingInOutPointers &io)
int16_t MemoryResOutput() const
void DumpTrackHits(std::ostream &out)
void DumpLinks(std::ostream &out, int32_t phase)
void DumpStartHits(std::ostream &out)
void DumpHitWeights(std::ostream &out)
int16_t MemoryResCommon() const
void DumpTrackingData(std::ostream &out)
void DumpTrackletHits(std::ostream &out)
GLintptr offset
Definition glcorearb.h:660
GLenum GLfloat param
Definition glcorearb.h:271
GLuint GLuint stream
Definition glcorearb.h:1806
constexpr T qStr2Tag(const char *str)
Definition strtag.h:22
GPUTPCTracker tpcTrackers[GPUCA_NSECTORS]
const GPUTPCHitId * sectorClusters[NSECTORS]
const o2::tpc::ClusterNativeAccess * clustersNative
const GPUTPCTrack * sectorTracks[NSECTORS]
const GPUSettingsTF * settingsTF
const GPUTPCClusterData * clusterData[NSECTORS]
size_t getIndex(const GPUOutputControl &v)
unsigned int nClustersSector[constants::MAXSECTOR]
unsigned int clusterOffset[constants::MAXSECTOR][constants::MAXGLOBALPADROW]