Project
Loading...
Searching...
No Matches
GPUChainTrackingSectorTracker.cxx
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
14
15#include "GPUChainTracking.h"
17#include "GPULogging.h"
18#include "GPUO2DataTypes.h"
22#include "GPUDefParametersRuntime.h"
33#include "utils/strtag.h"
34#include <fstream>
35
36using namespace o2::gpu;
37
38uint32_t GPUChainTracking::StreamForSector(uint32_t sector) const
39{
40 return sector % mRec->NStreams();
41}
42
43int32_t GPUChainTracking::ExtrapolationTracking(uint32_t iSector, bool blocking)
44{
45 const uint32_t stream = StreamForSector(iSector);
46 runKernel<GPUTPCExtrapolationTracking>({GetGridBlk(256, stream), {iSector}});
47 TransferMemoryResourceLinkToHost(RecoStep::TPCSectorTracking, processors()->tpcTrackers[iSector].MemoryResCommon(), stream);
48 if (blocking) {
50 }
51 return (0);
52}
53
55{
56 if (mRec->GPUStuck()) {
57 GPUWarning("This GPU is stuck, processing of tracking for this event is skipped!");
58 return (1);
59 }
60
61 const auto& threadContext = GetThreadContext();
62
63 int32_t retVal = RunTPCTrackingSectors_internal();
64 if (retVal) {
66 }
67 return (retVal != 0);
68}
69
70int32_t GPUChainTracking::RunTPCTrackingSectors_internal()
71{
72 if (GetProcessingSettings().debugLevel >= 2) {
73 GPUInfo("Running TPC Sector Tracker");
74 }
75 bool doGPU = GetRecoStepsGPU() & RecoStep::TPCSectorTracking;
76 for (uint32_t i = 0; i < NSECTORS; i++) {
78 if (doGPU) {
79 processorsShadow()->tpcTrackers[i].Data().SetClusterData(mIOPtrs.clustersNative->nClustersSector[i], mIOPtrs.clustersNative->clusterOffset[i][0]); // TODO: not needed I think, anyway copied in SetupGPUProcessor
80 }
81 }
83 GPUInfo("Event has %u TPC Clusters, %d TRD Tracklets", (uint32_t)mRec->MemoryScalers()->nTPCHits, mIOPtrs.nTRDTracklets);
84
85 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
86 processors()->tpcTrackers[iSector].SetMaxData(mIOPtrs); // First iteration to set data sizes
87 }
88 mRec->ComputeReuseMax(nullptr); // Resolve maximums for shared buffers
89 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
90 SetupGPUProcessor(&processors()->tpcTrackers[iSector], false); // Prepare custom allocation for 1st stack level
91 mRec->AllocateRegisteredMemory(processors()->tpcTrackers[iSector].MemoryResSectorScratch());
92 }
94 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
95 SetupGPUProcessor(&processors()->tpcTrackers[iSector], true); // Now we allocate
96 mRec->ResetRegisteredMemoryPointers(&processors()->tpcTrackers[iSector]); // TODO: The above call breaks the GPU ptrs to already allocated memory. This fixes them. Should actually be cleaned up at the source.
98 }
99
100 bool streamInit[GPUCA_MAX_STREAMS] = {false};
101 int32_t streamInitAndOccMap = mRec->NStreams() - 1;
102
103 if (doGPU) {
104 // Copy Tracker Object to GPU Memory
105 if (GetProcessingSettings().debugLevel >= 3) {
106 GPUInfo("Copying Tracker objects to GPU");
107 }
108 if (PrepareProfile()) {
109 return 2;
110 }
111
112 WriteToConstantMemory(RecoStep::TPCSectorTracking, (char*)processors()->tpcTrackers - (char*)processors(), processorsShadow()->tpcTrackers, sizeof(GPUTPCTracker) * NSECTORS, streamInitAndOccMap, &mEvents->init);
113
114 std::fill(streamInit, streamInit + mRec->NStreams(), false);
115 streamInit[streamInitAndOccMap] = true;
116 }
117
118 if (param().rec.tpc.occupancyMapTimeBins || param().rec.tpc.sysClusErrorC12Norm) {
120 }
121 if (param().rec.tpc.occupancyMapTimeBins) {
122 if (doGPU) {
124 }
125 uint32_t* ptr = doGPU ? mInputsShadow->mTPCClusterOccupancyMap : mInputsHost->mTPCClusterOccupancyMap;
126 auto* ptrTmp = (GPUTPCClusterOccupancyMapBin*)mRec->AllocateVolatileMemory(GPUTPCClusterOccupancyMapBin::getTotalSize(param()), doGPU);
127 runKernel<GPUMemClean16>(GetGridAutoStep(streamInitAndOccMap, RecoStep::TPCSectorTracking), ptrTmp, GPUTPCClusterOccupancyMapBin::getTotalSize(param()));
128 runKernel<GPUTPCCreateOccupancyMap, GPUTPCCreateOccupancyMap::fill>(GetGridBlk(GPUCA_NSECTORS * GPUCA_ROW_COUNT, streamInitAndOccMap), ptrTmp);
129 runKernel<GPUTPCCreateOccupancyMap, GPUTPCCreateOccupancyMap::fold>(GetGridBlk(GPUTPCClusterOccupancyMapBin::getNBins(param()), streamInitAndOccMap), ptrTmp, ptr + 2);
131 mInputsHost->mTPCClusterOccupancyMap[1] = param().rec.tpc.occupancyMapTimeBins * 0x10000 + param().rec.tpc.occupancyMapTimeBinsAverage;
132 if (doGPU) {
133 GPUMemCpy(RecoStep::TPCSectorTracking, mInputsHost->mTPCClusterOccupancyMap + 2, mInputsShadow->mTPCClusterOccupancyMap + 2, sizeof(*ptr) * GPUTPCClusterOccupancyMapBin::getNBins(mRec->GetParam()), streamInitAndOccMap, false, &mEvents->init);
134 } else {
135 TransferMemoryResourceLinkToGPU(RecoStep::TPCSectorTracking, mInputsHost->mResourceOccupancyMap, streamInitAndOccMap, &mEvents->init);
136 }
137 }
138 if (param().rec.tpc.occupancyMapTimeBins || param().rec.tpc.sysClusErrorC12Norm) {
139 uint32_t& occupancyTotal = *mInputsHost->mTPCClusterOccupancyMap;
140 occupancyTotal = CAMath::Float2UIntRn(mRec->MemoryScalers()->nTPCHits / (mIOPtrs.settingsTF && mIOPtrs.settingsTF->hasNHBFPerTF ? mIOPtrs.settingsTF->nHBFPerTF : 128));
141 mRec->UpdateParamOccupancyMap(param().rec.tpc.occupancyMapTimeBins ? mInputsHost->mTPCClusterOccupancyMap + 2 : nullptr, doGPU && param().rec.tpc.occupancyMapTimeBins ? mInputsShadow->mTPCClusterOccupancyMap + 2 : nullptr, occupancyTotal, streamInitAndOccMap);
142 }
143
144 int32_t streamMap[NSECTORS];
145
146 bool error = false;
147 mRec->runParallelOuterLoop(doGPU, NSECTORS, [&](uint32_t iSector) {
148 GPUTPCTracker& trk = processors()->tpcTrackers[iSector];
149 GPUTPCTracker& trkShadow = doGPU ? processorsShadow()->tpcTrackers[iSector] : trk;
150 int32_t useStream = StreamForSector(iSector);
151 if (GetProcessingSettings().amdMI100SerializationWorkaround) {
152 SynchronizeStream(useStream); // TODO: Remove this workaround once fixed on MI100
153 }
154
155 if (GetProcessingSettings().debugLevel >= 3) {
156 GPUInfo("Creating Sector Data (Sector %d)", iSector);
157 }
158 TransferMemoryResourcesToGPU(RecoStep::TPCSectorTracking, &trk, useStream);
159 runKernel<GPUTPCCreateTrackingData>({doGPU ? GetGridBlk(GPUCA_ROW_COUNT, useStream) : GetGridAuto(0), {iSector}, {nullptr, streamInit[useStream] ? nullptr : &mEvents->init}}); // TODO: Check why GetGridAuto(0) is much fast on CPU
160 streamInit[useStream] = true;
161 if (GetProcessingSettings().deterministicGPUReconstruction) {
162 runKernel<GPUTPCSectorDebugSortKernels, GPUTPCSectorDebugSortKernels::hitData>({GetGridBlk(GPUCA_ROW_COUNT, useStream), {iSector}});
163 }
164 if (!doGPU && trk.CheckEmptySector() && GetProcessingSettings().debugLevel == 0) {
165 return;
166 }
167
168 if (GetProcessingSettings().debugLevel >= 6) {
169 if ((GetProcessingSettings().debugMask & 63)) {
170 *mDebugFile << "\n\nReconstruction: Sector " << iSector << "/" << NSECTORS << std::endl;
171 }
173 if (doGPU) {
174 TransferMemoryResourcesToHost(RecoStep::TPCSectorTracking, &trk, -1, true);
175 }
177 }
178 }
179
180 runKernel<GPUMemClean16>(GetGridAutoStep(useStream, RecoStep::TPCSectorTracking), trkShadow.Data().HitWeights(), trkShadow.Data().NumberOfHitsPlusAlign() * sizeof(*trkShadow.Data().HitWeights()));
181 runKernel<GPUTPCNeighboursFinder>({GetGridBlk(GPUCA_ROW_COUNT, useStream), {iSector}, {nullptr, streamInit[useStream] ? nullptr : &mEvents->init}});
182 streamInit[useStream] = true;
183
184 if (GetProcessingSettings().keepDisplayMemory) {
185 TransferMemoryResourcesToHost(RecoStep::TPCSectorTracking, &trk, -1, true);
186 memcpy(trk.LinkTmpMemory(), mRec->Res(trk.MemoryResLinks()).Ptr(), mRec->Res(trk.MemoryResLinks()).Size());
188 trk.DumpLinks(*mDebugFile, 0);
189 }
190 }
191
192 runKernel<GPUTPCNeighboursCleaner>({GetGridBlk(GPUCA_ROW_COUNT - 2, useStream), {iSector}});
194
195 runKernel<GPUTPCStartHitsFinder>({GetGridBlk(GPUCA_ROW_COUNT - 6, useStream), {iSector}});
196 if (mRec->getGPUParameters(doGPU).par_SORT_STARTHITS) {
197 runKernel<GPUTPCStartHitsSorter>({GetGridAuto(useStream), {iSector}});
198 }
199 if (GetProcessingSettings().deterministicGPUReconstruction) {
200 runKernel<GPUTPCSectorDebugSortKernels, GPUTPCSectorDebugSortKernels::startHits>({GetGrid(1, 1, useStream), {iSector}});
201 }
203
204 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_INDIVIDUAL) {
205 trk.UpdateMaxData();
208 }
209
210 runKernel<GPUTPCTrackletConstructor>({GetGridAuto(useStream), {iSector}});
212 if (GetProcessingSettings().debugMask & GPUChainTrackingDebugFlags::TPCHitWeights && GetProcessingSettings().deterministicGPUReconstruction < 2) {
214 }
215
216 runKernel<GPUTPCTrackletSelector>({GetGridAuto(useStream), {iSector}});
217 runKernel<GPUTPCExtrapolationTrackingCopyNumbers>({{1, -ThreadCount(), useStream}, {iSector}}, 1);
218 if (GetProcessingSettings().deterministicGPUReconstruction) {
219 runKernel<GPUTPCSectorDebugSortKernels, GPUTPCSectorDebugSortKernels::sectorTracks>({GetGrid(1, 1, useStream), {iSector}});
220 }
221 TransferMemoryResourceLinkToHost(RecoStep::TPCSectorTracking, trk.MemoryResCommon(), useStream, &mEvents->sector[iSector]);
222 streamMap[iSector] = useStream;
223 if (GetProcessingSettings().debugLevel >= 3) {
224 GPUInfo("Sector %u, Number of tracks: %d", iSector, *trk.NTracks());
225 }
227 });
229 if (error) {
230 return (3);
231 }
232
233 if (doGPU || GetProcessingSettings().debugLevel >= 1) {
234 if (param().rec.tpc.extrapolationTracking) {
235 std::vector<bool> blocking(NSECTORS * mRec->NStreams());
236 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
237 for (uint32_t iStream = 0; iStream < mRec->NStreams(); iStream++) {
238 blocking[iSector * mRec->NStreams() + iStream] = StreamForSector(iSector) == iStream;
239 }
240 }
241 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
242 uint32_t tmpSector = GPUTPCExtrapolationTracking::ExtrapolationTrackingSectorOrder(iSector);
243 uint32_t sectorLeft, sectorRight;
244 GPUTPCExtrapolationTracking::ExtrapolationTrackingSectorLeftRight(tmpSector, sectorLeft, sectorRight);
245 if (doGPU && !blocking[tmpSector * mRec->NStreams() + StreamForSector(sectorLeft)]) {
246 StreamWaitForEvents(StreamForSector(tmpSector), &mEvents->sector[sectorLeft]);
247 blocking[tmpSector * mRec->NStreams() + StreamForSector(sectorLeft)] = true;
248 }
249 if (doGPU && !blocking[tmpSector * mRec->NStreams() + StreamForSector(sectorRight)]) {
250 StreamWaitForEvents(StreamForSector(tmpSector), &mEvents->sector[sectorRight]);
251 blocking[tmpSector * mRec->NStreams() + StreamForSector(sectorRight)] = true;
252 }
253 ExtrapolationTracking(tmpSector, false);
254 }
255 }
256 if (doGPU) {
258 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
259 ReleaseEvent(mEvents->sector[iSector]);
260 }
261 }
262 } else {
263 mRec->runParallelOuterLoop(doGPU, NSECTORS, [&](uint32_t iSector) {
264 if (param().rec.tpc.extrapolationTracking) {
265 ExtrapolationTracking(iSector, true);
266 }
267 });
269 }
270
271 if (param().rec.tpc.extrapolationTracking && GetProcessingSettings().debugLevel >= 3) {
272 for (uint32_t iSector = 0; iSector < NSECTORS; iSector++) {
273 GPUInfo("Sector %d - Tracks: Local %d Extrapolated %d - Hits: Local %d Extrapolated %d", iSector,
274 processors()->tpcTrackers[iSector].CommonMemory()->nLocalTracks, processors()->tpcTrackers[iSector].CommonMemory()->nTracks, processors()->tpcTrackers[iSector].CommonMemory()->nLocalTrackHits, processors()->tpcTrackers[iSector].CommonMemory()->nTrackHits);
275 }
276 }
277
278 if (DoProfile()) {
279 return (1);
280 }
281 for (uint32_t i = 0; i < NSECTORS; i++) {
282 mIOPtrs.nSectorTracks[i] = *processors()->tpcTrackers[i].NTracks();
284 mIOPtrs.nSectorClusters[i] = *processors()->tpcTrackers[i].NTrackHits();
285 mIOPtrs.sectorClusters[i] = processors()->tpcTrackers[i].TrackHits();
286 if (GetProcessingSettings().keepDisplayMemory && !GetProcessingSettings().keepAllMemory) {
287 TransferMemoryResourcesToHost(RecoStep::TPCSectorTracking, &processors()->tpcTrackers[i], -1, true);
288 }
289 }
290 if (GetProcessingSettings().debugLevel >= 2) {
291 GPUInfo("TPC Sector Tracker finished");
292 }
293 mRec->PopNonPersistentMemory(RecoStep::TPCSectorTracking, qStr2Tag("TPCSLTRK"));
294 return 0;
295}
int32_t i
#define GPUCA_MAX_STREAMS
int32_t retVal
#define GPUCA_NSECTORS
#define GPUCA_ROW_COUNT
TBranch * ptr
std::unique_ptr< GPUTrackingInputProvider > mInputsHost
std::array< GPUOutputControl *, GPUTrackingOutputs::count()> mSubOutputControls
std::unique_ptr< std::ofstream > mDebugFile
GPUTrackingInOutPointers & mIOPtrs
std::unique_ptr< GPUTrackingInputProvider > mInputsShadow
int32_t ExtrapolationTracking(uint32_t iSector, bool blocking)
void TransferMemoryResourceLinkToGPU(RecoStep step, int16_t res, int32_t stream=-1, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
Definition GPUChain.h:124
void GPUMemCpy(RecoStep step, void *dst, const void *src, size_t size, int32_t stream, int32_t toGPU, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
Definition GPUChain.h:128
bool DoDebugAndDump(RecoStep step, uint32_t mask, T &processor, S T::*func, Args &&... args)
Definition GPUChain.h:230
void SynchronizeGPU()
Definition GPUChain.h:110
GPUReconstruction::RecoStepField GetRecoStepsGPU() const
Definition GPUChain.h:72
void WriteToConstantMemory(RecoStep step, size_t offset, const void *src, size_t size, int32_t stream=-1, deviceEvent *ev=nullptr)
Definition GPUChain.h:127
void ReleaseEvent(deviceEvent ev, bool doGPU=true)
Definition GPUChain.h:111
uint32_t ThreadCount() const
Definition GPUChain.h:215
size_t AllocateRegisteredMemory(GPUProcessor *proc)
Definition GPUChain.h:217
virtual std::unique_ptr< GPUReconstructionProcessing::threadContext > GetThreadContext()
Definition GPUChain.h:109
GPUConstantMem * processors()
Definition GPUChain.h:84
void StreamWaitForEvents(int32_t stream, deviceEvent *evList, int32_t nEvents=1)
Definition GPUChain.h:117
GPUParam & param()
Definition GPUChain.h:87
void SetupGPUProcessor(T *proc, bool allocate)
Definition GPUChain.h:220
const GPUSettingsProcessing & GetProcessingSettings() const
Definition GPUChain.h:76
void SynchronizeStream(int32_t stream)
Definition GPUChain.h:89
GPUReconstructionCPU * mRec
Definition GPUChain.h:79
GPUConstantMem * processorsShadow()
Definition GPUChain.h:85
krnlExec GetGridAutoStep(int32_t stream, GPUDataTypes::RecoStep st=GPUDataTypes::RecoStep::NoRecoStep)
Definition GPUChain.cxx:47
static constexpr int32_t NSECTORS
Definition GPUChain.h:58
void TransferMemoryResourceLinkToHost(RecoStep step, int16_t res, int32_t stream=-1, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
Definition GPUChain.h:125
void TransferMemoryResourcesToHost(RecoStep step, GPUProcessor *proc, int32_t stream=-1, bool all=false)
Definition GPUChain.h:123
krnlExec GetGrid(uint32_t totalItems, uint32_t nThreads, int32_t stream, GPUReconstruction::krnlDeviceType d=GPUReconstruction::krnlDeviceType::Auto, GPUDataTypes::RecoStep st=GPUDataTypes::RecoStep::NoRecoStep)
Definition GPUChain.cxx:21
krnlExec GetGridAuto(int32_t stream, GPUReconstruction::krnlDeviceType d=GPUReconstruction::krnlDeviceType::Auto, GPUDataTypes::RecoStep st=GPUDataTypes::RecoStep::NoRecoStep)
Definition GPUChain.cxx:42
krnlExec GetGridBlk(uint32_t nBlocks, int32_t stream, GPUReconstruction::krnlDeviceType d=GPUReconstruction::krnlDeviceType::Auto, GPUDataTypes::RecoStep st=GPUDataTypes::RecoStep::NoRecoStep)
Definition GPUChain.cxx:32
GPUReconstruction * rec()
Definition GPUChain.h:66
void TransferMemoryResourcesToGPU(RecoStep step, GPUProcessor *proc, int32_t stream=-1, bool all=false)
Definition GPUChain.h:122
void UpdateParamOccupancyMap(const uint32_t *mapHost, const uint32_t *mapGPU, uint32_t occupancyTotal, int32_t stream=-1)
void runParallelOuterLoop(bool doGPU, uint32_t nThreads, std::function< void(uint32_t)> lambda)
const GPUDefParameters & getGPUParameters(bool doGPU) const override
void * AllocateVolatileMemory(size_t size, bool device)
void PopNonPersistentMemory(RecoStep step, uint64_t tag)
void ComputeReuseMax(GPUProcessor *proc)
void ResetRegisteredMemoryPointers(GPUProcessor *proc)
GPUMemoryResource & Res(int16_t num)
const GPUParam & GetParam() const
void PushNonPersistentMemory(uint64_t tag)
GPUMemorySizeScalers * MemoryScalers()
size_t AllocateRegisteredMemory(GPUProcessor *proc, bool resetCustom=false)
int16_t MemoryResTracklets() const
int16_t MemoryResLinks() const
void SetMaxData(const GPUTrackingInOutPointers &io)
int16_t MemoryResOutput() const
void DumpTrackHits(std::ostream &out)
void DumpLinks(std::ostream &out, int32_t phase)
void DumpStartHits(std::ostream &out)
void DumpHitWeights(std::ostream &out)
int16_t MemoryResCommon() const
void DumpTrackingData(std::ostream &out)
void DumpTrackletHits(std::ostream &out)
GLenum GLfloat param
Definition glcorearb.h:271
GLuint GLuint stream
Definition glcorearb.h:1806
constexpr T qStr2Tag(const char *str)
Definition strtag.h:22
GPUTPCTracker tpcTrackers[GPUCA_NSECTORS]
const GPUTPCHitId * sectorClusters[NSECTORS]
const o2::tpc::ClusterNativeAccess * clustersNative
const GPUTPCTrack * sectorTracks[NSECTORS]
const GPUSettingsTF * settingsTF
size_t getIndex(const GPUOutputControl &v)
unsigned int nClustersSector[constants::MAXSECTOR]
unsigned int clusterOffset[constants::MAXSECTOR][constants::MAXGLOBALPADROW]