Project
Loading...
Searching...
No Matches
GPUReconstruction.cxx
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
14
15#include <cstring>
16#include <cstdio>
17#include <iostream>
18#include <mutex>
19#include <string>
20#include <map>
21#include <queue>
22#include <mutex>
23#include <condition_variable>
24#include <array>
25
26#include "GPUReconstruction.h"
29#include "GPUReconstructionIO.h"
30#include "GPUROOTDumpCore.h"
31#include "GPUConfigDump.h"
32#include "GPUChainTracking.h"
33#include "GPUConstantMem.h"
34#include "GPUCommonHelpers.h"
35#include "GPUSettings.h"
36
37#include "GPUMemoryResource.h"
38#include "GPUChain.h"
40
41#include "GPULogging.h"
42#include "utils/strtag.h"
43#include "utils/stdspinlock.h"
44
45#ifndef GPUCA_STANDALONE
47#endif
48
50
51namespace o2::gpu
52{
53namespace // anonymous
54{
55struct GPUReconstructionPipelineQueue {
56 uint32_t op = 0; // For now, 0 = process, 1 = terminate
57 GPUChain* chain = nullptr;
58 std::mutex m;
59 std::condition_variable c;
60 bool done = false;
62};
63} // namespace
64
66 std::queue<GPUReconstructionPipelineQueue*> pipelineQueue;
67 std::mutex mutex;
68 std::condition_variable cond;
69 bool workerRunning = false;
70 bool terminate = false;
71};
72} // namespace o2::gpu
73
74using namespace o2::gpu;
75
76constexpr const char* const GPUReconstruction::GEOMETRY_TYPE_NAMES[];
77constexpr const char* const GPUReconstruction::IOTYPENAMES[];
79
80static ptrdiff_t ptrDiff(void* a, void* b) { return (char*)a - (char*)b; }
81
82GPUReconstruction::GPUReconstruction(const GPUSettingsDeviceBackend& cfg) : mHostConstantMem(new GPUConstantMem), mGRPSettings(new GPUSettingsGRP), mDeviceBackendSettings(new GPUSettingsDeviceBackend(cfg)), mProcessingSettings(new GPUSettingsProcessing)
83{
84 if (cfg.master) {
86 throw std::invalid_argument("device type of master and slave GPUReconstruction does not match");
87 }
88 if (cfg.master->mMaster) {
89 throw std::invalid_argument("Cannot be slave to a slave");
90 }
91 mMaster = cfg.master;
92 mSlaveId = cfg.master->mSlaves.size();
93 cfg.master->mSlaves.emplace_back(this);
94 }
97 for (uint32_t i = 0; i < NSECTORS; i++) {
98 processors()->tpcTrackers[i].SetSector(i); // TODO: Move to a better place
100#ifdef GPUCA_HAS_ONNX
101 processors()->tpcNNClusterer[i].mISector = i;
102#endif
103 }
104}
105
107{
108 if (mInitialized) {
109 GPUError("GPU Reconstruction not properly deinitialized!");
110 }
111}
112
113void GPUReconstruction::GetITSTraits(std::unique_ptr<o2::its::TrackerTraits<7>>* trackerTraits, std::unique_ptr<o2::its::VertexerTraits<7>>* vertexerTraits, std::unique_ptr<o2::its::TimeFrame<7>>* timeFrame)
114{
115 if (trackerTraits) {
116 trackerTraits->reset(new o2::its::TrackerTraits<7>);
117 }
118 if (vertexerTraits) {
119 vertexerTraits->reset(new o2::its::VertexerTraits<7>);
120 }
121 if (timeFrame) {
122 timeFrame->reset(new o2::its::TimeFrame<7>);
123 }
124}
125
127{
128 return std::max<int32_t>(0, tbb::this_task_arena::current_thread_index());
129}
130
132{
133 if (mMaster) {
134 throw std::runtime_error("Must not call init on slave!");
135 }
136#ifndef GPUCA_NO_ROOT
137 if (!mROOTDump) {
138 mROOTDump = GPUROOTDumpCore::getAndCreate(GetProcessingSettings().ROOTDumpFile.c_str());
139 }
140#endif
142 if (retVal) {
143 return retVal;
144 }
145 for (uint32_t i = 0; i < mSlaves.size(); i++) {
146 retVal = mSlaves[i]->InitPhaseBeforeDevice();
147 if (retVal) {
148 GPUError("Error initialization slave (before deviceinit)");
149 return retVal;
150 }
151 mNStreams = std::max(mNStreams, mSlaves[i]->mNStreams);
154 }
155 if (InitDevice()) {
156 return 1;
157 }
158 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_GLOBAL) {
161 } else {
163 }
165 return 1;
166 }
167 for (uint32_t i = 0; i < mSlaves.size(); i++) {
168 mSlaves[i]->mDeviceMemoryBase = mDeviceMemoryPermanent;
169 mSlaves[i]->mHostMemoryBase = mHostMemoryPermanent;
170 mSlaves[i]->mDeviceMemorySize = mDeviceMemorySize - ptrDiff(mSlaves[i]->mDeviceMemoryBase, mDeviceMemoryBase);
171 mSlaves[i]->mHostMemorySize = mHostMemorySize - ptrDiff(mSlaves[i]->mHostMemoryBase, mHostMemoryBase);
172 mSlaves[i]->mHostMemoryPoolEnd = mHostMemoryPoolEnd;
173 mSlaves[i]->mDeviceMemoryPoolEnd = mDeviceMemoryPoolEnd;
174 if (mSlaves[i]->InitDevice()) {
175 GPUError("Error initialization slave (deviceinit)");
176 return 1;
177 }
179 GPUError("Error initialization slave (permanent memory)");
180 return 1;
181 }
182 mDeviceMemoryPermanent = mSlaves[i]->mDeviceMemoryPermanent;
183 mHostMemoryPermanent = mSlaves[i]->mHostMemoryPermanent;
184 }
186 if (retVal) {
187 return retVal;
188 }
190 for (uint32_t i = 0; i < mSlaves.size(); i++) {
191 mSlaves[i]->mDeviceMemoryPermanent = mDeviceMemoryPermanent;
192 mSlaves[i]->mHostMemoryPermanent = mHostMemoryPermanent;
193 retVal = mSlaves[i]->InitPhaseAfterDevice();
194 if (retVal) {
195 GPUError("Error initialization slave (after device init)");
196 return retVal;
197 }
198 mSlaves[i]->ClearAllocatedMemory();
199 }
200 debugInit();
201 return 0;
202}
203
204namespace o2::gpu::internal
205{
206static uint32_t getDefaultNThreads()
207{
208 const char* tbbEnv = getenv("TBB_NUM_THREADS");
209 uint32_t tbbNum = tbbEnv ? atoi(tbbEnv) : 0;
210 if (tbbNum) {
211 return tbbNum;
212 }
213 const char* ompEnv = getenv("OMP_NUM_THREADS");
214 uint32_t ompNum = ompEnv ? atoi(ompEnv) : 0;
215 if (ompNum) {
216 return ompNum;
217 }
218 return tbb::info::default_concurrency();
219}
220} // namespace o2::gpu::internal
221
223{
224 if (GetProcessingSettings().printSettings) {
225 if (mSlaves.size() || mMaster) {
226 printf("\nConfig Dump %s\n", mMaster ? "Slave" : "Master");
227 }
228 const GPUChainTracking* chTrk;
229 for (uint32_t i = 0; i < mChains.size(); i++) {
230 if ((chTrk = dynamic_cast<GPUChainTracking*>(mChains[i].get()))) {
231 break;
232 }
233 }
234 GPUConfigDump::dumpConfig(&param().rec, mProcessingSettings.get(), chTrk ? chTrk->GetQAConfig() : nullptr, chTrk ? chTrk->GetEventDisplayConfig() : nullptr, mDeviceBackendSettings.get(), &mRecoSteps);
235 }
238 if (!IsGPU()) {
240 }
241
242 if (GetProcessingSettings().forceMemoryPoolSize >= 1024 || GetProcessingSettings().forceHostMemoryPoolSize >= 1024) {
244 }
245 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_AUTO) {
247 }
248 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_INDIVIDUAL) {
249 mProcessingSettings->forceMemoryPoolSize = mProcessingSettings->forceHostMemoryPoolSize = 0;
250 }
251 if (GetProcessingSettings().debugLevel >= 4) {
252 mProcessingSettings->keepAllMemory = true;
253 }
254 if (GetProcessingSettings().debugLevel >= 5 && GetProcessingSettings().allocDebugLevel < 2) {
255 mProcessingSettings->allocDebugLevel = 2;
256 }
258 mProcessingSettings->keepDisplayMemory = true;
259 }
260 if (GetProcessingSettings().debugLevel < 6) {
261 mProcessingSettings->debugMask = 0;
262 }
263 if (GetProcessingSettings().debugLevel < 1) {
264 mProcessingSettings->deviceTimers = false;
265 }
266 if (GetProcessingSettings().debugLevel > 0) {
267 mProcessingSettings->recoTaskTiming = true;
268 }
269 bool detMode = false;
270#ifdef GPUCA_DETERMINISTIC_MODE
271 detMode = true;
272#endif
273 if (GetProcessingSettings().deterministicGPUReconstruction == -1) {
274 mProcessingSettings->deterministicGPUReconstruction = detMode ? 1 : (GetProcessingSettings().debugLevel >= 6);
275 }
276 if (GetProcessingSettings().deterministicGPUReconstruction) {
277 if (!detMode) {
278 GPUError("WARNING, deterministicGPUReconstruction needs GPUCA_DETERMINISTIC_MODE for being fully deterministic, without only most indeterminism by concurrency is removed, but floating point effects remain!");
279 } else {
280 GPUInfo("GPU Deterministic Reconstruction is enabled");
281 }
282 if (mProcessingSettings->debugLevel >= 6 && ((mProcessingSettings->debugMask + 1) & mProcessingSettings->debugMask)) {
283 GPUError("WARNING: debugMask %d - debug output might not be deterministic with intermediate steps missing", mProcessingSettings->debugMask);
284 }
285 mProcessingSettings->overrideClusterizerFragmentLen = TPC_MAX_FRAGMENT_LEN_GPU;
286 if (GetProcessingSettings().createO2Output > 1) {
287 mProcessingSettings->createO2Output = 1;
288 }
289 mProcessingSettings->rtc.deterministic = 1;
290 } else {
291 if (detMode) {
292 GPUError("WARNING, compiled with GPUCA_DETERMINISTIC_MODE but deterministicGPUReconstruction not set, only compile-time determinism and deterministic math enforced, not fully deterministic!");
293 }
294 }
295 if (GetProcessingSettings().deterministicGPUReconstruction && GetProcessingSettings().debugLevel >= 6) {
296 mProcessingSettings->nTPCClustererLanes = 1;
297 }
298 if (GetProcessingSettings().createO2Output > 1 && GetProcessingSettings().runQA && GetProcessingSettings().qcRunFraction == 100.f) {
299 mProcessingSettings->createO2Output = 1;
300 }
301 if (!GetProcessingSettings().createO2Output || !IsGPU()) {
302 mProcessingSettings->clearO2OutputFromGPU = false;
303 }
305 mProcessingSettings->mergerSortTracks = false;
306 }
307 if (GetProcessingSettings().debugLevel > 3 || !IsGPU() || GetProcessingSettings().deterministicGPUReconstruction) {
308 mProcessingSettings->delayedOutput = false;
309 }
310 if (!GetProcessingSettings().rtc.enable) {
311 mProcessingSettings->rtc.optConstexpr = false;
312 }
313 if (GetProcessingSettings().allSanityChecks) {
314 mProcessingSettings->clusterizerZSSanityCheck = mProcessingSettings->mergerSanityCheck = mProcessingSettings->outputSanityCheck = true;
315 }
316
317 static_cast<GPUSettingsProcessingScaling&>(*mMemoryScalers) = GetProcessingSettings().scaling;
318 mMemoryScalers->scalingFactor = GetProcessingSettings().memoryScalingFactor;
319 mMemoryScalers->returnMaxVal = GetProcessingSettings().forceMaxMemScalers != 0;
320 if (GetProcessingSettings().forceMaxMemScalers > 1) {
321 mMemoryScalers->rescaleMaxMem(GetProcessingSettings().forceMaxMemScalers);
322 }
323
324 if (GetProcessingSettings().nHostThreads != -1 && GetProcessingSettings().ompThreads != -1) {
325 GPUFatal("Must not use both nHostThreads and ompThreads at the same time!");
326 } else if (GetProcessingSettings().ompThreads != -1) {
327 mProcessingSettings->nHostThreads = GetProcessingSettings().ompThreads;
328 GPUWarning("You are using the deprecated ompThreads option, please switch to nHostThreads!");
329 }
330
331 if (GetProcessingSettings().nHostThreads <= 0) {
332 mProcessingSettings->nHostThreads = internal::getDefaultNThreads();
333 } else {
334 mProcessingSettings->autoAdjustHostThreads = false;
335 }
336 mMaxHostThreads = GetProcessingSettings().nHostThreads;
337 if (mMaster == nullptr) {
338 mThreading = std::make_shared<GPUReconstructionThreading>();
339 mThreading->control = std::make_unique<tbb::global_control>(tbb::global_control::max_allowed_parallelism, mMaxHostThreads);
340 mThreading->allThreads = std::make_unique<tbb::task_arena>(mMaxHostThreads);
341 mThreading->activeThreads = std::make_unique<tbb::task_arena>(mMaxHostThreads);
342 } else {
344 }
346 if (IsGPU()) {
347 mNStreams = std::max<int32_t>(GetProcessingSettings().nStreams, 3);
348 }
349
350 if (GetProcessingSettings().nTPCClustererLanes == -1) {
351 mProcessingSettings->nTPCClustererLanes = (GetRecoStepsGPU() & RecoStep::TPCClusterFinding) ? 3 : std::max<int32_t>(1, std::min<int32_t>(GPUTPCGeometry::NSECTORS, GetProcessingSettings().inKernelParallel ? (mMaxHostThreads >= 4 ? std::min<int32_t>(mMaxHostThreads / 2, mMaxHostThreads >= 32 ? GPUTPCGeometry::NSECTORS : 4) : 1) : mMaxHostThreads));
352 }
353 if (GetProcessingSettings().overrideClusterizerFragmentLen == -1) {
354 mProcessingSettings->overrideClusterizerFragmentLen = ((GetRecoStepsGPU() & RecoStep::TPCClusterFinding) || (mMaxHostThreads / GetProcessingSettings().nTPCClustererLanes >= 3)) ? TPC_MAX_FRAGMENT_LEN_GPU : TPC_MAX_FRAGMENT_LEN_HOST;
355 }
356 if ((uint32_t)GetProcessingSettings().nTPCClustererLanes > GPUTPCGeometry::NSECTORS) {
357 GPUError("Invalid value for nTPCClustererLanes: %d", GetProcessingSettings().nTPCClustererLanes);
358 mProcessingSettings->nTPCClustererLanes = GPUTPCGeometry::NSECTORS;
359 }
360
361 if (GetProcessingSettings().doublePipeline) {
362 mProcessingSettings->rtctech.allowOptimizedSlaveReconstruction = true;
363 }
364 if (GetProcessingSettings().doublePipeline && (mChains.size() != 1 || mChains[0]->SupportsDoublePipeline() == false || !IsGPU() || GetProcessingSettings().memoryAllocationStrategy != GPUMemoryResource::ALLOCATION_GLOBAL)) {
365 GPUError("Must use double pipeline mode only with exactly one chain that must support it");
366 return 1;
367 }
368 if (mMaster == nullptr && GetProcessingSettings().doublePipeline) {
370 }
371
372 if (mMaster && GetProcessingSettings().rtc.enable && (GetProcessingSettings().rtc.optConstexpr || GetProcessingSettings().rtc.optSpecialCode) && !GetProcessingSettings().rtctech.allowOptimizedSlaveReconstruction) {
373 GPUError("Not allowed to create optimized RTC code with more than one GPUReconstruction instances");
374 return 1;
375 }
376
378 for (uint32_t i = 0; i < mChains.size(); i++) {
379 if (mChains[i]->EarlyConfigure()) {
380 return 1;
381 }
382 mChains[i]->RegisterPermanentMemoryAndProcessors();
383 size_t memPrimary, memPageLocked;
384 mChains[i]->MemorySize(memPrimary, memPageLocked);
385 if (!IsGPU() || mOutputControl.useInternal()) {
386 memPageLocked = memPrimary;
387 }
388 mDeviceMemorySize += memPrimary;
389 mHostMemorySize += memPageLocked;
390 }
391 if (GetProcessingSettings().forceMemoryPoolSize && GetProcessingSettings().forceMemoryPoolSize <= 2 && CanQueryMaxMemory()) {
392 mDeviceMemorySize = GetProcessingSettings().forceMemoryPoolSize;
393 } else if (GetProcessingSettings().forceMemoryPoolSize > 2) {
394 mDeviceMemorySize = GetProcessingSettings().forceMemoryPoolSize;
395 if (!IsGPU() || mOutputControl.useInternal()) {
397 }
398 }
399 if (GetProcessingSettings().forceHostMemoryPoolSize) {
400 mHostMemorySize = GetProcessingSettings().forceHostMemoryPoolSize;
401 }
402
403 for (uint32_t i = 0; i < mProcessors.size(); i++) {
404 (mProcessors[i].proc->*(mProcessors[i].RegisterMemoryAllocation))();
405 }
406
407 return 0;
408}
409
411{
412 if (IsGPU()) {
413 for (uint32_t i = 0; i < mChains.size(); i++) {
414 mChains[i]->RegisterGPUProcessors();
415 }
416 }
418 return 0;
419}
420
422{
423 if (GetProcessingSettings().forceMaxMemScalers <= 1 && GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_GLOBAL) {
425 }
426 for (uint32_t i = 0; i < mChains.size(); i++) {
427 if (mChains[i]->Init()) {
428 return 1;
429 }
430 }
431 for (uint32_t i = 0; i < mProcessors.size(); i++) {
432 (mProcessors[i].proc->*(mProcessors[i].InitializeProcessor))();
433 }
434
435 WriteConstantParams(); // Initialize with initial values, can optionally be updated later
436
437 mInitialized = true;
438 return 0;
439}
440
442{
443 if (IsGPU()) {
444 const auto threadContext = GetThreadContext();
445 WriteToConstantMemory(ptrDiff(&processors()->param, processors()), &param(), sizeof(param()), stream);
446 }
447}
448
450{
451 for (uint32_t i = 0; i < mChains.size(); i++) {
452 mChains[i]->Finalize();
453 }
454 return 0;
455}
456
458{
459 if (!mInitialized) {
460 return 1;
461 }
462 for (uint32_t i = 0; i < mSlaves.size(); i++) {
463 if (mSlaves[i]->Exit()) {
464 GPUError("Error exiting slave");
465 }
466 }
467
468 mChains.clear(); // Make sure we destroy a possible ITS GPU tracker before we call the destructors
469 mHostConstantMem.reset(); // Reset these explicitly before the destruction of other members unloads the library
470 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_INDIVIDUAL) {
471 for (uint32_t i = 0; i < mMemoryResources.size(); i++) {
472 if (mMemoryResources[i].mReuse >= 0) {
473 continue;
474 }
476 mMemoryResources[i].mPtr = mMemoryResources[i].mPtrDevice = nullptr;
477 }
478 }
479 mMemoryResources.clear();
480 if (mInitialized) {
481 ExitDevice();
482 }
483 debugExit();
484 mInitialized = false;
485 return 0;
486}
487
490
492{
493 for (auto it = mMemoryReuse1to1.begin(); it != mMemoryReuse1to1.end(); it++) {
494 auto& re = it->second;
495 if (proc == nullptr || re.proc == proc) {
496 GPUMemoryResource& resMain = mMemoryResources[re.res[0]];
497 resMain.mOverrideSize = 0;
498 for (uint32_t i = 0; i < re.res.size(); i++) {
500 resMain.mOverrideSize = std::max<size_t>(resMain.mOverrideSize, ptrDiff(res.SetPointers((void*)1), (char*)1));
501 }
502 }
503 }
504}
505
507{
509 if ((type & GPUMemoryResource::MEMORY_SCRATCH) && !GetProcessingSettings().keepDisplayMemory) { // keepAllMemory --> keepDisplayMemory
511 } else {
513 }
514 }
515 if (proc->mGPUProcessorType == GPUProcessor::PROCESSOR_TYPE_CPU) {
516 type &= ~GPUMemoryResource::MEMORY_GPU;
517 }
519 if (mMemoryResources.size() >= 32768) {
520 throw std::bad_alloc();
521 }
522 uint16_t retVal = mMemoryResources.size() - 1;
523 if (re.type != GPUMemoryReuse::NONE && !GetProcessingSettings().disableMemoryReuse) {
524 const auto& it = mMemoryReuse1to1.find(re.id);
525 if (it == mMemoryReuse1to1.end()) {
526 mMemoryReuse1to1[re.id] = {proc, retVal};
527 } else {
528 mMemoryResources[retVal].mReuse = it->second.res[0];
529 it->second.res.emplace_back(retVal);
530 }
531 }
532 return retVal;
533}
534
536{
537 if (GetProcessingSettings().debugLevel >= 5) {
538 GPUInfo("Allocating memory %p", (void*)proc);
539 }
540 size_t total = 0;
541 for (uint32_t i = 0; i < mMemoryResources.size(); i++) {
542 if (proc == nullptr ? !mMemoryResources[i].mProcessor->mAllocateAndInitializeLate : mMemoryResources[i].mProcessor == proc) {
544 total += AllocateRegisteredMemory(i);
545 } else if (resetCustom && (mMemoryResources[i].mPtr || mMemoryResources[i].mPtrDevice)) {
547 }
548 }
549 }
550 if (GetProcessingSettings().debugLevel >= 5) {
551 GPUInfo("Allocating memory done");
552 }
553 return total;
554}
555
557{
558 if (GetProcessingSettings().debugLevel >= 5) {
559 GPUInfo("Allocating Permanent Memory");
560 }
562 GPUError("Must not allocate permanent memory while volatile chunks are allocated");
563 throw std::bad_alloc();
564 }
565 int32_t total = 0;
566 for (uint32_t i = 0; i < mMemoryResources.size(); i++) {
567 if ((mMemoryResources[i].mType & GPUMemoryResource::MEMORY_PERMANENT) && mMemoryResources[i].mPtr == nullptr) {
568 total += AllocateRegisteredMemory(i);
569 }
570 }
573 if (GetProcessingSettings().debugLevel >= 5) {
574 GPUInfo("Permanent Memory Done");
575 }
576 return total;
577}
578
579size_t GPUReconstruction::AllocateRegisteredMemoryHelper(GPUMemoryResource* res, void*& ptr, void*& memorypool, void* memorybase, size_t memorysize, void* (GPUMemoryResource::*setPtr)(void*) const, void*& memorypoolend, const char* device)
580{
581 if (res->mReuse >= 0) {
582 ptr = (&ptr == &res->mPtrDevice) ? mMemoryResources[res->mReuse].mPtrDevice : mMemoryResources[res->mReuse].mPtr;
583 if (ptr == nullptr) {
584 GPUError("Invalid reuse ptr (%s)", res->mName);
585 throw std::bad_alloc();
586 }
587 size_t retVal = ptrDiff((res->*setPtr)(ptr), ptr);
588 if (retVal > mMemoryResources[res->mReuse].mSize) {
589 GPUError("Insufficient reuse memory %lu < %lu (%s) (%s)", mMemoryResources[res->mReuse].mSize, retVal, res->mName, device);
590 throw std::bad_alloc();
591 }
592 if (GetProcessingSettings().allocDebugLevel >= 2) {
593 std::cout << "Reused (" << device << ") " << res->mName << ": " << retVal << "\n";
594 }
595 return retVal;
596 }
597 if (memorypool == nullptr) {
598 GPUError("Cannot allocate memory from uninitialized pool");
599 throw std::bad_alloc();
600 }
601 size_t retVal;
602 stdspinlock spinlock(mMemoryMutex);
603 if ((res->mType & GPUMemoryResource::MEMORY_STACK) && memorypoolend) {
604 retVal = ptrDiff((res->*setPtr)((char*)1), (char*)(1));
605 memorypoolend = (void*)((char*)memorypoolend - GPUProcessor::getAlignmentMod<constants::GPU_MEMALIGN>(memorypoolend));
606 if (retVal < res->mOverrideSize) {
607 retVal = res->mOverrideSize;
608 }
609 retVal += GPUProcessor::getAlignment<constants::GPU_MEMALIGN>(retVal);
610 memorypoolend = (char*)memorypoolend - retVal;
611 ptr = memorypoolend;
612 retVal = std::max<size_t>(ptrDiff((res->*setPtr)(ptr), ptr), res->mOverrideSize);
613 } else {
614 ptr = memorypool;
615 memorypool = (char*)((res->*setPtr)(ptr));
616 retVal = ptrDiff(memorypool, ptr);
617 if (retVal < res->mOverrideSize) {
618 retVal = res->mOverrideSize;
619 memorypool = (char*)ptr + res->mOverrideSize;
620 }
621 memorypool = (void*)((char*)memorypool + GPUProcessor::getAlignment<constants::GPU_MEMALIGN>(memorypool));
622 }
623 if (memorypoolend ? (memorypool > memorypoolend) : ((size_t)ptrDiff(memorypool, memorybase) > memorysize)) {
624 std::cerr << "Memory pool size exceeded (" << device << ") (" << res->mName << ": " << (memorypoolend ? (memorysize + ptrDiff(memorypool, memorypoolend)) : ptrDiff(memorypool, memorybase)) << " > " << memorysize << "\n";
625 throw std::bad_alloc();
626 }
627 if (GetProcessingSettings().allocDebugLevel >= 2) {
628 std::cout << "Allocated (" << device << ") " << res->mName << ": " << retVal << " - available: " << (memorypoolend ? ptrDiff(memorypoolend, memorypool) : (memorysize - ptrDiff(memorypool, memorybase))) << "\n";
629 }
630 return retVal;
631}
632
634{
635 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_INDIVIDUAL && (control == nullptr || control->useInternal())) {
636 if (!(res->mType & GPUMemoryResource::MEMORY_EXTERNAL)) {
637 if (res->mPtrDevice && res->mReuse < 0) {
638 alignedDefaultBufferDeleter()(res->mPtrDevice);
639 }
640 res->mSize = std::max((size_t)res->SetPointers((void*)1) - 1, res->mOverrideSize);
641 if (res->mReuse >= 0) {
642 if (res->mSize > mMemoryResources[res->mReuse].mSize) {
643 GPUError("Invalid reuse, insufficient size: %ld < %ld", (int64_t)mMemoryResources[res->mReuse].mSize, (int64_t)res->mSize);
644 throw std::bad_alloc();
645 }
646 res->mPtrDevice = mMemoryResources[res->mReuse].mPtrDevice;
647 } else {
648 res->mPtrDevice = alignedDefaultBufferAllocator<char>(res->mSize + constants::GPU_BUFFER_ALIGNMENT);
649 }
650 res->mPtr = GPUProcessor::alignPointer<constants::GPU_BUFFER_ALIGNMENT>(res->mPtrDevice);
651 res->SetPointers(res->mPtr);
652 if (GetProcessingSettings().allocDebugLevel >= 2) {
653 std::cout << (res->mReuse >= 0 ? "Reused " : "Allocated ") << res->mName << ": " << res->mSize << " (individual" << ((res->mType & GPUMemoryResource::MEMORY_STACK) ? " stack" : "") << ")\n";
654 }
656 stdspinlock spinlock(mMemoryMutex);
658 }
659 if ((size_t)res->mPtr % constants::GPU_BUFFER_ALIGNMENT) {
660 GPUError("Got buffer with insufficient alignment");
661 throw std::bad_alloc();
662 }
663 }
664 } else {
665 if (res->mPtr != nullptr) {
666 GPUError("Double allocation! (%s)", res->mName);
667 throw std::bad_alloc();
668 }
669 if (IsGPU() && res->mOverrideSize < constants::GPU_BUFFER_ALIGNMENT) {
670 res->mOverrideSize = constants::GPU_BUFFER_ALIGNMENT;
671 }
672 if ((!IsGPU() || (res->mType & GPUMemoryResource::MEMORY_HOST) || GetProcessingSettings().keepDisplayMemory) && !(res->mType & GPUMemoryResource::MEMORY_EXTERNAL)) { // keepAllMemory --> keepDisplayMemory
673 if (control && control->useExternal()) {
674 if (control->allocator) {
675 res->mSize = std::max((size_t)res->SetPointers((void*)1) - 1, res->mOverrideSize);
676 res->mPtr = control->allocator(CAMath::nextMultipleOf<constants::GPU_BUFFER_ALIGNMENT>(res->mSize));
677 res->mSize = std::max<size_t>(ptrDiff(res->SetPointers(res->mPtr), res->mPtr), res->mOverrideSize);
678 if (GetProcessingSettings().allocDebugLevel >= 2) {
679 std::cout << "Allocated (from callback) " << res->mName << ": " << res->mSize << "\n";
680 }
681 } else {
682 void* dummy = nullptr;
683 res->mSize = AllocateRegisteredMemoryHelper(res, res->mPtr, control->ptrCurrent, control->ptrBase, control->size, &GPUMemoryResource::SetPointers, dummy, "host");
684 }
685 } else {
687 }
688 if ((size_t)res->mPtr % constants::GPU_BUFFER_ALIGNMENT) {
689 GPUError("Got buffer with insufficient alignment");
690 throw std::bad_alloc();
691 }
692 }
693 if (IsGPU() && (res->mType & GPUMemoryResource::MEMORY_GPU)) {
694 if (res->mProcessor->mLinkedProcessor == nullptr) {
695 GPUError("Device Processor not set (%s)", res->mName);
696 throw std::bad_alloc();
697 }
699 GPUError("Must not allocate non-stacked device memory while volatile chunks are allocated");
700 throw std::bad_alloc();
701 }
703
705 res->mSize = size;
706 } else if (size != res->mSize) {
707 GPUError("Inconsistent device memory allocation (%s: device %lu vs %lu)", res->mName, size, res->mSize);
708 throw std::bad_alloc();
709 }
710 if ((size_t)res->mPtrDevice % constants::GPU_BUFFER_ALIGNMENT) {
711 GPUError("Got buffer with insufficient alignment");
712 throw std::bad_alloc();
713 }
714 }
716 }
717}
718
720{
721 rec->AllocateRegisteredMemoryInternal(&rec->mMemoryResources[ires], control, this);
722}
723
725{
727 if ((res->mType & GPUMemoryResource::MEMORY_PERMANENT) && res->mPtr != nullptr) {
729 } else {
731 }
732 return res->mReuse >= 0 ? 0 : res->mSize;
733}
734
736{
737 stdspinlock spinlock(mMemoryMutex);
738 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_INDIVIDUAL) {
739 char* retVal = alignedDefaultBufferAllocator<char>(size);
742 } else {
744 }
745 return retVal;
746 }
747
749 throw std::runtime_error("Requested invalid memory typo for direct allocation");
750 }
752 GPUError("Must not allocate direct memory while volatile chunks are allocated");
753 throw std::bad_alloc();
754 }
755
758 char* retVal;
760 poolend = (char*)poolend - size; // TODO: Implement overflow check
761 poolend = (char*)poolend - GPUProcessor::getAlignmentMod<constants::GPU_MEMALIGN>(poolend);
762 retVal = (char*)poolend;
763 } else {
765 }
766 if (pool > poolend) {
767 GPUError("Insufficient unmanaged memory: missing %ld bytes", ptrDiff(pool, poolend));
768 throw std::bad_alloc();
769 }
771 if (GetProcessingSettings().allocDebugLevel >= 2) {
772 std::cout << "Allocated (unmanaged " << ((type & GPUMemoryResource::MEMORY_GPU) ? "gpu" : "host") << "): " << size << " - available: " << ptrDiff(poolend, pool) << "\n";
773 }
774 return retVal;
775}
776
778{
779 stdspinlock spinlock(mMemoryMutex);
780 if (mVolatileMemoryStart == nullptr) {
782 }
783 if (size == 0) {
784 return nullptr; // Future GPU memory allocation is volatile
785 }
786 char* retVal;
789 GPUError("Insufficient volatile device memory: missing %ld", ptrDiff(mDeviceMemoryPool, mDeviceMemoryPoolEnd));
790 throw std::bad_alloc();
791 }
793 if (GetProcessingSettings().allocDebugLevel >= 2) {
794 std::cout << "Allocated (volatile GPU): " << size << " - available: " << ptrDiff(mDeviceMemoryPoolEnd, mDeviceMemoryPool) << "\n";
795 }
796 return retVal;
797}
798
800{
801 if (device) {
803 }
804 char* retVal = alignedDefaultBufferAllocator<char>(size);
805 stdspinlock spinlock(mMemoryMutex);
807 return retVal;
808}
809
815
817{
821 mVolatileMemoryStart = nullptr;
822 }
823 if (GetProcessingSettings().allocDebugLevel >= 2) {
824 std::cout << "Freed (volatile GPU) - available: " << ptrDiff(mDeviceMemoryPoolEnd, mDeviceMemoryPool) << "\n";
825 }
826}
827
833
835{
836 for (uint32_t i = 0; i < mMemoryResources.size(); i++) {
837 if (proc == nullptr || mMemoryResources[i].mProcessor == proc) {
839 }
840 }
841}
842
844{
847 void* basePtr = res->mReuse >= 0 ? mMemoryResources[res->mReuse].mPtr : res->mPtr;
848 size_t size = ptrDiff(res->SetPointers(basePtr), basePtr);
849 if (basePtr && size > std::max(res->mSize, res->mOverrideSize)) {
850 std::cerr << "Updated pointers exceed available memory size: " << size << " > " << std::max(res->mSize, res->mOverrideSize) << " - host - " << res->mName << "\n";
851 throw std::bad_alloc();
852 }
853 }
854 if (IsGPU() && (res->mType & GPUMemoryResource::MEMORY_GPU)) {
855 void* basePtr = res->mReuse >= 0 ? mMemoryResources[res->mReuse].mPtrDevice : res->mPtrDevice;
856 size_t size = ptrDiff(res->SetDevicePointers(basePtr), basePtr);
857 if (basePtr && size > std::max(res->mSize, res->mOverrideSize)) {
858 std::cerr << "Updated pointers exceed available memory size: " << size << " > " << std::max(res->mSize, res->mOverrideSize) << " - GPU - " << res->mName << "\n";
859 throw std::bad_alloc();
860 }
861 }
862}
863
864void GPUReconstruction::FreeRegisteredMemory(GPUProcessor* proc, bool freeCustom, bool freePermanent)
865{
866 for (uint32_t i = 0; i < mMemoryResources.size(); i++) {
867 if ((proc == nullptr || mMemoryResources[i].mProcessor == proc) && (freeCustom || !(mMemoryResources[i].mType & GPUMemoryResource::MEMORY_CUSTOM)) && (freePermanent || !(mMemoryResources[i].mType & GPUMemoryResource::MEMORY_PERMANENT))) {
869 }
870 }
871}
872
877
879{
880 if (GetProcessingSettings().allocDebugLevel >= 2 && (res->mPtr || res->mPtrDevice)) {
881 std::cout << "Freeing " << res->mName << ": size " << res->mSize << " (reused " << res->mReuse << ")\n";
882 }
883 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_INDIVIDUAL && res->mReuse < 0) {
884 alignedDefaultBufferDeleter()(res->mPtrDevice);
885 }
886 res->mPtr = nullptr;
887 res->mPtrDevice = nullptr;
888}
889
894
896{
897 if (proc && GetProcessingSettings().memoryAllocationStrategy != GPUMemoryResource::ALLOCATION_INDIVIDUAL) {
898 GPUFatal("Processor-depending memory-free works only with allocation strategy ALLOCATION_INDIVIDUAL");
899 }
900 if (GetProcessingSettings().keepDisplayMemory || GetProcessingSettings().disableMemoryReuse) {
901 return;
902 }
903 if (mNonPersistentMemoryStack.size() == 0) {
904 GPUFatal("Trying to pop memory state from empty stack");
905 }
906 if (tag != 0 && std::get<4>(mNonPersistentMemoryStack.back()) != tag) {
907 GPUFatal("Tag mismatch when popping non persistent memory from stack : pop %s vs on stack %s", qTag2Str(tag).c_str(), qTag2Str(std::get<4>(mNonPersistentMemoryStack.back())).c_str());
908 }
909 if (!proc && (GetProcessingSettings().debugLevel >= 3 || GetProcessingSettings().allocDebugLevel) && (IsGPU() || GetProcessingSettings().forceHostMemoryPoolSize)) {
910 printf("Allocated memory after %30s (%8s) (Stack %zu): ", gpudatatypes::RECO_STEP_NAMES[getRecoStepNum(step, true)], qTag2Str(std::get<4>(mNonPersistentMemoryStack.back())).c_str(), mNonPersistentMemoryStack.size());
912 printf("%76s", "");
914 }
915 for (uint32_t i = std::get<2>(mNonPersistentMemoryStack.back()); i < mNonPersistentIndividualAllocations.size(); i++) {
917 if (proc && res->mProcessor != proc) {
918 continue;
919 }
920 if (GetProcessingSettings().allocDebugLevel >= 2 && (res->mPtr || res->mPtrDevice)) {
921 std::cout << "Freeing NonPersistent " << res->mName << ": size " << res->mSize << " (reused " << res->mReuse << ")\n";
922 }
923 if (res->mReuse < 0) {
924 alignedDefaultBufferDeleter()(res->mPtrDevice);
925 }
926 res->mPtr = nullptr;
927 res->mPtrDevice = nullptr;
928 }
929 if (!proc) {
930 stdspinlock spinlock(mMemoryMutex);
931 mHostMemoryPoolEnd = std::get<0>(mNonPersistentMemoryStack.back());
935 mNonPersistentMemoryStack.pop_back();
936 }
937}
938
940{
942 throw std::runtime_error("temporary memory stack already blocked");
943 }
944 mHostMemoryPoolBlocked = rec->mHostMemoryPoolEnd;
945 mDeviceMemoryPoolBlocked = rec->mDeviceMemoryPoolEnd;
946}
947
949{
950 if (mNonPersistentMemoryStack.size()) {
951 throw std::runtime_error("cannot unblock while there is stacked memory");
952 }
955 mHostMemoryPoolBlocked = nullptr;
956 mDeviceMemoryPoolBlocked = nullptr;
957}
958
960{
961 mMemoryResources[res].mPtr = ptr;
962}
963
965{
966 for (uint32_t i = 0; i < mMemoryResources.size(); i++) {
969 }
970 }
973 mDirectMemoryChunks.clear();
975 mVolatileChunks.clear();
976 mVolatileMemoryStart = nullptr;
977 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_GLOBAL) {
978 mHostMemoryPool = GPUProcessor::alignPointer<constants::GPU_MEMALIGN>(mHostMemoryPermanent);
979 mDeviceMemoryPool = GPUProcessor::alignPointer<constants::GPU_MEMALIGN>(mDeviceMemoryPermanent);
982 } else {
984 }
985}
986
992
994{
995 printf("Maximum Memory Allocation: Host %'zu / Device %'zu\n", mHostMemoryUsedMax, mDeviceMemoryUsedMax);
996}
997
999{
1000 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_GLOBAL) {
1001 printf("Memory Allocation: Host %'13zd / %'13zu (Permanent %'13zd, Data %'13zd, Scratch %'13zd), Device %'13zd / %'13zu, (Permanent %'13zd, Data %'13zd, Scratch %'13zd) %zu chunks\n",
1004 mMemoryResources.size());
1005 }
1006}
1007
1009{
1010 std::map<std::string, std::array<size_t, 3>> sizes;
1011 for (uint32_t i = 0; i < mMemoryResources.size(); i++) {
1012 auto& res = mMemoryResources[i];
1013 if (res.mReuse >= 0) {
1014 continue;
1015 }
1016 auto& x = sizes[res.mName];
1017 if (res.mPtr) {
1018 x[0] += res.mSize;
1019 }
1020 if (res.mPtrDevice) {
1021 x[1] += res.mSize;
1022 }
1024 x[2] = 1;
1025 }
1026 }
1027 printf("%59s CPU / %9s GPU\n", "", "");
1028 for (auto it = sizes.begin(); it != sizes.end(); it++) {
1029 printf("Allocation %50s %s: Size %'14zu / %'14zu\n", it->first.c_str(), it->second[2] ? "P" : " ", it->second[0], it->second[1]);
1030 }
1032 for (uint32_t i = 0; i < mChains.size(); i++) {
1033 mChains[i]->PrintMemoryStatistics();
1034 }
1035}
1036
1038{
1039 if (GetProcessingSettings().noGPUMemoryRegistration) {
1040 return 0;
1041 }
1043 if (retVal == 0) {
1044 mRegisteredMemoryPtrs.emplace(ptr);
1045 }
1046 return retVal;
1047}
1048
1050{
1051 if (GetProcessingSettings().noGPUMemoryRegistration) {
1052 return 0;
1053 }
1054 const auto& pos = mRegisteredMemoryPtrs.find(ptr);
1055 if (pos != mRegisteredMemoryPtrs.end()) {
1058 }
1059 return 1;
1060}
1061
1062namespace o2::gpu::internal
1063{
1064namespace // anonymous
1065{
1066template <class T>
1067constexpr static inline int32_t getStepNum(T step, bool validCheck, int32_t N, const char* err = "Invalid step num")
1068{
1069 static_assert(sizeof(step) == sizeof(uint32_t), "Invalid step enum size");
1070 int32_t retVal = 8 * sizeof(uint32_t) - 1 - CAMath::Clz((uint32_t)step);
1071 if ((uint32_t)step == 0 || retVal >= N) {
1072 if (!validCheck) {
1073 return -1;
1074 }
1075 throw std::runtime_error("Invalid General Step");
1076 }
1077 return retVal;
1078}
1079} // anonymous namespace
1080} // namespace o2::gpu::internal
1081
1082int32_t GPUReconstruction::getRecoStepNum(RecoStep step, bool validCheck) { return internal::getStepNum(step, validCheck, gpudatatypes::N_RECO_STEPS, "Invalid Reco Step"); }
1083int32_t GPUReconstruction::getGeneralStepNum(GeneralStep step, bool validCheck) { return internal::getStepNum(step, validCheck, gpudatatypes::N_GENERAL_STEPS, "Invalid General Step"); }
1084
1086{
1087 if (!mInitialized || !GetProcessingSettings().doublePipeline || mMaster != nullptr || !mSlaves.size()) {
1088 throw std::invalid_argument("Cannot start double pipeline mode");
1089 }
1090 if (GetProcessingSettings().debugLevel >= 3) {
1091 GPUInfo("Pipeline worker started");
1092 }
1093 bool terminate = false;
1094 while (!terminate) {
1095 {
1096 std::unique_lock<std::mutex> lk(mPipelineContext->mutex);
1097 mPipelineContext->cond.wait(lk, [this] { return this->mPipelineContext->pipelineQueue.size() > 0; });
1098 mPipelineContext->workerRunning = true;
1099 }
1100 GPUReconstructionPipelineQueue* q;
1101 {
1102 std::lock_guard<std::mutex> lk(mPipelineContext->mutex);
1103 q = mPipelineContext->pipelineQueue.front();
1104 mPipelineContext->pipelineQueue.pop();
1105 }
1106 if (q->op == 1) {
1107 terminate = 1;
1108 } else {
1109 q->retVal = q->chain->RunChain();
1110 }
1111 {
1112 std::lock_guard<std::mutex> lk(q->m);
1113 q->done = true;
1114 }
1115 q->c.notify_one();
1116 mPipelineContext->workerRunning = false;
1117 mPipelineContext->cond.notify_one();
1118 }
1119 if (GetProcessingSettings().debugLevel >= 3) {
1120 GPUInfo("Pipeline worker ended");
1121 }
1122}
1123
1128
1130{
1131 std::unique_lock<std::mutex> lk(mPipelineContext->mutex);
1132 mPipelineContext->cond.wait(lk, [this] { return this->mPipelineContext->pipelineQueue.empty() && !this->mPipelineContext->workerRunning; });
1133}
1134
1136{
1139 std::unique_ptr<GPUReconstructionPipelineQueue> qu(new GPUReconstructionPipelineQueue);
1140 GPUReconstructionPipelineQueue* q = qu.get();
1141 q->chain = terminate ? nullptr : mChains[0].get();
1142 q->op = terminate ? 1 : 0;
1143 std::unique_lock<std::mutex> lkdone(q->m);
1144 {
1145 std::lock_guard<std::mutex> lkpipe(rec->mPipelineContext->mutex);
1146 if (rec->mPipelineContext->terminate) {
1147 throw std::runtime_error("Must not enqueue work after termination request");
1148 }
1149 rec->mPipelineContext->pipelineQueue.push(q);
1150 rec->mPipelineContext->terminate = terminate;
1151 rec->mPipelineContext->cond.notify_one();
1152 }
1153 q->c.wait(lkdone, [&q]() { return q->done; });
1154 if (terminate || (q->retVal && (q->retVal != 3 || !GetProcessingSettings().ignoreNonFatalGPUErrors))) {
1155 return q->retVal;
1156 }
1157 int32_t retVal2 = mChains[0]->FinalizePipelinedProcessing();
1158 return retVal2 ? retVal2 : q->retVal;
1159}
1160
1162{
1164 std::lock_guard<std::mutex> lk(rec->mPipelineContext->mutex);
1165 return rec->mPipelineContext->pipelineQueue.size() && rec->mPipelineContext->pipelineQueue.front()->op == 0 ? rec->mPipelineContext->pipelineQueue.front()->chain : nullptr;
1166}
1167
1168void GPUReconstruction::PrepareEvent() // TODO: Clean this up, this should not be called from chainTracking but before
1169{
1171 for (uint32_t i = 0; i < mChains.size(); i++) {
1172 mChains[i]->PrepareEvent();
1173 }
1174 for (uint32_t i = 0; i < mProcessors.size(); i++) {
1175 if (mProcessors[i].proc->mAllocateAndInitializeLate) {
1176 continue;
1177 }
1178 (mProcessors[i].proc->*(mProcessors[i].SetMaxData))(mHostConstantMem->ioPtrs);
1179 if (mProcessors[i].proc->mGPUProcessorType != GPUProcessor::PROCESSOR_TYPE_DEVICE && mProcessors[i].proc->mLinkedProcessor) {
1180 (mProcessors[i].proc->mLinkedProcessor->*(mProcessors[i].SetMaxData))(mHostConstantMem->ioPtrs);
1181 }
1182 }
1183 ComputeReuseMax(nullptr);
1184 AllocateRegisteredMemory(nullptr);
1185}
1186
1187int32_t GPUReconstruction::CheckErrorCodes(bool cpuOnly, bool forceShowErrors, std::vector<std::array<uint32_t, 4>>* fillErrors)
1188{
1189 int32_t retVal = 0;
1190 for (uint32_t i = 0; i < mChains.size(); i++) {
1191 if (mChains[i]->CheckErrorCodes(cpuOnly, forceShowErrors, fillErrors)) {
1192 retVal++;
1193 }
1194 }
1195 return retVal;
1196}
1197
1198int32_t GPUReconstruction::GPUChkErrA(const int64_t retval, const char* file, int32_t line, bool failOnError)
1199{
1200 if (retval == 0 || !GPUChkErrInternal(retval, file, line)) {
1201 return 0;
1202 }
1203 if (failOnError) {
1204 if (mInitialized && mInErrorHandling == false) {
1205 mInErrorHandling = true;
1206 CheckErrorCodes(false, true);
1207 }
1208 throw std::runtime_error("GPU Backend Failure");
1209 }
1210 return 1;
1211}
1212
1214{
1215 std::string f;
1216 f = dir;
1217 f += "settings.dump";
1218 DumpStructToFile(mGRPSettings.get(), f.c_str());
1219 for (uint32_t i = 0; i < mChains.size(); i++) {
1220 mChains[i]->DumpSettings(dir);
1221 }
1222}
1223
1224void GPUReconstruction::UpdateDynamicSettings(const GPUSettingsRecDynamic* d)
1225{
1226 UpdateSettings(nullptr, nullptr, d);
1227}
1228
1229void GPUReconstruction::UpdateSettings(const GPUSettingsGRP* g, const GPUSettingsProcessing* p, const GPUSettingsRecDynamic* d)
1230{
1231 if (g) {
1232 *mGRPSettings = *g;
1233 }
1234 if (p) {
1235 mProcessingSettings->debugLevel = p->debugLevel;
1236 mProcessingSettings->resetTimers = p->resetTimers;
1237 }
1238 GPURecoStepConfiguration* w = nullptr;
1240 w = &mRecoSteps;
1241 }
1242 param().UpdateSettings(g, p, w, d);
1243 if (mInitialized) {
1245 }
1246}
1247
1249{
1250 std::string f;
1251 f = dir;
1252 f += "settings.dump";
1253 new (mGRPSettings.get()) GPUSettingsGRP;
1254 bool error;
1255 ReadStructFromFile(f.c_str(), mGRPSettings.get(), &error, true);
1256 if (error) {
1257 return 1;
1258 }
1260 for (uint32_t i = 0; i < mChains.size(); i++) {
1261 mChains[i]->ReadSettings(dir);
1262 }
1263 return 0;
1264}
1265
1266void GPUReconstruction::SetSettings(float solenoidBzNominalGPU, const GPURecoStepConfiguration* workflow)
1267{
1268#ifdef GPUCA_STANDALONE
1269 GPUSettingsGRP grp;
1270 grp.solenoidBzNominalGPU = solenoidBzNominalGPU;
1271 SetSettings(&grp, nullptr, nullptr, workflow);
1272#else
1274 config.ReadConfigurableParam(config);
1275 config.configGRP.solenoidBzNominalGPU = solenoidBzNominalGPU;
1276 SetSettings(&config.configGRP, &config.configReconstruction, &config.configProcessing, workflow);
1277#endif
1278}
1279
1280void GPUReconstruction::SetSettings(const GPUSettingsGRP* grp, const GPUSettingsRec* rec, const GPUSettingsProcessing* proc, const GPURecoStepConfiguration* workflow)
1281{
1282 if (mInitialized) {
1283 GPUError("Cannot update settings while initialized");
1284 throw std::runtime_error("Settings updated while initialized");
1285 }
1286 *mGRPSettings = *grp;
1287 if (proc) {
1289 }
1290 if (workflow) {
1291 mRecoSteps.steps = workflow->steps;
1293 mRecoSteps.inputs = workflow->inputs;
1294 mRecoSteps.outputs = workflow->outputs;
1295 }
1296 param().SetDefaults(mGRPSettings.get(), rec, proc, workflow);
1297}
1298
1300{
1301 GPUOutputControl outputControl;
1302 outputControl.set(ptr, size);
1303 SetOutputControl(outputControl);
1304}
1305
1309void GPUReconstruction::SetResetTimers(bool reset) { mProcessingSettings->resetTimers = reset; }
1314
1315ThrustVolatileAllocator::ThrustVolatileAllocator(GPUReconstruction* r)
1316{
1317 mAlloc = [&r](size_t n) { return (char*)r->AllocateVolatileDeviceMemory(n); };
1318}
unsigned char uint8_t
unsigned short int uint16_t
unsigned int uint32_t
int32_t i
uint32_t op
bool done
int32_t retVal
GPUChain * chain
configStandalone Light OpenGL display display_light Apply smoothing to points Enable Z buffer Highlight links Highlight seeds Highlight sector tracks Highlight final tracks Show individual Show related Highlight grid Show only individual collision Color clusters belonging to track objects Set line Enable drawing TPC data Enable drawing TOF data Enable drawing magnetic field Set field line step count Invert colors configStandalone Heavy OpenGL display display_heavy Show tracks with ITS contribution Show tracks with TOF contribution Enable propagation of loopers Show clusters at nominal x of pad row for early transformed data Split CE tracks when they cross the central Mark clusters Mark adjacent clusters Hide rejected tracks Apply filter on tracks to be displayed Separate increase X h configStandalone Camera window renderer OpenGL display display_camera Orient the camera such that the y axis is always upwards Full Screen MultiSample Anti Downsampling Anti Enable Vertical Sync Target frame or if the backend cannot limit to the refresh Use OpenGL indirect draws to reduce number of draw calls Display h configStandalone Vulkan display display_vulkan Have uniform buffers in host accessible device memory h configStandalone Internal display display_internal Number of updates the Qt GUI may send ahead of the last one the display has before it waits for the display Opacity of the console background Number of console lines the history is shortened to this when it reaches twice the size Time in seconds an info message fades out after setInfoTime h OpenGL display display Font size Do not use Freetype for font Renderer for event Frontend to use for event ROOT macros used as track filter Console script file
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error First event to Number of events to Number of iterations to Number of iterations to Number of initial iterations excluded from const char Directory with events to Events directory is and not inside events d
configStandalone Reconstruction rec_tpc multiplier for the combined cluster track error during track following area in cm for the search of for z only used if decrease removal by when above this local occupancy rowx Shift of TPC revert cluster flag edge bit distance to edge exceeds this ZOffset for secondary multiple of radius max P of electron K BB bands crossing max P of electron p BB bands crossing max P of electron d BB bands crossing max P of electron t BB bands crossing min accepted dEdxMaxTPC of the track Additive extra cluster error for Y2 if edge flag set Additive extra cluster error for Y2 if single set Additive extra cluster error for Y2 if or single set Additive extra cluster error for Z2 if or single set Extra errors to add to track when crossing depending on addErrorsCECrossing Inner portion of y range in sector that is not used in searching for extrapolated track candidates factor2YS for track merging factor2K for track merging Margin in cm of Y position when rejecting edge clusters based on uncorrected track Y Max fraction of shared clusters for tracklet Except pad from cluster finding if total number of charges in a fragment is above this Except pad from cluster finding if number of consecutive charges in a fragment is above this Threshold where a timebin is considered disabling the noisy pad check for that pad Treat signal above this minimum as saturated Smoothing factor for the exponential Highly Ionising Particle tail filter Number of timebins to use for the averaging Add additional custom track errors when crossing relative as fraction of Do N fit passes in final fit of< 0:reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking(prolong tracks to adjacent sectors to find short segments)") AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit(1:attachment, 2:propagation, 4:loop following, 8:mirroring)") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression(0=no, 1=strategy A, 2=strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits(1=truncate charge/width LSB, 2=differences, 4=track-model)") AddOptionRTC(compressionSortOrder, uint8_t, o2::gpu::GPUSettings::SortTime, "", 0, "Sort order of TPC compression(0=time, 1=pad, 2=Z-time-pad, 3=Z-pad-time, 4=no sorting(use incoming order))") AddOptionRTC(sigBitsCharge, uint8_t, 4, "", 0, "Number of significant bits for TPC cluster charge in compression mode 1") AddOptionRTC(sigBitsWidth, uint8_t, 3, "", 0, "Number of significant bits for TPC cluster width in compression mode 1") AddOptionRTC(dropLoopers, uint8_t, 0, "", 0, "Drop looping tracks starting from second loop") AddOptionRTC(mergerInterpolateErrors, uint8_t, 1, "", 0, "Use interpolation instead of extrapolation for chi2 based cluster rejection") AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF Merge tracks accross the central electrode Enable PID response Add sub threshold clusters in TPC dEdx computation OR mask of TPC flags that will reject the cluster in alternative dEdx Reject edge clusters based on uncorrected track Y during track fit Increase decrease removal by when on this number of pads from the edge h configStandalone Reconstruction rec_trd Max chi2 for TRD tracklets to be matched to a track Chi2 cut for strict matching mode Number of sigmas for ITS TPC track time error estimate Addition to search road around track prolongation along Y in cm Optimal tracklet rphi resolution in maximum chi2 per attached tracklet for TRD tracks take into account the full likelihood of the slope including exponential tails Tracks with less attached tracklets are discarded after the tracking Post trigger Pile up integration time in BCs fits all iterations but the last one in projections Use material lookup table for TPC refit Apply workarounds to avoid cyclic merge should not be needed tpc dyn rtc
configStandalone Reconstruction rec_tpc multiplier for the combined cluster track error during track following area in cm for the search of for z only used if decrease removal by when above this local occupancy rowx Shift of TPC revert cluster flag edge bit distance to edge exceeds this ZOffset for secondary multiple of radius max P of electron K BB bands crossing max P of electron p BB bands crossing max P of electron d BB bands crossing max P of electron t BB bands crossing min accepted dEdxMaxTPC of the track Additive extra cluster error for Y2 if edge flag set Additive extra cluster error for Y2 if single set Additive extra cluster error for Y2 if or single set Additive extra cluster error for Z2 if or single set Extra errors to add to track when crossing depending on addErrorsCECrossing Inner portion of y range in sector that is not used in searching for extrapolated track candidates factor2YS for track merging factor2K for track merging Margin in cm of Y position when rejecting edge clusters based on uncorrected track Y Max fraction of shared clusters for tracklet Except pad from cluster finding if total number of charges in a fragment is above this Except pad from cluster finding if number of consecutive charges in a fragment is above this Threshold where a timebin is considered disabling the noisy pad check for that pad Treat signal above this minimum as saturated Smoothing factor for the exponential Highly Ionising Particle tail filter Number of timebins to use for the averaging Add additional custom track errors when crossing relative as fraction of Do N fit passes in final fit of< 0:reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking(prolong tracks to adjacent sectors to find short segments)") AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit(1:attachment, 2:propagation, 4:loop following, 8:mirroring)") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression(0=no, 1=strategy A, 2=strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits(1=truncate charge/width LSB, 2=differences, 4=track-model)") AddOptionRTC(compressionSortOrder, uint8_t, o2::gpu::GPUSettings::SortTime, "", 0, "Sort order of TPC compression(0=time, 1=pad, 2=Z-time-pad, 3=Z-pad-time, 4=no sorting(use incoming order))") AddOptionRTC(sigBitsCharge, uint8_t, 4, "", 0, "Number of significant bits for TPC cluster charge in compression mode 1") AddOptionRTC(sigBitsWidth, uint8_t, 3, "", 0, "Number of significant bits for TPC cluster width in compression mode 1") AddOptionRTC(dropLoopers, uint8_t, 0, "", 0, "Drop looping tracks starting from second loop") AddOptionRTC(mergerInterpolateErrors, uint8_t, 1, "", 0, "Use interpolation instead of extrapolation for chi2 based cluster rejection") AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF Merge tracks accross the central electrode Enable PID response Add sub threshold clusters in TPC dEdx computation OR mask of TPC flags that will reject the cluster in alternative dEdx Reject edge clusters based on uncorrected track Y during track fit Increase decrease removal by when on this number of pads from the edge h configStandalone Reconstruction rec_trd Max chi2 for TRD tracklets to be matched to a track Chi2 cut for strict matching mode Number of sigmas for ITS TPC track time error estimate Addition to search road around track prolongation along Y in cm Optimal tracklet rphi resolution in maximum chi2 per attached tracklet for TRD tracks take into account the full likelihood of the slope including exponential tails Tracks with less attached tracklets are discarded after the tracking Post trigger Pile up integration time in BCs fits all iterations but the last one in projections Use material lookup table for TPC refit Apply workarounds to avoid cyclic merge should not be needed tpc dyn configStandalone Processing proc_rtc Replace constant variables static by constexpr expressions Compile RTC in deterministic with NO_FAST_MATH flags and GPUCA_DETERMINISTIC_MODE define Use RTC to optimize GPU code h configStandalone Processing proc_rtctech Use a file lock to serialize access to the cache folder int32_t
configStandalone Reconstruction rec_tpc multiplier for the combined cluster track error during track following area in cm for the search of for z only used if decrease removal by when above this local occupancy rowx Shift of TPC revert cluster flag edge bit distance to edge exceeds this ZOffset for secondary multiple of radius max P of electron K BB bands crossing max P of electron p BB bands crossing max P of electron d BB bands crossing max P of electron t BB bands crossing min accepted dEdxMaxTPC of the track Additive extra cluster error for Y2 if edge flag set Additive extra cluster error for Y2 if single set Additive extra cluster error for Y2 if or single set Additive extra cluster error for Z2 if or single set Extra errors to add to track when crossing depending on addErrorsCECrossing Inner portion of y range in sector that is not used in searching for extrapolated track candidates factor2YS for track merging factor2K for track merging Margin in cm of Y position when rejecting edge clusters based on uncorrected track Y Max fraction of shared clusters for tracklet Except pad from cluster finding if total number of charges in a fragment is above this Except pad from cluster finding if number of consecutive charges in a fragment is above this Threshold where a timebin is considered disabling the noisy pad check for that pad Treat signal above this minimum as saturated Smoothing factor for the exponential Highly Ionising Particle tail filter Number of timebins to use for the averaging Add additional custom track errors when crossing relative as fraction of Do N fit passes in final fit of< 0:reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking(prolong tracks to adjacent sectors to find short segments)") AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit(1:attachment, 2:propagation, 4:loop following, 8:mirroring)") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression(0=no, 1=strategy A, 2=strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits(1=truncate charge/width LSB, 2=differences, 4=track-model)") AddOptionRTC(compressionSortOrder, uint8_t, o2::gpu::GPUSettings::SortTime, "", 0, "Sort order of TPC compression(0=time, 1=pad, 2=Z-time-pad, 3=Z-pad-time, 4=no sorting(use incoming order))") AddOptionRTC(sigBitsCharge, uint8_t, 4, "", 0, "Number of significant bits for TPC cluster charge in compression mode 1") AddOptionRTC(sigBitsWidth, uint8_t, 3, "", 0, "Number of significant bits for TPC cluster width in compression mode 1") AddOptionRTC(dropLoopers, uint8_t, 0, "", 0, "Drop looping tracks starting from second loop") AddOptionRTC(mergerInterpolateErrors, uint8_t, 1, "", 0, "Use interpolation instead of extrapolation for chi2 based cluster rejection") AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF Merge tracks accross the central electrode Enable PID response Add sub threshold clusters in TPC dEdx computation OR mask of TPC flags that will reject the cluster in alternative dEdx Reject edge clusters based on uncorrected track Y during track fit Increase decrease removal by when on this number of pads from the edge h configStandalone Reconstruction rec_trd Max chi2 for TRD tracklets to be matched to a track Chi2 cut for strict matching mode Number of sigmas for ITS TPC track time error estimate Addition to search road around track prolongation along Y in cm Optimal tracklet rphi resolution in maximum chi2 per attached tracklet for TRD tracks take into account the full likelihood of the slope including exponential tails Tracks with less attached tracklets are discarded after the tracking Post trigger Pile up integration time in BCs fits all iterations but the last one in projections Use material lookup table for TPC refit Apply workarounds to avoid cyclic merge should not be needed tpc dyn configStandalone proc
configStandalone rec
configStandalone Reconstruction rec_tpc multiplier for the combined cluster track error during track following area in cm for the search of for z only used if decrease removal by when above this local occupancy rowx Shift of TPC revert cluster flag edge bit distance to edge exceeds this ZOffset for secondary multiple of radius max P of electron K BB bands crossing max P of electron p BB bands crossing max P of electron d BB bands crossing max P of electron t BB bands crossing min accepted dEdxMaxTPC of the track Additive extra cluster error for Y2 if edge flag set Additive extra cluster error for Y2 if single set Additive extra cluster error for Y2 if or single set Additive extra cluster error for Z2 if or single set Extra errors to add to track when crossing depending on addErrorsCECrossing Inner portion of y range in sector that is not used in searching for extrapolated track candidates factor2YS for track merging factor2K for track merging Margin in cm of Y position when rejecting edge clusters based on uncorrected track Y Max fraction of shared clusters for tracklet Except pad from cluster finding if total number of charges in a fragment is above this Except pad from cluster finding if number of consecutive charges in a fragment is above this Threshold where a timebin is considered disabling the noisy pad check for that pad Treat signal above this minimum as saturated Smoothing factor for the exponential Highly Ionising Particle tail filter Number of timebins to use for the averaging Add additional custom track errors when crossing relative as fraction of Do N fit passes in final fit of< 0:reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking(prolong tracks to adjacent sectors to find short segments)") AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit(1:attachment, 2:propagation, 4:loop following, 8:mirroring)") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression(0=no, 1=strategy A, 2=strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits(1=truncate charge/width LSB, 2=differences, 4=track-model)") AddOptionRTC(compressionSortOrder, uint8_t, o2::gpu::GPUSettings::SortTime, "", 0, "Sort order of TPC compression(0=time, 1=pad, 2=Z-time-pad, 3=Z-pad-time, 4=no sorting(use incoming order))") AddOptionRTC(sigBitsCharge, uint8_t, 4, "", 0, "Number of significant bits for TPC cluster charge in compression mode 1") AddOptionRTC(sigBitsWidth, uint8_t, 3, "", 0, "Number of significant bits for TPC cluster width in compression mode 1") AddOptionRTC(dropLoopers, uint8_t, 0, "", 0, "Drop looping tracks starting from second loop") AddOptionRTC(mergerInterpolateErrors, uint8_t, 1, "", 0, "Use interpolation instead of extrapolation for chi2 based cluster rejection") AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF Merge tracks accross the central electrode Enable PID response Add sub threshold clusters in TPC dEdx computation OR mask of TPC flags that will reject the cluster in alternative dEdx Reject edge clusters based on uncorrected track Y during track fit Increase decrease removal by when on this number of pads from the edge h configStandalone Reconstruction rec_trd Max chi2 for TRD tracklets to be matched to a track Chi2 cut for strict matching mode Number of sigmas for ITS TPC track time error estimate Addition to search road around track prolongation along Y in cm Optimal tracklet rphi resolution in maximum chi2 per attached tracklet for TRD tracks take into account the full likelihood of the slope including exponential tails Tracks with less attached tracklets are discarded after the tracking Post trigger Pile up integration time in BCs fits all iterations but the last one in projections Use material lookup table for TPC refit Apply workarounds to avoid cyclic merge should not be needed tpc dyn configStandalone Processing proc_rtc Replace constant variables static by constexpr expressions Compile RTC in deterministic with NO_FAST_MATH flags and GPUCA_DETERMINISTIC_MODE define Use RTC to optimize GPU code h configStandalone Processing proc_rtctech Use a file lock to serialize access to the cache folder Print launch bounds used for RTC code as debugging for exit after Allow RTC with slave GPUReconstruction instances with optConstexpr and optSpecialcode Maximum number of RTC cache files per backend in the cache the oldest are Prepend RTC compilation commands by this string Load a parameter object containing the launch bounds from a file h configStandalone Processing proc_param TPC Cluster Z Error Parameterization h configStandalone Processing settings for neural network proc_nn std::string Specify inference device(cpu(default), rocm, cuda)") AddOption(nnInferenceDeviceId
configStandalone Reconstruction rec_tpc multiplier for the combined cluster track error during track following area in cm for the search of for z only used if decrease removal by when above this local occupancy rowx Shift of TPC revert cluster flag edge bit distance to edge exceeds this ZOffset for secondary multiple of radius max P of electron K BB bands crossing max P of electron p BB bands crossing max P of electron d BB bands crossing max P of electron t BB bands crossing min accepted dEdxMaxTPC of the track Additive extra cluster error for Y2 if edge flag set Additive extra cluster error for Y2 if single set Additive extra cluster error for Y2 if or single set Additive extra cluster error for Z2 if or single set Extra errors to add to track when crossing depending on addErrorsCECrossing Inner portion of y range in sector that is not used in searching for extrapolated track candidates factor2YS for track merging factor2K for track merging Margin in cm of Y position when rejecting edge clusters based on uncorrected track Y Max fraction of shared clusters for tracklet Except pad from cluster finding if total number of charges in a fragment is above this Except pad from cluster finding if number of consecutive charges in a fragment is above this Threshold where a timebin is considered disabling the noisy pad check for that pad Treat signal above this minimum as saturated Smoothing factor for the exponential Highly Ionising Particle tail filter Number of timebins to use for the averaging Add additional custom track errors when crossing relative as fraction of Do N fit passes in final fit of< 0:reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking(prolong tracks to adjacent sectors to find short segments)") AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit(1:attachment, 2:propagation, 4:loop following, 8:mirroring)") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression(0=no, 1=strategy A, 2=strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits(1=truncate charge/width LSB, 2=differences, 4=track-model)") AddOptionRTC(compressionSortOrder, uint8_t, o2::gpu::GPUSettings::SortTime, "", 0, "Sort order of TPC compression(0=time, 1=pad, 2=Z-time-pad, 3=Z-pad-time, 4=no sorting(use incoming order))") AddOptionRTC(sigBitsCharge, uint8_t, 4, "", 0, "Number of significant bits for TPC cluster charge in compression mode 1") AddOptionRTC(sigBitsWidth, uint8_t, 3, "", 0, "Number of significant bits for TPC cluster width in compression mode 1") AddOptionRTC(dropLoopers, uint8_t, 0, "", 0, "Drop looping tracks starting from second loop") AddOptionRTC(mergerInterpolateErrors, uint8_t, 1, "", 0, "Use interpolation instead of extrapolation for chi2 based cluster rejection") AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF Merge tracks accross the central electrode Enable PID response Add sub threshold clusters in TPC dEdx computation OR mask of TPC flags that will reject the cluster in alternative dEdx Reject edge clusters based on uncorrected track Y during track fit Increase decrease removal by when on this number of pads from the edge h configStandalone Reconstruction rec_trd Max chi2 for TRD tracklets to be matched to a track Chi2 cut for strict matching mode Number of sigmas for ITS TPC track time error estimate Addition to search road around track prolongation along Y in cm Optimal tracklet rphi resolution in maximum chi2 per attached tracklet for TRD tracks take into account the full likelihood of the slope including exponential tails Tracks with less attached tracklets are discarded after the tracking Post trigger Pile up integration time in BCs fits all iterations but the last one in projections Use material lookup table for TPC refit Apply workarounds to avoid cyclic merge should not be needed tpc dyn configStandalone Processing proc_rtc Replace constant variables static by constexpr expressions Compile RTC in deterministic with NO_FAST_MATH flags and GPUCA_DETERMINISTIC_MODE define Use RTC to optimize GPU code h rtctech
char error[CURL_ERROR_SIZE]
uint16_t pos
Definition RawData.h:3
uint32_t res
Definition RawData.h:0
TBranch * ptr
void set(S v)
Definition bitfield.h:55
bool isSet(const bitfield &v) const
Definition bitfield.h:66
const GPUSettingsDisplay * GetEventDisplayConfig() const
const GPUSettingsQA * GetQAConfig() const
static void dumpConfig(const GPUSettingsRec *rec, const GPUSettingsProcessing *proc, const GPUSettingsQA *qa, const GPUSettingsDisplay *display, const GPUSettingsDeviceBackend *device, const GPURecoStepConfiguration *workflow)
void * SetDevicePointers(void *ptr) const
void * SetPointers(void *ptr) const
static void computePointerWithAlignment(T *&basePtr, S *&objPtr, size_t nEntries=1)
GPURecoStepConfiguration mRecoSteps
std::unordered_set< const void * > mRegisteredMemoryPtrs
int16_t RegisterMemoryAllocationHelper(GPUProcessor *proc, void *(GPUProcessor::*setPtr)(void *), int32_t type, const char *name, const GPUMemoryReuse &re)
std::vector< std::unique_ptr< GPUChain > > mChains
void * AllocateVolatileMemory(size_t size, bool device)
ThrustVolatileAllocator getThrustVolatileDeviceAllocator()
std::unique_ptr< GPUMemorySizeScalers > mMemoryScalers
void AllocateRegisteredForeignMemory(int16_t res, GPUReconstruction *rec, GPUOutputControl *control=nullptr)
std::unique_ptr< T > ReadStructFromFile(const char *file, T *obj=nullptr, bool *errorOnMissing=nullptr, bool allowSmaller=false)
void SetInputControl(void *ptr, size_t size)
GPUConstantMem * mDeviceConstantMem
void ConstructGPUProcessor(GPUProcessor *proc)
std::shared_ptr< GPUROOTDumpCore > mROOTDump
const GPUSettingsDeviceBackend & GetDeviceBackendSettings() const
void ComputeReuseMax(GPUProcessor *proc)
void SetMemoryExternalInput(int16_t res, void *ptr)
int32_t getGeneralStepNum(GeneralStep step, bool validCheck=true)
static constexpr uint32_t NSECTORS
RecoStepField GetRecoStepsGPU() const
void RegisterGPUDeviceProcessor(GPUProcessor *proc, GPUProcessor *slaveProcessor)
std::vector< GPUReconstruction * > mSlaves
std::vector< std::tuple< void *, void *, size_t, size_t, uint64_t > > mNonPersistentMemoryStack
alignedDeleter< char, constants::GPU_BUFFER_ALIGNMENT > alignedDefaultBufferDeleter
void UpdateDynamicSettings(const GPUSettingsRecDynamic *d)
std::unique_ptr< GPUSettingsDeviceBackend > mDeviceBackendSettings
std::vector< GPUMemoryResource > mMemoryResources
std::unique_ptr< GPUReconstructionPipelineContext > mPipelineContext
std::unique_ptr< GPUConstantMem > mHostConstantMem
void ResetRegisteredMemoryPointers(GPUProcessor *proc)
void DumpStructToFile(const T *obj, const char *file)
void AllocateRegisteredMemoryInternal(GPUMemoryResource *res, GPUOutputControl *control, GPUReconstruction *recPool)
std::vector< std::unique_ptr< char[], alignedDefaultBufferDeleter > > mVolatileChunks
virtual int32_t GPUChkErrInternal(const int64_t retval, const char *file, int32_t line) const
virtual int32_t registerMemoryForGPU_internal(const void *ptr, size_t size)=0
virtual size_t WriteToConstantMemory(size_t offset, const void *src, size_t size, int32_t stream=-1, gpu_reconstruction_kernels::deviceEvent *ev=nullptr)=0
std::unordered_map< GPUMemoryReuse::ID, MemoryReuseMeta > mMemoryReuse1to1
std::vector< ProcessorData > mProcessors
void * AllocateVolatileDeviceMemory(size_t size)
virtual int32_t InitDevice()=0
void SetSettings(float solenoidBzNominalGPU, const GPURecoStepConfiguration *workflow=nullptr)
const GPUCalibObjectsConst & GetCalib() const
const GPUTrackingInOutPointers GetIOPtrs() const
virtual std::unique_ptr< gpu_reconstruction_kernels::threadContext > GetThreadContext()=0
GPUReconstruction(const GPUReconstruction &)=delete
static constexpr GeometryType geometryType
void WriteConstantParams(int32_t stream=-1)
void FreeRegisteredMemory(GPUProcessor *proc, bool freeCustom=false, bool freePermanent=false)
virtual RecoStepField AvailableGPURecoSteps()
static constexpr const char *const IOTYPENAMES[]
void PopNonPersistentMemory(RecoStep step, uint64_t tag, const GPUProcessor *proc=nullptr)
void UpdateSettings(const GPUSettingsGRP *g, const GPUSettingsProcessing *p=nullptr, const GPUSettingsRecDynamic *d=nullptr)
int32_t CheckErrorCodes(bool cpuOnly=false, bool forceShowErrors=false, std::vector< std::array< uint32_t, 4 > > *fillErrors=nullptr)
const GPUParam & GetParam() const
void ClearAllocatedMemory(bool clearOutputs=true)
static constexpr const char *const GEOMETRY_TYPE_NAMES[]
virtual int32_t ExitDevice()=0
std::unique_ptr< GPUSettingsGRP > mGRPSettings
std::unique_ptr< GPUSettingsProcessing > mProcessingSettings
void PushNonPersistentMemory(uint64_t tag)
int32_t getRecoStepNum(RecoStep step, bool validCheck=true)
virtual int32_t unregisterMemoryForGPU_internal(const void *ptr)=0
void BlockStackedMemory(GPUReconstruction *rec)
const GPUSettingsProcessing & GetProcessingSettings() const
void DumpSettings(const char *dir="")
std::vector< std::unique_ptr< char[], alignedDefaultBufferDeleter > > mNonPersistentIndividualDirectAllocations
void * AllocateDirectMemory(size_t size, int32_t type)
virtual void GetITSTraits(std::unique_ptr< o2::its::TrackerTraits< 7 > > *trackerTraits, std::unique_ptr< o2::its::VertexerTraits< 7 > > *vertexerTraits, std::unique_ptr< o2::its::TimeFrame< 7 > > *timeFrame)
int32_t unregisterMemoryForGPU(const void *ptr)
int32_t registerMemoryForGPU(const void *ptr, size_t size)
void SetDebugLevelTmp(int32_t level)
size_t AllocateRegisteredMemoryHelper(GPUMemoryResource *res, void *&ptr, void *&memorypool, void *memorybase, size_t memorysize, void *(GPUMemoryResource::*SetPointers)(void *) const, void *&memorypoolend, const char *device)
std::vector< std::unique_ptr< char[], alignedDefaultBufferDeleter > > mDirectMemoryChunks
int32_t EnqueuePipeline(bool terminate=false)
std::shared_ptr< GPUReconstructionThreading > mThreading
std::vector< GPUMemoryResource * > mNonPersistentIndividualAllocations
int32_t GPUChkErrA(const int64_t error, const char *file, int32_t line, bool failOnError)
size_t AllocateRegisteredMemory(GPUProcessor *proc, bool resetCustom=false)
int32_t ReadSettings(const char *dir="")
void SetOutputControl(const GPUOutputControl &v)
void SetSector(int32_t iSector)
#define TPC_MAX_FRAGMENT_LEN_GPU
#define TPC_MAX_FRAGMENT_LEN_HOST
GLboolean GLboolean g
Definition glcorearb.h:1233
GLdouble n
Definition glcorearb.h:1982
GLint GLenum GLint x
Definition glcorearb.h:403
const GLfloat * m
Definition glcorearb.h:4066
GLsizeiptr size
Definition glcorearb.h:659
GLuint GLsizei const GLuint const GLintptr const GLsizeiptr * sizes
Definition glcorearb.h:2595
GLuint const GLchar * name
Definition glcorearb.h:781
GLdouble f
Definition glcorearb.h:310
GLboolean GLboolean GLboolean b
Definition glcorearb.h:1233
GLint GLint GLsizei GLint GLenum GLenum type
Definition glcorearb.h:275
typedef void(APIENTRYP PFNGLCULLFACEPROC)(GLenum mode)
GLint level
Definition glcorearb.h:275
GLboolean r
Definition glcorearb.h:1233
GLenum GLfloat param
Definition glcorearb.h:271
GLboolean GLboolean GLboolean GLboolean a
Definition glcorearb.h:1233
GLuint GLuint stream
Definition glcorearb.h:1806
GLubyte GLubyte GLubyte GLubyte w
Definition glcorearb.h:852
std::unique_ptr< GPUDisplayFrontendInterface > eventDisplay
std::string qTag2Str(const T tag)
Definition strtag.h:36
GPUTPCTracker tpcTrackers[GPUTPCGeometry::NSECTORS]
GPUCalibObjectsConst calibObjects
GPUTPCClusterFinder tpcClusterer[GPUTPCGeometry::NSECTORS]
void set(void *p, size_t s)
std::function< void *(size_t)> allocator
void SetDefaults(float solenoidBz, bool assumeConstantBz=false)
Definition GPUParam.cxx:33
void UpdateSettings(const GPUSettingsGRP *g, const GPUSettingsProcessing *p=nullptr, const GPURecoStepConfiguration *w=nullptr, const GPUSettingsRecDynamic *d=nullptr)
Definition GPUParam.cxx:95
gpudatatypes::RecoStepField steps
gpudatatypes::InOutTypeField inputs
gpudatatypes::RecoStepField stepsGPUMask
gpudatatypes::InOutTypeField outputs
std::queue< GPUReconstructionPipelineQueue * > pipelineQueue