Project
Loading...
Searching...
No Matches
GPUReconstruction.cxx
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
14
15#include <cstring>
16#include <cstdio>
17#include <iostream>
18#include <mutex>
19#include <string>
20#include <map>
21#include <queue>
22#include <mutex>
23#include <condition_variable>
24#include <array>
25
26#include "GPUReconstruction.h"
29#include "GPUReconstructionIO.h"
30#include "GPUROOTDumpCore.h"
31#include "GPUConfigDump.h"
32#include "GPUChainTracking.h"
33#include "GPUConstantMem.h"
34#include "GPUCommonHelpers.h"
35#include "GPUSettings.h"
36
37#include "GPUMemoryResource.h"
38#include "GPUChain.h"
40
41#include "GPULogging.h"
42#include "utils/strtag.h"
43
44#ifdef GPUCA_O2_LIB
46#endif
47
49
50namespace o2::gpu
51{
52namespace // anonymous
53{
54struct GPUReconstructionPipelineQueue {
55 uint32_t op = 0; // For now, 0 = process, 1 = terminate
56 GPUChain* chain = nullptr;
57 std::mutex m;
58 std::condition_variable c;
59 bool done = false;
60 int32_t retVal = 0;
61};
62} // namespace
63
65 std::queue<GPUReconstructionPipelineQueue*> queue;
66 std::mutex mutex;
67 std::condition_variable cond;
68 bool terminate = false;
69};
70} // namespace o2::gpu
71
72using namespace o2::gpu;
73
74constexpr const char* const GPUReconstruction::GEOMETRY_TYPE_NAMES[];
75constexpr const char* const GPUReconstruction::IOTYPENAMES[];
77
78static ptrdiff_t ptrDiff(void* a, void* b) { return (char*)a - (char*)b; }
79
80GPUReconstruction::GPUReconstruction(const GPUSettingsDeviceBackend& cfg) : mHostConstantMem(new GPUConstantMem), mGRPSettings(new GPUSettingsGRP), mDeviceBackendSettings(new GPUSettingsDeviceBackend(cfg)), mProcessingSettings(new GPUSettingsProcessing)
81{
82 if (cfg.master) {
84 throw std::invalid_argument("device type of master and slave GPUReconstruction does not match");
85 }
86 if (cfg.master->mMaster) {
87 throw std::invalid_argument("Cannot be slave to a slave");
88 }
89 mMaster = cfg.master;
90 mSlaveId = cfg.master->mSlaves.size();
91 cfg.master->mSlaves.emplace_back(this);
92 }
95 for (uint32_t i = 0; i < NSECTORS; i++) {
96 processors()->tpcTrackers[i].SetSector(i); // TODO: Move to a better place
98#ifdef GPUCA_HAS_ONNX
99 processors()->tpcNNClusterer[i].mISector = i;
100#endif
101 }
102#ifndef GPUCA_NO_ROOT
103 mROOTDump = GPUROOTDumpCore::getAndCreate();
104#endif
105}
106
108{
109 if (mInitialized) {
110 GPUError("GPU Reconstruction not properly deinitialized!");
111 }
112}
113
114void GPUReconstruction::GetITSTraits(std::unique_ptr<o2::its::TrackerTraits<7>>* trackerTraits, std::unique_ptr<o2::its::VertexerTraits>* vertexerTraits, std::unique_ptr<o2::its::TimeFrame<7>>* timeFrame)
115{
116 if (trackerTraits) {
117 trackerTraits->reset(new o2::its::TrackerTraits<7>);
118 }
119 if (vertexerTraits) {
120 vertexerTraits->reset(new o2::its::VertexerTraits);
121 }
122 if (timeFrame) {
123 timeFrame->reset(new o2::its::TimeFrame<7>);
124 }
125}
126
128{
129 return std::max<int32_t>(0, tbb::this_task_arena::current_thread_index());
130}
131
133{
134 if (mMaster) {
135 throw std::runtime_error("Must not call init on slave!");
136 }
137 int32_t retVal = InitPhaseBeforeDevice();
138 if (retVal) {
139 return retVal;
140 }
141 for (uint32_t i = 0; i < mSlaves.size(); i++) {
142 retVal = mSlaves[i]->InitPhaseBeforeDevice();
143 if (retVal) {
144 GPUError("Error initialization slave (before deviceinit)");
145 return retVal;
146 }
147 mNStreams = std::max(mNStreams, mSlaves[i]->mNStreams);
150 }
151 if (InitDevice()) {
152 return 1;
153 }
154 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_GLOBAL) {
157 } else {
159 }
161 return 1;
162 }
163 for (uint32_t i = 0; i < mSlaves.size(); i++) {
164 mSlaves[i]->mDeviceMemoryBase = mDeviceMemoryPermanent;
165 mSlaves[i]->mHostMemoryBase = mHostMemoryPermanent;
166 mSlaves[i]->mDeviceMemorySize = mDeviceMemorySize - ptrDiff(mSlaves[i]->mDeviceMemoryBase, mDeviceMemoryBase);
167 mSlaves[i]->mHostMemorySize = mHostMemorySize - ptrDiff(mSlaves[i]->mHostMemoryBase, mHostMemoryBase);
168 mSlaves[i]->mHostMemoryPoolEnd = mHostMemoryPoolEnd;
169 mSlaves[i]->mDeviceMemoryPoolEnd = mDeviceMemoryPoolEnd;
170 if (mSlaves[i]->InitDevice()) {
171 GPUError("Error initialization slave (deviceinit)");
172 return 1;
173 }
175 GPUError("Error initialization slave (permanent memory)");
176 return 1;
177 }
178 mDeviceMemoryPermanent = mSlaves[i]->mDeviceMemoryPermanent;
179 mHostMemoryPermanent = mSlaves[i]->mHostMemoryPermanent;
180 }
182 if (retVal) {
183 return retVal;
184 }
186 for (uint32_t i = 0; i < mSlaves.size(); i++) {
187 mSlaves[i]->mDeviceMemoryPermanent = mDeviceMemoryPermanent;
188 mSlaves[i]->mHostMemoryPermanent = mHostMemoryPermanent;
189 retVal = mSlaves[i]->InitPhaseAfterDevice();
190 if (retVal) {
191 GPUError("Error initialization slave (after device init)");
192 return retVal;
193 }
194 mSlaves[i]->ClearAllocatedMemory();
195 }
196 debugInit();
197 return 0;
198}
199
200namespace o2::gpu::internal
201{
202static uint32_t getDefaultNThreads()
203{
204 const char* tbbEnv = getenv("TBB_NUM_THREADS");
205 uint32_t tbbNum = tbbEnv ? atoi(tbbEnv) : 0;
206 if (tbbNum) {
207 return tbbNum;
208 }
209 const char* ompEnv = getenv("OMP_NUM_THREADS");
210 uint32_t ompNum = ompEnv ? atoi(ompEnv) : 0;
211 if (ompNum) {
212 return tbbNum;
213 }
214 return tbb::info::default_concurrency();
215}
216} // namespace o2::gpu::internal
217
219{
220 if (GetProcessingSettings().printSettings) {
221 if (mSlaves.size() || mMaster) {
222 printf("\nConfig Dump %s\n", mMaster ? "Slave" : "Master");
223 }
224 const GPUChainTracking* chTrk;
225 for (uint32_t i = 0; i < mChains.size(); i++) {
226 if ((chTrk = dynamic_cast<GPUChainTracking*>(mChains[i].get()))) {
227 break;
228 }
229 }
230 GPUConfigDump::dumpConfig(&param().rec, mProcessingSettings.get(), chTrk ? chTrk->GetQAConfig() : nullptr, chTrk ? chTrk->GetEventDisplayConfig() : nullptr, mDeviceBackendSettings.get(), &mRecoSteps);
231 }
234 if (!IsGPU()) {
235 mRecoSteps.stepsGPUMask.set((uint8_t)0);
236 }
237
238 if (GetProcessingSettings().forceMemoryPoolSize >= 1024 || GetProcessingSettings().forceHostMemoryPoolSize >= 1024) {
240 }
241 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_AUTO) {
243 }
244 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_INDIVIDUAL) {
245 mProcessingSettings->forceMemoryPoolSize = mProcessingSettings->forceHostMemoryPoolSize = 0;
246 }
247 if (GetProcessingSettings().debugLevel >= 4) {
248 mProcessingSettings->keepAllMemory = true;
249 }
250 if (GetProcessingSettings().debugLevel >= 5 && GetProcessingSettings().allocDebugLevel < 2) {
251 mProcessingSettings->allocDebugLevel = 2;
252 }
254 mProcessingSettings->keepDisplayMemory = true;
255 }
256 if (GetProcessingSettings().debugLevel < 6) {
257 mProcessingSettings->debugMask = 0;
258 }
259 if (GetProcessingSettings().debugLevel < 1) {
260 mProcessingSettings->deviceTimers = false;
261 }
262 if (GetProcessingSettings().debugLevel > 0) {
263 mProcessingSettings->recoTaskTiming = true;
264 }
265 if (GetProcessingSettings().deterministicGPUReconstruction == -1) {
266 mProcessingSettings->deterministicGPUReconstruction = GetProcessingSettings().debugLevel >= 6;
267 }
268 if (GetProcessingSettings().deterministicGPUReconstruction) {
269#ifndef GPUCA_DETERMINISTIC_MODE
270 GPUError("WARNING, deterministicGPUReconstruction needs GPUCA_DETERMINISTIC_MODE for being fully deterministic, without only most indeterminism by concurrency is removed, but floating point effects remain!");
271#endif
272 mProcessingSettings->overrideClusterizerFragmentLen = TPC_MAX_FRAGMENT_LEN_GPU;
273 param().rec.tpc.nWaysOuter = true;
274 if (param().rec.tpc.looperInterpolationInExtraPass == -1) {
275 param().rec.tpc.looperInterpolationInExtraPass = 0;
276 }
277 if (GetProcessingSettings().createO2Output > 1) {
278 mProcessingSettings->createO2Output = 1;
279 }
280 mProcessingSettings->rtc.deterministic = 1;
281 } else {
282#ifdef GPUCA_DETERMINISTIC_MODE
283 GPUError("WARNING, compiled with GPUCA_DETERMINISTIC_MODE but deterministicGPUReconstruction not set, only compile-time determinism and deterministic math enforced, not fully deterministic!");
284#endif
285 }
286 if (GetProcessingSettings().deterministicGPUReconstruction && GetProcessingSettings().debugLevel >= 6) {
287 mProcessingSettings->nTPCClustererLanes = 1;
288 }
289 if (GetProcessingSettings().createO2Output > 1 && GetProcessingSettings().runQA && GetProcessingSettings().qcRunFraction == 100.f) {
290 mProcessingSettings->createO2Output = 1;
291 }
292 if (!GetProcessingSettings().createO2Output || !IsGPU()) {
293 mProcessingSettings->clearO2OutputFromGPU = false;
294 }
296 mProcessingSettings->mergerSortTracks = false;
297 }
298
299 if (GetProcessingSettings().debugLevel > 3 || !IsGPU() || GetProcessingSettings().deterministicGPUReconstruction) {
300 mProcessingSettings->delayedOutput = false;
301 }
302
303 if (!GetProcessingSettings().rtc.enable) {
304 mProcessingSettings->rtc.optConstexpr = false;
305 }
306
307 mMemoryScalers->scalingFactor = GetProcessingSettings().memoryScalingFactor;
308 mMemoryScalers->conservative = GetProcessingSettings().conservativeMemoryEstimate;
309 mMemoryScalers->returnMaxVal = GetProcessingSettings().forceMaxMemScalers != 0;
310 if (GetProcessingSettings().forceMaxMemScalers > 1) {
311 mMemoryScalers->rescaleMaxMem(GetProcessingSettings().forceMaxMemScalers);
312 }
313
314 if (GetProcessingSettings().nHostThreads != -1 && GetProcessingSettings().ompThreads != -1) {
315 GPUFatal("Must not use both nHostThreads and ompThreads at the same time!");
316 } else if (GetProcessingSettings().ompThreads != -1) {
317 mProcessingSettings->nHostThreads = GetProcessingSettings().ompThreads;
318 GPUWarning("You are using the deprecated ompThreads option, please switch to nHostThreads!");
319 }
320
321 if (GetProcessingSettings().nHostThreads <= 0) {
322 mProcessingSettings->nHostThreads = internal::getDefaultNThreads();
323 } else {
324 mProcessingSettings->autoAdjustHostThreads = false;
325 }
326 mMaxHostThreads = GetProcessingSettings().nHostThreads;
327 if (mMaster == nullptr) {
328 mThreading = std::make_shared<GPUReconstructionThreading>();
329 mThreading->control = std::make_unique<tbb::global_control>(tbb::global_control::max_allowed_parallelism, mMaxHostThreads);
330 mThreading->allThreads = std::make_unique<tbb::task_arena>(mMaxHostThreads);
331 mThreading->activeThreads = std::make_unique<tbb::task_arena>(mMaxHostThreads);
332 } else {
334 }
336 if (IsGPU()) {
337 mNStreams = std::max<int32_t>(GetProcessingSettings().nStreams, 3);
338 }
339
340 if (GetProcessingSettings().nTPCClustererLanes == -1) {
341 mProcessingSettings->nTPCClustererLanes = (GetRecoStepsGPU() & RecoStep::TPCClusterFinding) ? 3 : std::max<int32_t>(1, std::min<int32_t>(GPUCA_NSECTORS, GetProcessingSettings().inKernelParallel ? (mMaxHostThreads >= 4 ? std::min<int32_t>(mMaxHostThreads / 2, mMaxHostThreads >= 32 ? GPUCA_NSECTORS : 4) : 1) : mMaxHostThreads));
342 }
343 if (GetProcessingSettings().overrideClusterizerFragmentLen == -1) {
344 mProcessingSettings->overrideClusterizerFragmentLen = ((GetRecoStepsGPU() & RecoStep::TPCClusterFinding) || (mMaxHostThreads / GetProcessingSettings().nTPCClustererLanes >= 3)) ? TPC_MAX_FRAGMENT_LEN_GPU : TPC_MAX_FRAGMENT_LEN_HOST;
345 }
346 if (GetProcessingSettings().nTPCClustererLanes > GPUCA_NSECTORS) {
347 GPUError("Invalid value for nTPCClustererLanes: %d", GetProcessingSettings().nTPCClustererLanes);
348 mProcessingSettings->nTPCClustererLanes = GPUCA_NSECTORS;
349 }
350
351 if (GetProcessingSettings().doublePipeline) {
352 mProcessingSettings->rtctech.allowOptimizedSlaveReconstruction = true;
353 }
354 if (GetProcessingSettings().doublePipeline && (mChains.size() != 1 || mChains[0]->SupportsDoublePipeline() == false || !IsGPU() || GetProcessingSettings().memoryAllocationStrategy != GPUMemoryResource::ALLOCATION_GLOBAL)) {
355 GPUError("Must use double pipeline mode only with exactly one chain that must support it");
356 return 1;
357 }
358 if (mMaster == nullptr && GetProcessingSettings().doublePipeline) {
360 }
361
362 if (mMaster && GetProcessingSettings().rtc.enable && (GetProcessingSettings().rtc.optConstexpr || GetProcessingSettings().rtc.optSpecialCode) && !GetProcessingSettings().rtctech.allowOptimizedSlaveReconstruction) {
363 GPUError("Not allowed to create optimized RTC code with more than one GPUReconstruction instances");
364 return 1;
365 }
366
368 for (uint32_t i = 0; i < mChains.size(); i++) {
369 if (mChains[i]->EarlyConfigure()) {
370 return 1;
371 }
372 mChains[i]->RegisterPermanentMemoryAndProcessors();
373 size_t memPrimary, memPageLocked;
374 mChains[i]->MemorySize(memPrimary, memPageLocked);
375 if (!IsGPU() || mOutputControl.useInternal()) {
376 memPageLocked = memPrimary;
377 }
378 mDeviceMemorySize += memPrimary;
379 mHostMemorySize += memPageLocked;
380 }
381 if (GetProcessingSettings().forceMemoryPoolSize && GetProcessingSettings().forceMemoryPoolSize <= 2 && CanQueryMaxMemory()) {
382 mDeviceMemorySize = GetProcessingSettings().forceMemoryPoolSize;
383 } else if (GetProcessingSettings().forceMemoryPoolSize > 2) {
384 mDeviceMemorySize = GetProcessingSettings().forceMemoryPoolSize;
385 if (!IsGPU() || mOutputControl.useInternal()) {
387 }
388 }
389 if (GetProcessingSettings().forceHostMemoryPoolSize) {
390 mHostMemorySize = GetProcessingSettings().forceHostMemoryPoolSize;
391 }
392
393 for (uint32_t i = 0; i < mProcessors.size(); i++) {
394 (mProcessors[i].proc->*(mProcessors[i].RegisterMemoryAllocation))();
395 }
396
397 return 0;
398}
399
401{
402 if (IsGPU()) {
403 for (uint32_t i = 0; i < mChains.size(); i++) {
404 mChains[i]->RegisterGPUProcessors();
405 }
406 }
408 return 0;
409}
410
412{
413 if (GetProcessingSettings().forceMaxMemScalers <= 1 && GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_GLOBAL) {
415 }
416 for (uint32_t i = 0; i < mChains.size(); i++) {
417 if (mChains[i]->Init()) {
418 return 1;
419 }
420 }
421 for (uint32_t i = 0; i < mProcessors.size(); i++) {
422 (mProcessors[i].proc->*(mProcessors[i].InitializeProcessor))();
423 }
424
425 WriteConstantParams(); // Initialize with initial values, can optionally be updated later
426
427 mInitialized = true;
428 return 0;
429}
430
432{
433 if (IsGPU()) {
434 const auto threadContext = GetThreadContext();
435 WriteToConstantMemory(ptrDiff(&processors()->param, processors()), &param(), sizeof(param()), -1);
436 }
437}
438
440{
441 for (uint32_t i = 0; i < mChains.size(); i++) {
442 mChains[i]->Finalize();
443 }
444 return 0;
445}
446
448{
449 if (!mInitialized) {
450 return 1;
451 }
452 for (uint32_t i = 0; i < mSlaves.size(); i++) {
453 if (mSlaves[i]->Exit()) {
454 GPUError("Error exiting slave");
455 }
456 }
457
458 mChains.clear(); // Make sure we destroy a possible ITS GPU tracker before we call the destructors
459 mHostConstantMem.reset(); // Reset these explicitly before the destruction of other members unloads the library
460 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_INDIVIDUAL) {
461 for (uint32_t i = 0; i < mMemoryResources.size(); i++) {
462 if (mMemoryResources[i].mReuse >= 0) {
463 continue;
464 }
465 operator delete(mMemoryResources[i].mPtrDevice, std::align_val_t(GPUCA_BUFFER_ALIGNMENT));
466 mMemoryResources[i].mPtr = mMemoryResources[i].mPtrDevice = nullptr;
467 }
468 }
469 mMemoryResources.clear();
470 if (mInitialized) {
471 ExitDevice();
472 }
473 debugExit();
474 mInitialized = false;
475 return 0;
476}
477
480
482{
483 for (auto it = mMemoryReuse1to1.begin(); it != mMemoryReuse1to1.end(); it++) {
484 auto& re = it->second;
485 if (proc == nullptr || re.proc == proc) {
486 GPUMemoryResource& resMain = mMemoryResources[re.res[0]];
487 resMain.mOverrideSize = 0;
488 for (uint32_t i = 0; i < re.res.size(); i++) {
490 resMain.mOverrideSize = std::max<size_t>(resMain.mOverrideSize, ptrDiff(res.SetPointers((void*)1), (char*)1));
491 }
492 }
493 }
494}
495
497{
499 if ((type & GPUMemoryResource::MEMORY_SCRATCH) && !GetProcessingSettings().keepDisplayMemory) { // keepAllMemory --> keepDisplayMemory
501 } else {
503 }
504 }
506 type &= ~GPUMemoryResource::MEMORY_GPU;
507 }
508 mMemoryResources.emplace_back(proc, setPtr, (GPUMemoryResource::MemoryType)type, name);
509 if (mMemoryResources.size() >= 32768) {
510 throw std::bad_alloc();
511 }
512 uint16_t retVal = mMemoryResources.size() - 1;
513 if (re.type != GPUMemoryReuse::NONE && !GetProcessingSettings().disableMemoryReuse) {
514 const auto& it = mMemoryReuse1to1.find(re.id);
515 if (it == mMemoryReuse1to1.end()) {
516 mMemoryReuse1to1[re.id] = {proc, retVal};
517 } else {
518 mMemoryResources[retVal].mReuse = it->second.res[0];
519 it->second.res.emplace_back(retVal);
520 }
521 }
522 return retVal;
523}
524
526{
527 if (GetProcessingSettings().debugLevel >= 5) {
528 GPUInfo("Allocating memory %p", (void*)proc);
529 }
530 size_t total = 0;
531 for (uint32_t i = 0; i < mMemoryResources.size(); i++) {
532 if (proc == nullptr ? !mMemoryResources[i].mProcessor->mAllocateAndInitializeLate : mMemoryResources[i].mProcessor == proc) {
534 total += AllocateRegisteredMemory(i);
535 } else if (resetCustom && (mMemoryResources[i].mPtr || mMemoryResources[i].mPtrDevice)) {
537 }
538 }
539 }
540 if (GetProcessingSettings().debugLevel >= 5) {
541 GPUInfo("Allocating memory done");
542 }
543 return total;
544}
545
547{
548 if (GetProcessingSettings().debugLevel >= 5) {
549 GPUInfo("Allocating Permanent Memory");
550 }
552 GPUError("Must not allocate permanent memory while volatile chunks are allocated");
553 throw std::bad_alloc();
554 }
555 int32_t total = 0;
556 for (uint32_t i = 0; i < mMemoryResources.size(); i++) {
557 if ((mMemoryResources[i].mType & GPUMemoryResource::MEMORY_PERMANENT) && mMemoryResources[i].mPtr == nullptr) {
558 total += AllocateRegisteredMemory(i);
559 }
560 }
563 if (GetProcessingSettings().debugLevel >= 5) {
564 GPUInfo("Permanent Memory Done");
565 }
566 return total;
567}
568
569size_t GPUReconstruction::AllocateRegisteredMemoryHelper(GPUMemoryResource* res, void*& ptr, void*& memorypool, void* memorybase, size_t memorysize, void* (GPUMemoryResource::*setPtr)(void*), void*& memorypoolend, const char* device)
570{
571 if (res->mReuse >= 0) {
572 ptr = (&ptr == &res->mPtrDevice) ? mMemoryResources[res->mReuse].mPtrDevice : mMemoryResources[res->mReuse].mPtr;
573 if (ptr == nullptr) {
574 GPUError("Invalid reuse ptr (%s)", res->mName);
575 throw std::bad_alloc();
576 }
577 size_t retVal = ptrDiff((res->*setPtr)(ptr), ptr);
578 if (retVal > mMemoryResources[res->mReuse].mSize) {
579 GPUError("Insufficient reuse memory %lu < %lu (%s) (%s)", mMemoryResources[res->mReuse].mSize, retVal, res->mName, device);
580 throw std::bad_alloc();
581 }
582 if (GetProcessingSettings().allocDebugLevel >= 2) {
583 std::cout << "Reused (" << device << ") " << res->mName << ": " << retVal << "\n";
584 }
585 return retVal;
586 }
587 if (memorypool == nullptr) {
588 GPUError("Cannot allocate memory from uninitialized pool");
589 throw std::bad_alloc();
590 }
591 size_t retVal;
592 if ((res->mType & GPUMemoryResource::MEMORY_STACK) && memorypoolend) {
593 retVal = ptrDiff((res->*setPtr)((char*)1), (char*)(1));
594 memorypoolend = (void*)((char*)memorypoolend - GPUProcessor::getAlignmentMod<GPUCA_MEMALIGN>(memorypoolend));
595 if (retVal < res->mOverrideSize) {
596 retVal = res->mOverrideSize;
597 }
598 retVal += GPUProcessor::getAlignment<GPUCA_MEMALIGN>(retVal);
599 memorypoolend = (char*)memorypoolend - retVal;
600 ptr = memorypoolend;
601 retVal = std::max<size_t>(ptrDiff((res->*setPtr)(ptr), ptr), res->mOverrideSize);
602 } else {
603 ptr = memorypool;
604 memorypool = (char*)((res->*setPtr)(ptr));
605 retVal = ptrDiff(memorypool, ptr);
606 if (retVal < res->mOverrideSize) {
607 retVal = res->mOverrideSize;
608 memorypool = (char*)ptr + res->mOverrideSize;
609 }
610 memorypool = (void*)((char*)memorypool + GPUProcessor::getAlignment<GPUCA_MEMALIGN>(memorypool));
611 }
612 if (memorypoolend ? (memorypool > memorypoolend) : ((size_t)ptrDiff(memorypool, memorybase) > memorysize)) {
613 std::cerr << "Memory pool size exceeded (" << device << ") (" << res->mName << ": " << (memorypoolend ? (memorysize + ptrDiff(memorypool, memorypoolend)) : ptrDiff(memorypool, memorybase)) << " > " << memorysize << "\n";
614 throw std::bad_alloc();
615 }
616 if (GetProcessingSettings().allocDebugLevel >= 2) {
617 std::cout << "Allocated (" << device << ") " << res->mName << ": " << retVal << " - available: " << (memorypoolend ? ptrDiff(memorypoolend, memorypool) : (memorysize - ptrDiff(memorypool, memorybase))) << "\n";
618 }
619 return retVal;
620}
621
623{
624 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_INDIVIDUAL && (control == nullptr || control->useInternal())) {
625 if (!(res->mType & GPUMemoryResource::MEMORY_EXTERNAL)) {
626 if (res->mPtrDevice && res->mReuse < 0) {
627 operator delete(res->mPtrDevice, std::align_val_t(GPUCA_BUFFER_ALIGNMENT));
628 }
629 res->mSize = std::max((size_t)res->SetPointers((void*)1) - 1, res->mOverrideSize);
630 if (res->mReuse >= 0) {
631 if (res->mSize > mMemoryResources[res->mReuse].mSize) {
632 GPUError("Invalid reuse, insufficient size: %ld < %ld", (int64_t)mMemoryResources[res->mReuse].mSize, (int64_t)res->mSize);
633 throw std::bad_alloc();
634 }
635 res->mPtrDevice = mMemoryResources[res->mReuse].mPtrDevice;
636 } else {
637 res->mPtrDevice = operator new(res->mSize + GPUCA_BUFFER_ALIGNMENT, std::align_val_t(GPUCA_BUFFER_ALIGNMENT));
638 }
639 res->mPtr = GPUProcessor::alignPointer<GPUCA_BUFFER_ALIGNMENT>(res->mPtrDevice);
640 res->SetPointers(res->mPtr);
641 if (GetProcessingSettings().allocDebugLevel >= 2) {
642 std::cout << (res->mReuse >= 0 ? "Reused " : "Allocated ") << res->mName << ": " << res->mSize << "\n";
643 }
646 }
647 if ((size_t)res->mPtr % GPUCA_BUFFER_ALIGNMENT) {
648 GPUError("Got buffer with insufficient alignment");
649 throw std::bad_alloc();
650 }
651 }
652 } else {
653 if (res->mPtr != nullptr) {
654 GPUError("Double allocation! (%s)", res->mName);
655 throw std::bad_alloc();
656 }
657 if (IsGPU() && res->mOverrideSize < GPUCA_BUFFER_ALIGNMENT) {
658 res->mOverrideSize = GPUCA_BUFFER_ALIGNMENT;
659 }
660 if ((!IsGPU() || (res->mType & GPUMemoryResource::MEMORY_HOST) || GetProcessingSettings().keepDisplayMemory) && !(res->mType & GPUMemoryResource::MEMORY_EXTERNAL)) { // keepAllMemory --> keepDisplayMemory
661 if (control && control->useExternal()) {
662 if (control->allocator) {
663 res->mSize = std::max((size_t)res->SetPointers((void*)1) - 1, res->mOverrideSize);
664 res->mPtr = control->allocator(CAMath::nextMultipleOf<GPUCA_BUFFER_ALIGNMENT>(res->mSize));
665 res->mSize = std::max<size_t>(ptrDiff(res->SetPointers(res->mPtr), res->mPtr), res->mOverrideSize);
666 if (GetProcessingSettings().allocDebugLevel >= 2) {
667 std::cout << "Allocated (from callback) " << res->mName << ": " << res->mSize << "\n";
668 }
669 } else {
670 void* dummy = nullptr;
671 res->mSize = AllocateRegisteredMemoryHelper(res, res->mPtr, control->ptrCurrent, control->ptrBase, control->size, &GPUMemoryResource::SetPointers, dummy, "host");
672 }
673 } else {
675 }
676 if ((size_t)res->mPtr % GPUCA_BUFFER_ALIGNMENT) {
677 GPUError("Got buffer with insufficient alignment");
678 throw std::bad_alloc();
679 }
680 }
681 if (IsGPU() && (res->mType & GPUMemoryResource::MEMORY_GPU)) {
682 if (res->mProcessor->mLinkedProcessor == nullptr) {
683 GPUError("Device Processor not set (%s)", res->mName);
684 throw std::bad_alloc();
685 }
687 GPUError("Must not allocate non-stacked device memory while volatile chunks are allocated");
688 throw std::bad_alloc();
689 }
691
693 res->mSize = size;
694 } else if (size != res->mSize) {
695 GPUError("Inconsistent device memory allocation (%s: device %lu vs %lu)", res->mName, size, res->mSize);
696 throw std::bad_alloc();
697 }
698 if ((size_t)res->mPtrDevice % GPUCA_BUFFER_ALIGNMENT) {
699 GPUError("Got buffer with insufficient alignment");
700 throw std::bad_alloc();
701 }
702 }
704 }
705}
706
711
713{
715 if ((res->mType & GPUMemoryResource::MEMORY_PERMANENT) && res->mPtr != nullptr) {
717 } else {
719 }
720 return res->mReuse >= 0 ? 0 : res->mSize;
721}
722
724{
725 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_INDIVIDUAL) {
726 char* retVal = new (std::align_val_t(GPUCA_BUFFER_ALIGNMENT)) char[size];
729 } else {
731 }
732 return retVal;
733 }
734
736 throw std::runtime_error("Requested invalid memory typo for direct allocation");
737 }
739 GPUError("Must not allocate direct memory while volatile chunks are allocated");
740 throw std::bad_alloc();
741 }
742
745 char* retVal;
747 poolend = (char*)poolend - size;
748 poolend = (char*)poolend - GPUProcessor::getAlignmentMod<GPUCA_MEMALIGN>(poolend);
749 retVal = (char*)poolend;
750 } else {
752 }
753 if (pool > poolend) {
754 GPUError("Insufficient unmanaged memory: missing %ld bytes", ptrDiff(pool, poolend));
755 throw std::bad_alloc();
756 }
758 if (GetProcessingSettings().allocDebugLevel >= 2) {
759 std::cout << "Allocated (unmanaged " << (type == GPUMemoryResource::MEMORY_GPU ? "gpu" : "host") << "): " << size << " - available: " << ptrDiff(poolend, pool) << "\n";
760 }
761 return retVal;
762}
763
765{
766 if (mVolatileMemoryStart == nullptr) {
768 }
769 if (size == 0) {
770 return nullptr; // Future GPU memory allocation is volatile
771 }
772 char* retVal;
775 GPUError("Insufficient volatile device memory: missing %ld", ptrDiff(mDeviceMemoryPool, mDeviceMemoryPoolEnd));
776 throw std::bad_alloc();
777 }
779 if (GetProcessingSettings().allocDebugLevel >= 2) {
780 std::cout << "Allocated (volatile GPU): " << size << " - available: " << ptrDiff(mDeviceMemoryPoolEnd, mDeviceMemoryPool) << "\n";
781 }
782 return retVal;
783}
784
786{
787 if (device) {
789 }
790 char* retVal = new (std::align_val_t(GPUCA_BUFFER_ALIGNMENT)) char[size];
791 mVolatileChunks.emplace_back(retVal, alignedDeleter());
792 return retVal;
793}
794
800
802{
806 mVolatileMemoryStart = nullptr;
807 }
808 if (GetProcessingSettings().allocDebugLevel >= 2) {
809 std::cout << "Freed (volatile GPU) - available: " << ptrDiff(mDeviceMemoryPoolEnd, mDeviceMemoryPool) << "\n";
810 }
811}
812
818
820{
821 for (uint32_t i = 0; i < mMemoryResources.size(); i++) {
822 if (proc == nullptr || mMemoryResources[i].mProcessor == proc) {
824 }
825 }
826}
827
829{
832 void* basePtr = res->mReuse >= 0 ? mMemoryResources[res->mReuse].mPtr : res->mPtr;
833 size_t size = ptrDiff(res->SetPointers(basePtr), basePtr);
834 if (basePtr && size > std::max(res->mSize, res->mOverrideSize)) {
835 std::cerr << "Updated pointers exceed available memory size: " << size << " > " << std::max(res->mSize, res->mOverrideSize) << " - host - " << res->mName << "\n";
836 throw std::bad_alloc();
837 }
838 }
839 if (IsGPU() && (res->mType & GPUMemoryResource::MEMORY_GPU)) {
840 void* basePtr = res->mReuse >= 0 ? mMemoryResources[res->mReuse].mPtrDevice : res->mPtrDevice;
841 size_t size = ptrDiff(res->SetDevicePointers(basePtr), basePtr);
842 if (basePtr && size > std::max(res->mSize, res->mOverrideSize)) {
843 std::cerr << "Updated pointers exceed available memory size: " << size << " > " << std::max(res->mSize, res->mOverrideSize) << " - GPU - " << res->mName << "\n";
844 throw std::bad_alloc();
845 }
846 }
847}
848
849void GPUReconstruction::FreeRegisteredMemory(GPUProcessor* proc, bool freeCustom, bool freePermanent)
850{
851 for (uint32_t i = 0; i < mMemoryResources.size(); i++) {
852 if ((proc == nullptr || mMemoryResources[i].mProcessor == proc) && (freeCustom || !(mMemoryResources[i].mType & GPUMemoryResource::MEMORY_CUSTOM)) && (freePermanent || !(mMemoryResources[i].mType & GPUMemoryResource::MEMORY_PERMANENT))) {
854 }
855 }
856}
857
862
864{
865 if (GetProcessingSettings().allocDebugLevel >= 2 && (res->mPtr || res->mPtrDevice)) {
866 std::cout << "Freeing " << res->mName << ": size " << res->mSize << " (reused " << res->mReuse << ")\n";
867 }
868 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_INDIVIDUAL && res->mReuse < 0) {
869 operator delete(res->mPtrDevice, std::align_val_t(GPUCA_BUFFER_ALIGNMENT));
870 }
871 res->mPtr = nullptr;
872 res->mPtrDevice = nullptr;
873}
874
879
881{
882 if (GetProcessingSettings().keepDisplayMemory || GetProcessingSettings().disableMemoryReuse) {
883 return;
884 }
885 if (mNonPersistentMemoryStack.size() == 0) {
886 GPUFatal("Trying to pop memory state from empty stack");
887 }
888 if (tag != 0 && std::get<4>(mNonPersistentMemoryStack.back()) != tag) {
889 GPUFatal("Tag mismatch when popping non persistent memory from stack : pop %s vs on stack %s", qTag2Str(tag).c_str(), qTag2Str(std::get<4>(mNonPersistentMemoryStack.back())).c_str());
890 }
891 if ((GetProcessingSettings().debugLevel >= 3 || GetProcessingSettings().allocDebugLevel) && (IsGPU() || GetProcessingSettings().forceHostMemoryPoolSize)) {
892 printf("Allocated memory after %30s (%8s) (Stack %zu): ", GPUDataTypes::RECO_STEP_NAMES[getRecoStepNum(step, true)], qTag2Str(std::get<4>(mNonPersistentMemoryStack.back())).c_str(), mNonPersistentMemoryStack.size());
894 printf("%76s", "");
896 }
897 mHostMemoryPoolEnd = std::get<0>(mNonPersistentMemoryStack.back());
899 for (uint32_t i = std::get<2>(mNonPersistentMemoryStack.back()); i < mNonPersistentIndividualAllocations.size(); i++) {
901 if (res->mReuse < 0) {
902 operator delete(res->mPtrDevice, std::align_val_t(GPUCA_BUFFER_ALIGNMENT));
903 }
904 res->mPtr = nullptr;
905 res->mPtrDevice = nullptr;
906 }
909 mNonPersistentMemoryStack.pop_back();
910}
911
913{
915 throw std::runtime_error("temporary memory stack already blocked");
916 }
919}
920
922{
923 if (mNonPersistentMemoryStack.size()) {
924 throw std::runtime_error("cannot unblock while there is stacked memory");
925 }
928 mHostMemoryPoolBlocked = nullptr;
929 mDeviceMemoryPoolBlocked = nullptr;
930}
931
933{
934 mMemoryResources[res].mPtr = ptr;
935}
936
938{
939 for (uint32_t i = 0; i < mMemoryResources.size(); i++) {
942 }
943 }
946 mDirectMemoryChunks.clear();
948 mVolatileChunks.clear();
949 mVolatileMemoryStart = nullptr;
950 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_GLOBAL) {
951 mHostMemoryPool = GPUProcessor::alignPointer<GPUCA_MEMALIGN>(mHostMemoryPermanent);
952 mDeviceMemoryPool = GPUProcessor::alignPointer<GPUCA_MEMALIGN>(mDeviceMemoryPermanent);
955 } else {
957 }
958}
959
965
967{
968 printf("Maximum Memory Allocation: Host %'zu / Device %'zu\n", mHostMemoryUsedMax, mDeviceMemoryUsedMax);
969}
970
972{
973 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_GLOBAL) {
974 printf("Memory Allocation: Host %'13zd / %'13zu (Permanent %'13zd, Data %'13zd, Scratch %'13zd), Device %'13zd / %'13zu, (Permanent %'13zd, Data %'13zd, Scratch %'13zd) %zu chunks\n",
977 mMemoryResources.size());
978 }
979}
980
982{
983 std::map<std::string, std::array<size_t, 3>> sizes;
984 for (uint32_t i = 0; i < mMemoryResources.size(); i++) {
985 auto& res = mMemoryResources[i];
986 if (res.mReuse >= 0) {
987 continue;
988 }
989 auto& x = sizes[res.mName];
990 if (res.mPtr) {
991 x[0] += res.mSize;
992 }
993 if (res.mPtrDevice) {
994 x[1] += res.mSize;
995 }
997 x[2] = 1;
998 }
999 }
1000 printf("%59s CPU / %9s GPU\n", "", "");
1001 for (auto it = sizes.begin(); it != sizes.end(); it++) {
1002 printf("Allocation %30s %s: Size %'14zu / %'14zu\n", it->first.c_str(), it->second[2] ? "P" : " ", it->second[0], it->second[1]);
1003 }
1005 for (uint32_t i = 0; i < mChains.size(); i++) {
1006 mChains[i]->PrintMemoryStatistics();
1007 }
1008}
1009
1011{
1012 if (GetProcessingSettings().noGPUMemoryRegistration) {
1013 return 0;
1014 }
1016 if (retVal == 0) {
1017 mRegisteredMemoryPtrs.emplace(ptr);
1018 }
1019 return retVal;
1020}
1021
1023{
1024 if (GetProcessingSettings().noGPUMemoryRegistration) {
1025 return 0;
1026 }
1027 const auto& pos = mRegisteredMemoryPtrs.find(ptr);
1028 if (pos != mRegisteredMemoryPtrs.end()) {
1031 }
1032 return 1;
1033}
1034
1035namespace o2::gpu::internal
1036{
1037namespace // anonymous
1038{
1039template <class T>
1040constexpr static inline int32_t getStepNum(T step, bool validCheck, int32_t N, const char* err = "Invalid step num")
1041{
1042 static_assert(sizeof(step) == sizeof(uint32_t), "Invalid step enum size");
1043 int32_t retVal = 8 * sizeof(uint32_t) - 1 - CAMath::Clz((uint32_t)step);
1044 if ((uint32_t)step == 0 || retVal >= N) {
1045 if (!validCheck) {
1046 return -1;
1047 }
1048 throw std::runtime_error("Invalid General Step");
1049 }
1050 return retVal;
1051}
1052} // anonymous namespace
1053} // namespace o2::gpu::internal
1054
1055int32_t GPUReconstruction::getRecoStepNum(RecoStep step, bool validCheck) { return internal::getStepNum(step, validCheck, GPUDataTypes::N_RECO_STEPS, "Invalid Reco Step"); }
1056int32_t GPUReconstruction::getGeneralStepNum(GeneralStep step, bool validCheck) { return internal::getStepNum(step, validCheck, GPUDataTypes::N_GENERAL_STEPS, "Invalid General Step"); }
1057
1059{
1060 if (!mInitialized || !GetProcessingSettings().doublePipeline || mMaster != nullptr || !mSlaves.size()) {
1061 throw std::invalid_argument("Cannot start double pipeline mode");
1062 }
1063 if (GetProcessingSettings().debugLevel >= 3) {
1064 GPUInfo("Pipeline worker started");
1065 }
1066 bool terminate = false;
1067 while (!terminate) {
1068 {
1069 std::unique_lock<std::mutex> lk(mPipelineContext->mutex);
1070 mPipelineContext->cond.wait(lk, [this] { return this->mPipelineContext->queue.size() > 0; });
1071 }
1072 GPUReconstructionPipelineQueue* q;
1073 {
1074 std::lock_guard<std::mutex> lk(mPipelineContext->mutex);
1075 q = mPipelineContext->queue.front();
1076 mPipelineContext->queue.pop();
1077 }
1078 if (q->op == 1) {
1079 terminate = 1;
1080 } else {
1081 q->retVal = q->chain->RunChain();
1082 }
1083 {
1084 std::lock_guard<std::mutex> lk(q->m);
1085 q->done = true;
1086 }
1087 q->c.notify_one();
1088 }
1089 if (GetProcessingSettings().debugLevel >= 3) {
1090 GPUInfo("Pipeline worker ended");
1091 }
1092}
1093
1098
1100{
1103 std::unique_ptr<GPUReconstructionPipelineQueue> qu(new GPUReconstructionPipelineQueue);
1104 GPUReconstructionPipelineQueue* q = qu.get();
1105 q->chain = terminate ? nullptr : mChains[0].get();
1106 q->op = terminate ? 1 : 0;
1107 std::unique_lock<std::mutex> lkdone(q->m);
1108 {
1109 std::lock_guard<std::mutex> lkpipe(rec->mPipelineContext->mutex);
1110 if (rec->mPipelineContext->terminate) {
1111 throw std::runtime_error("Must not enqueue work after termination request");
1112 }
1113 rec->mPipelineContext->queue.push(q);
1114 rec->mPipelineContext->terminate = terminate;
1115 rec->mPipelineContext->cond.notify_one();
1116 }
1117 q->c.wait(lkdone, [&q]() { return q->done; });
1118 if (q->retVal) {
1119 return q->retVal;
1120 }
1121 if (terminate) {
1122 return 0;
1123 } else {
1124 return mChains[0]->FinalizePipelinedProcessing();
1125 }
1126}
1127
1129{
1131 std::lock_guard<std::mutex> lk(rec->mPipelineContext->mutex);
1132 return rec->mPipelineContext->queue.size() && rec->mPipelineContext->queue.front()->op == 0 ? rec->mPipelineContext->queue.front()->chain : nullptr;
1133}
1134
1135void GPUReconstruction::PrepareEvent() // TODO: Clean this up, this should not be called from chainTracking but before
1136{
1138 for (uint32_t i = 0; i < mChains.size(); i++) {
1139 mChains[i]->PrepareEvent();
1140 }
1141 for (uint32_t i = 0; i < mProcessors.size(); i++) {
1142 if (mProcessors[i].proc->mAllocateAndInitializeLate) {
1143 continue;
1144 }
1145 (mProcessors[i].proc->*(mProcessors[i].SetMaxData))(mHostConstantMem->ioPtrs);
1146 if (mProcessors[i].proc->mGPUProcessorType != GPUProcessor::PROCESSOR_TYPE_DEVICE && mProcessors[i].proc->mLinkedProcessor) {
1147 (mProcessors[i].proc->mLinkedProcessor->*(mProcessors[i].SetMaxData))(mHostConstantMem->ioPtrs);
1148 }
1149 }
1150 ComputeReuseMax(nullptr);
1151 AllocateRegisteredMemory(nullptr);
1152}
1153
1154int32_t GPUReconstruction::CheckErrorCodes(bool cpuOnly, bool forceShowErrors, std::vector<std::array<uint32_t, 4>>* fillErrors)
1155{
1156 int32_t retVal = 0;
1157 for (uint32_t i = 0; i < mChains.size(); i++) {
1158 if (mChains[i]->CheckErrorCodes(cpuOnly, forceShowErrors, fillErrors)) {
1159 retVal++;
1160 }
1161 }
1162 return retVal;
1163}
1164
1165int32_t GPUReconstruction::GPUChkErrA(const int64_t error, const char* file, int32_t line, bool failOnError)
1166{
1167 if (error == 0 || !GPUChkErrInternal(error, file, line)) {
1168 return 0;
1169 }
1170 if (failOnError) {
1171 if (mInitialized && mInErrorHandling == false) {
1172 mInErrorHandling = true;
1173 CheckErrorCodes(false, true);
1174 }
1175 throw std::runtime_error("GPU Backend Failure");
1176 }
1177 return 1;
1178}
1179
1181{
1182 std::string f;
1183 f = dir;
1184 f += "settings.dump";
1185 DumpStructToFile(mGRPSettings.get(), f.c_str());
1186 for (uint32_t i = 0; i < mChains.size(); i++) {
1187 mChains[i]->DumpSettings(dir);
1188 }
1189}
1190
1191void GPUReconstruction::UpdateDynamicSettings(const GPUSettingsRecDynamic* d)
1192{
1193 UpdateSettings(nullptr, nullptr, d);
1194}
1195
1196void GPUReconstruction::UpdateSettings(const GPUSettingsGRP* g, const GPUSettingsProcessing* p, const GPUSettingsRecDynamic* d)
1197{
1198 if (g) {
1199 *mGRPSettings = *g;
1200 }
1201 if (p) {
1202 mProcessingSettings->debugLevel = p->debugLevel;
1203 mProcessingSettings->resetTimers = p->resetTimers;
1204 }
1205 GPURecoStepConfiguration* w = nullptr;
1207 w = &mRecoSteps;
1208 }
1209 param().UpdateSettings(g, p, w, d);
1210 if (mInitialized) {
1212 }
1213}
1214
1215int32_t GPUReconstruction::ReadSettings(const char* dir)
1216{
1217 std::string f;
1218 f = dir;
1219 f += "settings.dump";
1220 new (mGRPSettings.get()) GPUSettingsGRP;
1221 if (ReadStructFromFile(f.c_str(), mGRPSettings.get())) {
1222 return 1;
1223 }
1225 for (uint32_t i = 0; i < mChains.size(); i++) {
1226 mChains[i]->ReadSettings(dir);
1227 }
1228 return 0;
1229}
1230
1231void GPUReconstruction::SetSettings(float solenoidBzNominalGPU, const GPURecoStepConfiguration* workflow)
1232{
1233#ifdef GPUCA_O2_LIB
1235 config.ReadConfigurableParam(config);
1236 config.configGRP.solenoidBzNominalGPU = solenoidBzNominalGPU;
1237 SetSettings(&config.configGRP, &config.configReconstruction, &config.configProcessing, workflow);
1238#else
1239 GPUSettingsGRP grp;
1240 grp.solenoidBzNominalGPU = solenoidBzNominalGPU;
1241 SetSettings(&grp, nullptr, nullptr, workflow);
1242#endif
1243}
1244
1245void GPUReconstruction::SetSettings(const GPUSettingsGRP* grp, const GPUSettingsRec* rec, const GPUSettingsProcessing* proc, const GPURecoStepConfiguration* workflow)
1246{
1247 if (mInitialized) {
1248 GPUError("Cannot update settings while initialized");
1249 throw std::runtime_error("Settings updated while initialized");
1250 }
1251 *mGRPSettings = *grp;
1252 if (proc) {
1253 *mProcessingSettings = *proc;
1254 }
1255 if (workflow) {
1256 mRecoSteps.steps = workflow->steps;
1258 mRecoSteps.inputs = workflow->inputs;
1259 mRecoSteps.outputs = workflow->outputs;
1260 }
1261 param().SetDefaults(mGRPSettings.get(), rec, proc, workflow);
1262}
1263
1265{
1266 GPUOutputControl outputControl;
1267 outputControl.set(ptr, size);
1268 SetOutputControl(outputControl);
1269}
1270
1274void GPUReconstruction::SetResetTimers(bool reset) { mProcessingSettings->resetTimers = reset; }
1279
1280ThrustVolatileAllocator::ThrustVolatileAllocator(GPUReconstruction* r)
1281{
1282 mAlloc = [&r](size_t n) { return (char*)r->AllocateVolatileDeviceMemory(n); };
1283}
int32_t i
#define GPUCA_BUFFER_ALIGNMENT
uint32_t op
bool done
int32_t retVal
GPUChain * chain
#define GPUCA_NSECTORS
uint16_t pos
Definition RawData.h:3
uint32_t res
Definition RawData.h:0
TBranch * ptr
void set(S v)
Definition bitfield.h:55
bool isSet(const bitfield &v) const
Definition bitfield.h:66
const GPUSettingsDisplay * GetEventDisplayConfig() const
const GPUSettingsQA * GetQAConfig() const
static void dumpConfig(const GPUSettingsRec *rec, const GPUSettingsProcessing *proc, const GPUSettingsQA *qa, const GPUSettingsDisplay *display, const GPUSettingsDeviceBackend *device, const GPURecoStepConfiguration *workflow)
static constexpr const char *const RECO_STEP_NAMES[]
static constexpr int32_t N_RECO_STEPS
static constexpr int32_t N_GENERAL_STEPS
void * SetDevicePointers(void *ptr)
static void computePointerWithAlignment(T *&basePtr, S *&objPtr, size_t nEntries=1)
void InitGPUProcessor(GPUReconstruction *rec, ProcessorType type=PROCESSOR_TYPE_CPU, GPUProcessor *slaveProcessor=nullptr)
ProcessorType mGPUProcessorType
GPURecoStepConfiguration mRecoSteps
virtual void GetITSTraits(std::unique_ptr< o2::its::TrackerTraits< 7 > > *trackerTraits, std::unique_ptr< o2::its::VertexerTraits > *vertexerTraits, std::unique_ptr< o2::its::TimeFrame< 7 > > *timeFrame)
std::unordered_set< const void * > mRegisteredMemoryPtrs
int16_t RegisterMemoryAllocationHelper(GPUProcessor *proc, void *(GPUProcessor::*setPtr)(void *), int32_t type, const char *name, const GPUMemoryReuse &re)
std::vector< std::unique_ptr< GPUChain > > mChains
void * AllocateVolatileMemory(size_t size, bool device)
ThrustVolatileAllocator getThrustVolatileDeviceAllocator()
std::unique_ptr< GPUMemorySizeScalers > mMemoryScalers
void AllocateRegisteredForeignMemory(int16_t res, GPUReconstruction *rec, GPUOutputControl *control=nullptr)
void SetInputControl(void *ptr, size_t size)
GPUConstantMem * mDeviceConstantMem
void ConstructGPUProcessor(GPUProcessor *proc)
std::shared_ptr< GPUROOTDumpCore > mROOTDump
void PopNonPersistentMemory(RecoStep step, uint64_t tag)
size_t AllocateRegisteredMemoryHelper(GPUMemoryResource *res, void *&ptr, void *&memorypool, void *memorybase, size_t memorysize, void *(GPUMemoryResource::*SetPointers)(void *), void *&memorypoolend, const char *device)
const GPUSettingsDeviceBackend & GetDeviceBackendSettings() const
void ComputeReuseMax(GPUProcessor *proc)
void SetMemoryExternalInput(int16_t res, void *ptr)
int32_t getGeneralStepNum(GeneralStep step, bool validCheck=true)
static constexpr uint32_t NSECTORS
RecoStepField GetRecoStepsGPU() const
void RegisterGPUDeviceProcessor(GPUProcessor *proc, GPUProcessor *slaveProcessor)
std::vector< GPUReconstruction * > mSlaves
std::vector< std::tuple< void *, void *, size_t, size_t, uint64_t > > mNonPersistentMemoryStack
std::unique_ptr< T > ReadStructFromFile(const char *file)
void UpdateDynamicSettings(const GPUSettingsRecDynamic *d)
std::unique_ptr< GPUSettingsDeviceBackend > mDeviceBackendSettings
std::vector< GPUMemoryResource > mMemoryResources
std::unique_ptr< GPUReconstructionPipelineContext > mPipelineContext
std::unique_ptr< GPUConstantMem > mHostConstantMem
void ResetRegisteredMemoryPointers(GPUProcessor *proc)
void DumpStructToFile(const T *obj, const char *file)
void AllocateRegisteredMemoryInternal(GPUMemoryResource *res, GPUOutputControl *control, GPUReconstruction *recPool)
virtual int32_t registerMemoryForGPU_internal(const void *ptr, size_t size)=0
virtual size_t WriteToConstantMemory(size_t offset, const void *src, size_t size, int32_t stream=-1, gpu_reconstruction_kernels::deviceEvent *ev=nullptr)=0
std::unordered_map< GPUMemoryReuse::ID, MemoryReuseMeta > mMemoryReuse1to1
std::vector< ProcessorData > mProcessors
void * AllocateVolatileDeviceMemory(size_t size)
virtual int32_t InitDevice()=0
void SetSettings(float solenoidBzNominalGPU, const GPURecoStepConfiguration *workflow=nullptr)
const GPUCalibObjectsConst & GetCalib() const
const GPUTrackingInOutPointers GetIOPtrs() const
virtual std::unique_ptr< gpu_reconstruction_kernels::threadContext > GetThreadContext()=0
GPUReconstruction(const GPUReconstruction &)=delete
static constexpr GeometryType geometryType
std::vector< std::unique_ptr< char[], alignedDeleter > > mNonPersistentIndividualDirectAllocations
void FreeRegisteredMemory(GPUProcessor *proc, bool freeCustom=false, bool freePermanent=false)
std::vector< std::unique_ptr< char[], alignedDeleter > > mVolatileChunks
virtual RecoStepField AvailableGPURecoSteps()
static constexpr const char *const IOTYPENAMES[]
std::vector< std::unique_ptr< char[], alignedDeleter > > mDirectMemoryChunks
void UpdateSettings(const GPUSettingsGRP *g, const GPUSettingsProcessing *p=nullptr, const GPUSettingsRecDynamic *d=nullptr)
int32_t CheckErrorCodes(bool cpuOnly=false, bool forceShowErrors=false, std::vector< std::array< uint32_t, 4 > > *fillErrors=nullptr)
const GPUParam & GetParam() const
void ClearAllocatedMemory(bool clearOutputs=true)
static constexpr const char *const GEOMETRY_TYPE_NAMES[]
virtual int32_t ExitDevice()=0
std::unique_ptr< GPUSettingsGRP > mGRPSettings
std::unique_ptr< GPUSettingsProcessing > mProcessingSettings
void PushNonPersistentMemory(uint64_t tag)
int32_t getRecoStepNum(RecoStep step, bool validCheck=true)
virtual int32_t unregisterMemoryForGPU_internal(const void *ptr)=0
void BlockStackedMemory(GPUReconstruction *rec)
const GPUSettingsProcessing & GetProcessingSettings() const
void DumpSettings(const char *dir="")
void * AllocateDirectMemory(size_t size, int32_t type)
int32_t unregisterMemoryForGPU(const void *ptr)
int32_t registerMemoryForGPU(const void *ptr, size_t size)
void SetDebugLevelTmp(int32_t level)
int32_t EnqueuePipeline(bool terminate=false)
std::shared_ptr< GPUReconstructionThreading > mThreading
std::vector< GPUMemoryResource * > mNonPersistentIndividualAllocations
virtual int32_t GPUChkErrInternal(const int64_t error, const char *file, int32_t line) const
int32_t GPUChkErrA(const int64_t error, const char *file, int32_t line, bool failOnError)
size_t AllocateRegisteredMemory(GPUProcessor *proc, bool resetCustom=false)
int32_t ReadSettings(const char *dir="")
void SetOutputControl(const GPUOutputControl &v)
void SetSector(int32_t iSector)
#define TPC_MAX_FRAGMENT_LEN_GPU
#define TPC_MAX_FRAGMENT_LEN_HOST
GLdouble n
Definition glcorearb.h:1982
GLint GLenum GLint x
Definition glcorearb.h:403
const GLfloat * m
Definition glcorearb.h:4066
GLsizeiptr size
Definition glcorearb.h:659
GLuint GLsizei const GLuint const GLintptr const GLsizeiptr * sizes
Definition glcorearb.h:2595
GLuint const GLchar * name
Definition glcorearb.h:781
GLdouble f
Definition glcorearb.h:310
GLboolean GLboolean GLboolean b
Definition glcorearb.h:1233
GLint GLint GLsizei GLint GLenum GLenum type
Definition glcorearb.h:275
typedef void(APIENTRYP PFNGLCULLFACEPROC)(GLenum mode)
GLboolean GLboolean g
Definition glcorearb.h:1233
GLint level
Definition glcorearb.h:275
GLboolean r
Definition glcorearb.h:1233
GLenum GLfloat param
Definition glcorearb.h:271
GLboolean GLboolean GLboolean GLboolean a
Definition glcorearb.h:1233
GLubyte GLubyte GLubyte GLubyte w
Definition glcorearb.h:852
std::unique_ptr< GPUDisplayFrontendInterface > eventDisplay
GPUReconstruction * rec
std::string qTag2Str(const T tag)
Definition strtag.h:35
GPUTPCTracker tpcTrackers[GPUCA_NSECTORS]
GPUTPCClusterFinder tpcClusterer[GPUCA_NSECTORS]
GPUCalibObjectsConst calibObjects
void set(void *p, size_t s)
std::function< void *(size_t)> allocator
void UpdateSettings(const GPUSettingsGRP *g, const GPUSettingsProcessing *p=nullptr, const GPURecoStepConfiguration *w=nullptr, const GPUSettingsRecDynamic *d=nullptr)
Definition GPUParam.cxx:117
void SetDefaults(float solenoidBz, bool assumeConstantBz)
Definition GPUParam.cxx:33
GPUDataTypes::RecoStepField stepsGPUMask
GPUDataTypes::InOutTypeField outputs
GPUDataTypes::RecoStepField steps
GPUDataTypes::InOutTypeField inputs
std::queue< GPUReconstructionPipelineQueue * > queue