Project
Loading...
Searching...
No Matches
GPUReconstructionCPU.cxx
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
14
18#include "GPUChain.h"
19#include "GPUDefParametersRuntime.h"
20#include "GPUTPCClusterData.h"
22#include "GPUTPCGMMergedTrack.h"
24#include "GPUTRDTrackletWord.h"
26#include "GPUTPCMCInfo.h"
27#include "GPUTRDTrack.h"
28#include "GPUTRDTracker.h"
29#include "AliHLTTPCRawCluster.h"
31#include "GPUMemoryResource.h"
32#include "GPUConstantMem.h"
33#include "GPULogging.h"
35#include "GPUReconstructionProcessingKernels.inc"
36
37#include <atomic>
38#include <ctime>
39
40#ifndef _WIN32
41#include <unistd.h>
42#endif
43
44using namespace o2::gpu;
45
48
50
52{
53 Exit(); // Needs to be identical to GPU backend bahavior in order to avoid calling abstract methods later in the destructor
54}
55
56template <class T, int32_t I, typename... Args>
57inline void GPUReconstructionCPU::runKernelBackend(const krnlSetupTime& _xyz, const Args&... args)
58{
59 auto& x = _xyz.x;
60 auto& y = _xyz.y;
61 if (x.device == krnlDeviceType::Device) {
62 throw std::runtime_error("Cannot run device kernel on host");
63 }
64 if (x.nThreads != 1) {
65 throw std::runtime_error("Cannot run device kernel on host with nThreads != 1");
66 }
67 int32_t nThreads = getNKernelHostThreads(false);
68 if (nThreads > 1) {
69 if (GetProcessingSettings().debugLevel >= 5) {
70 printf("Running %d Threads\n", mThreading->activeThreads->max_concurrency());
71 }
72 tbb::this_task_arena::isolate([&] {
73 mThreading->activeThreads->execute([&] {
74 tbb::parallel_for(tbb::blocked_range<uint32_t>(0, x.nBlocks, 1), [&](const tbb::blocked_range<uint32_t>& r) {
75 typename T::GPUSharedMemory smem;
76 for (uint32_t iB = r.begin(); iB < r.end(); iB++) {
77 T::template Thread<I>(x.nBlocks, 1, iB, 0, smem, T::Processor(*mHostConstantMem)[y.index], args...);
78 }
79 });
80 });
81 });
82 } else {
83 for (uint32_t iB = 0; iB < x.nBlocks; iB++) {
84 typename T::GPUSharedMemory smem;
85 T::template Thread<I>(x.nBlocks, 1, iB, 0, smem, T::Processor(*mHostConstantMem)[y.index], args...);
86 }
87 }
88}
89
90template <>
91inline void GPUReconstructionCPU::runKernelBackend<GPUMemClean16, 0>(const krnlSetupTime& _xyz, void* const& ptr, uint64_t const& size)
92{
93 int32_t nThreads = std::max<int32_t>(1, std::min<int32_t>(size / (16 * 1024 * 1024), getNKernelHostThreads(true)));
94 if (nThreads > 1) {
95 tbb::parallel_for(0, nThreads, [&](int iThread) {
96 size_t threadSize = size / nThreads;
97 if (threadSize % 4096) {
98 threadSize += 4096 - threadSize % 4096;
99 }
100 size_t offset = threadSize * iThread;
101 size_t mySize = std::min<size_t>(threadSize, size - offset);
102 if (mySize) {
103 memset((char*)ptr + offset, 0, mySize);
104 } // clang-format off
105 }, tbb::static_partitioner()); // clang-format on
106 } else {
107 memset(ptr, 0, size);
108 }
109}
110
111template <class S, int32_t I>
113{
114 if (gpu == -1) {
115 gpu = IsGPU();
116 }
117 const auto num = GetKernelNum<S, I>();
118 const auto* p = gpu ? mParDevice : mParCPU;
119 GPUReconstructionProcessing::krnlProperties ret = {p->par_LB_maxThreads[num], p->par_LB_minBlocks[num], p->par_LB_forceBlocks[num]};
120 if (ret.nThreads == 0) {
121 ret.nThreads = gpu ? mThreadCount : 1u;
122 }
123 if (ret.minBlocks == 0) {
124 ret.minBlocks = 1;
125 }
126 return ret;
127}
128
129#define GPUCA_KRNL(x_class, x_attributes, x_arguments, x_forward, x_types, ...) \
130 template GPUReconstructionProcessing::krnlProperties GPUReconstructionCPU::getKernelProperties<GPUCA_M_KRNL_TEMPLATE(x_class)>(int gpu);
131#include "GPUReconstructionKernelList.h"
132#undef GPUCA_KRNL
133
134size_t GPUReconstructionCPU::TransferMemoryInternal(GPUMemoryResource* res, int32_t stream, deviceEvent* ev, deviceEvent* evList, int32_t nEvents, bool toGPU, const void* src, void* dst) { return 0; }
135size_t GPUReconstructionCPU::GPUMemCpy(void* dst, const void* src, size_t size, int32_t stream, int32_t toGPU, deviceEvent* ev, deviceEvent* evList, int32_t nEvents) { return 0; }
136size_t GPUReconstructionCPU::GPUMemCpyAlways(bool onGpu, void* dst, const void* src, size_t size, int32_t stream, int32_t toGPU, deviceEvent* ev, deviceEvent* evList, int32_t nEvents)
137{
138 memcpy(dst, src, size);
139 return 0;
140}
141size_t GPUReconstructionCPU::WriteToConstantMemory(size_t offset, const void* src, size_t size, int32_t stream, deviceEvent* ev) { return 0; }
142int32_t GPUReconstructionCPU::GPUDebug(const char* state, int32_t stream, bool force) { return 0; }
143size_t GPUReconstructionCPU::TransferMemoryResourcesHelper(GPUProcessor* proc, int32_t stream, bool all, bool toGPU)
144{
147 size_t n = 0;
148 for (uint32_t i = 0; i < mMemoryResources.size(); i++) {
150 if (res.mPtr == nullptr) {
151 continue;
152 }
153 if (proc && res.mProcessor != proc) {
154 continue;
155 }
157 continue;
158 }
159 if (!GetProcessingSettings().keepAllMemory && !all && (res.mType & exc) && !(res.mType & inc)) {
160 continue;
161 }
162 if (toGPU) {
164 } else {
166 }
167 }
168 return n;
169}
170
172{
173// Get Thread ID
174#if defined(__APPLE__)
175 return (0); // syscall is deprecated on MacOS..., only needed for GPU support which we don't do on Mac anyway
176#elif defined(_WIN32)
177 return ((int32_t)(size_t)GetCurrentThread());
178#else
179 return ((int32_t)syscall(SYS_gettid));
180#endif
181}
182
184{
186 mThreading->activeThreads = std::make_unique<tbb::task_arena>(mActiveHostKernelThreads);
187 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_GLOBAL) {
188 if (mMaster == nullptr) {
191 }
192 mHostMemoryBase = operator new(mHostMemorySize, std::align_val_t(GPUCA_BUFFER_ALIGNMENT));
193 }
196 }
197 if (GetProcessingSettings().inKernelParallel) {
199 }
201 return 0;
202}
203
205{
206 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_GLOBAL) {
207 if (mMaster == nullptr) {
208 operator delete(mHostMemoryBase, std::align_val_t(GPUCA_BUFFER_ALIGNMENT));
209 }
211 mHostMemorySize = 0;
212 }
213 return 0;
214}
215
217{
218 mMemoryScalers->temporaryFactor = 1.;
219 if (GetProcessingSettings().memoryScalingFuzz) {
220 static std::mt19937 rng;
221 static std::uniform_int_distribution<uint64_t> dist(0, 1000000);
222 uint64_t fuzzFactor = GetProcessingSettings().memoryScalingFuzz == 1 ? dist(rng) : GetProcessingSettings().memoryScalingFuzz;
223 GPUInfo("Fuzzing memory scaling factor with %lu", fuzzFactor);
224 mMemoryScalers->fuzzScalingFactor(fuzzFactor);
225 }
226
227 mStatNEvents++;
229
230 if (GetProcessingSettings().debugLevel >= 3 || GetProcessingSettings().allocDebugLevel) {
231 printf("Allocated memory when starting processing %34s", "");
233 }
235 const std::clock_t cpuTimerStart = std::clock();
236 if (GetProcessingSettings().doublePipeline) {
237 int32_t retVal = EnqueuePipeline();
238 if (retVal) {
239 return retVal;
240 }
241 } else {
242 if (mSlaves.size() || mMaster) {
243 WriteConstantParams(); // Reinitialize // TODO: Get this in sync with GPUChainTracking::DoQueuedUpdates, and consider the doublePipeline
244 }
245 for (uint32_t i = 0; i < mChains.size(); i++) {
246 int32_t retVal = mChains[i]->RunChain();
247 if (retVal) {
248 return retVal;
249 }
250 }
251 if (GetProcessingSettings().tpcFreeAllocatedMemoryAfterProcessing) {
253 }
254 }
256 mStatCPUTime += (double)(std::clock() - cpuTimerStart) / CLOCKS_PER_SEC;
257 if (GetProcessingSettings().debugLevel >= 3 || GetProcessingSettings().allocDebugLevel) {
258 printf("Allocated memory when ending processing %36s", "");
260 }
261
263 std::string nEventReport;
264 if (GetProcessingSettings().debugLevel >= 0 && mStatNEvents > 1) {
265 nEventReport += " (avergage of " + std::to_string(mStatNEvents) + " runs)";
266 }
267 double kernelTotal = 0;
268 std::vector<double> kernelStepTimes(GPUDataTypes::N_RECO_STEPS, 0.);
269
270 if (GetProcessingSettings().debugLevel >= 1) {
271 for (uint32_t i = 0; i < mTimers.size(); i++) {
272 double time = 0;
273 if (mTimers[i] == nullptr) {
274 continue;
275 }
276 for (int32_t j = 0; j < mTimers[i]->num; j++) {
277 HighResTimer& timer = mTimers[i]->timer[j];
278 time += timer.GetElapsedTime();
279 if (GetProcessingSettings().resetTimers) {
280 timer.Reset();
281 }
282 }
283
284 uint32_t type = mTimers[i]->type;
285 if (type == 0) {
286 kernelTotal += time;
287 int32_t stepNum = getRecoStepNum(mTimers[i]->step);
288 kernelStepTimes[stepNum] += time;
289 }
290 char bandwidth[256] = "";
291 if (mTimers[i]->memSize && mStatNEvents && time != 0.) {
292 snprintf(bandwidth, 256, " (%8.3f GB/s - %'14zu bytes - %'14zu per call)", mTimers[i]->memSize / time * 1e-9, mTimers[i]->memSize / mStatNEvents, mTimers[i]->memSize / mStatNEvents / mTimers[i]->count);
293 }
294 printf("Execution Time: Task (%c %8ux): %50s Time: %'10.0f us%s\n", type == 0 ? 'K' : 'C', mTimers[i]->count, mTimers[i]->name.c_str(), time * 1000000 / mStatNEvents, bandwidth);
295 if (GetProcessingSettings().resetTimers) {
296 mTimers[i]->count = 0;
297 mTimers[i]->memSize = 0;
298 }
299 }
300 }
301 if (GetProcessingSettings().recoTaskTiming) {
302 for (int32_t i = 0; i < GPUDataTypes::N_RECO_STEPS; i++) {
303 if (kernelStepTimes[i] != 0. || mTimersRecoSteps[i].timerTotal.GetElapsedTime() != 0.) {
304 printf("Execution Time: Step : %11s %38s Time: %'10.0f us %64s ( Total Time : %'14.0f us, CPU Time : %'14.0f us, %'7.2fx )\n", "Tasks",
305 GPUDataTypes::RECO_STEP_NAMES[i], kernelStepTimes[i] * 1000000 / mStatNEvents, "", mTimersRecoSteps[i].timerTotal.GetElapsedTime() * 1000000 / mStatNEvents, mTimersRecoSteps[i].timerCPU * 1000000 / mStatNEvents, mTimersRecoSteps[i].timerCPU / mTimersRecoSteps[i].timerTotal.GetElapsedTime());
306 }
307 if (mTimersRecoSteps[i].bytesToGPU) {
308 printf("Execution Time: Step (D %8ux): %11s %38s Time: %'10.0f us (%8.3f GB/s - %'14zu bytes - %'14zu per call)\n", mTimersRecoSteps[i].countToGPU, "DMA to GPU", GPUDataTypes::RECO_STEP_NAMES[i], mTimersRecoSteps[i].timerToGPU.GetElapsedTime() * 1000000 / mStatNEvents,
310 }
311 if (mTimersRecoSteps[i].bytesToHost) {
312 printf("Execution Time: Step (D %8ux): %11s %38s Time: %'10.0f us (%8.3f GB/s - %'14zu bytes - %'14zu per call)\n", mTimersRecoSteps[i].countToHost, "DMA to Host", GPUDataTypes::RECO_STEP_NAMES[i], mTimersRecoSteps[i].timerToHost.GetElapsedTime() * 1000000 / mStatNEvents,
314 }
315 if (GetProcessingSettings().resetTimers) {
323 }
324 }
325 for (int32_t i = 0; i < GPUDataTypes::N_GENERAL_STEPS; i++) {
326 if (mTimersGeneralSteps[i].GetElapsedTime() != 0.) {
327 printf("Execution Time: General Step : %50s Time: %'10.0f us\n", GPUDataTypes::GENERAL_STEP_NAMES[i], mTimersGeneralSteps[i].GetElapsedTime() * 1000000 / mStatNEvents);
328 }
329 }
330 if (GetProcessingSettings().debugLevel >= 1) {
331 mStatKernelTime = kernelTotal * 1000000 / mStatNEvents;
332 printf("Execution Time: Total : %50s Time: %'10.0f us%s\n", "Total Kernel", mStatKernelTime, nEventReport.c_str());
333 }
334 printf("Execution Time: Total : %50s Time: %'10.0f us ( CPU Time : %'10.0f us, %7.2fx ) %s\n", "Total Wall", mStatWallTime, mStatCPUTime * 1000000 / mStatNEvents, mStatCPUTime / mTimerTotal.GetElapsedTime(), nEventReport.c_str());
335 } else if (GetProcessingSettings().debugLevel >= 0) {
336 GPUInfo("Total Wall Time: %10.0f us%s", mStatWallTime, nEventReport.c_str());
337 }
338 if (GetProcessingSettings().resetTimers) {
339 mStatNEvents = 0;
340 mStatCPUTime = 0;
342 }
343
344 return 0;
345}
346
348{
349 for (uint32_t i = 0; i < mProcessors.size(); i++) {
350 if (mProcessors[i].proc->mGPUProcessorType != GPUProcessor::PROCESSOR_TYPE_DEVICE && mProcessors[i].proc->mLinkedProcessor) {
351 mProcessors[i].proc->mLinkedProcessor->InitGPUProcessor(this, GPUProcessor::PROCESSOR_TYPE_DEVICE);
352 }
353 }
354}
355
356void GPUReconstructionCPU::UpdateParamOccupancyMap(const uint32_t* mapHost, const uint32_t* mapGPU, uint32_t occupancyTotal, int32_t stream)
357{
358 param().occupancyMap = mapHost;
359 param().occupancyTotal = occupancyTotal;
360 if (IsGPU()) {
361 if (!((size_t)&param().occupancyTotal - (size_t)&param().occupancyMap == sizeof(param().occupancyMap) && sizeof(param().occupancyMap) == sizeof(size_t) && sizeof(param().occupancyTotal) < sizeof(size_t))) {
362 throw std::runtime_error("occupancy data not consecutive in GPUParam");
363 }
364 const auto holdContext = GetThreadContext();
365 size_t tmp[2] = {(size_t)mapGPU, 0};
366 memcpy(&tmp[1], &occupancyTotal, sizeof(occupancyTotal));
367 WriteToConstantMemory((char*)&processors()->param.occupancyMap - (char*)processors(), &tmp, sizeof(param().occupancyMap) + sizeof(param().occupancyTotal), stream);
368 }
369}
int16_t time
Definition RawEventData.h:4
int32_t i
#define GPUCA_BUFFER_ALIGNMENT
int32_t retVal
Online TRD tracker based on extrapolated TPC tracks.
Used for storing the MC labels for the TRD tracklets.
TRD Tracklet word for GPU tracker - 32bit tracklet info + half chamber ID + index.
uint32_t j
Definition RawData.h:0
uint32_t res
Definition RawData.h:0
TBranch * ptr
double num
void Reset()
Definition timer.cxx:101
void Start()
Definition timer.cxx:57
double GetElapsedTime()
Definition timer.cxx:108
void Stop()
Definition timer.cxx:69
static constexpr const char *const GENERAL_STEP_NAMES[]
static constexpr const char *const RECO_STEP_NAMES[]
static constexpr int32_t N_RECO_STEPS
static constexpr int32_t N_GENERAL_STEPS
ProcessorType mGPUProcessorType
virtual size_t GPUMemCpy(void *dst, const void *src, size_t size, int32_t stream, int32_t toGPU, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
void runKernelBackend(const krnlSetupTime &_xyz, const Args &... args)
virtual size_t GPUMemCpyAlways(bool onGpu, void *dst, const void *src, size_t size, int32_t stream, int32_t toGPU, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
static constexpr krnlRunRange krnlRunRangeNone
size_t TransferMemoryResourceToHost(GPUMemoryResource *res, int32_t stream=-1, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
void UpdateParamOccupancyMap(const uint32_t *mapHost, const uint32_t *mapGPU, uint32_t occupancyTotal, int32_t stream=-1)
size_t TransferMemoryResourceToGPU(GPUMemoryResource *res, int32_t stream=-1, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
GPUProcessorProcessors mProcShadow
krnlProperties getKernelProperties(int gpu=-1)
virtual int32_t GPUDebug(const char *state="UNKNOWN", int32_t stream=-1, bool force=false)
static constexpr krnlEvent krnlEventNone
size_t WriteToConstantMemory(size_t offset, const void *src, size_t size, int32_t stream=-1, deviceEvent *ev=nullptr) override
virtual size_t TransferMemoryInternal(GPUMemoryResource *res, int32_t stream, deviceEvent *ev, deviceEvent *evList, int32_t nEvents, bool toGPU, const void *src, void *dst)
RecoStepTimerMeta mTimersRecoSteps[GPUDataTypes::N_RECO_STEPS]
std::vector< std::unique_ptr< timerMeta > > mTimers
HighResTimer mTimersGeneralSteps[GPUDataTypes::N_GENERAL_STEPS]
virtual std::unique_ptr< threadContext > GetThreadContext() override
std::vector< std::unique_ptr< GPUChain > > mChains
std::unique_ptr< GPUMemorySizeScalers > mMemoryScalers
std::vector< GPUReconstruction * > mSlaves
std::vector< GPUMemoryResource > mMemoryResources
std::vector< ProcessorData > mProcessors
static GPUReconstruction * GPUReconstruction_Create_CPU(const GPUSettingsDeviceBackend &cfg)
void ClearAllocatedMemory(bool clearOutputs=true)
int32_t getRecoStepNum(RecoStep step, bool validCheck=true)
const GPUSettingsProcessing & GetProcessingSettings() const
int32_t EnqueuePipeline(bool terminate=false)
std::shared_ptr< GPUReconstructionThreading > mThreading
GLdouble n
Definition glcorearb.h:1982
GLint GLenum GLint x
Definition glcorearb.h:403
GLenum src
Definition glcorearb.h:1767
GLint GLsizei count
Definition glcorearb.h:399
GLsizeiptr size
Definition glcorearb.h:659
GLint GLint GLsizei GLint GLenum GLenum type
Definition glcorearb.h:275
GLenum GLenum dst
Definition glcorearb.h:1767
GLintptr offset
Definition glcorearb.h:660
GLboolean r
Definition glcorearb.h:1233
GLenum GLfloat param
Definition glcorearb.h:271
GLuint GLuint stream
Definition glcorearb.h:1806
std::string to_string(gsl::span< T, Size > span)
Definition common.h:52
const uint32_t * occupancyMap
Definition GPUParam.h:64
const int nEvents
Definition test_Fifo.cxx:27