Project
Loading...
Searching...
No Matches
GPUReconstructionCPU.cxx
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
14
18#include "GPUChain.h"
19#include "GPUDefParametersRuntime.h"
20#include "GPUTPCClusterData.h"
22#include "GPUTPCGMMergedTrack.h"
24#include "GPUTRDTrackletWord.h"
26#include "GPUTPCMCInfo.h"
27#include "GPUTRDTrack.h"
28#include "GPUTRDTracker.h"
29#include "AliHLTTPCRawCluster.h"
31#include "GPUMemoryResource.h"
32#include "GPUConstantMem.h"
33#include "GPULogging.h"
35#include "GPUReconstructionProcessingKernels.inc"
36
37#include <atomic>
38#include <ctime>
39
40#ifndef _WIN32
41#include <unistd.h>
42#endif
43
44using namespace o2::gpu;
45
48
50
52{
53 Exit(); // Needs to be identical to GPU backend bahavior in order to avoid calling abstract methods later in the destructor
54}
55
56template <class T, int32_t I, typename... Args>
57inline void GPUReconstructionCPU::runKernelBackend(const krnlSetupTime& _xyz, const Args&... args)
58{
59 auto& x = _xyz.x;
60 auto& y = _xyz.y;
61 if (x.device == krnlDeviceType::Device) {
62 throw std::runtime_error("Cannot run device kernel on host");
63 }
64 if (x.nThreads != 1) {
65 throw std::runtime_error("Cannot run device kernel on host with nThreads != 1");
66 }
67 int32_t nThreads = getNKernelHostThreads(false);
68 if (nThreads > 1) {
69 if (GetProcessingSettings().debugLevel >= 5) {
70 printf("Running %d Threads\n", mThreading->activeThreads->max_concurrency());
71 }
72 tbb::this_task_arena::isolate([&] {
73 mThreading->activeThreads->execute([&] {
74 tbb::parallel_for(tbb::blocked_range<uint32_t>(0, x.nBlocks, 1), [&](const tbb::blocked_range<uint32_t>& r) {
75 typename T::GPUSharedMemory smem;
76 for (uint32_t iB = r.begin(); iB < r.end(); iB++) {
77 T::template Thread<I>(x.nBlocks, 1, iB, 0, smem, T::Processor(*mHostConstantMem)[y.index], args...);
78 }
79 });
80 });
81 });
82 } else {
83 for (uint32_t iB = 0; iB < x.nBlocks; iB++) {
84 typename T::GPUSharedMemory smem;
85 T::template Thread<I>(x.nBlocks, 1, iB, 0, smem, T::Processor(*mHostConstantMem)[y.index], args...);
86 }
87 }
88}
89
90template <>
91inline void GPUReconstructionCPU::runKernelBackend<GPUMemClean16, 0>(const krnlSetupTime& _xyz, void* const& ptr, uint64_t const& size)
92{
93 int32_t nThreads = std::max<int32_t>(1, std::min<int32_t>(size / (16 * 1024 * 1024), getNKernelHostThreads(true)));
94 if (nThreads > 1) {
95 tbb::parallel_for(0, nThreads, [&](int iThread) {
96 size_t threadSize = size / nThreads;
97 if (threadSize % 4096) {
98 threadSize += 4096 - threadSize % 4096;
99 }
100 size_t offset = threadSize * iThread;
101 size_t mySize = std::min<size_t>(threadSize, size - offset);
102 if (mySize) {
103 memset((char*)ptr + offset, 0, mySize);
104 } // clang-format off
105 }, tbb::static_partitioner()); // clang-format on
106 } else {
107 memset(ptr, 0, size);
108 }
109}
110
111template <class S, int32_t I>
113{
114 if (gpu == -1) {
115 gpu = IsGPU();
116 }
117 const auto num = GetKernelNum<S, I>();
118 const auto* p = gpu ? mParDevice : mParCPU;
119 GPUReconstructionProcessing::krnlProperties ret = {p->par_LB_maxThreads[num], p->par_LB_minBlocks[num], p->par_LB_forceBlocks[num]};
120 if (ret.nThreads == 0) {
121 ret.nThreads = gpu ? mThreadCount : 1u;
122 }
123 if (ret.minBlocks == 0) {
124 ret.minBlocks = 1;
125 }
126 return ret;
127}
128
129#define GPUCA_KRNL(x_class, x_attributes, x_arguments, x_forward, x_types, ...) \
130 template GPUReconstructionProcessing::krnlProperties GPUReconstructionCPU::getKernelProperties<GPUCA_M_KRNL_TEMPLATE(x_class)>(int gpu);
131#include "GPUReconstructionKernelList.h"
132#undef GPUCA_KRNL
133
134size_t GPUReconstructionCPU::TransferMemoryInternal(GPUMemoryResource* res, int32_t stream, deviceEvent* ev, deviceEvent* evList, int32_t nEvents, bool toGPU, const void* src, void* dst) { return 0; }
135size_t GPUReconstructionCPU::GPUMemCpy(void* dst, const void* src, size_t size, int32_t stream, int32_t toGPU, deviceEvent* ev, deviceEvent* evList, int32_t nEvents) { return 0; }
136size_t GPUReconstructionCPU::GPUMemCpyAlways(bool onGpu, void* dst, const void* src, size_t size, int32_t stream, int32_t toGPU, deviceEvent* ev, deviceEvent* evList, int32_t nEvents)
137{
138 memcpy(dst, src, size);
139 return 0;
140}
141size_t GPUReconstructionCPU::WriteToConstantMemory(size_t offset, const void* src, size_t size, int32_t stream, deviceEvent* ev) { return 0; }
142int32_t GPUReconstructionCPU::GPUDebug(const char* state, int32_t stream, bool force) { return 0; }
143size_t GPUReconstructionCPU::TransferMemoryResourcesHelper(GPUProcessor* proc, int32_t stream, bool all, bool toGPU)
144{
147 size_t n = 0;
148 for (uint32_t i = 0; i < mMemoryResources.size(); i++) {
150 if (res.mPtr == nullptr) {
151 continue;
152 }
153 if (proc && res.mProcessor != proc) {
154 continue;
155 }
157 continue;
158 }
159 if (!GetProcessingSettings().keepAllMemory && !all && (res.mType & exc) && !(res.mType & inc)) {
160 continue;
161 }
162 if (toGPU) {
164 } else {
166 }
167 }
168 return n;
169}
170
172{
173// Get Thread ID
174#if defined(__APPLE__)
175 return (0); // syscall is deprecated on MacOS..., only needed for GPU support which we don't do on Mac anyway
176#elif defined(_WIN32)
177 return ((int32_t)(size_t)GetCurrentThread());
178#else
179 return ((int32_t)syscall(SYS_gettid));
180#endif
181}
182
184{
186 mThreading->activeThreads = std::make_unique<tbb::task_arena>(mActiveHostKernelThreads);
187 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_GLOBAL) {
188 if (mMaster == nullptr) {
191 }
192 mHostMemoryBase = operator new(mHostMemorySize, std::align_val_t(GPUCA_BUFFER_ALIGNMENT));
193 }
196 }
197 if (GetProcessingSettings().inKernelParallel) {
199 }
201 return 0;
202}
203
205{
206 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_GLOBAL) {
207 if (mMaster == nullptr) {
208 operator delete(mHostMemoryBase, std::align_val_t(GPUCA_BUFFER_ALIGNMENT));
209 }
211 mHostMemorySize = 0;
212 }
213 return 0;
214}
215
217{
218 mMemoryScalers->temporaryFactor = 1.;
219 mStatNEvents++;
221
222 if (GetProcessingSettings().debugLevel >= 3 || GetProcessingSettings().allocDebugLevel) {
223 printf("Allocated memory when starting processing %34s", "");
225 }
227 const std::clock_t cpuTimerStart = std::clock();
228 if (GetProcessingSettings().doublePipeline) {
229 int32_t retVal = EnqueuePipeline();
230 if (retVal) {
231 return retVal;
232 }
233 } else {
234 if (mSlaves.size() || mMaster) {
235 WriteConstantParams(); // Reinitialize // TODO: Get this in sync with GPUChainTracking::DoQueuedUpdates, and consider the doublePipeline
236 }
237 for (uint32_t i = 0; i < mChains.size(); i++) {
238 int32_t retVal = mChains[i]->RunChain();
239 if (retVal) {
240 return retVal;
241 }
242 }
243 if (GetProcessingSettings().tpcFreeAllocatedMemoryAfterProcessing) {
245 }
246 }
248 mStatCPUTime += (double)(std::clock() - cpuTimerStart) / CLOCKS_PER_SEC;
249 if (GetProcessingSettings().debugLevel >= 3 || GetProcessingSettings().allocDebugLevel) {
250 printf("Allocated memory when ending processing %36s", "");
252 }
253
255 std::string nEventReport;
256 if (GetProcessingSettings().debugLevel >= 0 && mStatNEvents > 1) {
257 nEventReport += " (avergage of " + std::to_string(mStatNEvents) + " runs)";
258 }
259 double kernelTotal = 0;
260 std::vector<double> kernelStepTimes(GPUDataTypes::N_RECO_STEPS, 0.);
261
262 if (GetProcessingSettings().debugLevel >= 1) {
263 for (uint32_t i = 0; i < mTimers.size(); i++) {
264 double time = 0;
265 if (mTimers[i] == nullptr) {
266 continue;
267 }
268 for (int32_t j = 0; j < mTimers[i]->num; j++) {
269 HighResTimer& timer = mTimers[i]->timer[j];
270 time += timer.GetElapsedTime();
271 if (GetProcessingSettings().resetTimers) {
272 timer.Reset();
273 }
274 }
275
276 uint32_t type = mTimers[i]->type;
277 if (type == 0) {
278 kernelTotal += time;
279 int32_t stepNum = getRecoStepNum(mTimers[i]->step);
280 kernelStepTimes[stepNum] += time;
281 }
282 char bandwidth[256] = "";
283 if (mTimers[i]->memSize && mStatNEvents && time != 0.) {
284 snprintf(bandwidth, 256, " (%8.3f GB/s - %'14zu bytes - %'14zu per call)", mTimers[i]->memSize / time * 1e-9, mTimers[i]->memSize / mStatNEvents, mTimers[i]->memSize / mStatNEvents / mTimers[i]->count);
285 }
286 printf("Execution Time: Task (%c %8ux): %50s Time: %'10.0f us%s\n", type == 0 ? 'K' : 'C', mTimers[i]->count, mTimers[i]->name.c_str(), time * 1000000 / mStatNEvents, bandwidth);
287 if (GetProcessingSettings().resetTimers) {
288 mTimers[i]->count = 0;
289 mTimers[i]->memSize = 0;
290 }
291 }
292 }
293 if (GetProcessingSettings().recoTaskTiming) {
294 for (int32_t i = 0; i < GPUDataTypes::N_RECO_STEPS; i++) {
295 if (kernelStepTimes[i] != 0. || mTimersRecoSteps[i].timerTotal.GetElapsedTime() != 0.) {
296 printf("Execution Time: Step : %11s %38s Time: %'10.0f us %64s ( Total Time : %'14.0f us, CPU Time : %'14.0f us, %'7.2fx )\n", "Tasks",
297 GPUDataTypes::RECO_STEP_NAMES[i], kernelStepTimes[i] * 1000000 / mStatNEvents, "", mTimersRecoSteps[i].timerTotal.GetElapsedTime() * 1000000 / mStatNEvents, mTimersRecoSteps[i].timerCPU * 1000000 / mStatNEvents, mTimersRecoSteps[i].timerCPU / mTimersRecoSteps[i].timerTotal.GetElapsedTime());
298 }
299 if (mTimersRecoSteps[i].bytesToGPU) {
300 printf("Execution Time: Step (D %8ux): %11s %38s Time: %'10.0f us (%8.3f GB/s - %'14zu bytes - %'14zu per call)\n", mTimersRecoSteps[i].countToGPU, "DMA to GPU", GPUDataTypes::RECO_STEP_NAMES[i], mTimersRecoSteps[i].timerToGPU.GetElapsedTime() * 1000000 / mStatNEvents,
302 }
303 if (mTimersRecoSteps[i].bytesToHost) {
304 printf("Execution Time: Step (D %8ux): %11s %38s Time: %'10.0f us (%8.3f GB/s - %'14zu bytes - %'14zu per call)\n", mTimersRecoSteps[i].countToHost, "DMA to Host", GPUDataTypes::RECO_STEP_NAMES[i], mTimersRecoSteps[i].timerToHost.GetElapsedTime() * 1000000 / mStatNEvents,
306 }
307 if (GetProcessingSettings().resetTimers) {
315 }
316 }
317 for (int32_t i = 0; i < GPUDataTypes::N_GENERAL_STEPS; i++) {
318 if (mTimersGeneralSteps[i].GetElapsedTime() != 0.) {
319 printf("Execution Time: General Step : %50s Time: %'10.0f us\n", GPUDataTypes::GENERAL_STEP_NAMES[i], mTimersGeneralSteps[i].GetElapsedTime() * 1000000 / mStatNEvents);
320 }
321 }
322 if (GetProcessingSettings().debugLevel >= 1) {
323 mStatKernelTime = kernelTotal * 1000000 / mStatNEvents;
324 printf("Execution Time: Total : %50s Time: %'10.0f us%s\n", "Total Kernel", mStatKernelTime, nEventReport.c_str());
325 }
326 printf("Execution Time: Total : %50s Time: %'10.0f us ( CPU Time : %'10.0f us, %7.2fx ) %s\n", "Total Wall", mStatWallTime, mStatCPUTime * 1000000 / mStatNEvents, mStatCPUTime / mTimerTotal.GetElapsedTime(), nEventReport.c_str());
327 } else if (GetProcessingSettings().debugLevel >= 0) {
328 GPUInfo("Total Wall Time: %10.0f us%s", mStatWallTime, nEventReport.c_str());
329 }
330 if (GetProcessingSettings().resetTimers) {
331 mStatNEvents = 0;
332 mStatCPUTime = 0;
334 }
335
336 return 0;
337}
338
340{
341 for (uint32_t i = 0; i < mProcessors.size(); i++) {
342 if (mProcessors[i].proc->mGPUProcessorType != GPUProcessor::PROCESSOR_TYPE_DEVICE && mProcessors[i].proc->mLinkedProcessor) {
343 mProcessors[i].proc->mLinkedProcessor->InitGPUProcessor(this, GPUProcessor::PROCESSOR_TYPE_DEVICE);
344 }
345 }
346}
347
348void GPUReconstructionCPU::UpdateParamOccupancyMap(const uint32_t* mapHost, const uint32_t* mapGPU, uint32_t occupancyTotal, int32_t stream)
349{
350 param().occupancyMap = mapHost;
351 param().occupancyTotal = occupancyTotal;
352 if (IsGPU()) {
353 if (!((size_t)&param().occupancyTotal - (size_t)&param().occupancyMap == sizeof(param().occupancyMap) && sizeof(param().occupancyMap) == sizeof(size_t) && sizeof(param().occupancyTotal) < sizeof(size_t))) {
354 throw std::runtime_error("occupancy data not consecutive in GPUParam");
355 }
356 const auto holdContext = GetThreadContext();
357 size_t tmp[2] = {(size_t)mapGPU, 0};
358 memcpy(&tmp[1], &occupancyTotal, sizeof(occupancyTotal));
359 WriteToConstantMemory((char*)&processors()->param.occupancyMap - (char*)processors(), &tmp, sizeof(param().occupancyMap) + sizeof(param().occupancyTotal), stream);
360 }
361}
int16_t time
Definition RawEventData.h:4
int32_t i
#define GPUCA_BUFFER_ALIGNMENT
int32_t retVal
Online TRD tracker based on extrapolated TPC tracks.
Used for storing the MC labels for the TRD tracklets.
TRD Tracklet word for GPU tracker - 32bit tracklet info + half chamber ID + index.
uint32_t j
Definition RawData.h:0
uint32_t res
Definition RawData.h:0
TBranch * ptr
double num
void Reset()
Definition timer.cxx:101
void Start()
Definition timer.cxx:57
double GetElapsedTime()
Definition timer.cxx:108
void Stop()
Definition timer.cxx:69
static constexpr const char *const GENERAL_STEP_NAMES[]
static constexpr const char *const RECO_STEP_NAMES[]
static constexpr int32_t N_RECO_STEPS
static constexpr int32_t N_GENERAL_STEPS
ProcessorType mGPUProcessorType
virtual size_t GPUMemCpy(void *dst, const void *src, size_t size, int32_t stream, int32_t toGPU, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
void runKernelBackend(const krnlSetupTime &_xyz, const Args &... args)
virtual size_t GPUMemCpyAlways(bool onGpu, void *dst, const void *src, size_t size, int32_t stream, int32_t toGPU, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
static constexpr krnlRunRange krnlRunRangeNone
size_t TransferMemoryResourceToHost(GPUMemoryResource *res, int32_t stream=-1, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
void UpdateParamOccupancyMap(const uint32_t *mapHost, const uint32_t *mapGPU, uint32_t occupancyTotal, int32_t stream=-1)
size_t TransferMemoryResourceToGPU(GPUMemoryResource *res, int32_t stream=-1, deviceEvent *ev=nullptr, deviceEvent *evList=nullptr, int32_t nEvents=1)
GPUProcessorProcessors mProcShadow
krnlProperties getKernelProperties(int gpu=-1)
virtual int32_t GPUDebug(const char *state="UNKNOWN", int32_t stream=-1, bool force=false)
static constexpr krnlEvent krnlEventNone
size_t WriteToConstantMemory(size_t offset, const void *src, size_t size, int32_t stream=-1, deviceEvent *ev=nullptr) override
virtual size_t TransferMemoryInternal(GPUMemoryResource *res, int32_t stream, deviceEvent *ev, deviceEvent *evList, int32_t nEvents, bool toGPU, const void *src, void *dst)
RecoStepTimerMeta mTimersRecoSteps[GPUDataTypes::N_RECO_STEPS]
std::vector< std::unique_ptr< timerMeta > > mTimers
HighResTimer mTimersGeneralSteps[GPUDataTypes::N_GENERAL_STEPS]
virtual std::unique_ptr< threadContext > GetThreadContext() override
std::vector< std::unique_ptr< GPUChain > > mChains
std::unique_ptr< GPUMemorySizeScalers > mMemoryScalers
std::vector< GPUReconstruction * > mSlaves
std::vector< GPUMemoryResource > mMemoryResources
std::vector< ProcessorData > mProcessors
static GPUReconstruction * GPUReconstruction_Create_CPU(const GPUSettingsDeviceBackend &cfg)
void ClearAllocatedMemory(bool clearOutputs=true)
int32_t getRecoStepNum(RecoStep step, bool validCheck=true)
const GPUSettingsProcessing & GetProcessingSettings() const
int32_t EnqueuePipeline(bool terminate=false)
std::shared_ptr< GPUReconstructionThreading > mThreading
GLdouble n
Definition glcorearb.h:1982
GLint GLenum GLint x
Definition glcorearb.h:403
GLenum src
Definition glcorearb.h:1767
GLint GLsizei count
Definition glcorearb.h:399
GLsizeiptr size
Definition glcorearb.h:659
GLint GLint GLsizei GLint GLenum GLenum type
Definition glcorearb.h:275
GLenum GLenum dst
Definition glcorearb.h:1767
GLintptr offset
Definition glcorearb.h:660
GLboolean r
Definition glcorearb.h:1233
GLenum GLfloat param
Definition glcorearb.h:271
GLuint GLuint stream
Definition glcorearb.h:1806
std::string to_string(gsl::span< T, Size > span)
Definition common.h:52
const uint32_t * occupancyMap
Definition GPUParam.h:62
const int nEvents
Definition test_Fifo.cxx:27