Project
Loading...
Searching...
No Matches
OrtInterface.cxx
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
15
16#include "ML/OrtInterface.h"
18
19// ONNX includes
20#include <onnxruntime_cxx_api.h>
21
22#include <sstream>
23
24namespace o2
25{
26
27namespace ml
28{
29
30OrtModel::OrtModel() = default;
31OrtModel::OrtModel(std::unordered_map<std::string, std::string> optionsMap) { init(optionsMap); }
32OrtModel::~OrtModel() = default;
33void OrtModel::init(std::unordered_map<std::string, std::string> optionsMap)
34{
35 initOptions(optionsMap);
37}
38
39struct OrtModel::OrtVariables { // The actual implementation is hidden in the .cxx file
40 // ORT runtime objects
41 Ort::RunOptions runOptions;
42 std::unique_ptr<Ort::Env> env = nullptr;
43 std::unique_ptr<Ort::Session> session = nullptr;
44 Ort::SessionOptions sessionOptions;
45 Ort::AllocatorWithDefaultOptions allocator;
46 Ort::MemoryInfo memoryInfo = Ort::MemoryInfo("Cpu", OrtAllocatorType::OrtDeviceAllocator, 0, OrtMemType::OrtMemTypeDefault);
47 std::unique_ptr<Ort::IoBinding> ioBinding = nullptr;
48};
49
50// General purpose
51void OrtModel::initOptions(std::unordered_map<std::string, std::string> optionsMap)
52{
53 mPImplOrt = std::make_unique<OrtVariables>();
54
55 // Load from options map
56 if (!optionsMap.contains("model-path")) {
57 LOG(fatal) << "(ORT) Model path cannot be empty!";
58 }
59
60 if (!optionsMap["model-path"].empty()) {
61 mModelPath = optionsMap["model-path"];
62 mDeviceType = (optionsMap.contains("device-type") ? optionsMap["device-type"] : "CPU");
63 mDeviceId = (optionsMap.contains("device-id") ? std::stoi(optionsMap["device-id"]) : -1);
64 mAllocateDeviceMemory = (optionsMap.contains("allocate-device-memory") ? std::stoi(optionsMap["allocate-device-memory"]) : 0);
65 mIntraOpNumThreads = (optionsMap.contains("intra-op-num-threads") ? std::stoi(optionsMap["intra-op-num-threads"]) : 0);
66 mInterOpNumThreads = (optionsMap.contains("inter-op-num-threads") ? std::stoi(optionsMap["inter-op-num-threads"]) : 0);
67 mLoggingLevel = (optionsMap.contains("logging-level") ? std::stoi(optionsMap["logging-level"]) : 0);
68 mEnableProfiling = (optionsMap.contains("enable-profiling") ? std::stoi(optionsMap["enable-profiling"]) : 0);
69 mEnableOptimizations = (optionsMap.contains("enable-optimizations") ? std::stoi(optionsMap["enable-optimizations"]) : 0);
70 mEnvName = (optionsMap.contains("onnx-environment-name") ? optionsMap["onnx-environment-name"] : "onnx_model_inference");
71
72 if (mDeviceType == "CPU") {
73 (mPImplOrt->sessionOptions).SetIntraOpNumThreads(mIntraOpNumThreads);
74 (mPImplOrt->sessionOptions).SetInterOpNumThreads(mInterOpNumThreads);
75 if (mIntraOpNumThreads > 1 || mInterOpNumThreads > 1) {
76 (mPImplOrt->sessionOptions).SetExecutionMode(ExecutionMode::ORT_PARALLEL);
77 } else if (mIntraOpNumThreads == 1) {
78 (mPImplOrt->sessionOptions).SetExecutionMode(ExecutionMode::ORT_SEQUENTIAL);
79 }
80 if (mLoggingLevel < 2) {
81 LOG(info) << "(ORT) CPU execution provider set with " << mIntraOpNumThreads << " (mIntraOpNumThreads) and " << mInterOpNumThreads << " (mInterOpNumThreads) threads";
82 }
83 }
84
85 // OrtROCMProviderOptions rocm_options{};
86 // (mPImplOrt->sessionOptions).AppendExecutionProvider_ROCM(rocm_options);
87
88 (mPImplOrt->sessionOptions).DisableMemPattern();
89 (mPImplOrt->sessionOptions).DisableCpuMemArena();
90
91 if (mEnableProfiling) {
92 if (optionsMap.contains("profiling-output-path")) {
93 (mPImplOrt->sessionOptions).EnableProfiling((optionsMap["profiling-output-path"] + "/ORT_LOG_").c_str());
94 } else {
95 LOG(warning) << "(ORT) If profiling is enabled, optionsMap[\"profiling-output-path\"] should be set. Disabling profiling for now.";
96 (mPImplOrt->sessionOptions).DisableProfiling();
97 }
98 } else {
99 (mPImplOrt->sessionOptions).DisableProfiling();
100 }
101
102 (mPImplOrt->sessionOptions).SetGraphOptimizationLevel(GraphOptimizationLevel(mEnableOptimizations));
103 (mPImplOrt->sessionOptions).SetLogSeverityLevel(OrtLoggingLevel(mLoggingLevel));
104
105 mInitialized = true;
106 } else {
107 LOG(fatal) << "(ORT) Model path cannot be empty!";
108 }
109}
110
112{
113 mPImplOrt->env = std::make_unique<Ort::Env>(
114 OrtLoggingLevel(mLoggingLevel),
115 (mEnvName.empty() ? "ORT" : mEnvName.c_str()),
116 // Integrate ORT logging into Fairlogger
117 [](void* param, OrtLoggingLevel severity, const char* category, const char* logid, const char* code_location, const char* message) {
118 if (severity == ORT_LOGGING_LEVEL_VERBOSE) {
119 LOG(debug) << "(ORT) [" << logid << "|" << category << "|" << code_location << "]: " << message;
120 } else if (severity == ORT_LOGGING_LEVEL_INFO) {
121 LOG(info) << "(ORT) [" << logid << "|" << category << "|" << code_location << "]: " << message;
122 } else if (severity == ORT_LOGGING_LEVEL_WARNING) {
123 LOG(warning) << "(ORT) [" << logid << "|" << category << "|" << code_location << "]: " << message;
124 } else if (severity == ORT_LOGGING_LEVEL_ERROR) {
125 LOG(error) << "(ORT) [" << logid << "|" << category << "|" << code_location << "]: " << message;
126 } else if (severity == ORT_LOGGING_LEVEL_FATAL) {
127 LOG(fatal) << "(ORT) [" << logid << "|" << category << "|" << code_location << "]: " << message;
128 } else {
129 LOG(info) << "(ORT) [" << logid << "|" << category << "|" << code_location << "]: " << message;
130 }
131 },
132 (void*)3);
133 (mPImplOrt->env)->DisableTelemetryEvents(); // Disable telemetry events
134}
135
137{
138 if (mAllocateDeviceMemory) {
139 memoryOnDevice(mDeviceId);
140 }
141 mPImplOrt->session = std::make_unique<Ort::Session>(*mPImplOrt->env, mModelPath.c_str(), mPImplOrt->sessionOptions);
142 mPImplOrt->ioBinding = std::make_unique<Ort::IoBinding>(*mPImplOrt->session);
143
144 setIO();
145
146 if (mLoggingLevel < 2) {
147 LOG(info) << "(ORT) Model loaded successfully! (inputs: " << printShape(mInputShapes, mInputNames) << ", outputs: " << printShape(mOutputShapes, mInputNames) << ")";
148 }
149}
150
151void OrtModel::memoryOnDevice(int32_t deviceIndex)
152{
153 if (deviceIndex >= 0) {
154 (mPImplOrt->runOptions).AddConfigEntry("disable_synchronize_execution_providers", "1");
155 (mPImplOrt->sessionOptions).AddConfigEntry("session.use_device_allocator_for_initializers", "1"); // See kOrtSessionOptionsUseDeviceAllocatorForInitializers, https://github.com/microsoft/onnxruntime/blob/main/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h
156 (mPImplOrt->sessionOptions).AddConfigEntry("session.use_env_allocators", "1"); // This should enable to use the volatile memory allocation defined in O2/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerHost.cxx; not working yet: ONNX still assigns new memory at init time
157 (mPImplOrt->sessionOptions).AddConfigEntry("session_options.enable_cpu_mem_arena", "0"); // This should enable to use the volatile memory allocation defined in O2/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerHost.cxx; not working yet: ONNX still assigns new memory at init time
158 // Arena memory shrinkage comes at performance cost
159 // For now prefer to use single allocation, enabled by O2/GPU/GPUTracking/Base/cuda/GPUReconstructionCUDA.cu -> SetONNXGPUStream -> rocm_options.arena_extend_strategy = 0;
160 (mPImplOrt->runOptions).AddConfigEntry("memory.enable_memory_arena_shrinkage", ("gpu:" + std::to_string(deviceIndex)).c_str()); // See kOrtRunOptionsConfigEnableMemoryArenaShrinkage, https://github.com/microsoft/onnxruntime/blob/90c263f471bbce724e77d8e62831d3a9fa838b2f/include/onnxruntime/core/session/onnxruntime_run_options_config_keys.h#L27
161
162 std::string dev_mem_str = "";
163 if (mDeviceType == "ROCM") {
164 dev_mem_str = "HipPinned";
165 }
166 if (mDeviceType == "CUDA") {
167 dev_mem_str = "Cuda";
168 }
169 mPImplOrt->memoryInfo = Ort::MemoryInfo(dev_mem_str.c_str(), OrtAllocatorType::OrtDeviceAllocator, deviceIndex, OrtMemType::OrtMemTypeDefault);
170 if (mLoggingLevel < 2) {
171 LOG(info) << "(ORT) Memory info set to on-device memory for device type " << mDeviceType << " with ID " << deviceIndex << " and mPImplOrt pointer " << mPImplOrt;
172 }
173 }
174}
175
177{
178 mPImplOrt->session = std::make_unique<Ort::Session>(*(mPImplOrt->env), mModelPath.c_str(), mPImplOrt->sessionOptions);
179}
180
181// Getters
182Ort::SessionOptions* OrtModel::getSessionOptions()
183{
184 return &mPImplOrt->sessionOptions;
185}
186
187Ort::MemoryInfo* OrtModel::getMemoryInfo()
188{
189 return &mPImplOrt->memoryInfo;
190}
191
193{
194 return (mPImplOrt->env).get();
195}
196
197template <class I, class O>
198std::vector<O> OrtModel::v2v(std::vector<I>& input, bool clearInput)
199{
200 if constexpr (std::is_same_v<I, O>) {
201 return input;
202 } else {
203 std::vector<O> output(input.size());
204 std::transform(std::begin(input), std::end(input), std::begin(output), [](I f) { return O(f); });
205 if (clearInput) {
206 input.clear();
207 }
208 return output;
209 }
210}
211
213{
214 for (size_t i = 0; i < (mPImplOrt->session)->GetInputCount(); ++i) {
215 mInputNames.push_back((mPImplOrt->session)->GetInputNameAllocated(i, mPImplOrt->allocator).get());
216 }
217 for (size_t i = 0; i < (mPImplOrt->session)->GetInputCount(); ++i) {
218 mInputShapes.emplace_back((mPImplOrt->session)->GetInputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape());
219 }
220 for (size_t i = 0; i < (mPImplOrt->session)->GetOutputCount(); ++i) {
221 mOutputNames.push_back((mPImplOrt->session)->GetOutputNameAllocated(i, mPImplOrt->allocator).get());
222 }
223 for (size_t i = 0; i < (mPImplOrt->session)->GetOutputCount(); ++i) {
224 mOutputShapes.emplace_back((mPImplOrt->session)->GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape());
225 }
226
227 mInputNamesChar.resize(mInputNames.size(), nullptr);
228 std::transform(std::begin(mInputNames), std::end(mInputNames), std::begin(mInputNamesChar),
229 [&](const std::string& str) { return str.c_str(); });
230 mOutputNamesChar.resize(mOutputNames.size(), nullptr);
231 std::transform(std::begin(mOutputNames), std::end(mOutputNames), std::begin(mOutputNamesChar),
232 [&](const std::string& str) { return str.c_str(); });
233
234 mInputShapesCopy = mInputShapes;
235 mOutputShapesCopy = mOutputShapes;
236 mInputSizePerNode.resize(mInputShapes.size(), 1);
237 mOutputSizePerNode.resize(mOutputShapes.size(), 1);
238 mInputsTotal = 1;
239 for (size_t i = 0; i < mInputShapes.size(); ++i) {
240 if (mInputShapes[i].size() > 0) {
241 for (size_t j = 1; j < mInputShapes[i].size(); ++j) {
242 if (mInputShapes[i][j] > 0) {
243 mInputsTotal *= mInputShapes[i][j];
244 mInputSizePerNode[i] *= mInputShapes[i][j];
245 }
246 }
247 }
248 }
249 mOutputsTotal = 1;
250 for (size_t i = 0; i < mOutputShapes.size(); ++i) {
251 if (mOutputShapes[i].size() > 0) {
252 for (size_t j = 1; j < mOutputShapes[i].size(); ++j) {
253 if (mOutputShapes[i][j] > 0) {
254 mOutputsTotal *= mOutputShapes[i][j];
255 mOutputSizePerNode[i] *= mOutputShapes[i][j];
256 }
257 }
258 }
259 }
260}
261
262void OrtModel::setEnv(Ort::Env* env)
263{
264 mPImplOrt->env.reset(env);
265}
266
267// Inference
268template <class I, class O>
269std::vector<O> OrtModel::inference(std::vector<I>& input)
270{
271 std::vector<int64_t> inputShape = mInputShapes[0];
272 inputShape[0] = input.size();
273 for (size_t i = 1; i < mInputShapes[0].size(); ++i) {
274 inputShape[0] /= mInputShapes[0][i];
275 }
276 std::vector<Ort::Value> inputTensor;
277 if constexpr (std::is_same_v<I, OrtDataType::Float16_t>) {
278 inputTensor.emplace_back(Ort::Value::CreateTensor<Ort::Float16_t>(mPImplOrt->memoryInfo, reinterpret_cast<Ort::Float16_t*>(input.data()), input.size(), inputShape.data(), inputShape.size()));
279 } else {
280 inputTensor.emplace_back(Ort::Value::CreateTensor<I>(mPImplOrt->memoryInfo, input.data(), input.size(), inputShape.data(), inputShape.size()));
281 }
282 // input.clear();
283 auto outputTensors = (mPImplOrt->session)->Run(mPImplOrt->runOptions, mInputNamesChar.data(), inputTensor.data(), inputTensor.size(), mOutputNamesChar.data(), mOutputNamesChar.size());
284 O* outputValues = outputTensors[0].template GetTensorMutableData<O>();
285 std::vector<O> outputValuesVec{outputValues, outputValues + inputShape[0] * mOutputShapes[0][1]};
286 outputTensors.clear();
287 return outputValuesVec;
288}
289
290template std::vector<float> OrtModel::inference<float, float>(std::vector<float>&);
291template std::vector<float> OrtModel::inference<OrtDataType::Float16_t, float>(std::vector<OrtDataType::Float16_t>&);
292template std::vector<OrtDataType::Float16_t> OrtModel::inference<OrtDataType::Float16_t, OrtDataType::Float16_t>(std::vector<OrtDataType::Float16_t>&);
293
294template <class I, class O>
295void OrtModel::inference(I* input, int64_t input_size, O* output)
296{
297 // std::vector<std::string> providers = Ort::GetAvailableProviders();
298 // for (const auto& provider : providers) {
299 // LOG(info) << "Available Execution Provider: " << provider;
300 // }
301 std::vector<int64_t> inputShape{input_size, (int64_t)mInputShapes[0][1]};
302 Ort::Value inputTensor = Ort::Value(nullptr);
303 if constexpr (std::is_same_v<I, OrtDataType::Float16_t>) {
304 inputTensor = Ort::Value::CreateTensor<Ort::Float16_t>(mPImplOrt->memoryInfo, reinterpret_cast<Ort::Float16_t*>(input), input_size * mInputShapes[0][1], inputShape.data(), inputShape.size());
305 } else {
306 inputTensor = Ort::Value::CreateTensor<I>(mPImplOrt->memoryInfo, input, input_size * mInputShapes[0][1], inputShape.data(), inputShape.size());
307 }
308 (mPImplOrt->ioBinding)->BindInput(mInputNames[0].c_str(), inputTensor);
309
310 std::vector<int64_t> outputShape{input_size, mOutputShapes[0][1]};
311 Ort::Value outputTensor = Ort::Value(nullptr);
312 if constexpr (std::is_same_v<O, OrtDataType::Float16_t>) {
313 outputTensor = Ort::Value::CreateTensor<Ort::Float16_t>(mPImplOrt->memoryInfo, reinterpret_cast<Ort::Float16_t*>(output), input_size * mOutputShapes[0][1], outputShape.data(), outputShape.size());
314 } else {
315 outputTensor = Ort::Value::CreateTensor<O>(mPImplOrt->memoryInfo, output, input_size * mOutputShapes[0][1], outputShape.data(), outputShape.size());
316 }
317 (mPImplOrt->ioBinding)->BindOutput(mOutputNames[0].c_str(), outputTensor);
318
319 (mPImplOrt->session)->Run(mPImplOrt->runOptions, *mPImplOrt->ioBinding);
320 // mPImplOrt->session->Run(
321 // mPImplOrt->runOptions,
322 // mInputNamesChar.data(),
323 // &inputTensor,
324 // mInputNamesChar.size(),
325 // mOutputNamesChar.data(),
326 // &outputTensor,
327 // mOutputNamesChar.size());
328}
329
330template void OrtModel::inference<OrtDataType::Float16_t, OrtDataType::Float16_t>(OrtDataType::Float16_t*, int64_t, OrtDataType::Float16_t*);
331template void OrtModel::inference<OrtDataType::Float16_t, float>(OrtDataType::Float16_t*, int64_t, float*);
332template void OrtModel::inference<float, OrtDataType::Float16_t>(float*, int64_t, OrtDataType::Float16_t*);
333template void OrtModel::inference<float, float>(float*, int64_t, float*);
334
335template <class I, class O>
336void OrtModel::inference(I** input, int64_t input_size, O* output)
337{
338 std::vector<Ort::Value> inputTensors(mInputShapesCopy.size());
339
340 for (size_t i = 0; i < mInputShapesCopy.size(); ++i) {
341
342 mInputShapesCopy[i][0] = input_size; // batch-size
343 mOutputShapesCopy[i][0] = input_size; // batch-size
344
345 if constexpr (std::is_same_v<I, OrtDataType::Float16_t>) {
346 inputTensors[i] = Ort::Value::CreateTensor<Ort::Float16_t>(
347 mPImplOrt->memoryInfo,
348 reinterpret_cast<Ort::Float16_t*>(input[i]),
349 mInputSizePerNode[i] * input_size,
350 mInputShapesCopy[i].data(),
351 mInputShapesCopy[i].size());
352 } else {
353 inputTensors[i] = Ort::Value::CreateTensor<I>(
354 mPImplOrt->memoryInfo,
355 input[i],
356 mInputSizePerNode[i] * input_size,
357 mInputShapesCopy[i].data(),
358 mInputShapesCopy[i].size());
359 }
360 }
361
362 Ort::Value outputTensor = Ort::Value(nullptr);
363 if constexpr (std::is_same_v<O, OrtDataType::Float16_t>) {
364 outputTensor = Ort::Value::CreateTensor<Ort::Float16_t>(
365 mPImplOrt->memoryInfo,
366 reinterpret_cast<Ort::Float16_t*>(output),
367 mOutputSizePerNode[0] * input_size, // assumes that there is only one output node
368 mOutputShapesCopy[0].data(),
369 mOutputShapesCopy[0].size());
370 } else {
371 outputTensor = Ort::Value::CreateTensor<O>(
372 mPImplOrt->memoryInfo,
373 output,
374 mOutputSizePerNode[0] * input_size, // assumes that there is only one output node
375 mOutputShapesCopy[0].data(),
376 mOutputShapesCopy[0].size());
377 }
378
379 // === Run inference ===
380 mPImplOrt->session->Run(
381 mPImplOrt->runOptions,
382 mInputNamesChar.data(),
383 inputTensors.data(),
384 mInputNamesChar.size(),
385 mOutputNamesChar.data(),
386 &outputTensor,
387 mOutputNamesChar.size());
388}
389
390template void OrtModel::inference<OrtDataType::Float16_t, OrtDataType::Float16_t>(OrtDataType::Float16_t**, int64_t, OrtDataType::Float16_t*);
391template void OrtModel::inference<OrtDataType::Float16_t, float>(OrtDataType::Float16_t**, int64_t, float*);
392template void OrtModel::inference<float, OrtDataType::Float16_t>(float**, int64_t, OrtDataType::Float16_t*);
393template void OrtModel::inference<float, float>(float**, int64_t, float*);
394
395template <class I, class O>
396std::vector<O> OrtModel::inference(std::vector<std::vector<I>>& inputs)
397{
398 std::vector<Ort::Value> input_tensors;
399
400 for (size_t i = 0; i < inputs.size(); ++i) {
401
402 mInputShapesCopy[i][0] = inputs[i].size() / mInputSizePerNode[i]; // batch-size
403
404 if constexpr (std::is_same_v<I, OrtDataType::Float16_t>) {
405 input_tensors.emplace_back(
406 Ort::Value::CreateTensor<Ort::Float16_t>(
407 mPImplOrt->memoryInfo,
408 reinterpret_cast<Ort::Float16_t*>(inputs[i].data()),
409 mInputSizePerNode[i] * mInputShapesCopy[i][0],
410 mInputShapesCopy[i].data(),
411 mInputShapesCopy[i].size()));
412 } else {
413 input_tensors.emplace_back(
414 Ort::Value::CreateTensor<I>(
415 mPImplOrt->memoryInfo,
416 inputs[i].data(),
417 mInputSizePerNode[i] * mInputShapesCopy[i][0],
418 mInputShapesCopy[i].data(),
419 mInputShapesCopy[i].size()));
420 }
421 }
422
423 int32_t totalOutputSize = mOutputsTotal * mInputShapesCopy[0][0];
424
425 // === Run inference ===
426 auto output_tensors = mPImplOrt->session->Run(
427 mPImplOrt->runOptions,
428 mInputNamesChar.data(),
429 input_tensors.data(),
430 input_tensors.size(),
431 mOutputNamesChar.data(),
432 mOutputNamesChar.size());
433
434 // === Extract output values ===
435 O* output_data = output_tensors[0].template GetTensorMutableData<O>();
436 std::vector<O> output_vec(output_data, output_data + totalOutputSize);
437 output_tensors.clear();
438 return output_vec;
439}
440
441template std::vector<float> OrtModel::inference<float, float>(std::vector<std::vector<float>>&);
442template std::vector<OrtDataType::Float16_t> OrtModel::inference<OrtDataType::Float16_t, OrtDataType::Float16_t>(std::vector<std::vector<OrtDataType::Float16_t>>&);
443
444// Release session
445void OrtModel::release(bool profilingEnabled)
446{
447 mPImplOrt.reset();
448}
449
450// private
451std::string OrtModel::printShape(const std::vector<int64_t>& v)
452{
453 std::stringstream ss("");
454 for (size_t i = 0; i < v.size() - 1; i++) {
455 ss << v[i] << "x";
456 }
457 ss << v[v.size() - 1];
458 return ss.str();
459}
460
461std::string OrtModel::printShape(const std::vector<std::vector<int64_t>>& v, std::vector<std::string>& n)
462{
463 std::stringstream ss("");
464 for (size_t i = 0; i < v.size(); i++) {
465 ss << n[i] << " -> (";
466 for (size_t j = 0; j < v[i].size() - 1; j++) {
467 ss << v[i][j] << "x";
468 }
469 ss << v[i][v[i].size() - 1] << "); ";
470 }
471 return ss.str();
472}
473
474} // namespace ml
475
476} // namespace o2
int32_t i
void output(const std::map< std::string, ChannelStat > &channels)
Definition rawdump.cxx:197
A header library for loading ONNX models and inferencing them on CPU and GPU.
uint32_t j
Definition RawData.h:0
std::ostringstream debug
void initOptions(std::unordered_map< std::string, std::string > optionsMap)
void memoryOnDevice(int32_t=0)
Ort::Env * getEnv()
void release(bool=false)
void setEnv(Ort::Env *)
std::vector< O > v2v(std::vector< I > &, bool=true)
Ort::MemoryInfo * getMemoryInfo()
std::vector< O > inference(std::vector< I > &)
virtual ~OrtModel()
void init(std::unordered_map< std::string, std::string > optionsMap)
Ort::SessionOptions * getSessionOptions()
GLdouble n
Definition glcorearb.h:1982
GLsizeiptr size
Definition glcorearb.h:659
const GLdouble * v
Definition glcorearb.h:832
GLdouble f
Definition glcorearb.h:310
GLuint GLsizei const GLchar * message
Definition glcorearb.h:2517
GLenum GLfloat param
Definition glcorearb.h:271
GLenum GLenum severity
Definition glcorearb.h:2513
a couple of static helper functions to create timestamp values for CCDB queries or override obsolete ...
std::string to_string(gsl::span< T, Size > span)
Definition common.h:52
void empty(int)
Ort::AllocatorWithDefaultOptions allocator
std::unique_ptr< Ort::Session > session
ONNX session.
Ort::SessionOptions sessionOptions
std::unique_ptr< Ort::IoBinding > ioBinding
std::unique_ptr< Ort::Env > env
LOG(info)<< "Compressed in "<< sw.CpuTime()<< " s"
const std::string str