20#include <onnxruntime_cxx_api.h>
42 std::unique_ptr<Ort::Env>
env =
nullptr;
43 std::unique_ptr<Ort::Session>
session =
nullptr;
46 Ort::MemoryInfo
memoryInfo = Ort::MemoryInfo(
"Cpu", OrtAllocatorType::OrtDeviceAllocator, 0, OrtMemType::OrtMemTypeDefault);
47 std::unique_ptr<Ort::IoBinding>
ioBinding =
nullptr;
53 mPImplOrt = std::make_unique<OrtVariables>();
56 if (!optionsMap.contains(
"model-path")) {
57 LOG(fatal) <<
"(ORT) Model path cannot be empty!";
60 if (!optionsMap[
"model-path"].
empty()) {
61 mModelPath = optionsMap[
"model-path"];
62 mDeviceType = (optionsMap.contains(
"device-type") ? optionsMap[
"device-type"] :
"CPU");
63 mDeviceId = (optionsMap.contains(
"device-id") ? std::stoi(optionsMap[
"device-id"]) : -1);
64 mAllocateDeviceMemory = (optionsMap.contains(
"allocate-device-memory") ? std::stoi(optionsMap[
"allocate-device-memory"]) : 0);
65 mIntraOpNumThreads = (optionsMap.contains(
"intra-op-num-threads") ? std::stoi(optionsMap[
"intra-op-num-threads"]) : 0);
66 mInterOpNumThreads = (optionsMap.contains(
"inter-op-num-threads") ? std::stoi(optionsMap[
"inter-op-num-threads"]) : 0);
67 mLoggingLevel = (optionsMap.contains(
"logging-level") ? std::stoi(optionsMap[
"logging-level"]) : 0);
68 mEnableProfiling = (optionsMap.contains(
"enable-profiling") ? std::stoi(optionsMap[
"enable-profiling"]) : 0);
69 mEnableOptimizations = (optionsMap.contains(
"enable-optimizations") ? std::stoi(optionsMap[
"enable-optimizations"]) : 0);
70 mEnvName = (optionsMap.contains(
"onnx-environment-name") ? optionsMap[
"onnx-environment-name"] :
"onnx_model_inference");
72 if (mDeviceType ==
"CPU") {
73 (mPImplOrt->sessionOptions).SetIntraOpNumThreads(mIntraOpNumThreads);
74 (mPImplOrt->sessionOptions).SetInterOpNumThreads(mInterOpNumThreads);
75 if (mIntraOpNumThreads > 1 || mInterOpNumThreads > 1) {
76 (mPImplOrt->sessionOptions).SetExecutionMode(ExecutionMode::ORT_PARALLEL);
77 }
else if (mIntraOpNumThreads == 1) {
78 (mPImplOrt->sessionOptions).SetExecutionMode(ExecutionMode::ORT_SEQUENTIAL);
80 if (mLoggingLevel < 2) {
81 LOG(info) <<
"(ORT) CPU execution provider set with " << mIntraOpNumThreads <<
" (mIntraOpNumThreads) and " << mInterOpNumThreads <<
" (mInterOpNumThreads) threads";
88 (mPImplOrt->sessionOptions).DisableMemPattern();
89 (mPImplOrt->sessionOptions).DisableCpuMemArena();
91 if (mEnableProfiling) {
92 if (optionsMap.contains(
"profiling-output-path")) {
93 (mPImplOrt->sessionOptions).EnableProfiling((optionsMap[
"profiling-output-path"] +
"/ORT_LOG_").c_str());
95 LOG(warning) <<
"(ORT) If profiling is enabled, optionsMap[\"profiling-output-path\"] should be set. Disabling profiling for now.";
96 (mPImplOrt->sessionOptions).DisableProfiling();
99 (mPImplOrt->sessionOptions).DisableProfiling();
102 (mPImplOrt->sessionOptions).SetGraphOptimizationLevel(GraphOptimizationLevel(mEnableOptimizations));
103 (mPImplOrt->sessionOptions).SetLogSeverityLevel(OrtLoggingLevel(mLoggingLevel));
107 LOG(fatal) <<
"(ORT) Model path cannot be empty!";
113 mPImplOrt->env = std::make_unique<Ort::Env>(
114 OrtLoggingLevel(mLoggingLevel),
115 (mEnvName.empty() ?
"ORT" : mEnvName.c_str()),
117 [](
void*
param, OrtLoggingLevel
severity,
const char* category,
const char* logid,
const char* code_location,
const char*
message) {
118 if (
severity == ORT_LOGGING_LEVEL_VERBOSE) {
119 LOG(
debug) <<
"(ORT) [" << logid <<
"|" << category <<
"|" << code_location <<
"]: " <<
message;
120 }
else if (
severity == ORT_LOGGING_LEVEL_INFO) {
121 LOG(info) <<
"(ORT) [" << logid <<
"|" << category <<
"|" << code_location <<
"]: " <<
message;
122 }
else if (
severity == ORT_LOGGING_LEVEL_WARNING) {
123 LOG(warning) <<
"(ORT) [" << logid <<
"|" << category <<
"|" << code_location <<
"]: " <<
message;
124 }
else if (
severity == ORT_LOGGING_LEVEL_ERROR) {
125 LOG(error) <<
"(ORT) [" << logid <<
"|" << category <<
"|" << code_location <<
"]: " <<
message;
126 }
else if (
severity == ORT_LOGGING_LEVEL_FATAL) {
127 LOG(fatal) <<
"(ORT) [" << logid <<
"|" << category <<
"|" << code_location <<
"]: " <<
message;
129 LOG(info) <<
"(ORT) [" << logid <<
"|" << category <<
"|" << code_location <<
"]: " <<
message;
133 (mPImplOrt->env)->DisableTelemetryEvents();
138 if (mAllocateDeviceMemory) {
141 mPImplOrt->session = std::make_unique<Ort::Session>(*mPImplOrt->env, mModelPath.c_str(), mPImplOrt->sessionOptions);
142 mPImplOrt->ioBinding = std::make_unique<Ort::IoBinding>(*mPImplOrt->session);
146 if (mLoggingLevel < 2) {
147 LOG(info) <<
"(ORT) Model loaded successfully! (inputs: " << printShape(mInputShapes, mInputNames) <<
", outputs: " << printShape(mOutputShapes, mInputNames) <<
")";
153 if (deviceIndex >= 0) {
154 (mPImplOrt->runOptions).AddConfigEntry(
"disable_synchronize_execution_providers",
"1");
155 (mPImplOrt->sessionOptions).AddConfigEntry(
"session.use_device_allocator_for_initializers",
"1");
156 (mPImplOrt->sessionOptions).AddConfigEntry(
"session.use_env_allocators",
"1");
157 (mPImplOrt->sessionOptions).AddConfigEntry(
"session_options.enable_cpu_mem_arena",
"0");
160 (mPImplOrt->runOptions).AddConfigEntry(
"memory.enable_memory_arena_shrinkage", (
"gpu:" +
std::to_string(deviceIndex)).c_str());
162 std::string dev_mem_str =
"";
163 if (mDeviceType ==
"ROCM") {
164 dev_mem_str =
"HipPinned";
166 if (mDeviceType ==
"CUDA") {
167 dev_mem_str =
"Cuda";
169 mPImplOrt->memoryInfo = Ort::MemoryInfo(dev_mem_str.c_str(), OrtAllocatorType::OrtDeviceAllocator, deviceIndex, OrtMemType::OrtMemTypeDefault);
170 if (mLoggingLevel < 2) {
171 LOG(info) <<
"(ORT) Memory info set to on-device memory for device type " << mDeviceType <<
" with ID " << deviceIndex <<
" and mPImplOrt pointer " << mPImplOrt;
178 mPImplOrt->session = std::make_unique<Ort::Session>(*(mPImplOrt->env), mModelPath.c_str(), mPImplOrt->sessionOptions);
184 return &mPImplOrt->sessionOptions;
189 return &mPImplOrt->memoryInfo;
194 return (mPImplOrt->env).get();
197template <
class I,
class O>
200 if constexpr (std::is_same_v<I, O>) {
203 std::vector<O>
output(input.size());
204 std::transform(std::begin(input), std::end(input), std::begin(
output), [](I
f) {
return O(
f); });
214 for (
size_t i = 0;
i < (mPImplOrt->session)->GetInputCount(); ++
i) {
215 mInputNames.push_back((mPImplOrt->session)->GetInputNameAllocated(
i, mPImplOrt->allocator).get());
217 for (
size_t i = 0;
i < (mPImplOrt->session)->GetInputCount(); ++
i) {
218 mInputShapes.emplace_back((mPImplOrt->session)->GetInputTypeInfo(
i).GetTensorTypeAndShapeInfo().GetShape());
220 for (
size_t i = 0;
i < (mPImplOrt->session)->GetOutputCount(); ++
i) {
221 mOutputNames.push_back((mPImplOrt->session)->GetOutputNameAllocated(
i, mPImplOrt->allocator).get());
223 for (
size_t i = 0;
i < (mPImplOrt->session)->GetOutputCount(); ++
i) {
224 mOutputShapes.emplace_back((mPImplOrt->session)->GetOutputTypeInfo(
i).GetTensorTypeAndShapeInfo().GetShape());
227 mInputNamesChar.resize(mInputNames.size(),
nullptr);
228 std::transform(std::begin(mInputNames), std::end(mInputNames), std::begin(mInputNamesChar),
229 [&](
const std::string&
str) {
return str.c_str(); });
230 mOutputNamesChar.resize(mOutputNames.size(),
nullptr);
231 std::transform(std::begin(mOutputNames), std::end(mOutputNames), std::begin(mOutputNamesChar),
232 [&](
const std::string&
str) {
return str.c_str(); });
234 mInputShapesCopy = mInputShapes;
235 mOutputShapesCopy = mOutputShapes;
236 mInputSizePerNode.resize(mInputShapes.size(), 1);
237 mOutputSizePerNode.resize(mOutputShapes.size(), 1);
239 for (
size_t i = 0;
i < mInputShapes.size(); ++
i) {
240 if (mInputShapes[
i].
size() > 0) {
241 for (
size_t j = 1;
j < mInputShapes[
i].size(); ++
j) {
242 if (mInputShapes[
i][
j] > 0) {
243 mInputsTotal *= mInputShapes[
i][
j];
244 mInputSizePerNode[
i] *= mInputShapes[
i][
j];
250 for (
size_t i = 0;
i < mOutputShapes.size(); ++
i) {
251 if (mOutputShapes[
i].
size() > 0) {
252 for (
size_t j = 1;
j < mOutputShapes[
i].size(); ++
j) {
253 if (mOutputShapes[
i][
j] > 0) {
254 mOutputsTotal *= mOutputShapes[
i][
j];
255 mOutputSizePerNode[
i] *= mOutputShapes[
i][
j];
264 mPImplOrt->env.reset(env);
268template <
class I,
class O>
271 std::vector<int64_t> inputShape = mInputShapes[0];
272 inputShape[0] = input.size();
273 for (
size_t i = 1;
i < mInputShapes[0].size(); ++
i) {
274 inputShape[0] /= mInputShapes[0][
i];
276 std::vector<Ort::Value> inputTensor;
277 if constexpr (std::is_same_v<I, OrtDataType::Float16_t>) {
278 inputTensor.emplace_back(Ort::Value::CreateTensor<Ort::Float16_t>(mPImplOrt->memoryInfo,
reinterpret_cast<Ort::Float16_t*
>(input.data()), input.size(), inputShape.data(), inputShape.size()));
280 inputTensor.emplace_back(Ort::Value::CreateTensor<I>(mPImplOrt->memoryInfo, input.data(), input.size(), inputShape.data(), inputShape.size()));
283 auto outputTensors = (mPImplOrt->session)->Run(mPImplOrt->runOptions, mInputNamesChar.data(), inputTensor.data(), inputTensor.size(), mOutputNamesChar.data(), mOutputNamesChar.size());
284 O* outputValues = outputTensors[0].template GetTensorMutableData<O>();
285 std::vector<O> outputValuesVec{outputValues, outputValues + inputShape[0] * mOutputShapes[0][1]};
286 outputTensors.clear();
287 return outputValuesVec;
290template std::vector<float> OrtModel::inference<float, float>(std::vector<float>&);
291template std::vector<float> OrtModel::inference<OrtDataType::Float16_t, float>(std::vector<OrtDataType::Float16_t>&);
292template std::vector<OrtDataType::Float16_t> OrtModel::inference<OrtDataType::Float16_t, OrtDataType::Float16_t>(std::vector<OrtDataType::Float16_t>&);
294template <
class I,
class O>
301 std::vector<int64_t> inputShape{input_size, (int64_t)mInputShapes[0][1]};
302 Ort::Value inputTensor = Ort::Value(
nullptr);
303 if constexpr (std::is_same_v<I, OrtDataType::Float16_t>) {
304 inputTensor = Ort::Value::CreateTensor<Ort::Float16_t>(mPImplOrt->memoryInfo,
reinterpret_cast<Ort::Float16_t*
>(input), input_size * mInputShapes[0][1], inputShape.data(), inputShape.size());
306 inputTensor = Ort::Value::CreateTensor<I>(mPImplOrt->memoryInfo, input, input_size * mInputShapes[0][1], inputShape.data(), inputShape.size());
308 (mPImplOrt->ioBinding)->BindInput(mInputNames[0].c_str(), inputTensor);
310 std::vector<int64_t> outputShape{input_size, mOutputShapes[0][1]};
311 Ort::Value outputTensor = Ort::Value(
nullptr);
312 if constexpr (std::is_same_v<O, OrtDataType::Float16_t>) {
313 outputTensor = Ort::Value::CreateTensor<Ort::Float16_t>(mPImplOrt->memoryInfo,
reinterpret_cast<Ort::Float16_t*
>(
output), input_size * mOutputShapes[0][1], outputShape.data(), outputShape.size());
315 outputTensor = Ort::Value::CreateTensor<O>(mPImplOrt->memoryInfo,
output, input_size * mOutputShapes[0][1], outputShape.data(), outputShape.size());
317 (mPImplOrt->ioBinding)->BindOutput(mOutputNames[0].c_str(), outputTensor);
319 (mPImplOrt->session)->Run(mPImplOrt->runOptions, *mPImplOrt->ioBinding);
330template void OrtModel::inference<OrtDataType::Float16_t, OrtDataType::Float16_t>(OrtDataType::Float16_t*, int64_t, OrtDataType::Float16_t*);
331template void OrtModel::inference<OrtDataType::Float16_t, float>(OrtDataType::Float16_t*, int64_t,
float*);
332template void OrtModel::inference<float, OrtDataType::Float16_t>(
float*, int64_t, OrtDataType::Float16_t*);
333template void OrtModel::inference<float, float>(
float*, int64_t,
float*);
335template <
class I,
class O>
338 std::vector<Ort::Value> inputTensors(mInputShapesCopy.size());
340 for (
size_t i = 0;
i < mInputShapesCopy.size(); ++
i) {
342 mInputShapesCopy[
i][0] = input_size;
343 mOutputShapesCopy[
i][0] = input_size;
345 if constexpr (std::is_same_v<I, OrtDataType::Float16_t>) {
346 inputTensors[
i] = Ort::Value::CreateTensor<Ort::Float16_t>(
347 mPImplOrt->memoryInfo,
348 reinterpret_cast<Ort::Float16_t*
>(input[
i]),
349 mInputSizePerNode[
i] * input_size,
350 mInputShapesCopy[
i].data(),
351 mInputShapesCopy[
i].size());
353 inputTensors[
i] = Ort::Value::CreateTensor<I>(
354 mPImplOrt->memoryInfo,
356 mInputSizePerNode[
i] * input_size,
357 mInputShapesCopy[
i].data(),
358 mInputShapesCopy[
i].size());
362 Ort::Value outputTensor = Ort::Value(
nullptr);
363 if constexpr (std::is_same_v<O, OrtDataType::Float16_t>) {
364 outputTensor = Ort::Value::CreateTensor<Ort::Float16_t>(
365 mPImplOrt->memoryInfo,
366 reinterpret_cast<Ort::Float16_t*
>(
output),
367 mOutputSizePerNode[0] * input_size,
368 mOutputShapesCopy[0].data(),
369 mOutputShapesCopy[0].size());
371 outputTensor = Ort::Value::CreateTensor<O>(
372 mPImplOrt->memoryInfo,
374 mOutputSizePerNode[0] * input_size,
375 mOutputShapesCopy[0].data(),
376 mOutputShapesCopy[0].size());
380 mPImplOrt->session->Run(
381 mPImplOrt->runOptions,
382 mInputNamesChar.data(),
384 mInputNamesChar.size(),
385 mOutputNamesChar.data(),
387 mOutputNamesChar.size());
390template void OrtModel::inference<OrtDataType::Float16_t, OrtDataType::Float16_t>(OrtDataType::Float16_t**, int64_t, OrtDataType::Float16_t*);
391template void OrtModel::inference<OrtDataType::Float16_t, float>(OrtDataType::Float16_t**, int64_t,
float*);
392template void OrtModel::inference<float, OrtDataType::Float16_t>(
float**, int64_t, OrtDataType::Float16_t*);
393template void OrtModel::inference<float, float>(
float**, int64_t,
float*);
395template <
class I,
class O>
398 std::vector<Ort::Value> input_tensors;
400 for (
size_t i = 0;
i < inputs.size(); ++
i) {
402 mInputShapesCopy[
i][0] = inputs[
i].size() / mInputSizePerNode[
i];
404 if constexpr (std::is_same_v<I, OrtDataType::Float16_t>) {
405 input_tensors.emplace_back(
406 Ort::Value::CreateTensor<Ort::Float16_t>(
407 mPImplOrt->memoryInfo,
408 reinterpret_cast<Ort::Float16_t*
>(inputs[
i].data()),
409 mInputSizePerNode[
i] * mInputShapesCopy[
i][0],
410 mInputShapesCopy[
i].data(),
411 mInputShapesCopy[
i].size()));
413 input_tensors.emplace_back(
414 Ort::Value::CreateTensor<I>(
415 mPImplOrt->memoryInfo,
417 mInputSizePerNode[
i] * mInputShapesCopy[
i][0],
418 mInputShapesCopy[
i].data(),
419 mInputShapesCopy[
i].size()));
423 int32_t totalOutputSize = mOutputsTotal * mInputShapesCopy[0][0];
426 auto output_tensors = mPImplOrt->session->Run(
427 mPImplOrt->runOptions,
428 mInputNamesChar.data(),
429 input_tensors.data(),
430 input_tensors.size(),
431 mOutputNamesChar.data(),
432 mOutputNamesChar.size());
435 O* output_data = output_tensors[0].template GetTensorMutableData<O>();
436 std::vector<O> output_vec(output_data, output_data + totalOutputSize);
437 output_tensors.clear();
441template std::vector<float> OrtModel::inference<float, float>(std::vector<std::vector<float>>&);
442template std::vector<OrtDataType::Float16_t> OrtModel::inference<OrtDataType::Float16_t, OrtDataType::Float16_t>(std::vector<std::vector<OrtDataType::Float16_t>>&);
451std::string OrtModel::printShape(
const std::vector<int64_t>&
v)
453 std::stringstream ss(
"");
454 for (
size_t i = 0;
i <
v.size() - 1;
i++) {
457 ss <<
v[
v.size() - 1];
461std::string OrtModel::printShape(
const std::vector<std::vector<int64_t>>&
v, std::vector<std::string>&
n)
463 std::stringstream ss(
"");
464 for (
size_t i = 0;
i <
v.size();
i++) {
465 ss <<
n[
i] <<
" -> (";
466 for (
size_t j = 0;
j <
v[
i].size() - 1;
j++) {
467 ss <<
v[
i][
j] <<
"x";
469 ss <<
v[
i][
v[
i].size() - 1] <<
"); ";
A header library for loading ONNX models and inferencing them on CPU and GPU.
void initOptions(std::unordered_map< std::string, std::string > optionsMap)
void memoryOnDevice(int32_t=0)
std::vector< O > v2v(std::vector< I > &, bool=true)
Ort::MemoryInfo * getMemoryInfo()
std::vector< O > inference(std::vector< I > &)
void init(std::unordered_map< std::string, std::string > optionsMap)
Ort::SessionOptions * getSessionOptions()
GLuint GLsizei const GLchar * message
a couple of static helper functions to create timestamp values for CCDB queries or override obsolete ...
std::string to_string(gsl::span< T, Size > span)
Ort::RunOptions runOptions
Ort::AllocatorWithDefaultOptions allocator
Ort::MemoryInfo memoryInfo
std::unique_ptr< Ort::Session > session
ONNX session.
Ort::SessionOptions sessionOptions
std::unique_ptr< Ort::IoBinding > ioBinding
std::unique_ptr< Ort::Env > env
LOG(info)<< "Compressed in "<< sw.CpuTime()<< " s"